1// Copyright 2020 the V8 project authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5/**
6 * @fileoverview Generate exceptions from full corpus test report.
7 */
8
9const program = require('commander');
10
11const assert = require('assert');
12const babelGenerator = require('@babel/generator').default;
13const babelTemplate = require('@babel/template').default;
14const babelTypes = require('@babel/types');
15const fs = require('fs');
16const p = require('path');
17const prettier = require("prettier");
18
19const SPLIT_LINES_RE = /^.*([\n\r]+|$)/gm;
20const PARSE_RE = /^Parsing (.*) sloppy took (\d+) ms\.\n$/;
21const MUTATE_RE = /^Mutating (.*) took (\d+) ms\.\n$/;
22const PARSE_FAILED_RE = /^WARNING: failed to sloppy parse (.*)\n$/;
23const PARSE_STRICT_FAILED_RE = /^WARNING: failed to strict parse (.*)\n$/;
24const MUTATE_FAILED_RE = /^ERROR: Exception during mutate: (.*)\n$/;
25
26// Add tests matching error regexp to result array.
27function matchError(regexp, line, resultArray){
28  const match = line.match(regexp);
29  if (!match) return false;
30  const relPath = match[1];
31  assert(relPath);
32  resultArray.push(relPath);
33  return true;
34}
35
36// Sum up total duration of tests matching the duration regexp and
37// map test -> duration in result map.
38function matchDuration(regexp, line, resultMap){
39  const match = line.match(regexp);
40  if (!match) return false;
41  const relPath = match[1];
42  assert(relPath);
43  resultMap[relPath] = (resultMap[relPath] || 0) + parseInt(match[2]);
44  return true;
45}
46
47// Create lists of failed and slow tests from stdout of a fuzzer run.
48function processFuzzOutput(outputFile){
49  const text = fs.readFileSync(outputFile, 'utf-8');
50  const lines = text.match(SPLIT_LINES_RE);
51
52  const failedParse = [];
53  const failedParseStrict = [];
54  const failedMutate = [];
55  const durationsMap = {};
56
57  for (const line of lines) {
58    if (matchError(PARSE_FAILED_RE, line, failedParse))
59      continue;
60    if (matchError(PARSE_STRICT_FAILED_RE, line, failedParseStrict))
61      continue;
62    if (matchError(MUTATE_FAILED_RE, line, failedMutate))
63      continue;
64    if (matchDuration(PARSE_RE, line, durationsMap))
65      continue;
66    if (matchDuration(MUTATE_RE, line, durationsMap))
67      continue;
68  }
69
70  // Tuples (absPath, duration).
71  const total = Object.entries(durationsMap);
72  // Tuples (absPath, duration) with 2s < duration <= 10s.
73  const slow = total.filter(t => t[1] > 2000 && t[1] <= 10000);
74  // Tuples (absPath, duration) with 10s < duration.
75  const verySlow = total.filter(t => t[1] > 10000);
76
77  // Assert there's nothing horribly wrong with the results.
78  // We have at least 2500 tests in the output.
79  assert(total.length > 2500);
80  // No more than 5% parse/mutation errors.
81  assert(failedParse.length + failedMutate.length < total.length / 20);
82  // No more than 10% slow tests
83  assert(slow.length < total.length / 10);
84  // No more than 2% very slow tests.
85  assert(verySlow.length < total.length / 50);
86
87  // Sort everything.
88  failedParse.sort();
89  failedParseStrict.sort();
90  failedMutate.sort();
91
92  function slowestFirst(a, b) {
93    return b[1] - a[1];
94  }
95
96  slow.sort(slowestFirst);
97  verySlow.sort(slowestFirst);
98
99  return [failedParse, failedParseStrict, failedMutate, slow, verySlow];
100}
101
102// List of string literals of failed tests.
103function getLiteralsForFailed(leadingComment, failedList) {
104  const result = failedList.map(path => babelTypes.stringLiteral(path));
105  if (result.length) {
106    babelTypes.addComment(result[0], 'leading', leadingComment);
107  }
108  return result;
109}
110
111// List of string literals of slow tests with duration comments.
112function getLiteralsForSlow(leadingComment, slowList) {
113  const result = slowList.map(([path, duration]) => {
114    const literal = babelTypes.stringLiteral(path);
115    babelTypes.addComment(
116        literal, 'trailing', ` ${duration / 1000}s`, true);
117    return literal;
118  });
119  if (result.length) {
120    babelTypes.addComment(result[0], 'leading', leadingComment);
121  }
122  return result;
123}
124
125function main() {
126  program
127    .version('0.0.1')
128    .parse(process.argv);
129
130  if (!program.args.length) {
131    console.log('Need to specify stdout reports of fuzz runs.');
132    return;
133  }
134
135  let skipped = [];
136  let softSkipped = [];
137  let sloppy = [];
138  for (const outputFile of program.args) {
139    const [failedParse, failedParseStrict, failedMutate, slow, verySlow] = (
140        processFuzzOutput(outputFile));
141    const name = p.basename(outputFile, p.extname(outputFile));
142
143    // Skip tests that fail to parse/mutate or are very slow.
144    skipped = skipped.concat(getLiteralsForFailed(
145        ` Tests with parse errors from ${name} `, failedParse));
146    skipped = skipped.concat(getLiteralsForFailed(
147        ` Tests with mutation errors from ${name} `, failedMutate));
148    skipped = skipped.concat(getLiteralsForSlow(
149        ` Very slow tests from ${name} `, verySlow));
150
151    // Soft-skip slow but not very slow tests.
152    softSkipped = softSkipped.concat(getLiteralsForSlow(
153        ` Slow tests from ${name} `, slow));
154
155    // Mark sloppy tests.
156    sloppy = sloppy.concat(getLiteralsForFailed(
157        ` Tests requiring sloppy mode from ${name} `, failedParseStrict));
158  }
159
160  const fileTemplate = babelTemplate(`
161    /**
162     * @fileoverview Autogenerated exceptions. Created with gen_exceptions.js.
163     */
164
165    'use strict';
166
167    const skipped = SKIPPED;
168
169    const softSkipped = SOFTSKIPPED;
170
171    const sloppy = SLOPPY;
172
173    module.exports = {
174      generatedSkipped: new Set(skipped),
175      generatedSoftSkipped: new Set(softSkipped),
176      generatedSloppy: new Set(sloppy),
177    }
178  `, {preserveComments: true});
179
180  const skippedArray = babelTypes.arrayExpression(skipped);
181  const softSkippedArray = babelTypes.arrayExpression(softSkipped);
182  const sloppyArray = babelTypes.arrayExpression(sloppy);
183
184  const statements = fileTemplate({
185    SKIPPED: skippedArray,
186    SOFTSKIPPED: softSkippedArray,
187    SLOPPY: sloppyArray,
188  });
189
190  const resultProgram = babelTypes.program(statements);
191  const code = babelGenerator(resultProgram, { comments: true }).code;
192  const prettyCode = prettier.format(code, { parser: "babel" });
193  fs.writeFileSync('generated/exceptions.js', prettyCode);
194}
195
196main();
197