1// Copyright 2020 the V8 project authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5/**
6 * @fileoverview Blacklists for fuzzer.
7 */
8
9'use strict';
10
11const fs = require('fs');
12const path = require('path');
13
14const random = require('./random.js');
15
16const {generatedSloppy, generatedSoftSkipped, generatedSkipped} = require(
17    './generated/exceptions.js');
18
19const SKIPPED_FILES = [
20    // Disabled for unexpected test behavior, specific to d8 shell.
21    'd8-os.js',
22    'd8-readbuffer.js',
23
24    // Passes JS flags.
25    'd8-arguments.js',
26
27    // Slow tests or tests that are too large to be used as input.
28    /numops-fuzz-part.*.js/,
29    'regexp-pcre.js',
30    'unicode-test.js',
31    'unicodelctest.js',
32    'unicodelctest-no-optimization.js',
33
34    // Unsupported modules.
35    /^modules.*\.js/,
36
37    // Unsupported property escapes.
38    /^regexp-property-.*\.js/,
39
40    // Bad testcases that just loads a script that always throws errors.
41    'regress-444805.js',
42    'regress-crbug-489597.js',
43    'regress-crbug-620253.js',
44
45    // Just recursively loads itself.
46    'regress-8510.js',
47];
48
49const SKIPPED_DIRECTORIES = [
50    // Slow tests or tests that are too large to be used as input.
51    'embenchen',
52    'poppler',
53    'sqlite',
54
55    // Causes lots of failures.
56    'test262',
57
58    // Unavailable debug.Debug.
59    'v8/test/debugger',
60    'v8/test/inspector',
61
62    // Unsupported modules.
63    'v8/test/js-perf-test/Modules',
64
65    // Contains tests expected to error out on parsing.
66    'v8/test/message',
67
68    // Needs specific dependencies for load of various tests.
69    'v8/test/mjsunit/tools',
70
71    // Unsupported e4x standard.
72    'mozilla/data/e4x',
73
74    // Bails out fast without ReadableStream support.
75    'spidermonkey/non262/ReadableStream',
76];
77
78// Files used with a lower probability.
79const SOFT_SKIPPED_FILES = [
80    // Tests with large binary content.
81    /^binaryen.*\.js/,
82
83    // Tests slow to parse.
84    // CrashTests:
85    /^jquery.*\.js/,
86    // Spidermonkey:
87    'regress-308085.js',
88    'regress-74474-002.js',
89    'regress-74474-003.js',
90    // V8:
91    'object-literal.js',
92];
93
94// Flags that lead to false positives or that are already passed by default.
95const DISALLOWED_FLAGS = [
96    // Disallowed because features prefixed with "experimental" are not
97    // stabilized yet and would cause too much noise when enabled.
98    /^--experimental-.*/,
99
100    // Disallowed due to noise. We explicitly add --harmony to job
101    // definitions, and all of these features are staged before launch.
102    /^--harmony-.*/,
103
104    // Disallowed because they are passed explicitly on the command line.
105    '--allow-natives-syntax',
106    '--debug-code',
107    '--harmony',
108    '--wasm-staging',
109    '--expose-gc',
110    '--expose_gc',
111    '--icu-data-file',
112    '--random-seed',
113
114    // Disallowed due to false positives.
115    '--check-handle-count',
116    '--correctness-fuzzer-suppressions',
117    '--expose-debug-as',
118    '--expose-natives-as',
119    '--expose-trigger-failure',
120    '--mock-arraybuffer-allocator',
121    'natives',  // Used in conjuction with --expose-natives-as.
122    /^--trace-path.*/,
123];
124
125// Flags only used with 25% probability.
126const LOW_PROB_FLAGS_PROB = 0.25;
127const LOW_PROB_FLAGS = [
128    // Flags that lead to slow test performance.
129    /^--gc-interval.*/,
130    /^--deopt-every-n-times.*/,
131];
132
133
134// Flags printing data, leading to false positives in differential fuzzing.
135const DISALLOWED_DIFFERENTIAL_FUZZ_FLAGS = [
136    /^--gc-interval.*/,
137    /^--perf.*/,
138    /^--print.*/,
139    /^--stress-runs.*/,
140    /^--trace.*/,
141    '--expose-externalize-string',
142    '--interpreted-frames-native-stack',
143    '--stress-opt',
144    '--validate-asm',
145];
146
147const MAX_FILE_SIZE_BYTES = 128 * 1024;  // 128KB
148const MEDIUM_FILE_SIZE_BYTES = 32 * 1024;  // 32KB
149
150function _findMatch(iterable, candidate) {
151  for (const entry of iterable) {
152    if (typeof entry === 'string') {
153      if (entry === candidate) {
154        return true;
155      }
156    } else {
157      if (entry.test(candidate)) {
158        return true;
159      }
160    }
161  }
162
163  return false;
164}
165
166function _doesntMatch(iterable, candidate) {
167  return !_findMatch(iterable, candidate);
168}
169
170// Convert Windows path separators.
171function normalize(testPath) {
172  return path.normalize(testPath).replace(/\\/g, '/');
173}
174
175function isTestSkippedAbs(absPath) {
176  const basename = path.basename(absPath);
177  if (_findMatch(SKIPPED_FILES, basename)) {
178    return true;
179  }
180
181  const normalizedTestPath = normalize(absPath);
182  for (const entry of SKIPPED_DIRECTORIES) {
183    if (normalizedTestPath.includes(entry))  {
184      return true;
185    }
186  }
187
188  // Avoid OOM/hangs through huge inputs.
189  const stat = fs.statSync(absPath);
190  return (stat && stat.size >= MAX_FILE_SIZE_BYTES);
191}
192
193function isTestSkippedRel(relPath) {
194  return generatedSkipped.has(normalize(relPath));
195}
196
197// For testing.
198function getSoftSkipped() {
199  return SOFT_SKIPPED_FILES;
200}
201
202// For testing.
203function getGeneratedSoftSkipped() {
204  return generatedSoftSkipped;
205}
206
207// For testing.
208function getGeneratedSloppy() {
209  return generatedSloppy;
210}
211
212function isTestSoftSkippedAbs(absPath) {
213  const basename = path.basename(absPath);
214  if (_findMatch(this.getSoftSkipped(), basename)) {
215    return true;
216  }
217
218  // Graylist medium size files.
219  const stat = fs.statSync(absPath);
220  return (stat && stat.size >= MEDIUM_FILE_SIZE_BYTES);
221}
222
223function isTestSoftSkippedRel(relPath) {
224  return this.getGeneratedSoftSkipped().has(normalize(relPath));
225}
226
227function isTestSloppyRel(relPath) {
228  return this.getGeneratedSloppy().has(normalize(relPath));
229}
230
231function filterFlags(flags) {
232  return flags.filter(flag => {
233    return (
234        _doesntMatch(DISALLOWED_FLAGS, flag) &&
235        (_doesntMatch(LOW_PROB_FLAGS, flag) ||
236         random.choose(LOW_PROB_FLAGS_PROB)));
237  });
238}
239
240function filterDifferentialFuzzFlags(flags) {
241  return flags.filter(
242      flag => _doesntMatch(DISALLOWED_DIFFERENTIAL_FUZZ_FLAGS, flag));
243}
244
245
246module.exports = {
247  filterDifferentialFuzzFlags: filterDifferentialFuzzFlags,
248  filterFlags: filterFlags,
249  getGeneratedSoftSkipped: getGeneratedSoftSkipped,
250  getGeneratedSloppy: getGeneratedSloppy,
251  getSoftSkipped: getSoftSkipped,
252  isTestSkippedAbs: isTestSkippedAbs,
253  isTestSkippedRel: isTestSkippedRel,
254  isTestSoftSkippedAbs: isTestSoftSkippedAbs,
255  isTestSoftSkippedRel: isTestSoftSkippedRel,
256  isTestSloppyRel: isTestSloppyRel,
257}
258