1#!/usr/bin/env python3
2# Copyright 2016 the V8 project authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6"""
7V8 correctness fuzzer launcher script.
8"""
9
10# for py2/py3 compatibility
11from __future__ import print_function
12
13import argparse
14import hashlib
15import itertools
16import json
17import os
18import random
19import re
20import sys
21import traceback
22
23from collections import namedtuple
24
25from v8_commands import Command, FailException, PassException
26import v8_suppressions
27
28PYTHON3 = sys.version_info >= (3, 0)
29
30CONFIGS = dict(
31  default=[],
32  ignition=[
33    '--turbo-filter=~',
34    '--no-opt',
35    '--no-sparkplug',
36    '--liftoff',
37    '--no-wasm-tier-up',
38  ],
39  ignition_asm=[
40    '--turbo-filter=~',
41    '--no-opt',
42    '--no-sparkplug',
43    '--validate-asm',
44    '--stress-validate-asm',
45  ],
46  ignition_eager=[
47    '--turbo-filter=~',
48    '--no-opt',
49    '--no-sparkplug',
50    '--no-lazy',
51    '--no-lazy-inner-functions',
52  ],
53  ignition_no_ic=[
54    '--turbo-filter=~',
55    '--no-opt',
56    '--no-sparkplug',
57    '--liftoff',
58    '--no-wasm-tier-up',
59    '--no-use-ic',
60    '--no-lazy-feedback-allocation',
61  ],
62  ignition_turbo=[],
63  ignition_turbo_no_ic=[
64    '--no-use-ic',
65  ],
66  ignition_turbo_opt=[
67    '--always-opt',
68    '--no-liftoff',
69  ],
70  ignition_turbo_opt_eager=[
71    '--always-opt',
72    '--no-lazy',
73    '--no-lazy-inner-functions',
74  ],
75  jitless=[
76    '--jitless',
77  ],
78  slow_path=[
79    '--force-slow-path',
80  ],
81  slow_path_opt=[
82    '--always-opt',
83    '--force-slow-path',
84  ],
85)
86
87BASELINE_CONFIG = 'ignition'
88DEFAULT_CONFIG = 'ignition_turbo'
89DEFAULT_D8 = 'd8'
90
91# Return codes.
92RETURN_PASS = 0
93RETURN_FAIL = 2
94
95BASE_PATH = os.path.dirname(os.path.abspath(__file__))
96SMOKE_TESTS = os.path.join(BASE_PATH, 'v8_smoke_tests.js')
97
98# Timeout for one d8 run.
99SMOKE_TEST_TIMEOUT_SEC = 1
100TEST_TIMEOUT_SEC = 3
101
102SUPPORTED_ARCHS = ['ia32', 'x64', 'arm', 'arm64']
103
104# Output for suppressed failure case.
105FAILURE_HEADER_TEMPLATE = """#
106# V8 correctness failure
107# V8 correctness configs: %(configs)s
108# V8 correctness sources: %(source_key)s
109# V8 correctness suppression: %(suppression)s
110"""
111
112# Extended output for failure case. The 'CHECK' is for the minimizer.
113FAILURE_TEMPLATE = FAILURE_HEADER_TEMPLATE + """#
114# CHECK
115#
116# Compared %(first_config_label)s with %(second_config_label)s
117#
118# Flags of %(first_config_label)s:
119%(first_config_flags)s
120# Flags of %(second_config_label)s:
121%(second_config_flags)s
122#
123# Difference:
124%(difference)s%(source_file_text)s
125#
126### Start of configuration %(first_config_label)s:
127%(first_config_output)s
128### End of configuration %(first_config_label)s
129#
130### Start of configuration %(second_config_label)s:
131%(second_config_output)s
132### End of configuration %(second_config_label)s
133"""
134
135SOURCE_FILE_TEMPLATE = """
136#
137# Source file:
138%s"""
139
140
141FUZZ_TEST_RE = re.compile(r'.*fuzz(-\d+\.js)')
142SOURCE_RE = re.compile(r'print\("v8-foozzie source: (.*)"\);')
143
144# The number of hex digits used from the hash of the original source file path.
145# Keep the number small to avoid duplicate explosion.
146ORIGINAL_SOURCE_HASH_LENGTH = 3
147
148# Placeholder string if no original source file could be determined.
149ORIGINAL_SOURCE_DEFAULT = 'none'
150
151# Placeholder string for failures from crash tests. If a failure is found with
152# this signature, the matching sources should be moved to the mapping below.
153ORIGINAL_SOURCE_CRASHTESTS = 'placeholder for CrashTests'
154
155# Mapping from relative original source path (e.g. CrashTests/path/to/file.js)
156# to a string key. Map to the same key for duplicate issues. The key should
157# have more than 3 characters to not collide with other existing hashes.
158# If a symptom from a particular original source file is known to map to a
159# known failure, it can be added to this mapping. This should be done for all
160# failures from CrashTests, as those by default map to the placeholder above.
161KNOWN_FAILURES = {
162  # Foo.caller with asm.js: https://crbug.com/1042556
163  'CrashTests/4782147262545920/494.js': '.caller',
164  'CrashTests/5637524389167104/01457.js': '.caller',
165  'CrashTests/5703451898085376/02176.js': '.caller',
166  'CrashTests/4846282433495040/04342.js': '.caller',
167  'CrashTests/5712410200899584/04483.js': '.caller',
168  'v8/test/mjsunit/regress/regress-105.js': '.caller',
169  # Flaky issue that almost never repros.
170  'CrashTests/5694376231632896/1033966.js': 'flaky',
171}
172
173# Flags that are already crashy during smoke tests should not be used.
174DISALLOWED_FLAGS = [
175  '--gdbjit',
176]
177
178
179def filter_flags(flags):
180  return [flag for flag in flags if flag not in DISALLOWED_FLAGS]
181
182
183def infer_arch(d8):
184  """Infer the V8 architecture from the build configuration next to the
185  executable.
186  """
187  with open(os.path.join(os.path.dirname(d8), 'v8_build_config.json')) as f:
188    arch = json.load(f)['v8_current_cpu']
189  arch = 'ia32' if arch == 'x86' else arch
190  assert arch in SUPPORTED_ARCHS
191  return arch
192
193
194class ExecutionArgumentsConfig(object):
195  def __init__(self, label):
196    self.label = label
197
198  def add_arguments(self, parser, default_config):
199    def add_argument(flag_template, help_template, **kwargs):
200      parser.add_argument(
201          flag_template % self.label,
202          help=help_template % self.label,
203          **kwargs)
204
205    add_argument(
206        '--%s-config',
207        '%s configuration',
208        default=default_config)
209    add_argument(
210        '--%s-config-extra-flags',
211        'additional flags passed to the %s run',
212        action='append',
213        default=[])
214    add_argument(
215        '--%s-d8',
216        'optional path to %s d8 executable, '
217        'default: bundled in the directory of this script',
218        default=DEFAULT_D8)
219
220  def make_options(self, options, default_config=None, default_d8=None):
221    def get(name):
222      return getattr(options, '%s_%s' % (self.label, name))
223
224    config = default_config or get('config')
225    assert config in CONFIGS
226
227    d8 = default_d8 or get('d8')
228    if not os.path.isabs(d8):
229      d8 = os.path.join(BASE_PATH, d8)
230    assert os.path.exists(d8)
231
232    flags = CONFIGS[config] + filter_flags(get('config_extra_flags'))
233
234    RunOptions = namedtuple('RunOptions', ['arch', 'config', 'd8', 'flags'])
235    return RunOptions(infer_arch(d8), config, d8, flags)
236
237
238class ExecutionConfig(object):
239  def __init__(self, options, label):
240    self.options = options
241    self.label = label
242    self.arch = getattr(options, label).arch
243    self.config = getattr(options, label).config
244    d8 = getattr(options, label).d8
245    flags = getattr(options, label).flags
246    self.command = Command(options, label, d8, flags)
247
248    # Options for a fallback configuration only exist when comparing
249    # different architectures.
250    fallback_label = label + '_fallback'
251    self.fallback = None
252    if getattr(options, fallback_label, None):
253      self.fallback = ExecutionConfig(options, fallback_label)
254
255  @property
256  def flags(self):
257    return self.command.flags
258
259  @property
260  def is_error_simulation(self):
261    return '--simulate-errors' in self.flags
262
263
264def parse_args():
265  first_config_arguments = ExecutionArgumentsConfig('first')
266  second_config_arguments = ExecutionArgumentsConfig('second')
267
268  parser = argparse.ArgumentParser()
269  parser.add_argument(
270    '--random-seed', type=int, required=True,
271    help='random seed passed to both runs')
272  parser.add_argument(
273      '--skip-smoke-tests', default=False, action='store_true',
274      help='skip smoke tests for testing purposes')
275  parser.add_argument(
276      '--skip-suppressions', default=False, action='store_true',
277      help='skip suppressions to reproduce known issues')
278
279  # Add arguments for each run configuration.
280  first_config_arguments.add_arguments(parser, BASELINE_CONFIG)
281  second_config_arguments.add_arguments(parser, DEFAULT_CONFIG)
282
283  parser.add_argument('testcase', help='path to test case')
284  options = parser.parse_args()
285
286  # Ensure we have a test case.
287  assert (os.path.exists(options.testcase) and
288          os.path.isfile(options.testcase)), (
289      'Test case %s doesn\'t exist' % options.testcase)
290
291  options.first = first_config_arguments.make_options(options)
292  options.second = second_config_arguments.make_options(options)
293  options.default = second_config_arguments.make_options(
294      options, default_config=DEFAULT_CONFIG)
295
296  # Use fallback configurations only on diffrent architectures. In this
297  # case we are going to re-test against the first architecture.
298  if options.first.arch != options.second.arch:
299    options.second_fallback = second_config_arguments.make_options(
300        options, default_d8=options.first.d8)
301    options.default_fallback = second_config_arguments.make_options(
302        options, default_config=DEFAULT_CONFIG, default_d8=options.first.d8)
303
304  # Ensure we make a valid comparison.
305  if (options.first.d8 == options.second.d8 and
306      options.first.config == options.second.config):
307    parser.error('Need either executable or config difference.')
308
309  return options
310
311
312def get_meta_data(content):
313  """Extracts original-source-file paths from test case content."""
314  sources = []
315  for line in content.splitlines():
316    match = SOURCE_RE.match(line)
317    if match:
318      sources.append(match.group(1))
319  return {'sources': sources}
320
321
322def content_bailout(content, ignore_fun):
323  """Print failure state and return if ignore_fun matches content."""
324  bug = (ignore_fun(content) or '').strip()
325  if bug:
326    raise FailException(FAILURE_HEADER_TEMPLATE % dict(
327        configs='', source_key='', suppression=bug))
328
329
330def fail_bailout(output, ignore_by_output_fun):
331  """Print failure state and return if ignore_by_output_fun matches output."""
332  bug = (ignore_by_output_fun(output.stdout) or '').strip()
333  if bug:
334    raise FailException(FAILURE_HEADER_TEMPLATE % dict(
335        configs='', source_key='', suppression=bug))
336
337
338def format_difference(
339    first_config, second_config,
340    first_config_output, second_config_output,
341    difference, source_key=None, source=None):
342  # The first three entries will be parsed by clusterfuzz. Format changes
343  # will require changes on the clusterfuzz side.
344  source_key = source_key or cluster_failures(source)
345  first_config_label = '%s,%s' % (first_config.arch, first_config.config)
346  second_config_label = '%s,%s' % (second_config.arch, second_config.config)
347  source_file_text = SOURCE_FILE_TEMPLATE % source if source else ''
348
349  if PYTHON3:
350    first_stdout = first_config_output.stdout
351    second_stdout = second_config_output.stdout
352  else:
353    first_stdout = first_config_output.stdout.decode('utf-8', 'replace')
354    second_stdout = second_config_output.stdout.decode('utf-8', 'replace')
355    difference = difference.decode('utf-8', 'replace')
356
357  text = (FAILURE_TEMPLATE % dict(
358      configs='%s:%s' % (first_config_label, second_config_label),
359      source_file_text=source_file_text,
360      source_key=source_key,
361      suppression='', # We can't tie bugs to differences.
362      first_config_label=first_config_label,
363      second_config_label=second_config_label,
364      first_config_flags=' '.join(first_config.flags),
365      second_config_flags=' '.join(second_config.flags),
366      first_config_output=first_stdout,
367      second_config_output=second_stdout,
368      source=source,
369      difference=difference,
370  ))
371  if PYTHON3:
372    return text
373  else:
374    return text.encode('utf-8', 'replace')
375
376
377def cluster_failures(source, known_failures=None):
378  """Returns a string key for clustering duplicate failures.
379
380  Args:
381    source: The original source path where the failure happened.
382    known_failures: Mapping from original source path to failure key.
383  """
384  known_failures = known_failures or KNOWN_FAILURES
385  # No source known. Typical for manually uploaded issues. This
386  # requires also manual issue creation.
387  if not source:
388    return ORIGINAL_SOURCE_DEFAULT
389  # Source is known to produce a particular failure.
390  if source in known_failures:
391    return known_failures[source]
392  # Subsume all other sources from CrashTests under one key. Otherwise
393  # failures lead to new crash tests which in turn lead to new failures.
394  if source.startswith('CrashTests'):
395    return ORIGINAL_SOURCE_CRASHTESTS
396
397  # We map all remaining failures to a short hash of the original source.
398  long_key = hashlib.sha1(source.encode('utf-8')).hexdigest()
399  return long_key[:ORIGINAL_SOURCE_HASH_LENGTH]
400
401
402class RepeatedRuns(object):
403  """Helper class for storing statistical data from repeated runs."""
404  def __init__(self, test_case, timeout, verbose):
405    self.test_case = test_case
406    self.timeout = timeout
407    self.verbose = verbose
408
409    # Stores if any run has crashed or was simulated.
410    self.has_crashed = False
411    self.simulated = False
412
413  def run(self, config):
414    comparison_output = config.command.run(
415        self.test_case, timeout=self.timeout, verbose=self.verbose)
416    self.has_crashed = self.has_crashed or comparison_output.HasCrashed()
417    self.simulated = self.simulated or config.is_error_simulation
418    return comparison_output
419
420  @property
421  def crash_state(self):
422    return '_simulated_crash_' if self.simulated else '_unexpected_crash_'
423
424
425def run_comparisons(suppress, execution_configs, test_case, timeout,
426                    verbose=True, ignore_crashes=True, source_key=None):
427  """Runs different configurations and bails out on output difference.
428
429  Args:
430    suppress: The helper object for textual suppressions.
431    execution_configs: Two or more configurations to run. The first one will be
432        used as baseline to compare all others to.
433    test_case: The test case to run.
434    timeout: Timeout in seconds for one run.
435    verbose: Prints the executed commands.
436    ignore_crashes: Typically we ignore crashes during fuzzing as they are
437        frequent. However, when running smoke tests we should not crash
438        and immediately flag crashes as a failure.
439    source_key: A fixed source key. If not given, it will be inferred from the
440        output.
441  """
442  runner = RepeatedRuns(test_case, timeout, verbose)
443
444  # Run the baseline configuration.
445  baseline_config = execution_configs[0]
446  baseline_output = runner.run(baseline_config)
447
448  # Iterate over the remaining configurations, run and compare.
449  for comparison_config in execution_configs[1:]:
450    comparison_output = runner.run(comparison_config)
451    difference, source = suppress.diff(baseline_output, comparison_output)
452
453    if difference:
454      # Only bail out due to suppressed output if there was a difference. If a
455      # suppression doesn't show up anymore in the statistics, we might want to
456      # remove it.
457      fail_bailout(baseline_output, suppress.ignore_by_output)
458      fail_bailout(comparison_output, suppress.ignore_by_output)
459
460      # Check if a difference also occurs with the fallback configuration and
461      # give it precedence. E.g. we always prefer x64 differences.
462      if comparison_config.fallback:
463        fallback_output = runner.run(comparison_config.fallback)
464        fallback_difference, fallback_source = suppress.diff(
465            baseline_output, fallback_output)
466        if fallback_difference:
467          fail_bailout(fallback_output, suppress.ignore_by_output)
468          source = fallback_source
469          comparison_config = comparison_config.fallback
470          comparison_output = fallback_output
471          difference = fallback_difference
472
473      raise FailException(format_difference(
474          baseline_config, comparison_config,
475          baseline_output, comparison_output,
476          difference, source_key, source))
477
478  if runner.has_crashed:
479    if ignore_crashes:
480      # Show if a crash has happened in one of the runs and no difference was
481      # detected. This is only for the statistics during experiments.
482      raise PassException('# V8 correctness - C-R-A-S-H')
483    else:
484      # Subsume simulated and unexpected crashes (e.g. during smoke tests)
485      # with one failure state.
486      raise FailException(FAILURE_HEADER_TEMPLATE % dict(
487          configs='', source_key='', suppression=runner.crash_state))
488
489
490def main():
491  options = parse_args()
492  suppress = v8_suppressions.get_suppression(options.skip_suppressions)
493
494  # Static bailout based on test case content or metadata.
495  kwargs = {}
496  if PYTHON3:
497    kwargs['encoding'] = 'utf-8'
498  with open(options.testcase, 'r', **kwargs) as f:
499    content = f.read()
500  content_bailout(get_meta_data(content), suppress.ignore_by_metadata)
501  content_bailout(content, suppress.ignore_by_content)
502
503  # Prepare the baseline, default and a secondary configuration to compare to.
504  # The default (turbofan) takes precedence as many of the secondary configs
505  # are based on the turbofan config with additional parameters.
506  execution_configs = [
507    ExecutionConfig(options, 'first'),
508    ExecutionConfig(options, 'default'),
509    ExecutionConfig(options, 'second'),
510  ]
511
512  # First, run some fixed smoke tests in all configs to ensure nothing
513  # is fundamentally wrong, in order to prevent bug flooding.
514  if not options.skip_smoke_tests:
515    run_comparisons(
516        suppress, execution_configs,
517        test_case=SMOKE_TESTS,
518        timeout=SMOKE_TEST_TIMEOUT_SEC,
519        verbose=False,
520        # Don't accept crashes during smoke tests. A crash would hint at
521        # a flag that might be incompatible or a broken test file.
522        ignore_crashes=False,
523        # Special source key for smoke tests so that clusterfuzz dedupes all
524        # cases on this in case it's hit.
525        source_key = 'smoke test failed',
526    )
527
528  # Second, run all configs against the fuzz test case.
529  run_comparisons(
530      suppress, execution_configs,
531      test_case=options.testcase,
532      timeout=TEST_TIMEOUT_SEC,
533  )
534
535  # TODO(machenbach): Figure out if we could also return a bug in case
536  # there's no difference, but one of the line suppressions has matched -
537  # and without the match there would be a difference.
538  print('# V8 correctness - pass')
539  return RETURN_PASS
540
541
542if __name__ == "__main__":
543  try:
544    result = main()
545  except FailException as e:
546    print(e.message)
547    result = RETURN_FAIL
548  except PassException as e:
549    print(e.message)
550    result = RETURN_PASS
551  except SystemExit:
552    # Make sure clusterfuzz reports internal errors and wrong usage.
553    # Use one label for all internal and usage errors.
554    print(FAILURE_HEADER_TEMPLATE % dict(
555        configs='', source_key='', suppression='wrong_usage'))
556    result = RETURN_FAIL
557  except MemoryError:
558    # Running out of memory happens occasionally but is not actionable.
559    print('# V8 correctness - pass')
560    result = RETURN_PASS
561  except Exception as e:
562    print(FAILURE_HEADER_TEMPLATE % dict(
563        configs='', source_key='', suppression='internal_error'))
564    print('# Internal error: %s' % e)
565    traceback.print_exc(file=sys.stdout)
566    result = RETURN_FAIL
567
568  sys.exit(result)
569