1#!/usr/bin/env python3
2# Copyright 2016 the V8 project authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6import os
7import random
8import subprocess
9import sys
10import unittest
11
12import v8_commands
13import v8_foozzie
14import v8_fuzz_config
15import v8_suppressions
16
17try:
18  basestring
19except NameError:
20  basestring = str
21
22PYTHON3 = sys.version_info >= (3, 0)
23
24BASE_DIR = os.path.dirname(os.path.abspath(__file__))
25FOOZZIE = os.path.join(BASE_DIR, 'v8_foozzie.py')
26TEST_DATA = os.path.join(BASE_DIR, 'testdata')
27
28KNOWN_BUILDS = [
29  'd8',
30  'clang_x86/d8',
31  'clang_x86_v8_arm/d8',
32  'clang_x64_v8_arm64/d8',
33  'clang_x64_pointer_compression/d8',
34]
35
36
37class ConfigTest(unittest.TestCase):
38  def testExperiments(self):
39    """Test integrity of probabilities and configs."""
40    CONFIGS = v8_foozzie.CONFIGS
41    EXPERIMENTS = v8_fuzz_config.FOOZZIE_EXPERIMENTS
42    FLAGS = v8_fuzz_config.ADDITIONAL_FLAGS
43    # Probabilities add up to 100%.
44    first_is_int = lambda x: type(x[0]) == int
45    assert all(map(first_is_int, EXPERIMENTS))
46    assert sum(x[0] for x in EXPERIMENTS) == 100
47    # Configs used in experiments are defined.
48    assert all(map(lambda x: x[1] in CONFIGS, EXPERIMENTS))
49    assert all(map(lambda x: x[2] in CONFIGS, EXPERIMENTS))
50    # The last config item points to a known build configuration.
51    assert all(map(lambda x: x[3] in KNOWN_BUILDS, EXPERIMENTS))
52    # All flags have a probability.
53    first_is_float = lambda x: type(x[0]) == float
54    assert all(map(first_is_float, FLAGS))
55    first_between_0_and_1 = lambda x: x[0] > 0 and x[0] < 1
56    assert all(map(first_between_0_and_1, FLAGS))
57    # Test consistent flags.
58    second_is_string = lambda x: isinstance(x[1], basestring)
59    assert all(map(second_is_string, FLAGS))
60    # We allow spaces to separate more flags. We don't allow spaces in the flag
61    # value.
62    is_flag = lambda x: x.startswith('--')
63    all_parts_are_flags = lambda x: all(map(is_flag, x[1].split()))
64    assert all(map(all_parts_are_flags, FLAGS))
65
66  def testConfig(self):
67    """Smoke test how to choose experiments."""
68    config = v8_fuzz_config.Config('foo', random.Random(42))
69    experiments = [
70      [25, 'ignition', 'jitless', 'd8'],
71      [75, 'ignition', 'ignition', 'clang_x86/d8'],
72    ]
73    flags = [
74      [0.1, '--flag'],
75      [0.3, '--baz'],
76      [0.3, '--foo --bar'],
77    ]
78    self.assertEqual(
79        [
80          '--first-config=ignition',
81          '--second-config=jitless',
82          '--second-d8=d8',
83          '--second-config-extra-flags=--baz',
84          '--second-config-extra-flags=--foo',
85          '--second-config-extra-flags=--bar',
86        ],
87        config.choose_foozzie_flags(experiments, flags),
88    )
89    self.assertEqual(
90        [
91          '--first-config=ignition',
92          '--second-config=jitless',
93          '--second-d8=d8',
94        ],
95        config.choose_foozzie_flags(experiments, flags),
96    )
97
98
99class UnitTest(unittest.TestCase):
100  def testCluster(self):
101    crash_test_example_path = 'CrashTests/path/to/file.js'
102    self.assertEqual(
103        v8_foozzie.ORIGINAL_SOURCE_DEFAULT,
104        v8_foozzie.cluster_failures(''))
105    self.assertEqual(
106        v8_foozzie.ORIGINAL_SOURCE_CRASHTESTS,
107        v8_foozzie.cluster_failures(crash_test_example_path))
108    self.assertEqual(
109        '_o_O_',
110        v8_foozzie.cluster_failures(
111            crash_test_example_path,
112            known_failures={crash_test_example_path: '_o_O_'}))
113    self.assertEqual(
114        '980',
115        v8_foozzie.cluster_failures('v8/test/mjsunit/apply.js'))
116
117  def testDiff(self):
118    def diff_fun(one, two, skip=False):
119      suppress = v8_suppressions.get_suppression(skip)
120      return suppress.diff_lines(one.splitlines(), two.splitlines())
121
122    one = ''
123    two = ''
124    diff = None, None
125    self.assertEqual(diff, diff_fun(one, two))
126
127    one = 'a \n  b\nc();'
128    two = 'a \n  b\nc();'
129    diff = None, None
130    self.assertEqual(diff, diff_fun(one, two))
131
132    # Ignore line before caret and caret position.
133    one = """
134undefined
135weird stuff
136      ^
137somefile.js: TypeError: suppressed message
138  undefined
139"""
140    two = """
141undefined
142other weird stuff
143            ^
144somefile.js: TypeError: suppressed message
145  undefined
146"""
147    diff = None, None
148    self.assertEqual(diff, diff_fun(one, two))
149
150    one = """
151Still equal
152Extra line
153"""
154    two = """
155Still equal
156"""
157    diff = '- Extra line', None
158    self.assertEqual(diff, diff_fun(one, two))
159
160    one = """
161Still equal
162"""
163    two = """
164Still equal
165Extra line
166"""
167    diff = '+ Extra line', None
168    self.assertEqual(diff, diff_fun(one, two))
169
170    one = """
171undefined
172somefile.js: TypeError: undefined is not a constructor
173"""
174    two = """
175undefined
176otherfile.js: TypeError: undefined is not a constructor
177"""
178    diff = """- somefile.js: TypeError: undefined is not a constructor
179+ otherfile.js: TypeError: undefined is not a constructor""", None
180    self.assertEqual(diff, diff_fun(one, two))
181
182    # Test that skipping suppressions works.
183    one = """
184v8-foozzie source: foo
185weird stuff
186      ^
187"""
188    two = """
189v8-foozzie source: foo
190other weird stuff
191            ^
192"""
193    self.assertEqual((None, 'foo'), diff_fun(one, two))
194    diff = ('-       ^\n+             ^', 'foo')
195    self.assertEqual(diff, diff_fun(one, two, skip=True))
196
197  def testOutputCapping(self):
198    def output(stdout, is_crash):
199      exit_code = -1 if is_crash else 0
200      return v8_commands.Output(exit_code=exit_code, stdout=stdout, pid=0)
201
202    def check(stdout1, stdout2, is_crash1, is_crash2, capped_lines1,
203              capped_lines2):
204      output1 = output(stdout1, is_crash1)
205      output2 = output(stdout2, is_crash2)
206      self.assertEqual(
207          (capped_lines1, capped_lines2),
208          v8_suppressions.get_output_capped(output1, output2))
209
210    # No capping, already equal.
211    check('1\n2', '1\n2', True, True, '1\n2', '1\n2')
212    # No crash, no capping.
213    check('1\n2', '1\n2\n3', False, False, '1\n2', '1\n2\n3')
214    check('1\n2\n3', '1\n2', False, False, '1\n2\n3', '1\n2')
215    # Cap smallest if all runs crash.
216    check('1\n2', '1\n2\n3', True, True, '1\n2', '1\n2')
217    check('1\n2\n3', '1\n2', True, True, '1\n2', '1\n2')
218    check('1\n2', '1\n23', True, True, '1\n2', '1\n2')
219    check('1\n23', '1\n2', True, True, '1\n2', '1\n2')
220    # Cap the non-crashy run.
221    check('1\n2\n3', '1\n2', False, True, '1\n2', '1\n2')
222    check('1\n2', '1\n2\n3', True, False, '1\n2', '1\n2')
223    check('1\n23', '1\n2', False, True, '1\n2', '1\n2')
224    check('1\n2', '1\n23', True, False, '1\n2', '1\n2')
225    # The crashy run has more output.
226    check('1\n2\n3', '1\n2', True, False, '1\n2\n3', '1\n2')
227    check('1\n2', '1\n2\n3', False, True, '1\n2', '1\n2\n3')
228    check('1\n23', '1\n2', True, False, '1\n23', '1\n2')
229    check('1\n2', '1\n23', False, True, '1\n2', '1\n23')
230    # Keep output difference when capping.
231    check('1\n2', '3\n4\n5', True, True, '1\n2', '3\n4')
232    check('1\n2\n3', '4\n5', True, True, '1\n2', '4\n5')
233    check('12', '345', True, True, '12', '34')
234    check('123', '45', True, True, '12', '45')
235
236
237def cut_verbose_output(stdout, n_comp):
238  # This removes the first lines containing d8 commands of `n_comp` comparison
239  # runs.
240  return '\n'.join(stdout.split('\n')[n_comp * 2:])
241
242
243def run_foozzie(second_d8_dir, *extra_flags, **kwargs):
244  second_config = 'ignition_turbo'
245  if 'second_config' in kwargs:
246    second_config = 'jitless'
247  kwargs = {}
248  if PYTHON3:
249    kwargs['text'] = True
250  return subprocess.check_output([
251    sys.executable, FOOZZIE,
252    '--random-seed', '12345',
253    '--first-d8', os.path.join(TEST_DATA, 'baseline', 'd8.py'),
254    '--second-d8', os.path.join(TEST_DATA, second_d8_dir, 'd8.py'),
255    '--first-config', 'ignition',
256    '--second-config', second_config,
257    os.path.join(TEST_DATA, 'fuzz-123.js'),
258  ] + list(extra_flags), **kwargs)
259
260class SystemTest(unittest.TestCase):
261  """This tests the whole correctness-fuzzing harness with fake build
262  artifacts.
263
264  Overview of fakes:
265    baseline: Example foozzie output including a syntax error.
266    build1: Difference to baseline is a stack trace difference expected to
267            be suppressed.
268    build2: Difference to baseline is a non-suppressed output difference
269            causing the script to fail.
270    build3: As build1 but with an architecture difference as well.
271  """
272  def testSyntaxErrorDiffPass(self):
273    stdout = run_foozzie('build1', '--skip-smoke-tests')
274    self.assertEqual('# V8 correctness - pass\n',
275                     cut_verbose_output(stdout, 3))
276    # Default comparison includes suppressions.
277    self.assertIn('v8_suppressions.js', stdout)
278    # Default comparison doesn't include any specific mock files.
279    self.assertNotIn('v8_mock_archs.js', stdout)
280    self.assertNotIn('v8_mock_webassembly.js', stdout)
281
282  def testDifferentOutputFail(self):
283    with open(os.path.join(TEST_DATA, 'failure_output.txt')) as f:
284      expected_output = f.read()
285    with self.assertRaises(subprocess.CalledProcessError) as ctx:
286      run_foozzie('build2', '--skip-smoke-tests',
287                  '--first-config-extra-flags=--flag1',
288                  '--first-config-extra-flags=--flag2=0',
289                  '--second-config-extra-flags=--flag3')
290    e = ctx.exception
291    self.assertEqual(v8_foozzie.RETURN_FAIL, e.returncode)
292    self.assertEqual(expected_output, cut_verbose_output(e.output, 2))
293
294  def testSmokeTest(self):
295    with open(os.path.join(TEST_DATA, 'smoke_test_output.txt')) as f:
296      expected_output = f.read()
297    with self.assertRaises(subprocess.CalledProcessError) as ctx:
298      run_foozzie('build2')
299    e = ctx.exception
300    self.assertEqual(v8_foozzie.RETURN_FAIL, e.returncode)
301    self.assertEqual(expected_output, e.output)
302
303  def testDifferentArch(self):
304    """Test that the architecture-specific mocks are passed to both runs when
305    we use executables with different architectures.
306    """
307    # Build 3 simulates x86, while the baseline is x64.
308    stdout = run_foozzie('build3', '--skip-smoke-tests')
309    lines = stdout.split('\n')
310    # TODO(machenbach): Don't depend on the command-lines being printed in
311    # particular lines.
312    self.assertIn('v8_mock_archs.js', lines[1])
313    self.assertIn('v8_mock_archs.js', lines[3])
314
315  def testDifferentArchFailFirst(self):
316    """Test that we re-test against x64. This tests the path that also fails
317    on x64 and then reports the error as x64.
318    """
319    with open(os.path.join(TEST_DATA, 'failure_output_arch.txt')) as f:
320      expected_output = f.read()
321    # Build 3 simulates x86 and produces a difference on --bad-flag, but
322    # the baseline build shows the same difference when --bad-flag is passed.
323    with self.assertRaises(subprocess.CalledProcessError) as ctx:
324      run_foozzie('build3', '--skip-smoke-tests',
325                  '--second-config-extra-flags=--bad-flag')
326    e = ctx.exception
327    self.assertEqual(v8_foozzie.RETURN_FAIL, e.returncode)
328    self.assertEqual(expected_output, cut_verbose_output(e.output, 3))
329
330  def testDifferentArchFailSecond(self):
331    """As above, but we test the path that only fails in the second (ia32)
332    run and not with x64 and then reports the error as ia32.
333    """
334    with open(os.path.join(TEST_DATA, 'failure_output_second.txt')) as f:
335      expected_output = f.read()
336    # Build 3 simulates x86 and produces a difference on --very-bad-flag,
337    # which the baseline build doesn't.
338    with self.assertRaises(subprocess.CalledProcessError) as ctx:
339      run_foozzie('build3', '--skip-smoke-tests',
340                  '--second-config-extra-flags=--very-bad-flag')
341    e = ctx.exception
342    self.assertEqual(v8_foozzie.RETURN_FAIL, e.returncode)
343    self.assertEqual(expected_output, cut_verbose_output(e.output, 3))
344
345  def testJitless(self):
346    """Test that webassembly is mocked out when comparing with jitless."""
347    stdout = run_foozzie(
348        'build1', '--skip-smoke-tests', second_config='jitless')
349    lines = stdout.split('\n')
350    # TODO(machenbach): Don't depend on the command-lines being printed in
351    # particular lines.
352    self.assertIn('v8_mock_webassembly.js', lines[1])
353    self.assertIn('v8_mock_webassembly.js', lines[3])
354
355  def testSkipSuppressions(self):
356    """Test that the suppressions file is not passed when skipping
357    suppressions.
358    """
359    # Compare baseline with baseline. This passes as there is no difference.
360    stdout = run_foozzie(
361        'baseline', '--skip-smoke-tests', '--skip-suppressions')
362    self.assertNotIn('v8_suppressions.js', stdout)
363
364    # Compare with a build that usually suppresses a difference. Now we fail
365    # since we skip suppressions.
366    with self.assertRaises(subprocess.CalledProcessError) as ctx:
367      run_foozzie(
368          'build1', '--skip-smoke-tests', '--skip-suppressions')
369    e = ctx.exception
370    self.assertEqual(v8_foozzie.RETURN_FAIL, e.returncode)
371    self.assertNotIn('v8_suppressions.js', e.output)
372
373
374if __name__ == '__main__':
375  unittest.main()
376