xref: /third_party/vixl/tools/test.py (revision b8021494) 
1#!/usr/bin/env python3
2
3# Copyright 2015, VIXL authors
4# All rights reserved.
5#
6# Redistribution and use in source and binary forms, with or without
7# modification, are permitted provided that the following conditions are met:
8#
9#   * Redistributions of source code must retain the above copyright notice,
10#     this list of conditions and the following disclaimer.
11#   * Redistributions in binary form must reproduce the above copyright notice,
12#     this list of conditions and the following disclaimer in the documentation
13#     and/or other materials provided with the distribution.
14#   * Neither the name of ARM Limited nor the names of its contributors may be
15#     used to endorse or promote products derived from this software without
16#     specific prior written permission.
17#
18# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND
19# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
20# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
21# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
22# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
24# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
25# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
26# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29import argparse
30import fcntl
31import itertools
32import multiprocessing
33import os
34from os.path import join
35import platform
36import subprocess
37import sys
38import time
39
40import config
41import clang_format
42import clang_tidy
43import lint
44import printer
45import test
46import test_runner
47import util
48
49
50dir_root = config.dir_root
51
52
53# Remove duplicates from a list
54def RemoveDuplicates(values):
55  # Convert the list into a set and back to list
56  # as sets guarantee items are unique.
57  return list(set(values))
58
59
60# Custom argparse.Action to automatically add and handle an 'all' option.
61# If no 'default' value is set, it will default to 'all.
62# If accepted options are set using 'choices' then only these values will be
63# allowed.
64# If they're set using 'soft_choices' then 'all' will default to these values,
65# but other values will also be accepted.
66class AllChoiceAction(argparse.Action):
67
68  # At least one option was set by the user.
69  WasSetByUser = False
70
71  def __init__(self, **kwargs):
72    if 'choices' in kwargs:
73      assert 'soft_choices' not in kwargs,\
74          "Can't have both 'choices' and 'soft_choices' options"
75      self.all_choices = list(kwargs['choices'])
76      kwargs['choices'].append('all')
77    else:
78      self.all_choices = kwargs['soft_choices']
79      kwargs['help'] += ' Supported values: {' + ','.join(
80          ['all'] + self.all_choices) + '}'
81      del kwargs['soft_choices']
82    if 'default' not in kwargs:
83      kwargs['default'] = self.all_choices
84    super(AllChoiceAction, self).__init__(**kwargs)
85
86  def __call__(self, parser, namespace, values, option_string=None):
87    AllChoiceAction.WasSetByUser = True
88    if 'all' in values:
89      # Substitute 'all' by the actual values.
90      values = self.all_choices + [value for value in values if value != 'all']
91
92    setattr(namespace, self.dest, RemoveDuplicates(values))
93
94
95def BuildOptions():
96  args = argparse.ArgumentParser(
97    description =
98    '''This tool runs all tests matching the specified filters for multiple
99    environment, build options, and runtime options configurations.''',
100    # Print default values.
101    formatter_class=argparse.ArgumentDefaultsHelpFormatter)
102
103  args.add_argument('filters', metavar='filter', nargs='*',
104                    help='Run tests matching all of the (regexp) filters.')
105
106  # We automatically build the script options from the options to be tested.
107  test_arguments = args.add_argument_group(
108    'Test options',
109    'These options indicate what should be tested')
110  test_arguments.add_argument(
111      '--negative_testing',
112      help='Tests with negative testing enabled.',
113      action='store_const',
114      const='on',
115      default='off')
116  test_arguments.add_argument(
117      '--compiler',
118      help='Test for the specified compilers.',
119      soft_choices=config.tested_compilers,
120      action=AllChoiceAction,
121      nargs="+")
122  test_arguments.add_argument(
123      '--mode',
124      help='Test with the specified build modes.',
125      choices=config.build_options_modes,
126      action=AllChoiceAction,
127      nargs="+")
128  test_arguments.add_argument(
129      '--std',
130      help='Test with the specified C++ standard.',
131      soft_choices=config.tested_cpp_standards,
132      action=AllChoiceAction,
133      nargs="+")
134  test_arguments.add_argument(
135      '--target',
136      help='Test with the specified isa enabled.',
137      soft_choices=config.build_options_target,
138      action=AllChoiceAction,
139      nargs="+")
140
141  general_arguments = args.add_argument_group('General options')
142  general_arguments.add_argument('--dry-run', action='store_true',
143                                 help='''Don't actually build or run anything,
144                                 but print the configurations that would be
145                                 tested.''')
146  general_arguments.add_argument('--verbose', action='store_true',
147                                 help='''Print extra information.''')
148  general_arguments.add_argument(
149    '--jobs', '-j', metavar='N', type=int, nargs='?',
150    default=multiprocessing.cpu_count(),
151    const=multiprocessing.cpu_count(),
152    help='''Runs the tests using N jobs. If the option is set but no value is
153    provided, the script will use as many jobs as it thinks useful.''')
154  general_arguments.add_argument('--clang-format',
155                                 default=clang_format.DEFAULT_CLANG_FORMAT,
156                                 help='Path to clang-format.')
157  general_arguments.add_argument('--clang-tidy',
158                                 default=clang_tidy.DEFAULT_CLANG_TIDY,
159                                 help='Path to clang-tidy.')
160  general_arguments.add_argument('--nobench', action='store_true',
161                                 help='Do not run benchmarks.')
162  general_arguments.add_argument('--nolint', action='store_true',
163                                 help='Do not run the linter.')
164  general_arguments.add_argument('--noclang-format', action='store_true',
165                                 help='Do not run clang-format.')
166  general_arguments.add_argument('--noclang-tidy', action='store_true',
167                                 help='Do not run clang-tidy.')
168  general_arguments.add_argument('--notest', action='store_true',
169                                 help='Do not run tests.')
170  general_arguments.add_argument('--nocheck-code-coverage', action='store_true',
171                                 help='Do not check code coverage results log.')
172  general_arguments.add_argument('--fail-early', action='store_true',
173                                 help='Exit as soon as a test fails.')
174  general_arguments.add_argument(
175    '--under_valgrind', action='store_true',
176    help='''Run the test-runner commands under Valgrind.
177            Note that a few tests are known to fail because of
178            issues in Valgrind''')
179  return args.parse_args()
180
181
182def RunCommand(command, environment_options = None):
183  # Create a copy of the environment. We do not want to pollute the environment
184  # of future commands run.
185  environment = os.environ.copy()
186
187  printable_command = ''
188  if environment_options:
189    # Add the environment options to the environment:
190    environment.update(environment_options)
191    printable_command += ' ' + DictToString(environment_options) + ' '
192  printable_command += ' '.join(command)
193
194  printable_command_orange = \
195    printer.COLOUR_ORANGE + printable_command + printer.NO_COLOUR
196  printer.PrintOverwritableLine(printable_command_orange)
197  sys.stdout.flush()
198
199  # Start a process for the command.
200  # Interleave `stderr` and `stdout`.
201  p = subprocess.Popen(command,
202                       stdout=subprocess.PIPE,
203                       stderr=subprocess.STDOUT,
204                       env=environment)
205
206  # We want to be able to display a continuously updated 'work indicator' while
207  # the process is running. Since the process can hang if the `stdout` pipe is
208  # full, we need to pull from it regularly. We cannot do so via the
209  # `readline()` function because it is blocking, and would thus cause the
210  # indicator to not be updated properly. So use file control mechanisms
211  # instead.
212  indicator = ' (still working: %d seconds elapsed)'
213
214  # Mark the process output as non-blocking.
215  flags = fcntl.fcntl(p.stdout, fcntl.F_GETFL)
216  fcntl.fcntl(p.stdout, fcntl.F_SETFL, flags | os.O_NONBLOCK)
217
218  t_start = time.time()
219  t_current = t_start
220  t_last_indication = t_start
221  t_current = t_start
222  process_output = b''
223
224  # Keep looping as long as the process is running.
225  while p.poll() is None:
226    # Avoid polling too often.
227    time.sleep(0.1)
228    # Update the progress indicator.
229    t_current = time.time()
230    if (t_current - t_start >= 2) and (t_current - t_last_indication >= 1):
231      printer.PrintOverwritableLine(
232        printable_command_orange + indicator % int(t_current - t_start))
233      sys.stdout.flush()
234      t_last_indication = t_current
235    # Pull from the process output.
236    while True:
237      try:
238        line = os.read(p.stdout.fileno(), 1024)
239      except OSError:
240        line = b''
241        break
242      if line == b'': break
243      process_output += line
244
245  # The process has exited. Don't forget to retrieve the rest of its output.
246  out, err = p.communicate()
247  rc = p.poll()
248  process_output += out
249
250  printable_command += ' (took %d seconds)' % int(t_current - t_start)
251  if rc == 0:
252    printer.Print(printer.COLOUR_GREEN + printable_command + printer.NO_COLOUR)
253  else:
254    printer.Print(printer.COLOUR_RED + printable_command + printer.NO_COLOUR)
255    printer.Print(process_output.decode())
256  return rc
257
258
259def RunLinter(jobs):
260  return lint.RunLinter([join(dir_root, x) for x in util.get_source_files()],
261                        jobs = args.jobs, progress_prefix = 'cpp lint: ')
262
263
264def RunClangFormat(clang_path, jobs):
265  return clang_format.ClangFormatFiles(util.get_source_files(exclude_dirs=['.*', '*/traces/*', '*/aarch32/*']),
266                                       clang_path,
267                                       jobs = jobs,
268                                       progress_prefix = 'clang-format: ')
269
270def RunClangTidy(clang_path, jobs):
271  return clang_tidy.ClangTidyFiles(util.get_source_files(exclude_dirs=['.*', '*/traces/*', '*/aarch32/*']),
272                                   clang_path,
273                                   jobs = jobs,
274                                   progress_prefix = 'clang-tidy: ')
275
276def CheckCodeCoverage():
277  command = ['tools/check_recent_coverage.sh']
278  return RunCommand(command)
279
280def BuildAll(build_options, jobs, environment_options):
281  scons_command = ['scons', '-C', dir_root, 'all', '-j', str(jobs)]
282  if util.IsCommandAvailable('ccache'):
283    scons_command += ['compiler_wrapper=ccache']
284    # Fixes warnings for ccache 3.3.1 and lower:
285    environment_options = environment_options.copy()
286    environment_options["CCACHE_CPP2"] = 'yes'
287  scons_command += DictToString(build_options).split()
288  return RunCommand(scons_command, environment_options)
289
290
291def CanRunAarch64(options, args):
292  for target in options['target']:
293    if target in ['aarch64', 'a64']:
294      return True
295
296  return False
297
298
299def CanRunAarch32(options, args):
300  for target in options['target']:
301    if target in ['aarch32', 'a32', 't32']:
302      return True
303  return False
304
305
306def RunBenchmarks(options, args):
307  rc = 0
308  if CanRunAarch32(options, args):
309    benchmark_names = util.ListCCFilesWithoutExt(config.dir_aarch32_benchmarks)
310    for bench in benchmark_names:
311      rc |= RunCommand(
312        [os.path.realpath(
313          join(config.dir_build_latest, 'benchmarks/aarch32', bench)), '10'])
314  if CanRunAarch64(options, args):
315    benchmark_names = util.ListCCFilesWithoutExt(config.dir_aarch64_benchmarks)
316    for bench in benchmark_names:
317      rc |= RunCommand(
318        [util.relrealpath(
319            join(config.dir_build_latest,
320                'benchmarks/aarch64', bench)), '10'])
321  return rc
322
323
324
325# It is a precommit run if the user did not specify any of the
326# options that would affect the automatically generated combinations.
327def IsPrecommitRun(args):
328  return args.negative_testing == "off" and not AllChoiceAction.WasSetByUser
329
330# Generate a list of all the possible combinations of the passed list:
331# ListCombinations( a = [a0, a1], b = [b0, b1] ) will return
332# [ {a : a0, b : b0}, {a : a0, b : b1}, {a: a1, b : b0}, {a : a1, b : b1}]
333def ListCombinations(**kwargs):
334  # End of recursion: no options passed
335  if not kwargs:
336    return [{}]
337  option, values = kwargs.popitem()
338  configs = ListCombinations(**kwargs)
339  retval = []
340  if not isinstance(values, list):
341    values = [values]
342  for value in values:
343    for config in configs:
344      new_config = config.copy()
345      new_config[option] = value
346      retval.append(new_config)
347  return retval
348
349# Convert a dictionary into a space separated string
350# {a : a0, b : b0} --> "a=a0 b=b0"
351def DictToString(options):
352  return " ".join(
353      ["{}={}".format(option, value) for option, value in options.items()])
354
355
356if __name__ == '__main__':
357  util.require_program('scons')
358
359  args = BuildOptions()
360
361  rc = util.ReturnCode(args.fail_early, printer.Print)
362
363  if args.under_valgrind:
364    util.require_program('valgrind')
365
366  if not args.nocheck_code_coverage:
367    rc.Combine(CheckCodeCoverage())
368
369  tests = test_runner.TestQueue()
370  if not args.nolint and not args.dry_run:
371    rc.Combine(RunLinter(args.jobs))
372
373  if not args.noclang_format and not args.dry_run:
374    rc.Combine(RunClangFormat(args.clang_format, args.jobs))
375
376  if not args.noclang_tidy and not args.dry_run:
377    rc.Combine(RunClangTidy(args.clang_tidy, args.jobs))
378
379  list_options = []
380  if IsPrecommitRun(args):
381    # Maximize the coverage for precommit testing.
382
383    # Debug builds with negative testing and all targets enabled.
384    list_options += ListCombinations(
385        compiler = args.compiler,
386        negative_testing = 'on',
387        std = args.std,
388        mode = 'debug',
389        target = 'a64,a32,t32')
390
391    # Release builds with all targets enabled.
392    list_options += ListCombinations(
393        compiler = args.compiler,
394        negative_testing = 'off',
395        std = args.std,
396        mode = 'release',
397        target = 'a64,a32,t32')
398
399    # Debug builds for individual targets.
400    list_options += ListCombinations(
401        compiler = args.compiler[0],
402        negative_testing = 'off',
403        std = args.std,
404        mode = 'debug',
405        target = ['a32', 't32', 'a64'])
406  else:
407    list_options = ListCombinations(
408        compiler = args.compiler,
409        negative_testing = args.negative_testing,
410        std = args.std,
411        mode = args.mode,
412        target = args.target)
413
414  for options in list_options:
415    if (args.dry_run):
416      print(DictToString(options))
417      continue
418    # Convert 'compiler' into an environment variable:
419    environment_options = {'CXX': options['compiler']}
420    del options['compiler']
421
422    # Avoid going through the build stage if we are not using the build
423    # result.
424    if not (args.notest and args.nobench):
425      build_rc = BuildAll(options, args.jobs, environment_options)
426      # Don't run the tests for this configuration if the build failed.
427      if build_rc != 0:
428        rc.Combine(build_rc)
429        continue
430
431      # Use the realpath of the test executable so that the commands printed
432      # can be copy-pasted and run.
433      test_executable = util.relrealpath(
434        join(config.dir_build_latest, 'test', 'test-runner'))
435
436      if not args.notest:
437        printer.Print(test_executable)
438        tests.AddTests(
439            test_executable,
440            args.filters,
441            list(),
442            args.under_valgrind)
443
444      if not args.nobench:
445        rc.Combine(RunBenchmarks(options, args))
446
447  rc.Combine(tests.Run(args.jobs, args.verbose))
448  if not args.dry_run:
449    rc.PrintStatus()
450
451  sys.exit(rc.Value)
452