1#!/usr/bin/env python3
2#
3# Copyright 2012 the V8 project authors. All rights reserved.
4# Redistribution and use in source and binary forms, with or without
5# modification, are permitted provided that the following conditions are
6# met:
7#
8#     * Redistributions of source code must retain the above copyright
9#       notice, this list of conditions and the following disclaimer.
10#     * Redistributions in binary form must reproduce the above
11#       copyright notice, this list of conditions and the following
12#       disclaimer in the documentation and/or other materials provided
13#       with the distribution.
14#     * Neither the name of Google Inc. nor the names of its
15#       contributors may be used to endorse or promote products derived
16#       from this software without specific prior written permission.
17#
18# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30import hashlib
31md5er = hashlib.md5
32
33
34import json
35import multiprocessing
36import optparse
37import os
38from os.path import abspath, join, dirname, basename, exists
39import pickle
40import re
41import subprocess
42from subprocess import PIPE
43import sys
44
45from testrunner.local import statusfile
46from testrunner.local import testsuite
47from testrunner.local import utils
48
49def decode(arg, encoding="utf-8"):
50  return arg.decode(encoding)
51
52# Special LINT rules diverging from default and reason.
53# build/header_guard: Our guards have the form "V8_FOO_H_", not "SRC_FOO_H_".
54#   We now run our own header guard check in PRESUBMIT.py.
55# build/include_what_you_use: Started giving false positives for variables
56#   named "string" and "map" assuming that you needed to include STL headers.
57# runtime/references: As of May 2020 the C++ style guide suggests using
58#   references for out parameters, see
59#   https://google.github.io/styleguide/cppguide.html#Inputs_and_Outputs.
60# whitespace/braces: Doesn't handle {}-initialization for custom types
61#   well; also should be subsumed by clang-format.
62
63LINT_RULES = """
64-build/header_guard
65-build/include_what_you_use
66-readability/fn_size
67-readability/multiline_comment
68-runtime/references
69-whitespace/braces
70-whitespace/comments
71""".split()
72
73LINT_OUTPUT_PATTERN = re.compile(r'^.+[:(]\d+[:)]')
74FLAGS_LINE = re.compile("//\s*Flags:.*--([A-z0-9-])+_[A-z0-9].*\n")
75ASSERT_OPTIMIZED_PATTERN = re.compile("assertOptimized")
76FLAGS_ENABLE_OPT = re.compile("//\s*Flags:.*--opt[^-].*\n")
77ASSERT_UNOPTIMIZED_PATTERN = re.compile("assertUnoptimized")
78FLAGS_NO_ALWAYS_OPT = re.compile("//\s*Flags:.*--no-?always-opt.*\n")
79
80TOOLS_PATH = dirname(abspath(__file__))
81DEPS_DEPOT_TOOLS_PATH = abspath(
82    join(TOOLS_PATH, '..', 'third_party', 'depot_tools'))
83
84
85def CppLintWorker(command):
86  try:
87    process = subprocess.Popen(command, stderr=subprocess.PIPE)
88    process.wait()
89    out_lines = ""
90    error_count = -1
91    while True:
92      out_line = decode(process.stderr.readline())
93      if out_line == '' and process.poll() != None:
94        if error_count == -1:
95          print("Failed to process %s" % command.pop())
96          return 1
97        break
98      if out_line.strip() == 'Total errors found: 0':
99        out_lines += "Done processing %s\n" % command.pop()
100        error_count += 1
101      else:
102        m = LINT_OUTPUT_PATTERN.match(out_line)
103        if m:
104          out_lines += out_line
105          error_count += 1
106    sys.stdout.write(out_lines)
107    return error_count
108  except KeyboardInterrupt:
109    process.kill()
110  except:
111    print('Error running cpplint.py. Please make sure you have depot_tools' +
112          ' in your third_party directory. Lint check skipped.')
113    process.kill()
114
115def TorqueLintWorker(command):
116  try:
117    process = subprocess.Popen(command, stderr=subprocess.PIPE)
118    process.wait()
119    out_lines = ""
120    error_count = 0
121    while True:
122      out_line = decode(process.stderr.readline())
123      if out_line == '' and process.poll() != None:
124        break
125      out_lines += out_line
126      error_count += 1
127    sys.stdout.write(out_lines)
128    if error_count != 0:
129      sys.stdout.write(
130          "warning: formatting and overwriting unformatted Torque files\n")
131    return error_count
132  except KeyboardInterrupt:
133    process.kill()
134  except:
135    print('Error running format-torque.py')
136    process.kill()
137
138def JSLintWorker(command):
139  def format_file(command):
140    try:
141      file_name = command[-1]
142      with open(file_name, "r") as file_handle:
143        contents = file_handle.read()
144
145      process = subprocess.Popen(command, stdout=PIPE, stderr=subprocess.PIPE)
146      output, err = process.communicate()
147      rc = process.returncode
148      if rc != 0:
149        sys.stdout.write("error code " + str(rc) + " running clang-format.\n")
150        return rc
151
152      if decode(output) != contents:
153        return 1
154
155      return 0
156    except KeyboardInterrupt:
157      process.kill()
158    except Exception:
159      print(
160          'Error running clang-format. Please make sure you have depot_tools' +
161          ' in your third_party directory. Lint check skipped.')
162      process.kill()
163
164  rc = format_file(command)
165  if rc == 1:
166    # There are files that need to be formatted, let's format them in place.
167    file_name = command[-1]
168    sys.stdout.write("Formatting %s.\n" % (file_name))
169    rc = format_file(command[:-1] + ["-i", file_name])
170  return rc
171
172class FileContentsCache(object):
173
174  def __init__(self, sums_file_name):
175    self.sums = {}
176    self.sums_file_name = sums_file_name
177
178  def Load(self):
179    try:
180      sums_file = None
181      try:
182        sums_file = open(self.sums_file_name, 'rb')
183        self.sums = pickle.load(sums_file)
184      except:
185        # Cannot parse pickle for any reason. Not much we can do about it.
186        pass
187    finally:
188      if sums_file:
189        sums_file.close()
190
191  def Save(self):
192    try:
193      sums_file = open(self.sums_file_name, 'wb')
194      pickle.dump(self.sums, sums_file)
195    except:
196      # Failed to write pickle. Try to clean-up behind us.
197      if sums_file:
198        sums_file.close()
199      try:
200        os.unlink(self.sums_file_name)
201      except:
202        pass
203    finally:
204      sums_file.close()
205
206  def FilterUnchangedFiles(self, files):
207    changed_or_new = []
208    for file in files:
209      try:
210        handle = open(file, "rb")
211        file_sum = md5er(handle.read()).digest()
212        if not file in self.sums or self.sums[file] != file_sum:
213          changed_or_new.append(file)
214          self.sums[file] = file_sum
215      finally:
216        handle.close()
217    return changed_or_new
218
219  def RemoveFile(self, file):
220    if file in self.sums:
221      self.sums.pop(file)
222
223
224class SourceFileProcessor(object):
225  """
226  Utility class that can run through a directory structure, find all relevant
227  files and invoke a custom check on the files.
228  """
229
230  def RunOnPath(self, path):
231    """Runs processor on all files under the given path."""
232
233    all_files = []
234    for file in self.GetPathsToSearch():
235      all_files += self.FindFilesIn(join(path, file))
236    return self.ProcessFiles(all_files)
237
238  def RunOnFiles(self, files):
239    """Runs processor only on affected files."""
240
241    # Helper for getting directory pieces.
242    dirs = lambda f: dirname(f).split(os.sep)
243
244    # Path offsets where to look (to be in sync with RunOnPath).
245    # Normalize '.' to check for it with str.startswith.
246    search_paths = [('' if p == '.' else p) for p in self.GetPathsToSearch()]
247
248    all_files = [
249      f.AbsoluteLocalPath()
250      for f in files
251      if (not self.IgnoreFile(f.LocalPath()) and
252          self.IsRelevant(f.LocalPath()) and
253          all(not self.IgnoreDir(d) for d in dirs(f.LocalPath())) and
254          any(map(f.LocalPath().startswith, search_paths)))
255    ]
256
257    return self.ProcessFiles(all_files)
258
259  def IgnoreDir(self, name):
260    return (name.startswith('.') or
261            name in ('buildtools', 'data', 'gmock', 'gtest', 'kraken',
262                     'octane', 'sunspider', 'traces-arm64'))
263
264  def IgnoreFile(self, name):
265    return name.startswith('.')
266
267  def FindFilesIn(self, path):
268    result = []
269    for (root, dirs, files) in os.walk(path):
270      for ignored in [x for x in dirs if self.IgnoreDir(x)]:
271        dirs.remove(ignored)
272      for file in files:
273        if not self.IgnoreFile(file) and self.IsRelevant(file):
274          result.append(join(root, file))
275    return result
276
277
278class CacheableSourceFileProcessor(SourceFileProcessor):
279  """Utility class that allows caching ProcessFiles() method calls.
280
281  In order to use it, create a ProcessFilesWithoutCaching method that returns
282  the files requiring intervention after processing the source files.
283  """
284
285  def __init__(self, use_cache, cache_file_path, file_type):
286    self.use_cache = use_cache
287    self.cache_file_path = cache_file_path
288    self.file_type = file_type
289
290  def GetProcessorWorker(self):
291    """Expected to return the worker function to run the formatter."""
292    raise NotImplementedError
293
294  def GetProcessorScript(self):
295    """Expected to return a tuple
296    (path to the format processor script, list of arguments)."""
297    raise NotImplementedError
298
299  def GetProcessorCommand(self):
300    format_processor, options = self.GetProcessorScript()
301    if not format_processor:
302      print('Could not find the formatter for % files' % self.file_type)
303      sys.exit(1)
304
305    command = [sys.executable, format_processor]
306    command.extend(options)
307
308    return command
309
310  def ProcessFiles(self, files):
311    if self.use_cache:
312      cache = FileContentsCache(self.cache_file_path)
313      cache.Load()
314      files = cache.FilterUnchangedFiles(files)
315
316    if len(files) == 0:
317      print('No changes in %s files detected. Skipping check' % self.file_type)
318      return True
319
320    files_requiring_changes = self.DetectFilesToChange(files)
321    print (
322      'Total %s files found that require formatting: %d' %
323      (self.file_type, len(files_requiring_changes)))
324    if self.use_cache:
325      for file in files_requiring_changes:
326        cache.RemoveFile(file)
327
328      cache.Save()
329
330    return files_requiring_changes == []
331
332  def DetectFilesToChange(self, files):
333    command = self.GetProcessorCommand()
334    worker = self.GetProcessorWorker()
335
336    commands = [command + [file] for file in files]
337    count = multiprocessing.cpu_count()
338    pool = multiprocessing.Pool(count)
339    try:
340      results = pool.map_async(worker, commands).get(timeout=240)
341    except KeyboardInterrupt:
342      print("\nCaught KeyboardInterrupt, terminating workers.")
343      pool.terminate()
344      pool.join()
345      sys.exit(1)
346
347    unformatted_files = []
348    for index, errors in enumerate(results):
349      if errors > 0:
350        unformatted_files.append(files[index])
351
352    return unformatted_files
353
354
355class CppLintProcessor(CacheableSourceFileProcessor):
356  """
357  Lint files to check that they follow the google code style.
358  """
359
360  def __init__(self, use_cache=True):
361    super(CppLintProcessor, self).__init__(
362      use_cache=use_cache, cache_file_path='.cpplint-cache', file_type='C/C++')
363
364  def IsRelevant(self, name):
365    return name.endswith('.cc') or name.endswith('.h')
366
367  def IgnoreDir(self, name):
368    return (super(CppLintProcessor, self).IgnoreDir(name)
369            or (name == 'third_party'))
370
371  IGNORE_LINT = [
372    'export-template.h',
373    'flag-definitions.h',
374    'gay-fixed.cc',
375    'gay-precision.cc',
376    'gay-shortest.cc',
377  ]
378
379  def IgnoreFile(self, name):
380    return (super(CppLintProcessor, self).IgnoreFile(name)
381              or (name in CppLintProcessor.IGNORE_LINT))
382
383  def GetPathsToSearch(self):
384    dirs = ['include', 'samples', 'src']
385    test_dirs = ['cctest', 'common', 'fuzzer', 'inspector', 'unittests']
386    return dirs + [join('test', dir) for dir in test_dirs]
387
388  def GetProcessorWorker(self):
389    return CppLintWorker
390
391  def GetProcessorScript(self):
392    filters = ','.join([n for n in LINT_RULES])
393    arguments = ['--filter', filters]
394
395    cpplint = os.path.join(DEPS_DEPOT_TOOLS_PATH, 'cpplint.py')
396    return cpplint, arguments
397
398
399class TorqueLintProcessor(CacheableSourceFileProcessor):
400  """
401  Check .tq files to verify they follow the Torque style guide.
402  """
403
404  def __init__(self, use_cache=True):
405    super(TorqueLintProcessor, self).__init__(
406      use_cache=use_cache, cache_file_path='.torquelint-cache',
407      file_type='Torque')
408
409  def IsRelevant(self, name):
410    return name.endswith('.tq')
411
412  def GetPathsToSearch(self):
413    dirs = ['third_party', 'src']
414    test_dirs = ['torque']
415    return dirs + [join('test', dir) for dir in test_dirs]
416
417  def GetProcessorWorker(self):
418    return TorqueLintWorker
419
420  def GetProcessorScript(self):
421    torque_tools = os.path.join(TOOLS_PATH, "torque")
422    torque_path = os.path.join(torque_tools, "format-torque.py")
423    arguments = ["-il"]
424    if os.path.isfile(torque_path):
425      return torque_path, arguments
426
427    return None, arguments
428
429class JSLintProcessor(CacheableSourceFileProcessor):
430  """
431  Check .{m}js file to verify they follow the JS Style guide.
432  """
433  def __init__(self, use_cache=True):
434    super(JSLintProcessor, self).__init__(
435      use_cache=use_cache, cache_file_path='.jslint-cache',
436      file_type='JavaScript')
437
438  def IsRelevant(self, name):
439    return name.endswith('.js') or name.endswith('.mjs')
440
441  def GetPathsToSearch(self):
442    return ['tools/system-analyzer', 'tools/heap-layout', 'tools/js']
443
444  def GetProcessorWorker(self):
445    return JSLintWorker
446
447  def GetProcessorScript(self):
448    jslint = os.path.join(DEPS_DEPOT_TOOLS_PATH, 'clang_format.py')
449    return jslint, []
450
451
452COPYRIGHT_HEADER_PATTERN = re.compile(
453    r'Copyright [\d-]*20[0-2][0-9] the V8 project authors. All rights reserved.')
454
455class SourceProcessor(SourceFileProcessor):
456  """
457  Check that all files include a copyright notice and no trailing whitespaces.
458  """
459
460  RELEVANT_EXTENSIONS = ['.js', '.cc', '.h', '.py', '.c', '.status', '.tq', '.g4']
461
462  def __init__(self):
463    self.runtime_function_call_pattern = self.CreateRuntimeFunctionCallMatcher()
464
465  def CreateRuntimeFunctionCallMatcher(self):
466    runtime_h_path = join(dirname(TOOLS_PATH), 'src/runtime/runtime.h')
467    pattern = re.compile(r'\s+F\(([^,]*),.*\)')
468    runtime_functions = []
469    with open(runtime_h_path) as f:
470      for line in f.readlines():
471        m = pattern.match(line)
472        if m:
473          runtime_functions.append(m.group(1))
474    if len(runtime_functions) < 250:
475      print ("Runtime functions list is suspiciously short. "
476             "Consider updating the presubmit script.")
477      sys.exit(1)
478    str = '(\%\s+(' + '|'.join(runtime_functions) + '))[\s\(]'
479    return re.compile(str)
480
481  # Overwriting the one in the parent class.
482  def FindFilesIn(self, path):
483    if os.path.exists(path+'/.git'):
484      output = subprocess.Popen('git ls-files --full-name',
485                                stdout=PIPE, cwd=path, shell=True)
486      result = []
487      for file in decode(output.stdout.read()).split():
488        for dir_part in os.path.dirname(file).replace(os.sep, '/').split('/'):
489          if self.IgnoreDir(dir_part):
490            break
491        else:
492          if (self.IsRelevant(file) and os.path.exists(file)
493              and not self.IgnoreFile(file)):
494            result.append(join(path, file))
495      if output.wait() == 0:
496        return result
497    return super(SourceProcessor, self).FindFilesIn(path)
498
499  def IsRelevant(self, name):
500    for ext in SourceProcessor.RELEVANT_EXTENSIONS:
501      if name.endswith(ext):
502        return True
503    return False
504
505  def GetPathsToSearch(self):
506    return ['.']
507
508  def IgnoreDir(self, name):
509    return (super(SourceProcessor, self).IgnoreDir(name) or
510            name in ('third_party', 'out', 'obj', 'DerivedSources'))
511
512  IGNORE_COPYRIGHTS = ['box2d.js',
513                       'cpplint.py',
514                       'copy.js',
515                       'corrections.js',
516                       'crypto.js',
517                       'daemon.py',
518                       'earley-boyer.js',
519                       'fannkuch.js',
520                       'fasta.js',
521                       'injected-script.cc',
522                       'injected-script.h',
523                       'libraries.cc',
524                       'libraries-empty.cc',
525                       'lua_binarytrees.js',
526                       'meta-123.js',
527                       'memops.js',
528                       'poppler.js',
529                       'primes.js',
530                       'raytrace.js',
531                       'regexp-pcre.js',
532                       'resources-123.js',
533                       'sqlite.js',
534                       'sqlite-change-heap.js',
535                       'sqlite-pointer-masking.js',
536                       'sqlite-safe-heap.js',
537                       'v8-debugger-script.h',
538                       'v8-inspector-impl.cc',
539                       'v8-inspector-impl.h',
540                       'v8-runtime-agent-impl.cc',
541                       'v8-runtime-agent-impl.h',
542                       'gnuplot-4.6.3-emscripten.js',
543                       'zlib.js']
544  IGNORE_TABS = IGNORE_COPYRIGHTS + ['unicode-test.js', 'html-comments.js']
545
546  IGNORE_COPYRIGHTS_DIRECTORIES = [
547      "test/test262/local-tests",
548      "test/mjsunit/wasm/bulk-memory-spec",
549  ]
550
551  def EndOfDeclaration(self, line):
552    return line == "}" or line == "};"
553
554  def StartOfDeclaration(self, line):
555    return line.find("//") == 0 or \
556           line.find("/*") == 0 or \
557           line.find(") {") != -1
558
559  def ProcessContents(self, name, contents):
560    result = True
561    base = basename(name)
562    if not base in SourceProcessor.IGNORE_TABS:
563      if '\t' in contents:
564        print("%s contains tabs" % name)
565        result = False
566    if not base in SourceProcessor.IGNORE_COPYRIGHTS and \
567        not any(ignore_dir in name for ignore_dir
568                in SourceProcessor.IGNORE_COPYRIGHTS_DIRECTORIES):
569      if not COPYRIGHT_HEADER_PATTERN.search(contents):
570        print("%s is missing a correct copyright header." % name)
571        result = False
572    if ' \n' in contents or contents.endswith(' '):
573      line = 0
574      lines = []
575      parts = contents.split(' \n')
576      if not contents.endswith(' '):
577        parts.pop()
578      for part in parts:
579        line += part.count('\n') + 1
580        lines.append(str(line))
581      linenumbers = ', '.join(lines)
582      if len(lines) > 1:
583        print("%s has trailing whitespaces in lines %s." % (name, linenumbers))
584      else:
585        print("%s has trailing whitespaces in line %s." % (name, linenumbers))
586      result = False
587    if not contents.endswith('\n') or contents.endswith('\n\n'):
588      print("%s does not end with a single new line." % name)
589      result = False
590    # Sanitize flags for fuzzer.
591    if (".js" in name or ".mjs" in name) and ("mjsunit" in name or "debugger" in name):
592      match = FLAGS_LINE.search(contents)
593      if match:
594        print("%s Flags should use '-' (not '_')" % name)
595        result = False
596      if (not "mjsunit/mjsunit.js" in name and
597          not "mjsunit/mjsunit_numfuzz.js" in name):
598        if ASSERT_OPTIMIZED_PATTERN.search(contents) and \
599            not FLAGS_ENABLE_OPT.search(contents):
600          print("%s Flag --opt should be set if " \
601                "assertOptimized() is used" % name)
602          result = False
603        if ASSERT_UNOPTIMIZED_PATTERN.search(contents) and \
604            not FLAGS_NO_ALWAYS_OPT.search(contents):
605          print("%s Flag --no-always-opt should be set if " \
606                "assertUnoptimized() is used" % name)
607          result = False
608
609      match = self.runtime_function_call_pattern.search(contents)
610      if match:
611        print("%s has unexpected spaces in a runtime call '%s'" % (name, match.group(1)))
612        result = False
613    return result
614
615  def ProcessFiles(self, files):
616    success = True
617    violations = 0
618    for file in files:
619      try:
620        handle = open(file, "rb")
621        contents = decode(handle.read(), "ISO-8859-1")
622        if len(contents) > 0 and not self.ProcessContents(file, contents):
623          success = False
624          violations += 1
625      finally:
626        handle.close()
627    print("Total violating files: %s" % violations)
628    return success
629
630def _CheckStatusFileForDuplicateKeys(filepath):
631  comma_space_bracket = re.compile(", *]")
632  lines = []
633  with open(filepath) as f:
634    for line in f.readlines():
635      # Skip all-comment lines.
636      if line.lstrip().startswith("#"): continue
637      # Strip away comments at the end of the line.
638      comment_start = line.find("#")
639      if comment_start != -1:
640        line = line[:comment_start]
641      line = line.strip()
642      # Strip away trailing commas within the line.
643      line = comma_space_bracket.sub("]", line)
644      if len(line) > 0:
645        lines.append(line)
646
647  # Strip away trailing commas at line ends. Ugh.
648  for i in range(len(lines) - 1):
649    if (lines[i].endswith(",") and len(lines[i + 1]) > 0 and
650        lines[i + 1][0] in ("}", "]")):
651      lines[i] = lines[i][:-1]
652
653  contents = "\n".join(lines)
654  # JSON wants double-quotes.
655  contents = contents.replace("'", '"')
656  # Fill in keywords (like PASS, SKIP).
657  for key in statusfile.KEYWORDS:
658    contents = re.sub(r"\b%s\b" % key, "\"%s\"" % key, contents)
659
660  status = {"success": True}
661  def check_pairs(pairs):
662    keys = {}
663    for key, value in pairs:
664      if key in keys:
665        print("%s: Error: duplicate key %s" % (filepath, key))
666        status["success"] = False
667      keys[key] = True
668
669  json.loads(contents, object_pairs_hook=check_pairs)
670  return status["success"]
671
672
673class StatusFilesProcessor(SourceFileProcessor):
674  """Checks status files for incorrect syntax and duplicate keys."""
675
676  def IsRelevant(self, name):
677    # Several changes to files under the test directories could impact status
678    # files.
679    return True
680
681  def GetPathsToSearch(self):
682    return ['test', 'tools/testrunner']
683
684  def ProcessFiles(self, files):
685    success = True
686    for status_file_path in sorted(self._GetStatusFiles(files)):
687      success &= statusfile.PresubmitCheck(status_file_path)
688      success &= _CheckStatusFileForDuplicateKeys(status_file_path)
689    return success
690
691  def _GetStatusFiles(self, files):
692    test_path = join(dirname(TOOLS_PATH), 'test')
693    testrunner_path = join(TOOLS_PATH, 'testrunner')
694    status_files = set()
695
696    for file_path in files:
697      if file_path.startswith(testrunner_path):
698        for suitepath in os.listdir(test_path):
699          suitename = os.path.basename(suitepath)
700          status_file = os.path.join(
701              test_path, suitename, suitename + ".status")
702          if os.path.exists(status_file):
703            status_files.add(status_file)
704        return status_files
705
706    for file_path in files:
707      if file_path.startswith(test_path):
708        # Strip off absolute path prefix pointing to test suites.
709        pieces = file_path[len(test_path):].lstrip(os.sep).split(os.sep)
710        if pieces:
711          # Infer affected status file name. Only care for existing status
712          # files. Some directories under "test" don't have any.
713          if not os.path.isdir(join(test_path, pieces[0])):
714            continue
715          status_file = join(test_path, pieces[0], pieces[0] + ".status")
716          if not os.path.exists(status_file):
717            continue
718          status_files.add(status_file)
719    return status_files
720
721
722def CheckDeps(workspace):
723  checkdeps_py = join(workspace, 'buildtools', 'checkdeps', 'checkdeps.py')
724  return subprocess.call([sys.executable, checkdeps_py, workspace]) == 0
725
726
727def FindTests(workspace):
728  scripts = []
729  # TODO(almuthanna): unskip valid tests when they are properly migrated
730  exclude = [
731      'tools/clang',
732      'tools/mb/mb_test.py',
733      'tools/cppgc/gen_cmake_test.py',
734      'tools/ignition/linux_perf_report_test.py',
735      'tools/ignition/bytecode_dispatches_report_test.py',
736      'tools/ignition/linux_perf_bytecode_annotate_test.py',
737  ]
738  scripts_without_excluded = []
739  for root, dirs, files in os.walk(join(workspace, 'tools')):
740    for f in files:
741      if f.endswith('_test.py'):
742        fullpath = os.path.join(root, f)
743        scripts.append(fullpath)
744  for script in scripts:
745    if not any(exc_dir in script for exc_dir in exclude):
746      scripts_without_excluded.append(script)
747  return scripts_without_excluded
748
749
750def PyTests(workspace):
751  result = True
752  for script in FindTests(workspace):
753    print('Running ' + script)
754    result &= subprocess.call(
755        [sys.executable, script], stdout=subprocess.PIPE) == 0
756
757  return result
758
759
760def GetOptions():
761  result = optparse.OptionParser()
762  result.add_option('--no-lint', help="Do not run cpplint", default=False,
763                    action="store_true")
764  result.add_option('--no-linter-cache', help="Do not cache linter results",
765                    default=False, action="store_true")
766
767  return result
768
769
770def Main():
771  workspace = abspath(join(dirname(sys.argv[0]), '..'))
772  parser = GetOptions()
773  (options, args) = parser.parse_args()
774  success = True
775  print("Running checkdeps...")
776  success &= CheckDeps(workspace)
777  use_linter_cache = not options.no_linter_cache
778  if not options.no_lint:
779    print("Running C++ lint check...")
780    success &= CppLintProcessor(use_cache=use_linter_cache).RunOnPath(workspace)
781
782  print("Running Torque formatting check...")
783  success &= TorqueLintProcessor(use_cache=use_linter_cache).RunOnPath(
784    workspace)
785  print("Running JavaScript formatting check...")
786  success &= JSLintProcessor(use_cache=use_linter_cache).RunOnPath(
787    workspace)
788  print("Running copyright header, trailing whitespaces and " \
789        "two empty lines between declarations check...")
790  success &= SourceProcessor().RunOnPath(workspace)
791  print("Running status-files check...")
792  success &= StatusFilesProcessor().RunOnPath(workspace)
793  print("Running python tests...")
794  success &= PyTests(workspace)
795  if success:
796    return 0
797  else:
798    return 1
799
800
801if __name__ == '__main__':
802  sys.exit(Main())
803