Lines Matching refs:self
81 def __init__(self):
82 self.chromium_src_dir = CHROMIUM_SRC_DIR
83 self.default_config = os.path.join(self.chromium_src_dir, 'infra', 'mb',
85 self.default_isolate_map = os.path.join(self.chromium_src_dir, 'infra',
87 self.executable = sys.executable
88 self.platform = sys.platform
89 self.sep = os.sep
90 self.args = argparse.Namespace()
91 self.configs = {}
92 self.luci_tryservers = {}
93 self.builder_groups = {}
94 self.mixins = {}
95 self.isolate_exe = 'isolate.exe' if self.platform.startswith(
98 def Main(self, args):
99 self.ParseArgs(args)
101 ret = self.args.func()
103 self.DumpInputFiles()
106 self.Print('interrupted, exiting')
109 self.DumpInputFiles()
112 self.Print(l)
115 def ParseArgs(self, argv):
129 default=self.default_config,
167 subp.set_defaults(func=self.CmdAnalyze)
173 default=self.default_config,
177 subp.set_defaults(func=self.CmdExport)
189 subp.set_defaults(func=self.CmdGen)
199 subp.set_defaults(func=self.CmdIsolate)
211 subp.set_defaults(func=self.CmdLookup)
257 subp.set_defaults(func=self.CmdRun)
262 default=self.default_config,
264 subp.set_defaults(func=self.CmdValidate)
270 default=self.default_config,
272 subp.set_defaults(func=self.CmdBuildbucket)
278 subp.set_defaults(func=self.CmdHelp)
280 self.args = parser.parse_args(argv)
282 def DumpInputFiles(self):
285 if path and self.Exists(path):
286 self.Print("\n# To recreate the file passed to %s:" % arg_name)
287 self.Print("%% cat > %s <<EOF" % path)
288 contents = self.ReadFile(path)
289 self.Print(contents)
290 self.Print("EOF\n%\n")
292 if getattr(self.args, 'input_path', None):
294 'argv[0] (input_path)', self.args.input_path[0])
295 if getattr(self.args, 'swarming_targets_file', None):
297 '--swarming-targets-file', self.args.swarming_targets_file)
299 def CmdAnalyze(self):
300 vals = self.Lookup()
301 return self.RunGNAnalyze(vals)
303 def CmdExport(self):
304 self.ReadConfigFile()
306 for builder_group, builders in self.builder_groups.items():
309 config = self.builder_groups[builder_group][builder]
314 args = {k: self.FlattenConfig(v)['gn_args']
319 args = self.FlattenConfig(config)['gn_args']
328 self.Print(s)
331 def CmdGen(self):
332 vals = self.Lookup()
333 return self.RunGNGen(vals)
335 def CmdHelp(self):
336 if self.args.subcommand:
337 self.ParseArgs([self.args.subcommand, '--help'])
339 self.ParseArgs(['--help'])
341 def CmdIsolate(self):
342 vals = self.GetConfig()
345 return self.RunGNIsolate()
347 def CmdLookup(self):
348 vals = self.Lookup()
349 gn_args = self.GNArgs(vals, expand_imports=self.args.recursive)
350 if self.args.quiet or self.args.recursive:
351 self.Print(gn_args, end='')
353 cmd = self.GNCmd('gen', '_path_')
354 self.Print('\nWriting """\\\n%s""" to _path_/args.gn.\n' % gn_args)
357 self.PrintCmd(cmd, env)
360 def CmdRun(self):
361 vals = self.GetConfig()
365 build_dir = self.args.path[0]
366 target = self.args.target[0]
368 if self.args.build:
369 ret = self.Build(target)
372 ret = self.RunGNIsolate()
376 return self._RunLocallyIsolated(build_dir, target)
378 def _RunLocallyIsolated(self, build_dir, target):
380 self.PathJoin(self.chromium_src_dir, 'tools', 'luci-go',
381 self.isolate_exe),
384 self.ToSrcRelPath('%s/%s.isolate' % (build_dir, target)),
386 if self.args.extra_args:
387 cmd += ['--'] + self.args.extra_args
388 ret, _, _ = self.Run(cmd, force_verbose=True, buffer_output=False)
391 def _DefaultDimensions(self):
392 if not self.args.default_dimensions:
396 if self.platform == 'darwin':
398 elif self.platform.startswith('linux'):
400 elif self.platform == 'win32':
403 raise MBErr('unrecognized platform string "%s"' % self.platform)
409 def CmdBuildbucket(self):
410 self.ReadConfigFile()
412 self.Print('# This file was generated using '
415 for luci_tryserver in sorted(self.luci_tryservers):
416 self.Print('[bucket "luci.%s"]' % luci_tryserver)
417 for bot in sorted(self.luci_tryservers[luci_tryserver]):
418 self.Print('\tbuilder = %s' % bot)
420 for builder_group in sorted(self.builder_groups):
422 self.Print('[bucket "builder_group.%s"]' % builder_group)
423 for bot in sorted(self.builder_groups[builder_group]):
424 self.Print('\tbuilder = %s' % bot)
428 def CmdValidate(self, print_ok=True):
432 self.ReadConfigFile()
436 for builder_group in self.builder_groups:
437 for config in self.builder_groups[builder_group].values():
447 if not self.Exists(self.ToAbsPath(config)):
450 elif not config in self.configs:
455 for config in self.configs:
462 for config, mixins in self.configs.items():
464 if not mixin in self.mixins:
469 for mixin in self.mixins:
470 for sub_mixin in self.mixins[mixin].get('mixins', []):
471 if not sub_mixin in self.mixins:
477 for mixin in self.mixins:
482 raise MBErr(('mb config file %s has problems:' % self.args.config_file) +
486 self.Print('mb config file %s looks ok.' % self.args.config_file)
489 def GetConfig(self):
490 build_dir = self.args.path[0]
492 vals = self.DefaultVals()
493 if self.args.builder or self.args.builder_group or self.args.config:
494 vals = self.Lookup()
497 self.RunGNGen(vals)
500 toolchain_path = self.PathJoin(self.ToAbsPath(build_dir),
502 if not self.Exists(toolchain_path):
503 self.Print('Must either specify a path to an existing GN build dir '
508 vals['gn_args'] = self.GNArgsFromDir(build_dir)
511 def GNArgsFromDir(self, build_dir):
513 gn_args_path = self.PathJoin(self.ToAbsPath(build_dir), 'args.gn')
514 if self.Exists(gn_args_path):
515 args_contents = self.ReadFile(gn_args_path)
525 def Lookup(self):
526 vals = self.ReadIOSBotConfig()
528 self.ReadConfigFile()
529 config = self.ConfigFromArgs()
531 if not self.Exists(self.ToAbsPath(config)):
533 vals = self.DefaultVals()
536 if not config in self.configs:
538 (config, self.args.config_file))
539 vals = self.FlattenConfig(config)
542 def ReadIOSBotConfig(self):
543 if not self.args.builder_group or not self.args.builder:
545 path = self.PathJoin(self.chromium_src_dir, 'ios', 'build', 'bots',
546 self.args.builder_group, self.args.builder + '.json')
547 if not self.Exists(path):
550 contents = json.loads(self.ReadFile(path))
553 vals = self.DefaultVals()
557 def ReadConfigFile(self):
558 if not self.Exists(self.args.config_file):
559 raise MBErr('config file not found at %s' % self.args.config_file)
562 contents = ast.literal_eval(self.ReadFile(self.args.config_file))
565 (self.args.config_file, e))
567 self.configs = contents['configs']
568 self.luci_tryservers = contents.get('luci_tryservers', {})
569 self.builder_groups = contents['builder_groups']
570 self.mixins = contents['mixins']
572 def ReadIsolateMap(self):
573 if not self.args.isolate_map_files:
574 self.args.isolate_map_files = [self.default_isolate_map]
576 for f in self.args.isolate_map_files:
577 if not self.Exists(f):
580 for isolate_map in self.args.isolate_map_files:
582 isolate_map = ast.literal_eval(self.ReadFile(isolate_map))
594 def ConfigFromArgs(self):
595 if self.args.config:
596 if self.args.builder_group or self.args.builder:
601 return self.args.config
603 if not self.args.builder_group or not self.args.builder:
607 if not self.args.builder_group in self.builder_groups:
609 (self.args.builder_group, self.args.config_file))
612 self.args.builder, self.builder_groups[self.args.builder_group])
617 (self.args.builder, self.args.builder_group, self.args.config_file))
620 if self.args.phase is None:
622 (self.args.builder, self.args.builder_group))
623 phase = str(self.args.phase)
626 (phase, self.args.builder, self.args.builder_group))
629 if self.args.phase is not None:
631 (self.args.builder, self.args.builder_group))
634 def FlattenConfig(self, config):
635 mixins = self.configs[config]
636 vals = self.DefaultVals()
639 self.FlattenMixins(mixins, vals, visited)
642 def DefaultVals(self):
649 def FlattenMixins(self, mixins, vals, visited):
651 if m not in self.mixins:
656 mixin_vals = self.mixins[m]
664 (self.args.builder, self.args.builder_group))
673 self.FlattenMixins(mixin_vals['mixins'], vals, visited)
676 def RunGNGen(self, vals, compute_grit_inputs_for_analyze=False):
677 build_dir = self.args.path[0]
679 cmd = self.GNCmd('gen', build_dir, '--check')
680 gn_args = self.GNArgs(vals)
685 self.MaybeMakeDirectory(self.ToAbsPath(build_dir))
687 gn_args_path = self.ToAbsPath(build_dir, 'args.gn')
688 self.WriteFile(gn_args_path, gn_args, force_verbose=True)
691 if getattr(self.args, 'swarming_targets_file', None):
696 path = self.args.swarming_targets_file
697 if not self.Exists(path):
698 self.WriteFailureAndRaise('"%s" does not exist' % path,
700 contents = self.ReadFile(path)
703 isolate_map = self.ReadIsolateMap()
704 err, labels = self.MapTargetsToLabels(isolate_map, swarming_targets)
708 gn_runtime_deps_path = self.ToAbsPath(build_dir, 'runtime_deps')
709 self.WriteFile(gn_runtime_deps_path, '\n'.join(labels) + '\n')
712 ret, output, _ = self.Run(cmd)
714 if self.args.json_output:
716 self.WriteJSON({'output': output}, self.args.json_output)
719 self.Print('GN gen failed: %d' % ret)
741 if self.platform == 'win32':
745 elif self.platform == 'win32':
751 runtime_deps_path = self.ToAbsPath(build_dir, r)
752 if self.Exists(runtime_deps_path):
758 runtime_deps = self.ReadFile(runtime_deps_path).splitlines()
760 self.WriteIsolateFiles(build_dir, target, runtime_deps)
764 def RunGNIsolate(self):
765 target = self.args.target[0]
766 isolate_map = self.ReadIsolateMap()
767 err, labels = self.MapTargetsToLabels(isolate_map, [target])
772 build_dir = self.args.path[0]
774 cmd = self.GNCmd('desc', build_dir, label, 'runtime_deps')
775 ret, out, _ = self.Call(cmd)
778 self.Print(out)
783 self.WriteIsolateFiles(build_dir, target, runtime_deps)
785 ret, _, _ = self.Run([
786 self.PathJoin(self.chromium_src_dir, 'tools', 'luci-go',
787 self.isolate_exe),
790 self.ToSrcRelPath('%s/%s.isolate' % (build_dir, target))],
795 def WriteIsolateFiles(self, build_dir, target, runtime_deps):
796 isolate_path = self.ToAbsPath(build_dir, target + '.isolate')
797 self.WriteFile(isolate_path,
804 self.WriteJSON(
808 self.ToSrcRelPath('%s/%s.isolate' % (build_dir, target)),
810 'dir': self.chromium_src_dir,
816 def MapTargetsToLabels(self, isolate_map, targets):
837 def GNCmd(self, subcommand, path, *args):
838 if self.platform.startswith('linux'):
840 elif self.platform == 'darwin':
847 self.platform.startswith('aix')):
851 gn_path = self.PathJoin(self.chromium_src_dir, 'buildtools', subdir, exe)
855 def GNArgs(self, vals, expand_imports=False):
866 if self.args.goma_dir:
867 gn_args += ' goma_dir="%s"' % self.args.goma_dir
869 android_version_code = self.args.android_version_code
873 android_version_name = self.args.android_version_name
883 content = self.ReadFile(self.ToAbsPath(args_file))
896 def ToAbsPath(self, build_path, *comps):
897 return self.PathJoin(self.chromium_src_dir,
898 self.ToSrcRelPath(build_path),
901 def ToSrcRelPath(self, path):
904 return path[2:].replace('/', self.sep)
905 return self.RelPath(path, self.chromium_src_dir)
907 def RunGNAnalyze(self, vals):
910 ret = self.RunGNGen(vals, compute_grit_inputs_for_analyze=True)
914 build_path = self.args.path[0]
915 input_path = self.args.input_path[0]
917 output_path = self.args.output_path[0]
920 inp = self.ReadInputJSON(['files', 'test_targets',
922 if self.args.verbose:
923 self.Print()
924 self.Print('analyze input:')
925 self.PrintJSON(inp)
926 self.Print()
933 self.Print('Warning: No files modified in patch, bailing out early.')
934 self.WriteJSON({
944 isolate_map = self.ReadIsolateMap()
945 err, gn_inp['additional_compile_targets'] = self.MapTargetsToLabels(
950 err, gn_inp['test_targets'] = self.MapTargetsToLabels(
959 self.WriteJSON(gn_inp, gn_input_path)
960 cmd = self.GNCmd('analyze', build_path, gn_input_path, gn_output_path)
961 ret, output, _ = self.Run(cmd, force_verbose=True)
963 if self.args.json_output:
965 self.WriteJSON({'output': output}, self.args.json_output)
968 gn_outp_str = self.ReadFile(gn_output_path)
972 self.Print("Failed to parse the JSON string GN returned: %s\n%s"
1004 self.Print('WARNING: targets with non-default toolchains were '
1018 self.Print('WARNING: Too many compile targets were affected.')
1019 self.Print('WARNING: Building everything instead to avoid '
1028 if self.args.verbose:
1029 self.Print()
1030 self.Print('analyze output:')
1031 self.PrintJSON(outp)
1032 self.Print()
1034 self.WriteJSON(outp, output_path)
1037 if self.Exists(gn_input_path):
1038 self.RemoveFile(gn_input_path)
1039 if self.Exists(gn_output_path):
1040 self.RemoveFile(gn_output_path)
1044 def ReadInputJSON(self, required_keys):
1045 path = self.args.input_path[0]
1046 output_path = self.args.output_path[0]
1047 if not self.Exists(path):
1048 self.WriteFailureAndRaise('"%s" does not exist' % path, output_path)
1051 inp = json.loads(self.ReadFile(path))
1053 self.WriteFailureAndRaise('Failed to read JSON input from "%s": %s' %
1058 self.WriteFailureAndRaise('input file is missing a "%s" key' % k,
1063 def WriteFailureAndRaise(self, msg, output_path):
1065 self.WriteJSON({'error': msg}, output_path, force_verbose=True)
1068 def WriteJSON(self, obj, path, force_verbose=False):
1070 self.WriteFile(path, json.dumps(obj, indent=2, sort_keys=True) + '\n',
1076 def CheckCompile(self, builder_group, builder):
1077 url_template = self.args.url_template + '/{builder}/builds/_all?as_text=1'
1082 builds = json.loads(self.Fetch(url))
1098 def PrintCmd(self, cmd, env):
1099 if self.platform == 'win32':
1110 self.Print('%s%s=%s' % (env_prefix, var, env_quoter(env[var])))
1114 if cmd[0] == self.executable:
1116 self.Print(*[shell_quoter(arg) for arg in cmd])
1118 def PrintJSON(self, obj):
1119 self.Print(json.dumps(obj, indent=2, sort_keys=True))
1121 def Build(self, target):
1122 build_dir = self.ToSrcRelPath(self.args.path[0])
1124 if self.args.jobs:
1125 ninja_cmd.extend(['-j', '%d' % self.args.jobs])
1127 ret, _, _ = self.Run(ninja_cmd, force_verbose=False, buffer_output=False)
1130 def Run(self, cmd, env=None, force_verbose=True, buffer_output=True):
1132 if self.args.dryrun or self.args.verbose or force_verbose:
1133 self.PrintCmd(cmd, env)
1134 if self.args.dryrun:
1137 ret, out, err = self.Call(cmd, env=env, buffer_output=buffer_output)
1138 if self.args.verbose or force_verbose:
1140 self.Print(' -> returned %d' % ret)
1142 self.Print(out, end='')
1144 self.Print(err, end='', file=sys.stderr)
1147 def Call(self, cmd, env=None, buffer_output=True):
1149 p = subprocess.Popen(cmd, shell=False, cwd=self.chromium_src_dir,
1156 p = subprocess.Popen(cmd, shell=False, cwd=self.chromium_src_dir,
1162 def ExpandUser(self, path):
1166 def Exists(self, path):
1170 def Fetch(self, url):
1177 def MaybeMakeDirectory(self, path):
1184 def PathJoin(self, *comps):
1188 def Print(self, *args, **kwargs):
1194 def ReadFile(self, path):
1199 def RelPath(self, path, start='.'):
1203 def RemoveFile(self, path):
1207 def RemoveDirectory(self, abs_path):
1208 if self.platform == 'win32':
1214 self.Run(['cmd.exe', '/c', 'rmdir', '/q', '/s', abs_path])
1218 def TempFile(self, mode='w'):
1222 def WriteFile(self, path, contents, force_verbose=False):
1224 if self.args.dryrun or self.args.verbose or force_verbose:
1225 self.Print('\nWriting """\\\n%s""" to %s.\n' % (contents, path))