1#!/bin/sh
2# Copyright 2019 The LUCI Authors. All rights reserved.
3# Use of this source code is governed under the Apache License, Version 2.0
4# that can be found in the LICENSE file.
5
6# We want to run python in unbuffered mode; however shebangs on linux grab the
7# entire rest of the shebang line as a single argument, leading to errors like:
8#
9#   /usr/bin/env: 'python3 -u': No such file or directory
10#
11# This little shell hack is a triple-quoted noop in python, but in sh it
12# evaluates to re-exec'ing this script in unbuffered mode.
13# pylint: disable=pointless-string-statement
14''''exec python3 -u -- "$0" ${1+"$@"} # '''
15# vi: syntax=python
16"""Bootstrap script to clone and forward to the recipe engine tool.
17
18*******************
19** DO NOT MODIFY **
20*******************
21
22This is a copy of https://chromium.googlesource.com/infra/luci/recipes-py/+/main/recipes.py.
23To fix bugs, fix in the googlesource repo then run the autoroller.
24"""
25
26# pylint: disable=wrong-import-position
27import argparse
28import errno
29import json
30import logging
31import os
32import subprocess
33import sys
34
35from collections import namedtuple
36from io import open  # pylint: disable=redefined-builtin
37
38try:
39  import urllib.parse as urlparse
40except ImportError:
41  import urlparse
42
43# The dependency entry for the recipe_engine in the client repo's recipes.cfg
44#
45# url (str) - the url to the engine repo we want to use.
46# revision (str) - the git revision for the engine to get.
47# branch (str) - the branch to fetch for the engine as an absolute ref (e.g.
48#   refs/heads/main)
49EngineDep = namedtuple('EngineDep', 'url revision branch')
50
51
52class MalformedRecipesCfg(Exception):
53
54  def __init__(self, msg, path):
55    full_message = 'malformed recipes.cfg: %s: %r' % (msg, path)
56    super(MalformedRecipesCfg, self).__init__(full_message)
57
58
59def parse(repo_root, recipes_cfg_path):
60  """Parse is a lightweight a recipes.cfg file parser.
61
62  Args:
63    repo_root (str) - native path to the root of the repo we're trying to run
64      recipes for.
65    recipes_cfg_path (str) - native path to the recipes.cfg file to process.
66
67  Returns (as tuple):
68    engine_dep (EngineDep|None): The recipe_engine dependency, or None, if the
69      current repo IS the recipe_engine.
70    recipes_path (str) - native path to where the recipes live inside of the
71      current repo (i.e. the folder containing `recipes/` and/or
72      `recipe_modules`)
73    py3_only (bool) - True if this repo has been marked as ONLY supporting
74      python3.
75  """
76  with open(recipes_cfg_path, 'r') as fh:
77    pb = json.load(fh)
78  py3_only = pb.get('py3_only', False)
79
80  try:
81    if pb['api_version'] != 2:
82      raise MalformedRecipesCfg('unknown version %d' % pb['api_version'],
83                                recipes_cfg_path)
84
85    # If we're running ./recipes.py from the recipe_engine repo itself, then
86    # return None to signal that there's no EngineDep.
87    repo_name = pb.get('repo_name')
88    if not repo_name:
89      repo_name = pb['project_id']
90    if repo_name == 'recipe_engine':
91      return None, pb.get('recipes_path', ''), py3_only
92
93    engine = pb['deps']['recipe_engine']
94
95    if 'url' not in engine:
96      raise MalformedRecipesCfg(
97          'Required field "url" in dependency "recipe_engine" not found',
98          recipes_cfg_path)
99
100    engine.setdefault('revision', '')
101    engine.setdefault('branch', 'refs/heads/main')
102    recipes_path = pb.get('recipes_path', '')
103
104    # TODO(iannucci): only support absolute refs
105    if not engine['branch'].startswith('refs/'):
106      engine['branch'] = 'refs/heads/' + engine['branch']
107
108    recipes_path = os.path.join(repo_root,
109                                recipes_path.replace('/', os.path.sep))
110    return EngineDep(**engine), recipes_path, py3_only
111  except KeyError as ex:
112    raise MalformedRecipesCfg(str(ex), recipes_cfg_path)
113
114
115IS_WIN = sys.platform.startswith(('win', 'cygwin'))
116
117_BAT = '.bat' if IS_WIN else ''
118GIT = 'git' + _BAT
119CIPD = 'cipd' + _BAT
120REQUIRED_BINARIES = {GIT, CIPD}
121
122
123def _is_executable(path):
124  return os.path.isfile(path) and os.access(path, os.X_OK)
125
126
127# TODO: Use shutil.which once we switch to Python3.
128def _is_on_path(basename):
129  for path in os.environ['PATH'].split(os.pathsep):
130    full_path = os.path.join(path, basename)
131    if _is_executable(full_path):
132      return True
133  return False
134
135
136def _subprocess_call(argv, **kwargs):
137  logging.info('Running %r', argv)
138  return subprocess.call(argv, **kwargs)
139
140
141def _git_check_call(argv, **kwargs):
142  argv = [GIT] + argv
143  logging.info('Running %r', argv)
144  subprocess.check_call(argv, **kwargs)
145
146
147def _git_output(argv, **kwargs):
148  argv = [GIT] + argv
149  logging.info('Running %r', argv)
150  return subprocess.check_output(argv, **kwargs)
151
152
153def parse_args(argv):
154  """This extracts a subset of the arguments that this bootstrap script cares
155  about. Currently this consists of:
156    * an override for the recipe engine in the form of `-O recipe_engine=/path`
157    * the --package option.
158  """
159  PREFIX = 'recipe_engine='
160
161  p = argparse.ArgumentParser(add_help=False)
162  p.add_argument('-O', '--project-override', action='append')
163  p.add_argument('--package', type=os.path.abspath)
164  args, _ = p.parse_known_args(argv)
165  for override in args.project_override or ():
166    if override.startswith(PREFIX):
167      return override[len(PREFIX):], args.package
168  return None, args.package
169
170
171def checkout_engine(engine_path, repo_root, recipes_cfg_path):
172  """Checks out the recipe_engine repo pinned in recipes.cfg.
173
174  Returns the path to the recipe engine repo and the py3_only boolean.
175  """
176  dep, recipes_path, py3_only = parse(repo_root, recipes_cfg_path)
177  if dep is None:
178    # we're running from the engine repo already!
179    return os.path.join(repo_root, recipes_path), py3_only
180
181  url = dep.url
182
183  if not engine_path and url.startswith('file://'):
184    engine_path = urlparse.urlparse(url).path
185
186  if not engine_path:
187    revision = dep.revision
188    branch = dep.branch
189
190    # Ensure that we have the recipe engine cloned.
191    engine_path = os.path.join(recipes_path, '.recipe_deps', 'recipe_engine')
192
193    with open(os.devnull, 'w') as NUL:
194      # Note: this logic mirrors the logic in recipe_engine/fetch.py
195      _git_check_call(['init', engine_path], stdout=NUL)
196
197      try:
198        _git_check_call(['rev-parse', '--verify',
199                         '%s^{commit}' % revision],
200                        cwd=engine_path,
201                        stdout=NUL,
202                        stderr=NUL)
203      except subprocess.CalledProcessError:
204        _git_check_call(['fetch', '--quiet', url, branch],
205                        cwd=engine_path,
206                        stdout=NUL)
207
208    try:
209      _git_check_call(['diff', '--quiet', revision], cwd=engine_path)
210    except subprocess.CalledProcessError:
211      index_lock = os.path.join(engine_path, '.git', 'index.lock')
212      try:
213        os.remove(index_lock)
214      except OSError as exc:
215        if exc.errno != errno.ENOENT:
216          logging.warn('failed to remove %r, reset will fail: %s', index_lock,
217                       exc)
218      _git_check_call(['reset', '-q', '--hard', revision], cwd=engine_path)
219
220    # If the engine has refactored/moved modules we need to clean all .pyc files
221    # or things will get squirrely.
222    _git_check_call(['clean', '-qxf'], cwd=engine_path)
223
224  return engine_path, py3_only
225
226
227def main():
228  for required_binary in REQUIRED_BINARIES:
229    if not _is_on_path(required_binary):
230      return 'Required binary is not found on PATH: %s' % required_binary
231
232  if '--verbose' in sys.argv:
233    logging.getLogger().setLevel(logging.INFO)
234
235  args = sys.argv[1:]
236  engine_override, recipes_cfg_path = parse_args(args)
237
238  if recipes_cfg_path:
239    # calculate repo_root from recipes_cfg_path
240    repo_root = os.path.dirname(
241        os.path.dirname(os.path.dirname(recipes_cfg_path)))
242  else:
243    # find repo_root with git and calculate recipes_cfg_path
244    repo_root = (
245        _git_output(['rev-parse', '--show-toplevel'],
246                    cwd=os.path.abspath(os.path.dirname(__file__))).strip())
247    repo_root = os.path.abspath(repo_root).decode()
248    recipes_cfg_path = os.path.join(repo_root, 'infra', 'config', 'recipes.cfg')
249    args = ['--package', recipes_cfg_path] + args
250  engine_path, py3_only = checkout_engine(engine_override, repo_root, recipes_cfg_path)
251
252  using_py3 = py3_only or os.getenv('RECIPES_USE_PY3') == 'true'
253  vpython = ('vpython' + ('3' if using_py3 else '') + _BAT)
254  if not _is_on_path(vpython):
255    return 'Required binary is not found on PATH: %s' % vpython
256
257  argv = ([
258    vpython, '-u', os.path.join(engine_path, 'recipe_engine', 'main.py'),
259  ] + args)
260
261  if IS_WIN:
262    # No real 'exec' on windows; set these signals to ignore so that they
263    # propagate to our children but we still wait for the child process to quit.
264    import signal
265    signal.signal(signal.SIGBREAK, signal.SIG_IGN)
266    signal.signal(signal.SIGINT, signal.SIG_IGN)
267    signal.signal(signal.SIGTERM, signal.SIG_IGN)
268    return _subprocess_call(argv)
269  else:
270    os.execvp(argv[0], argv)
271
272
273if __name__ == '__main__':
274  sys.exit(main())
275