1# Copyright 2018 The Chromium Authors. All rights reserved. 2# Use of this source code is governed under the Apache License, Version 2.0 3# that can be found in the LICENSE file. 4"""Recipe for building GN.""" 5 6from recipe_engine.recipe_api import Property 7 8DEPS = [ 9 'recipe_engine/buildbucket', 10 'recipe_engine/cas', 11 'recipe_engine/cipd', 12 'recipe_engine/context', 13 'recipe_engine/file', 14 'recipe_engine/json', 15 'recipe_engine/path', 16 'recipe_engine/platform', 17 'recipe_engine/properties', 18 'recipe_engine/raw_io', 19 'recipe_engine/step', 20 'target', 21 'macos_sdk', 22 'windows_sdk', 23] 24 25PROPERTIES = { 26 'repository': Property(kind=str, default='https://gn.googlesource.com/gn'), 27} 28 29# On select platforms, link the GN executable against jemalloc for a drastic speed boost. 30JEMALLOC_GIT_URL = 'https://fuchsia.googlesource.com/third_party/github.com/jemalloc/jemalloc.git' 31JEMALLOC_TAG = '5.3.0' 32 33def _get_libcxx_include_path(api): 34 # Run the preprocessor with an empty input and print all include paths. 35 lines = api.step( 36 'xcrun toolchain', [ 37 'xcrun', '--toolchain', 'clang', 'clang++', '-xc++', '-fsyntax-only', 38 '-Wp,-v', '/dev/null' 39 ], 40 stderr=api.raw_io.output_text(name='toolchain', add_output_log=True), 41 step_test_data=lambda: api.raw_io.test_api.stream_output_text( 42 str(api.macos_sdk.sdk_dir.join('include', 'c++', 'v1')), 43 stream='stderr')).stderr.splitlines() 44 # Iterate over all include paths and look for the SDK libc++ one. 45 sdk_dir = str(api.macos_sdk.sdk_dir) 46 for line in lines: 47 line = line.strip() 48 if line.startswith(sdk_dir) and 'include/c++/v1' in line: 49 return line 50 return None # pragma: no cover 51 52 53def _get_compilation_environment(api, target, cipd_dir): 54 if target.is_linux: 55 triple = '--target=%s' % target.triple 56 if target.triple == 'riscv64-linux-gnu': 57 sysroot = '--sysroot=%s' % cipd_dir.join('sysroot-focal') 58 else: 59 sysroot = '--sysroot=%s' % cipd_dir.join('sysroot') 60 env = { 61 'CC': cipd_dir.join('bin', 'clang'), 62 'CXX': cipd_dir.join('bin', 'clang++'), 63 'AR': cipd_dir.join('bin', 'llvm-ar'), 64 'CFLAGS': '%s %s' % (triple, sysroot), 65 'LDFLAGS': '%s %s -static-libstdc++' % (triple, sysroot), 66 } 67 elif target.is_mac: 68 triple = '--target=%s' % target.triple 69 sysroot = '--sysroot=%s' % api.step( 70 'xcrun sdk-path', ['xcrun', '--show-sdk-path'], 71 stdout=api.raw_io.output_text(name='sdk-path', add_output_log=True), 72 step_test_data=lambda: api.raw_io.test_api.stream_output_text( 73 '/some/xcode/path')).stdout.strip() 74 stdlib = cipd_dir.join('lib', 'libc++.a') 75 cxx_include = _get_libcxx_include_path(api) 76 env = { 77 'CC': 78 cipd_dir.join('bin', 'clang'), 79 'CXX': 80 cipd_dir.join('bin', 'clang++'), 81 'AR': 82 cipd_dir.join('bin', 'llvm-ar'), 83 'CFLAGS': 84 '%s %s -nostdinc++ -cxx-isystem %s' % 85 (triple, sysroot, cxx_include), 86 # TODO(phosek): Use the system libc++ temporarily until we 87 # have universal libc++.a for macOS that supports both x86_64 88 # and arm64. 89 'LDFLAGS': 90 '%s %s' % (triple, sysroot), 91 } 92 else: 93 env = {} 94 95 return env 96 97 98def RunSteps(api, repository): 99 src_dir = api.path['start_dir'].join('gn') 100 101 # TODO: Verify that building and linking jemalloc works on OS X and Windows as 102 # well. 103 with api.step.nest('git'), api.context(infra_steps=True): 104 api.step('init', ['git', 'init', src_dir]) 105 106 with api.context(cwd=src_dir): 107 build_input = api.buildbucket.build_input 108 ref = ( 109 build_input.gitiles_commit.id 110 if build_input.gitiles_commit else 'refs/heads/master') 111 # Fetch tags so `git describe` works. 112 api.step('fetch', ['git', 'fetch', '--tags', repository, ref]) 113 api.step('checkout', ['git', 'checkout', 'FETCH_HEAD']) 114 revision = api.step( 115 'rev-parse', ['git', 'rev-parse', 'HEAD'], 116 stdout=api.raw_io.output_text()).stdout.strip() 117 for change in build_input.gerrit_changes: 118 api.step('fetch %s/%s' % (change.change, change.patchset), [ 119 'git', 'fetch', repository, 120 'refs/changes/%s/%s/%s' % 121 (str(change.change)[-2:], change.change, change.patchset) 122 ]) 123 api.step('checkout %s/%s' % (change.change, change.patchset), 124 ['git', 'checkout', 'FETCH_HEAD']) 125 126 with api.context(infra_steps=True): 127 cipd_dir = api.path['start_dir'].join('cipd') 128 pkgs = api.cipd.EnsureFile() 129 pkgs.add_package('infra/ninja/${platform}', 'version:1.8.2') 130 if api.platform.is_linux or api.platform.is_mac: 131 pkgs.add_package('fuchsia/third_party/clang/${platform}', 'integration') 132 if api.platform.is_linux: 133 pkgs.add_package('fuchsia/third_party/sysroot/linux', 134 'git_revision:c912d089c3d46d8982fdef76a50514cca79b6132', 135 'sysroot') 136 # RISCV64 support starts in focal. 137 pkgs.add_package('fuchsia/third_party/sysroot/focal', 138 'git_revision:fa7a5a9710540f30ff98ae48b62f2cdf72ed2acd', 139 'sysroot-focal') 140 api.cipd.ensure(cipd_dir, pkgs) 141 142 def release_targets(): 143 if api.platform.is_linux: 144 return [ 145 api.target('linux-amd64'), 146 api.target('linux-arm64'), 147 api.target('linux-riscv64') 148 ] 149 elif api.platform.is_mac: 150 return [api.target('mac-amd64'), api.target('mac-arm64')] 151 else: 152 return [api.target.host] 153 154 # The order is important since release build will get uploaded to CIPD. 155 configs = [ 156 { 157 'name': 'debug', 158 'args': ['-d'], 159 'targets': [api.target.host], 160 }, 161 { 162 'name': 'release', 163 'args': ['--use-lto', '--use-icf'], 164 'targets': release_targets(), 165 # TODO: Enable this for OS X and Windows. 166 'use_jemalloc': api.platform.is_linux, 167 }, 168 ] 169 170 use_jemalloc = any(c.get('use_jemalloc', False) for c in configs) 171 172 with api.macos_sdk(), api.windows_sdk(): 173 # Build the jemalloc static library if needed. 174 if use_jemalloc: 175 # Maps a target.platform string to the location of the corresponding 176 # jemalloc static library. 177 jemalloc_static_libs = {} 178 179 # Get the list of all target platforms that are listed in `configs` 180 # above. Note that this is a list of Target instances, some of them 181 # may refer to the same platform string (e.g. linux-amd64). 182 # 183 # For each platform, a version of jemalloc will be built if necessary, 184 # but doing this properly requires having a valid target instance to 185 # call _get_compilation_environment. So create a { platform -> Target } 186 # map to do that later. 187 all_config_platforms = {} 188 for c in configs: 189 if not c.get('use_jemalloc', False): 190 continue 191 for t in c['targets']: 192 if t.platform not in all_config_platforms: 193 all_config_platforms[t.platform] = t 194 195 jemalloc_src_dir = api.path['start_dir'].join('jemalloc') 196 with api.step.nest('jemalloc'): 197 api.step('init', ['git', 'init', jemalloc_src_dir]) 198 with api.context(cwd=jemalloc_src_dir, infra_steps=True): 199 api.step( 200 'fetch', 201 ['git', 'fetch', JEMALLOC_GIT_URL, 'refs/tags/' + JEMALLOC_TAG, '--depth=1']) 202 api.step('checkout', ['git', 'checkout', 'FETCH_HEAD']) 203 api.step('autoconf', ['autoconf']) 204 205 for platform in all_config_platforms: 206 # Convert target architecture and os to jemalloc format. 207 jemalloc_build_dir = jemalloc_src_dir.join('build-' + platform) 208 209 target = all_config_platforms[platform] 210 host_target = api.target.host 211 env = _get_compilation_environment(api, target, cipd_dir) 212 213 # Prepare environment for configuring and building jemalloc 214 # 215 # CFLAGS: -Wno-error is required to avoid warnings that break 216 # compilation with Clang 17. 217 # 218 # CXXFLAGS: Required to pass the right --target=<target> argument 219 # to the C++ compiler when building jemalloc_cpp.o, 220 # otherwise that file will not be properly cross-compiled. 221 # 222 # Note that -flto is never passed to CFLAGS/CXXFLAGS/LDFLAGS, since 223 # benchmarking shows that this leads to no performance difference 224 # for the resulting GN binary. 225 # 226 env['CFLAGS'] += " -Wno-error" 227 env['CXXFLAGS'] = env['CFLAGS'] 228 229 api.step('prepare %s build' % platform, ['mkdir', '-p', jemalloc_build_dir]) 230 231 with api.step.nest('build jemalloc-' + platform), api.context( 232 env=env, cwd=jemalloc_build_dir): 233 api.step( 234 'configure', 235 [ 236 '../configure', 237 '--build=' + host_target.triple, 238 '--host=' + target.triple, 239 '--disable-shared', 240 '--enable-static', 241 '--disable-syscall', 242 '--disable-stats', 243 ]) 244 api.step('build', ['make', '-j%d' % api.platform.cpu_count, 'build_lib_static']) 245 jemalloc_static_lib = jemalloc_build_dir.join('lib', 'libjemalloc.a') 246 247 jemalloc_static_libs[platform] = jemalloc_static_lib 248 249 for config in configs: 250 with api.step.nest(config['name']): 251 for target in config['targets']: 252 env = _get_compilation_environment(api, target, cipd_dir) 253 with api.step.nest(target.platform), api.context( 254 env=env, cwd=src_dir): 255 args = config['args'] 256 if config.get('use_jemalloc', False): 257 args = args[:] + [ 258 '--link-lib=%s' % jemalloc_static_libs[target.platform] 259 ] 260 261 api.step('generate', 262 ['python3', '-u', src_dir.join('build', 'gen.py')] + args) 263 264 # Windows requires the environment modifications when building too. 265 api.step('build', 266 [cipd_dir.join('ninja'), '-C', 267 src_dir.join('out')]) 268 269 if target.is_host: 270 api.step('test', [src_dir.join('out', 'gn_unittests')]) 271 272 if config['name'] != 'release': 273 continue 274 275 with api.step.nest('upload'): 276 gn = 'gn' + ('.exe' if target.is_win else '') 277 278 if build_input.gerrit_changes: 279 # Upload to CAS from CQ. 280 api.cas.archive('upload binary to CAS', src_dir.join('out'), 281 src_dir.join('out', gn)) 282 continue 283 284 cipd_pkg_name = 'gn/gn/%s' % target.platform 285 286 pkg_def = api.cipd.PackageDefinition( 287 package_name=cipd_pkg_name, 288 package_root=src_dir.join('out'), 289 install_mode='copy') 290 pkg_def.add_file(src_dir.join('out', gn)) 291 pkg_def.add_version_file('.versions/%s.cipd_version' % gn) 292 293 cipd_pkg_file = api.path['cleanup'].join('gn.cipd') 294 295 api.cipd.build_from_pkg( 296 pkg_def=pkg_def, 297 output_package=cipd_pkg_file, 298 ) 299 300 if api.buildbucket.builder_id.project == 'infra-internal': 301 cipd_pin = api.cipd.search(cipd_pkg_name, 302 'git_revision:' + revision) 303 if cipd_pin: 304 api.step('Package is up-to-date', cmd=None) 305 continue 306 307 api.cipd.register( 308 package_name=cipd_pkg_name, 309 package_path=cipd_pkg_file, 310 refs=['latest'], 311 tags={ 312 'git_repository': repository, 313 'git_revision': revision, 314 }, 315 ) 316 317 318def GenTests(api): 319 for platform in ('linux', 'mac', 'win'): 320 yield (api.test('ci_' + platform) + api.platform.name(platform) + 321 api.buildbucket.ci_build( 322 project='gn', 323 git_repo='gn.googlesource.com/gn', 324 )) 325 326 yield (api.test('cq_' + platform) + api.platform.name(platform) + 327 api.buildbucket.try_build( 328 project='gn', 329 git_repo='gn.googlesource.com/gn', 330 )) 331 332 yield (api.test('cipd_exists') + api.buildbucket.ci_build( 333 project='infra-internal', 334 git_repo='gn.googlesource.com/gn', 335 revision='a' * 40, 336 ) + api.step_data( 337 'git.rev-parse', api.raw_io.stream_output_text('a' * 40) 338 ) + api.step_data( 339 'release.linux-amd64.upload.cipd search gn/gn/linux-amd64 git_revision:' + 340 'a' * 40, 341 api.cipd.example_search('gn/gn/linux-amd64', 342 ['git_revision:' + 'a' * 40]))) 343 344 yield (api.test('cipd_register') + api.buildbucket.ci_build( 345 project='infra-internal', 346 git_repo='gn.googlesource.com/gn', 347 revision='a' * 40, 348 ) + api.step_data( 349 'git.rev-parse', api.raw_io.stream_output_text('a' * 40) 350 ) + api.step_data( 351 'release.linux-amd64.upload.cipd search gn/gn/linux-amd64 git_revision:' + 352 'a' * 40, api.cipd.example_search('gn/gn/linux-amd64', []))) 353