1#!/usr/bin/env python3 2# -*- coding: utf-8 -*- 3 4# 5# Copyright (c) 2023 Huawei Device Co., Ltd. 6# Licensed under the Apache License, Version 2.0 (the "License"); 7# you may not use this file except in compliance with the License. 8# You may obtain a copy of the License at 9# 10# http://www.apache.org/licenses/LICENSE-2.0 11# 12# Unless required by applicable law or agreed to in writing, software 13# distributed under the License is distributed on an "AS IS" BASIS, 14# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15# See the License for the specific language governing permissions and 16# limitations under the License. 17 18import os 19import json 20import copy 21 22from services.interface.load_interface import LoadInterface 23from containers.status import throw_exception 24from exceptions.ohos_exception import OHOSException 25from util.loader import platforms_loader # noqa: E402 26from util.loader import generate_targets_gn # noqa: E402 27from util.loader import load_ohos_build # noqa: E402 28from util.loader import subsystem_scan # noqa: E402 29from util.loader import subsystem_info # noqa: E402 30from scripts.util.file_utils import read_json_file, write_json_file, write_file # noqa: E402, E501 31from util.log_util import LogUtil 32from resources.config import Config 33 34 35class OHOSLoader(LoadInterface): 36 37 def __init__(self): 38 super().__init__() 39 self.source_root_dir = "" 40 self.gn_root_out_dir = "" 41 self.os_level = "" 42 self.target_cpu = "" 43 self.target_os = "" 44 self.config_output_relpath = "" 45 self.config_output_dir = "" 46 self.target_arch = "" 47 self.subsystem_config_file = "" 48 self.subsystem_config_overlay_file = "" 49 self.platforms_config_file = "" 50 self.exclusion_modules_config_file = "" 51 self.example_subsystem_file = "" 52 self.build_example = "" 53 self.scalable_build = "" 54 self.build_platform_name = "" 55 self.build_xts = "" 56 self.ignore_api_check = "" 57 self.load_test_config = "" 58 self.subsystem_configs = "" 59 self._subsystem_info = "" 60 self.skip_partlist_check = "" 61 62 def __post_init__(self): 63 self.source_root_dir = self.config.root_path + '/' 64 self.gn_root_out_dir = self.config.out_path if not self.config.out_path.startswith( 65 '/') else os.path.relpath(self.config.out_path, self.config.root_path) 66 self.os_level = self.config.os_level if self.config.os_level else "standard" 67 self.target_cpu = self.config.target_cpu if self.config.target_cpu else "arm" 68 self.target_os = self.config.target_os if self.config.target_os else "ohos" 69 self.config_output_relpath = os.path.join( 70 self.gn_root_out_dir, 'build_configs') 71 self.config_output_dir = os.path.join( 72 self.source_root_dir, self.config_output_relpath) 73 self.target_arch = '{}_{}'.format(self.target_os, self.target_cpu) 74 self.subsystem_config_file = os.path.join( 75 self.config.root_path, 'out/preloader', self.config.product, 'subsystem_config.json') 76 self.platforms_config_file = os.path.join( 77 self.config.root_path, 'out/preloader', self.config.product, 'platforms.build') 78 self.exclusion_modules_config_file = os.path.join( 79 self.config.root_path, 'out/preloader', self.config.product, 'exclusion_modules.json') 80 self.example_subsystem_file = os.path.join( 81 self.config.root_path, 'build', 'subsystem_config_example.json') 82 self.parts_src_file = os.path.join( 83 self.config_output_dir, 'parts_src_flag.json') 84 self.auto_install_file = os.path.join( 85 self.config_output_dir, 'auto_install_parts.json') 86 self.components_file = os.path.join( 87 self.config_output_dir, 'parts_info', 'components.json') 88 89 compile_standard_allow_file = os.path.join( 90 self.config.root_path, 'out/preloader', self.config.product, 'compile_standard_whitelist.json') 91 compile_standard_allow_info = read_json_file(compile_standard_allow_file) 92 bundle_subsystem_allow_list = compile_standard_allow_info.get("bundle_subsystem_error", []) 93 94 # check config args 95 self._check_args() 96 97 self.build_example = self.args_dict.get('build_example') 98 if not self.build_example: 99 self.example_subsystem_file = "" 100 self.scalable_build = self.args_dict.get('scalable_build') 101 self.build_platform_name = self.args_dict.get('build_platform_name') 102 self.build_xts = self.args_dict.get('build_xts') 103 self.ignore_api_check = self.args_dict.get('ignore_api_check') 104 self.load_test_config = self.args_dict.get('load_test_config') 105 self.skip_partlist_check = self.args_dict.get('skip_partlist_check') 106 107 self._subsystem_info = subsystem_info.get_subsystem_info( 108 self.subsystem_config_file, 109 self.example_subsystem_file, 110 self.source_root_dir, 111 self.config_output_relpath, 112 self.os_level) 113 overrided_components = self._override_components() 114 115 self._platforms_info = platforms_loader.get_platforms_info( 116 self.platforms_config_file, 117 self.source_root_dir, 118 self.gn_root_out_dir, 119 self.target_arch, 120 self.config_output_relpath, 121 self.scalable_build) 122 self.variant_toolchains = self._platforms_info.get( 123 'variant_toolchain_info').get('platform_toolchain') 124 self._all_platforms = self.variant_toolchains.keys() 125 self.build_platforms = self._get_build_platforms() 126 self.parts_config_info = load_ohos_build.get_parts_info( 127 self.source_root_dir, 128 self.config_output_relpath, 129 self._subsystem_info, 130 self.variant_toolchains, 131 self.target_arch, 132 self.ignore_api_check, 133 self.exclusion_modules_config_file, 134 self.load_test_config, 135 overrided_components, 136 bundle_subsystem_allow_list, 137 self.skip_partlist_check, 138 self.build_xts) 139 self.parts_targets = self.parts_config_info.get('parts_targets') 140 self.phony_targets = self.parts_config_info.get('phony_target') 141 self.parts_info = self.parts_config_info.get('parts_info') 142 self.target_platform_parts = self._get_platforms_all_parts() 143 self.target_platform_stubs = self._get_platforms_all_stubs() 144 self.required_parts_targets_list = self._get_required_build_parts_list() 145 self.required_phony_targets = self._get_required_phony_targets() 146 self.required_parts_targets = self._get_required_build_targets() 147 148 149 @throw_exception 150 def _merge_components_info(self, components): 151 config = Config() 152 sdk_components_file = os.path.join(config.root_path, "out/products_ext/components.json") 153 if not os.path.exists(sdk_components_file): 154 return 155 156 sdk_components_info = read_json_file(sdk_components_file) 157 for name, val in sdk_components_info.items(): 158 if name not in components.keys(): 159 components[name] = val 160 161 162 @throw_exception 163 def _cropping_components(self): 164 src_parts = read_json_file(self.parts_src_file) 165 166 auto_parts = read_json_file(self.auto_install_file) 167 168 self.third_party_file = os.path.join(self.config.root_path, "out/products_ext/third_party_allow_list.json") 169 if not os.path.exists(self.third_party_file): 170 self.third_party_file = os.path.join(self.config.root_path, 'build/third_party_allow_list.json') 171 cropping_parts = read_json_file(self.third_party_file) 172 173 components_data = read_json_file(self.components_file) 174 175 new_components_data = copy.deepcopy(components_data) 176 177 for component, component_value in components_data.items(): 178 if component not in src_parts and component not in auto_parts and component not in cropping_parts: 179 del new_components_data[component] 180 self._merge_components_info(new_components_data) 181 os.remove(self.components_file) 182 write_json_file(self.components_file, new_components_data) 183 184 185# check method 186 187 '''Description: Check the parameters passed in config. If the parameters are not 188 specified or the file content pointed to by the parameters does not 189 exist, an exception will be thrown directly. 190 @parameter:none 191 @return :none 192 ''' 193 @throw_exception 194 def _check_args(self): 195 LogUtil.hb_info("Checking all build args...") 196 # check subsystem_config_file 197 if not read_json_file(self.subsystem_config_file): 198 self.subsystem_config_file = os.path.join( 199 self.source_root_dir, 'build/subsystem_config.json') 200 if not read_json_file(self.subsystem_config_file): 201 raise OHOSException("Cannot get the content from platform config file, \ 202 please check whether the corresponding file('out/preloader/{}/subsystem_config.json' or \ 203 'build/subsystem_config.json') is written correctly.".format(self.config.product), "2001") 204 205 # check gn_root_out_dir 206 if not self.gn_root_out_dir: 207 raise OHOSException("Args gn_root_out_dir is required.", "2002") 208 if not os.path.realpath(self.gn_root_out_dir).startswith(self.source_root_dir): 209 raise OHOSException("Args gn_root_out_dir is incorrect.", "2003") 210 211 # check platform config file 212 if not read_json_file(self.platforms_config_file): 213 raise OHOSException("Cannot get the content from platform config file, \ 214 please check whether the corresponding file('out/preloader/${product_name}/platforms.build') \ 215 is written correctly.".format(self.config.product), "2004") 216 217 # check example subsystem file 218 if not read_json_file(self.example_subsystem_file): 219 raise OHOSException("Cannot get the content from example subsystem file, please check whether \ 220 the corresponding file ('build/subsystem_config_example.json') exists.", "2005") 221 222 @throw_exception 223 def _check_product_part_feature(self): 224 LogUtil.hb_info("Checking all product features...") 225 product_preloader_dir = os.path.dirname(self.platforms_config_file) 226 _preloader_feature_file = os.path.join(product_preloader_dir, 227 'features.json') 228 _preloader_feature_info = read_json_file(_preloader_feature_file) 229 part_to_feature = _preloader_feature_info.get('part_to_feature') 230 _feature_whitelist_file = os.path.join( 231 self.source_root_dir, "out/products_ext", "component_feature_whitelist.json" 232 ) 233 if not os.path.exists(_feature_whitelist_file): 234 _feature_whitelist_file = os.path.join( 235 self.source_root_dir, "build/", "component_feature_whitelist.json" 236 ) 237 _feature_whitelist_info = read_json_file(_feature_whitelist_file) 238 _feature_whitelist_list = [] 239 if _feature_whitelist_info: 240 _feature_whitelist_list = list(_feature_whitelist_info.keys()) 241 for key, vals in part_to_feature.items(): 242 part = self.parts_info.get(key) 243 if part is None: 244 continue 245 _p_info = part[0] 246 def_feature_list = _p_info.get('feature_list') 247 if vals and not def_feature_list: 248 message = "The product use a feature vals='{}', but that is not defined " \ 249 "in this part bundle.json file, part_name='{}'".format(vals, key) 250 if key not in _feature_whitelist_list: 251 raise OHOSException(message, "2006") 252 LogUtil.hb_warning(message) 253 continue 254 for _f_name in vals: 255 if _f_name not in def_feature_list: 256 raise OHOSException( 257 "The product use a feature that is not supported" 258 " by this part, part_name='{}', feature='{}'".format( 259 key, _f_name), "2006") 260 261 @throw_exception 262 def _check_parts_config_info(self): 263 LogUtil.hb_info("Checking parts config...") 264 if not ('parts_info' in self.parts_config_info 265 and 'subsystem_parts' in self.parts_config_info 266 and 'parts_variants' in self.parts_config_info 267 and 'parts_kits_info' in self.parts_config_info 268 and 'parts_inner_kits_info' in self.parts_config_info 269 and 'parts_targets' in self.parts_config_info): 270 raise OHOSException( 271 "Loading ohos.build information is incorrect.", "2007") 272 273# generate method 274 275 '''Description: Generate SystemCapability.json & syscap.json & syscap.para, dir:[ 276 (//out/preloader/${product_name}/system/etc/SystemCapability.json), 277 (//out/preloader/${product_name}/system/etc/syscap.json), 278 (//out/preloader/${product_name}/system/etc/param/syscap.para)] 279 @parameter:none 280 @return :none 281 ''' 282 @throw_exception 283 def _generate_syscap_files(self): 284 pre_syscap_info_path = os.path.dirname(self.platforms_config_file) 285 system_path = os.path.join(self.source_root_dir, os.path.join( 286 os.path.dirname(self.platforms_config_file), "system/")) 287 syscap_product_dict = read_json_file( 288 os.path.join(pre_syscap_info_path, "syscap.json")) 289 syscap_info_list = self.parts_config_info.get('syscap_info') 290 target_syscap_with_part_name_list = [] 291 target_syscap_list = [] 292 target_syscap_for_init_list = [] 293 all_syscap_list = [] 294 for syscap in syscap_info_list: 295 if syscap['component'] not in self.required_parts_targets_list: 296 continue 297 if 'syscap' not in syscap or syscap['syscap'] is None \ 298 or len(syscap['syscap']) == 0 or syscap['syscap'] == [""]: 299 continue 300 for syscap_string in syscap['syscap']: 301 all_syscap_list.append(syscap_string.split('=')[0].strip()) 302 303 for key, value in syscap_product_dict['part_to_syscap'].items(): 304 part = self.parts_info.get(key) 305 if part is None: 306 continue 307 for syscap in value: 308 if syscap not in all_syscap_list: 309 raise OHOSException( 310 "In config.json of part [{}],the syscap[{}] is incorrect, \ 311 please check the syscap name".format(key, syscap), "2008") 312 313 for syscap in syscap_info_list: 314 remove_list = [] 315 if syscap['component'] not in self.required_parts_targets_list: 316 continue 317 if 'syscap' not in syscap or syscap['syscap'] is None \ 318 or len(syscap['syscap']) == 0 or syscap['syscap'] == [""]: 319 continue 320 for syscap_string in syscap['syscap']: 321 if syscap_string.startswith("SystemCapability.") is True: 322 target_syscap_init_str = "const." 323 syscap_name = syscap_string.split('=')[0].strip() 324 all_syscap_product = syscap_product_dict['syscap'] 325 if syscap_name in all_syscap_product and not all_syscap_product[syscap_name]: 326 remove_list.append(syscap_string) 327 continue 328 elif syscap_name in all_syscap_product and all_syscap_product[syscap_name]: 329 target_syscap_init_str += syscap_name + '=true\n' 330 else: 331 if syscap_string.endswith('true'): 332 target_syscap_init_str += syscap_name + '=true\n' 333 elif syscap_string.endswith('false'): 334 remove_list.append(syscap_string) 335 continue 336 else: 337 target_syscap_init_str += syscap_string + "=true\n" 338 if target_syscap_init_str not in target_syscap_for_init_list: 339 target_syscap_for_init_list.append( 340 target_syscap_init_str) 341 else: 342 raise OHOSException("""In bundle.json of part [{}], The syscap string [{}] is incorrect, 343 need start with \"SystemCapability.\"""".format(syscap['component'], syscap_string), "2009") 344 345 for remove_str in remove_list: 346 syscap['syscap'].remove(remove_str) 347 for i in range(len(syscap['syscap'])): 348 if syscap['syscap'][i].endswith('true') or syscap['syscap'][i].endswith('false'): 349 syscap['syscap'][i] = syscap['syscap'][i].split('=')[ 350 0].strip() 351 352 syscap['syscap'].sort() 353 target_syscap_with_part_name_list.append(syscap) 354 target_syscap_list.extend(syscap['syscap']) 355 356 # Generate SystemCapability.json & syscap.json & syscap.para 357 target_syscap_list.sort() 358 syscap_info_dict = read_json_file(os.path.join( 359 pre_syscap_info_path, "SystemCapability.json")) 360 syscap_info_dict.update({'syscap': {'os': target_syscap_list}}) 361 system_etc_path = os.path.join(system_path, "etc/") 362 if not os.path.exists(system_path): 363 os.mkdir(system_path) 364 if not os.path.exists(system_etc_path): 365 os.mkdir(system_etc_path) 366 syscap_info_json = os.path.join( 367 system_etc_path, "SystemCapability.json") 368 write_json_file(syscap_info_json, syscap_info_dict) 369 LogUtil.hb_info( 370 "generate syscap info file to '{}'".format(syscap_info_json), mode=self.config.log_mode) 371 target_syscap_with_part_name_list.sort( 372 key=lambda syscap: syscap['component']) 373 syscap_info_with_part_name_file = os.path.join( 374 system_etc_path, "syscap.json") 375 write_json_file(syscap_info_with_part_name_file, { 376 'components': target_syscap_with_part_name_list}) 377 LogUtil.hb_info("generate syscap info with part name list to '{}'".format( 378 syscap_info_with_part_name_file), mode=self.config.log_mode) 379 if not os.path.exists(os.path.join(system_etc_path, "param/")): 380 os.mkdir(os.path.join(system_etc_path, "param/")) 381 target_syscap_for_init_file = os.path.join( 382 system_etc_path, "param/syscap.para") 383 with open(target_syscap_for_init_file, "w") as file: 384 file.writelines(sorted(target_syscap_for_init_list)) 385 LogUtil.hb_info("generate target syscap for init list to '{}'".format( 386 target_syscap_for_init_file), mode=self.config.log_mode) 387 388# get method 389 @throw_exception 390 def _get_build_platforms(self) -> list: 391 build_platforms = [] 392 if self.build_platform_name == 'all': 393 build_platforms = self._all_platforms 394 elif self.build_platform_name in self._all_platforms: 395 build_platforms = [self.build_platform_name] 396 else: 397 raise OHOSException( 398 "The target_platform is incorrect, only allows [{}].".format( 399 ', '.join(self._all_platforms)), "2010") 400 return build_platforms 401 402 '''Description: output infos for testfwk into a json file. \ 403 (/out/${product_name}/build_configs/infos_for_testfwk.json) 404 @parameter:none 405 @return :none 406 ''' 407 408 def _generate_infos_for_testfwk(self): 409 infos_for_testfwk_file = os.path.join(self.config_output_dir, 410 "infos_for_testfwk.json") 411 parts_info = self.parts_config_info.get('parts_info') 412 parts_info_dict = {} 413 for _part_name, _parts in parts_info.items(): 414 for _info in _parts: 415 parts_info_dict[_info.get('part_name')] = _info 416 _output_infos = {} 417 for _platform, _parts in self.target_platform_parts.items(): 418 result = self._output_infos_by_platform(_parts, parts_info_dict) 419 _output_infos[_platform] = result 420 write_json_file(infos_for_testfwk_file, 421 _output_infos, check_changes=True) 422 LogUtil.hb_info("generate infos for testfwk to '{}'".format( 423 infos_for_testfwk_file), mode=self.config.log_mode) 424 425 '''Description: output all target platform parts into a json file \ 426 (/out/${product_name}/build_configs/target_platforms_parts.json) 427 @parameter:none 428 @return :none 429 ''' 430 431 def _generate_target_platform_parts(self): 432 target_platform_parts_file = os.path.join(self.config_output_dir, 433 "target_platforms_parts.json") 434 write_json_file(target_platform_parts_file, 435 self.target_platform_parts, 436 check_changes=True) 437 LogUtil.hb_info("generate target platform parts to '{}'".format( 438 target_platform_parts_file), mode=self.config.log_mode) 439 440 '''Description: Generate parts differences in different platforms, using phone as base. \ 441 (/out/${product_name}/build_configs/parts_different_info.json) 442 @parameter: none 443 @return :none 444 ''' 445 446 def _generate_part_different_info(self): 447 parts_different_info = self._get_parts_by_platform() 448 parts_different_info_file = os.path.join(self.config_output_dir, 449 "parts_different_info.json") 450 write_json_file(parts_different_info_file, 451 parts_different_info, 452 check_changes=True) 453 LogUtil.hb_info("generate part different info to '{}'".format( 454 parts_different_info_file), mode=self.config.log_mode) 455 456 '''Description: output platforms list into a gni file. \ 457 (/out/${product_name}/build_configs/platforms_list.gni) 458 @parameter: none 459 @return: none 460 ''' 461 462 def _generate_platforms_list(self): 463 platforms_list_gni_file = os.path.join(self.config_output_dir, 464 "platforms_list.gni") 465 _platforms = set(self.build_platforms) 466 _gni_file_content = ['target_platform_list = [', ' "{}"'.format('",\n "'.join(_platforms)), ']', 467 'kits_platform_list = [', ' "{}",'.format('",\n "'.join(_platforms))] 468 if 'phone' not in self.build_platforms: 469 _gni_file_content.append(' "phone"') 470 _gni_file_content.append(']') 471 write_file(platforms_list_gni_file, '\n'.join(_gni_file_content)) 472 LogUtil.hb_info("generate platforms list to '{}'".format( 473 platforms_list_gni_file), mode=self.config.log_mode) 474 475 '''Description: output auto install part into a json file. \ 476 (/out/${product_name}/build_configs/auto_install_parts.json) 477 @parameter: none 478 @return: none 479 ''' 480 481 def _generate_auto_install_part(self): 482 parts_path_info = self.parts_config_info.get("parts_path_info") 483 auto_install_part_list = [] 484 for part, path in parts_path_info.items(): 485 if str(path).startswith("drivers/interface") or \ 486 str(path).startswith("third_party"): 487 auto_install_part_list.append(part) 488 auto_install_list_file = os.path.join( 489 self.config_output_dir, "auto_install_parts.json") 490 write_json_file(auto_install_list_file, auto_install_part_list) 491 LogUtil.hb_info("generate auto install part to '{}'".format( 492 auto_install_list_file), mode=self.config.log_mode) 493 494 '''Description: output src flag into a json file. \ 495 (/out/${product_name}/build_configs/parts_src_flag.json) 496 @parameter: none 497 @return :none 498 ''' 499 500 def _generate_src_flag(self): 501 parts_src_flag_file = os.path.join(self.config_output_dir, 502 "parts_src_flag.json") 503 write_json_file(parts_src_flag_file, 504 self._get_parts_src_list(), 505 check_changes=True) 506 LogUtil.hb_info( 507 "generated parts src flag to '{}/subsystem_info/parts_src_flag.json'".format( 508 self.config_output_dir), mode=self.config.log_mode) 509 510 '''Description: output build target list into a json file.\ 511 (/out/${product_name}/build_configs/required_parts_targets_list.json) 512 @parameter: none 513 @return :none 514 ''' 515 516 def _generate_required_parts_targets_list(self): 517 build_targets_list_file = os.path.join(self.config_output_dir, 518 "required_parts_targets_list.json") 519 write_json_file(build_targets_list_file, 520 list(self.required_parts_targets.values())) 521 LogUtil.hb_info("generate build targets list file to '{}'".format( 522 build_targets_list_file), mode=self.config.log_mode) 523 524 '''Description: output build target info into a json file. \ 525 (/out/${product_name}/build_configs/required_parts_targets.json) 526 @parameter: none 527 @return: none 528 ''' 529 530 def _generate_required_parts_targets(self): 531 build_targets_info_file = os.path.join(self.config_output_dir, 532 "required_parts_targets.json") 533 write_json_file(build_targets_info_file, self.required_parts_targets) 534 LogUtil.hb_info("generate required parts targets to '{}'".format( 535 build_targets_info_file), mode=self.config.log_mode) 536 537 '''Description: output platforms part by src into a json file. \ 538 (/out/${product_name}/build_configs/platforms_parts_by_src.json) 539 @parameter: none 540 @return :none 541 ''' 542 543 def _generate_platforms_part_by_src(self): 544 platforms_parts_by_src = self._get_platforms_parts() 545 platforms_parts_by_src_file = os.path.join(self.source_root_dir, 546 self.config_output_relpath, 547 "platforms_parts_by_src.json") 548 write_json_file(platforms_parts_by_src_file, 549 platforms_parts_by_src, 550 check_changes=True) 551 LogUtil.hb_info("generated platforms parts by src to '{}'".format( 552 platforms_parts_by_src_file), mode=self.config.log_mode) 553 554 '''Description: output system configs info into 4 files:[ 555 (/out/${product_name}/build_configs/subsystem_info/parts_list.gni), 556 (/out/${product_name}/build_configs/subsystem_info/inner_kits_list.gni), 557 (/out/${product_name}/build_configs/subsystem_info/system_kits_list.gni), 558 (/out/${product_name}/build_configs/subsystem_info/parts_test_list.gni), 559 (/out/${product_name}/build_configs/subsystem_info/BUILD.gn)] 560 @parameter: none 561 @return :none 562 ''' 563 564 def _generate_target_gn(self): 565 generate_targets_gn.gen_targets_gn(self.required_parts_targets, 566 self.config_output_dir) 567 568 '''Description: output phony targets build file. \ 569 (/out/${product_name}/build_configs/phony_target/BUILD.gn) 570 @parameter: none 571 @return :none 572 ''' 573 574 def _generate_phony_targets_build_file(self): 575 generate_targets_gn.gen_phony_targets(self.required_phony_targets, 576 self.config_output_dir) 577 578 '''Description: output system configs info into 2 files:[ 579 (/out/${product_name}/build_configs/subsystem_info/${platform}-stub/BUILG.gn), 580 (/out/${product_name}/build_configs/subsystem_info/${platform}-stub/zframework_stub_exists.gni)] 581 @parameter: none 582 @return :none 583 ''' 584 585 def _generate_stub_targets(self): 586 generate_targets_gn.gen_stub_targets( 587 self.parts_config_info.get('parts_kits_info'), 588 self.target_platform_stubs, 589 self.config_output_dir) 590 591 '''Description: output system capabilities into a json file. \ 592 (/out/${product_name}/build_configs/${platform}_system_capabilities.json) 593 @parameter: none 594 @return :none 595 ''' 596 597 def _generate_system_capabilities(self): 598 for platform in self.build_platforms: 599 platform_parts = self.target_platform_parts.get(platform) 600 platform_capabilities = [] 601 for _, origin in platform_parts.items(): 602 # parts_info.get() might be None if the part is a binary package 603 all_parts_variants = self.parts_info.get(origin) 604 if all_parts_variants is None: 605 continue 606 part = all_parts_variants[0] 607 if part.get('system_capabilities'): 608 entry = part.get('system_capabilities') 609 if len(entry) > 0: 610 platform_capabilities.extend(entry) 611 platform_part_json_file = os.path.join( 612 self.config_output_dir, "{0}_system_capabilities.json".format(platform)) 613 write_json_file(platform_part_json_file, 614 sorted(platform_capabilities), 615 check_changes=True) 616 LogUtil.hb_info( 617 "generated system capabilities to '{}/{}_system_capabilities.json'".format( 618 self.config_output_dir, platform), mode=self.config.log_mode) 619 620 '''Description: output system configs info into three json files:[ 621 (/out/${product_name}/build_configs/subsystem_info/subsystem_build_config.json), 622 (/out/${product_name}/build_configs/subsystem_info/src_subsystem_info.json), 623 (/out/${product_name}/build_configs/subsystem_info/no_src_subsystem_info.json)] 624 @parameter: none 625 @return :none 626 ''' 627 628 def _generate_subsystem_configs(self): 629 630 # The function has been implemented in module util/loader/subsystem_info.py 631 LogUtil.hb_info( 632 "generated subsystem build config to '{}/subsystem_info/subsystem_build_config.json'".format( 633 self.config_output_dir), mode=self.config.log_mode) 634 LogUtil.hb_info( 635 "generated src subsystem info to '{}/subsystem_info/src_subsystem_info.json'".format( 636 self.config_output_dir), mode=self.config.log_mode) 637 LogUtil.hb_info( 638 "generated no src subsystem info to '{}/subsystem_info/no_src_subsystem_info.json'".format( 639 self.config_output_dir), mode=self.config.log_mode) 640 641 def _get_parts_by_platform(self) -> dict: 642 parts_info = {} 643 if 'phone' in self.target_platform_parts: 644 phone_parts_list = self.target_platform_parts.get('phone').keys() 645 else: 646 phone_parts_list = [] 647 for _platform, _parts_info in self.target_platform_parts.items(): 648 base_parts_list = [] 649 curr_parts_list = [] 650 for _real_name, _original_name in _parts_info.items(): 651 if _real_name in phone_parts_list: 652 base_parts_list.append(_real_name) 653 elif _original_name in phone_parts_list: 654 base_parts_list.append(_real_name) 655 else: 656 curr_parts_list.append(_real_name) 657 result_data = { 658 "base_parts_list": base_parts_list, 659 "curr_parts_list": curr_parts_list 660 } 661 parts_info[_platform] = result_data 662 return parts_info 663 664 def _get_platforms_all_parts(self) -> dict: 665 _dist_parts_variants = self._load_component_dist() 666 target_platform_parts = {} 667 all_parts = self._platforms_info.get('all_parts') 668 parts_variants = self.parts_config_info.get('parts_variants') 669 for _platform, _parts in all_parts.items(): 670 if _platform not in self.build_platforms: 671 continue 672 part_name_info = {} 673 for part_def in _parts: 674 real_name, original_name = self._get_real_part_name( 675 part_def, _platform, parts_variants) 676 if real_name is None: 677 # find this from component_dist 678 real_name, original_name = self._get_real_part_name( 679 part_def, _platform, _dist_parts_variants) 680 if real_name is None: 681 continue 682 part_name_info[real_name] = original_name 683 target_platform_parts[_platform] = part_name_info 684 return target_platform_parts 685 686 def _get_platforms_all_stubs(self) -> dict: 687 _dist_parts_variants = self._load_component_dist() 688 platform_stubs = {} 689 all_stubs = self._platforms_info.get('all_stubs') 690 parts_variants = self.parts_config_info.get('parts_variants') 691 for _platform, _part_names in all_stubs.items(): 692 if _platform not in self.build_platforms: 693 continue 694 stub_parts_from_src = [] 695 stub_parts_from_dist = [] 696 for part_name in _part_names: 697 real_name, original_name = self._get_real_part_name( 698 part_name, _platform, parts_variants) 699 # real_name=None means part_name doesn't exist in source tree, 700 # use binary in component_dist then. 701 if real_name is None: 702 # find this from component_dist 703 real_name, original_name = self._get_real_part_name( 704 part_name, _platform, _dist_parts_variants) 705 if real_name is None: 706 continue 707 else: 708 stub_sources = os.path.join( 709 self.source_root_dir, 710 "component_dist/{}-{}/api_stubs/{}/stubs_sources_list.txt" # noqa: E501 711 .format(self.target_os, self.target_cpu, real_name)) 712 stub_parts_from_dist.append( 713 '"{}"'.format(stub_sources)) 714 else: 715 stub_parts_from_src.append(real_name) 716 platform_stubs[_platform] = { 717 "src": stub_parts_from_src, 718 "dist": stub_parts_from_dist, 719 } 720 return platform_stubs 721 722 def _get_platforms_parts(self) -> dict: 723 platforms_parts = {} 724 src_parts_targets = self.parts_targets 725 src_all_parts = src_parts_targets.keys() 726 for _platform, _all_parts in self.target_platform_parts.items(): 727 src_parts_list = [] 728 no_src_parts_list = [] 729 for _part in _all_parts.keys(): 730 if _part in src_all_parts: 731 src_parts_list.append(_part) 732 else: 733 no_src_parts_list.append(_part) 734 _data = { 735 'src_parts': src_parts_list, 736 'no_src_parts': no_src_parts_list 737 } 738 platforms_parts[_platform] = _data 739 return platforms_parts 740 741 def _get_parts_src_list(self) -> list: 742 parts_name_map = {} 743 for _list in self.parts_info.values(): 744 for _info in _list: 745 parts_name_map[_info.get('part_name')] = _info.get( 746 'origin_part_name') 747 _src_set = set() 748 for _name in self.required_parts_targets.keys(): 749 _origin_name = parts_name_map.get(_name) 750 if _origin_name is None: 751 continue 752 _src_set.add(_origin_name) 753 return list(_src_set) 754 755 def _get_required_build_targets(self) -> dict: 756 required_build_targets = {} 757 for _p_name, _info in self.parts_targets.items(): 758 if _p_name not in self.required_parts_targets_list: 759 continue 760 required_build_targets[_p_name] = _info 761 return required_build_targets 762 763 def _get_required_phony_targets(self) -> dict: 764 required_build_targets = {} 765 for _p_name, _info in self.phony_targets.items(): 766 if _p_name not in self.required_parts_targets_list: 767 continue 768 required_build_targets[_p_name] = _info 769 return required_build_targets 770 771 def _get_required_build_parts_list(self) -> list: 772 parts_set = set() 773 for _parts_list in self.target_platform_parts.values(): 774 parts_set.update(_parts_list) 775 return list(parts_set) 776 777# util method 778 779 def _load_component_dist(self) -> dict: 780 _parts_variants_info = {} 781 _dir = "component_dist/{}-{}/packages_to_install".format( 782 self.target_os, self.target_cpu) 783 _file_name = "dist_parts_info.json" 784 _dist_parts_info_file = os.path.join( 785 self.source_root_dir, _dir, _file_name) 786 if not os.path.exists(_dist_parts_info_file): 787 # If the file does not exist, do nothing and return 788 return _parts_variants_info 789 _parts_info = read_json_file(_dist_parts_info_file) 790 if _parts_info is None: 791 raise Exception("read file '{}' failed.".format( 792 _dist_parts_info_file)) 793 for _part_info in _parts_info: 794 origin_part_name = _part_info.get('origin_part_name') 795 if origin_part_name in _parts_variants_info: 796 variants = _parts_variants_info.get(origin_part_name) 797 else: 798 variants = [] 799 _variant_name = _part_info.get('variant_name') 800 variants.append(_variant_name) 801 _parts_variants_info[origin_part_name] = variants 802 return _parts_variants_info 803 804 def _get_real_part_name(self, original_part_name: str, current_platform: str, parts_variants: dict): 805 part_info = parts_variants.get(original_part_name) 806 if part_info is None: 807 return None, None 808 if current_platform in part_info and current_platform != 'phone': 809 real_name = '{}_{}'.format(original_part_name, current_platform) 810 else: 811 real_name = original_part_name 812 return real_name, original_part_name 813 814 '''Description: called by _out_infos_for_testfwk, output information by platform 815 @parameter:none 816 @return :none 817 ''' 818 819 def _output_infos_by_platform(self, part_name_infos: dict, parts_info_dict: dict): 820 required_parts = {} 821 subsystem_infos = {} 822 for part_name, origin_part_name in part_name_infos.items(): 823 part_info = parts_info_dict.get(part_name) 824 if part_info is None: 825 continue 826 if origin_part_name != part_info.get('origin_part_name'): 827 raise Exception("part configuration is incorrect.") 828 required_parts[origin_part_name] = part_info 829 _subsystem_name = part_info.get('subsystem_name') 830 if _subsystem_name in subsystem_infos: 831 p_list = subsystem_infos.get(_subsystem_name) 832 else: 833 p_list = [] 834 p_list.append(origin_part_name) 835 subsystem_infos[_subsystem_name] = p_list 836 result = {} 837 result['subsystem_infos'] = subsystem_infos 838 result['part_infos'] = required_parts 839 return result 840 841 def _execute_loader_args_display(self): 842 LogUtil.hb_info('Loading configuration file...') 843 args = [] 844 args.append('platforms_config_file="{}"'.format( 845 self.platforms_config_file)) 846 args.append('subsystem_config_file="{}"'.format( 847 self.subsystem_config_file)) 848 args.append('example_subsystem_file="{}"'.format( 849 self.example_subsystem_file)) 850 args.append('exclusion_modules_config_file="{}"'.format( 851 self.exclusion_modules_config_file)) 852 args.append('source_root_dir="{}"'.format(self.source_root_dir)) 853 args.append('gn_root_out_dir="{}"'.format(self.gn_root_out_dir)) 854 args.append('build_platform_name={}'.format(self.build_platform_name)) 855 args.append('build_xts={}'.format(self.build_xts)) 856 args.append('load_test_config={}'.format(self.load_test_config)) 857 args.append('target_os={}'.format(self.target_os)) 858 args.append('target_cpu={}'.format(self.target_cpu)) 859 args.append('os_level={}'.format(self.os_level)) 860 args.append('ignore_api_check={}'.format(self.ignore_api_check)) 861 args.append('scalable_build={}'.format(self.scalable_build)) 862 args.append('skip_partlist_check={}'.format(self.skip_partlist_check)) 863 LogUtil.write_log(self.config.log_path, 864 'loader args:{}'.format(args), 'info') 865 866 def _override_components(self): 867 '''Description: Check whether there are components that need to be replaced, and if so, 868 replace the component configuration file bundle.json in subsystem_info and update 869 the component list generated by the preloader. 870 @parameter:none 871 @return :overrided_components 872 ''' 873 parts_file = self.platforms_config_file.replace( 874 "platforms.build", "parts.json") 875 all_parts = read_json_file(parts_file) 876 if "parts" not in all_parts: 877 LogUtil.hb_warning("{} does not contain parts!".format(parts_file)) 878 return {} 879 overrided = False 880 overrided_components = {} 881 all_parts = all_parts["parts"] 882 component_override_map = {} 883 all_component_override_map = {} 884 for subsystem_name, build_config_info in self._subsystem_info.items(): 885 if "build_files" not in build_config_info: 886 continue 887 888 # scan all bundle.json or ohos.build files with named groups 889 for build_file in build_config_info["build_files"]: 890 891 # ohos.build does not support overrided components 892 if not build_file.endswith('bundle.json'): 893 continue 894 895 # Only device or vendor components can do named groups extensions 896 if (not build_file.startswith(self.source_root_dir + 'device/')) \ 897 and (not build_file.startswith(self.source_root_dir + 'vendor/')): 898 continue 899 900 # "subsystem", "name" and "override" is required 901 component = read_json_file(build_file).get("component") 902 903 if (not component) or (not all(key in component for key in ("subsystem", "name", "override"))): 904 continue 905 906 full_part_name = f"{component.get('subsystem')}:{component.get('name')}" 907 if full_part_name not in all_parts: 908 LogUtil.hb_warning("{} was not configured for this product: {}".format( 909 build_file, full_part_name)) 910 continue 911 912 if self._override_one_component(self._subsystem_info, component, build_file, all_parts, overrided_components, component_override_map): 913 overrided = True 914 915 if overrided: 916 # Update parts.json and parts_config.json generated by preloader 917 write_json_file(parts_file, {"parts": all_parts}) 918 parts_file = self.platforms_config_file.replace( 919 "platforms.build", "parts_config.json") 920 self._output_parts_config_json(all_parts, parts_file) 921 all_component_override_map.update(component_override_map) 922 write_json_file( 923 f"{self.config_output_dir}/component_override_map.json", all_component_override_map) 924 return overrided_components 925 926 def _override_one_component(self, subsystem_info: dict, component: dict, build_file: str, all_parts: dict, overrided_components: dict, component_override_map: dict): 927 '''Description: Perform a replacement of a single component and return the component list update result. 928 @parameter:subsystem_info, component, build_file, all_parts, overrided_components 929 @return :True or False(Whether replacement has been performed) 930 ''' 931 splits = component["override"].split(":") 932 if len(splits) != 2: 933 LogUtil.hb_warning( 934 "{} override value is invalid format. Skip override process".format(build_file)) 935 return False 936 overrided_subsystem = splits[0] 937 overrided_component = splits[1] 938 if overrided_subsystem not in subsystem_info: 939 LogUtil.hb_warning( 940 "{} override invalid subsystem. Skip override process".format(build_file)) 941 return False 942 943 founded_bundle = "" 944 945 for bundle in subsystem_info[overrided_subsystem]["build_files"]: 946 if not bundle.endswith('bundle.json'): 947 continue 948 949 bundle_obj = read_json_file(bundle) 950 951 if bundle_obj.get("component", {}).get("name") == overrided_component: 952 founded_bundle = bundle 953 break 954 955 if founded_bundle: 956 origin_component = read_json_file(build_file).get('component') 957 LogUtil.hb_warning( 958 f"You are trying to override \"{component['override']}\" with \"{origin_component.get('subsystem')}:{origin_component.get('name')}\". \nPlease ensure that the modules in \"{component['override']}\" only rely on the interfaces of other components through \"external_deps\"") 959 960 # replace bundle.json in subsystem_info's build_files 961 subsystem_info[overrided_subsystem]["build_files"].remove( 962 founded_bundle) 963 964 # Update parts.json generated by preloader, which means that new added components will not be installed 965 # Ensure that the overrided components will be installed 966 full_partname = f"{overrided_subsystem}:{overrided_component}" 967 if full_partname in all_parts: 968 all_parts.remove(full_partname) 969 970 overrided_components[f"{component['subsystem']}:{component['name']}"] = { 971 'subsystem': overrided_subsystem, 972 'partName': overrided_component 973 } 974 component_override_map[overrided_component] = component["name"] 975 return True 976 LogUtil.hb_warning( 977 "{}:{} is not configured in product, \new add component will be installed!".format( 978 overrided_subsystem, overrided_component)) 979 return False 980 981 def _output_parts_config_json(self, all_parts: dict, output_file: dict): 982 '''Description: Update the parts list file generated by preloader 983 @parameter: all_parts, output_file 984 @return :none 985 ''' 986 parts_config = {} 987 for part in all_parts: 988 part = part.replace(":", "_") 989 part = part.replace("-", "_") 990 part = part.replace(".", "_") 991 part = part.replace("/", "_") 992 parts_config[part] = True 993 write_json_file(output_file, parts_config) 994