1#!/usr/bin/env python 2# -*- coding: utf-8 -*- 3# Copyright (c) 2021 Huawei Device Co., Ltd. 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15 16from collections import defaultdict 17import argparse 18import hashlib 19import os 20import os.path 21import sys 22import gzip 23import shutil 24import glob 25import re 26import subprocess 27 28sys.path.append( 29 os.path.dirname(os.path.dirname(os.path.dirname( 30 os.path.abspath(__file__))))) 31from scripts.util import build_utils # noqa: E402 32from scripts.util.file_utils import write_json_file, read_json_file # noqa: E402 33 34xml_escape_table = { 35 "&": "&", 36 '"': """, 37 "'": "'", 38 ">": ">", 39 "<": "<", 40} 41 42 43def copy_static_library_notices(options, depfiles: list): 44 valid_notices = [] 45 basenames = [] 46 # add sort method 47 files = build_utils.get_all_files(options.static_library_notice_dir) 48 files.sort() 49 for file in files: 50 if os.stat(file).st_size == 0: 51 continue 52 if not file.endswith('.a.txt'): 53 continue 54 notice_file_name = os.path.basename(file) 55 if file not in basenames: 56 basenames.append(notice_file_name) 57 valid_notices.append(file) 58 depfiles.append(file) 59 60 for file in valid_notices: 61 if options.image_name == "system": 62 if options.target_cpu == "arm64" or options.target_cpu == "x64": 63 install_dir = "system/lib64" 64 elif options.target_cpu == "arm": 65 install_dir = "system/lib" 66 else: 67 continue 68 elif options.image_name == "sdk": 69 install_dir = "toolchains/lib" 70 elif options.image_name == "ndk": 71 install_dir = "sysroot/usr/lib" 72 else: 73 continue 74 dest = os.path.join(options.notice_root_dir, install_dir, 75 os.path.basename(file)) 76 os.makedirs(os.path.dirname(dest), exist_ok=True) 77 shutil.copyfile(file, dest) 78 if os.path.isfile("{}.json".format(file)): 79 os.makedirs(os.path.dirname("{}.json".format(dest)), exist_ok=True) 80 shutil.copyfile("{}.json".format(file), "{}.json".format(dest)) 81 82 83def write_file(file: str, string: str): 84 print(string, file=file) 85 86 87def compute_hash(file: str): 88 sha256 = hashlib.sha256() 89 with open(file, 'rb') as file_fd: 90 for line in file_fd: 91 sha256.update(line) 92 return sha256.hexdigest() 93 94 95def get_entity(text: str): 96 return "".join(xml_escape_table.get(c, c) for c in text) 97 98 99def generate_txt_notice_files(file_hash: str, input_dir: str, output_filename: str, 100 notice_title: str): 101 with open(output_filename, "w") as output_file: 102 write_file(output_file, notice_title) 103 for value in file_hash: 104 write_file(output_file, '=' * 60) 105 write_file(output_file, "Notices for file(s):") 106 for filename in value: 107 write_file( 108 output_file, '/{}'.format( 109 re.sub('.txt.*', '', 110 os.path.relpath(filename, input_dir)))) 111 write_file(output_file, '-' * 60) 112 write_file(output_file, "Notices for software(s):") 113 software_list = [] 114 for filename in value: 115 json_filename = '{}.json'.format(filename) 116 contents = read_json_file(json_filename) 117 if contents is not None and contents not in software_list: 118 software_list.append(contents) 119 software_dict = {} 120 for contents_value in software_list: 121 if len(contents_value) > 0: 122 for val in contents_value: 123 if val.get('Software'): 124 software_name = val.get('Software').strip() 125 if software_name not in software_dict: 126 software_dict[software_name] = {"_version": "", "_path": []} 127 else: 128 write_file(output_file, "Software: ") 129 if val.get('Version'): 130 version = val.get('Version').strip() 131 software_dict[software_name]["_version"] = version 132 else: 133 write_file(output_file, "Version: ") 134 if val.get('Path'): 135 notice_source_path = val.get('Path').strip() 136 software_dict[software_name]["_path"].append(notice_source_path) 137 for software, software_value in software_dict.items(): 138 write_file(output_file, f"Software: {software}") 139 write_file(output_file, f"Version: {software_value.get('_version')}") 140 if software_value.get("_path"): 141 for path in software_value.get("_path"): 142 write_file(output_file, f"Path: {path}") 143 write_file(output_file, '-' * 60) 144 with open(value[0], errors='ignore') as temp_file_hd: 145 write_file(output_file, temp_file_hd.read()) 146 147 148def generate_xml_notice_files(files_with_same_hash: dict, input_dir: str, 149 output_filename: str): 150 id_table = {} 151 for file_key in files_with_same_hash.keys(): 152 for filename in files_with_same_hash[file_key]: 153 id_table[filename] = file_key 154 with open(output_filename, "w") as output_file: 155 write_file(output_file, '<?xml version="1.0" encoding="utf-8"?>') 156 write_file(output_file, "<licenses>") 157 158 # Flatten the lists into a single filename list 159 sorted_filenames = sorted(id_table.keys()) 160 161 # write out a table of contents 162 for filename in sorted_filenames: 163 stripped_filename = re.sub('.txt.*', '', 164 os.path.relpath(filename, input_dir)) 165 write_file( 166 output_file, '<file-name content_id="%s">%s</file-name>' % 167 (id_table.get(filename), stripped_filename)) 168 169 write_file(output_file, '') 170 write_file(output_file, '') 171 172 processed_file_keys = [] 173 # write the notice file lists 174 for filename in sorted_filenames: 175 file_key = id_table.get(filename) 176 if file_key in processed_file_keys: 177 continue 178 processed_file_keys.append(file_key) 179 180 with open(filename, errors='ignore') as temp_file_hd: 181 write_file( 182 output_file, 183 '<file-content content_id="{}"><![CDATA[{}]]></file-content>' 184 .format(file_key, get_entity(temp_file_hd.read()))) 185 write_file(output_file, '') 186 187 # write the file complete node. 188 write_file(output_file, "</licenses>") 189 190 191def compress_file_to_gz(src_file_name: str, gz_file_name: str): 192 with open(src_file_name, mode='rb') as src_file_fd: 193 with gzip.open(gz_file_name, mode='wb') as gz_file_fd: 194 gz_file_fd.writelines(src_file_fd) 195 196 197def handle_zipfile_notices(zip_file: str): 198 notice_file = '{}.txt'.format(zip_file[:-4]) 199 with build_utils.temp_dir() as tmp_dir: 200 build_utils.extract_all(zip_file, tmp_dir, no_clobber=False) 201 files = build_utils.get_all_files(tmp_dir) 202 contents = [] 203 for file in files: 204 with open(file, 'r') as fd: 205 data = fd.read() 206 if data not in contents: 207 contents.append(data) 208 with open(notice_file, 'w') as merged_notice: 209 merged_notice.write('\n\n'.join(contents)) 210 return notice_file 211 212 213def do_merge_notice(args, zipfiles: str, txt_files: str): 214 notice_dir = args.notice_root_dir 215 notice_txt = args.output_notice_txt 216 notice_gz = args.output_notice_gz 217 notice_title = args.notice_title 218 219 if not notice_txt.endswith('.txt'): 220 raise Exception( 221 'Error: input variable output_notice_txt must ends with .txt') 222 if not notice_gz.endswith('.xml.gz'): 223 raise Exception( 224 'Error: input variable output_notice_gz must ends with .xml.gz') 225 226 notice_xml = notice_gz.replace('.gz', '') 227 228 files_with_same_hash = defaultdict(list) 229 for file in zipfiles: 230 txt_files.append(handle_zipfile_notices(file)) 231 232 for file in txt_files: 233 if os.stat(file).st_size == 0: 234 continue 235 file_hash = compute_hash(file) 236 files_with_same_hash[file_hash].append(file) 237 238 file_sets = [ 239 sorted(files_with_same_hash[hash]) 240 for hash in sorted(files_with_same_hash.keys()) 241 ] 242 243 if file_sets is not None: 244 generate_txt_notice_files(file_sets, notice_dir, notice_txt, 245 notice_title) 246 247 if files_with_same_hash is not None: 248 generate_xml_notice_files(files_with_same_hash, notice_dir, notice_xml) 249 compress_file_to_gz(notice_xml, args.output_notice_gz) 250 251 if args.notice_module_info: 252 module_install_info_list = [] 253 module_install_info = {} 254 module_install_info['type'] = 'notice' 255 module_install_info['source'] = args.output_notice_txt 256 module_install_info['install_enable'] = True 257 module_install_info['dest'] = [ 258 os.path.join(args.notice_install_dir, 259 os.path.basename(args.output_notice_txt)) 260 ] 261 module_install_info_list.append(module_install_info) 262 write_json_file(args.notice_module_info, module_install_info_list) 263 264 if args.lite_product: 265 current_dir_cmd = ['pwd'] 266 process = subprocess.Popen(current_dir_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 267 stdout, stderr = process.communicate(timeout=600) 268 current_dir = stdout.decode().strip() 269 dest = f"{current_dir}/system/etc/NOTICE.txt" 270 if os.path.isfile(notice_txt): 271 os.makedirs(os.path.dirname(dest), exist_ok=True) 272 shutil.copyfile(notice_txt, dest) 273 274 275def parse_args(): 276 """Parses command-line arguments.""" 277 parser = argparse.ArgumentParser() 278 parser.add_argument('--image-name') 279 parser.add_argument('--collected-notice-zipfile', 280 action='append', 281 help='zipfile stors collected notice files') 282 parser.add_argument('--notice-root-dir', help='where notice files store') 283 parser.add_argument('--output-notice-txt', help='output notice.txt') 284 parser.add_argument('--output-notice-gz', help='output notice.txt') 285 parser.add_argument('--notice-title', help='title of notice.txt') 286 parser.add_argument('--static-library-notice-dir', 287 help='path to static library notice files') 288 parser.add_argument('--target-cpu', help='cpu arch') 289 parser.add_argument('--depfile', help='depfile') 290 parser.add_argument('--notice-module-info', 291 help='module info file for notice target') 292 parser.add_argument('--notice-install-dir', 293 help='install directories of notice file') 294 parser.add_argument('--lite-product', help='', default="") 295 296 297 return parser.parse_args() 298 299 300def main(): 301 """Main function to merge and generate notice files.""" 302 args = parse_args() 303 304 notice_dir = args.notice_root_dir 305 depfiles = [] 306 if args.collected_notice_zipfile: 307 for zip_file in args.collected_notice_zipfile: 308 build_utils.extract_all(zip_file, notice_dir, no_clobber=False) 309 else: 310 depfiles += build_utils.get_all_files(notice_dir) 311 # Copy notice of static targets to notice_root_dir 312 if args.static_library_notice_dir: 313 copy_static_library_notices(args, depfiles) 314 315 zipfiles = glob.glob('{}/**/*.zip'.format(notice_dir), recursive=True) 316 317 txt_files = glob.glob('{}/**/*.txt'.format(notice_dir), recursive=True) 318 txt_files += glob.glob('{}/**/*.txt.?'.format(notice_dir), recursive=True) 319 320 outputs = [args.output_notice_txt, args.output_notice_gz] 321 if args.notice_module_info: 322 outputs.append(args.notice_module_info) 323 build_utils.call_and_write_depfile_if_stale( 324 lambda: do_merge_notice(args, zipfiles, txt_files), 325 args, 326 depfile_deps=depfiles, 327 input_paths=depfiles, 328 input_strings=args.notice_title + args.target_cpu, 329 output_paths=(outputs)) 330 331 332if __name__ == "__main__": 333 main() 334