1#!/usr/bin/env python3
2# encoding: utf-8
3# Copyright 2024 Huawei Technologies Co., Ltd
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9# http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16# ============================================================================
17
18import os
19import shutil
20import zipfile
21import argparse
22import hashlib
23import subprocess
24
25def extract_source(in_zip_path, out_src_path):
26    "depress source code form release package"
27    print('Extracting zipped release package...')
28    f = zipfile.ZipFile(in_zip_path, "r")
29    f.extractall(path=out_src_path)
30    old_src_dir = out_src_path + "/mindspore-v2.1.0/"
31    new_src_dir = out_src_path + "/source/"
32    os.rename(old_src_dir, new_src_dir)
33    print("Done extraction.")
34
35def do_patch(patch_dir, target_dir):
36    patches = [
37        '0001-build-gn-c-api-for-OHOS.patch',
38        '0002-train-and-build.patch',
39        '0003-add-js-api.patch',
40        '0004-cross-compile-ndkso-fp16-nnrt-train_capi.patch',
41        '0005-micro-for-ohos.patch',
42        '0006-remove-lite-expression-fix-double-loadso.patch',
43        '0007-deobfuscator.patch',
44        '0008-upgrade-flatbuffers-fix_crash.patch',
45        '0009-npu-zero-copy.patch',
46        '0010-micro-dynamic-shape-support-discrete-value.patch',
47        '0011-fix-npu-infer-memory-leak-delete-liteGraph.patch',
48        '0012-add-mindir-ops.patch',
49        '0013-hiappevent.patch',
50        '0014-DynamicQuant-strategy-optimization.patch',
51        '0015-bugfix-for-cpu-kernel.patch',
52        '0016-bugfix-for-argminmax-swish-int8-and-vad-asan.patch',
53        '0017-bugfix-for-onnx-parser.patch',
54        '0018-nnrt-litegraph-dequant.patch',
55        '0019-adaper-NNCore-Api.patch',
56        '0020-fix-ocr-gcn-model-crash.patch',
57        '0021-add-mindir-ops.patch',
58        '0022-adapter-HiAI-Foundation-NPU.patch',
59        '0023-support-x86-emulator-build.patch',
60        '0024-fix-gcn-model-squeeze-transpose-infershape-not-do.patch',
61        '0025-support-kirin-npu-dynamic-dims.patch',
62        '0026-fix-depthwise-conv-kernel.patch',
63        '0027-reduce-memory-when-npu-compilation-with-cache.patch',
64        '0028-fix-onnx-parser-and-cpu-kernel.patch',
65        '0029-remove-recursive-lock.patch',
66        '0030-generate-flatbuffer-notice.patch',
67        '0031-fix-matmul-assemble-can-not-protect-stack-in-mutil-thread.patch',
68        '0032-fix-for-concat-bool-type.patch',
69        '0033-fix-nullptr-etal.patch',
70        '0034-fix-semicolon-and-nullptr-etal.patch',
71        '0035-fix-ArgMaxFusion_get_max_parameter_uninitialized.patch',
72        '0036-fix-split-non-uniform-split.patch',
73        '0037-fix-for-fuzz-problem.patch',
74        '0038-refactor-nnrt-allocator-to-singleton.patch',
75        '0039-add-compile-option-SP-extensions-white-list.patch',
76        '0040-fix-context-double-free.patch',
77        '0041-fix-MutableData-memory-leak.patch',
78        '0042-scatterND-indices-illegal.patch',
79        '0043-fix-too-many-hi-app-event-reports.patch',
80    ]
81
82    cwd = os.getcwd()
83    os.chdir(target_dir)
84    print('Change dir to', os.getcwd())
85    subprocess.run(['git', 'init', '.'])
86    subprocess.run(['git', 'add', '.'])
87    subprocess.run(['git', 'commit', '-m', '"init"'])
88
89    for patch in patches:
90        print('Applying ', patch, '...')
91        ret = subprocess.run(['git', 'apply', '{0}/{1}'.format(patch_dir, patch)])
92        if ret.returncode != 0:
93            raise Exception("Apply patch {0} failed, ret: {1}".format(patch, ret))
94        subprocess.run(['git', 'add', '.'])
95        subprocess.run(['git', 'commit', '-m', "auto-apply {0}".format(patch)])
96        print('Done')
97    os.chdir(cwd)
98
99def create_status_file(out_src_path):
100    with open("{0}/.status".format(out_src_path), 'w+') as f:
101        f.write('ok')
102
103
104def compute_md5(file):
105    m = hashlib.md5()
106    with open(file, 'rb') as f:
107        m.update(f.read())
108    return m.hexdigest()
109
110
111def save_md5s(folder_path, out_path):
112    files_list = []
113    for file_name in os.listdir(folder_path):
114        if (file_name.endswith(".patch")):
115            files_list.append(file_name)
116
117    os.makedirs(out_path, exist_ok=True)
118    for pf in files_list:
119        md5_path = os.path.join(out_path, pf.replace(".patch", ".md5"))
120        with open(md5_path, 'w') as f:
121            f.write(compute_md5(os.path.join(folder_path, pf)))
122
123
124def md5_changed(patch_path, md5_path):
125    if not os.path.exists(md5_path):
126        return True
127    patch_list = []
128    md5_list = []
129    for file_name in os.listdir(patch_path):
130        if (file_name.endswith(".patch")):
131            patch_list.append(file_name)
132    for file_name in os.listdir(md5_path):
133        if (file_name.endswith(".md5")):
134            md5_list.append(file_name)
135    if (len(patch_list) != len(md5_list)):
136        return True
137
138    for md5_file in md5_list:
139        if not os.path.exists(os.path.join(patch_path, md5_file.replace(".md5", ".patch"))):
140            return True
141        with open(os.path.join(md5_path, md5_file), 'r') as f:
142            origin_v = f.read().strip()
143            if (origin_v != compute_md5(os.path.join(patch_path, md5_file.replace(".md5", ".patch")))):
144                return True
145    return False
146
147
148def source_has_changed(out_src_path, patch_path, md5_path):
149    if not os.path.exists(os.path.join(out_src_path, ".status")):
150        print(".status not exist.")
151        return True
152    return md5_changed(patch_path, md5_path)
153
154
155def main_work():
156    parser = argparse.ArgumentParser(description="mindspore build helper")
157    parser.add_argument('--in_zip_path')
158    parser.add_argument('--out_src_path')
159    parser.add_argument('--patch_dir')
160    args = vars(parser.parse_args())
161
162    in_zip_path = os.path.realpath(args['in_zip_path'])
163    out_src_path = args['out_src_path']
164    patch_dir = os.path.realpath(args['patch_dir'])
165
166    md5_dir = os.path.join(out_src_path, "patches_md5")
167    if source_has_changed(out_src_path, patch_dir, md5_dir):
168        print("remove ", out_src_path)
169        if os.path.exists(out_src_path):
170            shutil.rmtree(out_src_path)
171        save_md5s(patch_dir, md5_dir)
172
173    if os.path.exists(os.path.join(out_src_path, ".status")):
174        print("patch files not changed and " + os.path.join(out_src_path, ".status") + " exists.")
175        return
176
177    os.makedirs(out_src_path, exist_ok=True)
178    out_src_path = os.path.realpath(out_src_path)
179
180    extract_source(in_zip_path, out_src_path)
181
182    do_patch(patch_dir, out_src_path + '/source/')
183
184    create_status_file(out_src_path)
185
186
187if __name__ == "__main__":
188    main_work()
189
190