1#!/usr/bin/env python3
2# -*- coding: utf-8 -*-
3# Copyright (c) 2022 Huawei Device Co., Ltd.
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8#     http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15
16# This file contains a RomAnalyzer for rom analyzation of standard device.
17
18import argparse
19import json
20import os
21import sys
22import typing
23from copy import deepcopy
24from typing import Dict, Any, Iterator, List, Text
25import re
26import subprocess
27from pkgs.rom_ram_baseline_collector import RomRamBaselineCollector
28from pkgs.basic_tool import BasicTool, unit_adaptive
29from pkgs.gn_common_tool import GnCommonTool, GnVariableParser
30from pkgs.simple_excel_writer import SimpleExcelWriter
31
32debug = bool(sys.gettrace())
33
34NOTFOUND = "NOTFOUND"
35
36
37class PreCollector:
38    """
39    collect some info that system_module_info.json dosn't contains
40    """
41
42    def __init__(self, project_path: str) -> None:
43        self.info_dict: Dict[str, Any] = dict()
44        self.project_path = BasicTool.get_abs_path(project_path)
45        self.result_dict = dict()
46
47    def collect_sa_profile(self):
48        grep_kw = r"ohos_sa_profile"
49        grep_cmd = f"grep -rn '{grep_kw}' --include=BUILD.gn {self.project_path}"
50        content = BasicTool.execute(
51            grep_cmd, post_processor=lambda x: x.split('\n'))
52        for item in content:
53            if not item:
54                continue
55            self._process_single_sa(item, start_pattern=grep_kw)
56
57    def _process_single_sa(self, item: str, start_pattern: str):
58        gn, _, _ = item.split(':')
59        with open(gn, 'r', encoding='utf-8') as f:
60            content = f.read()
61        p_itr: Iterator[re.Match] = BasicTool.match_paragraph(
62            content=content, start_pattern=start_pattern)
63        for p in p_itr:
64            p_content = p.group()
65            files: List[str] = GnVariableParser.list_parser(
66                "sources", p_content)
67            component_name, subsystem_name = GnCommonTool.find_part_subsystem(
68                gn, self.project_path)
69            for f in files:
70                f = f.split('/')[-1]
71                self.result_dict[f] = {
72                    "subsystem_name": subsystem_name,
73                    "component_name": component_name,
74                    "gn_path": gn
75                }
76
77
78class RomAnalyzer:
79    @classmethod
80    def analysis(cls, system_module_info_json: Text, product_dirs: List[str],
81                 project_path: Text, product_name: Text, output_file: Text, output_execel: bool, add_baseline: bool,
82                 unit_adapt: bool):
83        """
84        system_module_info_json: json文件
85        product_dirs：要处理的产物的路径列表如["vendor", "system/"]
86        project_path: 项目根路径
87        product_name: eg，rk3568
88        output_file: basename of output file
89        """
90        project_path = BasicTool.get_abs_path(project_path)
91        rom_baseline_dict: Dict[str, Any] = RomRamBaselineCollector.collect(
92            project_path)
93        with os.fdopen(os.open("rom_ram_baseline.json", os.O_WRONLY | os.O_CREAT, mode=0o640), 'w', encoding='utf-8') as f:
94            json.dump(rom_baseline_dict, f, indent=4)
95        phone_dir = os.path.join(
96            project_path, "out", product_name, "packages", "phone")
97        product_dirs = [os.path.join(phone_dir, d) for d in product_dirs]
98        pre_collector = PreCollector(project_path)
99        pre_collector.collect_sa_profile()
100        extra_product_info_dict: Dict[str, Dict] = pre_collector.result_dict
101        product_info_dict = cls.__collect_product_info(
102            system_module_info_json, project_path,
103            extra_info=extra_product_info_dict)  # collect product info from json file
104        result_dict: Dict[Text:Dict] = dict()
105        for d in product_dirs:
106            file_list: List[Text] = BasicTool.find_all_files(d)
107            for f in file_list:
108                size = os.path.getsize(f)
109                relative_filepath = f.replace(phone_dir, "").lstrip(os.sep)
110                unit: Dict[Text, Any] = product_info_dict.get(
111                    relative_filepath)
112                if not unit:
113                    bf = f.split('/')[-1]
114                    unit: Dict[Text, Any] = product_info_dict.get(bf)
115                if not unit:
116                    unit = dict()
117                unit["size"] = size
118                unit["relative_filepath"] = relative_filepath
119                cls.__put(unit, result_dict, rom_baseline_dict, add_baseline)
120        output_dir, _ = os.path.split(output_file)
121        if len(output_dir) != 0:
122            os.makedirs(output_dir, exist_ok=True)
123        if unit_adapt:
124            cls.result_unit_adaptive(result_dict)
125        with os.fdopen(os.open(output_file + ".json", os.O_WRONLY | os.O_CREAT, mode=0o640), 'w', encoding='utf-8') as f:
126            f.write(json.dumps(result_dict, indent=4))
127        if output_execel:
128            cls.__save_result_as_excel(result_dict, output_file, add_baseline)
129
130    @classmethod
131    def result_unit_adaptive(self, result_dict: Dict[str, Dict]) -> None:
132        for subsystem_name, subsystem_info in result_dict.items():
133            size = unit_adaptive(subsystem_info["size"])
134            count = subsystem_info["file_count"]
135            if "size" in subsystem_info.keys():
136                del subsystem_info["size"]
137            if "file_count" in subsystem_info.keys():
138                del subsystem_info["file_count"]
139            for component_name, component_info in subsystem_info.items():
140                component_info["size"] = unit_adaptive(component_info["size"])
141            subsystem_info["size"] = size
142            subsystem_info["file_count"] = count
143
144    @classmethod
145    def __collect_product_info(cls, system_module_info_json: Text,
146                               project_path: Text, extra_info: Dict[str, Dict]) -> Dict[Text, Dict[Text, Text]]:
147        """
148        根据system_module_info.json生成target字典
149        format:
150            {
151                "{file_name}":{
152                    "{subsytem_name}": abc,
153                    "{component_name}": def,
154                    "{gn_path}": ghi
155                }
156            }
157        if the unit of system_module_info.json has not field "label" and the "type" is "sa_profile",
158        find the subsystem_name and component_name in the BUILD.gn
159        """
160        with open(system_module_info_json, 'r', encoding='utf-8') as f:
161            product_list = json.loads(f.read())
162        project_path = BasicTool.get_abs_path(project_path)
163        product_info_dict: Dict[Text, Dict[Text, Text]] = dict()
164        for unit in product_list:
165            cs_flag = False
166            dest: List = unit.get("dest")
167            if not dest:
168                print("warning: keyword 'dest' not found in {}".format(
169                    system_module_info_json))
170                continue
171            label: Text = unit.get("label")
172            gn_path = component_name = subsystem_name = None
173            if label:
174                cs_flag = True
175                gn_path = os.path.join(project_path, label.split(':')[
176                    0].lstrip('/'), "BUILD.gn")
177                component_name = unit.get("part_name")
178                subsystem_name = unit.get("subsystem_name")
179                if not component_name:
180                    cn, sn = GnCommonTool.find_part_subsystem(
181                        gn_path, project_path)
182                    component_name = cn
183                if not subsystem_name:
184                    cn, sn = GnCommonTool.find_part_subsystem(
185                        gn_path, project_path)
186                    subsystem_name = sn
187            else:
188                print("warning: keyword 'label' not found in {}".format(unit))
189            for target in dest:
190                if cs_flag:
191                    product_info_dict[target] = {
192                        "component_name": component_name,
193                        "subsystem_name": subsystem_name,
194                        "gn_path": gn_path,
195                    }
196                    continue
197                tmp = target.split('/')[-1]
198                pre_info = extra_info.get(tmp)
199                if not pre_info:
200                    continue
201                else:
202                    product_info_dict[target] = pre_info
203        return product_info_dict
204
205    @classmethod
206    def __inside_save_result_as_excel(cls, add_baseline, subsystem_name, component_name,
207                                      baseline, file_name, size):
208        if add_baseline:
209            return [subsystem_name, component_name,
210                    baseline, file_name, size]
211        else:
212            return [subsystem_name, component_name, file_name, size]
213
214    @classmethod
215    def __save_result_as_excel(cls, result_dict: dict, output_name: str, add_baseline: bool):
216        header = ["subsystem_name", "component_name",
217                  "output_file", "size(Byte)"]
218        if add_baseline:
219            header = ["subsystem_name", "component_name", "baseline",
220                      "output_file", "size(Byte)"]
221        tmp_dict = deepcopy(result_dict)
222        excel_writer = SimpleExcelWriter("rom")
223        excel_writer.set_sheet_header(headers=header)
224        subsystem_start_row = 1
225        subsystem_end_row = 0
226        subsystem_col = 0
227        component_start_row = 1
228        component_end_row = 0
229        component_col = 1
230        if add_baseline:
231            baseline_col = 2
232        for subsystem_name in tmp_dict.keys():
233            subsystem_dict = tmp_dict.get(subsystem_name)
234            subsystem_size = subsystem_dict.get("size")
235            subsystem_file_count = subsystem_dict.get("file_count")
236            del subsystem_dict["file_count"]
237            del subsystem_dict["size"]
238            subsystem_end_row += subsystem_file_count
239
240            for component_name in subsystem_dict.keys():
241                component_dict: Dict[str, int] = subsystem_dict.get(
242                    component_name)
243                component_size = component_dict.get("size")
244                component_file_count = component_dict.get("file_count")
245                baseline = component_dict.get("baseline")
246                del component_dict["file_count"]
247                del component_dict["size"]
248                if add_baseline:
249                    del component_dict["baseline"]
250                component_end_row += component_file_count
251
252                for file_name, size in component_dict.items():
253                    line = cls.__inside_save_result_as_excel(add_baseline, subsystem_name, component_name,
254                                                             baseline, file_name, size)
255                    excel_writer.append_line(line)
256                excel_writer.write_merge(component_start_row, component_col, component_end_row, component_col,
257                                         component_name)
258                if add_baseline:
259                    excel_writer.write_merge(component_start_row, baseline_col, component_end_row, baseline_col,
260                                             baseline)
261                component_start_row = component_end_row + 1
262            excel_writer.write_merge(subsystem_start_row, subsystem_col, subsystem_end_row, subsystem_col,
263                                     subsystem_name)
264            subsystem_start_row = subsystem_end_row + 1
265        excel_writer.save(output_name + ".xls")
266
267    @classmethod
268    def __put(cls, unit: typing.Dict[Text, Any], result_dict: typing.Dict[Text, Dict], baseline_dict: Dict[str, Any],
269              baseline: bool):
270
271        component_name = NOTFOUND if unit.get(
272            "component_name") is None else unit.get("component_name")
273        subsystem_name = NOTFOUND if unit.get(
274            "subsystem_name") is None else unit.get("subsystem_name")
275
276        def get_rom_baseline():
277            if (not baseline_dict.get(subsystem_name)) or (not baseline_dict.get(subsystem_name).get(component_name)):
278                return str()
279            return baseline_dict.get(subsystem_name).get(component_name).get("rom")
280
281        size = unit.get("size")
282        relative_filepath = unit.get("relative_filepath")
283        if result_dict.get(subsystem_name) is None:  # 子系统
284            result_dict[subsystem_name] = dict()
285            result_dict[subsystem_name]["size"] = 0
286            result_dict[subsystem_name]["file_count"] = 0
287
288        if result_dict.get(subsystem_name).get(component_name) is None:  # 部件
289            result_dict[subsystem_name][component_name] = dict()
290            result_dict[subsystem_name][component_name]["size"] = 0
291            result_dict[subsystem_name][component_name]["file_count"] = 0
292            if baseline:
293                result_dict[subsystem_name][component_name]["baseline"] = get_rom_baseline(
294                )
295
296        result_dict[subsystem_name]["size"] += size
297        result_dict[subsystem_name]["file_count"] += 1
298        result_dict[subsystem_name][component_name]["size"] += size
299        result_dict[subsystem_name][component_name]["file_count"] += 1
300        result_dict[subsystem_name][component_name][relative_filepath] = size
301
302
303def get_args():
304    version_num = 2.0
305    parser = argparse.ArgumentParser(
306        description=f"analyze rom size of component.\n")
307    parser.add_argument("-v", "-version", action="version",
308                        version=f"version {version_num}")
309    parser.add_argument("-p", "--project_path", type=str, required=True,
310                        help="root path of openharmony. eg: -p ~/openharmony")
311    parser.add_argument("-j", "--module_info_json", required=True, type=str,
312                        help="path of out/{product_name}/packages/phone/system_module_info.json")
313    parser.add_argument("-n", "--product_name", required=True,
314                        type=str, help="product name. eg: -n rk3568")
315    parser.add_argument("-d", "--product_dir", required=True, action="append",
316                        help="subdirectories of out/{product_name}/packages/phone to be counted."
317                             "eg: -d system -d vendor")
318    parser.add_argument("-b", "--baseline", action="store_true",
319                        help="add baseline of component to the result(-b) or not.")
320    parser.add_argument("-o", "--output_file", type=str, default="rom_analysis_result",
321                        help="basename of output file, default: rom_analysis_result. eg: demo/rom_analysis_result")
322    parser.add_argument("-u", "--unit_adaptive",
323                        action="store_true", help="unit adaptive")
324    parser.add_argument("-e", "--excel", type=bool, default=False,
325                        help="if output result as excel, default: False. eg: -e True")
326    args = parser.parse_args()
327    return args
328
329
330if __name__ == '__main__':
331    args = get_args()
332    module_info_json = args.module_info_json
333    project_origin_path = args.project_path
334    product = args.product_name
335    product_dirs = args.product_dir
336    output_file_name = args.output_file
337    output_excel = args.excel
338    baseline_path = args.baseline
339    unit_adaptiv = args.unit_adaptive
340    RomAnalyzer.analysis(module_info_json, product_dirs,
341                         project_origin_path, product, output_file_name, output_excel, baseline_path, unit_adaptiv)
342