1#!/usr/bin/env python3
2# -*- coding: utf-8 -*-
3#
4# Copyright (c) 2024 Huawei Device Co., Ltd.
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9# http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16
17from abc import ABC, abstractmethod
18import argparse
19import glob
20import json
21import os
22import re
23import shutil
24import stat
25import subprocess
26import sys
27import threading
28
29SUCCESS_STATUS = 0
30CRASH_STATUS = -1
31TIMEOUT_STATUS = -2
32
33WORKLOAD_URL = 'https://gitee.com/xliu-huanwei/ark-workload.git'
34BENCHMARK_PATH = '../benchmark'
35SKIP_TEST = 'skip_tests.json'
36DISCREPANCY_REPORT = 'discrepancy_report'
37HTML_CONTENT = \
38"""
39<!DOCTYPE html>
40<html>
41<head>
42    <title>Instruction Discrepancy Report</title>
43    <style>
44        table {
45            width: 50%;
46            border-collapse: collapse;
47            margin: auto;
48        }
49        th, td {
50            padding: 8px;
51            text-align: center;
52            border: 1px solid black;
53            white-space: nowrap;
54        }
55        th:nth-child(2), td:nth-child(2) {
56            text-align: left;
57        }
58        h1 {
59            text-align: center;
60        }
61    </style>
62</head>
63<body>
64    <h1>Instruction Discrepancy Report</h1>
65    <table>
66        <tr>
67            <th>No</th>
68            <th>Case Path</th>
69            <th>es2abc Instruction Number</th>
70            <th>v8 Instruction Number</th>
71            <th>es2abc/v8 Ratio</th>
72        </tr>
73"""
74
75
76def is_file(parser, arg):
77    if not os.path.isfile(arg):
78        parser.error(f'[ERROR]: The file "{arg}" does not exist!')
79    return os.path.abspath(arg)
80
81
82def is_directory(parser, arg):
83    if not os.path.isdir(arg):
84        parser.error(f'[ERROR]: The directory "{arg}" does not exist!')
85    return os.path.abspath(arg)
86
87
88def check_timeout(val):
89    val = int(val)
90    if val <= 0:
91        raise argparse.ArgumentTypeError(f'[ERROR]: {val} is an invalid timeout value')
92    return val
93
94
95def get_args():
96    description = "Generate a report on the difference between the number of v8 and es2abc bytecode instructions."
97    parser = argparse.ArgumentParser(description=description)
98    parser.add_argument('-v8', '-d8', '--d8_path', dest='d8_path', type=lambda arg : is_file(parser, arg),
99                        help='Path to the V8 executable d8', required=True)
100    parser.add_argument('-es2abc', '--es2abc_path', dest='es2abc_path', type=lambda arg : is_file(parser, arg),
101                        help='Path to the executable program es2abc', required=True)
102    parser.add_argument('--timeout', dest='timeout', type=check_timeout, default=180,
103                        help='Time limits for use case execution (In seconds)')
104    parser.add_argument('--add_case', dest='case', type=lambda arg : is_file(parser, arg),
105                        help='Add the file path of a single test case to be executed', nargs='+')
106    parser.add_argument('--add_case_dir', dest='case_dir', type=lambda arg : is_directory(parser, arg),
107                        help='Add the directory where the test cases are to be executed', nargs='+')
108    return parser.parse_args()
109
110
111def assemble_command(command_list: list = None, executable_program_path: str = None,
112                     file_path: str = None, parameters: list = None):
113    if command_list is None:
114        command_list = [executable_program_path, file_path]
115        if parameters:
116            command_list.extend(parameters)
117    else:
118        if executable_program_path:
119            command_list[0] = executable_program_path
120        if file_path:
121            command_list[1] = file_path
122    return command_list
123
124
125class CaseManager:
126    def __init__(self, args, skip_test_flag=True):
127        self.test_root = os.path.dirname(os.path.abspath(__file__))
128        self.args = args
129        self.case_list = []
130        self.skip_tests_info = os.path.join(self.test_root, SKIP_TEST)
131        self.report_path = os.path.join(self.test_root, DISCREPANCY_REPORT)
132
133        self.get_test_case()
134
135        if skip_test_flag and os.path.exists(self.skip_tests_info):
136            self.skip_cases()
137
138        self.case_list.sort()
139
140        # Creating dictionary: {file_path : [d8 status, es2abc status]}
141        self.crash_dict = {file_path : [SUCCESS_STATUS, SUCCESS_STATUS] for file_path in self.case_list}
142
143    def get_test_case(self):
144        def add_directory_to_case_list(case_dir, case_list, extension='js', recursive=True):
145            if not os.path.isdir(case_dir):
146                print(f'[ERROR]: add_directory_to_case_list failed! {case_dir} does not exist!')
147                return False
148            glob_expression = os.path.join(case_dir, f'**/*.{extension}')
149            files_list = glob.glob(glob_expression, recursive=recursive)
150            for file in files_list:
151                abs_file_path = os.path.abspath(file)
152                if abs_file_path not in case_list:   # make sure no duplicate case
153                    case_list.append(abs_file_path)
154            return True
155
156        if self.args.case is not None:
157            for case in self.args.case:
158                abs_file_path = os.path.abspath(case)
159                if abs_file_path not in self.case_list:
160                    self.case_list.append(abs_file_path)
161
162        if self.args.case_dir is not None:
163            for case in self.args.case_dir:
164                add_directory_to_case_list(case, self.case_list)
165
166        cur_dir = os.getcwd()
167        os.chdir(self.test_root)
168
169        # add workload cases
170        case_dir_path = self.pull_cases_from_repo(WORKLOAD_URL, 'test_cases/ark-workload')
171        if case_dir_path:
172            print('[INFO]: pull workload cases Success!')
173            case_dir_path = os.path.join(case_dir_path, 'weekly_workload', 'js')
174            add_directory_to_case_list(case_dir_path, self.case_list)
175
176        # add benchmark cases
177        sys.path.insert(0, BENCHMARK_PATH)
178        from utils import DEFAULT_TESTCASES_DIR, pull_cases, clear_folder_shutil
179        benchmark_case_path = os.path.join(BENCHMARK_PATH, DEFAULT_TESTCASES_DIR)
180        clear_folder_shutil(benchmark_case_path)
181        pull_benchmark_cases_success = pull_cases()
182        if pull_benchmark_cases_success:
183            print('[INFO]: pull benchmark cases Success!')
184            add_directory_to_case_list(benchmark_case_path, self.case_list)
185
186        os.chdir(cur_dir)
187
188    def git_clone(self, git_url, code_dir, pull=False):
189        cur_dir = os.getcwd()
190        cmd = ['git', 'clone', git_url, code_dir]
191        if pull:
192            os.chdir(code_dir)
193            cmd = ['git', 'pull']
194        process = subprocess.Popen(cmd)
195        process.wait()
196        os.chdir(cur_dir)
197        result = True
198        if process.returncode:
199            print(f"\n[ERROR]: git clone or pull '{git_url}' Failed!")
200            result = False
201        return result
202
203    def pull_cases_from_repo(self, case_url, case_dir):
204        dir_path = os.path.join(self.test_root, case_dir)
205        pull = False
206        if os.path.exists(dir_path):
207            pull = True
208        clone_result = self.git_clone(case_url, dir_path, pull)
209        if not clone_result:
210            return None
211        return dir_path
212
213    def skip_cases(self):
214        with open(self.skip_tests_info, 'r') as f:
215            data = json.load(f)
216
217        skip_case_list = []
218        for reason_files_dict in data:
219            skip_case_list.extend([os.path.abspath(os.path.join(self.test_root, case))
220                                   for case in reason_files_dict['files']])
221
222        self.case_list = [case for case in self.case_list if case not in skip_case_list]
223
224    def calculate_ratio_for_discrepancy_report(self, d8_instruction_number, es2abc_instruction_number):
225        if d8_instruction_number <= 0 or es2abc_instruction_number <= 0:
226            return 'Invalid Ratio'
227        ratio = (float(es2abc_instruction_number) / d8_instruction_number) * 100
228        return f'{ratio:.2f}%'
229
230    def generate_discrepancy_report(self, d8_output, es2abc_output):
231        global HTML_CONTENT
232        case_number = 1
233        for d8_item, es2abc_item in zip(d8_output, es2abc_output):
234            d8_instruction_number = d8_item[1]
235            es2abc_instruction_number = es2abc_item[1]
236
237            if self.crash_dict[d8_item[0]][0] == CRASH_STATUS:
238                d8_instruction_number = \
239                    "<span style='color:Red; font-weight:bold;'>{}</span>".format(d8_item[1])
240            elif self.crash_dict[d8_item[0]][0] == TIMEOUT_STATUS:
241                d8_instruction_number = \
242                    "<span style='color:DarkRed; font-weight:bold;'>{}</span>".format(d8_item[1])
243
244            if self.crash_dict[es2abc_item[0]][1] == CRASH_STATUS:
245                es2abc_instruction_number = \
246                    "<span style='color:Red; font-weight:bold;'>{}</span>".format(es2abc_item[1])
247            elif self.crash_dict[es2abc_item[0]][1] == TIMEOUT_STATUS:
248                es2abc_instruction_number = \
249                    "<span style='color:DarkRed; font-weight:bold;'>{}</span>".format(es2abc_item[1])
250            case_path = os.path.relpath(d8_item[0], self.test_root)
251            html_content_of_case_info = f"""<tr>
252                                    <td>{case_number}</td>
253                                    <td>{case_path}</td>
254                                    <td>{es2abc_instruction_number}</td>
255                                    <td>{d8_instruction_number}</td>
256                                    <td>{self.calculate_ratio_for_discrepancy_report(int(d8_item[1]),
257                                         int(es2abc_item[1]))}</td>
258                                </tr>"""
259            HTML_CONTENT = "{}{}".format(HTML_CONTENT, html_content_of_case_info)
260            case_number += 1
261
262        html_content_of_end_tag = "</table></body></html>"
263        HTML_CONTENT = "{}{}".format(HTML_CONTENT, html_content_of_end_tag)
264
265        flags = os.O_RDWR | os.O_CREAT
266        mode = stat.S_IWUSR | stat.S_IRUSR
267        with os.fdopen(os.open(self.report_path + '.html', flags, mode), 'w') as f:
268            f.truncate()
269            f.write(HTML_CONTENT)
270
271
272class Runner(ABC):
273    def __init__(self, command_list:list, case_manager:CaseManager):
274        self.output = []
275        self.command_list = command_list
276        self.case_manager = case_manager
277
278    @abstractmethod
279    def run(self):
280        pass
281
282
283class D8Runner(Runner):
284    def __init__(self, command_list, case_manager):
285        super().__init__(command_list, case_manager)
286
287    def run(self):
288        for file_path in self.case_manager.case_list:
289            d8_case_path = file_path.replace('.js', '.mjs')
290            shutil.copyfile(file_path, d8_case_path)
291            instruction_number = RecognizeInstructionMethod.recognize_d8_bytecode_instruction(
292                self.command_list, d8_case_path, self.case_manager)
293            if os.path.exists(d8_case_path):
294                os.remove(d8_case_path)
295            self.output.append((file_path, instruction_number))
296
297
298class ES2ABCRunner(Runner):
299    def __init__(self, command_list, case_manager):
300        super().__init__(command_list, case_manager)
301
302    def run(self):
303        for file_path in self.case_manager.case_list:
304            instruction_number = RecognizeInstructionMethod.recognize_es2abc_bytecode_instruction(
305                self.command_list, file_path, self.case_manager)
306            self.output.append((file_path, instruction_number))
307
308
309class RecognizeInstructionMethod:
310    @staticmethod
311    def recognize_d8_bytecode_instruction(command_list, file_path, case_manager):
312        original_file_path = file_path.replace('.mjs', '.js')
313        command_list = assemble_command(command_list, file_path=file_path)
314        process = subprocess.Popen(command_list, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
315        try:
316            output, _ = process.communicate(timeout=case_manager.args.timeout)
317        except subprocess.TimeoutExpired:
318            print(f"[WARNING]: v8 - Killed! Timeout: {file_path}")
319            process.kill()
320            case_manager.crash_dict[original_file_path][0] = TIMEOUT_STATUS
321            return -1   # Script execution timeout sets the instruction number to -1
322        process.wait()
323
324        if process.returncode != 0:
325            case_manager.crash_dict[original_file_path][0] = CRASH_STATUS
326            print(f'[WARNING]: v8 - Crashed! {file_path}')
327
328        decoded_output = output.decode('utf-8', errors='ignore')
329
330        instruction_pattern = r'0x[0-9a-fA-F]+ @\s+\d+\s+:\s+([0-9a-fA-F\s]+)\s+(.*)$'
331        result = re.finditer(instruction_pattern, decoded_output, re.MULTILINE)
332        instruction_number = sum(1 for _ in result)
333        return instruction_number
334
335    @staticmethod
336    def recognize_es2abc_bytecode_instruction(command_list, file_path, case_manager):
337        command_list = assemble_command(command_list, file_path=file_path)
338        process = subprocess.Popen(command_list, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
339        try:
340            output, _ = process.communicate(timeout=case_manager.args.timeout)
341        except subprocess.TimeoutExpired:
342            print(f"[WARNING]: es2abc - Killed! Timeout: {file_path}")
343            process.kill()
344            case_manager.crash_dict[file_path][1] = TIMEOUT_STATUS
345            return -1    # Script execution timeout sets the instructions number to -1
346        process.wait()
347
348        if process.returncode != 0:
349            case_manager.crash_dict[file_path][1] = CRASH_STATUS
350            print(f'[WARNING]: es2abc - Crashed! {file_path}')
351
352        decoded_output = output.decode('utf-8', errors='ignore')
353        lines = decoded_output.split('\n')
354        instruction_number = 0
355        for line in lines:
356            if 'instructions_number:' in line:
357                instruction_number = line.split('instructions_number:')[-1].strip()
358                break
359        return instruction_number
360
361
362def main():
363    args = get_args()
364
365    d8_command = assemble_command(executable_program_path=args.d8_path, parameters=['--print-bytecode'])
366    es2abc_command = assemble_command(executable_program_path=args.es2abc_path,
367                                      parameters=['--module', '--dump-size-stat', '--output=/dev/null'])
368
369    case_manager = CaseManager(args)
370    d8_runner = D8Runner(d8_command, case_manager)
371    es2abc_runner = ES2ABCRunner(es2abc_command, case_manager)
372
373    if len(case_manager.case_list):
374        thread_d8 = threading.Thread(target=d8_runner.run)
375        thread_es2abc = threading.Thread(target=es2abc_runner.run)
376        thread_d8.start()
377        thread_es2abc.start()
378        thread_d8.join()
379        thread_es2abc.join()
380
381    case_manager.generate_discrepancy_report(d8_runner.output, es2abc_runner.output)
382
383
384if __name__ == "__main__":
385    main()