1#!/usr/bin/env python 2# -*- coding: utf-8 -*- 3""" 4Copyright (c) 2024 Huawei Device Co., Ltd. 5Licensed under the Apache License, Version 2.0 (the "License"); 6you may not use this file except in compliance with the License. 7You may obtain a copy of the License at 8 9 http://www.apache.org/licenses/LICENSE-2.0 10 11Unless required by applicable law or agreed to in writing, software 12distributed under the License is distributed on an "AS IS" BASIS, 13WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14See the License for the specific language governing permissions and 15limitations under the License. 16 17Description: Use ark to execute workload test suite 18""" 19 20import argparse 21import datetime 22import json 23import logging 24import os 25import shutil 26import stat 27import subprocess 28import sys 29from typing import Union, List, Tuple 30from collections import namedtuple 31from openpyxl import Workbook, load_workbook 32from openpyxl.styles import PatternFill 33 34 35def get_logger(logger_name, log_file_path, level=logging.INFO): 36 """Create logger for this script""" 37 formatter = logging.Formatter(fmt='[%(asctime)s] [%(levelname)s] %(message)s', 38 datefmt='%Y-%m-%d %H:%M:%S') 39 40 fh = logging.FileHandler(encoding='utf-8', mode='a', filename=log_file_path) 41 fh.setFormatter(formatter) 42 fh.setLevel(logging.DEBUG) 43 # console output 44 ch = logging.StreamHandler() 45 ch.setFormatter(formatter) 46 ch.setLevel(logging.DEBUG) 47 log = logging.getLogger(logger_name) 48 log.addHandler(fh) 49 log.addHandler(ch) 50 log.setLevel(level) 51 52 return log 53 54 55class Constants: 56 logger = None 57 CUR_PATH = os.path.abspath(os.path.dirname(__file__)) 58 TMP_PATH = os.path.join(os.getcwd(), "tmp") 59 REPORT_NAME_HEAD_FIX = "js_perf_test_result" 60 RET_OK = 0 61 RET_FAILED = 1 62 BINARY_PATH = "" 63 OUTPUT_PATH = "" 64 LOG_PATH = "" 65 TODAY_EXCEL_PATH = "" 66 YESTERDAY_EXCEL_PATH = "" 67 DETERIORATION_BOUNDARY_VALUE = 0.05 68 TODAY_EXCUTE_INFO = {} 69 YESTERDAY_EXCUTE_TIME_DICT = {} 70 V_8_EXCUTE_TIME_DICT = {} 71 V_8_JITLESS_EXCUTE_TIME_DICT = {} 72 JS_FILE_SUPER_LINK_DICT = {} 73 HYPERLINK_HEAD = "https://gitee.com/dov1s/arkjs-perf-test/tree/builtins_test1110/js-perf-test" 74 PASS = 'pass' 75 FAIL = 'fail' 76 SOLID = 'solid' 77 NA_FIX = 'NA' 78 # 1e-6 s 79 COMPARISON_ACCURACY = 0.001 80 ICU_DATA_PATH = "" 81 FIX_STR = "8/d" 82 V_8_ENGINED_PATH = '/usr/bin/v{}8'.format(FIX_STR) 83 VER_PLATFORM = "full_x86_64" 84 ES2ABC_PATH = "" 85 ARK_JS_VM_PATH = "" 86 ETS_RUNTIME = "" 87 LD_LIBRARY_PATH = "" 88 HDC_PATH: str = "hdc" 89 DEVICE_WORKDIR: str = "/data/local/tmp/jsperftest" 90 LIBS_LIST: List[str] = [] 91 STUB_AN: str = "" 92 TASKSET_MASK: str = "" 93 CaseTestDataType = namedtuple('test', ['exec_status', 'exec_time']) 94 95 96def get_js_file_class_api_scenes(js_file_path): 97 """Get all cases for one benchmark file""" 98 scenes = [] 99 with open(js_file_path, 'r') as f: 100 for line in f: 101 if "scene_output" in line: 102 str_array = line.split(':') 103 mid_str = str_array[1].strip() 104 elements = mid_str.split(' ') 105 main_key = '/'.join([elements[0], elements[1] + '.js', elements[2]]).lower() 106 scenes.append(main_key) 107 return scenes 108 109 110def degraded_str(yesterday_excute_time, exec_time): 111 is_degraded_str = Constants.NA_FIX 112 if len(str(yesterday_excute_time).strip()) != 0: 113 if abs(float(yesterday_excute_time)) <= Constants.COMPARISON_ACCURACY: 114 is_degraded_str = str(True) if abs(float(exec_time)) >= DETERIORATION_BOUNDARY_VALUE else str(False) 115 else: 116 is_degraded_tmp = float(exec_time) / float(yesterday_excute_time) >= (1 + DETERIORATION_BOUNDARY_VALUE) 117 is_degraded_str = str(True) if is_degraded_tmp else str(False) 118 119 return is_degraded_str 120 121 122def v_8_excute_time_compute(main_key): 123 v_8_excute_time_str = '' 124 if len(Constants.V_8_EXCUTE_TIME_DICT) > 0 and main_key in Constants.V_8_EXCUTE_TIME_DICT.keys(): 125 v_8_excute_time_str = Constants.V_8_EXCUTE_TIME_DICT[main_key].strip() 126 127 if len(v_8_excute_time_str) == 0: 128 v_8_excute_time = ' ' 129 else: 130 v_8_excute_time = v_8_excute_time_str 131 132 return v_8_excute_time 133 134 135def v_8_gitless_excute_time_compute(main_key): 136 v_8_jitless_excute_time_str = '' 137 if len(Constants.V_8_JITLESS_EXCUTE_TIME_DICT) > 0 and main_key in Constants.V_8_JITLESS_EXCUTE_TIME_DICT.keys(): 138 v_8_jitless_excute_time_str = Constants.V_8_JITLESS_EXCUTE_TIME_DICT[main_key].strip() 139 140 if len(v_8_jitless_excute_time_str) == 0: 141 v_8_jitless_excute_time = ' ' 142 else: 143 v_8_jitless_excute_time = v_8_jitless_excute_time_str 144 145 return v_8_jitless_excute_time 146 147 148def ark_divide_v_8_compute(exec_time, v_8_excute_time): 149 if len(exec_time) == 0 or len(v_8_excute_time.strip()) == 0: 150 ark_divide_v_8 = Constants.NA_FIX 151 elif abs(float(exec_time)) <= Constants.COMPARISON_ACCURACY: 152 if abs(float(v_8_excute_time)) <= Constants.COMPARISON_ACCURACY: 153 ark_divide_v_8 = '1' 154 else: 155 ark_divide_v_8 = '0' 156 else: 157 v_8_excute_time = v_8_excute_time.strip() 158 if len(v_8_excute_time) == 0 or abs(float(v_8_excute_time)) <= Constants.COMPARISON_ACCURACY: 159 ark_divide_v_8 = Constants.NA_FIX 160 else: 161 ark_divide_v_8 = str("{:.2f}".format(float(exec_time) / float(v_8_excute_time))) 162 163 return ark_divide_v_8 164 165 166def cast_to_float_or_str(value: str) -> Union[float, str]: 167 """Return float value by str if it is possible, return input str otherwise""" 168 try: 169 result = float(value) 170 except ValueError: 171 result = value 172 return result 173 174 175def append_row_data(report_file, case_test_data): 176 wb = load_workbook(report_file) 177 ws = wb.worksheets[0] 178 for main_key in case_test_data.keys(): 179 str_arr = main_key.split('/') 180 class_name = str_arr[0] 181 api_name = str_arr[1] 182 scene = str_arr[2] 183 js_case_name = '/'.join([class_name, api_name]) 184 excute_status = case_test_data[main_key].exec_status 185 exec_time = case_test_data[main_key].exec_time.strip() 186 yesterday_excute_time = '' 187 if (len(Constants.YESTERDAY_EXCUTE_TIME_DICT) > 0 and 188 Constants.YESTERDAY_EXCUTE_TIME_DICT.get(main_key) is not None): 189 yesterday_excute_time = str(Constants.YESTERDAY_EXCUTE_TIME_DICT[main_key]) 190 is_degraded_str = degraded_str(yesterday_excute_time, exec_time) 191 v_8_excute_time = v_8_excute_time_compute(main_key) 192 v_8_jitless_excute_time = v_8_gitless_excute_time_compute(main_key) 193 ark_divide_v_8 = ark_divide_v_8_compute(exec_time, v_8_excute_time) 194 if len(exec_time) == 0 or len(v_8_jitless_excute_time.strip()) == 0: 195 ark_divide_v_8_with_jitless = Constants.NA_FIX 196 elif abs(float(exec_time)) <= Constants.COMPARISON_ACCURACY: 197 if abs(float(v_8_jitless_excute_time)) <= Constants.COMPARISON_ACCURACY: 198 ark_divide_v_8_with_jitless = '1' 199 else: 200 ark_divide_v_8_with_jitless = '0' 201 else: 202 v_8_jitless_excute_time = v_8_jitless_excute_time.strip() 203 if (len(v_8_jitless_excute_time) == 0 or 204 abs(float(v_8_jitless_excute_time)) <= Constants.COMPARISON_ACCURACY): 205 ark_divide_v_8_with_jitless = Constants.NA_FIX 206 else: 207 ark_divide_v_8_with_jitless = str("{:.2f}".format(float(exec_time) / float(v_8_jitless_excute_time))) 208 jis_case_file_name_with_class = Constants.JS_FILE_SUPER_LINK_DICT['/'.join([class_name, api_name])] 209 js_file_super_link = '/'.join([Constants.HYPERLINK_HEAD, jis_case_file_name_with_class]) 210 new_row = [js_case_name, scene, excute_status, cast_to_float_or_str(exec_time), yesterday_excute_time, 211 is_degraded_str, cast_to_float_or_str(v_8_excute_time), 212 cast_to_float_or_str(v_8_jitless_excute_time), cast_to_float_or_str(ark_divide_v_8), 213 cast_to_float_or_str(ark_divide_v_8_with_jitless), js_file_super_link, ' '] 214 ws.append(new_row) 215 check(is_degraded_str, ark_divide_v_8, ark_divide_v_8_with_jitless, ws) 216 wb.save(report_file) 217 return Constants.RET_OK 218 219 220def check(is_degraded_str, ark_divide_v_8, ark_divide_v_8_with_jitless, ws): 221 if is_degraded_str is str(True): 222 ws.cell(row=ws.max_row, column=6).fill = PatternFill(start_color='FF0000', end_color='FF0000', 223 fill_type=Constants.SOLID) 224 if (ark_divide_v_8 != Constants.NA_FIX and 225 (float(ark_divide_v_8) > 2 or abs(float(ark_divide_v_8) - 2) <= Constants.COMPARISON_ACCURACY)): 226 ws.cell(row=ws.max_row, column=9).fill = PatternFill(start_color='FFFF00', end_color='FFFF00', 227 fill_type=Constants.SOLID) 228 if (ark_divide_v_8_with_jitless != Constants.NA_FIX and 229 (float(ark_divide_v_8_with_jitless) > 2 or 230 abs(float(ark_divide_v_8_with_jitless) - 2) <= Constants.COMPARISON_ACCURACY)): 231 ws.cell(row=ws.max_row, column=10).fill = PatternFill(start_color='FF00FF', end_color='FF00FF', 232 fill_type=Constants.SOLID) 233 234 235def get_ark_js_cmd(abc_file: str) -> List[str]: 236 """Get command for ark js vm""" 237 cmd: List[str] = [] 238 if Constants.VER_PLATFORM.find("arm64") != -1: 239 cmd = [Constants.HDC_PATH, "shell"] 240 run_cmd = f"LD_LIBRARY_PATH={Constants.LD_LIBRARY_PATH}" 241 if len(Constants.TASKSET_MASK) != 0: 242 run_cmd += f" taskset -a {Constants.TASKSET_MASK}" 243 run_cmd += " " + os.path.join(Constants.DEVICE_WORKDIR, "ark_js_vm") 244 run_cmd += " --stub-file " + os.path.join(Constants.DEVICE_WORKDIR, "lib", "stub.an") 245 run_cmd += " --icu-data-path " + os.path.join(Constants.DEVICE_WORKDIR, "data") 246 run_cmd += " " + abc_file 247 cmd.append(run_cmd) 248 else: 249 cmd = [Constants.ARK_JS_VM_PATH, 250 "--log-level=info", 251 "--enable-runtime-stat=true", 252 "--stub-file", Constants.STUB_AN, 253 "--icu-data-path", ICU_DATA_PATH, 254 abc_file] 255 if len(Constants.TASKSET_MASK) != 0: 256 cmd = ["taskset", "-a", Constants.TASKSET_MASK] + cmd 257 return cmd 258 259 260def prepare_for_ark_run(class_name: str, api_name: str) -> Tuple[str, str]: 261 """Prepare workspace for benchmark""" 262 fangzhou_test_path = os.path.join(Constants.TMP_PATH, "fangzhou_test") # for abc file 263 if os.path.exists(fangzhou_test_path): 264 shutil.rmtree(fangzhou_test_path) 265 os.makedirs(fangzhou_test_path) 266 class_folder_path = os.path.join(fangzhou_test_path, class_name) 267 if not os.path.exists(class_folder_path): 268 os.makedirs(class_folder_path) 269 return (os.path.join(Constants.CUR_PATH, api_name + ".abc"), 270 os.path.join(class_folder_path, api_name + ".log")) 271 272 273def run_es2panda(abc_file: str, js_file: str) -> int: 274 """Run es2panda for one benchmark file""" 275 cmd = [Constants.ES2ABC_PATH, "--output", abc_file, js_file] 276 logger.info("run cmd: %s", cmd) 277 ret = subprocess.run(cmd, check=False) 278 if ret.returncode != 0: 279 logger.error("ret = %s, %s generate abc file failed. cmd: %s", str(ret), js_file, cmd) 280 return ret.returncode 281 282 283def run_js_case_via_ark(js_file_path, class_name, api_name, iterations, report_file): 284 """Run js perf benchmark via ark js vm""" 285 composite_scenes = get_js_file_class_api_scenes(js_file_path) 286 case_test_data = {} 287 execute_status = Constants.FAIL 288 execute_time = ' ' 289 290 for _, composite_scene in enumerate(composite_scenes): 291 case_test_data[composite_scene] = Constants.CaseTestDataType(execute_status, execute_time) 292 293 js_file_name = class_name + '/' + api_name + '.js' 294 cur_abc_file, api_log_path = prepare_for_ark_run(class_name, api_name) 295 using_abc_file = cur_abc_file 296 297 ret = run_es2panda(cur_abc_file, js_file_path) 298 if ret != 0: 299 append_row_data(report_file, case_test_data) 300 return case_test_data 301 if Constants.VER_PLATFORM.find("arm64") != -1: 302 using_abc_file = os.path.join(Constants.DEVICE_WORKDIR, os.path.basename(cur_abc_file)) 303 if hdc_send(cur_abc_file, using_abc_file) != 0: 304 append_row_data(report_file, case_test_data) 305 return case_test_data 306 # execute abc 307 cmd = get_ark_js_cmd(using_abc_file) 308 309 logger.info("run cmd: %s", cmd) 310 flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL 311 modes = stat.S_IWUSR | stat.S_IRUSR 312 313 data = {} 314 for _ in range(iterations): 315 if os.path.exists(api_log_path): 316 os.remove(api_log_path) 317 with os.fdopen(os.open(api_log_path, flags, modes), 'wb') as outfile: 318 ret = subprocess.run(cmd, stdout=outfile) 319 320 if ret.returncode != 0: 321 logger.error("%s execute abc file failed. cmd: %s", js_file_name, cmd) 322 append_row_data(report_file, case_test_data) 323 return case_test_data 324 if os.path.exists(api_log_path): 325 data = update_data_by_log(data, api_log_path, js_file_name[:-3]) 326 327 case_test_data.clear() 328 execute_status = Constants.PASS 329 for k, time_value in data.items(): 330 case_test_data[k] = Constants.CaseTestDataType(execute_status, str(time_value / iterations)) 331 append_row_data(report_file, case_test_data) 332 logger.info("%s execute abc file successfully. cmd: %s case_test_data: %s", 333 js_file_name, cmd, case_test_data) 334 os.remove(cur_abc_file) 335 return case_test_data 336 337 338def run_via_ark(jspath, report_file, iterations): 339 if not os.path.exists(jspath): 340 logger.error("js perf cases path is not exist. jspath: %s", jspath) 341 logger.info("begin to run js perf test via ark. js perf cases path: %s", jspath) 342 for root, _, files in os.walk(jspath): 343 if "TestCaseError" in root: 344 continue 345 for file in files: 346 if not file.endswith('.js'): 347 continue 348 349 file_path = os.path.join(root, file) 350 results = file_path.split("/") 351 class_name = results[-2] 352 api_name = results[-1].split(".")[0] 353 js_case_name = '/'.join([class_name, results[-1]]) 354 logger.info("begin to execute %s.", js_case_name) 355 test_data = run_js_case_via_ark(file_path, class_name, api_name, iterations, report_file) 356 for _, key in enumerate(test_data.keys()): 357 Constants.TODAY_EXCUTE_INFO[key] = test_data.get(key) 358 logger.info("finish executing %s. executing info: %s.", js_case_name, Constants.TODAY_EXCUTE_INFO) 359 360 361def get_js_case_super_link_data(jspath): 362 logger.info("get js case super link data") 363 for root, _, files in os.walk(jspath): 364 for file in files: 365 if not file.endswith('.js'): 366 continue 367 368 file_path = os.path.join(root, file) 369 results = file_path.split("/") 370 class_name = results[-2] 371 js_case_name = '/'.join([class_name, results[-1]]) 372 key = js_case_name.lower() 373 Constants.JS_FILE_SUPER_LINK_DICT[key] = js_case_name 374 375 376def export_sumary_info_for_notifying_email(json_path, total_cases_num, ark_divide_v_8_num, ark_divide_v_8_jitless_num): 377 data = {} 378 data['kind'] = 'V 8 js-perf-test' 379 data['Total'] = total_cases_num 380 data['Ark劣化v 8'] = ark_divide_v_8_num 381 data['Ark劣化v 8 jitless'] = ark_divide_v_8_jitless_num 382 flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL 383 modes = stat.S_IWUSR | stat.S_IRUSR 384 if os.path.exists(json_path): 385 os.remove(json_path) 386 with os.fdopen(os.open(json_path, flags, modes), 'w', encoding='utf-8') as f: 387 json.dump(data, f, indent=4, ensure_ascii=False) 388 logger.info("export summary info to json file successfully.") 389 390 391def get_umary_info_json_file_path(daily_report_file_path): 392 dir_path = os.path.dirname(daily_report_file_path) 393 json_file_name = 'jsperftest_notifying_info_in_email.json' 394 json_file_path = os.path.join(dir_path, json_file_name) 395 return json_file_path 396 397 398def append_summary_info(report_file, total_cost_time): 399 """ 400 summary info: 401 pass count: 402 fail count: 403 totle count: 404 degraded count: 405 total excute time is(s) : 406 degraded percentage upper limit: 407 ark/v 8 degraded count: 408 ark/v 8 jitless degraded count: 409 """ 410 wb = load_workbook(report_file) 411 ws = wb.worksheets[0] 412 413 totle_num = 0 414 degraded_upper_limit = DETERIORATION_BOUNDARY_VALUE 415 pass_num = 0 416 failed_num = 0 417 degraded_num = 0 418 ark_divide_v_8_degraded_count = 0 419 ark_divide_v_8_jitless_degraded_count = 0 420 421 last_bench_line = ws.max_row 422 for row_num in range(2, last_bench_line + 1): 423 excu_status = str(ws.cell(row=row_num, column=3).value) 424 is_degraded = str(ws.cell(row=row_num, column=6).value) 425 if is_degraded == str(True): 426 degraded_num += 1 427 428 if excu_status == Constants.PASS: 429 pass_num += 1 430 totle_num += 1 431 elif excu_status == Constants.FAIL: 432 failed_num += 1 433 totle_num += 1 434 435 obj = ws.cell(row=row_num, column=9).value 436 if obj is None: 437 obj = 0 438 ark_divide_v_8 = obj 439 if ark_divide_v_8 != Constants.NA_FIX and float(ark_divide_v_8) > 1: 440 ark_divide_v_8_degraded_count += 1 441 obj = ws.cell(row=row_num, column=10).value 442 if obj is None: 443 obj = 0 444 ark_divide_v_8_jitless = obj 445 if ark_divide_v_8_jitless != Constants.NA_FIX and float(ark_divide_v_8_jitless) > 1: 446 ark_divide_v_8_jitless_degraded_count += 1 447 448 avg_funcs = ['AVERAGE', 'GEOMEAN', 'MEDIAN'] 449 for avg_func in avg_funcs: 450 new_row = [' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', 451 f'=\"{avg_func}: \"&{avg_func}(I2:I{last_bench_line})', 452 f'=\"{avg_func}: \"&{avg_func}(J2:J{last_bench_line})', ' ', ' '] 453 ws.append(new_row) 454 new_row = ['劣化判定比率上限', degraded_upper_limit, ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' '] 455 ws.append(new_row) 456 new_row = ['js 用例总数', totle_num, ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' '] 457 ws.append(new_row) 458 new_row = ['Pass 数量', pass_num, ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' '] 459 ws.append(new_row) 460 new_row = ['Fail 数量', failed_num, ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' '] 461 ws.append(new_row) 462 new_row = ['ark今日劣化数量', degraded_num, ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' '] 463 ws.append(new_row) 464 new_row = ['Total excute time(时:分:秒.微妙)', total_cost_time, ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' '] 465 ws.append(new_row) 466 new_row = ['ark/v 8 劣化数量', ark_divide_v_8_degraded_count, ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' '] 467 ws.append(new_row) 468 new_row = ['ark/v 8 jitless 劣化数量', ark_divide_v_8_jitless_degraded_count, ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', 469 ' ', ' '] 470 ws.append(new_row) 471 472 ws.column_dimensions.group('E', hidden=True) 473 wb.save(report_file) 474 475 json_file_path = get_umary_info_json_file_path(report_file) 476 export_sumary_info_for_notifying_email(json_file_path, totle_num, ark_divide_v_8_degraded_count, 477 ark_divide_v_8_jitless_degraded_count) 478 return Constants.RET_OK 479 480 481def process_args(args: argparse.Namespace) -> argparse.Namespace: 482 """Process and check argument values""" 483 if not os.path.exists(args.binarypath): 484 logger.error("parameter --binarypath is not exist. Please check it! binary path: %s", args.binarypath) 485 raise RuntimeError("error bad parameters --binarypath") 486 if args.output_folder_path is None: 487 args.output_folder_path = os.getcwd() 488 if not os.path.isabs(args.output_folder_path): 489 args.output_folder_path = os.path.abspath(args.output_folder_path) 490 if args.ver_platform.find("arm64") == -1 and not os.path.exists(args.d_8_binary_path): 491 logger.error("parameter --d_8_binary_path is not exist. Please check it! d 8 binary path: %s", 492 args.d_8_binary_path) 493 raise RuntimeError("error bad parameters --d_8_binary_path: {}".format(args.d_8_binary_path)) 494 if args.iterations <= 0: 495 logger.error("parameter --iterations <= 0. Please check it! iterations: %s", 496 args.iterations) 497 raise RuntimeError(f"error bad parameters --iterations: {args.iterations}") 498 return args 499 500 501def get_args(): 502 parser = argparse.ArgumentParser() 503 parser.add_argument( 504 "--binarypath", "-bp", required=True, 505 help="path of binary folder. refer to harmony root folder path", 506 ) 507 parser.add_argument( 508 "--jspath", "-p", required=True, 509 help="path of js scripts, support folder and file", 510 ) 511 parser.add_argument( 512 "--deterioration_boundary_value", "-d", default=0.05, 513 help="deterioration boundary value, default 0.05", 514 ) 515 parser.add_argument( 516 "--output_folder_path", "-o", default=None, 517 help="output folder for executing js cases, default current folder", 518 ) 519 parser.add_argument( 520 "--d_8_binary_path", "-v", default=None, 521 help="v 8 engine d 8 binary path", 522 ) 523 parser.add_argument( 524 "--ver_platform", "-e", default="full_x86_64", 525 help="Code repository version and platform", 526 ) 527 parser.add_argument( 528 "--iterations", "-n", default=1, type=int, 529 help="Number of benchmark launches" 530 ) 531 parser.add_argument( 532 "--hdc", default="hdc", type=str, 533 help="path to hdc" 534 ) 535 parser.add_argument( 536 "--taskset", "-t", default="", type=str, 537 help="Use taskset mask for affinity on specific CPUs" 538 ) 539 parser.add_argument("--config", "-c", required=True, type=str, help="config json-file") 540 return process_args(parser.parse_args()) 541 542 543def init_report(report_file): 544 try: 545 today_wb = load_workbook(report_file) 546 today_ws = today_wb.worksheets[0] 547 except FileNotFoundError: 548 headers_row = ['用例名称', '场景', '执行状态', 'ark用例执行耗时(ms)', '昨日ark用例执行耗时(ms)', '是否劣化', 549 'v 8(ms)', 'v 8 --jitless(ms)', 'ark/v 8', 'ark/v 8 jitless', 'hyperlink', '备注'] 550 today_wb = Workbook() 551 today_ws = today_wb.active 552 553 today_ws.column_dimensions['A'].width = 35.0 554 today_ws.column_dimensions['B'].width = 15.0 555 today_ws.column_dimensions['C'].width = 15.0 556 today_ws.column_dimensions['D'].width = 15.0 557 today_ws.column_dimensions['E'].width = 25.0 558 today_ws.column_dimensions['F'].width = 15.0 559 today_ws.column_dimensions['G'].width = 15.0 560 today_ws.column_dimensions['H'].width = 15.0 561 today_ws.column_dimensions['I'].width = 15.0 562 today_ws.column_dimensions['J'].width = 15.0 563 today_ws.column_dimensions['K'].width = 50.0 564 today_ws.column_dimensions['L'].width = 15.0 565 today_ws.append(headers_row) 566 today_ws.freeze_panes = 'A2' 567 today_wb.save(report_file) 568 569 570def append_date_label(target_str, date_input): 571 formatted_date = date_input.strftime('%Y%m%d') 572 new_str = target_str + "_{}".format(formatted_date) 573 574 return new_str 575 576 577def get_v_8_benchmark_daily_report_path(): 578 ''' 579 get v 8 based data. v 8 based data obtained on 1,11,21 day for dayevery month.that is to say, in 1,11,21, 580 v 8 executes js cases. 581 ''' 582 now = datetime.datetime.now(tz=datetime.timezone.utc) 583 today_str = now.strftime("%Y.%m.%d") 584 str_list = today_str.split('.') 585 year_str = str_list[0] 586 month_str = str_list[1] 587 day = int(str_list[2]) 588 based_day = 0 589 if day > 21: 590 based_day = 21 591 elif day > 11: 592 based_day = 11 593 else: 594 based_day = 1 595 596 based_date = year_str + month_str + str(based_day) 597 base_date_file = based_date + '.xlsx' 598 based_report_name = '_'.join([Constants.REPORT_NAME_HEAD_FIX, base_date_file]) 599 report_file_path = os.path.join(OUTPUT_PATH, based_report_name) 600 return report_file_path 601 602 603def get_given_date_report_name(date_input): 604 report_name_head = append_date_label(Constants.REPORT_NAME_HEAD_FIX, date_input) 605 return report_name_head + ".xlsx" 606 607 608def get_given_date_report_path(date_input): 609 report_file_name = get_given_date_report_name(date_input) 610 report_file_path = os.path.join(OUTPUT_PATH, report_file_name) 611 return report_file_path 612 613 614def get_yesterday_excute_times(yesterday_report): 615 if not os.path.exists(yesterday_report) or not os.path.isfile(yesterday_report): 616 return 617 618 wb = load_workbook(yesterday_report) 619 ws = wb.worksheets[0] 620 for row_num in range(2, ws.max_row + 1): 621 js_case = ws.cell(row=row_num, column=1).value 622 scene = ws.cell(row=row_num, column=2).value 623 exec_status = ws.cell(row=row_num, column=3).value 624 if exec_status == Constants.PASS or exec_status == Constants.FAIL: 625 main_key = '/'.join([js_case, scene]).lower() 626 excute_time = ws.cell(row=row_num, column=4).value 627 Constants.YESTERDAY_EXCUTE_TIME_DICT[main_key] = excute_time 628 629 630def update_data_by_log(data: dict, log_path: str, js_name: str) -> dict: 631 """Update execution time data by log file""" 632 with open(log_path, 'r') as f: 633 for line in f: 634 if "scene_output" not in line: 635 continue 636 str_array = line.split(':') 637 mid_str = str_array[1].strip() 638 scene = mid_str.split()[2] 639 exec_time = str_array[2] 640 key_str = '/'.join([js_name + '.js', scene]).lower() 641 if key_str not in data: 642 data[key_str] = float(exec_time) 643 else: 644 data[key_str] += float(exec_time) 645 return data 646 647 648def get_v_8_cmd(parameter: str, js_file_path: str) -> List[str]: 649 """Get command for v 8""" 650 cmd: List[str] = [] 651 if Constants.VER_PLATFORM.find("arm64") != -1: 652 cmd = [Constants.HDC_PATH, "shell"] 653 if len(Constants.TASKSET_MASK) != 0: 654 cmd += ["taskset", "-a", Constants.TASKSET_MASK] 655 if len(parameter) == 0: 656 cmd += [Constants.V_8_ENGINED_PATH, js_file_path] 657 else: 658 cmd += [Constants.V_8_ENGINED_PATH, parameter, js_file_path] 659 return cmd 660 661 662def run_v_8_single_js_case(js_file_path, cmd_para, js_case_name, iterations: int): 663 """Run single js case for v 8 based with parameters""" 664 v_8_exec_time_dict = {} 665 scenes = get_js_file_class_api_scenes(js_file_path) 666 667 v_8_log_path = os.path.join(Constants.CUR_PATH, "v_8.log") 668 669 flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL 670 modes = stat.S_IWUSR | stat.S_IRUSR 671 used_js_file = js_file_path 672 673 if Constants.VER_PLATFORM.find("arm64") != -1: 674 js_file_path_device = os.path.join(Constants.DEVICE_WORKDIR, 675 os.path.basename(js_case_name + '.js')) 676 used_js_file = js_file_path_device 677 if hdc_send(js_file_path, js_file_path_device) != 0: 678 for elem in enumerate(scenes): 679 v_8_exec_time_dict[elem] = 0 680 logger.error("Couldn't send file %s to device", js_file_path) 681 return v_8_exec_time_dict 682 683 cmd = get_v_8_cmd(cmd_para, used_js_file) 684 logger.info("run cmd:%s", cmd) 685 data = {} 686 for _ in range(iterations): 687 if os.path.exists(v_8_log_path): 688 os.remove(v_8_log_path) 689 with os.fdopen(os.open(v_8_log_path, flags, modes), 'wb') as outfile: 690 ret = subprocess.run(cmd, stdout=outfile, check=False) 691 if ret.returncode != 0: 692 for elem in enumerate(scenes): 693 v_8_exec_time_dict[elem] = 0 694 logger.error("execute cmd failed. cmd: %s", cmd) 695 return v_8_exec_time_dict 696 data = update_data_by_log(data, v_8_log_path, js_case_name) 697 os.remove(v_8_log_path) 698 699 for k, time_value in data.items(): 700 v_8_exec_time_dict[k] = str(time_value / iterations) 701 logger.info("v 8 excute %s successfully. cmd: %s", js_file_path, cmd) 702 return v_8_exec_time_dict 703 704 705def get_given_column_data(report_file, column_index): 706 column_data = {} 707 if os.path.exists(report_file) and report_file.endswith('.xlsx'): 708 wb = load_workbook(report_file) 709 ws = wb.worksheets[0] 710 711 for row_num in range(2, ws.max_row + 1): 712 js_case_name = str(ws.cell(row=row_num, column=1).value) 713 scene = str(ws.cell(row=row_num, column=2).value) 714 exec_status = str(ws.cell(row=row_num, column=3).value) 715 time = str(ws.cell(row=row_num, column=column_index).value) 716 if exec_status == Constants.PASS or exec_status == Constants.FAIL: 717 main_key = '/'.join([js_case_name, scene]) 718 column_data[main_key] = time 719 720 return column_data 721 722 723def get_v_8_excute_times(jspath, v_8_based_report_file, iterations): 724 if os.path.exists(v_8_based_report_file) and os.path.isfile(v_8_based_report_file): 725 # Generate v 8 benchmark data on the 1st, 11th, and 21st of each month.The testing at other times refers to 726 # these V 8 benchmark data 727 v_8_exec_time_dict = get_given_column_data(v_8_based_report_file, 7) 728 for key in v_8_exec_time_dict.keys(): 729 Constants.V_8_EXCUTE_TIME_DICT[key] = v_8_exec_time_dict[key] 730 return Constants.RET_OK 731 732 file_list = [] 733 for root, _, files in os.walk(jspath): 734 for file in files: 735 if not file.endswith('.js'): 736 continue 737 file_path = os.path.join(root, file) 738 file_list.append(file_path) 739 for _, file_path in enumerate(file_list): 740 results = file_path.split("/") 741 class_name = results[-2] 742 api_name = results[-1].split(".")[0] 743 js_case_name = '/'.join([class_name, api_name]) 744 745 v_8_exec_time_dict = run_v_8_single_js_case(file_path, '', js_case_name, iterations) 746 for key in v_8_exec_time_dict.keys(): 747 Constants.V_8_EXCUTE_TIME_DICT[key] = v_8_exec_time_dict[key] 748 749 return Constants.RET_OK 750 751 752def get_v_8_jitless_excute_times(jspath, v_8_based_report_file_path, iterations): 753 if os.path.exists(v_8_based_report_file_path) and os.path.isfile(v_8_based_report_file_path): 754 # Generate v 8 benchmark data on the 1st, 11th, and 21st of each month.The testing at other times refers to 755 # these V 8 benchmark data 756 v_8_exec_time_dict = get_given_column_data(v_8_based_report_file_path, 8) 757 for key in v_8_exec_time_dict.keys(): 758 Constants.V_8_JITLESS_EXCUTE_TIME_DICT[key] = v_8_exec_time_dict[key] 759 return Constants.RET_OK 760 761 file_list = [] 762 for root, _, files in os.walk(jspath): 763 for file in files: 764 if not file.endswith('.js'): 765 continue 766 file_path = os.path.join(root, file) 767 file_list.append(file_path) 768 769 for _, file_path in enumerate(file_list): 770 results = file_path.split("/") 771 class_name = results[-2] 772 api_name = results[-1].split(".")[0] 773 js_case_name = '/'.join([class_name, api_name]) 774 775 v_8_exec_time_dict = run_v_8_single_js_case(file_path, '--jitless', js_case_name, iterations) 776 for key in v_8_exec_time_dict.keys(): 777 Constants.V_8_JITLESS_EXCUTE_TIME_DICT[key] = v_8_exec_time_dict[key] 778 779 return Constants.RET_OK 780 781 782def hdc_send(source: str, destination: str) -> int: 783 """Run hdc send command""" 784 hdc_cmd: List[str] = [Constants.HDC_PATH, "file", "send"] 785 hdc_cmd += [source, destination] 786 logger.info("run cmd: %s", hdc_cmd) 787 return subprocess.run(hdc_cmd, check=False).returncode 788 789 790def hdc_run(cmd: List[str]) -> int: 791 """Run command on device via hdc shell""" 792 hdc_cmd = [Constants.HDC_PATH, "shell"] + cmd 793 return subprocess.run(hdc_cmd).returncode 794 795 796def prepare_device(): 797 """Preapare device workdir for js perf testing""" 798 if hdc_send(Constants.ARK_JS_VM_PATH, Constants.DEVICE_WORKDIR) != 0: 799 logger.error("Couldn't send ark_js_vm to device") 800 sys.exit(1) 801 hdc_run(["chmod", "u+x", os.path.join(Constants.DEVICE_WORKDIR, "ark_js_vm")]) 802 arkjsvm_lib = os.path.join(Constants.ETS_RUNTIME, "libark_jsruntime.so") 803 if hdc_send(arkjsvm_lib, os.path.join(Constants.DEVICE_WORKDIR, "lib")) != 0: 804 logger.error("Couldn't send libark_jsruntime.so to device") 805 sys.exit(1) 806 if hdc_send(ICU_DATA_PATH, Constants.DEVICE_WORKDIR) != 0: 807 logger.error("Couldn't send icu data to device") 808 sys.exit(1) 809 thirdparty_path = os.path.join(Constants.DEVICE_WORKDIR, "thirdparty") 810 for lib in Constants.LIBS_LIST: 811 if not os.path.isdir(lib): 812 logger.error("Couldn't find lib from config %s", lib) 813 sys.exit(1) 814 if hdc_send(lib, thirdparty_path) != 0: 815 logger.error("Couldn't send %s lib to device", lib) 816 sys.exit(1) 817 if hdc_send(Constants.STUB_AN, os.path.join(Constants.DEVICE_WORKDIR, "lib")) != 0: 818 logger.error("Couldn't send %s file to device", Constants.STUB_AN) 819 sys.exit(1) 820 821 822def get_config(parameters: argparse.Namespace): 823 """Get config from arguments and json file""" 824 Constants.V_8_ENGINED_PATH = parameters.d_8_binary_path 825 Constants.VER_PLATFORM = parameters.ver_platform 826 Constants.HDC_PATH = parameters.hdc 827 Constants.TASKSET_MASK = parameters.taskset 828 with open(parameters.config, 'r', encoding='UTF-8') as f: 829 json_data = json.load(f) 830 831 Constants.ES2ABC_PATH = os.path.join(BINARY_PATH, json_data[Constants.VER_PLATFORM]["ES2ABC"]) 832 Constants.ETS_RUNTIME = os.path.join(BINARY_PATH, 833 json_data[Constants.VER_PLATFORM]["ETS_RUNTIME_PATH"]) 834 Constants.ARK_JS_VM_PATH = os.path.join(Constants.ETS_RUNTIME, "ark_js_vm") 835 Constants.STUB_AN = os.path.join(BINARY_PATH, json_data[Constants.VER_PLATFORM]["STUB_AN"]) 836 libs = json_data[Constants.VER_PLATFORM]["LIBS_LIST"] 837 for lib in libs: 838 Constants.LIBS_LIST.append(os.path.normpath(os.path.join(BINARY_PATH, lib))) 839 if Constants.VER_PLATFORM.find("x86_64") != -1: 840 old_ld_library_path = os.environ.get('LD_LIBRARY_PATH', '') 841 Constants.LD_LIBRARY_PATH = Constants.ETS_RUNTIME + ":" 842 if len(Constants.LIBS_LIST) != 0: 843 Constants.LD_LIBRARY_PATH += ":".join(Constants.LIBS_LIST) 844 if len(old_ld_library_path) != 0: 845 Constants.LD_LIBRARY_PATH += f":{old_ld_library_path}" 846 os.environ['LD_LIBRARY_PATH'] = Constants.LD_LIBRARY_PATH 847 elif Constants.VER_PLATFORM.find("arm64") != -1: 848 Constants.LD_LIBRARY_PATH = os.path.join(Constants.DEVICE_WORKDIR, "lib") 849 for lib in Constants.LIBS_LIST: 850 lib = os.path.normpath(lib) 851 Constants.LD_LIBRARY_PATH += ":" +\ 852 os.path.join(Constants.DEVICE_WORKDIR, "thirdparty", os.path.basename(lib)) 853 854 855if __name__ == "__main__": 856 LOG_PATH = os.path.join(Constants.TMP_PATH, "test.log") 857 if os.path.exists(LOG_PATH): 858 os.remove(LOG_PATH) 859 logger = get_logger("jstest", LOG_PATH) 860 861 paras = get_args() 862 logger.info("execute arguments: %s", paras) 863 864 DETERIORATION_BOUNDARY_VALUE = paras.deterioration_boundary_value 865 BINARY_PATH = paras.binarypath 866 ICU_DATA_PATH = os.path.join(BINARY_PATH, "third_party/icu/ohos_icu4j/data") 867 OUTPUT_PATH = Constants.CUR_PATH 868 get_config(paras) 869 if not os.path.exists(Constants.ARK_JS_VM_PATH): 870 logger.error("%s does not exist", Constants.ARK_JS_VM_PATH) 871 sys.exit(1) 872 if Constants.VER_PLATFORM.find("arm64") != -1: 873 prepare_device() 874 875 if paras.output_folder_path is not None: 876 OUTPUT_PATH = paras.output_folder_path 877 878 if not os.path.exists(OUTPUT_PATH): 879 os.makedirs(OUTPUT_PATH) 880 881 today = datetime.date.today() 882 yesterday = today - datetime.timedelta(days=1) 883 TODAY_EXCEL_PATH = get_given_date_report_path(today) 884 YESTERDAY_EXCEL_PATH = get_given_date_report_path(yesterday) 885 886 if os.path.exists(TODAY_EXCEL_PATH): 887 os.remove(TODAY_EXCEL_PATH) 888 889 get_js_case_super_link_data(paras.jspath) 890 start_time = datetime.datetime.now(tz=datetime.timezone.utc) 891 init_report(TODAY_EXCEL_PATH) 892 get_yesterday_excute_times(YESTERDAY_EXCEL_PATH) 893 v_8_based_report_path = get_v_8_benchmark_daily_report_path() 894 get_v_8_excute_times(paras.jspath, v_8_based_report_path, paras.iterations) 895 get_v_8_jitless_excute_times(paras.jspath, v_8_based_report_path, paras.iterations) 896 897 run_via_ark(paras.jspath, TODAY_EXCEL_PATH, paras.iterations) 898 end_time = datetime.datetime.now(tz=datetime.timezone.utc) 899 900 totol_time = u"%s" % (end_time - start_time) 901 append_summary_info(TODAY_EXCEL_PATH, totol_time) 902 903 logger.info("run js perf test finished. Please check details in report.") 904 shutil.rmtree(Constants.TMP_PATH) 905