1#!/usr/bin/env python 2# -*- coding: utf-8 -*- 3""" 4Copyright (c) 2023 Huawei Device Co., Ltd. 5Licensed under the Apache License, Version 2.0 (the "License"); 6you may not use this file except in compliance with the License. 7You may obtain a copy of the License at 8 9 http://www.apache.org/licenses/LICENSE-2.0 10 11Unless required by applicable law or agreed to in writing, software 12distributed under the License is distributed on an "AS IS" BASIS, 13WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14See the License for the specific language governing permissions and 15limitations under the License. 16 17Description: Use ark to execute workload test suite 18""" 19 20import argparse 21import datetime 22import json 23import logging 24import os 25import shutil 26import stat 27import subprocess 28from collections import namedtuple 29from openpyxl import Workbook, load_workbook 30from openpyxl.styles import PatternFill 31 32 33def get_logger(logger_name, log_file_path, level=logging.INFO): 34 formatter = logging.Formatter(fmt='[%(asctime)s] [%(levelname)s] %(message)s', 35 datefmt='%Y-%m-%d %H:%M:%S') 36 37 fh = logging.FileHandler(encoding='utf-8', mode='a', filename=log_file_path) 38 fh.setFormatter(formatter) 39 fh.setLevel(logging.DEBUG) 40 # console output 41 ch = logging.StreamHandler() 42 ch.setFormatter(formatter) 43 ch.setLevel(logging.DEBUG) 44 log = logging.getLogger(logger_name) 45 log.addHandler(fh) 46 log.addHandler(ch) 47 log.setLevel(level) 48 49 return log 50 51 52class Constants: 53 logger = None 54 CUR_PATH = os.path.abspath(os.path.dirname(__file__)) 55 TMP_PATH = os.path.join(os.getcwd(), "tmp") 56 REPORT_NAME_HEAD_FIX = "js_perf_test_result" 57 RET_OK = 0 58 RET_FAILED = 1 59 BINARY_PATH = "" 60 OUTPUT_PATH = "" 61 LOG_PATH = "" 62 TODAY_EXCEL_PATH = "" 63 YESTERDAY_EXCEL_PATH = "" 64 DETERIORATION_BOUNDARY_VALUE = 0.05 65 TODAY_EXCUTE_INFO = {} 66 YESTERDAY_EXCUTE_TIME_DICT = {} 67 V_8_EXCUTE_TIME_DICT = {} 68 V_8_JITLESS_EXCUTE_TIME_DICT = {} 69 JS_FILE_SUPER_LINK_DICT = {} 70 HYPERLINK_HEAD = "https://gitee.com/dov1s/arkjs-perf-test/tree/builtins_test1110/js-perf-test" 71 PASS = 'pass' 72 FAIL = 'fail' 73 SOLID = 'solid' 74 NA_FIX = 'NA' 75 # 1e-6 s 76 COMPARISON_ACCURACY = 0.001 77 ICU_DATA_PATH = "" 78 FIX_STR = "8/d" 79 V_8_ENGINED_PATH = '/usr/bin/v{}8'.format(FIX_STR) 80 VER_PLATFORM = "full_x86_64" 81 ES2ABC_PATH = "" 82 ARK_JS_VM_PATH = "" 83 ARK_AOT_COMPILER_PATH = "" 84 CaseTestDataType = namedtuple('test', ['exec_status', 'exec_time']) 85 86 87def get_js_file_class_api_scenes(js_file_path): 88 scenes = [] 89 with open(js_file_path, 'r') as f: 90 for line in f: 91 if "scene_output" in line: 92 str_array = line.split(':') 93 mid_str = str_array[1].strip() 94 elements = mid_str.split(' ') 95 main_key = '/'.join([elements[0], elements[1] + '.js', elements[2]]).lower() 96 scenes.append(main_key) 97 return scenes 98 99 100def degraded_str(yesterday_excute_time, exec_time): 101 is_degraded_str = Constants.NA_FIX 102 if len(str(yesterday_excute_time).strip()) != 0: 103 if abs(float(yesterday_excute_time)) <= Constants.COMPARISON_ACCURACY: 104 is_degraded_str = str(True) if abs(float(exec_time)) >= DETERIORATION_BOUNDARY_VALUE else str(False) 105 else: 106 is_degraded_tmp = float(exec_time) / float(yesterday_excute_time) >= (1 + DETERIORATION_BOUNDARY_VALUE) 107 is_degraded_str = str(True) if is_degraded_tmp else str(False) 108 109 return is_degraded_str 110 111 112def v_8_excute_time_compute(main_key): 113 v_8_excute_time_str = '' 114 if len(Constants.V_8_EXCUTE_TIME_DICT) > 0 and main_key in Constants.V_8_EXCUTE_TIME_DICT.keys(): 115 v_8_excute_time_str = Constants.V_8_EXCUTE_TIME_DICT[main_key].strip() 116 117 if len(v_8_excute_time_str) == 0: 118 v_8_excute_time = ' ' 119 else: 120 v_8_excute_time = v_8_excute_time_str 121 122 return v_8_excute_time 123 124 125def v_8_gitless_excute_time_compute(main_key): 126 v_8_jitless_excute_time_str = '' 127 if len(Constants.V_8_JITLESS_EXCUTE_TIME_DICT) > 0 and main_key in Constants.V_8_JITLESS_EXCUTE_TIME_DICT.keys(): 128 v_8_jitless_excute_time_str = Constants.V_8_JITLESS_EXCUTE_TIME_DICT[main_key].strip() 129 130 if len(v_8_jitless_excute_time_str) == 0: 131 v_8_jitless_excute_time = ' ' 132 else: 133 v_8_jitless_excute_time = v_8_jitless_excute_time_str 134 135 return v_8_jitless_excute_time 136 137 138def ark_divide_v_8_compute(exec_time, v_8_excute_time): 139 if len(exec_time) == 0 or len(v_8_excute_time.strip()) == 0: 140 ark_divide_v_8 = Constants.NA_FIX 141 elif abs(float(exec_time)) <= Constants.COMPARISON_ACCURACY: 142 if abs(float(v_8_excute_time)) <= Constants.COMPARISON_ACCURACY: 143 ark_divide_v_8 = '1' 144 else: 145 ark_divide_v_8 = '0' 146 else: 147 v_8_excute_time = v_8_excute_time.strip() 148 if len(v_8_excute_time) == 0 or abs(float(v_8_excute_time)) <= Constants.COMPARISON_ACCURACY: 149 ark_divide_v_8 = Constants.NA_FIX 150 else: 151 ark_divide_v_8 = str("{:.2f}".format(float(exec_time) / float(v_8_excute_time))) 152 153 return ark_divide_v_8 154 155 156def append_row_data(report_file, case_test_data): 157 wb = load_workbook(report_file) 158 ws = wb.worksheets[0] 159 for main_key in case_test_data.keys(): 160 str_arr = main_key.split('/') 161 class_name = str_arr[0] 162 api_name = str_arr[1] 163 scene = str_arr[2] 164 js_case_name = '/'.join([class_name, api_name]) 165 excute_status = case_test_data[main_key].exec_status 166 exec_time = case_test_data[main_key].exec_time.strip() 167 yesterday_excute_time = '' 168 if (len(Constants.YESTERDAY_EXCUTE_TIME_DICT) > 0 and 169 Constants.YESTERDAY_EXCUTE_TIME_DICT.get(main_key) is not None): 170 yesterday_excute_time = str(Constants.YESTERDAY_EXCUTE_TIME_DICT[main_key]) 171 is_degraded_str = degraded_str(yesterday_excute_time, exec_time) 172 v_8_excute_time = v_8_excute_time_compute(main_key) 173 v_8_jitless_excute_time = v_8_gitless_excute_time_compute(main_key) 174 ark_divide_v_8 = ark_divide_v_8_compute(exec_time, v_8_excute_time) 175 if len(exec_time) == 0 or len(v_8_jitless_excute_time.strip()) == 0: 176 ark_divide_v_8_with_jitless = Constants.NA_FIX 177 elif abs(float(exec_time)) <= Constants.COMPARISON_ACCURACY: 178 if abs(float(v_8_jitless_excute_time)) <= Constants.COMPARISON_ACCURACY: 179 ark_divide_v_8_with_jitless = '1' 180 else: 181 ark_divide_v_8_with_jitless = '0' 182 else: 183 v_8_jitless_excute_time = v_8_jitless_excute_time.strip() 184 if (len(v_8_jitless_excute_time) == 0 or 185 abs(float(v_8_jitless_excute_time)) <= Constants.COMPARISON_ACCURACY): 186 ark_divide_v_8_with_jitless = Constants.NA_FIX 187 else: 188 ark_divide_v_8_with_jitless = str("{:.2f}".format(float(exec_time) / float(v_8_jitless_excute_time))) 189 jis_case_file_name_with_class = Constants.JS_FILE_SUPER_LINK_DICT['/'.join([class_name, api_name])] 190 js_file_super_link = '/'.join([Constants.HYPERLINK_HEAD, jis_case_file_name_with_class]) 191 new_row = [js_case_name, scene, excute_status, exec_time, yesterday_excute_time, 192 is_degraded_str, v_8_excute_time, v_8_jitless_excute_time, ark_divide_v_8, 193 ark_divide_v_8_with_jitless, js_file_super_link, ' '] 194 ws.append(new_row) 195 if is_degraded_str is str(True): 196 ws.cell(row=ws.max_row, column=6).fill = PatternFill(start_color='FF0000', end_color='FF0000', 197 fill_type=Constants.SOLID) 198 if (ark_divide_v_8 != Constants.NA_FIX and 199 (float(ark_divide_v_8) > 2 or 200 abs(float(ark_divide_v_8) - 2) <= Constants.COMPARISON_ACCURACY)): 201 ws.cell(row=ws.max_row, column=9).fill = PatternFill(start_color='FFFF00', end_color='FFFF00', 202 fill_type=Constants.SOLID) 203 if (ark_divide_v_8_with_jitless != Constants.NA_FIX and 204 (float(ark_divide_v_8_with_jitless) > 2 or 205 abs(float(ark_divide_v_8_with_jitless) - 2) <= Constants.COMPARISON_ACCURACY)): 206 ws.cell(row=ws.max_row, column=10).fill = PatternFill(start_color='FF00FF', end_color='FF00FF', 207 fill_type=Constants.SOLID) 208 wb.save(report_file) 209 return Constants.RET_OK 210 211 212def run_js_case_via_ark(binary_path, js_file_path, class_name, api_name, report_file): 213 composite_scenes = get_js_file_class_api_scenes(js_file_path) 214 case_test_data = {} 215 execute_status = Constants.FAIL 216 execute_time = ' ' 217 218 for _, composite_scene in enumerate(composite_scenes): 219 case_test_data[composite_scene] = Constants.CaseTestDataType(execute_status, execute_time) 220 221 js_file_name = class_name + '/' + api_name + '.js' 222 fangzhou_test_path = os.path.join(Constants.TMP_PATH, "fangzhou_test") # for abc file 223 if os.path.exists(fangzhou_test_path): 224 shutil.rmtree(fangzhou_test_path) 225 os.makedirs(fangzhou_test_path) 226 227 class_folder_path = os.path.join(fangzhou_test_path, class_name) 228 api_path = os.path.join(class_folder_path, api_name) 229 if not os.path.exists(class_folder_path): 230 os.makedirs(class_folder_path) 231 abc_file_path = api_path + ".abc" 232 cur_abc_file = os.path.join(Constants.CUR_PATH, api_name + ".abc") 233 cur_ap_file = os.path.join(Constants.CUR_PATH, api_name + ".ap") 234 cur_an_file = os.path.join(Constants.CUR_PATH, api_name + ".an") 235 cur_ai_file = os.path.join(Constants.CUR_PATH, api_name + ".ai") 236 api_log_path = os.path.join(class_folder_path, api_name + ".log") 237 238 es2abc_path = Constants.ES2ABC_PATH 239 # tranmit abc 240 cmd = [es2abc_path, "--module", "--merge-abc", "--extension=js", js_file_path] 241 242 logger.info("run cmd: %s", cmd) 243 ret = subprocess.run(cmd) 244 if ret.returncode != 0: 245 logger.error("ret = %s, %s generate abc file failed. cmd: %s", str(ret), js_file_name, cmd) 246 append_row_data(report_file, case_test_data) 247 return case_test_data 248 249 cmd2 = ["cp", cur_abc_file, abc_file_path] 250 ret = subprocess.run(cmd2) 251 if ret.returncode != 0: 252 logger.error("ret.returncode = %s, %s generate abc file failed. cmd: %s", str(ret.returncode), js_file_name, 253 cmd2) 254 append_row_data(report_file, case_test_data) 255 return case_test_data 256 # execute pgo 257 ark_js_vm_path = Constants.ARK_JS_VM_PATH 258 cmd = [ark_js_vm_path, 259 "--enable-pgo-profiler=true", 260 f"--compiler-pgo-profiler-path={api_name}.ap", 261 f"--entry-point={api_name}", 262 "--icu-data-path", ICU_DATA_PATH, 263 cur_abc_file] 264 logger.info("run cmd: %s", cmd) 265 ret = subprocess.run(cmd) 266 if ret.returncode != 0: 267 logger.error("ret = %s, %s pgo file failed. cmd: %s", str(ret), cur_abc_file, cmd) 268 append_row_data(report_file, case_test_data) 269 return case_test_data 270 271 # excute first time aot 272 ark_aot_compiler_path = Constants.ARK_AOT_COMPILER_PATH 273 cmd = [ark_aot_compiler_path, 274 f"--aot-file={api_name}", 275 f"--compiler-pgo-profiler-path={api_name}.ap", 276 "--icu-data-path", ICU_DATA_PATH, 277 cur_abc_file] 278 logger.info("run cmd: %s", cmd) 279 ret = subprocess.run(cmd) 280 if ret.returncode != 0: 281 logger.error("ret = %s, %s aot file failed. cmd: %s", str(ret), cur_abc_file, cmd) 282 append_row_data(report_file, case_test_data) 283 return case_test_data 284 285 # excute second time pgo 286 ark_js_vm_path = Constants.ARK_JS_VM_PATH 287 cmd = [ark_js_vm_path, 288 "--enable-pgo-profiler=true", 289 f"--compiler-pgo-profiler-path={api_name}.ap", 290 f"--entry-point={api_name}", 291 f"--aot-file={api_name}", 292 "--icu-data-path", ICU_DATA_PATH, 293 cur_abc_file] 294 logger.info("run cmd: %s", cmd) 295 ret = subprocess.run(cmd) 296 if ret.returncode != 0: 297 logger.error("ret = %s, %s sencond time pgo file failed. cmd: %s", str(ret), cur_abc_file, cmd) 298 append_row_data(report_file, case_test_data) 299 return case_test_data 300 301 # excute second time aot 302 ark_aot_compiler_path = Constants.ARK_AOT_COMPILER_PATH 303 cmd = [ark_aot_compiler_path, 304 f"--aot-file={api_name}", 305 f"--compiler-pgo-profiler-path={api_name}.ap", 306 "--icu-data-path", ICU_DATA_PATH, 307 cur_abc_file] 308 logger.info("run cmd: %s", cmd) 309 ret = subprocess.run(cmd) 310 if ret.returncode != 0: 311 logger.error("ret = %s, %s second time aot file failed. cmd: %s", str(ret), cur_abc_file, cmd) 312 append_row_data(report_file, case_test_data) 313 return case_test_data 314 315 #excute final abc 316 ark_js_vm_path = Constants.ARK_JS_VM_PATH 317 cmd = [ark_js_vm_path, 318 "--log-level=info", 319 "--compiler-trace-deopt=true", 320 f"--entry-point={api_name}", 321 f"--aot-file={api_name}", 322 "--icu-data-path", ICU_DATA_PATH, 323 cur_abc_file] 324 logger.info("run cmd: %s", cmd) 325 flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL 326 modes = stat.S_IWUSR | stat.S_IRUSR 327 if os.path.exists(api_log_path): 328 os.remove(api_log_path) 329 with os.fdopen(os.open(api_log_path, flags, modes), 'wb') as outfile: 330 ret = subprocess.run(cmd, stdout=outfile) 331 332 if ret.returncode != 0: 333 logger.error("%s execute abc file failed. cmd: %s", js_file_name, cmd) 334 append_row_data(report_file, case_test_data) 335 return case_test_data 336 else: 337 case_test_data.clear() 338 if os.path.exists(api_log_path): 339 with open(api_log_path, 'r') as f: 340 for line in f: 341 if "scene_output" not in line: 342 continue 343 344 mid_str = line.split(':')[1].strip() 345 scene = mid_str.split()[2] 346 main_key = '/'.join([js_file_name, scene]).lower() 347 execute_time = line.split(':')[2] 348 execute_status = Constants.PASS 349 case_test_data[main_key] = Constants.CaseTestDataType(execute_status, execute_time) 350 351 append_row_data(report_file, case_test_data) 352 logger.info("%s execute abc file successfully. cmd: %s case_test_data: %s", js_file_name, cmd, case_test_data) 353 354 os.remove(cur_abc_file) 355 os.remove(cur_ap_file) 356 os.remove(cur_an_file) 357 os.remove(cur_ai_file) 358 359 return case_test_data 360 361 362def run_via_ark(jspath, report_file): 363 if not os.path.exists(jspath): 364 logger.error("js perf cases path is not exist. jspath: %s", jspath) 365 logger.info("begin to run js perf test via ark. js perf cases path: %s", jspath) 366 for root, _, files in os.walk(jspath): 367 if "TestCaseError" in root: 368 continue 369 for file in files: 370 if not file.endswith('.js'): 371 continue 372 373 file_path = os.path.join(root, file) 374 results = file_path.split("/") 375 class_name = results[-2] 376 api_name = results[-1].split(".")[0] 377 js_case_name = '/'.join([class_name, results[-1]]) 378 logger.info("begin to execute %s.", js_case_name) 379 test_data = run_js_case_via_ark(BINARY_PATH, file_path, class_name, api_name, report_file) 380 for _, key in enumerate(test_data.keys()): 381 Constants.TODAY_EXCUTE_INFO[key] = test_data.get(key) 382 logger.info("finish executing %s. executing info: %s.", js_case_name, Constants.TODAY_EXCUTE_INFO) 383 384 385def get_js_case_super_link_data(jspath): 386 logger.info("get js case super link data") 387 for root, _, files in os.walk(jspath): 388 for file in files: 389 if not file.endswith('.js'): 390 continue 391 392 file_path = os.path.join(root, file) 393 results = file_path.split("/") 394 class_name = results[-2] 395 js_case_name = '/'.join([class_name, results[-1]]) 396 key = js_case_name.lower() 397 Constants.JS_FILE_SUPER_LINK_DICT[key] = js_case_name 398 399 400def export_sumary_info_for_notifying_email(json_path, total_cases_num, ark_divide_v_8_num, ark_divide_v_8_jitless_num): 401 data = {} 402 data['kind'] = 'V 8 js-perf-test' 403 data['Total'] = total_cases_num 404 data['Ark劣化v 8'] = ark_divide_v_8_num 405 data['Ark劣化v 8 jitless'] = ark_divide_v_8_jitless_num 406 flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL 407 modes = stat.S_IWUSR | stat.S_IRUSR 408 if os.path.exists(json_path): 409 os.remove(json_path) 410 with os.fdopen(os.open(json_path, flags, modes), 'w', encoding='utf-8') as f: 411 json.dump(data, f, indent=4, ensure_ascii=False) 412 logger.info("export summary info to json file successfully.") 413 414 415def get_umary_info_json_file_path(daily_report_file_path): 416 dir_path = os.path.dirname(daily_report_file_path) 417 json_file_name = 'jsperftest_notifying_info_in_email.json' 418 json_file_path = os.path.join(dir_path, json_file_name) 419 return json_file_path 420 421 422def append_summary_info(report_file, total_cost_time): 423 """ 424 summary info: 425 pass count: 426 fail count: 427 totle count: 428 degraded count: 429 total excute time is(s) : 430 degraded percentage upper limit: 431 ark/v 8 degraded count: 432 ark/v 8 jitless degraded count: 433 """ 434 wb = load_workbook(report_file) 435 ws = wb.worksheets[0] 436 437 totle_num = 0 438 degraded_upper_limit = DETERIORATION_BOUNDARY_VALUE 439 pass_num = 0 440 failed_num = 0 441 degraded_num = 0 442 ark_divide_v_8_degraded_count = 0 443 ark_divide_v_8_jitless_degraded_count = 0 444 445 for row_num in range(2, ws.max_row + 1): 446 excu_status = str(ws.cell(row=row_num, column=3).value) 447 is_degraded = str(ws.cell(row=row_num, column=6).value) 448 if is_degraded == str(True): 449 degraded_num += 1 450 451 if excu_status == Constants.PASS: 452 pass_num += 1 453 totle_num += 1 454 elif excu_status == Constants.FAIL: 455 failed_num += 1 456 totle_num += 1 457 458 obj = ws.cell(row=row_num, column=9).value 459 if obj is None: 460 obj = 0 461 ark_divide_v_8 = obj 462 if ark_divide_v_8 != Constants.NA_FIX and float(ark_divide_v_8) > 1: 463 ark_divide_v_8_degraded_count += 1 464 obj = ws.cell(row=row_num, column=10).value 465 if obj is None: 466 obj = 0 467 ark_divide_v_8_jitless = obj 468 if ark_divide_v_8_jitless != Constants.NA_FIX and float(ark_divide_v_8_jitless) > 1: 469 ark_divide_v_8_jitless_degraded_count += 1 470 471 count = 3 472 for _ in range(count): 473 new_row = [' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' '] 474 ws.append(new_row) 475 new_row = ['degraded_upper_limit', degraded_upper_limit, ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' '] 476 ws.append(new_row) 477 new_row = ['js test cases count', totle_num, ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' '] 478 ws.append(new_row) 479 new_row = ['Pass num', pass_num, ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' '] 480 ws.append(new_row) 481 new_row = ['Fail num', failed_num, ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' '] 482 ws.append(new_row) 483 new_row = ['ark today degraded_num', degraded_num, ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' '] 484 ws.append(new_row) 485 new_row = ['Total excute time(h:m:s.ms)', total_cost_time, ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' '] 486 ws.append(new_row) 487 new_row = ['ark/v_8 bad nums', ark_divide_v_8_degraded_count, ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' '] 488 ws.append(new_row) 489 new_row = ['ark/v_8 jitless badnums', ark_divide_v_8_jitless_degraded_count, ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', 490 ' ', ' '] 491 ws.append(new_row) 492 493 ws.column_dimensions.group('E', hidden=True) 494 wb.save(report_file) 495 496 json_file_path = get_umary_info_json_file_path(report_file) 497 export_sumary_info_for_notifying_email(json_file_path, totle_num, ark_divide_v_8_degraded_count, 498 ark_divide_v_8_jitless_degraded_count) 499 return Constants.RET_OK 500 501 502def get_args(): 503 parser = argparse.ArgumentParser() 504 parser.add_argument( 505 "--binarypath", 506 "-bp", 507 required=True, 508 help="path of binary folder. refer to harmony root folder path", 509 ) 510 parser.add_argument( 511 "--jspath", 512 "-p", 513 required=True, 514 help="path of js scripts, support folder and file", 515 ) 516 parser.add_argument( 517 "--deterioration_boundary_value", 518 "-d", 519 default=0.05, 520 help="deterioration boundary value, default 0.05", 521 ) 522 parser.add_argument( 523 "--output_folder_path", 524 "-o", 525 default=None, 526 help="output folder for executing js cases, default current folder", 527 ) 528 parser.add_argument( 529 "--d_8_binary_path", 530 "-v", 531 default=None, 532 help="v 8 engine d 8 binary path", 533 ) 534 parser.add_argument( 535 "--ver_platform", 536 "-e", 537 default="full_x86_64", 538 help="Code repository version and platform", 539 ) 540 args = parser.parse_args() 541 542 if not os.path.exists(args.binarypath): 543 logger.error("parameter --binarypath is not exist. Please check it! binary path: %s", args.binarypath) 544 raise RuntimeError("error bad parameters --binarypath") 545 546 if args.output_folder_path is None: 547 args.output_folder_path = os.getcwd() 548 549 if not os.path.isabs(args.output_folder_path): 550 args.output_folder_path = os.path.abspath(args.output_folder_path) 551 552 if not os.path.exists(args.d_8_binary_path): 553 logger.error("parameter --d_8_binary_path is not exist. Please check it! d 8 binary path: %s", 554 args.d_8_binary_path) 555 raise RuntimeError("error bad parameters --d_8_binary_path: {}".format(args.d_8_binary_path)) 556 557 return args 558 559 560def init_report(report_file): 561 try: 562 today_wb = load_workbook(report_file) 563 today_ws = today_wb.worksheets[0] 564 except FileNotFoundError: 565 headers_row = ['caseName', 'scene', 'status', 'ark_aot excute(ms)', 'last day excute(ms)', 'detorioration?', 566 'v 8(ms)', 'v 8 --jitless(ms)', 'ark_aot/v 8', 'ark_aot/v 8 jitless', 'hyperlink', 'note'] 567 today_wb = Workbook() 568 today_ws = today_wb.active 569 570 today_ws.column_dimensions['A'].width = 35.0 571 today_ws.column_dimensions['B'].width = 15.0 572 today_ws.column_dimensions['C'].width = 15.0 573 today_ws.column_dimensions['D'].width = 15.0 574 today_ws.column_dimensions['E'].width = 25.0 575 today_ws.column_dimensions['F'].width = 15.0 576 today_ws.column_dimensions['G'].width = 15.0 577 today_ws.column_dimensions['H'].width = 15.0 578 today_ws.column_dimensions['I'].width = 15.0 579 today_ws.column_dimensions['J'].width = 15.0 580 today_ws.column_dimensions['K'].width = 50.0 581 today_ws.column_dimensions['L'].width = 15.0 582 today_ws.append(headers_row) 583 today_ws.freeze_panes = 'A2' 584 today_wb.save(report_file) 585 586 587def append_date_label(target_str, date_input): 588 formatted_date = date_input.strftime('%Y%m%d') 589 new_str = target_str + "_{}".format(formatted_date) 590 591 return new_str 592 593 594def get_v_8_benchmark_daily_report_path(): 595 ''' 596 get v 8 based data. v 8 based data obtained on 1,11,21 day for dayevery month.that is to say, in 1,11,21, 597 v 8 executes js cases. 598 ''' 599 now = datetime.datetime.now(tz=datetime.timezone.utc) 600 today_str = now.strftime("%Y.%m.%d") 601 str_list = today_str.split('.') 602 year_str = str_list[0] 603 month_str = str_list[1] 604 day = int(str_list[2]) 605 based_day = 0 606 if day > 21: 607 based_day = 21 608 elif day > 11: 609 based_day = 11 610 else: 611 based_day = 1 612 613 based_date = year_str + month_str + str(based_day) 614 base_date_file = based_date + '.xlsx' 615 based_report_name = '_'.join([Constants.REPORT_NAME_HEAD_FIX, base_date_file]) 616 report_file_path = os.path.join(OUTPUT_PATH, based_report_name) 617 return report_file_path 618 619 620def get_given_date_report_name(date_input): 621 report_name_head = append_date_label(Constants.REPORT_NAME_HEAD_FIX, date_input) 622 return report_name_head + ".xlsx" 623 624 625def get_given_date_report_path(date_input): 626 report_file_name = get_given_date_report_name(date_input) 627 report_file_path = os.path.join(OUTPUT_PATH, report_file_name) 628 return report_file_path 629 630 631def get_yesterday_excute_times(yesterday_report): 632 if not os.path.exists(yesterday_report) or not os.path.isfile(yesterday_report): 633 return 634 635 wb = load_workbook(yesterday_report) 636 ws = wb.worksheets[0] 637 for row_num in range(2, ws.max_row + 1): 638 js_case = ws.cell(row=row_num, column=1).value 639 scene = ws.cell(row=row_num, column=2).value 640 exec_status = ws.cell(row=row_num, column=3).value 641 if exec_status == Constants.PASS or exec_status == Constants.FAIL: 642 main_key = '/'.join([js_case, scene]).lower() 643 excute_time = ws.cell(row=row_num, column=4).value 644 Constants.YESTERDAY_EXCUTE_TIME_DICT[main_key] = excute_time 645 646 647def run_v_8_single_js_case(js_file_path, cmd_para, js_case_name): 648 v_8_exec_time_dict = {} 649 scenes = get_js_file_class_api_scenes(js_file_path) 650 651 v_8_log_path = os.path.join(Constants.CUR_PATH, "v_8.log") 652 if os.path.exists(v_8_log_path): 653 os.remove(v_8_log_path) 654 655 flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL 656 modes = stat.S_IWUSR | stat.S_IRUSR 657 with os.fdopen(os.open(v_8_log_path, flags, modes), 'wb') as outfile: 658 if len(cmd_para) == 0: 659 cmd = [Constants.V_8_ENGINED_PATH, js_file_path] 660 else: 661 cmd = [Constants.V_8_ENGINED_PATH, cmd_para, js_file_path] 662 logger.info("run cmd:%s", cmd) 663 ret = subprocess.run(cmd, stdout=outfile) 664 665 if ret.returncode != 0: 666 for elem in enumerate(scenes): 667 v_8_exec_time_dict[elem] = 0 668 logger.error("execute cmd failed. cmd: %s", cmd) 669 return v_8_exec_time_dict 670 671 logger.info("v 8 excute %s successfully. cmd: %s", js_file_path, cmd) 672 673 with open(v_8_log_path, 'r') as f: 674 for line in f: 675 if "scene_output" not in line: 676 continue 677 str_array = line.split(':') 678 mid_str = str_array[1].strip() 679 scene = mid_str.split()[2] 680 exec_time = str_array[2] 681 key_str = '/'.join([js_case_name + '.js', scene]).lower() 682 v_8_exec_time_dict[key_str] = exec_time 683 684 os.remove(v_8_log_path) 685 return v_8_exec_time_dict 686 687 688def get_given_column_data(report_file, column_index): 689 column_data = {} 690 if os.path.exists(report_file) and report_file.endswith('.xlsx'): 691 wb = load_workbook(report_file) 692 ws = wb.worksheets[0] 693 694 for row_num in range(2, ws.max_row + 1): 695 js_case_name = str(ws.cell(row=row_num, column=1).value) 696 scene = str(ws.cell(row=row_num, column=2).value) 697 exec_status = str(ws.cell(row=row_num, column=3).value) 698 time = str(ws.cell(row=row_num, column=column_index).value) 699 if exec_status == Constants.PASS or exec_status == Constants.FAIL: 700 main_key = '/'.join([js_case_name, scene]) 701 column_data[main_key] = time 702 703 return column_data 704 705 706def get_v_8_excute_times(jspath, v_8_based_report_file): 707 if os.path.exists(v_8_based_report_file) and os.path.isfile(v_8_based_report_file): 708 # Generate v 8 benchmark data on the 1st, 11th, and 21st of each month.The testing at other times refers to 709 # these V 8 benchmark data 710 v_8_exec_time_dict = get_given_column_data(v_8_based_report_file, 7) 711 for key in v_8_exec_time_dict.keys(): 712 Constants.V_8_EXCUTE_TIME_DICT[key] = v_8_exec_time_dict[key] 713 return Constants.RET_OK 714 715 file_list = [] 716 for root, _, files in os.walk(jspath): 717 for file in files: 718 if not file.endswith('.js'): 719 continue 720 file_path = os.path.join(root, file) 721 file_list.append(file_path) 722 for _, file_path in enumerate(file_list): 723 results = file_path.split("/") 724 class_name = results[-2] 725 api_name = results[-1].split(".")[0] 726 js_case_name = '/'.join([class_name, api_name]) 727 728 v_8_exec_time_dict = run_v_8_single_js_case(file_path, '', js_case_name) 729 for key in v_8_exec_time_dict.keys(): 730 Constants.V_8_EXCUTE_TIME_DICT[key] = v_8_exec_time_dict[key] 731 732 return Constants.RET_OK 733 734 735def get_v_8_jitless_excute_times(jspath, v_8_based_report_file_path): 736 if os.path.exists(v_8_based_report_file_path) and os.path.isfile(v_8_based_report_file_path): 737 # Generate v 8 benchmark data on the 1st, 11th, and 21st of each month.The testing at other times refers to 738 # these V 8 benchmark data 739 v_8_exec_time_dict = get_given_column_data(v_8_based_report_file_path, 8) 740 for key in v_8_exec_time_dict.keys(): 741 Constants.V_8_JITLESS_EXCUTE_TIME_DICT[key] = v_8_exec_time_dict[key] 742 return Constants.RET_OK 743 744 file_list = [] 745 for root, _, files in os.walk(jspath): 746 for file in files: 747 if not file.endswith('.js'): 748 continue 749 file_path = os.path.join(root, file) 750 file_list.append(file_path) 751 752 for _, file_path in enumerate(file_list): 753 results = file_path.split("/") 754 class_name = results[-2] 755 api_name = results[-1].split(".")[0] 756 js_case_name = '/'.join([class_name, api_name]) 757 758 v_8_exec_time_dict = run_v_8_single_js_case(file_path, '--jitless', js_case_name) 759 for key in v_8_exec_time_dict.keys(): 760 Constants.V_8_JITLESS_EXCUTE_TIME_DICT[key] = v_8_exec_time_dict[key] 761 762 return Constants.RET_OK 763 764 765def get_config(): 766 config_json_path = os.path.join(Constants.CUR_PATH, "config.json") 767 with open(config_json_path, 'r', encoding='UTF-8') as f: 768 json_data = json.load(f) 769 770 Constants.ES2ABC_PATH = os.path.join(BINARY_PATH, json_data[Constants.VER_PLATFORM]["ES2ABC"]) 771 Constants.ARK_JS_VM_PATH = os.path.join(BINARY_PATH, json_data[Constants.VER_PLATFORM]["ETS_RUNTIME_PATH"], 772 "ark_js_vm") 773 Constants.ARK_AOT_COMPILER_PATH = os.path.join(BINARY_PATH, json_data[Constants.VER_PLATFORM]["ETS_RUNTIME_PATH"], 774 "ark_aot_compiler") 775 ETS_RUNTIME_PATH = os.path.join(BINARY_PATH, json_data[Constants.VER_PLATFORM]["ETS_RUNTIME_PATH"]) 776 ICU_PATH = os.path.join(BINARY_PATH, json_data[Constants.VER_PLATFORM]["ICU_PATH"]) 777 ZLIB_PATH = os.path.join(BINARY_PATH, json_data[Constants.VER_PLATFORM]["ZLIB_PATH"]) 778 LIB_PATH = os.path.join(BINARY_PATH, json_data[Constants.VER_PLATFORM]["LIB_PATH"]) 779 old_ld_library_path = os.environ.get('LD_LIBRARY_PATH', '') 780 os.environ['LD_LIBRARY_PATH'] = f'{ETS_RUNTIME_PATH}:' + f'{ICU_PATH}:' + f'{ZLIB_PATH}:' + f'{LIB_PATH}:'\ 781 + old_ld_library_path 782 783 784if __name__ == "__main__": 785 """ 786 command format: python3 run_js_test.py -bp /home/out -p /home/arkjs-perf-test/js-perf-test -o output_path 787 -v d_8_binary_path -e ver_platform 788 notes: all paths must be absolute path 789 """ 790 LOG_PATH = os.path.join(Constants.TMP_PATH, "test.log") 791 if os.path.exists(LOG_PATH): 792 os.remove(LOG_PATH) 793 logger = get_logger("jstest", LOG_PATH) 794 795 paras = get_args() 796 logger.info("execute arguments: %s", paras) 797 798 DETERIORATION_BOUNDARY_VALUE = paras.deterioration_boundary_value 799 BINARY_PATH = paras.binarypath 800 ICU_DATA_PATH = os.path.join(BINARY_PATH, "third_party/icu/ohos_icu4j/data/") 801 OUTPUT_PATH = Constants.CUR_PATH 802 Constants.V_8_ENGINED_PATH = paras.d_8_binary_path 803 Constants.VER_PLATFORM = paras.ver_platform 804 get_config() 805 806 if paras.output_folder_path is not None: 807 OUTPUT_PATH = paras.output_folder_path 808 809 if not os.path.exists(OUTPUT_PATH): 810 os.makedirs(OUTPUT_PATH) 811 812 today = datetime.date.today() 813 yesterday = today - datetime.timedelta(days=1) 814 TODAY_EXCEL_PATH = get_given_date_report_path(today) 815 YESTERDAY_EXCEL_PATH = get_given_date_report_path(yesterday) 816 817 if os.path.exists(TODAY_EXCEL_PATH): 818 os.remove(TODAY_EXCEL_PATH) 819 820 get_js_case_super_link_data(paras.jspath) 821 start_time = datetime.datetime.now(tz=datetime.timezone.utc) 822 init_report(TODAY_EXCEL_PATH) 823 get_yesterday_excute_times(YESTERDAY_EXCEL_PATH) 824 v_8_based_report_path = get_v_8_benchmark_daily_report_path() 825 get_v_8_excute_times(paras.jspath, v_8_based_report_path) 826 get_v_8_jitless_excute_times(paras.jspath, v_8_based_report_path) 827 828 run_via_ark(paras.jspath, TODAY_EXCEL_PATH) 829 end_time = datetime.datetime.now(tz=datetime.timezone.utc) 830 831 totol_time = u"%s" % (end_time - start_time) 832 append_summary_info(TODAY_EXCEL_PATH, totol_time) 833 834 logger.info("run js perf test finished. Please check details in report.") 835 shutil.rmtree(Constants.TMP_PATH) 836