1#!/usr/bin/env python3 2# coding: utf-8 3 4""" 5Copyright (c) 2023 Huawei Device Co., Ltd. 6Licensed under the Apache License, Version 2.0 (the "License"); 7you may not use this file except in compliance with the License. 8You may obtain a copy of the License at 9 10 http://www.apache.org/licenses/LICENSE-2.0 11 12Unless required by applicable law or agreed to in writing, software 13distributed under the License is distributed on an "AS IS" BASIS, 14WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15See the License for the specific language governing permissions and 16limitations under the License. 17 18Description: output test results 19""" 20 21import copy 22import logging 23import os 24import time 25 26import pandas 27 28import options 29 30full_compile_tests = ["full_compile", 31 "import_ordinary_ohpm_package", 32 "import_special_ohpm_package", 33 "import_static_library", 34 "import_share_library", 35 "import_so_file", 36 "has_syntax_error_in_js", 37 "use_normalize_ohmurl", 38 "module_name_is_inconsistent", 39 ] 40 41incremental_compile_tests = ["no_change", 42 "add_oneline", 43 "add_file", 44 "add_nonexistent_file", 45 "delete_file", 46 "modify_error_then_fix", 47 "add_error_page_then_fix", 48 "add_error_non_page_then_fix", 49 "build_entry_then_har", 50 "build_har_then_entry", 51 "build_entry_then_hsp", 52 "build_hsp_then_entry", 53 "build_hsp_then_ohos", 54 "build_entry_then_ohos", 55 "build_entry_then_preview_build", 56 "reverse_hap_mode", 57 "change_module_name", 58 "modify_sdk_version", 59 ] 60 61bytecode_har_compile_tests = ["build_bytecode_har", 62 "build_har_then_bytecode_har", 63 "import_bytecode_static_library", 64 ] 65 66external_compile_tests = ["import_external_share_library", 67 "import_external_static_library", 68 "full_compile_external_static_library", 69 "full_compile_external_share_library" 70 ] 71 72preview_compile_tests = ["preview_compile", 73 "build_entry_then_preview", 74 "build_modify_file_name", 75 "build_generate_sourcemap", 76 "tigger_incremental_build", 77 "has_arkui_error", 78 "sdk_path_has_special_char", 79 "modify_hello_world_then_fix" 80 ] 81 82other_tests = ["binary_consistency", 83 "break_continue_compile", 84 "compile_with_error", 85 "compile_with_exceed_length", 86 "ohos_test" 87 ] 88 89 90class TestResult: 91 def __init__(self): 92 self.passed = [] 93 self.failed = [] 94 self.time = 0.0 95 96 97def print_result(test_result, test_tasks): 98 logging.info("========================================") 99 logging.info("Test finished. The result is as following:") 100 logging.info("=====> Summary") 101 logging.info("Total test number: %s, took time: %.3f s", 102 len(test_tasks), test_result.time) 103 logging.info("Passed test number: %s", len(test_result.passed)) 104 logging.info("Failed test number: %s", len(test_result.failed)) 105 106 logging.info("=====> Detail Information") 107 logging.info("-----") 108 idx = 1 109 for task in test_tasks: 110 logging.info("task index: %d", idx) 111 idx = idx + 1 112 logging.info("task name: %s", task.name) 113 logging.info("task type: %s", task.type) 114 # print full compile result 115 logging.info("--full compilation result:") 116 for full_task in task.full_compilation_info.values(): 117 logging.info("full test: %s", full_task.name) 118 logging.info("debug: %s, abc_size(byte) %s, time(s) %s, error message: %s", 119 full_task.debug_info.result, 120 full_task.debug_info.abc_size, 121 full_task.debug_info.time, 122 full_task.debug_info.error_message) 123 logging.info("release: %s, abc_size(byte) %s, time(s) %s, error message: %s", 124 full_task.release_info.result, 125 full_task.release_info.abc_size, 126 full_task.release_info.time, 127 full_task.debug_info.error_message) 128 129 # print incremental compile result 130 logging.info("--incremental compilation result:") 131 for inc_task in task.incre_compilation_info.values(): 132 logging.info("incre test: %s", inc_task.name) 133 logging.info("debug: %s, abc_size(byte) %s, time(s) %s, error message: %s", 134 inc_task.debug_info.result, 135 inc_task.debug_info.abc_size, 136 inc_task.debug_info.time, 137 inc_task.debug_info.error_message) 138 logging.info("release: %s, abc_size(byte) %s, time(s) %s, error message: %s", 139 inc_task.release_info.result, 140 inc_task.release_info.abc_size, 141 inc_task.release_info.time, 142 inc_task.release_info.error_message) 143 144 # print bytecode har compile result 145 logging.info("--bytecode har compilation result:") 146 for byte_task in task.bytecode_har_compilation_info.values(): 147 logging.info("bytecode har test: %s", byte_task.name) 148 logging.info("debug: %s, abc_size(byte) %s, time(s) %s, error message: %s", 149 byte_task.debug_info.result, 150 byte_task.debug_info.abc_size, 151 byte_task.debug_info.time, 152 byte_task.debug_info.error_message) 153 logging.info("release: %s, abc_size(byte) %s, time(s) %s, error message: %s", 154 byte_task.release_info.result, 155 byte_task.release_info.abc_size, 156 byte_task.release_info.time, 157 byte_task.release_info.error_message) 158 159 # print external compile result 160 logging.info("--external compilation result:") 161 for external_task in task.external_compilation_info.values(): 162 logging.info("external test: %s", external_task.name) 163 logging.info("debug: %s, abc_size(byte) %s, time(s) %s, error message: %s", 164 external_task.debug_info.result, 165 external_task.debug_info.abc_size, 166 external_task.debug_info.time, 167 external_task.debug_info.error_message) 168 logging.info("release: %s, abc_size(byte) %s, time(s) %s, error message: %s", 169 external_task.release_info.result, 170 external_task.release_info.abc_size, 171 external_task.release_info.time, 172 external_task.release_info.error_message) 173 174 # print preview compile result 175 for name, task_info in task.preview_compilation_info.items(): 176 logging.info("--test name: %s", name) 177 logging.info("result: %s, error message: %s", 178 task_info.result, 179 task_info.error_message) 180 181 # print other tests result 182 for name, task_info in task.other_tests.items(): 183 logging.info("--test name: %s", name) 184 logging.info("result: %s, error message: %s", 185 task_info.result, 186 task_info.error_message) 187 188 logging.info("-----") 189 logging.info("========================================") 190 191 192def is_compilation_passed(task_info, compile_mode): 193 if options.arguments.compile_mode not in ['all', compile_mode]: 194 return True 195 196 if len(task_info) == 0: 197 return False 198 199 passed_debug = True 200 passed_release = True 201 for task_name, inc_task in task_info.items(): 202 if options.arguments.hap_mode in ['all', 'release']: 203 if inc_task.release_info.result == options.TaskResult.undefined: 204 continue 205 passed_release = passed_release and inc_task.release_info.result == options.TaskResult.passed 206 if options.arguments.hap_mode == ['all', 'debug']: 207 if inc_task.debug_info.result == options.TaskResult.undefined: 208 continue 209 passed_debug = passed_debug and inc_task.debug_info.result == options.TaskResult.passed 210 211 return passed_debug and passed_release 212 213 214def is_task_passed(task): 215 passed = is_compilation_passed(task.full_compilation_info, 'full') and \ 216 is_compilation_passed(task.incre_compilation_info, 'incremental') and \ 217 is_compilation_passed(task.bytecode_har_compilation_info, 'bytecode_har') and \ 218 is_compilation_passed(task.external_compilation_info, 'external') 219 220 for test in task.preview_compilation_info.values(): 221 passed = passed and (test.result == options.TaskResult.passed) 222 223 for test in task.other_tests.values(): 224 passed = passed and (test.result == options.TaskResult.passed) 225 226 return passed 227 228 229def collect_result(test_result, test_tasks, start_time): 230 for task in test_tasks: 231 if not is_task_passed(task): 232 test_result.failed.append(task) 233 else: 234 test_result.passed.append(task) 235 236 end_time = time.time() 237 test_result.time = round(end_time - start_time, 3) 238 239 240def get_result_symbol(result_type): 241 if result_type == options.TaskResult.passed: 242 return '√' 243 elif result_type == options.TaskResult.failed: 244 return '×' 245 else: 246 return '-' 247 248 249def generate_summary_data(test_result, test_tasks): 250 # collect summary data 251 passed_task_name_list = [] 252 for task in test_result.passed: 253 passed_task_name_list.append(task.name) 254 failed_task_name_list = [] 255 for task in test_result.failed: 256 failed_task_name_list.append(task.name) 257 258 summary_data = { 259 'Total Test Number': len(test_tasks), 260 'Passed Test Number': len(test_result.passed), 261 'Failed Test Number': len(test_result.failed), 262 'Passed Tests': ','.join(passed_task_name_list), 263 'Failed Tests': ','.join(failed_task_name_list), 264 'Test Took Time(s)': test_result.time 265 } 266 267 return summary_data 268 269 270def generate_detail_data(test_tasks): 271 time_size_data = [] 272 result_data = [] 273 274 idx = 0 275 for task in test_tasks: 276 idx += 1 277 task_time_size_data = { 278 'Task Index': idx, 279 'Task Name': task.name 280 } 281 task_result_data = copy.deepcopy(task_time_size_data) 282 task_result_data['Task Type'] = ','.join(task.type) 283 284 full_compilation_debug, full_compilation_release = get_full_build_test_result(task, task_result_data, 285 task_time_size_data) 286 287 get_incremental_build_test_result(task, task_result_data, task_time_size_data) 288 289 get_bytecode_har_build_test_result(task, task_result_data, task_time_size_data) 290 291 get_external_build_test_result(task, task_result_data, task_time_size_data) 292 293 get_preview_build_test_result(task, task_result_data) 294 295 get_other_test_result(task, task_result_data) 296 297 task_time_size_data['[Abc Size(byte)]\n[Debug]'] = full_compilation_debug.abc_size 298 task_time_size_data['[Abc Size(byte)]\n[Release]'] = full_compilation_release.abc_size 299 time_size_data.append(task_time_size_data) 300 result_data.append(task_result_data) 301 302 detail_data = { 303 'result_data': result_data, 304 'time_size_data': time_size_data 305 } 306 return detail_data 307 308 309def get_test_tesult(test, task_result_data, compilation_info): 310 debug_result = options.TaskResult.undefined 311 debug_runtime_result = options.TaskResult.undefined 312 release_result = options.TaskResult.undefined 313 release_runtime_result = options.TaskResult.undefined 314 if test in compilation_info.keys(): 315 task_info = compilation_info[test] 316 debug_result = task_info.debug_info.result 317 debug_runtime_result = task_info.debug_info.runtime_result 318 release_result = task_info.release_info.result 319 release_runtime_result = task_info.release_info.runtime_result 320 task_result_data[f'[Debug]\n{test}'] = get_result_symbol( 321 debug_result) 322 task_result_data[f'[Debug-runtime]\n{test}'] = get_result_symbol( 323 debug_runtime_result) 324 task_result_data[f'[Release]\n{test}'] = get_result_symbol( 325 release_result) 326 task_result_data[f'[Release-runtime]\n{test}'] = get_result_symbol( 327 release_runtime_result) 328 329 330def get_full_build_test_result(task, task_result_data, task_time_size_data): 331 for test in full_compile_tests: 332 get_test_tesult(test, task_result_data, task.full_compilation_info) 333 334 if test == 'full_compile': 335 debug_test_time = 0 336 release_test_time = 0 337 if test in task.full_compilation_info.keys(): 338 full_task_info = task.full_compilation_info[test] 339 debug_test_time = full_task_info.debug_info.time 340 release_test_time = full_task_info.release_info.time 341 342 task_time_size_data[ 343 '[Full Compilation]\n[Debug]\n[Compilation Time(s)]'] = debug_test_time 344 task_time_size_data[ 345 '[Full Compilation]\n[Release]\n[Compilation Time(s)]'] = release_test_time 346 347 return (task.full_compilation_info['full_compile'].debug_info, 348 task.full_compilation_info['full_compile'].release_info) 349 350 351def get_incremental_build_test_result(task, task_result_data, task_time_size_data): 352 for test in incremental_compile_tests: 353 get_test_tesult(test, task_result_data, task.incre_compilation_info) 354 355 if test == 'add_oneline': 356 debug_test_time = 0 357 release_test_time = 0 358 if test in task.incre_compilation_info.keys(): 359 inc_task_info = task.incre_compilation_info[test] 360 debug_test_time = inc_task_info.debug_info.time 361 release_test_time = inc_task_info.release_info.time 362 363 task_time_size_data[ 364 '[Incremental Compilation]\n[Debug]\n[Compilation Time(s)]'] = debug_test_time 365 task_time_size_data[ 366 '[Incremental Compilation]\n[Release]\n[Compilation Time(s)]'] = release_test_time 367 368 369def get_bytecode_har_build_test_result(task, task_result_data, task_time_size_data): 370 for test in bytecode_har_compile_tests: 371 get_test_tesult(test, task_result_data, task.bytecode_har_compilation_info) 372 373 if test == 'build_bytecode_har': 374 debug_test_time = 0 375 release_test_time = 0 376 if test in task.bytecode_har_compilation_info.keys(): 377 inc_task_info = task.bytecode_har_compilation_info[test] 378 debug_test_time = inc_task_info.debug_info.time 379 release_test_time = inc_task_info.release_info.time 380 381 task_time_size_data[ 382 '[Bytecode Har Compilation]\n[Debug]\n[Compilation Time(s)]'] = debug_test_time 383 task_time_size_data[ 384 '[Bytecode Har Compilation]\n[Release]\n[Compilation Time(s)]'] = release_test_time 385 386 387def get_external_build_test_result(task, task_result_data, task_time_size_data): 388 for test in external_compile_tests: 389 get_test_tesult(test, task_result_data, task.external_compilation_info) 390 391 if test == 'import_external_share_library': 392 debug_test_time = 0 393 release_test_time = 0 394 if test in task.external_compilation_info.keys(): 395 inc_task_info = task.external_compilation_info[test] 396 debug_test_time = inc_task_info.debug_info.time 397 release_test_time = inc_task_info.release_info.time 398 399 task_time_size_data[ 400 '[External Compilation]\n[Debug]\n[Compilation Time(s)]'] = debug_test_time 401 task_time_size_data[ 402 '[External Compilation]\n[Release]\n[Compilation Time(s)]'] = release_test_time 403 404 405def get_preview_build_test_result(task, task_result_data): 406 for test in preview_compile_tests: 407 result = options.TaskResult.undefined 408 runtime_result = options.TaskResult.undefined 409 if test in task.preview_compilation_info.keys(): 410 task_info = task.preview_compilation_info[test] 411 result = task_info.result 412 runtime_result = task_info.runtime_result 413 414 task_result_data[f'{test}'] = get_result_symbol(result) 415 task_result_data[f'{test}-runtime'] = get_result_symbol(runtime_result) 416 417 418def get_other_test_result(task, task_result_data): 419 for test in other_tests: 420 result = options.TaskResult.undefined 421 runtime_result = options.TaskResult.undefined 422 if test in task.other_tests.keys(): 423 task_info = task.other_tests[test] 424 result = task_info.result 425 runtime_result = task_info.runtime_result 426 task_result_data[f'{test}'] = get_result_symbol(result) 427 task_result_data[f'{test}-runtime'] = get_result_symbol(runtime_result) 428 429 430def rotate_data(df): 431 num_rows, num_cols = df.shape 432 rotated_df = pandas.DataFrame(columns=range(num_rows), index=range(num_cols)) 433 for i in range(num_rows): 434 for j in range(num_cols): 435 rotated_df.iloc[j, i] = df.iloc[i, j] 436 return rotated_df 437 438 439def get_merge_data(rotated_df): 440 data = rotated_df.iloc[3:, :].values.tolist() 441 merged_data = [] 442 for i in range(0, len(data) - 1, 2): 443 row = [value for sublist in zip(data[i], data[i + 1]) for value in sublist] 444 merged_data.append(row) 445 return merged_data 446 447 448def generate_content_section(section_title, tests, start_index, merged_data): 449 section_content = f'<tr><th rowspan={len(tests) * 2}>{section_title}</th>' 450 for index, item in enumerate(tests): 451 debug_result = ''.join([f'<th>{column}</th>' for column in merged_data[start_index]]) 452 section_content = ''.join( 453 [section_content, f'<th rowspan="2">{item}</th><th>[Debug]</th>', debug_result, '</tr>']) 454 release_result = ''.join([f'<th>{column}</th>' for column in merged_data[start_index + 1]]) 455 section_content = ''.join([section_content, '<tr><th>[Release]</th>', release_result, '</tr>']) 456 start_index += 2 457 return section_content, start_index 458 459 460def get_result_table_content(result_df_rotate): 461 start_index = 0 462 merged_data = get_merge_data(result_df_rotate) 463 # Full Compilation section 464 full_compile_section, start_index = generate_content_section("Full Compilation", full_compile_tests, start_index, 465 merged_data) 466 content = full_compile_section 467 468 # Incremental Compilation section 469 incremental_compile_section, start_index = generate_content_section("Incremental Compilation", 470 incremental_compile_tests, start_index, 471 merged_data) 472 content += incremental_compile_section 473 474 # Bytecode Har Compilation section 475 bytecode_har_compile_section, start_index = generate_content_section("Bytecode Har Compilation", 476 bytecode_har_compile_tests, start_index, 477 merged_data) 478 content += bytecode_har_compile_section 479 480 # External Compilation section 481 external_compile_section, start_index = generate_content_section("External Compilation", 482 external_compile_tests, start_index, 483 merged_data) 484 content += external_compile_section 485 486 content += f'<tr><th colspan=2 rowspan={len(preview_compile_tests)}>Preview Compilation</th>' 487 for index, item in enumerate(preview_compile_tests): 488 preview_result = ''.join([f'<th>{column}</th>' for column in merged_data[start_index]]) 489 content = ''.join([content, f'<th>{item}</th>', preview_result, '</tr>']) 490 start_index = start_index + 1 491 492 # Other Test section 493 content += f'<tr><th colspan=2 rowspan={len(other_tests)}>Other Tests</th>' 494 for index, item in enumerate(other_tests): 495 other_result = ''.join([f'<th>{column}</th>' for column in merged_data[start_index]]) 496 content = ''.join([content, f'<th>{item}</th>', other_result, '</tr>']) 497 start_index = start_index + 1 498 499 return content 500 501 502def generate_data_html(summary_data, detail_data): 503 # summary table 504 key_value_pairs = [ 505 f'<tr><td>{key}</td><td>{value}</td></tr>' for key, value in summary_data.items()] 506 summary_table_content = ''.join(key_value_pairs) 507 summary_table = f'<table id=sdk>{summary_table_content}</table>' 508 509 # time and size table 510 time_size_data = detail_data.get('time_size_data') 511 time_size_df = pandas.DataFrame(time_size_data) 512 513 time_size_table_header = '<tr>' + \ 514 ''.join( 515 [f'<th rowspan="2">{column}</th>' for column in time_size_df.columns[:2]]) 516 time_size_table_header += '<th colspan="2">Full Compilation Time(s)</th>' + \ 517 f'<th colspan="2">Incremental Compilation Time(s)</th>' + \ 518 f'<th colspan="2">Bytecode Har Compilation Time(s)</th>' + \ 519 f'<th colspan="2">External Compilation Time(s)</th>' + \ 520 f'<th colspan="2">Abc Size(byte)</th></tr>' 521 time_size_table_sub_header = '<tr>' + \ 522 f'<th>[Debug]</th><th>[Release]</th>' * 5 + '</tr>' 523 524 time_size_table_content = ''.join([ 525 '<tr>' + ''.join([f'<td>{value}</td>' for _, 526 value in row.items()]) + '</tr>' 527 for _, row in time_size_df.iterrows() 528 ]) 529 time_size_table = f'<table id=sdk> \ 530 {time_size_table_header}{time_size_table_sub_header}{time_size_table_content}</table>' 531 532 # result table 533 result_data = detail_data.get('result_data') 534 result_df = pandas.DataFrame(result_data) 535 result_df_rotate = rotate_data(result_df) 536 537 result_table_header = '<tr><th colspan="3">Task Index</th>' + \ 538 ''.join( 539 [f'<th colspan="2">{column}</th>' for column in result_df.iloc[:, 0].tolist()]) + '</tr>' 540 result_table_header += '<tr><th colspan="3">Task Name</th>' + \ 541 ''.join( 542 [f'<th colspan="2">{column}</th>' for column in result_df.iloc[:, 1].tolist()]) + '</tr>' 543 result_table_header += '<tr><th colspan="3">Task Type</th>' + \ 544 ''.join( 545 [f'<th colspan="2">{column}</th>' for column in result_df.iloc[:, 2].tolist()]) + '</tr>' 546 result_table_sub_header = f"<tr><th colspan=3>Build && Run </th>{'<th>[build]</th><th>[runtime]</th>' * result_df.shape[0]}</tr>" 547 result_table_content = get_result_table_content(result_df_rotate) 548 549 result_table = f'<table id=sdk> \ 550 {result_table_header}{result_table_sub_header}{result_table_content}</table>' 551 552 return summary_table, time_size_table, result_table 553 554 555def get_html_style(): 556 html_style = ''' 557 #sdk body { 558 font-family: Arial, sans-serif; 559 margin: 20px; 560 } 561 #sdk h2 { 562 color: #333; 563 } 564 #sdk { 565 border-collapse: collapse; 566 width: 100%; 567 margin-bottom: 20px; 568 } 569 #sdk th, #sdk td { 570 padding: 8px; 571 border: 1px solid #ddd; 572 } 573 #sdk th { 574 background-color: #f2f2f2; 575 font-weight: bold; 576 } 577 #sdk tr:nth-child(odd) { 578 background-color: #f9f9f9; 579 } 580 ''' 581 return html_style 582 583 584def generate_report_html(summary_data, detail_data): 585 [summary_table, time_size_table, result_table] = generate_data_html( 586 summary_data, detail_data) 587 html_style = get_html_style() 588 589 html_content = f''' 590 <html> 591 <head> 592 <meta http-equiv="Content-Type" content="text/html; charset=utf-8"> 593 <style> 594 {html_style} 595 </style> 596 </head> 597 <body> 598 <h2>SDK Test Results</h2> 599 <h3>Summary</h3> 600 {summary_table} 601 <h3>Detail Information</h3> 602 <h4>Test Result</h4> 603 {result_table} 604 <h4>Compilation Time And Abc Size</h4> 605 {time_size_table} 606 <p> 607 Notes:<br> 608 1. Incremental compilation time refers to add-one line incremental compile.<br> 609 2. For details compile output or error message during compile, please refer to attachment of log file.<br> 610 3. For sdk commit tags, please refer to attachment of manifest file(to be added). 611 </p> 612 </body> 613 </html> 614 ''' 615 616 daily_report_file = options.configs.get('output_html_file') 617 with open(daily_report_file, 'w', encoding='utf-8') as report: 618 report.write(html_content) 619 620 621def generate_log_file(): 622 logger = logging.getLogger() 623 if not hasattr(logger.handlers[0], 'baseFilename'): 624 return 625 log_file = logger.handlers[0].baseFilename 626 logger.handlers[0].close() 627 output_log_file = options.configs.get('log_file') 628 os.rename(log_file, output_log_file) 629 630 631def generate_result_reports(test_result, test_tasks): 632 summary_data = generate_summary_data(test_result, test_tasks) 633 detail_data = generate_detail_data(test_tasks) 634 generate_report_html(summary_data, detail_data) 635 generate_log_file() 636 637 638def process_test_result(test_tasks, start_time): 639 test_result = TestResult() 640 641 collect_result(test_result, test_tasks, start_time) 642 print_result(test_result, test_tasks) 643 generate_result_reports(test_result, test_tasks) 644 645