1#!/usr/bin/env python 2# coding=utf-8 3############################################## 4# Copyright (c) 2021-2022 Huawei Device Co., Ltd. 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16############################################## 17import json 18import os.path 19import random 20import sys 21import time 22import ast 23import re 24from subprocess import Popen 25from subprocess import PIPE 26 27 28def print_help(): 29 content = "\n" \ 30 "Usage: signtool.jar -scope <simple|all|`component`> -n <round, default 1> <--random>\n" \ 31 " signtool.jar : Main progress jar file\n" \ 32 " component: \n" \ 33 " --random: random test, default false" \ 34 "\n" \ 35 "Example: \n" \ 36 " signtool.jar \n" \ 37 " signtool.jar -runtest\n" \ 38 " signtool.jar -scope all -n 1000\n" \ 39 " signtool.jar -scope generate-profile-cert\n" \ 40 " signtool.jar -n 50 --random\n" \ 41 "\n" 42 43 print(content) 44 pass 45 46 47random_scope = { 48 'generate-keypair': { 49 'required': { 50 'keyAlias': 'oh-app1-key-v1', 51 'keyAlg': ["RSA", "ECC"], 52 'keySize': ["2048", "3072", "4096", "NIST-P-256", "NIST-P-384"], 53 'keystoreFile': ['ohtest.jks', 'ohtest.p12'] 54 }, 55 'others': { 56 'keyPwd': '123456', 57 'keystorePwd': '123456' 58 } 59 }, 60 'generate-csr': { 61 'required': { 62 'keyAlias': 'oh-app1-key-v1', 63 'signAlg': ["SHA256withRSA", "SHA384withRSA", "SHA256withECDSA", "SHA384withECDSA"], 64 'subject': "C=CN,O=OpenHarmony,OU=OpenHarmony Community,CN=App1 Release", 65 'keystoreFile': ['ohtest.jks', 'ohtest.p12'], 66 'outFile': 'oh-app1-key-v1.csr' 67 }, 68 'others': { 69 'keyPwd': '123456', 70 'keystorePwd': '132456' 71 } 72 }, 73 'generate-ca': { 74 'required': { 75 'keyAlias': ['oh-ca-key-v1', "oh-app-sign-srv-ca-key-v1"], 76 'signAlg': ["SHA256withRSA", "SHA384withRSA", "SHA256withECDSA", "SHA384withECDSA"], 77 'keyAlg': ['RSA', 'ECC'], 78 'keySize': ["2048", "3072", "4096", "NIST-P-256", "NIST-P-384"], 79 'subject': ["C=CN,O=OpenHarmony,OU=OpenHarmony Community,CN=App1 Release", 80 "C=CN,O=OpenHarmony,OU=OpenHarmony Community,CN=Root CA"], 81 'keystoreFile': ['ohtest.jks', 'ohtest.p12'], 82 'outFile': 'app-sign-srv-ca.cer' 83 }, 84 'others': { 85 'keyPwd': '123456', 86 'keystorePwd': '132456', 87 'issuer': 'C=CN,O=OpenHarmony,OU=OpenHarmony Community,CN=Application Signature Service CA', 88 'issuerKeyAlias': 'oh-app-sign-srv-ca-key-v1', 89 'issuerKeyPwd': '123456', 90 'validity': '365', 91 'basicConstraintsPathLen': '2' 92 } 93 }, 94 'generate-cert': { 95 'required': { 96 'keyAlias': ['oh-sub-key-v1', 'oh-ca-key-v1'], 97 'signAlg': ["SHA256withRSA", "SHA384withRSA", "SHA256withECDSA", "SHA384withECDSA"], 98 'subject': "C=CN,O=OpenHarmony,OU=OpenHarmony Community,CN=App1 Release", 99 'issuer': 'C=CN,O=OpenHarmony,OU=OpenHarmony Community,CN=Application Signature Service CA', 100 'issuerKeyAlias': 'oh-ca-key-v1', 101 'extKeyUsage': 'codeSignature', 102 'keyUsage': ['digitalSignature,nonRepudiation,keyEncipherment', 103 'dataEncipherment,keyAgreement, certificateSignature, crlSignature', 104 'encipherOnly, encipherOnly'], 105 'keystoreFile': ['ohtest.jks', 'ohtest.p12'], 106 'outFile': 'app1.cer' 107 }, 108 'others': { 109 'extKeyUsage': ['serverAuthentication', 'clientAuthentication', 'emailProtection'], 110 'extKeyUsageCritical': ['false', 'true'], 111 'keyUsageCritical': ['false', 'true'], 112 'issuerKeyPwd': '123456', 113 'keyPwd': '123456', 114 'validity': '365', 115 'keystorePwd': '123456' 116 } 117 } 118} 119 120simple_scope = { 121 'generate-keypair': [ 122 'generate-keypair -keyAlias "oh-app1-key-v1" -keyPwd 123456 -keyAlg ECC -keySize NIST-P-256 ' 123 '-keystoreFile "./test1/ohtest.jks" -keystorePwd 123456', 124 125 'generate-keypair -keyAlias "oh-profile1-key-v1" -keyPwd 123456 -keyAlg ECC -keySize NIST-P-384 ' 126 '-keystoreFile "./test1/ohtest.jks" -keystorePwd 123456', 127 128 'generate-keypair -keyAlias "oh-app2-key-v1" -keyPwd 123456 -keyAlg RSA -keySize 2048 ' 129 '-keystoreFile "./test2/ohtest.p12" -keystorePwd 123456', 130 131 'generate-keypair -keyAlias "oh-profile2-key-v1" -keyPwd 123456 -keyAlg RSA -keySize 4096 ' 132 '-keystoreFile "./test2/ohtest.p12" -keystorePwd 123456' 133 ], 134 'generate-csr': [ 135 'generate-csr -keyAlias "oh-app1-key-v1" -keyPwd 123456 -subject ' 136 '"C=CN,O=OpenHarmony,OU=OpenHarmony Community,CN=App1 Release" -signAlg SHA256withECDSA ' 137 '-keystoreFile "./test1/ohtest.jks" -keystorePwd 123456 -outFile "./test1/oh-app1-key-v1.csr"', 138 139 'generate-csr -keyAlias "oh-profile2-key-v1" -keyPwd 123456 -subject ' 140 '"C=CN,O=OpenHarmony,OU=OpenHarmony Community,CN=App1 Release" -signAlg SHA256withRSA ' 141 '-keystoreFile "./test2/ohtest.p12" -keystorePwd 123456 -outFile "./test2/oh-profile2-key-v1.csr"' 142 ], 143 'generate-ca': [ 144 # Root CA in ohtest.jks 145 'generate-ca -keyAlias "oh-root-ca-key-v1" -subject "C=CN,O=OpenHarmony,OU=OpenHarmony Community,CN=Root CA" ' 146 '-validity 365 -signAlg SHA384withECDSA -keystoreFile "./test1/ohtest.jks" -keystorePwd 123456 ' 147 '-outFile "./test1/root-ca1.cer" -keyAlg ECC -keySize NIST-P-256', 148 # Sub app cert in ohtest.jks 149 'generate-ca -keyAlias "oh-app-sign-srv-ca-key-v1" -issuer "C=CN,O=OpenHarmony,OU=OpenHarmony Community,' 150 'CN=Root CA" -issuerKeyAlias "oh-root-ca-key-v1" -subject "C=CN,O=OpenHarmony,OU=OpenHarmony Community,' 151 'CN= Application Signature Service CA" -validity 365 -signAlg SHA384withECDSA ' 152 '-keystoreFile "./test1/ohtest.jks" -keystorePwd 123456 -outFile "./test1/app-sign-srv-ca1.cer" ' 153 '-keyAlg ECC -keySize NIST-P-256', 154 # Sub profile cert in ohtest.jks 155 'generate-ca -keyAlias "oh-profile-sign-srv-ca-key-v1" -issuer "C=CN,O=OpenHarmony,OU=OpenHarmony Community,' 156 'CN=Root CA" -issuerKeyAlias "oh-root-ca-key-v1" -subject "C=CN,O=OpenHarmony,OU=OpenHarmony Community,' 157 'CN= Profile Signature Service CA" -validity 365 -signAlg SHA384withECDSA -keystoreFile "./test1/ohtest.jks" ' 158 '-keystorePwd 123456 -outFile "./test1/profile-sign-srv-ca1.cer" -keyAlg ECC -keySize NIST-P-384', 159 160 # Root CA in ohtest.p12 161 'generate-ca -keyAlias "oh-root-ca2-key-v1" -subject "C=CN,O=OpenHarmony,OU=OpenHarmony Community,CN=Root CA" ' 162 '-validity 365 -signAlg SHA384withRSA -keystoreFile "./test2/ohtest.p12" -keystorePwd 123456 ' 163 '-outFile "./test2/root-ca2.cer" -keyAlg RSA -keySize 2048', 164 # Sub app cert in ohtest.p12 165 'generate-ca -keyAlias "oh-app-sign-srv-ca2-key-v1" -issuer "C=CN,O=OpenHarmony,OU=OpenHarmony Community,' 166 'CN=Root CA" -issuerKeyAlias "oh-root-ca2-key-v1" -subject "C=CN,O=OpenHarmony,OU=OpenHarmony Community,' 167 'CN= Application Signature Service CA" -validity 365 -signAlg SHA384withRSA ' 168 '-keystoreFile "./test2/ohtest.p12" -keystorePwd 123456 -outFile "./test2/app-sign-srv-ca2.cer" ' 169 '-keyAlg RSA -keySize 2048', 170 # Sub profile cert in ohtest.p12 171 'generate-ca -keyAlias "oh-profile-sign-srv-ca2-key-v1" -issuer "C=CN,O=OpenHarmony,OU=OpenHarmony Community,' 172 'CN=Root CA" -issuerKeyAlias "oh-root-ca2-key-v1" -subject "C=CN,O=OpenHarmony,OU=OpenHarmony Community,' 173 'CN= Profile Signature Service CA" -validity 365 -signAlg SHA384withRSA -keystoreFile "./test2/ohtest.p12" ' 174 '-keystorePwd 123456 -outFile "./test2/profile-sign-srv-ca2.cer" -keyAlg RSA -keySize 2048' 175 ], 176 'generate-cert': [ 177 # Self-Definition cert - Root CA 178 'generate-cert -keyAlias "oh-app1-key-v1" -issuer "C=CN,O=OpenHarmony,OU=OpenHarmony Community,CN=Root CA" ' 179 '-issuerKeyAlias "oh-app1-key-v1" -issuerKeyPwd 123456 ' 180 '-subject "C=CN,O=OpenHarmony,OU=OpenHarmony Community,CN=Root CA" -validity 365 ' 181 '-keyUsage "certificateSignature, crlSignature" -signAlg SHA256withECDSA ' 182 '-keystoreFile "./test1/ohtest.jks" -keystorePwd 123456 -outFile "./test1/single-root.cer" -keyPwd 123456', 183 # Self-definition sign cert - app cert 184 'generate-cert -keyAlias "oh-app1-key-v1" -issuer "C=CN,O=OpenHarmony,OU=OpenHarmony Community,' 185 'CN=Application Signature Service CA" -issuerKeyAlias "oh-app-sign-srv-ca-key-v1" ' 186 '-subject "C=CN,O=OpenHarmony,OU=OpenHarmony Community,CN=App1 Release" -validity 365 ' 187 '-keyUsage digitalSignature -extKeyUsage codeSignature -signAlg SHA256withECDSA ' 188 '-keystoreFile "./test1/ohtest.jks" -keystorePwd 123456 -outFile "./test1/single-app1.cer" -keyPwd 123456' 189 ], 190 'generate-app-cert': [ 191 # App sign cert via ohtest.jks 192 'generate-app-cert -keyAlias "oh-app1-key-v1" -issuer "C=CN,O=OpenHarmony,OU=OpenHarmony Community,' 193 'CN=Application Signature Service CA" -issuerKeyAlias "oh-app-sign-srv-ca-key-v1" -subject ' 194 '"C=CN,O=OpenHarmony,OU=OpenHarmony Community,CN=App1 Release" -validity 365 -signAlg SHA256withECDSA ' 195 '-keystoreFile "./test1/ohtest.jks" -keystorePwd 123456 -outFile "./test1/app1.cer" -keyPwd 123456 ' 196 '-outForm cert ', 197 # App sign cert chain via ohtest.jks 198 'generate-app-cert -keyAlias "oh-app1-key-v1" -issuer "C=CN,O=OpenHarmony,OU=OpenHarmony Community,' 199 'CN=Application Signature Service CA" -issuerKeyAlias "oh-app-sign-srv-ca-key-v1" -subject ' 200 '"C=CN,O=OpenHarmony,OU=OpenHarmony Community,CN=App1 Release" -validity 365 -signAlg SHA256withECDSA ' 201 '-keystoreFile "./test1/ohtest.jks" -keystorePwd 123456 -outFile "./test1/app-release1.pem" ' 202 '-subCaCertFile ./test1/app-sign-srv-ca1.cer -outForm certChain ' 203 '-rootCaCertFile ./test1/root-ca1.cer -keyPwd 123456', 204 # App sign cert via ohtest.p12 205 'generate-app-cert -keyAlias "oh-app2-key-v1" -issuer "C=CN,O=OpenHarmony,OU=OpenHarmony Community,' 206 'CN=Application Signature Service CA" -issuerKeyAlias "oh-app-sign-srv-ca2-key-v1" -subject ' 207 '"C=CN,O=OpenHarmony,OU=OpenHarmony Community,CN=App1 Release" -validity 365 -signAlg SHA256withECDSA ' 208 '-keystoreFile "./test2/ohtest.p12" -keystorePwd 123456 -outFile "./test2/app2.cer" -keyPwd 123456 ' 209 '-outForm cert ', 210 # App sign cert chain via ohtest.p12 211 'generate-app-cert -keyAlias "oh-app2-key-v1" -issuer "C=CN,O=OpenHarmony,OU=OpenHarmony Community,' 212 'CN=Application Signature Service CA" -issuerKeyAlias "oh-app-sign-srv-ca2-key-v1" -subject ' 213 '"C=CN,O=OpenHarmony,OU=OpenHarmony Community,CN=App1 Release" -validity 365 -signAlg SHA256withECDSA ' 214 '-keystoreFile "./test2/ohtest.p12" -keystorePwd 123456 -outFile "./test2/app-release2.pem" ' 215 '-subCaCertFile ./test2/app-sign-srv-ca2.cer -outForm certChain ' 216 '-rootCaCertFile ./test2/root-ca2.cer -keyPwd 123456' 217 ], 218 'generate-profile-cert': [ 219 # Profile sign cert via ohtest.jks 220 'generate-profile-cert -keyAlias "oh-profile1-key-v1" -issuer "C=CN,O=OpenHarmony,OU=OpenHarmony Community,' 221 'CN=Profile Signature Service CA" -issuerKeyAlias "oh-profile-sign-srv-ca-key-v1" ' 222 '-subject "C=CN,O=OpenHarmony,OU=OpenHarmony Community,CN=Profile1 Release" ' 223 '-validity 365 -signAlg SHA256withECDSA -keystoreFile "./test1/ohtest.jks" ' 224 '-keystorePwd 123456 -outFile "./test1/profile1.cer" -keyPwd 123456 -outForm cert ', 225 # Profile sign cert chain via ohtest.jks 226 'generate-profile-cert -keyAlias "oh-profile1-key-v1" -issuer "C=CN,O=OpenHarmony,OU=OpenHarmony Community,' 227 'CN=Profile Signature Service CA" -issuerKeyAlias "oh-profile-sign-srv-ca-key-v1" ' 228 '-subject "C=CN,O=OpenHarmony,OU=OpenHarmony Community,CN=Profile1 Release" -validity 365 ' 229 '-signAlg SHA256withECDSA -keystoreFile "./test1/ohtest.jks" ' 230 '-keystorePwd 123456 -outFile "./test1/profile-release1.pem" ' 231 '-subCaCertFile "./test1/profile-sign-srv-ca1.cer" -outForm certChain ' 232 '-rootCaCertFile "./test1/root-ca1.cer" -keyPwd 123456', 233 # Profile sign cert via ohtest.p12 234 'generate-profile-cert -keyAlias "oh-profile2-key-v1" -issuer "C=CN,O=OpenHarmony,OU=OpenHarmony Community,' 235 'CN=Profile Signature Service CA" -issuerKeyAlias "oh-profile-sign-srv-ca2-key-v1" ' 236 '-subject "C=CN,O=OpenHarmony,OU=OpenHarmony Community,CN=Profile2 Release" ' 237 '-validity 365 -signAlg SHA256withECDSA -keystoreFile "./test2/ohtest.p12" ' 238 '-keystorePwd 123456 -outFile "./test2/profile2.cer" -keyPwd 123456 -outForm cert ', 239 # Profile sign cert chain via ohtest.p12 240 'generate-profile-cert -keyAlias "oh-profile2-key-v1" -issuer "C=CN,O=OpenHarmony,OU=OpenHarmony Community,' 241 'CN=Profile Signature Service CA" -issuerKeyAlias "oh-profile-sign-srv-ca2-key-v1" ' 242 '-subject "C=CN,O=OpenHarmony,OU=OpenHarmony Community,CN=Profile2 Release" -validity 365 ' 243 '-signAlg SHA256withECDSA -keystoreFile "./test2/ohtest.p12" ' 244 '-keystorePwd 123456 -outFile "./test2/profile-release2.pem" -subCaCertFile "./test2/profile-sign-srv-ca2.cer" ' 245 '-outForm certChain -rootCaCertFile "./test2/root-ca2.cer" -keyPwd 123456' 246 ], 247 'sign-profile': [ 248 'sign-profile -mode localSign -keyAlias "oh-profile1-key-v1" -profileCertFile "./test1/profile-release1.pem" ' 249 '-inFile "profile.json" -signAlg SHA256withECDSA -keystoreFile "./test1/ohtest.jks" -keystorePwd 123456 ' 250 '-outFile "./test1/app1-profile1.p7b" -keyPwd 123456' 251 ], 252 'verify-profile': [ 253 'verify-profile -inFile "./test1/app1-profile1.p7b"' 254 ] 255} 256 257 258def get_test_scope_from_file(): 259 with open('commands.config', 'r', encoding='utf-8') as f: 260 content = f.read() 261 return ast.literal_eval(content) 262 263 264test_scope = get_test_scope_from_file() 265 266test_result = {} 267 268 269def run_target(case, cmd): 270 if not test_result.get(case, None): 271 test_result[case] = {'times': 0, 'total_cost': 0, 'success': 0, 'fail': 0} 272 273 case_result = test_result.get(case) 274 case_result['times'] = case_result['times'] + 1 275 start = time.time() 276 277 command = Popen(cmd, stdout=PIPE, stderr=PIPE, stdin=PIPE, shell=False) 278 279 out = command.stdout.readlines() 280 with open("log.txt", mode='a+', encoding='utf-8') as f: 281 if len(out) > 0: 282 f.writelines(' '.join(cmd) + "\r\n") 283 for line in out: 284 f.writelines(str(line.strip()) + "\r\n") 285 286 success = True 287 error = command.stderr.readlines() 288 with open("error.txt", mode='a+', encoding='utf-8') as f: 289 if len(error) > 0: 290 f.writelines(' '.join(cmd) + "\r\n") 291 for line in error: 292 f.writelines(str(line.strip()) + "\r\n") 293 294 code = command.wait() 295 if code != 0: 296 success = False 297 end = time.time() 298 case_result['total_cost'] = case_result['total_cost'] + (end - start) 299 300 if success: 301 case_result['success'] = case_result['success'] + 1 302 else: 303 case_result['fail'] = case_result['fail'] + 1 304 return success 305 306 307def run_simple_case(case, jar_file): 308 test_case = simple_scope.get(case, None) 309 if not test_case: 310 print("Not found test case: {}".format(case)) 311 exit(0) 312 313 for k in test_case: 314 cmd = ['java', '-jar', jar_file] + [s.replace('"', '') for s in (re.split(r' \s*(?![^"]*\" )', k.strip()))] 315 print("== Run command: {}".format(cmd)) 316 result = run_target(case, cmd) 317 print("== Done command: {}".format(result)) 318 319 320def run_test_case(case, jar_file): 321 test_case = test_scope.get(case, None) 322 if not test_case: 323 print("Not found test case: {}".format(case)) 324 exit(0) 325 326 for k in test_case: 327 cmd = ['java', '-jar', jar_file] + [s.replace('"', '') for s in (re.split(r' \s*(?![^"]*\" )', k.strip()))] 328 print("== Run command: {}".format(cmd)) 329 result = run_target(case, cmd) 330 331 with open('test_result.log', 'r', encoding='utf-8') as f: 332 content = f.read() 333 test_result_dict = ast.literal_eval(content) 334 335 if case == 'case-assert-true': 336 if result: 337 print("== Done command: Expected True and tested True") 338 else: 339 test_result_dict['commands_expected_True_but_tested_False'].append(cmd) 340 print("== Done command: Expected True but tested False") 341 else: 342 if result: 343 test_result_dict['commands_expected_False_but_tested_True'].append(cmd) 344 print("== Done command: Expected False but tested True") 345 else: 346 print("== Done command: Expected False and tested False") 347 348 with open("test_result.log", mode='w', encoding='utf-8') as tr: 349 tr.write(json.dumps(test_result_dict, indent=4)) 350 351 352def random_str(): 353 strs = "abcdefghjiklmnopqstuvwxyzABCDEFGHIJKLMNOPQRS TUVWXYZ1234567890~!@#ls%^&*()_+,./<>?;':" 354 result = '' 355 for i in range(random.randint(1, 30)): 356 result = result + random.choice(strs) 357 return result 358 359 360def run_random_case(case, jar_file): 361 test_case = random_scope.get(case, None) 362 if not test_case: 363 print("Not found test case: {}".format(case)) 364 exit(0) 365 366 cmd = ['java', '-jar', jar_file, case] 367 for k, v in test_case.get('required').items(): 368 r = random.choice(['none', 'choice', 'choice', 'random']) 369 if r == 'choice': 370 cmd.append('-'.join(k)) 371 cmd.append(random.choice(v) if isinstance(v, list) else v) 372 elif r == 'random': 373 cmd.append('-'.join(k)) 374 cmd.append(random_str()) 375 376 for k, v in test_case.get('others').items(): 377 r = random.choice(['none', 'choice', 'choice', 'random']) 378 if r == 'choice': 379 cmd.append('-'.join(k)) 380 cmd.append(random.choice(v) if isinstance(v, list) else v) 381 elif r == 'random': 382 cmd.append('-'.join(k)) 383 cmd.append(random_str()) 384 385 print("== Run command: {}".format(cmd)) 386 result = run_target(case, cmd) 387 print("== Done command: {}".format(result)) 388 389 390def run_all_case(case, jar_file): 391 test_case = random_scope.get(case, None) 392 if not test_case: 393 print("Not found test case: {}".format(case)) 394 exit(0) 395 396 cmd = ['java', '-jar', jar_file, case] 397 for ak, av in test_case.get('required').items(): 398 cmd.append('-'.join(ak)) 399 cmd.append(random.choice(av) if isinstance(av, list) else av) 400 401 print("== Run command: {}".format(cmd)) 402 result = run_target(case, cmd) 403 print("== Done command: {}".format(result)) 404 405 406def prepare_env(): 407 test_dirs = ['test1', 'test2'] 408 for test_dir in test_dirs: 409 if not os.path.exists(test_dir): 410 os.mkdir(test_dir) 411 412 for key_file in ['ohtest.jks', 'ohtest.p12']: 413 target_file = os.path.join(test_dir, key_file) 414 if os.path.exists(target_file): 415 os.remove(target_file) 416 if os.path.exists(key_file): 417 os.remove(key_file) 418 419 420def process_cmd(args): 421 run_round: int = 1 422 run_scope: str = 'simple' 423 is_random: bool = False 424 425 if len(args) <= 1 or ('.jar' not in args[1]) or '--help' == args[1] or '-h' == args[1]: 426 print_help() 427 exit(0) 428 429 jar_file = args[1] 430 if not os.path.exists(jar_file): 431 print("Jar file '{}' not found".format(jar_file)) 432 exit(0) 433 434 if len(args) >= 3: 435 temp_round, temp_scope, temp_random = get_run_format(args) 436 run_round: int = temp_round 437 run_scope: str = temp_scope 438 is_random: bool = temp_random 439 440 print('=== Start testing ===') 441 print('Scope: {}. Round: {}. Random: {}'.format(run_scope, run_round, is_random)) 442 443 if os.path.exists('log.txt'): 444 os.remove('log.txt') 445 if os.path.exists('error.txt'): 446 os.remove('error.txt') 447 448 for i in range(run_round): 449 if run_scope == 'all': 450 for r_scope, _ in random_scope.items(): 451 run_all_case(r_scope, jar_file) 452 elif is_random: 453 for r_scope, _ in random_scope.items(): 454 run_random_case(r_scope, jar_file) 455 elif run_scope == 'simple': 456 prepare_env() 457 for s_scope, _ in simple_scope.items(): 458 run_simple_case(s_scope, jar_file) 459 elif run_scope == 'runtest': 460 prepare_env() 461 with open("test_result.log", mode='w', encoding='utf-8') as file_result: 462 result_dict = { 463 'commands_expected_True_but_tested_False': [], 464 'commands_expected_False_but_tested_True': [] 465 } 466 file_result.write(str(result_dict)) 467 for t_scope, _ in test_scope.items(): 468 run_test_case(t_scope, jar_file) 469 else: 470 run_simple_case(run_scope, jar_file) 471 472 473def get_run_format(args): 474 run_round: int = 1 475 run_scope: str = 'simple' 476 is_random: bool = False 477 try: 478 for i in range(2, len(args), 1): 479 if args[i] == '-n': 480 run_round = int(args[i + 1]) 481 elif args[i] == '-scope': 482 run_scope = args[i + 1] 483 elif args[i] == '--random': 484 is_random = True 485 elif args[i] == '-runtest': 486 run_scope = 'runtest' 487 except IndexError: 488 print_help() 489 exit(0) 490 return run_round, run_scope, is_random 491 492 493if __name__ == '__main__': 494 process_cmd(sys.argv) 495 print("All test done") 496 print("========================") 497 for rk, rv in test_result.items(): 498 times = rv['times'] 499 if times != 0: 500 print("Case {}, run times: {}, avg cost: {}s, total success: {}, total fail: {}".format(rk, times, round( 501 rv['total_cost'] / times, 2), rv['success'], rv['fail'])) 502 print("========================") 503 print("See log.txt / error.txt") 504