1/* 2 * Copyright (c) 2022 Huawei Device Co., Ltd. 3 * Licensed under the Apache License, Version 2.0 (the "License"); 4 * you may not use this file except in compliance with the License. 5 * You may obtain a copy of the License at 6 * 7 * http://www.apache.org/licenses/LICENSE-2.0 8 * 9 * Unless required by applicable law or agreed to in writing, software 10 * distributed under the License is distributed on an "AS IS" BASIS, 11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 * See the License for the specific language governing permissions and 13 * limitations under the License. 14 */ 15 16package rec 17 18import ( 19 "context" 20 "errors" 21 "fmt" 22 "fotff/pkg" 23 "fotff/res" 24 "fotff/tester" 25 "fotff/utils" 26 "github.com/sirupsen/logrus" 27 "math" 28 "sync" 29) 30 31type cancelCtx struct { 32 ctx context.Context 33 fn context.CancelFunc 34} 35 36// FindOutTheFirstFail returns the first issue URL that introduce the failure. 37// 'fellows' are optional, these testcases may be tested with target testcase together. 38func FindOutTheFirstFail(m pkg.Manager, t tester.Tester, testCase string, successPkg string, failPkg string, fellows ...string) (string, error) { 39 if successPkg == "" { 40 return "", fmt.Errorf("can not get a success package for %s", testCase) 41 } 42 steps, err := m.Steps(successPkg, failPkg) 43 if err != nil { 44 return "", err 45 } 46 return findOutTheFirstFail(m, t, testCase, steps, fellows...) 47} 48 49// findOutTheFirstFail is the recursive implementation to find out the first issue URL that introduce the failure. 50// Arg steps' length must be grater than 1. The last step is a pre-known failure, while the rests are not tested. 51// 'fellows' are optional. In the last recursive term, they have the same result as what the target testcases has. 52// These fellows can be tested with target testcase together in this term to accelerate testing. 53func findOutTheFirstFail(m pkg.Manager, t tester.Tester, testcase string, steps []string, fellows ...string) (string, error) { 54 if len(steps) == 0 { 55 return "", errors.New("steps are no between (success, failure], perhaps the failure is occasional") 56 } 57 logrus.Infof("now use %d-section search to find out the first fault, the length of range is %d, between [%s, %s]", res.Num()+1, len(steps), steps[0], steps[len(steps)-1]) 58 if len(steps) == 1 { 59 return m.LastIssue(steps[0]) 60 } 61 // calculate gaps between every check point of N-section search. At least 1, or will cause duplicated tests. 62 gapLen := float64(len(steps)-1) / float64(res.Num()+1) 63 if gapLen < 1 { 64 gapLen = 1 65 } 66 // 'success' and 'fail' record the left/right steps indexes of the next term recursive call. 67 // Here defines functions and surrounding helpers to update success/fail indexes and cancel un-needed tests. 68 success, fail := -1, len(steps)-1 69 var lock sync.Mutex 70 var contexts []cancelCtx 71 updateRange := func(pass bool, index int) { 72 lock.Lock() 73 defer lock.Unlock() 74 if pass && index > success { 75 success = index 76 for _, ctx := range contexts { 77 if ctx.ctx.Value("index").(int) < success { 78 ctx.fn() 79 } 80 } 81 } 82 if !pass && index < fail { 83 fail = index 84 for _, ctx := range contexts { 85 if ctx.ctx.Value("index").(int) > fail { 86 ctx.fn() 87 } 88 } 89 } 90 } 91 // Now, start all tests concurrently. 92 var wg sync.WaitGroup 93 start := make(chan struct{}) 94 for i := 1; i <= res.Num(); i++ { 95 // Since the last step is a pre-known failure, we start index from the tail to avoid testing the last one. 96 // Otherwise, if the last step is the only one we test this term, we can not narrow ranges to continue. 97 index := len(steps) - 1 - int(math.Round(float64(i)*gapLen)) 98 if index < 0 { 99 break 100 } 101 ctx, fn := context.WithCancel(context.WithValue(context.TODO(), "index", index)) 102 contexts = append(contexts, cancelCtx{ctx: ctx, fn: fn}) 103 wg.Add(1) 104 go func(index int, ctx context.Context) { 105 defer wg.Done() 106 // Start after all test goroutine's contexts are registered. 107 // Otherwise, contexts that not registered yet may out of controlling. 108 <-start 109 var pass bool 110 var err error 111 pass, fellows, err = flashAndTest(m, t, steps[index], testcase, ctx, fellows...) 112 if err != nil { 113 if errors.Is(err, context.Canceled) { 114 logrus.Warnf("abort to flash %s and test %s: %v", steps[index], testcase, err) 115 } else { 116 logrus.Errorf("flash %s and test %s fail: %v", steps[index], testcase, err) 117 } 118 return 119 } 120 updateRange(pass, index) 121 }(index, ctx) 122 } 123 close(start) 124 wg.Wait() 125 if fail-success == len(steps) { 126 return "", errors.New("all judgements failed, can not narrow ranges to continue") 127 } 128 return findOutTheFirstFail(m, t, testcase, steps[success+1:fail+1], fellows...) 129} 130 131func flashAndTest(m pkg.Manager, t tester.Tester, pkg string, testcase string, ctx context.Context, fellows ...string) (bool, []string, error) { 132 var newFellows []string 133 if result, found := utils.CacheGet("testcase_result", testcase+"__at__"+pkg); found { 134 logrus.Infof("get testcase result %s from cache done, result is %s", result.(tester.Result).TestCaseName, result.(tester.Result).Status) 135 for _, fellow := range fellows { 136 if fellowResult, fellowFound := utils.CacheGet("testcase_result", fellow+"__at__"+pkg); fellowFound { 137 logrus.Infof("get testcase result %s from cache done, result is %s", fellowResult.(tester.Result).TestCaseName, fellowResult.(tester.Result).Status) 138 if fellowResult.(tester.Result).Status == result.(tester.Result).Status { 139 newFellows = append(newFellows, fellow) 140 } 141 } 142 } 143 return result.(tester.Result).Status == tester.ResultPass, newFellows, nil 144 } 145 var results []tester.Result 146 device := res.GetDevice() 147 defer res.ReleaseDevice(device) 148 if err := m.Flash(device, pkg, ctx); err != nil && !errors.Is(err, context.Canceled) { 149 return false, newFellows, err 150 } else { 151 if err = t.Prepare(m.PkgDir(pkg), device, ctx); err != nil { 152 return false, newFellows, err 153 } 154 results, err = t.DoTestCases(device, append(fellows, testcase), ctx) 155 if err != nil { 156 return false, newFellows, err 157 } 158 } 159 var testcaseStatus tester.ResultStatus 160 for _, result := range results { 161 logrus.Infof("do testcase %s at %s done, result is %s", result.TestCaseName, device, result.Status) 162 if result.TestCaseName == testcase { 163 testcaseStatus = result.Status 164 } 165 utils.CacheSet("testcase_result", result.TestCaseName+"__at__"+pkg, result) 166 } 167 for _, result := range results { 168 if result.TestCaseName != testcase && result.Status == testcaseStatus { 169 newFellows = append(newFellows, result.TestCaseName) 170 } 171 } 172 return testcaseStatus == tester.ResultPass, newFellows, nil 173} 174