1 /*
2  * Copyright (c) 2023-2023 Huawei Device Co., Ltd. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without modification,
5  * are permitted provided that the following conditions are met:
6  *
7  * 1. Redistributions of source code must retain the above copyright notice, this list of
8  * conditions and the following disclaimer.
9  *
10  * 2. Redistributions in binary form must reproduce the above copyright notice, this list
11  * of conditions and the following disclaimer in the documentation and/or other materials
12  * provided with the distribution.
13  *
14  * 3. Neither the name of the copyright holder nor the names of its contributors may be used
15  * to endorse or promote products derived from this software without specific prior written
16  * permission.
17  *
18  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
20  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
21  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
22  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
23  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
24  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
25  * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
26  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
27  * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
28  * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29  */
30 
31 #include <iostream>
32 #include <vector>
33 #include <string>
34 #include <algorithm>
35 #include <cstdlib>
36 #include <cstdio>
37 #include <dirent.h>
38 #include <unistd.h>
39 #include <sys/wait.h>
40 #include <securec.h>
41 
42 using namespace std;
43 const static int TEST_PATH_MAX = 255;
44 const static int TEST_PARAM_NUM = 5;
45 const static int TEST_CASE_MODE_FULL = 0;
46 const static int TEST_CASE_MODE_SMOKE = 1;
47 const static int TEST_DEFAULT_CASE_MAX = 5;
48 const static int TEST_T_PARAM_LEN = 2;
49 const static unsigned int TEST_CASE_FLAGS_ALL = 1;
50 const static unsigned int TEST_CASE_FLAGS_SPECIFY = 2;
51 const static unsigned int TEST_CASE_FAILED_COUNT_MAX = 500;
52 static unsigned int g_testsuitesCount = 0;
53 static unsigned int g_testsuitesFailedCount = 0;
54 
55 struct TestCase {
56     char bin[TEST_PATH_MAX];
57     char *param[TEST_PARAM_NUM];
58     int caseLen;
59 };
60 
61 struct TestsuitesParam {
62     char testsuitesBin[TEST_PATH_MAX];
63     char testsuitesDir[TEST_PATH_MAX];
64     int testsuitesDirLen;
65     struct TestCase *testCase[TEST_DEFAULT_CASE_MAX];
66     int testCaseNum;
67     int testMode;
68     int repeat;
69 };
70 
71 static struct TestsuitesParam g_param;
72 static char *g_testsuitesFailedCase[TEST_CASE_FAILED_COUNT_MAX];
73 
GetAllTestsuites(string testsuitesDir)74 static vector<string> GetAllTestsuites(string testsuitesDir)
75 {
76     vector<string> vFiles;
77     int suffixLen = strlen(".bin");
78     DIR *dir = opendir(testsuitesDir.c_str());
79     if (dir == nullptr) {
80         cout << "opendir " <<  testsuitesDir << " failed" << endl;
81         return vFiles;
82     }
83 
84     do {
85         struct dirent *pDir = readdir(dir);
86         if (pDir == nullptr) {
87             break;
88         }
89 
90         if ((strcmp(pDir->d_name, ".") == 0) || (strcmp(pDir->d_name, "..") == 0)) {
91             continue;
92         }
93 
94         if (pDir->d_type == DT_REG) {
95             int pathLen = strlen(pDir->d_name);
96             if (pathLen <= suffixLen) {
97                 continue;
98             }
99             if (strcmp(".bin", (char *)(pDir->d_name + (pathLen - suffixLen))) != 0) {
100                 continue;
101             }
102 
103             vFiles.push_back(testsuitesDir + "/" + pDir->d_name);
104         }
105     } while (1);
106     closedir(dir);
107     return vFiles;
108 }
109 
TestsuitesIsSmoke(string testsuite, int fileLen, string smokeTest, int smokeTestLen)110 static bool TestsuitesIsSmoke(string testsuite, int fileLen, string smokeTest, int smokeTestLen)
111 {
112     if (fileLen <= smokeTestLen) {
113         return false;
114     }
115 
116     if (strcmp(smokeTest.c_str(), (char *)(testsuite.c_str() + (fileLen - smokeTestLen))) == 0) {
117         return true;
118     }
119     return false;
120 }
121 
RunCase(const char *testCase, char *param[])122 static void RunCase(const char *testCase, char *param[])
123 {
124     int ret;
125     int size;
126     int status = 0;
127     pid_t pid = fork();
128     if (pid < 0) {
129         cout << "fork failed, " << strerror(errno) << endl;
130         return;
131     }
132 
133     g_testsuitesCount++;
134     if (pid == 0) {
135         cout << testCase << ":" << endl;
136         ret = execve(param[0], param, nullptr);
137         if (ret != 0) {
138             cout << "execl: " << testCase << " failed, " << strerror(errno) << endl;
139             exit(0);
140         }
141     }
142 
143     ret = waitpid(pid, &status, 0);
144     if (ret != pid) {
145         cout << "waitpid failed, " << strerror(errno) << endl;
146         return;
147     }
148 
149     if (WEXITSTATUS(status) == 0) {
150         return;
151     }
152 
153     if (g_testsuitesFailedCount >= TEST_CASE_FAILED_COUNT_MAX) {
154         cout << "[UNITTEST_RUN] Failure cases more than upper limit!\n";
155         return;
156     }
157 
158     size = strlen(testCase) + 1;
159     g_testsuitesFailedCase[g_testsuitesFailedCount] = static_cast<char *>(malloc(size));
160     if (g_testsuitesFailedCase[g_testsuitesFailedCount] == nullptr) {
161         cout << "[UNITTEST_RUN] malloc failed!\n";
162         return;
163     }
164 
165     if (memcpy_s(g_testsuitesFailedCase[g_testsuitesFailedCount], size, testCase, size) != EOK) {
166         cout << "[UNITTEST_RUN] memcpy failed!\n";
167         return;
168     }
169     g_testsuitesFailedCount++;
170 }
171 
RunAllTestCase(vector<string> files, struct TestsuitesParam *param)172 static void RunAllTestCase(vector<string> files, struct TestsuitesParam *param)
173 {
174     const char *testMode = nullptr;
175     const char *smokeTest = "door.bin";
176     const char *unittestRun = g_param.testsuitesBin;
177     int unittestRunLen = strlen(unittestRun);
178     int smokeTestLen = strlen(smokeTest);
179     char *execParam[TEST_PARAM_NUM];
180 
181     if (param->testMode == TEST_CASE_MODE_SMOKE) {
182         testMode = "door.bin";
183     } else {
184         testMode = ".bin";
185     }
186 
187     for (size_t i = 0; i < files.size(); i++) {
188         int fileLen = strlen(files[i].c_str());
189         if (fileLen >= unittestRunLen) {
190             if (strcmp((char *)(files[i].c_str() + (fileLen - unittestRunLen)), unittestRun) == 0) {
191                 continue;
192             }
193         }
194 
195         if (strcmp(testMode, smokeTest) == 0) {
196             if (!TestsuitesIsSmoke(files[i], fileLen, smokeTest, smokeTestLen)) {
197                 continue;
198             }
199         } else {
200             if (TestsuitesIsSmoke(files[i], fileLen, smokeTest, smokeTestLen)) {
201                 continue;
202             }
203         }
204 
205         (void)memset_s(execParam, sizeof(char *) * TEST_PARAM_NUM, 0, sizeof(char *) * TEST_PARAM_NUM);
206         execParam[0] = (char *)files[i].c_str();
207         RunCase(files[i].c_str(), execParam);
208     }
209 }
210 
TestsuitesToolsUsage(void)211 static void TestsuitesToolsUsage(void)
212 {
213     cout << "Usage: " << endl;
214     cout << "liteos_unittest_run.bin [testsuites_dir] [options]" << endl;
215     cout << "options:" << endl;
216     cout << " -r [1-1000]             --- The number of repeated runs of the test program." << endl;
217     cout << " -m [smoke/full]         --- Run the smoke or full test case in this directory." << endl;
218     cout << " -t [case] [args] -t ... --- Runs the specified executable program name." << endl;
219 }
220 
TestsuitesDirFormat(char *testDir, int len)221 static int TestsuitesDirFormat(char *testDir, int len)
222 {
223     if (memcpy_s(g_param.testsuitesDir, TEST_PATH_MAX, testDir, len + 1) != EOK) {
224         cout << "testsuites dir: " << strerror(ENAMETOOLONG) << endl;
225         return -1;
226     }
227 
228     char *end = g_param.testsuitesDir + len;
229     while ((testDir != end) && (*end == '/')) {
230         *end = '\0';
231         end--;
232         len--;
233     }
234 
235     g_param.testsuitesDirLen = len;
236     if (len <= 0) {
237         return -1;
238     }
239     return 0;
240 }
241 
GetTestsuitesToolsName(const char *argv[])242 static int GetTestsuitesToolsName(const char *argv[])
243 {
244     const char *testBin = static_cast<const char *>(argv[0]);
245     const char *end = testBin + strlen(testBin);
246     while (*end != '/') {
247         end--;
248     }
249     end++;
250 
251     if (memcpy_s(g_param.testsuitesBin, TEST_PATH_MAX, end, strlen(end)) != EOK) {
252         cout << "testsuites dir: " << strerror(ENAMETOOLONG) << endl;
253         return -1;
254     }
255     return 0;
256 }
257 
ParseTestCaseAndParam(int argc, const char *argv[], int *index)258 static int ParseTestCaseAndParam(int argc, const char *argv[], int *index)
259 {
260     int j;
261 
262     (*index)++;
263     if (*index >= argc) {
264         return -1;
265     }
266 
267     if (g_param.testCaseNum >= TEST_DEFAULT_CASE_MAX) {
268         return -1;
269     }
270 
271     g_param.testCase[g_param.testCaseNum] = (struct TestCase *)malloc(sizeof(struct TestCase));
272     if (g_param.testCase[g_param.testCaseNum] == nullptr) {
273         cout << "test case " << strerror(ENOMEM) << endl;
274         return -1;
275     }
276     (void)memset_s(g_param.testCase[g_param.testCaseNum], sizeof(struct TestCase), 0, sizeof(struct TestCase));
277     struct TestCase *testCase = g_param.testCase[g_param.testCaseNum];
278     testCase->caseLen = strlen(argv[*index]);
279     if (memcpy_s(testCase->bin, TEST_PATH_MAX, argv[*index], testCase->caseLen + 1) != EOK) {
280         testCase->caseLen = 0;
281         cout << "test case " << strerror(ENAMETOOLONG) << endl;
282         return -1;
283     }
284     testCase->param[0] = testCase->bin;
285     (*index)++;
286     for (j = 1; (j < TEST_PARAM_NUM) && (*index < argc) && (strncmp("-t", argv[*index], TEST_T_PARAM_LEN) != 0); j++) {
287         testCase->param[j] = (char *)argv[*index];
288         (*index)++;
289     }
290     g_param.testCaseNum++;
291     if (((*index) < argc) && (strncmp("-t", argv[*index], TEST_T_PARAM_LEN) == 0)) {
292         (*index)--;
293     }
294     return 0;
295 }
296 
TestsuitesParamCheck(int argc, const char *argv[])297 static int TestsuitesParamCheck(int argc, const char *argv[])
298 {
299     int ret;
300     unsigned int mask = 0;
301     g_param.testMode = TEST_CASE_MODE_FULL;
302     g_param.repeat = 1;
303     ret = TestsuitesDirFormat((char *)argv[1], strlen(argv[1]));
304     if (ret < 0) {
305         return -1;
306     }
307 
308     for (int i = 2; i < argc; i++) { /* 2: param index */
309         if (strcmp("-m", argv[i]) == 0) {
310             i++;
311             if (i >= argc) {
312                 return -1;
313             }
314             mask |= TEST_CASE_FLAGS_ALL;
315             if (strcmp("smoke", argv[i]) == 0) {
316                 g_param.testMode = TEST_CASE_MODE_SMOKE;
317             } else if (strcmp("full", argv[i]) == 0) {
318                 g_param.testMode = TEST_CASE_MODE_FULL;
319             } else {
320                 return -1;
321             }
322         } else if (strcmp("-t", argv[i]) == 0) {
323             mask |= TEST_CASE_FLAGS_SPECIFY;
324             ret = ParseTestCaseAndParam(argc, argv, &i);
325             if (ret < 0) {
326                 return ret;
327             }
328         } else if (strcmp("-r", argv[i]) == 0) {
329             i++;
330             if (i >= argc) {
331                 return -1;
332             }
333             g_param.repeat = atoi(argv[i]);
334             if ((g_param.repeat <= 0) || (g_param.repeat > 1000)) { /* 1000: repeat limit */
335                 return -1;
336             }
337         }
338     }
339 
340     if (((mask & TEST_CASE_FLAGS_ALL) != 0) && ((mask & TEST_CASE_FLAGS_SPECIFY) != 0)) {
341         cout << "Invalid parameter combination" << endl;
342         return -1;
343     }
344     return 0;
345 }
346 
IsCase(vector<string> files, struct TestCase *testCase)347 static void IsCase(vector<string> files, struct TestCase *testCase)
348 {
349     for (size_t i = 0; i < files.size(); i++) {
350         string file = files[i];
351         int fileLen = strlen(file.c_str());
352         if (fileLen <= testCase->caseLen) {
353             continue;
354         }
355 
356         const string &suffix = file.c_str() + (fileLen - testCase->caseLen);
357         if (strcmp(suffix.c_str(), testCase->bin) != 0) {
358             continue;
359         }
360 
361         if (memcpy_s(testCase->bin, TEST_PATH_MAX, file.c_str(), fileLen + 1) != EOK) {
362             testCase->caseLen = 0;
363             return;
364         }
365         testCase->caseLen = fileLen;
366         g_param.testCaseNum++;
367         return;
368     }
369     cout << "liteos_unittest_run.bin: not find test case: " << testCase->bin << endl;
370     return;
371 }
372 
FindTestCase(vector<string> files)373 static int FindTestCase(vector<string> files)
374 {
375     int count = g_param.testCaseNum;
376     g_param.testCaseNum = 0;
377 
378     for (int i = 0; i < count; i++) {
379         IsCase(files, g_param.testCase[i]);
380     }
381 
382     if (g_param.testCaseNum == 0) {
383         cout << "Not find test case !" << endl;
384         return -1;
385     }
386     return 0;
387 }
388 
FreeTestCaseMem(void)389 static void FreeTestCaseMem(void)
390 {
391     for (int index = 0; index < g_param.testCaseNum; index++) {
392         free(g_param.testCase[index]);
393         g_param.testCase[index] = nullptr;
394     }
395 }
396 
ShowTestLog(int count)397 static void ShowTestLog(int count)
398 {
399     cout << "[UNITTEST_RUN] Repeats: " << count << " Succeed count: "
400         << g_testsuitesCount  - g_testsuitesFailedCount
401         << " Failed count: " << g_testsuitesFailedCount << endl;
402 
403     if (g_testsuitesFailedCount == 0) {
404         return;
405     }
406     cout << "[UNITTEST_RUN] Failed testcase: " << endl;
407     for (int i = 0; i < g_testsuitesFailedCount; i++) {
408         cout << "[" << i << "] -> " << g_testsuitesFailedCase[i] << endl;
409         free(g_testsuitesFailedCase[i]);
410         g_testsuitesFailedCase[i] = nullptr;
411     }
412 }
413 
main(int argc, const char *argv[])414 int main(int argc, const char *argv[])
415 {
416     int ret;
417     int count = 0;
418 
419     if ((argc < 2) || (argv == nullptr)) { /* 2: param index */
420         cout << argv[0] << ": " << strerror(EINVAL) << endl;
421         return -1;
422     }
423 
424     if ((strcmp("--h", argv[1]) == 0) || (strcmp("--help", argv[1]) == 0)) {
425         TestsuitesToolsUsage();
426         return 0;
427     }
428 
429     (void)memset_s(&g_param, sizeof(struct TestsuitesParam), 0, sizeof(struct TestsuitesParam));
430 
431     ret = GetTestsuitesToolsName(argv);
432     if (ret < 0) {
433         return -1;
434     }
435 
436     ret = TestsuitesParamCheck(argc, argv);
437     if (ret < 0) {
438         cout << strerror(EINVAL) << endl;
439         FreeTestCaseMem();
440         return -1;
441     }
442 
443     vector<string> files = GetAllTestsuites(g_param.testsuitesDir);
444 
445     if (g_param.testCaseNum != 0) {
446         ret = FindTestCase(files);
447         if (ret < 0) {
448             files.clear();
449             FreeTestCaseMem();
450             return -1;
451         }
452     }
453 
454     while (count < g_param.repeat) {
455         if (g_param.testCaseNum == 0) {
456             RunAllTestCase(files, &g_param);
457         } else {
458             for (int index = 0; index < g_param.testCaseNum; index++) {
459                 RunCase(g_param.testCase[index]->bin, g_param.testCase[index]->param);
460             }
461         }
462         count++;
463     }
464     files.clear();
465     FreeTestCaseMem();
466     ShowTestLog(count);
467     return 0;
468 }
469