1#!/usr/bin/env ruby
2# Copyright (c) 2021-2022 Huawei Device Co., Ltd.
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14
15# Huawei Technologies Co.,Ltd.
16
17require 'optparse'
18require 'fileutils'
19require 'ostruct'
20require 'yaml'
21require 'open3'
22require 'pathname'
23require 'thread'
24require 'timeout'
25require 'securerandom'
26
27require_relative 'runner/runner'
28require_relative 'runner/single_test_runner'
29require_relative 'runner/reporters/test_reporter'
30require_relative 'runner/reporters/jtr_reporter'
31require_relative 'runner/reporters/allure_reporter'
32
33def check_option(optparser, options, key)
34  return if options[key]
35
36  puts "Missing option: --#{key}"
37  puts optparser
38  exit false
39end
40
41def check_option_limit(optparser, options, key, min, max)
42  return unless options[key]
43  return if options[key] >= min && options[key] <= max
44
45  puts "Incorrect value for option: --#{key} [#{min}, #{max}]"
46  puts optparser
47  exit false
48end
49
50def check_option_enum(optparser, options, key, enum)
51  return unless options[key]
52  return if enum.include?(options[key])
53  puts "Incorrect value for option: --#{key} #{enum}"
54  puts optparser
55  exit false
56end
57
58Encoding.default_external = Encoding::UTF_8
59Encoding.default_internal = Encoding::UTF_8
60
61TestRunner.runner_class = TestRunner::SingleTestRunner
62
63options = OpenStruct.new
64options[:exclude_tag] = []
65options[:include_tag] = []
66options[:bug_ids] = []
67options[:panda_options] = []
68
69optparser = OptionParser.new do |opts|
70  opts.banner = 'Usage: test-runner.rb [options]'
71  opts.on('-p', '--panda-build DIR', 'Path to panda build directory (required)')
72  opts.on('-t', '--test-dir DIR', 'Path to test directory to search tests recursively, or path to single test (required)')
73  opts.on('-x', '--temp-dir DIR', 'Temporary files location, defaults to /tmp')
74  opts.on('-v', '--verbose LEVEL', Integer, 'Set verbose level 1..5')
75  opts.on('--verbose-verifier', 'Allow verifier to produce extended checking log')
76  opts.on('--aot-mode', 'Perform AOT compilation on test sources')
77  opts.on('--timeout SECONDS', Integer, 'Set process timeout, default is 30 seconds')
78  opts.on('--dump-timeout SECONDS', Integer, 'Set process completion timeout, default is 30 seconds')
79  opts.on('--enable-core-dump', 'Enable core dumps')
80  opts.on('--verify-tests', 'Run verifier against positive tests (option for test checking)')
81  opts.on('--with-quickener', 'Run quickener tool after assembly')
82  opts.on('--global-timeout SECONDS', Integer, 'Set testing timeout, default is 0 (ulimited)')
83  opts.on('-a', '--run-all', 'Run all tests, ignore "runner-option: ignore" tag in test definition')
84  opts.on('--run-ignored', 'Run ignored tests, which have "runner-option: ignore" tag in test definition')
85  opts.on('--reporter TYPE', "Reporter for test results (default 'log', available: 'log', 'jtr', 'allure')")
86  opts.on('--report-dir DIR', "Where to put results, applicable for 'jtr' and 'allure' logger")
87  opts.on('--verifier-config PATH', "Path to verifier config file")
88  opts.on('-e', '--exclude-tag TAG', Array, 'Exclude tags for tests') do |f|
89    options[:exclude_tag] |= [*f]
90  end
91  opts.on('-o', '--panda-options OPTION', Array, 'Panda options') do |f|
92    options[:panda_options] |= [*f]
93  end
94  opts.on('-i', '--include-tag TAG', Array, 'Include tags for tests') do |f|
95    options[:include_tag] |= [*f]
96  end
97  opts.on('-b', '--bug_id BUGID', Array, 'Include tests with specified bug ids') do |f|
98    options[:bug_ids] |= [*f]
99  end
100  opts.on('-j', '--jobs N', 'Amount of concurrent jobs for test execution (default 8)', Integer)
101  opts.on('--prlimit OPTS', "Run panda via prlimit with options")
102  opts.on('--plugins PLUGINS', Array, 'Paths to runner plugins') do |plugins|
103    plugins.each do |plugin|
104      require plugin
105      TestRunner.plugins.last.add_options(opts, options)
106    end
107  end
108  opts.on('-h', '--help', 'Prints this help') do
109    puts opts
110    exit
111  end
112end
113
114optparser.parse!(into: options)
115
116check_option_enum(optparser, options, 'reporter', ['log', 'jtr', 'allure'])
117
118check_option(optparser, options, 'panda-build')
119check_option(optparser, options, 'test-dir')
120check_option_limit(optparser, options, 'jobs', 1, 20)
121check_option_limit(optparser, options, 'timeout', 1, 1000)
122options['verbose'] = 1 unless options['verbose']
123options['timeout'] = 30 unless options['timeout']
124options['temp-dir'] = '/tmp' unless options['temp-dir']
125options['dump-timeout'] = 30 unless options['dump-timeout']
126options['global-timeout'] = 0 unless options['global-timeout']
127options['jobs'] = 8 unless options['jobs']
128options['reporter'] = 'log'  unless options['reporter']
129
130# TODO refactor to avoid global vars
131$VERBOSITY = options['verbose']
132$TIMEOUT = options['timeout']
133$DUMP_TIMEOUT = options['dump-timeout']
134$GLOBAL_TIMEOUT = options['global-timeout']
135$CONCURRENCY = options['jobs']
136
137$path_to_panda = options['panda-build']
138$pandasm = "#{$path_to_panda}/bin/ark_asm"
139$panda = if options['prlimit']
140  "prlimit #{options['prlimit']} #{$path_to_panda}/bin/ark"
141else
142  "#{$path_to_panda}/bin/ark"
143end
144$verifier = "#{$path_to_panda}/bin/verifier"
145$verifier_config = options['verifier-config'] || ''
146$paoc = if options['aot-mode']
147  # Use paoc on host for x86
148  "#{$path_to_panda}/bin/ark_aot"
149else
150  false
151end
152$quickener = if options['with-quickener']
153  # Use quickener tool
154  "#{$path_to_panda}/bin/arkquick"
155else
156  false
157end
158
159TestRunner.log 2, "Path to panda: #{$path_to_panda}"
160TestRunner.log 2, "Path to verifier debug config: #{$verifier_config}"
161
162TestRunner::plugins.each { |p| p.process(options) }
163
164# TODO refactor to avoid global vars
165$run_all = options['run-all']
166$run_ignore = options['run-ignored']
167$enable_core = !!options['enable-core-dump']
168$force_verifier = !!options['verify-tests']
169$verbose_verifier = !!options['verbose-verifier']
170$exclude_list = options[:exclude_tag]
171$include_list = options[:include_tag]
172$bug_ids = options[:bug_ids]
173$panda_options = options[:panda_options]
174$root_dir = options['test-dir']
175$report_dir = options['report-dir'] || ''
176$reporter = options['reporter']
177
178# path_to_tests = '/mnt/d/linux/work/panda/tests/cts-generator/cts-generated/'
179path_to_tests = $root_dir
180TestRunner::log 2, "Path to tests: #{path_to_tests}"
181
182TestRunner::log 2, "pandasm: #{$pandasm}"
183TestRunner::log 2, "panda: #{$panda}"
184TestRunner::log 2, "verifier: #{$verifier}"
185
186$tmp_dir  = "#{options['temp-dir']}#{File::SEPARATOR}#{SecureRandom.uuid}#{File::SEPARATOR}"
187TestRunner::log 2, "tmp_dir: #{$tmp_dir}"
188TestRunner::log 3, "Make dir - #{$tmp_dir}"
189FileUtils.mkdir_p $tmp_dir unless File.exist? $tmp_dir
190
191interrupted = false
192timeouted = false
193
194TestRunner::log 2, 'Walk through directories'
195
196files = if File.file?(path_to_tests)
197  Dir.glob("#{path_to_tests}")
198else
199  Dir.glob("#{path_to_tests}/**/*.pa")
200end
201
202# TODO should be configured
203reporter_factory = if $reporter == 'jtr'
204    TestRunner::JtrTestReporter
205  elsif $reporter == 'allure'
206    TestRunner::AllureTestReporter
207  else
208    TestRunner::LogTestReporter
209  end
210
211FileUtils.rm_r $report_dir, force: true if File.exist? $report_dir
212
213start_time = Time.now
214queue = Queue.new
215files.each { |x| queue.push x}
216
217# TestRunner::Result holds execution statistic
218
219def create_executor_threads(queue, id, reporter_factory)
220  Thread.new do
221    begin
222      while file = queue.pop(true)
223        runner = TestRunner.create_runner(file, id, reporter_factory, $root_dir, $report_dir)
224        runner.process_single
225      end
226    rescue ThreadError => e # for queue.pop, suppress
227    rescue Interrupt => e
228      TestRunner.print_exception e
229      interrupted = true
230    rescue SignalException => e
231      TestRunner.print_exception e
232      interrupted = true
233    rescue Exception => e
234      TestRunner.print_exception e
235      interrupted = true
236    end
237  end
238end
239
240if $CONCURRENCY > 1
241  runner_threads = (1..$CONCURRENCY).map do |id|
242    create_executor_threads queue, id, reporter_factory
243  end
244  begin
245    if $GLOBAL_TIMEOUT == 0
246      runner_threads.map(&:join)
247    else
248      has_active_tread = false
249      loop do
250        # Wait a bit
251        sleep 1
252        # Check if there are any working thread
253        has_active_tread = false
254        runner_threads.each do |t|
255          has_active_tread = true if t.status != false
256        end
257        # If we reach timeout or there no active threads, break
258        if (Time.now - start_time >= $GLOBAL_TIMEOUT) | !has_active_tread
259          break
260        end
261      end
262
263      # We have active treads, kill them
264      if has_active_tread == true
265        runner_threads.each do |t|
266          status = t.status
267          if status != false
268            timeouted = true
269             TestRunner::log 1, "Kill test executor tread #{t}"
270            t.kill
271          end
272        end
273      end
274    end
275  rescue SignalException => e
276    interrupted = true
277    TestRunner.print_exception e
278  end
279else
280  begin
281    while file = queue.pop(true)
282      runner = TestRunner.create_runner(file, 1, reporter_factory, $root_dir, $report_dir)
283      runner.process_single
284      if ($GLOBAL_TIMEOUT > 0 && (Time.now - start_time >= $GLOBAL_TIMEOUT))
285        puts "Global timeout reached, finish test execution"
286        break
287      end
288    end
289  rescue ThreadError => e # for queue.pop, suppress
290  rescue Interrupt => e
291    TestRunner.print_exception e
292    interrupted = true
293  rescue SignalException => e
294    TestRunner.print_exception e
295    interrupted = true
296  rescue Exception => e
297    TestRunner.print_exception e
298    interrupted = true
299  end
300end
301
302# Write result report
303
304TestRunner::log 1, '----------------------------------------'
305TestRunner::log 1, "Testing done in #{Time.now - start_time} sec"
306TestRunner::log 2, "Remove tmp dir if empty: #{$tmp_dir}"
307FileUtils.rm_rf $tmp_dir if File.exist?($tmp_dir) && Dir.children($tmp_dir).empty?
308
309TestRunner::Result.write_report
310
311TestRunner::log 1, "Testing timeout reached, so testing failed" if timeouted
312TestRunner::log 1, "Testing interrupted and failed" if interrupted
313
314exit 1 if interrupted || timeouted || TestRunner::Result.failed?
315