1
2#!/usr/bin/env python3
3#
4# Copyright © 2020 Google LLC
5#
6# Permission is hereby granted, free of charge, to any person obtaining a
7# copy of this software and associated documentation files (the "Software"),
8# to deal in the Software without restriction, including without limitation
9# the rights to use, copy, modify, merge, publish, distribute, sublicense,
10# and/or sell copies of the Software, and to permit persons to whom the
11# Software is furnished to do so, subject to the following conditions:
12#
13# The above copyright notice and this permission notice (including the next
14# paragraph) shall be included in all copies or substantial portions of the
15# Software.
16#
17# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
18# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
19# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
20# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
21# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
22# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
23# IN THE SOFTWARE.
24
25import argparse
26import queue
27import re
28from serial_buffer import SerialBuffer
29import sys
30import threading
31
32
33class CrosServoRun:
34    def __init__(self, cpu, ec, test_timeout):
35        self.cpu_ser = SerialBuffer(
36            cpu, "results/serial.txt", "R SERIAL-CPU> ")
37        # Merge the EC serial into the cpu_ser's line stream so that we can
38        # effectively poll on both at the same time and not have to worry about
39        self.ec_ser = SerialBuffer(
40            ec, "results/serial-ec.txt", "R SERIAL-EC> ", line_queue=self.cpu_ser.line_queue)
41        self.test_timeout = test_timeout
42
43    def close(self):
44        self.ec_ser.close()
45        self.cpu_ser.close()
46
47    def ec_write(self, s):
48        print("W SERIAL-EC> %s" % s)
49        self.ec_ser.serial.write(s.encode())
50
51    def cpu_write(self, s):
52        print("W SERIAL-CPU> %s" % s)
53        self.cpu_ser.serial.write(s.encode())
54
55    def print_error(self, message):
56        RED = '\033[0;31m'
57        NO_COLOR = '\033[0m'
58        print(RED + message + NO_COLOR)
59
60    def run(self):
61        # Flush any partial commands in the EC's prompt, then ask for a reboot.
62        self.ec_write("\n")
63        self.ec_write("reboot\n")
64
65        bootloader_done = False
66        # This is emitted right when the bootloader pauses to check for input.
67        # Emit a ^N character to request network boot, because we don't have a
68        # direct-to-netboot firmware on cheza.
69        for line in self.cpu_ser.lines(timeout=120, phase="bootloader"):
70            if re.search("load_archive: loading locale_en.bin", line):
71                self.cpu_write("\016")
72                bootloader_done = True
73                break
74
75            # If the board has a netboot firmware and we made it to booting the
76            # kernel, proceed to processing of the test run.
77            if re.search("Booting Linux", line):
78                bootloader_done = True
79                break
80
81            # The Cheza boards have issues with failing to bring up power to
82            # the system sometimes, possibly dependent on ambient temperature
83            # in the farm.
84            if re.search("POWER_GOOD not seen in time", line):
85                self.print_error(
86                    "Detected intermittent poweron failure, restarting run...")
87                return 2
88
89        if not bootloader_done:
90            print("Failed to make it through bootloader, restarting run...")
91            return 2
92
93        tftp_failures = 0
94        for line in self.cpu_ser.lines(timeout=self.test_timeout, phase="test"):
95            if re.search("---. end Kernel panic", line):
96                return 1
97
98            # The Cheza firmware seems to occasionally get stuck looping in
99            # this error state during TFTP booting, possibly based on amount of
100            # network traffic around it, but it'll usually recover after a
101            # reboot.
102            if re.search("R8152: Bulk read error 0xffffffbf", line):
103                tftp_failures += 1
104                if tftp_failures >= 100:
105                    self.print_error(
106                        "Detected intermittent tftp failure, restarting run...")
107                    return 2
108
109            # There are very infrequent bus errors during power management transitions
110            # on cheza, which we don't expect to be the case on future boards.
111            if re.search("Kernel panic - not syncing: Asynchronous SError Interrupt", line):
112                self.print_error(
113                    "Detected cheza power management bus error, restarting run...")
114                return 2
115
116            # If the network device dies, it's probably not graphics's fault, just try again.
117            if re.search("NETDEV WATCHDOG", line):
118                self.print_error(
119                    "Detected network device failure, restarting run...")
120                return 2
121
122            # These HFI response errors started appearing with the introduction
123            # of piglit runs.  CosmicPenguin says:
124            #
125            # "message ID 106 isn't a thing, so likely what happened is that we
126            # got confused when parsing the HFI queue.  If it happened on only
127            # one run, then memory corruption could be a possible clue"
128            #
129            # Given that it seems to trigger randomly near a GPU fault and then
130            # break many tests after that, just restart the whole run.
131            if re.search("a6xx_hfi_send_msg.*Unexpected message id .* on the response queue", line):
132                self.print_error(
133                    "Detected cheza power management bus error, restarting run...")
134                return 2
135
136            if re.search("coreboot.*bootblock starting", line):
137                self.print_error(
138                    "Detected spontaneous reboot, restarting run...")
139                return 2
140
141            if re.search("arm-smmu 5040000.iommu: TLB sync timed out -- SMMU may be deadlocked", line):
142                self.print_error("Detected cheza MMU fail, restarting run...")
143                return 2
144
145            result = re.search("hwci: mesa: (\S*)", line)
146            if result:
147                if result.group(1) == "pass":
148                    return 0
149                else:
150                    return 1
151
152        self.print_error(
153            "Reached the end of the CPU serial log without finding a result")
154        return 2
155
156
157def main():
158    parser = argparse.ArgumentParser()
159    parser.add_argument('--cpu', type=str,
160                        help='CPU Serial device', required=True)
161    parser.add_argument(
162        '--ec', type=str, help='EC Serial device', required=True)
163    parser.add_argument(
164        '--test-timeout', type=int, help='Test phase timeout (minutes)', required=True)
165    args = parser.parse_args()
166
167    servo = CrosServoRun(args.cpu, args.ec, args.test_timeout * 60)
168
169    while True:
170        retval = servo.run()
171        if retval != 2:
172            break
173
174    # power down the CPU on the device
175    servo.ec_write("power off\n")
176
177    servo.close()
178
179    sys.exit(retval)
180
181
182if __name__ == '__main__':
183    main()
184