1namespace Harness { 2 const fs: typeof import("fs") = require("fs"); 3 const path: typeof import("path") = require("path"); 4 const del: typeof import("del") = require("del"); 5 6 interface ExecResult { 7 stdout: Buffer; 8 stderr: Buffer; 9 status: number | null; 10 } 11 12 interface UserConfig { 13 types: string[]; 14 cloneUrl: string; 15 branch?: string; 16 path?: string; 17 } 18 19 abstract class ExternalCompileRunnerBase extends RunnerBase { 20 abstract testDir: string; 21 abstract report(result: ExecResult): string | null; 22 enumerateTestFiles() { 23 return IO.getDirectories(this.testDir); 24 } 25 /** Setup the runner's tests so that they are ready to be executed by the harness 26 * The first test should be a describe/it block that sets up the harness's compiler instance appropriately 27 */ 28 initializeTests(): void { 29 // Read in and evaluate the test list 30 const testList = this.tests && this.tests.length ? this.tests : this.getTestFiles(); 31 32 // eslint-disable-next-line @typescript-eslint/no-this-alias 33 const cls = this; 34 describe(`${this.kind()} code samples`, function (this: Mocha.Suite) { 35 this.timeout(600_000); // 10 minutes 36 for (const test of testList) { 37 cls.runTest(typeof test === "string" ? test : test.file); 38 } 39 }); 40 } 41 private runTest(directoryName: string) { 42 // eslint-disable-next-line @typescript-eslint/no-this-alias 43 const cls = this; 44 const timeout = 600_000; // 10 minutes 45 describe(directoryName, function (this: Mocha.Suite) { 46 this.timeout(timeout); 47 const cp: typeof import("child_process") = require("child_process"); 48 49 it("should build successfully", () => { 50 let cwd = path.join(IO.getWorkspaceRoot(), cls.testDir, directoryName); 51 const originalCwd = cwd; 52 const stdio = isWorker ? "pipe" : "inherit"; 53 let types: string[] | undefined; 54 if (fs.existsSync(path.join(cwd, "test.json"))) { 55 const config = JSON.parse(fs.readFileSync(path.join(cwd, "test.json"), { encoding: "utf8" })) as UserConfig; 56 ts.Debug.assert(!!config.types, "Bad format from test.json: Types field must be present."); 57 ts.Debug.assert(!!config.cloneUrl, "Bad format from test.json: cloneUrl field must be present."); 58 const submoduleDir = path.join(cwd, directoryName); 59 if (!fs.existsSync(submoduleDir)) { 60 exec("git", ["--work-tree", submoduleDir, "clone", "-b", config.branch || "master", config.cloneUrl, path.join(submoduleDir, ".git")], { cwd }); 61 } 62 else { 63 exec("git", ["--git-dir", path.join(submoduleDir, ".git"), "--work-tree", submoduleDir, "checkout", config.branch || "master"], { cwd: submoduleDir }); 64 exec("git", ["--git-dir", path.join(submoduleDir, ".git"), "--work-tree", submoduleDir, "reset", "HEAD", "--hard"], { cwd: submoduleDir }); 65 exec("git", ["--git-dir", path.join(submoduleDir, ".git"), "--work-tree", submoduleDir, "clean", "-f"], { cwd: submoduleDir }); 66 exec("git", ["--git-dir", path.join(submoduleDir, ".git"), "--work-tree", submoduleDir, "pull", "-f"], { cwd: submoduleDir }); 67 } 68 69 types = config.types; 70 71 cwd = config.path ? path.join(cwd, config.path) : submoduleDir; 72 } 73 const npmVersionText = exec("npm", ["--version"], { cwd, stdio: "pipe" })?.trim(); 74 const npmVersion = npmVersionText ? ts.Version.tryParse(npmVersionText.trim()) : undefined; 75 const isV7OrLater = !!npmVersion && npmVersion.major >= 7; 76 if (fs.existsSync(path.join(cwd, "package.json"))) { 77 if (fs.existsSync(path.join(cwd, "package-lock.json"))) { 78 fs.unlinkSync(path.join(cwd, "package-lock.json")); 79 } 80 if (fs.existsSync(path.join(cwd, "node_modules"))) { 81 del.sync(path.join(cwd, "node_modules"), { force: true }); 82 } 83 exec("npm", ["i", "--ignore-scripts", ...(isV7OrLater ? ["--legacy-peer-deps"] : [])], { cwd, timeout: timeout / 2 }); // NPM shouldn't take the entire timeout - if it takes a long time, it should be terminated and we should log the failure 84 } 85 const args = [path.join(IO.getWorkspaceRoot(), "built/local/tsc.js")]; 86 if (types) { 87 args.push("--types", types.join(",")); 88 // Also actually install those types (for, eg, the js projects which need node) 89 if (types.length) { 90 exec("npm", ["i", ...types.map(t => `@types/${t}`), "--no-save", "--ignore-scripts", ...(isV7OrLater ? ["--legacy-peer-deps"] : [])], { cwd: originalCwd, timeout: timeout / 2 }); // NPM shouldn't take the entire timeout - if it takes a long time, it should be terminated and we should log the failure 91 } 92 } 93 args.push("--noEmit"); 94 Baseline.runBaseline(`${cls.kind()}/${directoryName}.log`, cls.report(cp.spawnSync(`node`, args, { cwd, timeout, shell: true }))); 95 96 function exec(command: string, args: string[], options: { cwd: string, timeout?: number, stdio?: import("child_process").StdioOptions }): string | undefined { 97 const res = cp.spawnSync(isWorker ? `${command} 2>&1` : command, args, { shell: true, stdio, ...options }); 98 if (res.status !== 0) { 99 throw new Error(`${command} ${args.join(" ")} for ${directoryName} failed: ${res.stdout && res.stdout.toString()}`); 100 } 101 return options.stdio === "pipe" ? res.stdout.toString("utf8") : undefined; 102 } 103 }); 104 }); 105 } 106 } 107 108 export class UserCodeRunner extends ExternalCompileRunnerBase { 109 readonly testDir = "tests/cases/user/"; 110 kind(): TestRunnerKind { 111 return "user"; 112 } 113 report(result: ExecResult) { 114 // eslint-disable-next-line no-null/no-null 115 return result.status === 0 && !result.stdout.length && !result.stderr.length ? null : `Exit Code: ${result.status} 116Standard output: 117${sortErrors(stripAbsoluteImportPaths(result.stdout.toString().replace(/\r\n/g, "\n")))} 118 119 120Standard error: 121${stripAbsoluteImportPaths(result.stderr.toString().replace(/\r\n/g, "\n"))}`; 122 } 123 } 124 125 export class DockerfileRunner extends ExternalCompileRunnerBase { 126 readonly testDir = "tests/cases/docker/"; 127 kind(): TestRunnerKind { 128 return "docker"; 129 } 130 initializeTests(): void { 131 // Read in and evaluate the test list 132 const testList = this.tests && this.tests.length ? this.tests : this.getTestFiles(); 133 134 // eslint-disable-next-line @typescript-eslint/no-this-alias 135 const cls = this; 136 describe(`${this.kind()} code samples`, function (this: Mocha.Suite) { 137 this.timeout(cls.timeout); // 20 minutes 138 before(() => { 139 cls.exec("docker", ["build", ".", "-t", "typescript/typescript"], { cwd: IO.getWorkspaceRoot() }); // cached because workspace is hashed to determine cacheability 140 }); 141 for (const test of testList) { 142 const directory = typeof test === "string" ? test : test.file; 143 const cwd = path.join(IO.getWorkspaceRoot(), cls.testDir, directory); 144 it(`should build ${directory} successfully`, () => { 145 const imageName = `tstest/${directory}`; 146 cls.exec("docker", ["build", "--no-cache", ".", "-t", imageName], { cwd }); // --no-cache so the latest version of the repos referenced is always fetched 147 const cp: typeof import("child_process") = require("child_process"); 148 Baseline.runBaseline(`${cls.kind()}/${directory}.log`, cls.report(cp.spawnSync(`docker`, ["run", imageName], { cwd, timeout: cls.timeout, shell: true }))); 149 }); 150 } 151 }); 152 } 153 154 private timeout = 1_200_000; // 20 minutes; 155 private exec(command: string, args: string[], options: { cwd: string }): void { 156 const cp: typeof import("child_process") = require("child_process"); 157 const stdio = isWorker ? "pipe" : "inherit"; 158 const res = cp.spawnSync(isWorker ? `${command} 2>&1` : command, args, { timeout: this.timeout, shell: true, stdio, ...options }); 159 if (res.status !== 0) { 160 throw new Error(`${command} ${args.join(" ")} for ${options.cwd} failed: ${res.stdout && res.stdout.toString()}`); 161 } 162 } 163 report(result: ExecResult) { 164 // eslint-disable-next-line no-null/no-null 165 return result.status === 0 && !result.stdout.length && !result.stderr.length ? null : `Exit Code: ${result.status} 166Standard output: 167${sanitizeDockerfileOutput(result.stdout.toString())} 168 169 170Standard error: 171${sanitizeDockerfileOutput(result.stderr.toString())}`; 172 } 173 } 174 175 function sanitizeDockerfileOutput(result: string): string { 176 return [ 177 normalizeNewlines, 178 stripANSIEscapes, 179 stripRushStageNumbers, 180 stripWebpackHash, 181 sanitizeVersionSpecifiers, 182 sanitizeTimestamps, 183 sanitizeSizes, 184 sanitizeUnimportantGulpOutput, 185 stripAbsoluteImportPaths, 186 ].reduce((result, f) => f(result), result); 187 } 188 189 function normalizeNewlines(result: string): string { 190 return result.replace(/\r\n/g, "\n"); 191 } 192 193 function stripANSIEscapes(result: string): string { 194 return result.replace(/\x1b\[[0-9;]*[a-zA-Z]/g, ""); 195 } 196 197 function stripRushStageNumbers(result: string): string { 198 return result.replace(/\d+ of \d+:/g, "XX of XX:"); 199 } 200 201 function stripWebpackHash(result: string): string { 202 return result.replace(/Hash: \w+/g, "Hash: [redacted]"); 203 } 204 205 function sanitizeSizes(result: string): string { 206 return result.replace(/\d+(\.\d+)? ((Ki|M)B|bytes)/g, "X KiB"); 207 } 208 209 /** 210 * Gulp's output order within a `parallel` block is nondeterministic (and there's no way to configure it to execute in series), 211 * so we purge as much of the gulp output as we can 212 */ 213 function sanitizeUnimportantGulpOutput(result: string): string { 214 return result.replace(/^.*(\] (Starting)|(Finished)).*$/gm, "") // "gulp" task start/end messages (nondeterministic order) 215 .replace(/^.*(\] . (finished)|(started)).*$/gm, "") // "just" task start/end messages (nondeterministic order) 216 .replace(/^.*\] Respawned to PID: \d+.*$/gm, "") // PID of child is OS and system-load dependent (likely stableish in a container but still dangerous) 217 .replace(/\n+/g, "\n") 218 .replace(/\/tmp\/yarn--.*?\/node/g, ""); 219 } 220 221 function sanitizeTimestamps(result: string): string { 222 return result.replace(/\[\d?\d:\d\d:\d\d (A|P)M\]/g, "[XX:XX:XX XM]") 223 .replace(/\[\d?\d:\d\d:\d\d\]/g, "[XX:XX:XX]") 224 .replace(/\/\d+-\d+-[\d_TZ]+-debug.log/g, "\/XXXX-XX-XXXXXXXXX-debug.log") 225 .replace(/\d+\/\d+\/\d+ \d+:\d+:\d+ (AM|PM)/g, "XX/XX/XX XX:XX:XX XM") 226 .replace(/\d+(\.\d+)? sec(onds?)?/g, "? seconds") 227 .replace(/\d+(\.\d+)? min(utes?)?/g, "") 228 .replace(/\d+(\.\d+)? ?m?s/g, "?s") 229 .replace(/ \(\?s\)/g, ""); 230 } 231 232 function sanitizeVersionSpecifiers(result: string): string { 233 return result 234 .replace(/\d+.\d+.\d+-insiders.\d\d\d\d\d\d\d\d/g, "X.X.X-insiders.xxxxxxxx") 235 .replace(/Rush Multi-Project Build Tool (\d+)\.\d+\.\d+/g, "Rush Multi-Project Build Tool $1.X.X") 236 .replace(/([@v\()])\d+\.\d+\.\d+/g, "$1X.X.X") 237 .replace(/webpack (\d+)\.\d+\.\d+/g, "webpack $1.X.X") 238 .replace(/Webpack version: (\d+)\.\d+\.\d+/g, "Webpack version: $1.X.X"); 239 } 240 241 /** 242 * Import types and some other error messages use absolute paths in errors as they have no context to be written relative to; 243 * This is problematic for error baselines, so we grep for them and strip them out. 244 */ 245 function stripAbsoluteImportPaths(result: string) { 246 const workspaceRegexp = new RegExp(IO.getWorkspaceRoot().replace(/\\/g, "\\\\"), "g"); 247 return result 248 .replace(/import\(".*?\/tests\/cases\/user\//g, `import("/`) 249 .replace(/Module '".*?\/tests\/cases\/user\//g, `Module '"/`) 250 .replace(workspaceRegexp, "../../.."); 251 } 252 253 function sortErrors(result: string) { 254 return ts.flatten(splitBy(result.split("\n"), s => /^\S+/.test(s)).sort(compareErrorStrings)).join("\n"); 255 } 256 257 const errorRegexp = /^(.+\.[tj]sx?)\((\d+),(\d+)\)(: error TS.*)/; 258 function compareErrorStrings(a: string[], b: string[]) { 259 ts.Debug.assertGreaterThanOrEqual(a.length, 1); 260 ts.Debug.assertGreaterThanOrEqual(b.length, 1); 261 const matchA = a[0].match(errorRegexp); 262 if (!matchA) { 263 return -1; 264 } 265 const matchB = b[0].match(errorRegexp); 266 if (!matchB) { 267 return 1; 268 } 269 const [, errorFileA, lineNumberStringA, columnNumberStringA, remainderA] = matchA; 270 const [, errorFileB, lineNumberStringB, columnNumberStringB, remainderB] = matchB; 271 return ts.comparePathsCaseSensitive(errorFileA, errorFileB) || 272 ts.compareValues(parseInt(lineNumberStringA), parseInt(lineNumberStringB)) || 273 ts.compareValues(parseInt(columnNumberStringA), parseInt(columnNumberStringB)) || 274 ts.compareStringsCaseSensitive(remainderA, remainderB) || 275 ts.compareStringsCaseSensitive(a.slice(1).join("\n"), b.slice(1).join("\n")); 276 } 277 278 export class DefinitelyTypedRunner extends ExternalCompileRunnerBase { 279 readonly testDir = "../DefinitelyTyped/types/"; 280 workingDirectory = this.testDir; 281 kind(): TestRunnerKind { 282 return "dt"; 283 } 284 report(result: ExecResult) { 285 // eslint-disable-next-line no-null/no-null 286 return !result.stdout.length && !result.stderr.length ? null : `Exit Code: ${result.status} 287Standard output: 288${result.stdout.toString().replace(/\r\n/g, "\n")} 289 290 291Standard error: 292${result.stderr.toString().replace(/\r\n/g, "\n")}`; 293 } 294 } 295 296 /** 297 * Split an array into multiple arrays whenever `isStart` returns true. 298 * @example 299 * splitBy([1,2,3,4,5,6], isOdd) 300 * ==> [[1, 2], [3, 4], [5, 6]] 301 * where 302 * const isOdd = n => !!(n % 2) 303 */ 304 function splitBy<T>(xs: T[], isStart: (x: T) => boolean): T[][] { 305 const result = []; 306 let group: T[] = []; 307 for (const x of xs) { 308 if (isStart(x)) { 309 if (group.length) { 310 result.push(group); 311 } 312 group = [x]; 313 } 314 else { 315 group.push(x); 316 } 317 } 318 if (group.length) { 319 result.push(group); 320 } 321 return result; 322 } 323} 324