xref: /arkcompiler/ets_frontend/test262/harness.patch (revision 3af6ab5f)

diff --git a/bin/run.js b/bin/run.js
index 650f19a..7284fa1 100755
--- a/bin/run.js
+++ b/bin/run.js
@@ -76,6 +76,7 @@ if (argv.prelude) {
 let hostType;
 let hostPath;
 let features;
+let mode;
 
 if (argv.hostType) {
   hostType = argv.hostType;
@@ -123,6 +124,12 @@ if (argv.features) {
   features = argv.features.split(',').map(feature => feature.trim());
 }
 
+mode = "only strict mode"
+
+if (argv.mode) {
+  mode = argv.mode
+}
+
 // Show help if no arguments provided
 if (!argv._.length) {
   cli.showHelp();
@@ -130,17 +137,35 @@ if (!argv._.length) {
   return;
 }
 
-// Test Pipeline
-const pool = new AgentPool(
-  Number(argv.threads), hostType, argv.hostArgs, hostPath, { tempDir, timeout, transform }
-);
 
 if (!test262Dir) {
   test262Dir = test262Finder(argv._[0]);
 }
+
 reporterOpts.test262Dir = test262Dir;
+reporterOpts.tempDir = tempDir
+
+// Test Pipeline
+const pool = new AgentPool(
+  Number(argv.threads), hostType, argv.hostArgs, hostPath, { tempDir, timeout, transform, test262Dir }
+);
 
 const remove = path.relative(process.cwd(), test262Dir);
+if (argv.isTestListSet) {
+    let fileName = argv._[0]
+    if (!fs.existsSync(fileName)){
+        if (fileName.startsWith("test262/")) {
+            fileName = path.join(__dirname, "../../../", fileName)
+        } else {
+            fileName = path.join(__dirname, "../../", fileName)
+        }
+    }
+    const data = fs.readFileSync(fileName, "utf8")
+    argv._ = data
+        .split("\n")
+        .map(line => line.trim())
+        .filter(line => line.length > 0 && !line.startsWith("#"))
+}
 argv._ = argv._.map(p => path.relative(remove, p));
 
 let test262Version;
@@ -166,6 +191,7 @@ if (preprocessor) {
   tests = tests.pipe(filter(preprocessor));
 }
 
+tests = tests.pipe(filter(operMode));
 const results = zip(pool, tests).pipe(
   flatMap(pair => {
     return pool.runTest(pair);
@@ -209,3 +235,11 @@ function hasFeatures(test) {
   }
   return features.filter(feature => (test.attrs.features || []).includes(feature)).length > 0;
 }
+
+function operMode(test) {
+  test_scenario = test.scenario
+  if (mode.indexOf(test_scenario) != -1) {
+    return true;
+  }
+  return false;
+}
diff --git a/lib/agent-pool.js b/lib/agent-pool.js
index ad14b84..1b8a184 100644
--- a/lib/agent-pool.js
+++ b/lib/agent-pool.js
@@ -1,6 +1,6 @@
 'use strict';
 const {Subject} = require('rxjs');
-const eshost = require('eshost');
+const eshost = require('../../eshost/lib/eshost');
 
 const internal = new WeakMap();
 
@@ -18,6 +18,7 @@ class AgentPool extends Subject {
         shortName: '$262',
         transform: options.transform,
         out: options.tempDir,
+        test262Dir: options.test262Dir,
       })
       .then(agent => {
         this.agents.push(agent);
diff --git a/lib/cli.js b/lib/cli.js
index 4a74309..a330271 100644
--- a/lib/cli.js
+++ b/lib/cli.js
@@ -1,4 +1,4 @@
-const { supportedHosts } = require("eshost");
+const { supportedHosts } = require("./../../eshost/lib/eshost");
 const yargs = require('yargs');
 const yargv = yargs
   .strict()
@@ -22,6 +22,9 @@ const yargv = yargs
   .nargs('threads', 1)
   .default('threads', 1)
   .alias('threads', 't')
+  .nargs('mode', 1)
+  .default('mode', 1)
+  .alias('mode', 'm')
   .describe('reporter', 'format of data written to standard output')
   .choices('reporter', ['simple', 'json'])
   .nargs('reporter', 1)
@@ -33,6 +36,8 @@ const yargv = yargs
   .describe('timeout', 'test timeout (in ms, default 10000)')
   .nargs('timeout', 1)
   .describe('acceptVersion', 'override for supported Test262 version')
+  .boolean('isTestListSet')
+  .describe('isTestListSet', 'Set if positional argument contains test-list file')
   .boolean('saveCompiledTests')
   .describe('saveCompiledTests', 'Write the compiled version of path/to/test.js as path/to/test.js.<hostType>.<default|strict>.<pass|fail> so that it can be easily re-run under that host')
   .boolean('saveOnlyFailed')
diff --git a/lib/reporters/simple.js b/lib/reporters/simple.js
index 08f9a55..1579861 100644
--- a/lib/reporters/simple.js
+++ b/lib/reporters/simple.js
@@ -1,22 +1,30 @@
 'use strict';
 const path = require('path');
+const fs = require('fs');
 const saveCompiledTest = require('../saveCompiledTest');
+var xmlbuilder = require('xmlbuilder');
 
 function simpleReporter(results, opts) {
   let passed = 0;
   let failed = 0;
   let lastPassed = true;
+  let xmll = xmlbuilder.create("testsuite");
+  xmll.att("name", "Test 262");
 
   results.on('pass', function (test) {
     passed++;
 
     clearPassed();
     lastPassed = true;
-    process.stdout.write(`PASS ${test.file}`);
+    let mess = `PASS ${test.file} (${test.scenario})\n`
+    console.log(mess);
+    writeStatistics(mess, opts);
+
+    xmll.ele("testcase").att("name", test.file);
 
     if (opts.saveCompiledTests && !opts.saveOnlyFailed) {
       test.savedTestPath = saveCompiledTest(test, opts);
-      process.stdout.write(`\nSaved compiled passed test as ${test.savedTestPath}\n`);
+      // process.stdout.write(`\nSaved compiled passed test as ${test.savedTestPath}\n`);
     }
   });
 
@@ -24,14 +32,27 @@ function simpleReporter(results, opts) {
     failed++;
     clearPassed();
     lastPassed = false;
-    console.log(`FAIL ${test.file} (${test.scenario})`);
-    console.log(`  ${test.result.message}`);
+
+    let mess = `FAIL ${test.file} (${test.scenario})\n`
+    saveInfoToFile(test,opts);
+
+    console.log(mess);
+    console.log(`${test.result.message}`);
     console.log('');
 
+    writeStatistics(mess, opts);
+
+    var tc = xmll.ele("testcase");
+    tc.att("name", test.file);
+    var ff = tc.ele("failure");
+    var cd = `error message = ${test.result.message}\nOUT: ${test.result.stdout}\nERR: ${test.result.stderr}`
+    ff.cdata(cd)
+
     if (opts.saveCompiledTests) {
       test.savedTestPath = saveCompiledTest(test, opts);
-      process.stdout.write(`Saved compiled failed test as ${test.savedTestPath}\n`);
+      // process.stdout.write(`Saved compiled failed test as ${test.savedTestPath}\n`);
     }
+
   });
 
   results.on('end', function () {
@@ -40,6 +61,11 @@ function simpleReporter(results, opts) {
     console.log(`Ran ${(passed + failed)} tests`);
     console.log(`${passed} passed`);
     console.log(`${failed} failed`);
+
+    xmll.att("tests", passed + failed);
+    xmll.att("failures", failed);
+
+    fs.writeFileSync(path.join(opts.tempDir,"result.xml"), xmll.end({pretty: true}));
   });
 
   function clearPassed() {
@@ -52,6 +78,29 @@ function simpleReporter(results, opts) {
       }
     }
   }
+
+  function saveInfoToFile(test,opts){
+    let filePath = test.file;
+    let tmps = filePath.split(opts.test262Dir);
+    let outFile = path.join(opts.tempDir,tmps[1]);
+    let scenario = test.scenario === 'strict mode' ? 'strict' : test.scenario;
+    let outcome = 'err';
+    let savedTestPath = path.normalize(
+      `${outFile}.${opts.hostType}.${scenario}.${outcome}`
+    );
+    fs.writeFileSync(savedTestPath, `  ${test.result.message}`);
+  }
+
+  function writeStatistics(data, opts) {
+    let save_file = path.join(opts.tempDir,"result.txt");
+    fs.appendFile(save_file, data, 'utf8', function(err){
+      if(err)
+      {
+          console.error(err);
+      }
+    });
+  }
+
 }
 
 module.exports = simpleReporter;
diff --git a/lib/saveCompiledTest.js b/lib/saveCompiledTest.js
index c233adb..7739946 100644
--- a/lib/saveCompiledTest.js
+++ b/lib/saveCompiledTest.js
@@ -6,8 +6,11 @@ const path = require('path');
 module.exports = function saveCompiledTest(test, options) {
   let outcome = test.result.pass ? 'pass' : 'fail';
   let scenario = test.scenario === 'strict mode' ? 'strict' : test.scenario;
+  let filePath = test.file;
+  let tmps = filePath.split(options.test262Dir);
+  let outFile = path.join(options.tempDir,tmps[1]);
   let savedTestPath = path.normalize(
-    `${test.file}.${options.hostType}.${scenario}.${outcome}`
+    `${outFile}.${options.hostType}.${scenario}.${outcome}`
   );
   fs.writeFileSync(savedTestPath, test.compiled);
   return savedTestPath;
diff --git a/lib/validator.js b/lib/validator.js
index e7cb695..d4671a3 100644
--- a/lib/validator.js
+++ b/lib/validator.js
@@ -35,7 +35,7 @@ module.exports = function validate(test) {
       } else {
         return {
           pass: false,
-          message: `Expected no error, got ${result.error.name}: ${result.error.message}`,
+          message: `Expected no error, but got ${result.error.name}: \n ${result.stderr}`,
         };
       }
     } else if (!ranToFinish && !test.attrs.flags.raw) {
@@ -46,7 +46,7 @@ module.exports = function validate(test) {
       }
       return {
         pass: false,
-        message,
+        message: `Expected no error, but got : \n ${result.stderr}`,
       };
     } else {
       return {
@@ -78,7 +78,7 @@ module.exports = function validate(test) {
       } else {
         return {
           pass: false,
-          message: `Expected test to throw error of type ${test.attrs.negative.type}, got ${result.error.name}: ${result.error.message}`,
+          message: `Expected test to throw error of type ${test.attrs.negative.type}, but got ${result.error.name}: \n ${result.stderr}`,
         };
       }
     }
diff --git a/package.json b/package.json
index 60ef715..9e7ef0c 100644
--- a/package.json
+++ b/package.json
@@ -15,11 +15,11 @@
     "minimatch": "^3.0.4",
     "rxjs": "^6.4.0",
     "test262-stream": "^1.3.0",
-    "yargs": "^13.2.2"
+    "yargs": "^13.2.2",
+    "xmlbuilder": "^15.1.1"
   },
   "author": "Brian Terlson",
-  "license": "BSD-3-Clause",
-  "files": [
+  "license": "BSD-3-Clause",  "files": [
     "index.js",
     "bin",
     "lib",