feat(runner): Integrate Test262 execution with wptrunner and add smoke tests

jcscottiii · jcscottiii · commit 94dd56717fdb · 2025-12-19T18:46:18.000Z
This commit completes the core integration of Test262 into the WPT runner (`wptrunner`), enabling the execution of Test262 JavaScript conformance tests within the browser environment. This work builds upon the manifest tooling (PR 1 #56840) and server-side handling (PR 2 #56841) to provide a full end-to-end testing solution for Test262. Key components introduced and integrated: - **Client-Side Harness:** New JavaScript files (`harness-adapter.js`, `testharness-client.js`, `testharness.js`) under `resources/test262/` provide the necessary environment for Test262 tests to execute in an `iframe` and report results back to the main WPT testharness. - **Test262 Assertion Libraries (Vendored):** For the purpose of enabling these initial smoke tests, the standard Test262 assertion libraries (`assert.js` and `sta.js`) are directly placed into `third_party/test262/harness/`. This provides the immediate dependencies required by the smoke tests. An autoamted vendoring solution, including version tracking via `vendored.toml`, will be implemented in a subsequent PR focused on automated updates, as described in the RFC. - **Wptrunner Integration:** Modifications to `tools/wptrunner/wpttest.py` and `tools/wptrunner/browsers/*.py` enable `wptrunner` to recognize the `test262` test type and execute it using existing testharness executors. - **Smoke Tests and Metadata:** A suite of basic Test262 smoke tests (`infrastructure/test262/`) and their corresponding expected results (`infrastructure/metadata/infrastructure/test262/`) are added to verify the correct functioning of the entire integration. - **Linting Exemption:** `lint.ignore` is updated to exclude the vendored Test262 harness files from linting, respecting their upstream style. This integration allows browser vendors to run Test262 directly with `wptrunner`, streamlining conformance testing efforts as specified in the RFC: web-platform-tests/rfcs#229 This commit is the third in a series of smaller PRs split from the larger, original implementation in #55997.
diff --git a/infrastructure/metadata/infrastructure/test262/basic-failure.js.ini b/infrastructure/metadata/infrastructure/test262/basic-failure.js.ini
@@ -0,0 +1,4 @@
+[basic-failure.test262.html]
+  expected: OK
+  [Test]
+    expected: FAIL
diff --git a/infrastructure/metadata/infrastructure/test262/basic.js.ini b/infrastructure/metadata/infrastructure/test262/basic.js.ini
@@ -0,0 +1,4 @@
+[basic.test262.html]
+  expected: OK
+  [Test]
+    expected: PASS
diff --git a/infrastructure/metadata/infrastructure/test262/error.js.ini b/infrastructure/metadata/infrastructure/test262/error.js.ini
@@ -0,0 +1,4 @@
+[error.test262.html]
+  expected: OK
+  [Test]
+    expected: FAIL
diff --git a/infrastructure/metadata/infrastructure/test262/module-error.js.ini b/infrastructure/metadata/infrastructure/test262/module-error.js.ini
@@ -0,0 +1,4 @@
+[module-error.test262-module.html]
+  expected: ERROR
+  [Test]
+    expected: FAIL
diff --git a/infrastructure/metadata/infrastructure/test262/module-failure.js.ini b/infrastructure/metadata/infrastructure/test262/module-failure.js.ini
@@ -0,0 +1,4 @@
+[module-failure.test262-module.html]
+  expected: OK
+  [Test]
+    expected: FAIL
diff --git a/infrastructure/metadata/infrastructure/test262/module.js.ini b/infrastructure/metadata/infrastructure/test262/module.js.ini
@@ -0,0 +1,4 @@
+[module.test262-module.html]
+  expected: OK
+  [Test]
+    expected: PASS
diff --git a/infrastructure/metadata/infrastructure/test262/negative-error.js.ini b/infrastructure/metadata/infrastructure/test262/negative-error.js.ini
@@ -0,0 +1,4 @@
+[negative-error.test262.html]
+  expected: ERROR
+  [Test]
+    expected: FAIL
diff --git a/infrastructure/metadata/infrastructure/test262/negative-failure.js.ini b/infrastructure/metadata/infrastructure/test262/negative-failure.js.ini
@@ -0,0 +1,4 @@
+[negative-failure.test262.html]
+  expected: OK
+  [Test]
+    expected: FAIL
diff --git a/infrastructure/metadata/infrastructure/test262/negative.js.ini b/infrastructure/metadata/infrastructure/test262/negative.js.ini
@@ -0,0 +1,4 @@
+[negative.test262.html]
+  expected: OK
+  [Test]
+    expected: PASS
diff --git a/infrastructure/metadata/infrastructure/test262/strict-error.js.ini b/infrastructure/metadata/infrastructure/test262/strict-error.js.ini
@@ -0,0 +1,4 @@
+[strict-error.test262.strict.html]
+  expected: ERROR
+  [Test]
+    expected: FAIL
diff --git a/infrastructure/metadata/infrastructure/test262/strict-failure.js.ini b/infrastructure/metadata/infrastructure/test262/strict-failure.js.ini
@@ -0,0 +1,4 @@
+[strict-failure.test262.strict.html]
+  expected: OK
+  [Test]
+    expected: FAIL
diff --git a/infrastructure/metadata/infrastructure/test262/strict.js.ini b/infrastructure/metadata/infrastructure/test262/strict.js.ini
@@ -0,0 +1,4 @@
+[strict.test262.html]
+  expected: OK
+  [Test]
+    expected: PASS
diff --git a/infrastructure/metadata/infrastructure/test262/unexpected-error.js.ini b/infrastructure/metadata/infrastructure/test262/unexpected-error.js.ini
@@ -0,0 +1,4 @@
+[unexpected-error.test262.html]
+  expected: ERROR
+  [Test]
+    expected: FAIL
diff --git a/infrastructure/test262/basic-failure.js b/infrastructure/test262/basic-failure.js
@@ -0,0 +1,6 @@
+/*---
+description: A basic Test262 smoketest that fails
+features: [Test262]
+---*/
+
+assert.sameValue(1, 2, "One should be two");
diff --git a/infrastructure/test262/basic.js b/infrastructure/test262/basic.js
@@ -0,0 +1,6 @@
+/*---
+description: A basic Test262 smoketest
+features: [Test262]
+---*/
+
+assert.sameValue(1, 1, "One should be one");
diff --git a/infrastructure/test262/error.js b/infrastructure/test262/error.js
@@ -0,0 +1,5 @@
+/*---
+description: A smoketest that throws a Test262Error
+---*/
+
+throw new Test262Error("This is a deliberate test failure.");
diff --git a/infrastructure/test262/module-error.js b/infrastructure/test262/module-error.js
@@ -0,0 +1,6 @@
+/*---
+description: A module smoketest that throws an unexpected ReferenceError.
+flags: [module]
+---*/
+import { a } from "./support/module-helper.js";
+foo.bar();
diff --git a/infrastructure/test262/module-failure.js b/infrastructure/test262/module-failure.js
@@ -0,0 +1,6 @@
+/*---
+description: A module smoketest that fails an assertion.
+flags: [module]
+---*/
+import { a } from "./support/module-helper.js";
+assert.sameValue(a, 2, "a should be 2 in module");
diff --git a/infrastructure/test262/module.js b/infrastructure/test262/module.js
@@ -0,0 +1,5 @@
+/*---
+description: A module smoketest
+flags: [module]
+---*/
+import { a } from "./support/module-helper.js";
diff --git a/infrastructure/test262/negative-error.js b/infrastructure/test262/negative-error.js
@@ -0,0 +1,8 @@
+/*---
+description: A negative test that throws a different error than expected.
+negative:
+  phase: runtime
+  type: TypeError
+---*/
+
+throw new RangeError();
diff --git a/infrastructure/test262/negative-failure.js b/infrastructure/test262/negative-failure.js
@@ -0,0 +1,6 @@
+/*---
+description: A negative smoketest that is expected to fail because it does not throw an error.
+negative:
+  phase: runtime
+  type: Test262Error
+---*/
diff --git a/infrastructure/test262/negative.js b/infrastructure/test262/negative.js
@@ -0,0 +1,7 @@
+/*---
+description: A negative smoketest
+negative:
+  phase: runtime
+  type: Test262Error
+---*/
+throw new Test262Error();
diff --git a/infrastructure/test262/strict-error.js b/infrastructure/test262/strict-error.js
@@ -0,0 +1,6 @@
+/*---
+description: A strict mode smoketest that throws an unexpected ReferenceError.
+flags: [onlyStrict]
+---*/
+"use strict";
+foo.bar();
diff --git a/infrastructure/test262/strict-failure.js b/infrastructure/test262/strict-failure.js
@@ -0,0 +1,6 @@
+/*---
+description: A strict mode smoketest that fails an assertion.
+flags: [onlyStrict]
+---*/
+"use strict";
+assert.sameValue(1, 2, "One should be two in strict mode");
diff --git a/infrastructure/test262/strict.js b/infrastructure/test262/strict.js
@@ -0,0 +1,8 @@
+/*---
+description: A strict-mode smoketest from https://github.com/tc39/test262/blob/main/INTERPRETING.md
+negative:
+  phase: parse
+  type: SyntaxError
+---*/
+$DONOTEVALUATE();
+var a\u2E2F;
diff --git a/infrastructure/test262/support/module-helper.js b/infrastructure/test262/support/module-helper.js
@@ -0,0 +1 @@
+export const a = 1;
diff --git a/infrastructure/test262/unexpected-error.js b/infrastructure/test262/unexpected-error.js
@@ -0,0 +1,5 @@
+/*---
+description: A smoketest that throws an unexpected ReferenceError.
+---*/
+
+foo.bar();
diff --git a/lint.ignore b/lint.ignore
@@ -841,3 +841,5 @@ PROMISE_REJECTS: wasm/core/js/harness/testharness.js
 
 # Legitimate use of test_driver_internal
 TEST DRIVER INTERNAL: resources/testdriver.js
+
+*: third_party/test262/*
diff --git a/resources/test262/harness-adapter.js b/resources/test262/harness-adapter.js
@@ -0,0 +1,173 @@
+function installAPI(global) {
+  global.$262 = {
+    createRealm: function() {
+      var iframe = global.document.createElement('iframe');
+      iframe.style.cssText = 'display: none';
+      iframe.src = '';  // iframeSrc;
+      if (global.document.body === null) {
+        global.document.body = global.document.createElement('body');
+      }
+      global.document.body.appendChild(iframe);
+      return installAPI(iframe.contentWindow);
+    },
+    evalScript: function(src) {
+      var script = global.document.createElement('script');
+      script.text = src;
+      window.__test262_evalScript_error_ = undefined;
+      global.document.head.appendChild(script);
+      // Errors in the above appendChild bubble up to the global error handler.
+      // Our testharnes-client.js stashes them in a global var for rethrowing.
+      if (window.__test262_evalScript_error_) {
+          err = window.__test262_evalScript_error_;
+          window.__test262_evalScript_error_ = undefined;
+          throw err;
+      }
+    },
+    detachArrayBuffer: function(buffer) {
+      if (typeof postMessage !== 'function') {
+        throw new Error('No method available to detach an ArrayBuffer');
+      } else {
+        postMessage(null, '*', [buffer]);
+        /*
+          See
+          https://html.spec.whatwg.org/multipage/comms.html#dom-window-postmessage
+          which calls
+          https://html.spec.whatwg.org/multipage/infrastructure.html#structuredclonewithtransfer
+          which calls
+          https://html.spec.whatwg.org/multipage/infrastructure.html#transfer-abstract-op
+          which calls the DetachArrayBuffer abstract operation
+          https://tc39.github.io/ecma262/#sec-detacharraybuffer
+        */
+      }
+    },
+    gc: function() {
+      if (typeof gc !== 'function') {
+        throw new Error('No method available to invoke a GC');
+      }
+      gc();
+    },
+    AbstractModuleSource: function() {
+      throw new Error('AbstractModuleSource not available');
+    },
+    agent: (function () {
+      var workers = [];
+      var i32a = null;
+      var pendingReports = [];
+
+      // Agents call Atomics.wait on this location to sleep.
+      var SLEEP_LOC = 0;
+      // 1 if the started worker is ready, 0 otherwise.
+      var START_LOC = 1;
+      // The number of workers that have received the broadcast.
+      var BROADCAST_LOC = 2;
+      // Each worker has a count of outstanding reports; worker N uses memory
+      // location [WORKER_REPORT_LOC + N].
+      var WORKER_REPORT_LOC = 3;
+
+      function workerScript(script) {
+        return `
+          var index;
+          var i32a = null;
+          var broadcasts = [];
+          var pendingReceiver = null;
+
+          function handleBroadcast() {
+            if (pendingReceiver && broadcasts.length > 0) {
+              pendingReceiver.apply(null, broadcasts.shift());
+              pendingReceiver = null;
+            }
+          };
+
+          var onmessage = function({data:msg}) {
+            switch (msg.kind) {
+              case 'start':
+                i32a = msg.i32a;
+                index = msg.index;
+                (0, eval)(\`${script}\`);
+                break;
+
+              case 'broadcast':
+                Atomics.add(i32a, ${BROADCAST_LOC}, 1);
+                broadcasts.push([msg.sab, msg.id]);
+                handleBroadcast();
+                break;
+            }
+          };
+
+          var $262 = {
+            agent: {
+              receiveBroadcast(receiver) {
+                pendingReceiver = receiver;
+                handleBroadcast();
+              },
+
+              report(msg) {
+                postMessage(String(msg));
+                Atomics.add(i32a, ${WORKER_REPORT_LOC} + index, 1);
+              },
+
+              sleep(s) { Atomics.wait(i32a, ${SLEEP_LOC}, 0, s); },
+
+              leaving() {},
+
+              monotonicNow() {
+                return performance.now();
+              }
+            }
+          };`;
+      }
+
+      var agent = {
+        start(script) {
+          if (i32a === null) {
+            i32a = new Int32Array(new SharedArrayBuffer(256));
+          }
+          var w = new Worker(workerScript(script), {type: 'string'});
+          w.index = workers.length;
+          w.postMessage({kind: 'start', i32a: i32a, index: w.index});
+          workers.push(w);
+        },
+
+        broadcast(sab, id) {
+          if (!(sab instanceof SharedArrayBuffer)) {
+            throw new TypeError('sab must be a SharedArrayBuffer.');
+          }
+
+          Atomics.store(i32a, BROADCAST_LOC, 0);
+
+          for (var w of workers) {
+            w.postMessage({kind: 'broadcast', sab: sab, id: id|0});
+          }
+
+          while (Atomics.load(i32a, BROADCAST_LOC) != workers.length) {}
+        },
+
+        getReport() {
+          for (var w of workers) {
+            while (Atomics.load(i32a, WORKER_REPORT_LOC + w.index) > 0) {
+              pendingReports.push(w.getMessage());
+              Atomics.sub(i32a, WORKER_REPORT_LOC + w.index, 1);
+            }
+          }
+
+          return pendingReports.shift() || null;
+        },
+
+        sleep(s) { Atomics.wait(i32a, SLEEP_LOC, 0, s); },
+
+        monotonicNow() {
+          return performance.now();
+        }
+      };
+      return agent;
+
+    })(),
+    global: global
+  };
+  global.$DONE = function() {}
+
+  return global.$262;
+}
+
+installAPI(window);
+
diff --git a/resources/test262/testharness-client.js b/resources/test262/testharness-client.js
diff --git a/resources/test262/testharness.js b/resources/test262/testharness.js
diff --git a/third_party/test262/harness/assert.js b/third_party/test262/harness/assert.js
diff --git a/third_party/test262/harness/sta.js b/third_party/test262/harness/sta.js
diff --git a/tools/manifest/__init__.py b/tools/manifest/__init__.py
diff --git a/tools/manifest/requirements.txt b/tools/manifest/requirements.txt
diff --git a/tools/wptrunner/wptrunner/browsers/chrome.py b/tools/wptrunner/wptrunner/browsers/chrome.py
diff --git a/tools/wptrunner/wptrunner/browsers/edge.py b/tools/wptrunner/wptrunner/browsers/edge.py
diff --git a/tools/wptrunner/wptrunner/browsers/firefox.py b/tools/wptrunner/wptrunner/browsers/firefox.py
diff --git a/tools/wptrunner/wptrunner/browsers/safari.py b/tools/wptrunner/wptrunner/browsers/safari.py
diff --git a/tools/wptrunner/wptrunner/browsers/servo.py b/tools/wptrunner/wptrunner/browsers/servo.py
diff --git a/tools/wptrunner/wptrunner/browsers/webkit.py b/tools/wptrunner/wptrunner/browsers/webkit.py
diff --git a/tools/wptrunner/wptrunner/wpttest.py b/tools/wptrunner/wptrunner/wpttest.py
