Add testing harness & improve typing

.eslintrc.cjs

@@ -107,5 +107,9 @@ module.exports = {
         "@typescript-eslint/no-var-requires": "off",
       },
     },
+    {
+      files: ["scripts/*.mjs"],
+      env: { node: true },
+    },
   ],
 };

.gitignore

@@ -1,2 +1,2 @@
 node_modules/
-dist/behaviors.js
+dist

package.json

@@ -5,24 +5,28 @@
   "author": "Webrecorder Software",
   "license": "AGPL-3.0-or-later",
   "devDependencies": {
+    "@types/query-selector-shadow-dom": "^1.0.4",
     "@typescript-eslint/eslint-plugin": "^8.28.0",
     "@typescript-eslint/parser": "^8.28.0",
     "@webpack-cli/init": "^1.1.3",
+    "eslint": "^8.56.0",
+    "eslint-config-prettier": "^9.1.0",
+    "prettier": "^3.5.3",
     "ts-loader": "^9.4.2",
     "typescript": "^5.7.3",
     "webpack": "^5.75.0",
     "webpack-cli": "^4.5.0",
-    "webpack-dev-server": "^3.11.2",
-    "eslint": "^8.56.0",
-    "eslint-config-prettier": "^9.1.0"
+    "webpack-dev-server": "^3.11.2"
   },
   "scripts": {
     "build": "webpack --mode production",
     "build-dev": "webpack --mode development",
     "build-dev-copy": "webpack --mode development && cat ./dist/behaviors.js | pbcopy",
     "watch": "webpack watch --mode production",
     "watch-dev": "webpack watch --mode development",
-    "lint": "eslint ./src/**/*.ts webpack.config.js"
+    "lint": "eslint ./src/**/*.ts webpack.config.js",
+    "format": "prettier --write ./src/**/*.ts webpack.config.js",
+    "test": "node ./scripts/test-harness.mjs"
   },
   "description": "Browsertrix Behaviors",
   "files": [
@@ -32,5 +36,9 @@
   ],
   "dependencies": {
     "query-selector-shadow-dom": "^1.0.1"
+  },
+  "optionalDependencies": {
+    "memfs": "^4.17.0",
+    "puppeteer": "^24.7.2"
   }
 }

scripts/test-harness.mjs

@@ -0,0 +1,72 @@
+import puppeteer from "puppeteer";
+import Webpack from "webpack";
+import { fs } from "memfs";
+
+import webpackConfig from "../webpack.config.js";
+
+/**
+ * Validate a URL
+ * @param {URL} url
+ * @returns {boolean}
+ */
+const validateUrl = (url) => {
+  try {
+    return new URL(url);
+  } catch (_e) {
+    return false;
+  }
+};
+
+if (!process.argv[2]) {
+  console.error("Usage: yarn test '<url>'");
+  process.exit(1);
+}
+
+if (!validateUrl(process.argv[2])) {
+  console.error("Invalid URL (hint: include http:// or https://)");
+  process.exit(1);
+}
+
+const config = webpackConfig({}, { mode: "development" });
+
+const compiler = Webpack(config);
+compiler.outputFileSystem = fs;
+
+const browser = await puppeteer.launch({ headless: false, devtools: true });
+const page = await browser.newPage();
+
+const _watching = compiler.watch({}, async (err, stats) => {
+  if (err) {
+    console.error(err);
+    console.error("Not opening browser");
+    return;
+  }
+  console.log(
+    stats.toString({
+      colors: true,
+      preset: "summary",
+    }),
+  );
+  const behaviorScript = fs.readFileSync("dist/behaviors.js", "utf8");
+
+  await page.goto(validateUrl(process.argv[2]));
+
+  await page.evaluate(
+    behaviorScript +
+      `
+  self.__bx_behaviors.init({
+    autofetch: true,
+    autoplay: true,
+    autoscroll: true,
+    siteSpecific: true,
+  });
+  `,
+  );
+
+  // call and await run on top frame and all child iframes
+  await Promise.allSettled(
+    page
+      .frames()
+      .map(async (frame) => frame.evaluate("self.__bx_behaviors.run()")),
+  );
+});

src/autoclick.ts

@@ -3,8 +3,7 @@ import { addToExternalSet, sleep } from "./lib/utils";
 
 declare let getEventListeners: any;
 
-export class AutoClick extends BackgroundBehavior
-{
+export class AutoClick extends BackgroundBehavior {
   _donePromise: Promise<void>;
   _markDone: () => void;
   selector: string;
@@ -15,10 +14,12 @@ export class AutoClick extends BackgroundBehavior
   constructor(selector = "a") {
     super();
     this.selector = selector;
-    this._donePromise = new Promise<void>((resolve) => this._markDone = resolve);
+    this._donePromise = new Promise<void>(
+      (resolve) => (this._markDone = resolve),
+    );
   }
 
-  nextSameOriginLink() : HTMLAnchorElement | null {
+  nextSameOriginLink(): HTMLAnchorElement | null {
     try {
       const allLinks = document.querySelectorAll(this.selector);
       for (const el of allLinks) {
@@ -49,7 +50,7 @@ export class AutoClick extends BackgroundBehavior
 
   async start() {
     const origHref = self.location.href;
-    
+
     const beforeUnload = (event) => {
       event.preventDefault();
       return false;
@@ -92,7 +93,7 @@ export class AutoClick extends BackgroundBehavior
 
     if (elem.href) {
       // skip if already clicked this URL, tracked in external state
-      if (!await addToExternalSet(elem.href)) {
+      if (!(await addToExternalSet(elem.href))) {
         return;
       }
 
@@ -107,18 +108,23 @@ export class AutoClick extends BackgroundBehavior
 
     if (self.location.href != origHref) {
       await new Promise((resolve) => {
-        window.addEventListener("popstate", () => {
-          resolve(null);
-        }, { once: true });
+        window.addEventListener(
+          "popstate",
+          () => {
+            resolve(null);
+          },
+          { once: true },
+        );
 
         window.history.back();
       });
     }
-  } catch (e) {
+  }
+  catch(e) {
     this.debug(e.toString());
   }
 
-  done() {
+  async done() {
     return this._donePromise;
   }
 }

src/autofetcher.ts

@@ -9,7 +9,8 @@ import { querySelectorAllDeep } from "query-selector-shadow-dom";
 import { BackgroundBehavior } from "./lib/behavior";
 import { doExternalFetch, sleep, xpathNodes } from "./lib/utils";
 
-const SRC_SET_SELECTOR = "img[srcset], img[data-srcset], img[data-src], noscript > img[src], img[loading='lazy'], " +
+const SRC_SET_SELECTOR =
+  "img[srcset], img[data-srcset], img[data-src], noscript > img[src], img[loading='lazy'], " +
   "video[srcset], video[data-srcset], video[data-src], audio[srcset], audio[data-srcset], audio[data-src], " +
   "picture > source[srcset], picture > source[data-srcset], picture > source[data-src], " +
   "video > source[srcset], video > source[data-srcset], video > source[data-src], " +
@@ -22,18 +23,17 @@ const IMPORT_REGEX = /(@import\s*[\\"']*)([^)'";]+)([\\"']*\s*;?)/gi;
 
 const MAX_CONCURRENT = 6;
 
-
 // ===========================================================================
 export class AutoFetcher extends BackgroundBehavior {
-  urlSet: Set<string> = new Set();
+  urlSet = new Set<string>();
   pendingQueue: string[] = [];
   waitQueue: string[] = [];
-  mutationObserver: MutationObserver;
-  numPending: number = 0;
-  numDone: number = 0;
+  mutationObserver?: MutationObserver;
+  numPending = 0;
+  numDone = 0;
   headers: object;
   _donePromise: Promise<null>;
-  _markDone: (value: any) => void;
+  _markDone?: (value: any) => void;
   active: boolean;
   running = false;
 
@@ -44,7 +44,7 @@ export class AutoFetcher extends BackgroundBehavior {
 
     this.headers = headers || {};
 
-    this._donePromise = new Promise((resolve) => this._markDone = resolve);
+    this._donePromise = new Promise((resolve) => (this._markDone = resolve));
 
     this.active = active;
     if (this.active && startEarly) {
@@ -72,7 +72,7 @@ export class AutoFetcher extends BackgroundBehavior {
     });
   }
 
-  done() {
+  async done() {
     return this._donePromise;
   }
 
@@ -93,10 +93,10 @@ export class AutoFetcher extends BackgroundBehavior {
     return url && (url.startsWith("http:") || url.startsWith("https:"));
   }
 
-  queueUrl(url: string, immediate: boolean = false) {
+  queueUrl(url: string, immediate = false) {
     try {
       url = new URL(url, document.baseURI).href;
-    } catch (e) {
+    } catch (_e) {
       return false;
     }
 
@@ -122,7 +122,10 @@ export class AutoFetcher extends BackgroundBehavior {
   // fetch with default CORS mode, read entire stream
   async doFetchStream(url: string) {
     try {
-      const resp = await fetch(url, { "credentials": "include", "referrerPolicy": "origin-when-cross-origin" });
+      const resp = await fetch(url, {
+        credentials: "include",
+        referrerPolicy: "origin-when-cross-origin",
+      });
       this.debug(`Autofetch: started ${url}`);
 
       const reader = resp.body.getReader();
@@ -133,7 +136,6 @@ export class AutoFetcher extends BackgroundBehavior {
       this.debug(`Autofetch: finished ${url}`);
 
       return true;
-
     } catch (e) {
       this.debug(e);
 
@@ -146,15 +148,15 @@ export class AutoFetcher extends BackgroundBehavior {
     try {
       const abort = new AbortController();
       await fetch(url, {
-        "mode": "no-cors",
-        "credentials": "include",
-        "referrerPolicy": "origin-when-cross-origin",
-        "headers": this.headers,
-        abort
+        mode: "no-cors",
+        credentials: "include",
+        referrerPolicy: "origin-when-cross-origin",
+        headers: this.headers,
+        abort,
       } as {});
       abort.abort();
       this.debug(`Autofetch: started non-cors stream for ${url}`);
-    } catch (e) {
+    } catch (_e) {
       this.debug(`Autofetch: failed non-cors for ${url}`);
     }
   }
@@ -167,7 +169,7 @@ export class AutoFetcher extends BackgroundBehavior {
 
         this.numPending++;
 
-        let success = await doExternalFetch(url);
+        const success = await doExternalFetch(url);
 
         if (!success) {
           await this.doFetchNonCors(url);
@@ -186,7 +188,9 @@ export class AutoFetcher extends BackgroundBehavior {
     if (this.mutationObserver) {
       return;
     }
-    this.mutationObserver = new MutationObserver((changes) => this.observeChange(changes));
+    this.mutationObserver = new MutationObserver((changes) =>
+      this.observeChange(changes),
+    );
 
     this.mutationObserver.observe(document.documentElement, {
       characterData: false,
@@ -195,7 +199,7 @@ export class AutoFetcher extends BackgroundBehavior {
       attributeOldValue: true,
       subtree: true,
       childList: true,
-      attributeFilter: ["srcset", "loading"]
+      attributeFilter: ["srcset", "loading"],
     });
   }
 
@@ -282,7 +286,10 @@ export class AutoFetcher extends BackgroundBehavior {
     // check regular src in case of <noscript> only to avoid duplicate loading
     const src = elem.getAttribute("src");
 
-    if (src && (srcset || data_srcset || elem.parentElement.tagName === "NOSCRIPT")) {
+    if (
+      src &&
+      (srcset || data_srcset || elem.parentElement.tagName === "NOSCRIPT")
+    ) {
       this.queueUrl(src);
     }
   }
@@ -309,7 +316,7 @@ export class AutoFetcher extends BackgroundBehavior {
 
     try {
       rules = sheet.cssRules || sheet.rules;
-    } catch (e) {
+    } catch (_e) {
       this.debug("Can't access stylesheet");
       return;
     }
@@ -331,12 +338,12 @@ export class AutoFetcher extends BackgroundBehavior {
   }
 
   extractDataAttributes(root) {
-    const QUERY = "//@*[starts-with(name(), 'data-') and " +
-    "(starts-with(., 'http') or starts-with(., '/') or starts-with(., './') or starts-with(., '../'))]";
-
+    const QUERY =
+      "//@*[starts-with(name(), 'data-') and " +
+      "(starts-with(., 'http') or starts-with(., '/') or starts-with(., './') or starts-with(., '../'))]";
+
     for (const attr of xpathNodes(QUERY, root)) {
       this.queueUrl(attr.value);
     }
   }
 }
-