diff --git a/.eslintrc.cjs b/.eslintrc.cjs index c719a88..f35e704 100644 --- a/.eslintrc.cjs +++ b/.eslintrc.cjs @@ -107,5 +107,9 @@ module.exports = { "@typescript-eslint/no-var-requires": "off", }, }, + { + files: ["scripts/*.mjs"], + env: { node: true }, + }, ], }; diff --git a/.gitignore b/.gitignore index 2a6dddb..940d56a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,2 @@ node_modules/ -dist/behaviors.js +dist diff --git a/package.json b/package.json index 0db2d3b..021472e 100644 --- a/package.json +++ b/package.json @@ -5,16 +5,18 @@ "author": "Webrecorder Software", "license": "AGPL-3.0-or-later", "devDependencies": { + "@types/query-selector-shadow-dom": "^1.0.4", "@typescript-eslint/eslint-plugin": "^8.28.0", "@typescript-eslint/parser": "^8.28.0", "@webpack-cli/init": "^1.1.3", + "eslint": "^8.56.0", + "eslint-config-prettier": "^9.1.0", + "prettier": "^3.5.3", "ts-loader": "^9.4.2", "typescript": "^5.7.3", "webpack": "^5.75.0", "webpack-cli": "^4.5.0", - "webpack-dev-server": "^3.11.2", - "eslint": "^8.56.0", - "eslint-config-prettier": "^9.1.0" + "webpack-dev-server": "^3.11.2" }, "scripts": { "build": "webpack --mode production", @@ -22,7 +24,9 @@ "build-dev-copy": "webpack --mode development && cat ./dist/behaviors.js | pbcopy", "watch": "webpack watch --mode production", "watch-dev": "webpack watch --mode development", - "lint": "eslint ./src/**/*.ts webpack.config.js" + "lint": "eslint ./src/**/*.ts webpack.config.js", + "format": "prettier --write ./src/**/*.ts webpack.config.js", + "test": "node ./scripts/test-harness.mjs" }, "description": "Browsertrix Behaviors", "files": [ @@ -32,5 +36,9 @@ ], "dependencies": { "query-selector-shadow-dom": "^1.0.1" + }, + "optionalDependencies": { + "memfs": "^4.17.0", + "puppeteer": "^24.7.2" } } diff --git a/scripts/test-harness.mjs b/scripts/test-harness.mjs new file mode 100644 index 0000000..f974d3b --- /dev/null +++ b/scripts/test-harness.mjs @@ -0,0 +1,72 @@ +import puppeteer from "puppeteer"; +import Webpack from "webpack"; +import { fs } from "memfs"; + +import webpackConfig from "../webpack.config.js"; + +/** + * Validate a URL + * @param {URL} url + * @returns {boolean} + */ +const validateUrl = (url) => { + try { + return new URL(url); + } catch (_e) { + return false; + } +}; + +if (!process.argv[2]) { + console.error("Usage: yarn test ''"); + process.exit(1); +} + +if (!validateUrl(process.argv[2])) { + console.error("Invalid URL (hint: include http:// or https://)"); + process.exit(1); +} + +const config = webpackConfig({}, { mode: "development" }); + +const compiler = Webpack(config); +compiler.outputFileSystem = fs; + +const browser = await puppeteer.launch({ headless: false, devtools: true }); +const page = await browser.newPage(); + +const _watching = compiler.watch({}, async (err, stats) => { + if (err) { + console.error(err); + console.error("Not opening browser"); + return; + } + console.log( + stats.toString({ + colors: true, + preset: "summary", + }), + ); + const behaviorScript = fs.readFileSync("dist/behaviors.js", "utf8"); + + await page.goto(validateUrl(process.argv[2])); + + await page.evaluate( + behaviorScript + + ` + self.__bx_behaviors.init({ + autofetch: true, + autoplay: true, + autoscroll: true, + siteSpecific: true, + }); + `, + ); + + // call and await run on top frame and all child iframes + await Promise.allSettled( + page + .frames() + .map(async (frame) => frame.evaluate("self.__bx_behaviors.run()")), + ); +}); diff --git a/src/autoclick.ts b/src/autoclick.ts index 28e80df..a0be360 100644 --- a/src/autoclick.ts +++ b/src/autoclick.ts @@ -3,8 +3,7 @@ import { addToExternalSet, sleep } from "./lib/utils"; declare let getEventListeners: any; -export class AutoClick extends BackgroundBehavior -{ +export class AutoClick extends BackgroundBehavior { _donePromise: Promise; _markDone: () => void; selector: string; @@ -15,10 +14,12 @@ export class AutoClick extends BackgroundBehavior constructor(selector = "a") { super(); this.selector = selector; - this._donePromise = new Promise((resolve) => this._markDone = resolve); + this._donePromise = new Promise( + (resolve) => (this._markDone = resolve), + ); } - nextSameOriginLink() : HTMLAnchorElement | null { + nextSameOriginLink(): HTMLAnchorElement | null { try { const allLinks = document.querySelectorAll(this.selector); for (const el of allLinks) { @@ -49,7 +50,7 @@ export class AutoClick extends BackgroundBehavior async start() { const origHref = self.location.href; - + const beforeUnload = (event) => { event.preventDefault(); return false; @@ -92,7 +93,7 @@ export class AutoClick extends BackgroundBehavior if (elem.href) { // skip if already clicked this URL, tracked in external state - if (!await addToExternalSet(elem.href)) { + if (!(await addToExternalSet(elem.href))) { return; } @@ -107,18 +108,23 @@ export class AutoClick extends BackgroundBehavior if (self.location.href != origHref) { await new Promise((resolve) => { - window.addEventListener("popstate", () => { - resolve(null); - }, { once: true }); + window.addEventListener( + "popstate", + () => { + resolve(null); + }, + { once: true }, + ); window.history.back(); }); } - } catch (e) { + } + catch(e) { this.debug(e.toString()); } - done() { + async done() { return this._donePromise; } } diff --git a/src/autofetcher.ts b/src/autofetcher.ts index d36b9b8..b4c8ec7 100644 --- a/src/autofetcher.ts +++ b/src/autofetcher.ts @@ -9,7 +9,8 @@ import { querySelectorAllDeep } from "query-selector-shadow-dom"; import { BackgroundBehavior } from "./lib/behavior"; import { doExternalFetch, sleep, xpathNodes } from "./lib/utils"; -const SRC_SET_SELECTOR = "img[srcset], img[data-srcset], img[data-src], noscript > img[src], img[loading='lazy'], " + +const SRC_SET_SELECTOR = + "img[srcset], img[data-srcset], img[data-src], noscript > img[src], img[loading='lazy'], " + "video[srcset], video[data-srcset], video[data-src], audio[srcset], audio[data-srcset], audio[data-src], " + "picture > source[srcset], picture > source[data-srcset], picture > source[data-src], " + "video > source[srcset], video > source[data-srcset], video > source[data-src], " + @@ -22,18 +23,17 @@ const IMPORT_REGEX = /(@import\s*[\\"']*)([^)'";]+)([\\"']*\s*;?)/gi; const MAX_CONCURRENT = 6; - // =========================================================================== export class AutoFetcher extends BackgroundBehavior { - urlSet: Set = new Set(); + urlSet = new Set(); pendingQueue: string[] = []; waitQueue: string[] = []; - mutationObserver: MutationObserver; - numPending: number = 0; - numDone: number = 0; + mutationObserver?: MutationObserver; + numPending = 0; + numDone = 0; headers: object; _donePromise: Promise; - _markDone: (value: any) => void; + _markDone?: (value: any) => void; active: boolean; running = false; @@ -44,7 +44,7 @@ export class AutoFetcher extends BackgroundBehavior { this.headers = headers || {}; - this._donePromise = new Promise((resolve) => this._markDone = resolve); + this._donePromise = new Promise((resolve) => (this._markDone = resolve)); this.active = active; if (this.active && startEarly) { @@ -72,7 +72,7 @@ export class AutoFetcher extends BackgroundBehavior { }); } - done() { + async done() { return this._donePromise; } @@ -93,10 +93,10 @@ export class AutoFetcher extends BackgroundBehavior { return url && (url.startsWith("http:") || url.startsWith("https:")); } - queueUrl(url: string, immediate: boolean = false) { + queueUrl(url: string, immediate = false) { try { url = new URL(url, document.baseURI).href; - } catch (e) { + } catch (_e) { return false; } @@ -122,7 +122,10 @@ export class AutoFetcher extends BackgroundBehavior { // fetch with default CORS mode, read entire stream async doFetchStream(url: string) { try { - const resp = await fetch(url, { "credentials": "include", "referrerPolicy": "origin-when-cross-origin" }); + const resp = await fetch(url, { + credentials: "include", + referrerPolicy: "origin-when-cross-origin", + }); this.debug(`Autofetch: started ${url}`); const reader = resp.body.getReader(); @@ -133,7 +136,6 @@ export class AutoFetcher extends BackgroundBehavior { this.debug(`Autofetch: finished ${url}`); return true; - } catch (e) { this.debug(e); @@ -146,15 +148,15 @@ export class AutoFetcher extends BackgroundBehavior { try { const abort = new AbortController(); await fetch(url, { - "mode": "no-cors", - "credentials": "include", - "referrerPolicy": "origin-when-cross-origin", - "headers": this.headers, - abort + mode: "no-cors", + credentials: "include", + referrerPolicy: "origin-when-cross-origin", + headers: this.headers, + abort, } as {}); abort.abort(); this.debug(`Autofetch: started non-cors stream for ${url}`); - } catch (e) { + } catch (_e) { this.debug(`Autofetch: failed non-cors for ${url}`); } } @@ -167,7 +169,7 @@ export class AutoFetcher extends BackgroundBehavior { this.numPending++; - let success = await doExternalFetch(url); + const success = await doExternalFetch(url); if (!success) { await this.doFetchNonCors(url); @@ -186,7 +188,9 @@ export class AutoFetcher extends BackgroundBehavior { if (this.mutationObserver) { return; } - this.mutationObserver = new MutationObserver((changes) => this.observeChange(changes)); + this.mutationObserver = new MutationObserver((changes) => + this.observeChange(changes), + ); this.mutationObserver.observe(document.documentElement, { characterData: false, @@ -195,7 +199,7 @@ export class AutoFetcher extends BackgroundBehavior { attributeOldValue: true, subtree: true, childList: true, - attributeFilter: ["srcset", "loading"] + attributeFilter: ["srcset", "loading"], }); } @@ -282,7 +286,10 @@ export class AutoFetcher extends BackgroundBehavior { // check regular src in case of