From 7c852b1f0e0ad0cb93973cd5f30a6116ca773ea4 Mon Sep 17 00:00:00 2001 From: emma Date: Mon, 28 Apr 2025 19:04:22 -0400 Subject: [PATCH 1/5] wip --- .gitignore | 2 +- package.json | 11 ++- src/autoclick.ts | 28 +++--- src/autofetcher.ts | 57 ++++++----- src/behavior.d.ts | 42 ++++++++ src/index.ts | 121 +++++++++++++++++------ src/lib/behavior.ts | 95 ++++++++++-------- src/lib/utils.ts | 220 ++++++++++++++++++++++++++++-------------- src/site/facebook.ts | 86 +++++++++++------ src/site/index.ts | 4 +- src/site/instagram.ts | 118 ++++++++++++++-------- src/site/telegram.ts | 19 +++- src/site/tiktok.ts | 30 +++--- src/site/twitter.ts | 123 +++++++++++++++-------- tsconfig.json | 20 +--- webpack.config.js | 8 +- yarn.lock | 10 ++ 17 files changed, 661 insertions(+), 333 deletions(-) create mode 100644 src/behavior.d.ts diff --git a/.gitignore b/.gitignore index 2a6dddb..940d56a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,2 @@ node_modules/ -dist/behaviors.js +dist diff --git a/package.json b/package.json index 0db2d3b..90cf7ab 100644 --- a/package.json +++ b/package.json @@ -5,16 +5,18 @@ "author": "Webrecorder Software", "license": "AGPL-3.0-or-later", "devDependencies": { + "@types/query-selector-shadow-dom": "^1.0.4", "@typescript-eslint/eslint-plugin": "^8.28.0", "@typescript-eslint/parser": "^8.28.0", "@webpack-cli/init": "^1.1.3", + "eslint": "^8.56.0", + "eslint-config-prettier": "^9.1.0", + "prettier": "^3.5.3", "ts-loader": "^9.4.2", "typescript": "^5.7.3", "webpack": "^5.75.0", "webpack-cli": "^4.5.0", - "webpack-dev-server": "^3.11.2", - "eslint": "^8.56.0", - "eslint-config-prettier": "^9.1.0" + "webpack-dev-server": "^3.11.2" }, "scripts": { "build": "webpack --mode production", @@ -22,7 +24,8 @@ "build-dev-copy": "webpack --mode development && cat ./dist/behaviors.js | pbcopy", "watch": "webpack watch --mode production", "watch-dev": "webpack watch --mode development", - "lint": "eslint ./src/**/*.ts webpack.config.js" + "lint": "eslint ./src/**/*.ts webpack.config.js", + "format": "prettier --write ./src/**/*.ts webpack.config.js" }, "description": "Browsertrix Behaviors", "files": [ diff --git a/src/autoclick.ts b/src/autoclick.ts index 28e80df..a0be360 100644 --- a/src/autoclick.ts +++ b/src/autoclick.ts @@ -3,8 +3,7 @@ import { addToExternalSet, sleep } from "./lib/utils"; declare let getEventListeners: any; -export class AutoClick extends BackgroundBehavior -{ +export class AutoClick extends BackgroundBehavior { _donePromise: Promise; _markDone: () => void; selector: string; @@ -15,10 +14,12 @@ export class AutoClick extends BackgroundBehavior constructor(selector = "a") { super(); this.selector = selector; - this._donePromise = new Promise((resolve) => this._markDone = resolve); + this._donePromise = new Promise( + (resolve) => (this._markDone = resolve), + ); } - nextSameOriginLink() : HTMLAnchorElement | null { + nextSameOriginLink(): HTMLAnchorElement | null { try { const allLinks = document.querySelectorAll(this.selector); for (const el of allLinks) { @@ -49,7 +50,7 @@ export class AutoClick extends BackgroundBehavior async start() { const origHref = self.location.href; - + const beforeUnload = (event) => { event.preventDefault(); return false; @@ -92,7 +93,7 @@ export class AutoClick extends BackgroundBehavior if (elem.href) { // skip if already clicked this URL, tracked in external state - if (!await addToExternalSet(elem.href)) { + if (!(await addToExternalSet(elem.href))) { return; } @@ -107,18 +108,23 @@ export class AutoClick extends BackgroundBehavior if (self.location.href != origHref) { await new Promise((resolve) => { - window.addEventListener("popstate", () => { - resolve(null); - }, { once: true }); + window.addEventListener( + "popstate", + () => { + resolve(null); + }, + { once: true }, + ); window.history.back(); }); } - } catch (e) { + } + catch(e) { this.debug(e.toString()); } - done() { + async done() { return this._donePromise; } } diff --git a/src/autofetcher.ts b/src/autofetcher.ts index d36b9b8..a79ff5e 100644 --- a/src/autofetcher.ts +++ b/src/autofetcher.ts @@ -9,7 +9,8 @@ import { querySelectorAllDeep } from "query-selector-shadow-dom"; import { BackgroundBehavior } from "./lib/behavior"; import { doExternalFetch, sleep, xpathNodes } from "./lib/utils"; -const SRC_SET_SELECTOR = "img[srcset], img[data-srcset], img[data-src], noscript > img[src], img[loading='lazy'], " + +const SRC_SET_SELECTOR = + "img[srcset], img[data-srcset], img[data-src], noscript > img[src], img[loading='lazy'], " + "video[srcset], video[data-srcset], video[data-src], audio[srcset], audio[data-srcset], audio[data-src], " + "picture > source[srcset], picture > source[data-srcset], picture > source[data-src], " + "video > source[srcset], video > source[data-srcset], video > source[data-src], " + @@ -22,18 +23,17 @@ const IMPORT_REGEX = /(@import\s*[\\"']*)([^)'";]+)([\\"']*\s*;?)/gi; const MAX_CONCURRENT = 6; - // =========================================================================== export class AutoFetcher extends BackgroundBehavior { - urlSet: Set = new Set(); + urlSet = new Set(); pendingQueue: string[] = []; waitQueue: string[] = []; - mutationObserver: MutationObserver; - numPending: number = 0; - numDone: number = 0; + mutationObserver?: MutationObserver; + numPending = 0; + numDone = 0; headers: object; _donePromise: Promise; - _markDone: (value: any) => void; + _markDone?: (value: any) => void; active: boolean; running = false; @@ -44,7 +44,7 @@ export class AutoFetcher extends BackgroundBehavior { this.headers = headers || {}; - this._donePromise = new Promise((resolve) => this._markDone = resolve); + this._donePromise = new Promise((resolve) => (this._markDone = resolve)); this.active = active; if (this.active && startEarly) { @@ -72,7 +72,7 @@ export class AutoFetcher extends BackgroundBehavior { }); } - done() { + async done() { return this._donePromise; } @@ -93,7 +93,7 @@ export class AutoFetcher extends BackgroundBehavior { return url && (url.startsWith("http:") || url.startsWith("https:")); } - queueUrl(url: string, immediate: boolean = false) { + queueUrl(url: string, immediate = false) { try { url = new URL(url, document.baseURI).href; } catch (e) { @@ -122,7 +122,10 @@ export class AutoFetcher extends BackgroundBehavior { // fetch with default CORS mode, read entire stream async doFetchStream(url: string) { try { - const resp = await fetch(url, { "credentials": "include", "referrerPolicy": "origin-when-cross-origin" }); + const resp = await fetch(url, { + credentials: "include", + referrerPolicy: "origin-when-cross-origin", + }); this.debug(`Autofetch: started ${url}`); const reader = resp.body.getReader(); @@ -133,7 +136,6 @@ export class AutoFetcher extends BackgroundBehavior { this.debug(`Autofetch: finished ${url}`); return true; - } catch (e) { this.debug(e); @@ -146,11 +148,11 @@ export class AutoFetcher extends BackgroundBehavior { try { const abort = new AbortController(); await fetch(url, { - "mode": "no-cors", - "credentials": "include", - "referrerPolicy": "origin-when-cross-origin", - "headers": this.headers, - abort + mode: "no-cors", + credentials: "include", + referrerPolicy: "origin-when-cross-origin", + headers: this.headers, + abort, } as {}); abort.abort(); this.debug(`Autofetch: started non-cors stream for ${url}`); @@ -167,7 +169,7 @@ export class AutoFetcher extends BackgroundBehavior { this.numPending++; - let success = await doExternalFetch(url); + const success = await doExternalFetch(url); if (!success) { await this.doFetchNonCors(url); @@ -186,7 +188,9 @@ export class AutoFetcher extends BackgroundBehavior { if (this.mutationObserver) { return; } - this.mutationObserver = new MutationObserver((changes) => this.observeChange(changes)); + this.mutationObserver = new MutationObserver((changes) => + this.observeChange(changes), + ); this.mutationObserver.observe(document.documentElement, { characterData: false, @@ -195,7 +199,7 @@ export class AutoFetcher extends BackgroundBehavior { attributeOldValue: true, subtree: true, childList: true, - attributeFilter: ["srcset", "loading"] + attributeFilter: ["srcset", "loading"], }); } @@ -282,7 +286,10 @@ export class AutoFetcher extends BackgroundBehavior { // check regular src in case of