diff --git a/extension/package.json b/extension/package.json index c557b9c..cbf50f5 100644 --- a/extension/package.json +++ b/extension/package.json @@ -24,14 +24,14 @@ }, "devDependencies": { "@tailwindcss/vite": "^4.2.1", - "@tsconfig/svelte": "^5.0.7", + "@tsconfig/svelte": "^5.0.8", "@wxt-dev/module-svelte": "^2.0.4", "daisyui": "^5.5.19", - "svelte": "^5.50.0", - "svelte-check": "^4.3.6", + "svelte": "^5.53.10", + "svelte-check": "^4.4.5", "tailwindcss": "^4.2.1", "tslib": "^2.8.1", "typescript": "^5.9.3", - "wxt": "^0.20.14" + "wxt": "^0.20.18" } } diff --git a/extension/src/entrypoints/background.ts b/extension/src/entrypoints/background.ts index 2443c43..c9212ec 100644 --- a/extension/src/entrypoints/background.ts +++ b/extension/src/entrypoints/background.ts @@ -106,36 +106,34 @@ function setup_inference_ai_text() { } async function configure_ai_text(config: ExtensionConfigAiText) { - if (config.enabled) { - const loader = config.lite ? inference_ai_text_lite : inference_ai_text; - const handler = config.lite ? AiTextLite : AiText; - const capability = config.lite ? AiTextLiteCapability : AiTextCapability; - const have = config.lite ? AI_TEXT_STATE.have_lite_model : AI_TEXT_STATE.have_normal_model; - const set_have = config.lite - ? () => (AI_TEXT_STATE.have_lite_model = true) - : () => (AI_TEXT_STATE.have_normal_model = true); - - if (!have) { - await loader(); - if (await guess_has_webgpu()) { - console.warn( - "WebGPU detected, but use is not currently possible due to burn APIs being sync. Check the issue progress in burn and burn-onnx repos. In the mean time will use NdArray anyway.", - ); - //await capability.assert_ndarray(); - await capability.assert_webgpu(); - } else { - await capability.assert_ndarray(); - } - set_have(); - } - - AI_TEXT_STATE.handle = new handler(); - } else { - AI_TEXT_STATE.handle = null; - if (AI_TEXT_STATE.queue_inference_timer) { - clearTimeout(AI_TEXT_STATE.queue_inference_timer); - } - } + // if (config.enabled) { + // const loader = config.lite ? inference_ai_text_lite : inference_ai_text; + // const handler = config.lite ? AiTextLite : AiText; + // const capability = config.lite ? AiTextLiteCapability : AiTextCapability; + // const have = config.lite ? AI_TEXT_STATE.have_lite_model : AI_TEXT_STATE.have_normal_model; + // const set_have = config.lite + // ? () => (AI_TEXT_STATE.have_lite_model = true) + // : () => (AI_TEXT_STATE.have_normal_model = true); + // if (!have) { + // await loader(); + // if (await guess_has_webgpu()) { + // console.warn( + // "WebGPU detected, but use is not currently possible due to burn APIs being sync. Check the issue progress in burn and burn-onnx repos. In the mean time will use NdArray anyway.", + // ); + // //await capability.assert_ndarray(); + // await capability.assert_webgpu(); + // } else { + // await capability.assert_ndarray(); + // } + // set_have(); + // } + // AI_TEXT_STATE.handle = new handler(); + // } else { + // AI_TEXT_STATE.handle = null; + // if (AI_TEXT_STATE.queue_inference_timer) { + // clearTimeout(AI_TEXT_STATE.queue_inference_timer); + // } + // } } async function guess_has_webgpu() { diff --git a/extension/src/entrypoints/content.ts b/extension/src/entrypoints/content.ts index 6dccca9..435a1e4 100644 --- a/extension/src/entrypoints/content.ts +++ b/extension/src/entrypoints/content.ts @@ -1,21 +1,37 @@ import { EXTENSION_CONFIG, ExtensionConfigAiImage, ExtensionConfigAiText } from "@/utils/config"; import { traverse_text } from "@/utils/domtraverse"; +const console = ExtConsole("content"); + export default defineContentScript({ matches: ["http://*/*", "https://*/*"], main() { - const trigger = setTimeout(() => { - text_ai_traverse(); - }, 1000); - - document.addEventListener("load", () => { - clearTimeout(trigger); - - text_ai_traverse(); + DOM_OBSERVATORY.observer = new MutationObserver((_) => { + if (DOM_OBSERVATORY.timer) { + clearTimeout(DOM_OBSERVATORY.timer); + } + DOM_OBSERVATORY.timer = setTimeout(() => { + console.debug("retraverse, cause: mutation"); + text_ai_traverse(); + }, 1000); }); + + setTimeout(() => { + text_ai_traverse(); + + DOM_OBSERVATORY.observer?.observe(document.body, { + subtree: true, + characterData: true, + }); + }, 1000); }, }); +const DOM_OBSERVATORY = { + observer: null as MutationObserver | null, + timer: null as number | null, +}; + const TRAVERSE_VIEWPORT_OBSERVER = new IntersectionObserver( (entries, observer) => { for (const entry of entries) { @@ -23,27 +39,25 @@ const TRAVERSE_VIEWPORT_OBSERVER = new IntersectionObserver( const stack = TRAVERSE_PARENT_TO_STACK.get(entry.target); if (stack) { - sendMessage("inference_aiText_classify", { text: stack.text }) - .then((result) => { - observer.unobserve(entry.target); + sendMessage("inference_aiText_classify", { text: stack.text }).then((result) => { + observer.unobserve(entry.target); - if ( - EXTENSION_CONFIG.ai_text.show_ai && - result.prob_ai >= EXTENSION_CONFIG.ai_text.threshold && - result.prob_ai >= result.prob_human - ) { - ai_set_badge(entry.target, result.prob_ai, undefined); - } else if ( - EXTENSION_CONFIG.ai_text.show_human && - result.prob_human >= EXTENSION_CONFIG.ai_text.threshold && - result.prob_human >= result.prob_ai - ) { - ai_set_badge(entry.target, undefined, result.prob_human); - } else { - ai_clear_badge(entry.target); - } - }) - .catch((e) => {}); + if ( + EXTENSION_CONFIG.ai_text.show_ai && + result.prob_ai >= EXTENSION_CONFIG.ai_text.threshold && + result.prob_ai >= result.prob_human + ) { + ai_set_badge(entry.target, result.prob_ai, undefined); + } else if ( + EXTENSION_CONFIG.ai_text.show_human && + result.prob_human >= EXTENSION_CONFIG.ai_text.threshold && + result.prob_human >= result.prob_ai + ) { + ai_set_badge(entry.target, undefined, result.prob_human); + } else { + ai_clear_badge(entry.target); + } + }); } } } @@ -63,6 +77,7 @@ interface WeakStack { } const TRAVERSE_PARENT_TO_STACK: WeakMap = new WeakMap(); function text_ai_traverse() { + console.debug("text_ai_traverse: traverse DOM"); for (const stack of traverse_text(600, 20)) { let target = stack.first_node.parentElement || stack.root.parentElement; if (target) { @@ -77,6 +92,7 @@ const SET_BADGES: WeakMap = new WeakMap(); function ai_set_badge(node: Node, ai?: number, human?: number) { const badge = document.createElement("span"); + badge.setAttribute(TRAVERSE_BEHAVOUR_ATTR, TRAVERSE_BEHAVOUR_IGNORE_MAGIC); badge.style.display = "inline"; badge.style.borderRadius = "4px"; badge.style.textAlign = "center"; @@ -84,20 +100,14 @@ function ai_set_badge(node: Node, ai?: number, human?: number) { badge.style.fontSize = "12px"; badge.style.padding = "1px"; badge.style.marginRight = "4px"; - if (ai || human) { - if (ai) { - badge.style.backgroundColor = "#ffc36d"; - badge.style.color = "#fff"; - badge.textContent = `🤖: ${Math.trunc(ai * 100)}%`; - } else if (human) { - badge.style.backgroundColor = "#8fe1a1"; - badge.style.color = "#fff"; - badge.textContent = `👨‍💼: ${Math.trunc(human * 100)}%`; - } else { - badge.style.backgroundColor = "#ddd"; - badge.style.color = "#fff"; - badge.textContent = "🦊: confused"; - } + if (ai) { + badge.style.backgroundColor = "#ffc36d"; + badge.style.color = "#fff"; + badge.textContent = `🤖: ${Math.trunc(ai * 100)}%`; + } else if (human) { + badge.style.backgroundColor = "#8fe1a1"; + badge.style.color = "#fff"; + badge.textContent = `👨‍💼: ${Math.trunc(human * 100)}%`; } else { badge.style.backgroundColor = "#ddd"; badge.style.color = "#fff"; diff --git a/extension/src/entrypoints/popup/App.svelte b/extension/src/entrypoints/popup/App.svelte index 0cb4aa2..8eb283f 100644 --- a/extension/src/entrypoints/popup/App.svelte +++ b/extension/src/entrypoints/popup/App.svelte @@ -5,71 +5,65 @@ class Config { key: Key; - value: (typeof extensionConfig)[Key]["fallback"]; + value: any; + insync = false; constructor(key: Key) { this.key = key; - this.value = $state(extensionConfig[key].fallback); + this.value = $state((extensionConfig[key] as any).fallback); this.load(); $effect(() => { - console.log("RESYNC"); - extensionConfig[key].setValue(this.value as any); + const value = $state.snapshot(this.value); + const timeout = setTimeout(() => { + if (!this.insync || this.value == value) return; + extensionConfig[this.key].setValue(value).catch((e) => { + console.error("Failed to write state back to storage", e); + }); + }, 20); + + return () => clearTimeout(timeout); }); } async load() { - const config = await extensionConfig[this.key].getValue(); + const config = await extensionConfig[this.key].getValue().catch((e) => { + console.error("Failed to sync state from storage", e); + }); this.value = config; + this.insync = true; - extensionConfig[this.key].watch((value) => { + (extensionConfig[this.key] as any).watch((value: any) => { if (value != this.value) this.value = value; }); } } const config_ai_text = new Config("ai_text"); - $inspect(config_ai_text.value); - const config_ai_image = new Config("ai_image"); - $inspect(config_ai_image.value); - - const config_ai = new Config("ai"); - $inspect(config_ai.value); + const domain_whitelist = new Config("domain_whitelist"); -
-
+
+
🦊
- +
- AI Text detection + General - - +
+
+ Mode + + +
+
+ Misc + +
+
+ +
- AI Image detection + General - - +
+
+ Mode + + +
+
+ Misc + +
+
Do not check on these sites: -
- {#each Object.entries(config_ai.value.domain_blacklist) as domain (domain[0])} +
+ {#each domain_whitelist.value as domain}
- {domain[0]} + {domain}
{/each}
- +

- The source code of this extension is available here +
+

+ The source code is available here under terms of Apache-2.0 license


- The following people/organizations made this fun extention possible to create by training and - providing ML models on huggingface: + The following people/organizations made it possible to create this by training and providing + ML models on huggingface:


diff --git a/extension/src/entrypoints/popup/app.css b/extension/src/entrypoints/popup/app.css index 4c1b0c2..c5c1f17 100644 --- a/extension/src/entrypoints/popup/app.css +++ b/extension/src/entrypoints/popup/app.css @@ -1,2 +1,70 @@ @import "tailwindcss"; @plugin "daisyui"; +@plugin "daisyui/theme" { + name: "spushk-dim"; + default: false; + prefersdark: true; + color-scheme: "dark"; + --color-base-100: #1f273e; + --color-base-200: #455474; + --color-base-300: #4f658a; + --color-base-content: oklch(96% 0.001 286.375); + --color-primary: #fe9735; + --color-primary-content: oklch(17.226% 0.028 139.549); + --color-secondary: oklch(83% 0.128 66.29); + --color-secondary-content: oklch(14.675% 0.033 35.353); + --color-accent: oklch(74.229% 0.133 311.379); + --color-accent-content: oklch(14.845% 0.026 311.379); + --color-neutral: oklch(24.731% 0.02 264.094); + --color-neutral-content: oklch(87% 0.006 286.286); + --color-info: oklch(86.078% 0.142 206.182); + --color-info-content: oklch(17.215% 0.028 206.182); + --color-success: oklch(86.171% 0.142 166.534); + --color-success-content: oklch(17.234% 0.028 166.534); + --color-warning: oklch(92% 0.12 95.746); + --color-warning-content: oklch(17.232% 0.028 94.818); + --color-error: oklch(89% 0.058 10.001); + --color-error-content: oklch(16.483% 0.019 33.756); + --radius-selector: 0.5rem; + --radius-field: 0.5rem; + --radius-box: 0.5rem; + --size-selector: 0.1875rem; + --size-field: 0.1875rem; + --border: 0.5px; + --depth: 0; + --noise: 0; +} +@plugin "daisyui/theme" { + name: "spushk-pastel"; + default: true; + prefersdark: false; + color-scheme: "light"; + --color-base-100: oklch(100% 0 0); + --color-base-200: oklch(98.462% 0.001 247.838); + --color-base-300: oklch(92.462% 0.001 247.838); + --color-base-content: oklch(20% 0 0); + --color-primary: #fe9735; + --color-primary-content: oklch(14% 0 0); + --color-secondary: oklch(83% 0.128 66.29); + --color-secondary-content: oklch(14% 0 0); + --color-accent: #c792e9; + --color-accent-content: oklch(14% 0 0); + --color-neutral: #455474; + --color-neutral-content: oklch(98% 0.002 247.839); + --color-info: oklch(86% 0.127 207.078); + --color-info-content: oklch(25% 0.09 281.288); + --color-success: oklch(87% 0.15 154.449); + --color-success-content: oklch(26% 0.065 152.934); + --color-warning: oklch(79% 0.184 86.047); + --color-warning-content: oklch(28% 0.066 53.813); + --color-error: oklch(51% 0.222 16.935); + --color-error-content: oklch(94% 0.03 12.58); + --radius-selector: 0.5rem; + --radius-field: 0.5rem; + --radius-box: 0.5rem; + --size-selector: 0.1875rem; + --size-field: 0.1875rem; + --border: 0.5px; + --depth: 0; + --noise: 0; +} diff --git a/extension/src/utils/config.ts b/extension/src/utils/config.ts index e40c5e6..4b41183 100644 --- a/extension/src/utils/config.ts +++ b/extension/src/utils/config.ts @@ -1,9 +1,6 @@ -import { ExtensionConfigSchema } from "#imports"; -import { defineExtensionStorage, ExtensionStorage } from "@webext-core/storage"; - export interface ExtensionConfigAiText { enabled: boolean; - lite: boolean; + mode: "lite" | "mixed" | "hard"; show_human: boolean; show_ai: boolean; threshold: number; @@ -11,53 +8,30 @@ export interface ExtensionConfigAiText { export interface ExtensionConfigAiImage { enabled: boolean; - lite: boolean; + mode: "lite" | "mixed" | "hard"; show_human: boolean; show_ai: boolean; threshold: number; } -export interface ExtensionConfigAi { - /// shall contain domain qualified name, separated by dots - /// indexed by domain name - /// e.g. { - /// "github.com": ["github", "com"] - /// } - domain_blacklist: { - [domain: string]: string[]; - }; -} - -// export interface ExtensionConfigSchema { -// version: string; -// ai_text: ExtensionConfigAiText; -// ai_image: ExtensionConfigAiImage; -// } - -//export const extensionConfig = defineExtensionStorage(browser.storage.local); - export const extensionConfig = { - ai: storage.defineItem("local:config:ai", { - fallback: { - domain_blacklist: {}, - }, - init: () => ({ - domain_blacklist: {}, - }), + domain_whitelist: storage.defineItem("local:config:domain_whitelist", { + fallback: [], + init: () => ["asd.asd", "lkj.asd", "oiua.asd", "oi1u2e.aoisdu", "did0.asdiho"], version: 1, migrations: {}, }), ai_text: storage.defineItem("local:config:ai_text", { fallback: { - enabled: true, - lite: true, + enabled: false, + mode: "lite", show_ai: true, show_human: false, threshold: 0.6, }, init: () => ({ enabled: true, - lite: true, + mode: "lite", show_ai: true, show_human: false, threshold: 0.6, @@ -67,39 +41,39 @@ export const extensionConfig = { }), ai_image: storage.defineItem("local:config:ai_image", { fallback: { - enabled: true, - lite: true, + enabled: false, + mode: "lite", show_ai: true, show_human: false, threshold: 0.6, }, init: () => ({ enabled: true, - lite: true, + mode: "lite", show_ai: true, - show_human: true, + show_human: false, threshold: 0.6, }), version: 1, migrations: {}, }), -} as const; +}; export const EXTENSION_CONFIG: { - ai: ExtensionConfigAi; + domain_whitelist: Set; ai_text: ExtensionConfigAiText; ai_image: ExtensionConfigAiImage; } = { - ai: extensionConfig.ai.fallback, + domain_whitelist: new Set(extensionConfig.domain_whitelist.fallback), ai_text: extensionConfig.ai_text.fallback, ai_image: extensionConfig.ai_image.fallback, }; -extensionConfig.ai.getValue().then((config) => { - EXTENSION_CONFIG.ai = config; +extensionConfig.domain_whitelist.getValue().then((config) => { + EXTENSION_CONFIG.domain_whitelist = new Set(config); - extensionConfig.ai.watch((config) => { - EXTENSION_CONFIG.ai = config; + extensionConfig.domain_whitelist.watch((config) => { + EXTENSION_CONFIG.domain_whitelist = new Set(config); }); }); @@ -118,41 +92,3 @@ extensionConfig.ai_image.getValue().then((config) => { EXTENSION_CONFIG.ai_image = config; }); }); - -// extensionConfig.getItem("version").then(async (v) => { -// const current_version = browser.runtime.getManifest().version; -// if (v != current_version) { -// switch (v) { -// default: -// await config_migrate_fresh(); -// } -// await extensionConfig.setItem("version", current_version); -// } -// }); - -// async function config_migrate_fresh() { -// await extensionConfig.clear(); -// await extensionConfig.setItem("ai_text", { -// enabled: true, -// lite: true, -// show_ai: true, -// show_human: false, -// threshold: 0.6, -// }); -// await extensionConfig.setItem("ai_image", { -// enabled: true, -// lite: true, -// show_ai: true, -// show_human: false, -// threshold: 0.6, -// }); -// await extensionConfig.setItem("fun_githief", { -// enabled: false, -// }); -// await extensionConfig.setItem("fun_microslop", { -// enabled: false, -// }); -// await extensionConfig.setItem("fun_huyandex", { -// enabled: false, -// }); -// } diff --git a/extension/src/utils/domtraverse.ts b/extension/src/utils/domtraverse.ts index 1abe88f..1a49e0b 100644 --- a/extension/src/utils/domtraverse.ts +++ b/extension/src/utils/domtraverse.ts @@ -1,19 +1,8 @@ -/// Extracts meaningful portions of text from DOM, yielding the top matched nodes -/// -/// e.g. -/// -/// ```html -///
-/// my hair are NOT -/// -/// blue! -/// -///
-/// ``` -/// -/// Shall yield a reference to the `div` with text "my hair are NOT blue!" - const IS_DEV = import.meta.env.MODE == "development"; +const console = ExtConsole("domtraverse"); + +export const TRAVERSE_BEHAVOUR_IGNORE_MAGIC = Math.floor(Math.random() * 1000000).toString(); +export const TRAVERSE_BEHAVOUR_ATTR = "data-firaid-moz-traverse-behavour"; const TRAVERSE_STACK_BANS: Set = new Set([ "IMG", @@ -50,28 +39,56 @@ const TRAVERSE_ALLOWED_ELEMENT_TAGS: Set = new Set([ "I", "STRONG", "EM", + "DEL", ]); +const KNOWN_STACK_ROOTS: WeakMap< + Node, + { + csum: number; + } +> = new WeakMap(); + export function* traverse_text(soft_max_words: number, hard_min_words: number) { const walker = document.createTreeWalker(document.body, NodeFilter.SHOW_TEXT, { acceptNode: (node) => { // sanity check - if (!node.isConnected) { - if (IS_DEV) { - node.parentElement?.setAttribute("data-firaid-moz-traverse", "REJECT-!IS_CONNECTED"); - } + if (node.parentNode?.nodeType != Node.ELEMENT_NODE || !node.parentElement) { return NodeFilter.FILTER_REJECT; } - if (node.parentNode?.nodeType != Node.ELEMENT_NODE) { + if (!node.isConnected) { if (IS_DEV) { - node.parentElement?.setAttribute("data-firaid-moz-traverse", "REJECT-!ELEMENT_NODE"); + node.parentElement.setAttribute("data-firaid-moz-traverse", "REJECT:disconnected"); } return NodeFilter.FILTER_REJECT; } // filter elements - if (!TRAVERSE_ALLOWED_ELEMENT_TAGS.has(node.parentElement?.tagName)) { + if (!TRAVERSE_ALLOWED_ELEMENT_TAGS.has(node.parentElement.tagName)) { if (IS_DEV) { - node.parentElement?.setAttribute("data-firaid-moz-traverse", "REJECT-!ALLOWED_TAG"); + node.parentElement.setAttribute("data-firaid-moz-traverse", "REJECT:tag_not_allowed"); + } + return NodeFilter.FILTER_REJECT; + } + + // ignored elements + if (node.parentElement.getAttribute(TRAVERSE_BEHAVOUR_ATTR) == TRAVERSE_BEHAVOUR_IGNORE_MAGIC) { + if (IS_DEV) { + node.parentElement.setAttribute("data-firaid-moz-traverse", "REJECT:behavour_magic"); + } + return NodeFilter.FILTER_REJECT; + } + + // check visibility (my best effort, still mistakes sometimes) + if ( + !node.parentElement.checkVisibility({ + visibilityProperty: true, + }) || + !node.parentElement.offsetParent || + !node.parentElement.getBoundingClientRect().height + ) { + // check visibility + if (IS_DEV) { + node.parentElement.setAttribute("data-firaid-moz-traverse", "REJECT:invisible"); } return NodeFilter.FILTER_REJECT; } @@ -80,17 +97,14 @@ export function* traverse_text(soft_max_words: number, hard_min_words: number) { // Ignore anything inside of banned elements. // // For now I'll just limit the depth to 16 - let checking = node.parentElement; + let checking: Element | null = node.parentElement; for (let i = 0; i < 16; i++) { if (!checking || checking.tagName == "BODY") { break; } if (TRAVERSE_STACK_BANS.has(checking.tagName)) { if (IS_DEV) { - node.parentElement?.setAttribute( - "data-firaid-moz-traverse", - "REJECT-BANNED_ANCESTOR", - ); + node.parentElement.setAttribute("data-firaid-moz-traverse", "REJECT:banned_ancestor"); } return NodeFilter.FILTER_REJECT; } @@ -98,16 +112,16 @@ export function* traverse_text(soft_max_words: number, hard_min_words: number) { } // drop with nonsense content - if ((node.textContent?.trim()?.length || 0) < 8) { - if (IS_DEV) { - node.parentElement?.setAttribute("data-firaid-moz-traverse", "REJECT-SHORT_TEXT"); - } - return NodeFilter.FILTER_REJECT; - } + // if (!node.textContent?.trim()) { + // if (IS_DEV) { + // node.parentElement?.setAttribute("data-firaid-moz-traverse", "REJECT-NO_TEXT"); + // } + // return NodeFilter.FILTER_REJECT; + // } // finally good ones if (IS_DEV) { - node.parentElement?.setAttribute("data-firaid-moz-traverse", "ACCEPT"); + node.parentElement.setAttribute("data-firaid-moz-traverse", "ACCEPT"); } return NodeFilter.FILTER_ACCEPT; }, @@ -127,25 +141,15 @@ export function* traverse_text(soft_max_words: number, hard_min_words: number) { rootable_parent = rootable_parent.parentElement; } if (!rootable_parent) { - if (IS_DEV) { - node.parentElement?.setAttribute("data-firaid-moz-stack", "REJECTED-!ROOTABLE_PARENT"); - } continue; } let nodeText = node.textContent?.trim(); if (!nodeText) { - if (IS_DEV) { - node.parentElement?.setAttribute("data-firaid-moz-stack", "REJECTED-!TEXT"); - } continue; } let nodeTextWords = count_words_naive(nodeText); - if (IS_DEV) { - node.parentElement?.setAttribute("data-firaid-moz-stack", "VISITED"); - } - if (!stack) { stack = { root: rootable_parent, @@ -155,20 +159,19 @@ export function* traverse_text(soft_max_words: number, hard_min_words: number) { words: nodeTextWords, }; } else if (stack.root.contains(rootable_parent)) { - if (IS_DEV) { - node.parentElement?.setAttribute("data-firaid-moz-stack", "MEMBER"); - } stack.text += " " + nodeText; stack.words += nodeTextWords; stack.last_node = node; } else { if (stack.words >= hard_min_words) { - yield stack; - if (IS_DEV) { - stack.root.parentElement?.setAttribute("data-firaid-moz-stack", "ROOT"); + const known = KNOWN_STACK_ROOTS.get(stack.root); + const csum = text_csum(stack.text); + if (csum !== known?.csum) { + KNOWN_STACK_ROOTS.set(stack.root, { csum, ...known }); + yield stack; + } else { + console.debug("Refuse to yield stack as matching csum exists"); } - } else if (IS_DEV) { - stack.root.parentElement?.setAttribute("data-firaid-moz-stack", "SKIP-SHORT_TEXT"); } stack = { root: rootable_parent, @@ -180,13 +183,16 @@ export function* traverse_text(soft_max_words: number, hard_min_words: number) { continue; } - if (stack.words > soft_max_words) { + if (stack && stack.words > soft_max_words) { if (stack.words >= hard_min_words) { - if (IS_DEV) { - stack.root.parentElement?.setAttribute("data-firaid-moz-stack", "ROOT"); + const known = KNOWN_STACK_ROOTS.get(stack.root); + const csum = text_csum(stack.text); + if (csum !== known?.csum) { + KNOWN_STACK_ROOTS.set(stack.root, { csum, ...known }); + yield stack; + } else { + console.debug("Refuse to yield stack as matching csum exists"); } - } else if (IS_DEV) { - stack.root.parentElement?.setAttribute("data-firaid-moz-stack", "SKIP-SHORT_TEXT"); } stack = null; } @@ -209,3 +215,13 @@ function count_words_naive(text: string): number { return count; } + +function text_csum(text: string): number { + let csum = 5381; + for (let i = 0; i < text.length; i++) { + csum += (csum << 5) + text.charCodeAt(i); + csum &= csum; + } + + return csum >>> 0; +}