firaid-moz/extension/src/utils/domtraverse.ts

const IS_DEV = import.meta.env.MODE == "development";
const console = ExtConsole("domtraverse");

export const TRAVERSE_BEHAVOUR_IGNORE_MAGIC = Math.floor(Math.random() * 1000000).toString();
export const TRAVERSE_BEHAVOUR_ATTR = "data-firaid-moz-traverse-behavour";

const TRAVERSE_STACK_BANS: Set<string | null | undefined> = new Set([
    "IMG",
    "FIGURE",
    "SVG",
    "VIDEO",
    "META",
]);
const TRAVERSE_STACK_ROOTS: Set<string | null | undefined> = new Set([
    "DIV",
    "ARTICLE",
    "SECTION",
    "MAIN",
    "HEADER",
    "BLOCKQUOTE",
]);
const TRAVERSE_ALLOWED_ELEMENT_TAGS: Set<string | null | undefined> = new Set([
    ...TRAVERSE_STACK_ROOTS,
    "SPAN",
    "P",
    "H1",
    "H2",
    "H3",
    "H4",
    "H5",
    "H6",
    "H7",
    "H8",
    "UL",
    "OL",
    "LI",
    "DEL",
    "B",
    "I",
    "STRONG",
    "EM",
    "DEL",
]);
const KNOWN_STACK_ROOTS: WeakMap<
    Node,
    {
        csum: number;
    }
> = new WeakMap();

export function* traverse_text(soft_max_words: number, hard_min_words: number) {
    const walker = document.createTreeWalker(document.body, NodeFilter.SHOW_TEXT, {
        acceptNode: (node) => {
            // sanity check
            if (node.parentNode?.nodeType != Node.ELEMENT_NODE || !node.parentElement) {
                return NodeFilter.FILTER_REJECT;
            }
            if (!node.isConnected) {
                if (IS_DEV) {
                    node.parentElement.setAttribute("data-firaid-moz-traverse", "REJECT:disconnected");
                }
                return NodeFilter.FILTER_REJECT;
            }

            // filter elements
            if (!TRAVERSE_ALLOWED_ELEMENT_TAGS.has(node.parentElement.tagName)) {
                if (IS_DEV) {
                    node.parentElement.setAttribute("data-firaid-moz-traverse", "REJECT:tag_not_allowed");
                }
                return NodeFilter.FILTER_REJECT;
            }

            // ignored elements
            if (node.parentElement.getAttribute(TRAVERSE_BEHAVOUR_ATTR) == TRAVERSE_BEHAVOUR_IGNORE_MAGIC) {
                if (IS_DEV) {
                    node.parentElement.setAttribute("data-firaid-moz-traverse", "REJECT:behavour_magic");
                }
                return NodeFilter.FILTER_REJECT;
            }

            // check visibility (my best effort, still mistakes sometimes)
            if (
                !node.parentElement.checkVisibility({
                    visibilityProperty: true,
                }) ||
                !node.parentElement.offsetParent ||
                !node.parentElement.getBoundingClientRect().height
            ) {
                // check visibility
                if (IS_DEV) {
                    node.parentElement.setAttribute("data-firaid-moz-traverse", "REJECT:invisible");
                }
                return NodeFilter.FILTER_REJECT;
            }

            // TODO: figure out more performant way to check this.
            // Ignore anything inside of banned elements.
            //
            // For now I'll just limit the depth to 16
            let checking: Element | null = node.parentElement;
            for (let i = 0; i < 16; i++) {
                if (!checking || checking.tagName == "BODY") {
                    break;
                }
                if (TRAVERSE_STACK_BANS.has(checking.tagName)) {
                    if (IS_DEV) {
                        node.parentElement.setAttribute("data-firaid-moz-traverse", "REJECT:banned_ancestor");
                    }
                    return NodeFilter.FILTER_REJECT;
                }
                checking = checking.parentElement;
            }

            // drop with nonsense content
            // if (!node.textContent?.trim()) {
            //     if (IS_DEV) {
            //         node.parentElement?.setAttribute("data-firaid-moz-traverse", "REJECT-NO_TEXT");
            //     }
            //     return NodeFilter.FILTER_REJECT;
            // }

            // finally good ones
            if (IS_DEV) {
                node.parentElement.setAttribute("data-firaid-moz-traverse", "ACCEPT");
            }
            return NodeFilter.FILTER_ACCEPT;
        },
    });

    let stack: { root: Node; first_node: Node; last_node: Node; text: string; words: number } | null = null;

    while (walker.nextNode()) {
        const node = walker.currentNode;

        let rootable_parent = node.parentElement;
        while (
            rootable_parent &&
            rootable_parent.tagName != "HTML" &&
            !TRAVERSE_STACK_ROOTS.has(rootable_parent.tagName)
        ) {
            rootable_parent = rootable_parent.parentElement;
        }
        if (!rootable_parent) {
            continue;
        }

        let nodeText = node.textContent?.trim();
        if (!nodeText) {
            continue;
        }
        let nodeTextWords = count_words_naive(nodeText);

        if (!stack) {
            stack = {
                root: rootable_parent,
                first_node: node,
                last_node: node,
                text: nodeText,
                words: nodeTextWords,
            };
        } else if (stack.root.contains(rootable_parent)) {
            stack.text += " " + nodeText;
            stack.words += nodeTextWords;
            stack.last_node = node;
        } else {
            if (stack.words >= hard_min_words) {
                const known = KNOWN_STACK_ROOTS.get(stack.root);
                const csum = text_csum(stack.text);
                if (csum !== known?.csum) {
                    KNOWN_STACK_ROOTS.set(stack.root, { csum, ...known });
                    yield stack;
                } else {
                    console.debug("Refuse to yield stack as matching csum exists");
                }
            }
            stack = {
                root: rootable_parent,
                first_node: node,
                last_node: node,
                text: nodeText,
                words: nodeTextWords,
            };
            continue;
        }

        if (stack && stack.words > soft_max_words) {
            if (stack.words >= hard_min_words) {
                const known = KNOWN_STACK_ROOTS.get(stack.root);
                const csum = text_csum(stack.text);
                if (csum !== known?.csum) {
                    KNOWN_STACK_ROOTS.set(stack.root, { csum, ...known });
                    yield stack;
                } else {
                    console.debug("Refuse to yield stack as matching csum exists");
                }
            }
            stack = null;
        }
    }
}

function count_words_naive(text: string): number {
    const is_space = /\s/;

    let count = 0;
    let word_marker = false;
    for (let i = 0; i < text.length; i++) {
        if (is_space.test(text[i])) {
            word_marker = false;
        } else if (!word_marker) {
            count += 1;
            word_marker = true;
        }
    }

    return count;
}

function text_csum(text: string): number {
    let csum = 5381;
    for (let i = 0; i < text.length; i++) {
        csum += (csum << 5) + text.charCodeAt(i);
        csum &= csum;
    }

    return csum >>> 0;
}