ribbit/src/ts/hopdown.ts

/*
 * hopdown.ts — configurable markdown↔HTML converter.
 *
 * HopDown orchestrates markdown↔HTML conversion using a tokenizer for
 * inline parsing and a serializer for HTML→markdown. Block-level parsing
 * uses Tag definitions directly. The tokenizer/serializer architecture
 * ensures correct round-trips by separating structural delimiters from
 * literal text at the type level.
 */

import type { Converter, MatchContext, Tag, DelimiterMatch } from './types';
import { defaultBlockTags, defaultInlineTags, defaultTags, escapeHtml } from './tags';
import { buildMacroTags, processInlineMacros, type MacroDef } from './macros';
import { InlineTokenizer, type InlineToken, type DelimiterDef } from './tokenizer';
import { MarkdownSerializer, type SerializerTagDef } from './serializer';

export type TagMap = Record<string, Tag>;

export interface HopDownOptions {
    tags?: TagMap;
    exclude?: string[];
    macros?: MacroDef[];
}

/**
 * Configurable markdown↔HTML converter. Uses a tokenizer for inline
 * parsing (markdown→HTML) and a serializer for HTML→markdown. Block
 * parsing delegates to Tag definitions.
 *
 *   const converter = new HopDown();
 *   converter.toHTML('**bold**');
 *   converter.toMarkdown('<strong>bold</strong>');
 */
export class HopDown {
    private blockTags: Tag[];
    private inlineTags: Tag[];
    private tags: Map<string, Tag>;
    private macroMap: Map<string, MacroDef>;
    private referenceLinks: Map<string, { url: string; title?: string }>;
    private tokenizer: InlineTokenizer;
    private serializer: MarkdownSerializer;
    private cachedConverter: Converter;
    private delimiterRegexes: { tag: Tag; htmlTag: string; complete: RegExp; open: RegExp }[];
    private editableSelectorCache: string;

    constructor(options: HopDownOptions = {}) {
        let tagMap: TagMap;

        if (options.tags) {
            tagMap = options.tags;
        } else if (options.exclude) {
            const excluded = new Set(options.exclude);
            tagMap = Object.fromEntries(
                Object.entries(defaultTags).filter(([, tag]) => !excluded.has(tag.name))
            );
        } else {
            tagMap = defaultTags;
        }

        this.macroMap = new Map();
        this.referenceLinks = new Map();
        if (options.macros && options.macros.length > 0) {
            const { blockTag, selectorTag, macroMap } = buildMacroTags(options.macros);
            this.macroMap = macroMap;
            tagMap['[data-macro]'] = selectorTag;
            tagMap['_macro'] = blockTag;
        }

        const allTags = Object.values(tagMap);
        const defaultBlockNames = new Set(Object.values(defaultBlockTags).map(tag => tag.name));
        const defaultInlineNames = new Set(Object.values(defaultInlineTags).map(tag => tag.name));

        this.blockTags = allTags.filter(tag =>
            defaultBlockNames.has(tag.name) || tag.name === 'macro' ||
            (!defaultInlineNames.has(tag.name) && !tag.pattern)
        );

        // Macro block tag must run after fencedCode (so code blocks aren't
        // parsed as macros) but before paragraph (the catch-all)
        this.blockTags.sort((a, b) => {
            const order = (tag: Tag) => {
                if (tag.name === 'fencedCode') {
                    return 0;
                }
                if (tag.name === 'macro') {
                    return 1;
                }
                if (tag.name === 'paragraph') {
                    return 99;
                }
                return 50;
            };
            return order(a) - order(b);
        });

        this.inlineTags = allTags.filter(tag =>
            defaultInlineNames.has(tag.name) || tag.pattern
        );

        this.tags = new Map();
        this.registerSelectors(tagMap);
        this.validateInlineTags();

        this.tokenizer = this.buildTokenizer();
        this.serializer = this.buildSerializer();
        this.cachedConverter = this.makeConverter();
        this.delimiterRegexes = this.buildDelimiterRegexes();
        this.editableSelectorCache = this.buildEditableSelector();
    }

    private registerSelectors(tagMap: TagMap): void {
        for (const [selector, tag] of Object.entries(tagMap)) {
            const parts = selector.split(',').map(part => part.trim()).filter(Boolean);
            for (const part of parts) {
                if (part.startsWith('_')) {
                    continue;
                }
                const existing = this.tags.get(part);
                if (existing && existing !== tag) {
                    throw new Error(
                        `HTML tag "${part}" is claimed by both "${existing.name}" and "${tag.name}". ` +
                        `Use the exclude option to remove one before adding the other.`
                    );
                }
                this.tags.set(part, tag);
            }
        }
    }

    private validateInlineTags(): void {
        const withDelimiters = this.inlineTags
            .filter(tag => tag.delimiter)
            .map(tag => ({
                name: tag.name,
                delimiter: tag.delimiter as string,
                precedence: tag.precedence as number ?? 50,
            }));

        for (let outer = 0; outer < withDelimiters.length; outer++) {
            for (let inner = outer + 1; inner < withDelimiters.length; inner++) {
                const first = withDelimiters[outer];
                const second = withDelimiters[inner];
                const firstIsPrefix = second.delimiter.startsWith(first.delimiter);
                const secondIsPrefix = first.delimiter.startsWith(second.delimiter);
                if (!firstIsPrefix && !secondIsPrefix) {
                    continue;
                }
                const longer = first.delimiter.length > second.delimiter.length ? first : second;
                const shorter = first.delimiter.length > second.delimiter.length ? second : first;
                if (longer.precedence >= shorter.precedence) {
                    throw new Error(
                        `Inline tag "${longer.name}" (delimiter "${longer.delimiter}") must have ` +
                        `lower precedence than "${shorter.name}" (delimiter "${shorter.delimiter}") ` +
                        `because its delimiter is a prefix match. ` +
                        `Got ${longer.name}=${longer.precedence}, ${shorter.name}=${shorter.precedence}.`
                    );
                }
            }
        }
    }

    /**
     * Convert a markdown string to HTML.
     *
     *   converter.toHTML('# Hello\n\n**bold** text')
     */
    toHTML(markdown: string): string {
        return this.processBlocks(markdown);
    }

    /**
     * Convert an HTML string back to markdown. Uses the serializer
     * which produces correctly-escaped output via typed tokens.
     *
     *   converter.toMarkdown('<h1>Hello</h1><p><strong>bold</strong> text</p>')
     */
    toMarkdown(html: string): string {
        const container = document.createElement('div');
        container.innerHTML = html;
        return this.serializeNode(container).replace(/\n{3,}/g, '\n\n').trim();
    }

    /**
     * The registered block-level tags. Used by the WYSIWYG editor
     * to detect block syntax patterns during live editing.
     *
     *   converter.getBlockTags().forEach(tag => console.log(tag.name))
     */
    getBlockTags(): Tag[] {
        return this.blockTags;
    }

    /**
     * The registered inline tags. Used by the WYSIWYG editor to
     * build delimiter regexes for speculative rendering.
     *
     *   converter.getInlineTags().filter(tag => tag.delimiter)
     */
    getInlineTags(): Tag[] {
        return this.inlineTags;
    }

    /**
     * Find the first complete delimiter pair in the text.
     *
     *   converter.findCompletePair('hello **world** end')
     */
    findCompletePair(text: string): DelimiterMatch | null {
        for (const entry of this.delimiterRegexes) {
            const match = text.match(entry.complete);
            if (match && match.index !== undefined) {
                return {
                    tag: entry.tag,
                    htmlTag: entry.htmlTag,
                    content: match[1],
                    index: match.index,
                    length: match[0].length,
                    delimiter: entry.tag.delimiter!,
                };
            }
        }
        return null;
    }

    /**
     * Find the first unclosed delimiter opener in the text.
     *
     *   converter.findUnmatchedOpener('hello **world')
     */
    findUnmatchedOpener(text: string): DelimiterMatch | null {
        for (const entry of this.delimiterRegexes) {
            const match = text.match(entry.open);
            if (match && match.index !== undefined) {
                const before = text.slice(0, match.index);
                if (before.endsWith('<') || before.endsWith('/')) {
                    continue;
                }
                return {
                    tag: entry.tag,
                    htmlTag: entry.htmlTag,
                    content: match[1],
                    index: match.index,
                    length: match[0].length,
                    delimiter: entry.tag.delimiter!,
                };
            }
        }
        return null;
    }

    /**
     * Look up the Tag definition for an HTML element by its tag name.
     *
     *   converter.getTagForElement(strongElement)
     */
    getTagForElement(element: HTMLElement): Tag | null {
        const tag = this.tags.get(element.tagName);
        if (tag && tag.delimiter) {
            return tag;
        }
        return null;
    }

    /**
     * CSS selector string matching all elements that should show
     * editing context.
     *
     *   element.matches(converter.getEditableSelector())
     */
    getEditableSelector(): string {
        return this.editableSelectorCache;
    }

    /**
     * Split markdown into lines, match each against block tags in
     * priority order, and concatenate the resulting HTML.
     */
    private processBlocks(markdown: string): string {
        const lines = markdown.replace(/\r\n/g, '\n').split('\n');
        const output: string[] = [];
        const blankLine = /^\s*$/;
        const refDefinition = /^\[(?<label>[^\]]+)\]:\s+(?<url>\S+)(?:\s+"(?<title>[^"]*)")?$/;
        let lineIndex = 0;

        // Collect reference link definitions
        this.referenceLinks = new Map();
        for (const line of lines) {
            const match = line.match(refDefinition);
            if (match?.groups) {
                this.referenceLinks.set(
                    match.groups.label.toLowerCase(),
                    {
                        url: match.groups.url,
                        title: match.groups.title,
                    },
                );
            }
        }

        while (lineIndex < lines.length) {
            if (blankLine.test(lines[lineIndex]) || refDefinition.test(lines[lineIndex])) {
                lineIndex++;
                continue;
            }

            let matched = false;
            for (const tag of this.blockTags) {
                const context: MatchContext = {
                    lines,
                    index: lineIndex,
                    text: '',
                    offset: 0,
                };
                const token = tag.match(context);
                if (!token) {
                    continue;
                }
                output.push(tag.toHTML(token, this.cachedConverter));
                lineIndex += token.consumed;
                matched = true;
                break;
            }

            if (!matched) {
                lineIndex++;
            }
        }

        return output.join('\n');
    }

    /**
     * Convert inline markdown to HTML using the tokenizer.
     * Tokenizes the source, then walks the token stream to build HTML.
     * Open/close delimiter pairs are matched using a stack.
     */
    private processInline(source: string): string {
        let text = source;

        // Process inline macros before tokenizing — they produce HTML
        // that should pass through without further parsing
        if (this.macroMap.size > 0) {
            const placeholders: string[] = [];
            text = processInlineMacros(text, this.macroMap, this.cachedConverter, placeholders);
            // Restore placeholders to their HTML content
            const placeholderPattern = /\x00P(?<index>\d+)\x00/g;
            text = text.replace(placeholderPattern, (_, index: string) =>
                placeholders[parseInt(index)]
            );
        }

        // Resolve reference links before tokenizing
        text = this.resolveReferenceLinks(text);
        // Normalize _ emphasis to *
        text = this.normalizeUnderscores(text);
        const tokens = this.tokenizer.tokenize(text);
        return this.tokensToHTML(tokens);
    }

    /**
     * Replace [text][ref] and [text][] with [text](url) using the
     * reference definitions collected during block parsing.
     */
    private resolveReferenceLinks(text: string): string {
        if (this.referenceLinks.size === 0) {
            return text;
        }
        const refLink = /\[(?<text>[^\[\]]+)\]\[(?<label>[^\]]*)\]/g;
        return text.replace(refLink, (...args) => {
            const groups = args[args.length - 1] as Record<string, string>;
            const label = (groups.label || groups.text).toLowerCase();
            const ref = this.referenceLinks.get(label);
            if (!ref) {
                return args[0];
            }
            const titlePart = ref.title ? ` "${ref.title}"` : '';
            return `[${groups.text}](${ref.url}${titlePart})`;
        });
    }

    /**
     * Normalize flanking underscore runs to asterisks so the tokenizer
     * only needs to handle * delimiters for emphasis.
     */
    private normalizeUnderscores(text: string): string {
        // Protect backslash-escaped underscores from normalization
        const escapePlaceholder = '\x00U\x00';
        const safeText = text.replace(/\\_/g, escapePlaceholder);

        const punctuation = `[\\s.,;:!?'"()\\[\\]{}<>\\-/\\\\~#@&^|]`;
        const openRun = new RegExp(
            `(?<=^|${punctuation})` +  // preceded by start, space, or punctuation
            `(_+)` +                   // one or more underscores
            `(?=\\S)`,                 // followed by non-whitespace
            'g'
        );
        const closeRun = new RegExp(
            `(?<=\\S)` +               // preceded by non-whitespace
            `(_+)` +                   // one or more underscores
            `(?=$|${punctuation})`,    // followed by end, space, or punctuation
            'g'
        );
        const toAsterisks = (_: string, run: string) => '*'.repeat(run.length);
        const normalized = safeText
            .replace(openRun, toAsterisks)
            .replace(closeRun, toAsterisks);

        return normalized.replace(/\x00U\x00/g, '\\_');
    }

    /**
     * Convert a token stream to HTML. Matches open/close delimiter
     * pairs and wraps their content in the appropriate HTML tags.
     * Unmatched delimiters are emitted as literal text.
     */
    private tokensToHTML(tokens: InlineToken[]): string {
        // Build a map from delimiter string to tag info
        const delimiterToTag = new Map<string, { htmlTag: string; name: string }>();
        for (const tag of this.inlineTags) {
            if (tag.delimiter) {
                const htmlTag = tag.name === 'boldItalic'
                    ? 'em'
                    : (tag.selector as string).split(',')[0].toLowerCase();
                delimiterToTag.set(tag.delimiter, {
                    htmlTag,
                    name: tag.name,
                });
            }
        }

        // First pass: match open/close pairs using a stack
        const paired = this.pairDelimiters(tokens);

        // Second pass: build HTML from paired tokens
        let html = '';
        for (const token of paired) {
            switch (token.role) {
                case 'text':
                    html += escapeHtml(token.value);
                    break;
                case 'open': {
                    const info = delimiterToTag.get(token.delimiter!);
                    if (info) {
                        if (info.name === 'boldItalic') {
                            html += '<em><strong>';
                        } else {
                            html += `<${info.htmlTag}>`;
                        }
                    } else {
                        html += escapeHtml(token.value);
                    }
                    break;
                }
                case 'close': {
                    const info = delimiterToTag.get(token.delimiter!);
                    if (info) {
                        if (info.name === 'boldItalic') {
                            html += '</strong></em>';
                        } else {
                            html += `</${info.htmlTag}>`;
                        }
                    } else {
                        html += escapeHtml(token.value);
                    }
                    break;
                }
                case 'code':
                    html += `<code>${escapeHtml(token.content || '')}</code>`;
                    break;
                case 'link': {
                    const titleAttr = token.title
                        ? ` title="${escapeHtml(token.title)}"`
                        : '';
                    // Process link text for nested inline formatting
                    const innerTokens = this.tokenizer.tokenize(token.value);
                    const innerHtml = this.tokensToHTML(innerTokens);
                    // Strip any nested <a> tags (links can't contain links)
                    const nestedLink = /<a[^>]*>|<\/a>/g;
                    const cleanInner = innerHtml.replace(nestedLink, '');
                    html += `<a href="${escapeHtml(token.href!)}"${titleAttr}>${cleanInner}</a>`;
                    break;
                }
                case 'autolink':
                    html += `<a href="${escapeHtml(token.href!)}">${escapeHtml(token.value)}</a>`;
                    break;
                case 'html':
                    html += token.value;
                    break;
                case 'break':
                    html += '<br>';
                    break;
                default:
                    html += escapeHtml(token.value);
            }
        }
        return html;
    }

    /**
     * Match open/close delimiter pairs in a token stream. Unmatched
     * openers/closers are converted to text tokens so they render
     * as literal characters.
     */
    private pairDelimiters(tokens: InlineToken[]): InlineToken[] {
        const openStack: number[] = [];
        const result = [...tokens];

        // Track which delimiter types are currently open to prevent
        // forbidden nesting (e.g. <del> inside <del>, <em> inside <em>)
        const openDelimiters = new Set<string>();

        for (let index = 0; index < result.length; index++) {
            const token = result[index];
            if (token.role === 'open') {
                // Don't open a delimiter that's already open (prevents nesting)
                if (openDelimiters.has(token.delimiter!)) {
                    result[index] = {
                        role: 'text',
                        value: token.value,
                    };
                    continue;
                }
                openStack.push(index);
                openDelimiters.add(token.delimiter!);
            } else if (token.role === 'close') {
                let matched = false;
                for (let stackIndex = openStack.length - 1; stackIndex >= 0; stackIndex--) {
                    const openerIndex = openStack[stackIndex];
                    if (result[openerIndex].delimiter === token.delimiter) {
                        openStack.splice(stackIndex, 1);
                        openDelimiters.delete(token.delimiter!);
                        matched = true;
                        break;
                    }
                }
                if (!matched) {
                    result[index] = {
                        role: 'text',
                        value: token.value,
                    };
                }
            }
        }

        // Any remaining unmatched openers become literal text
        for (const openerIndex of openStack) {
            result[openerIndex] = {
                role: 'text',
                value: result[openerIndex].value,
            };
        }

        return result;
    }

    /**
     * Serialize a DOM node to markdown using the serializer for inline
     * content and custom logic for block-level elements.
     */
    private serializeNode(node: Node): string {
        if (node.nodeType === 3) {
            return this.serializer.serialize(node);
        }
        if (node.nodeType !== 1) {
            return '';
        }
        const element = node as HTMLElement;

        // CSS selectors (e.g. [data-macro]) are more specific
        const cssSelectorMatch = this.matchCssSelector(element);
        if (cssSelectorMatch) {
            return cssSelectorMatch.toMarkdown(element, this.cachedConverter);
        }

        // Inline elements: use the serializer which handles escaping
        // via typed tokens (text vs delimiter separation)
        const inlineTag = this.tags.get(element.nodeName);
        if (inlineTag && (inlineTag.delimiter || inlineTag.name === 'link'
            || inlineTag.name === 'code' || inlineTag.name === 'hardBreak')) {
            return this.serializer.serialize(element);
        }

        // Block elements: use the tag's toMarkdown
        const tag = this.tags.get(element.nodeName);
        if (tag) {
            return tag.toMarkdown(element, this.cachedConverter);
        }

        return this.serializeChildren(node);
    }

    private matchCssSelector(element: HTMLElement): Tag | null {
        for (const [selector, tag] of this.tags.entries()) {
            if (!selector.includes('[') && !selector.includes('.') && !selector.includes('#')) {
                continue;
            }
            const uppercaseTagName = /^[A-Z]+/;
            const normalized = selector.replace(uppercaseTagName, part => part.toLowerCase());
            try {
                if (element.matches(normalized)) {
                    return tag;
                }
            } catch {
                // Invalid selector — skip
            }
        }
        return null;
    }

    private serializeChildren(node: Node): string {
        return Array.from(node.childNodes)
            .map(child => this.serializeNode(child))
            .join('');
    }

    /**
     * Build the inline tokenizer from registered delimiter-based tags.
     */
    private buildTokenizer(): InlineTokenizer {
        const hasCodeTag = this.inlineTags.some(tag => tag.name === 'code');
        const delimiterDefs: DelimiterDef[] = this.inlineTags
            .filter(tag => tag.delimiter && tag.name !== 'code')
            .map(tag => ({
                delimiter: tag.delimiter!,
                htmlTag: tag.name === 'boldItalic'
                    ? 'em'
                    : (tag.selector as string).split(',')[0].toLowerCase(),
                recursive: tag.recursive !== false,
                precedence: tag.precedence ?? 50,
            }));
        return new InlineTokenizer(delimiterDefs, { codeSpans: hasCodeTag });
    }

    /**
     * Build the markdown serializer from registered tags. Maps HTML
     * element names to their serialization strategy (delimiter wrap
     * or custom function).
     */
    private buildSerializer(): MarkdownSerializer {
        const tagMap = new Map<string, SerializerTagDef>();
        const delimiterChars = new Set<string>();

        for (const [selector, tag] of this.tags.entries()) {
            if (tag.delimiter) {
                delimiterChars.add(tag.delimiter[0]);
                // Delimiter-based tags: emit delimiter + children + delimiter
                for (const part of selector.split(',').map(part => part.trim())) {
                    tagMap.set(part, { delimiter: tag.delimiter });
                }
            } else if (tag.name === 'link') {
                tagMap.set('A', {
                    serialize: (element, children) => {
                        const href = element.getAttribute('href') || '';
                        const title = element.getAttribute('title');
                        const titlePart = title ? ` "${title}"` : '';
                        return '[' + children() + '](' + href + titlePart + ')';
                    },
                });
            } else if (tag.name === 'hardBreak') {
                tagMap.set('BR', {
                    serialize: () => '  \n',
                });
            } else if (tag.name === 'fencedCode') {
                tagMap.set('PRE', {
                    serialize: (element) => {
                        const code = element.querySelector('code');
                        const langMatch = (code?.getAttribute('class') || '').match(/language-(\S+)/);
                        const lang = langMatch ? langMatch[1] : '';
                        const content = code?.textContent || element.textContent || '';
                        return '\n\n```' + lang + '\n' + content + '\n```\n\n';
                    },
                });
            }
        }

        // CODE gets a custom serializer because its content is literal
        tagMap.set('CODE', {
            serialize: (element) => {
                // Code inside <pre> is handled by the PRE serializer
                if (element.parentNode?.nodeName === 'PRE') {
                    return element.textContent || '';
                }
                return '`' + (element.textContent || '') + '`';
            },
        });

        return new MarkdownSerializer(tagMap, delimiterChars);
    }

    private buildDelimiterRegexes(): { tag: Tag; htmlTag: string; complete: RegExp; open: RegExp }[] {
        const escapeRegex = /[.*+?^${}()|[\]\\]/g;
        const sorted = this.inlineTags
            .filter(tag => tag.delimiter)
            .sort((first, second) => (first.precedence ?? 50) - (second.precedence ?? 50));

        return sorted.map(tag => {
            const delimiter = tag.delimiter!;
            const escaped = delimiter.replace(escapeRegex, '\\$&');
            const escapedChar = delimiter[0].replace(escapeRegex, '\\$&');
            const htmlTag = tag.name === 'boldItalic'
                ? 'em'
                : (tag.selector as string).split(',')[0].toLowerCase();
            return {
                tag,
                htmlTag,
                complete: new RegExp(
                    `(?<!${escapedChar})` +
                    `${escaped}` +
                    `(?!${escapedChar})` +
                    `([^\\x01\\x02]+?)` +
                    `(?<!${escapedChar})` +
                    `${escaped}`
                ),
                open: new RegExp(
                    `(?<!${escapedChar})` +
                    `${escaped}` +
                    `(?!${escapedChar})` +
                    `([^\\x01\\x02]+)$`
                ),
            };
        });
    }

    private buildEditableSelector(): string {
        return [
            ...this.inlineTags,
            ...this.blockTags,
        ].filter(tag => typeof tag.selector === 'string')
         .map(tag => (tag.selector as string).toLowerCase())
         .join(', ');
    }

    private makeConverter(): Converter {
        return {
            inline: (source) => this.processInline(source),
            block: (markdown) => this.processBlocks(markdown),
            children: (node) => this.serializeChildren(node),
            node: (node) => this.serializeNode(node),
        };
    }
}