Reimplement as a tokenizer with GFM parity
This commit is contained in:
gsb
+564
-172
@@ -1,18 +1,18 @@
|
||||
/*
|
||||
* hopdown.ts — configurable markdown↔HTML converter.
|
||||
*
|
||||
* Usage:
|
||||
* const converter = new HopDown();
|
||||
* const converter = new HopDown({ exclude: ['table'] });
|
||||
* const converter = new HopDown({ tags: { ...defaultTags, 'DEL,S,STRIKE': strikethrough } });
|
||||
*
|
||||
* converter.toHTML('**bold**');
|
||||
* converter.toMarkdown('<strong>bold</strong>');
|
||||
* HopDown orchestrates markdown↔HTML conversion using a tokenizer for
|
||||
* inline parsing and a serializer for HTML→markdown. Block-level parsing
|
||||
* uses Tag definitions directly. The tokenizer/serializer architecture
|
||||
* ensures correct round-trips by separating structural delimiters from
|
||||
* literal text at the type level.
|
||||
*/
|
||||
|
||||
import type { Converter, MatchContext, Tag } from './types';
|
||||
import { defaultBlockTags, defaultInlineTags, defaultTags, escapeHtml, parseListBlock } from './tags';
|
||||
import type { Converter, MatchContext, Tag, DelimiterMatch } from './types';
|
||||
import { defaultBlockTags, defaultInlineTags, defaultTags, escapeHtml } from './tags';
|
||||
import { buildMacroTags, processInlineMacros, type MacroDef } from './macros';
|
||||
import { InlineTokenizer, type InlineToken, type DelimiterDef } from './tokenizer';
|
||||
import { MarkdownSerializer, type SerializerTagDef } from './serializer';
|
||||
|
||||
export type TagMap = Record<string, Tag>;
|
||||
|
||||
@@ -23,17 +23,25 @@ export interface HopDownOptions {
|
||||
}
|
||||
|
||||
/**
|
||||
* A configurable markdown↔HTML converter.
|
||||
* Configurable markdown↔HTML converter. Uses a tokenizer for inline
|
||||
* parsing (markdown→HTML) and a serializer for HTML→markdown. Block
|
||||
* parsing delegates to Tag definitions.
|
||||
*
|
||||
* By default includes all standard tags. Pass options to customize:
|
||||
* - tags: a mapping of HTML selectors to Tag definitions
|
||||
* - exclude: remove specific tags by name from the defaults
|
||||
* const converter = new HopDown();
|
||||
* converter.toHTML('**bold**');
|
||||
* converter.toMarkdown('<strong>bold</strong>');
|
||||
*/
|
||||
export class HopDown {
|
||||
private blockTags: Tag[];
|
||||
private inlineTags: Tag[];
|
||||
private tags: Map<string, Tag>;
|
||||
private macroMap: Map<string, MacroDef>;
|
||||
private referenceLinks: Map<string, { url: string; title?: string }>;
|
||||
private tokenizer: InlineTokenizer;
|
||||
private serializer: MarkdownSerializer;
|
||||
private cachedConverter: Converter;
|
||||
private delimiterRegexes: { tag: Tag; htmlTag: string; complete: RegExp; open: RegExp }[];
|
||||
private editableSelectorCache: string;
|
||||
|
||||
constructor(options: HopDownOptions = {}) {
|
||||
let tagMap: TagMap;
|
||||
@@ -49,8 +57,8 @@ export class HopDown {
|
||||
tagMap = defaultTags;
|
||||
}
|
||||
|
||||
// Build macro tags if macros are provided
|
||||
this.macroMap = new Map();
|
||||
this.referenceLinks = new Map();
|
||||
if (options.macros && options.macros.length > 0) {
|
||||
const { blockTag, selectorTag, macroMap } = buildMacroTags(options.macros);
|
||||
this.macroMap = macroMap;
|
||||
@@ -59,20 +67,27 @@ export class HopDown {
|
||||
}
|
||||
|
||||
const allTags = Object.values(tagMap);
|
||||
const defaultBlockNames = new Set(Object.values(defaultBlockTags).map(t => t.name));
|
||||
const defaultInlineNames = new Set(Object.values(defaultInlineTags).map(t => t.name));
|
||||
const defaultBlockNames = new Set(Object.values(defaultBlockTags).map(tag => tag.name));
|
||||
const defaultInlineNames = new Set(Object.values(defaultInlineTags).map(tag => tag.name));
|
||||
|
||||
this.blockTags = allTags.filter(tag =>
|
||||
defaultBlockNames.has(tag.name) || tag.name === 'macro' ||
|
||||
(!defaultInlineNames.has(tag.name) && !tag.pattern)
|
||||
);
|
||||
|
||||
// Ensure macro block tag runs after fencedCode but before everything else
|
||||
// Macro block tag must run after fencedCode (so code blocks aren't
|
||||
// parsed as macros) but before paragraph (the catch-all)
|
||||
this.blockTags.sort((a, b) => {
|
||||
const order = (t: Tag) => {
|
||||
if (t.name === 'fencedCode') return 0;
|
||||
if (t.name === 'macro') return 1;
|
||||
if (t.name === 'paragraph') return 99;
|
||||
const order = (tag: Tag) => {
|
||||
if (tag.name === 'fencedCode') {
|
||||
return 0;
|
||||
}
|
||||
if (tag.name === 'macro') {
|
||||
return 1;
|
||||
}
|
||||
if (tag.name === 'paragraph') {
|
||||
return 99;
|
||||
}
|
||||
return 50;
|
||||
};
|
||||
return order(a) - order(b);
|
||||
@@ -83,30 +98,35 @@ export class HopDown {
|
||||
);
|
||||
|
||||
this.tags = new Map();
|
||||
this.registerSelectors(tagMap);
|
||||
this.validateInlineTags();
|
||||
|
||||
this.tokenizer = this.buildTokenizer();
|
||||
this.serializer = this.buildSerializer();
|
||||
this.cachedConverter = this.makeConverter();
|
||||
this.delimiterRegexes = this.buildDelimiterRegexes();
|
||||
this.editableSelectorCache = this.buildEditableSelector();
|
||||
}
|
||||
|
||||
private registerSelectors(tagMap: TagMap): void {
|
||||
for (const [selector, tag] of Object.entries(tagMap)) {
|
||||
for (const sel of selector.split(',').map(s => s.trim()).filter(Boolean)) {
|
||||
if (sel.startsWith('_')) {
|
||||
const parts = selector.split(',').map(part => part.trim()).filter(Boolean);
|
||||
for (const part of parts) {
|
||||
if (part.startsWith('_')) {
|
||||
continue;
|
||||
}
|
||||
const existing = this.tags.get(sel);
|
||||
const existing = this.tags.get(part);
|
||||
if (existing && existing !== tag) {
|
||||
throw new Error(
|
||||
`HTML tag "${sel}" is claimed by both "${existing.name}" and "${tag.name}". ` +
|
||||
`HTML tag "${part}" is claimed by both "${existing.name}" and "${tag.name}". ` +
|
||||
`Use the exclude option to remove one before adding the other.`
|
||||
);
|
||||
}
|
||||
this.tags.set(sel, tag);
|
||||
this.tags.set(part, tag);
|
||||
}
|
||||
}
|
||||
|
||||
this.validateInlineTags();
|
||||
}
|
||||
|
||||
/**
|
||||
* Verify that no two inline tags have colliding delimiters without
|
||||
* correct precedence ordering. If delimiter A is a prefix of delimiter B,
|
||||
* B must have lower (earlier) precedence so the longer match wins.
|
||||
*/
|
||||
private validateInlineTags(): void {
|
||||
const withDelimiters = this.inlineTags
|
||||
.filter(tag => tag.delimiter)
|
||||
@@ -116,17 +136,17 @@ export class HopDown {
|
||||
precedence: tag.precedence as number ?? 50,
|
||||
}));
|
||||
|
||||
for (let i = 0; i < withDelimiters.length; i++) {
|
||||
for (let j = i + 1; j < withDelimiters.length; j++) {
|
||||
const a = withDelimiters[i];
|
||||
const b = withDelimiters[j];
|
||||
const aPrefix = b.delimiter.startsWith(a.delimiter);
|
||||
const bPrefix = a.delimiter.startsWith(b.delimiter);
|
||||
if (!aPrefix && !bPrefix) {
|
||||
for (let outer = 0; outer < withDelimiters.length; outer++) {
|
||||
for (let inner = outer + 1; inner < withDelimiters.length; inner++) {
|
||||
const first = withDelimiters[outer];
|
||||
const second = withDelimiters[inner];
|
||||
const firstIsPrefix = second.delimiter.startsWith(first.delimiter);
|
||||
const secondIsPrefix = first.delimiter.startsWith(second.delimiter);
|
||||
if (!firstIsPrefix && !secondIsPrefix) {
|
||||
continue;
|
||||
}
|
||||
const longer = a.delimiter.length > b.delimiter.length ? a : b;
|
||||
const shorter = a.delimiter.length > b.delimiter.length ? b : a;
|
||||
const longer = first.delimiter.length > second.delimiter.length ? first : second;
|
||||
const shorter = first.delimiter.length > second.delimiter.length ? second : first;
|
||||
if (longer.precedence >= shorter.precedence) {
|
||||
throw new Error(
|
||||
`Inline tag "${longer.name}" (delimiter "${longer.delimiter}") must have ` +
|
||||
@@ -141,42 +161,145 @@ export class HopDown {
|
||||
|
||||
/**
|
||||
* Convert a markdown string to HTML.
|
||||
*
|
||||
* converter.toHTML('# Hello\n\n**bold** text')
|
||||
*/
|
||||
toHTML(md: string): string {
|
||||
return this.processBlocks(md);
|
||||
toHTML(markdown: string): string {
|
||||
return this.processBlocks(markdown);
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert an HTML string back to markdown.
|
||||
* Convert an HTML string back to markdown. Uses the serializer
|
||||
* which produces correctly-escaped output via typed tokens.
|
||||
*
|
||||
* converter.toMarkdown('<h1>Hello</h1><p><strong>bold</strong> text</p>')
|
||||
*/
|
||||
toMarkdown(html: string): string {
|
||||
const container = document.createElement('div');
|
||||
container.innerHTML = html;
|
||||
return this.nodeToMd(container).replace(/\n{3,}/g, '\n\n').trim();
|
||||
return this.serializeNode(container).replace(/\n{3,}/g, '\n\n').trim();
|
||||
}
|
||||
|
||||
/**
|
||||
* Return the block tags for external iteration (e.g. speculative rendering).
|
||||
* The registered block-level tags. Used by the WYSIWYG editor
|
||||
* to detect block syntax patterns during live editing.
|
||||
*
|
||||
* converter.getBlockTags().forEach(tag => console.log(tag.name))
|
||||
*/
|
||||
getBlockTags(): Tag[] {
|
||||
return this.blockTags;
|
||||
}
|
||||
|
||||
/**
|
||||
* Return the inline tags for external iteration (e.g. speculative rendering).
|
||||
* The registered inline tags. Used by the WYSIWYG editor to
|
||||
* build delimiter regexes for speculative rendering.
|
||||
*
|
||||
* converter.getInlineTags().filter(tag => tag.delimiter)
|
||||
*/
|
||||
getInlineTags(): Tag[] {
|
||||
return this.inlineTags;
|
||||
}
|
||||
|
||||
private processBlocks(md: string): string {
|
||||
const lines = md.replace(/\r\n/g, '\n').split('\n');
|
||||
const output: string[] = [];
|
||||
let index = 0;
|
||||
/**
|
||||
* Find the first complete delimiter pair in the text.
|
||||
*
|
||||
* converter.findCompletePair('hello **world** end')
|
||||
*/
|
||||
findCompletePair(text: string): DelimiterMatch | null {
|
||||
for (const entry of this.delimiterRegexes) {
|
||||
const match = text.match(entry.complete);
|
||||
if (match && match.index !== undefined) {
|
||||
return {
|
||||
tag: entry.tag,
|
||||
htmlTag: entry.htmlTag,
|
||||
content: match[1],
|
||||
index: match.index,
|
||||
length: match[0].length,
|
||||
delimiter: entry.tag.delimiter!,
|
||||
};
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
while (index < lines.length) {
|
||||
if (/^\s*$/.test(lines[index])) {
|
||||
index++;
|
||||
/**
|
||||
* Find the first unclosed delimiter opener in the text.
|
||||
*
|
||||
* converter.findUnmatchedOpener('hello **world')
|
||||
*/
|
||||
findUnmatchedOpener(text: string): DelimiterMatch | null {
|
||||
for (const entry of this.delimiterRegexes) {
|
||||
const match = text.match(entry.open);
|
||||
if (match && match.index !== undefined) {
|
||||
const before = text.slice(0, match.index);
|
||||
if (before.endsWith('<') || before.endsWith('/')) {
|
||||
continue;
|
||||
}
|
||||
return {
|
||||
tag: entry.tag,
|
||||
htmlTag: entry.htmlTag,
|
||||
content: match[1],
|
||||
index: match.index,
|
||||
length: match[0].length,
|
||||
delimiter: entry.tag.delimiter!,
|
||||
};
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Look up the Tag definition for an HTML element by its tag name.
|
||||
*
|
||||
* converter.getTagForElement(strongElement)
|
||||
*/
|
||||
getTagForElement(element: HTMLElement): Tag | null {
|
||||
const tag = this.tags.get(element.tagName);
|
||||
if (tag && tag.delimiter) {
|
||||
return tag;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* CSS selector string matching all elements that should show
|
||||
* editing context.
|
||||
*
|
||||
* element.matches(converter.getEditableSelector())
|
||||
*/
|
||||
getEditableSelector(): string {
|
||||
return this.editableSelectorCache;
|
||||
}
|
||||
|
||||
/**
|
||||
* Split markdown into lines, match each against block tags in
|
||||
* priority order, and concatenate the resulting HTML.
|
||||
*/
|
||||
private processBlocks(markdown: string): string {
|
||||
const lines = markdown.replace(/\r\n/g, '\n').split('\n');
|
||||
const output: string[] = [];
|
||||
const blankLine = /^\s*$/;
|
||||
const refDefinition = /^\[(?<label>[^\]]+)\]:\s+(?<url>\S+)(?:\s+"(?<title>[^"]*)")?$/;
|
||||
let lineIndex = 0;
|
||||
|
||||
// Collect reference link definitions
|
||||
this.referenceLinks = new Map();
|
||||
for (const line of lines) {
|
||||
const match = line.match(refDefinition);
|
||||
if (match?.groups) {
|
||||
this.referenceLinks.set(
|
||||
match.groups.label.toLowerCase(),
|
||||
{
|
||||
url: match.groups.url,
|
||||
title: match.groups.title,
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
while (lineIndex < lines.length) {
|
||||
if (blankLine.test(lines[lineIndex]) || refDefinition.test(lines[lineIndex])) {
|
||||
lineIndex++;
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -184,166 +307,435 @@ export class HopDown {
|
||||
for (const tag of this.blockTags) {
|
||||
const context: MatchContext = {
|
||||
lines,
|
||||
index,
|
||||
index: lineIndex,
|
||||
text: '',
|
||||
offset: 0,
|
||||
};
|
||||
const token = tag.match(context);
|
||||
if (!token) continue;
|
||||
|
||||
if (tag.name === 'list') {
|
||||
const result = parseListBlock(lines, index, 0, (source) => this.processInline(source));
|
||||
output.push(result.html);
|
||||
index = result.end;
|
||||
} else {
|
||||
|
||||
output.push(tag.toHTML(token, this.makeConverter()));
|
||||
index += token.consumed;
|
||||
if (!token) {
|
||||
continue;
|
||||
}
|
||||
output.push(tag.toHTML(token, this.cachedConverter));
|
||||
lineIndex += token.consumed;
|
||||
matched = true;
|
||||
break;
|
||||
}
|
||||
|
||||
if (!matched) {
|
||||
index++;
|
||||
lineIndex++;
|
||||
}
|
||||
}
|
||||
|
||||
return output.join('\n');
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert inline markdown to HTML using the tokenizer.
|
||||
* Tokenizes the source, then walks the token stream to build HTML.
|
||||
* Open/close delimiter pairs are matched using a stack.
|
||||
*/
|
||||
private processInline(source: string): string {
|
||||
const sorted = [...this.inlineTags].sort((a, b) =>
|
||||
((a as any).precedence ?? 50) - ((b as any).precedence ?? 50)
|
||||
);
|
||||
|
||||
const placeholders: string[] = [];
|
||||
let text = source;
|
||||
|
||||
// Extract inline macros before other processing
|
||||
// Process inline macros before tokenizing — they produce HTML
|
||||
// that should pass through without further parsing
|
||||
if (this.macroMap.size > 0) {
|
||||
text = processInlineMacros(text, this.macroMap, this.makeConverter(), placeholders);
|
||||
const placeholders: string[] = [];
|
||||
text = processInlineMacros(text, this.macroMap, this.cachedConverter, placeholders);
|
||||
// Restore placeholders to their HTML content
|
||||
const placeholderPattern = /\x00P(?<index>\d+)\x00/g;
|
||||
text = text.replace(placeholderPattern, (_, index: string) =>
|
||||
placeholders[parseInt(index)]
|
||||
);
|
||||
}
|
||||
|
||||
// Pass 1: extract links and non-recursive tags into placeholders before escaping
|
||||
for (const tag of sorted) {
|
||||
const recursive = tag.recursive ?? true;
|
||||
|
||||
if (tag.name === 'link') {
|
||||
text = text.replace(/\[([^\]]+)\]\(([^)]+)\)/g, (_, linkText: string, href: string) => {
|
||||
let inner = linkText;
|
||||
const hasPlaceholders = /\x00P\d+\x00/.test(inner);
|
||||
if (hasPlaceholders) {
|
||||
inner = inner.replace(/\x00P(\d+)\x00/g, (__, idx: string) => placeholders[parseInt(idx)]);
|
||||
} else {
|
||||
inner = this.processInline(inner);
|
||||
}
|
||||
placeholders.push('<a href="' + escapeHtml(href) + '">' + inner + '</a>');
|
||||
return '\x00P' + (placeholders.length - 1) + '\x00';
|
||||
});
|
||||
} else if (!recursive && tag.pattern) {
|
||||
const globalPattern = tag.pattern as RegExp;
|
||||
globalPattern.lastIndex = 0;
|
||||
|
||||
text = text.replace(globalPattern, (_, content: string) => {
|
||||
placeholders.push(tag.toHTML(
|
||||
{ content, raw: '', consumed: 0 },
|
||||
this.makeConverter(),
|
||||
));
|
||||
return '\x00P' + (placeholders.length - 1) + '\x00';
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
text = escapeHtml(text);
|
||||
|
||||
// Pass 2: apply recursive tags in precedence order.
|
||||
// Content is already HTML-escaped from pass 1, so we wrap directly
|
||||
// without re-processing through convert.inline().
|
||||
for (const tag of sorted) {
|
||||
const recursive = tag.recursive ?? true;
|
||||
if (tag.name === 'link' || !recursive) {
|
||||
continue;
|
||||
}
|
||||
const globalPattern = tag.pattern as RegExp | undefined;
|
||||
if (globalPattern) {
|
||||
globalPattern.lastIndex = 0;
|
||||
text = text.replace(globalPattern, (_, content: string) => {
|
||||
const restored = content.replace(/\x00P(\d+)\x00/g, (__, idx: string) => placeholders[parseInt(idx)]);
|
||||
const htmlTag = tag.name === 'boldItalic'
|
||||
? null
|
||||
: ((tag.selector as string) || '').split(',')[0].toLowerCase();
|
||||
if (tag.name === 'boldItalic') {
|
||||
return '<em><strong>' + restored + '</strong></em>';
|
||||
}
|
||||
return `<${htmlTag}>${restored}</${htmlTag}>`;
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
text = text.replace(/\x00P(\d+)\x00/g, (_, index: string) => placeholders[parseInt(index)]);
|
||||
return text;
|
||||
// Resolve reference links before tokenizing
|
||||
text = this.resolveReferenceLinks(text);
|
||||
// Normalize _ emphasis to *
|
||||
text = this.normalizeUnderscores(text);
|
||||
const tokens = this.tokenizer.tokenize(text);
|
||||
return this.tokensToHTML(tokens);
|
||||
}
|
||||
|
||||
private nodeToMd(node: Node): string {
|
||||
/**
|
||||
* Replace [text][ref] and [text][] with [text](url) using the
|
||||
* reference definitions collected during block parsing.
|
||||
*/
|
||||
private resolveReferenceLinks(text: string): string {
|
||||
if (this.referenceLinks.size === 0) {
|
||||
return text;
|
||||
}
|
||||
const refLink = /\[(?<text>[^\[\]]+)\]\[(?<label>[^\]]*)\]/g;
|
||||
return text.replace(refLink, (...args) => {
|
||||
const groups = args[args.length - 1] as Record<string, string>;
|
||||
const label = (groups.label || groups.text).toLowerCase();
|
||||
const ref = this.referenceLinks.get(label);
|
||||
if (!ref) {
|
||||
return args[0];
|
||||
}
|
||||
const titlePart = ref.title ? ` "${ref.title}"` : '';
|
||||
return `[${groups.text}](${ref.url}${titlePart})`;
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalize flanking underscore runs to asterisks so the tokenizer
|
||||
* only needs to handle * delimiters for emphasis.
|
||||
*/
|
||||
private normalizeUnderscores(text: string): string {
|
||||
// Protect backslash-escaped underscores from normalization
|
||||
const escapePlaceholder = '\x00U\x00';
|
||||
const safeText = text.replace(/\\_/g, escapePlaceholder);
|
||||
|
||||
const punctuation = `[\\s.,;:!?'"()\\[\\]{}<>\\-/\\\\~#@&^|]`;
|
||||
const openRun = new RegExp(
|
||||
`(?<=^|${punctuation})` + // preceded by start, space, or punctuation
|
||||
`(_+)` + // one or more underscores
|
||||
`(?=\\S)`, // followed by non-whitespace
|
||||
'g'
|
||||
);
|
||||
const closeRun = new RegExp(
|
||||
`(?<=\\S)` + // preceded by non-whitespace
|
||||
`(_+)` + // one or more underscores
|
||||
`(?=$|${punctuation})`, // followed by end, space, or punctuation
|
||||
'g'
|
||||
);
|
||||
const toAsterisks = (_: string, run: string) => '*'.repeat(run.length);
|
||||
const normalized = safeText
|
||||
.replace(openRun, toAsterisks)
|
||||
.replace(closeRun, toAsterisks);
|
||||
|
||||
return normalized.replace(/\x00U\x00/g, '\\_');
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert a token stream to HTML. Matches open/close delimiter
|
||||
* pairs and wraps their content in the appropriate HTML tags.
|
||||
* Unmatched delimiters are emitted as literal text.
|
||||
*/
|
||||
private tokensToHTML(tokens: InlineToken[]): string {
|
||||
// Build a map from delimiter string to tag info
|
||||
const delimiterToTag = new Map<string, { htmlTag: string; name: string }>();
|
||||
for (const tag of this.inlineTags) {
|
||||
if (tag.delimiter) {
|
||||
const htmlTag = tag.name === 'boldItalic'
|
||||
? 'em'
|
||||
: (tag.selector as string).split(',')[0].toLowerCase();
|
||||
delimiterToTag.set(tag.delimiter, {
|
||||
htmlTag,
|
||||
name: tag.name,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// First pass: match open/close pairs using a stack
|
||||
const paired = this.pairDelimiters(tokens);
|
||||
|
||||
// Second pass: build HTML from paired tokens
|
||||
let html = '';
|
||||
for (const token of paired) {
|
||||
switch (token.role) {
|
||||
case 'text':
|
||||
html += escapeHtml(token.value);
|
||||
break;
|
||||
case 'open': {
|
||||
const info = delimiterToTag.get(token.delimiter!);
|
||||
if (info) {
|
||||
if (info.name === 'boldItalic') {
|
||||
html += '<em><strong>';
|
||||
} else {
|
||||
html += `<${info.htmlTag}>`;
|
||||
}
|
||||
} else {
|
||||
html += escapeHtml(token.value);
|
||||
}
|
||||
break;
|
||||
}
|
||||
case 'close': {
|
||||
const info = delimiterToTag.get(token.delimiter!);
|
||||
if (info) {
|
||||
if (info.name === 'boldItalic') {
|
||||
html += '</strong></em>';
|
||||
} else {
|
||||
html += `</${info.htmlTag}>`;
|
||||
}
|
||||
} else {
|
||||
html += escapeHtml(token.value);
|
||||
}
|
||||
break;
|
||||
}
|
||||
case 'code':
|
||||
html += `<code>${escapeHtml(token.content || '')}</code>`;
|
||||
break;
|
||||
case 'link': {
|
||||
const titleAttr = token.title
|
||||
? ` title="${escapeHtml(token.title)}"`
|
||||
: '';
|
||||
// Process link text for nested inline formatting
|
||||
const innerTokens = this.tokenizer.tokenize(token.value);
|
||||
const innerHtml = this.tokensToHTML(innerTokens);
|
||||
// Strip any nested <a> tags (links can't contain links)
|
||||
const nestedLink = /<a[^>]*>|<\/a>/g;
|
||||
const cleanInner = innerHtml.replace(nestedLink, '');
|
||||
html += `<a href="${escapeHtml(token.href!)}"${titleAttr}>${cleanInner}</a>`;
|
||||
break;
|
||||
}
|
||||
case 'autolink':
|
||||
html += `<a href="${escapeHtml(token.href!)}">${escapeHtml(token.value)}</a>`;
|
||||
break;
|
||||
case 'html':
|
||||
html += token.value;
|
||||
break;
|
||||
case 'break':
|
||||
html += '<br>';
|
||||
break;
|
||||
default:
|
||||
html += escapeHtml(token.value);
|
||||
}
|
||||
}
|
||||
return html;
|
||||
}
|
||||
|
||||
/**
|
||||
* Match open/close delimiter pairs in a token stream. Unmatched
|
||||
* openers/closers are converted to text tokens so they render
|
||||
* as literal characters.
|
||||
*/
|
||||
private pairDelimiters(tokens: InlineToken[]): InlineToken[] {
|
||||
const openStack: number[] = [];
|
||||
const result = [...tokens];
|
||||
|
||||
// Track which delimiter types are currently open to prevent
|
||||
// forbidden nesting (e.g. <del> inside <del>, <em> inside <em>)
|
||||
const openDelimiters = new Set<string>();
|
||||
|
||||
for (let index = 0; index < result.length; index++) {
|
||||
const token = result[index];
|
||||
if (token.role === 'open') {
|
||||
// Don't open a delimiter that's already open (prevents nesting)
|
||||
if (openDelimiters.has(token.delimiter!)) {
|
||||
result[index] = {
|
||||
role: 'text',
|
||||
value: token.value,
|
||||
};
|
||||
continue;
|
||||
}
|
||||
openStack.push(index);
|
||||
openDelimiters.add(token.delimiter!);
|
||||
} else if (token.role === 'close') {
|
||||
let matched = false;
|
||||
for (let stackIndex = openStack.length - 1; stackIndex >= 0; stackIndex--) {
|
||||
const openerIndex = openStack[stackIndex];
|
||||
if (result[openerIndex].delimiter === token.delimiter) {
|
||||
openStack.splice(stackIndex, 1);
|
||||
openDelimiters.delete(token.delimiter!);
|
||||
matched = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!matched) {
|
||||
result[index] = {
|
||||
role: 'text',
|
||||
value: token.value,
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Any remaining unmatched openers become literal text
|
||||
for (const openerIndex of openStack) {
|
||||
result[openerIndex] = {
|
||||
role: 'text',
|
||||
value: result[openerIndex].value,
|
||||
};
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Serialize a DOM node to markdown using the serializer for inline
|
||||
* content and custom logic for block-level elements.
|
||||
*/
|
||||
private serializeNode(node: Node): string {
|
||||
if (node.nodeType === 3) {
|
||||
return node.textContent || '';
|
||||
return this.serializer.serialize(node);
|
||||
}
|
||||
if (node.nodeType !== 1) {
|
||||
return '';
|
||||
}
|
||||
const element = node as HTMLElement;
|
||||
|
||||
// Check CSS selectors first (macro selectors are more specific)
|
||||
for (const [selector, selectorTag] of this.tags.entries()) {
|
||||
if (selector.includes('[') || selector.includes('.') || selector.includes('#')) {
|
||||
// Lowercase only the tag name portion for case-insensitive matching
|
||||
const normalized = selector.replace(/^[A-Z]+/, s => s.toLowerCase());
|
||||
try {
|
||||
if (element.matches(normalized)) {
|
||||
return selectorTag.toMarkdown(element, this.makeConverter());
|
||||
}
|
||||
} catch {
|
||||
// invalid selector, skip
|
||||
// CSS selectors (e.g. [data-macro]) are more specific
|
||||
const cssSelectorMatch = this.matchCssSelector(element);
|
||||
if (cssSelectorMatch) {
|
||||
return cssSelectorMatch.toMarkdown(element, this.cachedConverter);
|
||||
}
|
||||
|
||||
// Inline elements: use the serializer which handles escaping
|
||||
// via typed tokens (text vs delimiter separation)
|
||||
const inlineTag = this.tags.get(element.nodeName);
|
||||
if (inlineTag && (inlineTag.delimiter || inlineTag.name === 'link'
|
||||
|| inlineTag.name === 'code' || inlineTag.name === 'hardBreak')) {
|
||||
return this.serializer.serialize(element);
|
||||
}
|
||||
|
||||
// Block elements: use the tag's toMarkdown
|
||||
const tag = this.tags.get(element.nodeName);
|
||||
if (tag) {
|
||||
return tag.toMarkdown(element, this.cachedConverter);
|
||||
}
|
||||
|
||||
return this.serializeChildren(node);
|
||||
}
|
||||
|
||||
private matchCssSelector(element: HTMLElement): Tag | null {
|
||||
for (const [selector, tag] of this.tags.entries()) {
|
||||
if (!selector.includes('[') && !selector.includes('.') && !selector.includes('#')) {
|
||||
continue;
|
||||
}
|
||||
const uppercaseTagName = /^[A-Z]+/;
|
||||
const normalized = selector.replace(uppercaseTagName, part => part.toLowerCase());
|
||||
try {
|
||||
if (element.matches(normalized)) {
|
||||
return tag;
|
||||
}
|
||||
} catch {
|
||||
// Invalid selector — skip
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private serializeChildren(node: Node): string {
|
||||
return Array.from(node.childNodes)
|
||||
.map(child => this.serializeNode(child))
|
||||
.join('');
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the inline tokenizer from registered delimiter-based tags.
|
||||
*/
|
||||
private buildTokenizer(): InlineTokenizer {
|
||||
const hasCodeTag = this.inlineTags.some(tag => tag.name === 'code');
|
||||
const delimiterDefs: DelimiterDef[] = this.inlineTags
|
||||
.filter(tag => tag.delimiter && tag.name !== 'code')
|
||||
.map(tag => ({
|
||||
delimiter: tag.delimiter!,
|
||||
htmlTag: tag.name === 'boldItalic'
|
||||
? 'em'
|
||||
: (tag.selector as string).split(',')[0].toLowerCase(),
|
||||
recursive: tag.recursive !== false,
|
||||
precedence: tag.precedence ?? 50,
|
||||
}));
|
||||
return new InlineTokenizer(delimiterDefs, { codeSpans: hasCodeTag });
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the markdown serializer from registered tags. Maps HTML
|
||||
* element names to their serialization strategy (delimiter wrap
|
||||
* or custom function).
|
||||
*/
|
||||
private buildSerializer(): MarkdownSerializer {
|
||||
const tagMap = new Map<string, SerializerTagDef>();
|
||||
const delimiterChars = new Set<string>();
|
||||
|
||||
for (const [selector, tag] of this.tags.entries()) {
|
||||
if (tag.delimiter) {
|
||||
delimiterChars.add(tag.delimiter[0]);
|
||||
// Delimiter-based tags: emit delimiter + children + delimiter
|
||||
for (const part of selector.split(',').map(part => part.trim())) {
|
||||
tagMap.set(part, { delimiter: tag.delimiter });
|
||||
}
|
||||
} else if (tag.name === 'link') {
|
||||
tagMap.set('A', {
|
||||
serialize: (element, children) => {
|
||||
const href = element.getAttribute('href') || '';
|
||||
const title = element.getAttribute('title');
|
||||
const titlePart = title ? ` "${title}"` : '';
|
||||
return '[' + children() + '](' + href + titlePart + ')';
|
||||
},
|
||||
});
|
||||
} else if (tag.name === 'hardBreak') {
|
||||
tagMap.set('BR', {
|
||||
serialize: () => ' \n',
|
||||
});
|
||||
} else if (tag.name === 'fencedCode') {
|
||||
tagMap.set('PRE', {
|
||||
serialize: (element) => {
|
||||
const code = element.querySelector('code');
|
||||
const langMatch = (code?.getAttribute('class') || '').match(/language-(\S+)/);
|
||||
const lang = langMatch ? langMatch[1] : '';
|
||||
const content = code?.textContent || element.textContent || '';
|
||||
return '\n\n```' + lang + '\n' + content + '\n```\n\n';
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Then check by element name
|
||||
const tag = this.tags.get(element.nodeName);
|
||||
if (tag) {
|
||||
return tag.toMarkdown(element, this.makeConverter());
|
||||
}
|
||||
// CODE gets a custom serializer because its content is literal
|
||||
tagMap.set('CODE', {
|
||||
serialize: (element) => {
|
||||
// Code inside <pre> is handled by the PRE serializer
|
||||
if (element.parentNode?.nodeName === 'PRE') {
|
||||
return element.textContent || '';
|
||||
}
|
||||
return '`' + (element.textContent || '') + '`';
|
||||
},
|
||||
});
|
||||
|
||||
return this.childrenToMd(node);
|
||||
return new MarkdownSerializer(tagMap, delimiterChars);
|
||||
}
|
||||
|
||||
private childrenToMd(node: Node): string {
|
||||
return Array.from(node.childNodes).map(child => this.nodeToMd(child)).join('');
|
||||
private buildDelimiterRegexes(): { tag: Tag; htmlTag: string; complete: RegExp; open: RegExp }[] {
|
||||
const escapeRegex = /[.*+?^${}()|[\]\\]/g;
|
||||
const sorted = this.inlineTags
|
||||
.filter(tag => tag.delimiter)
|
||||
.sort((first, second) => (first.precedence ?? 50) - (second.precedence ?? 50));
|
||||
|
||||
return sorted.map(tag => {
|
||||
const delimiter = tag.delimiter!;
|
||||
const escaped = delimiter.replace(escapeRegex, '\\$&');
|
||||
const escapedChar = delimiter[0].replace(escapeRegex, '\\$&');
|
||||
const htmlTag = tag.name === 'boldItalic'
|
||||
? 'em'
|
||||
: (tag.selector as string).split(',')[0].toLowerCase();
|
||||
return {
|
||||
tag,
|
||||
htmlTag,
|
||||
complete: new RegExp(
|
||||
`(?<!${escapedChar})` +
|
||||
`${escaped}` +
|
||||
`(?!${escapedChar})` +
|
||||
`([^\\x01\\x02]+?)` +
|
||||
`(?<!${escapedChar})` +
|
||||
`${escaped}`
|
||||
),
|
||||
open: new RegExp(
|
||||
`(?<!${escapedChar})` +
|
||||
`${escaped}` +
|
||||
`(?!${escapedChar})` +
|
||||
`([^\\x01\\x02]+)$`
|
||||
),
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
private buildEditableSelector(): string {
|
||||
return [
|
||||
...this.inlineTags,
|
||||
...this.blockTags,
|
||||
].filter(tag => typeof tag.selector === 'string')
|
||||
.map(tag => (tag.selector as string).toLowerCase())
|
||||
.join(', ');
|
||||
}
|
||||
|
||||
private makeConverter(): Converter {
|
||||
return {
|
||||
inline: (source) => this.processInline(source),
|
||||
block: (md) => this.processBlocks(md),
|
||||
children: (node) => this.childrenToMd(node),
|
||||
node: (node) => this.nodeToMd(node),
|
||||
block: (markdown) => this.processBlocks(markdown),
|
||||
children: (node) => this.serializeChildren(node),
|
||||
node: (node) => this.serializeNode(node),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* A default HopDown instance with all standard tags enabled.
|
||||
* Use this for simple cases where no configuration is needed.
|
||||
*/
|
||||
const hopdown = new HopDown();
|
||||
|
||||
export function toHTML(md: string): string {
|
||||
return hopdown.toHTML(md);
|
||||
}
|
||||
|
||||
export function toMarkdown(html: string): string {
|
||||
return hopdown.toMarkdown(html);
|
||||
}
|
||||
|
||||
export default hopdown;
|
||||
|
||||
Reference in New Issue
Block a user