Reimplement as a tokenizer with GFM parity
This commit is contained in:
gsb
@@ -0,0 +1,447 @@
|
||||
/*
|
||||
* tokenizer.ts — Inline markdown tokenizer.
|
||||
*
|
||||
* Scans markdown text left-to-right producing a typed token stream.
|
||||
* Tokens carry their semantic role (delimiter, text, code, link, etc.)
|
||||
* so downstream consumers can make correct escaping and pairing
|
||||
* decisions without regex heuristics.
|
||||
*
|
||||
* const tokenizer = new InlineTokenizer(delimiterDefs);
|
||||
* const tokens = tokenizer.tokenize('hello **bold** end');
|
||||
* // [text "hello "] [open "**"] [text "bold"] [close "**"] [text " end"]
|
||||
*/
|
||||
|
||||
/**
|
||||
* A single token in the inline token stream. The `role` field
|
||||
* distinguishes structural markers from literal content, which
|
||||
* is the key insight that makes round-trip escaping correct.
|
||||
*/
|
||||
export interface InlineToken {
|
||||
role: 'text' | 'open' | 'close' | 'code' | 'link' | 'autolink' | 'html' | 'break';
|
||||
value: string;
|
||||
/** For link tokens: the href and optional title. */
|
||||
href?: string;
|
||||
title?: string;
|
||||
/** For delimiter tokens: which delimiter this is (e.g. '**'). */
|
||||
delimiter?: string;
|
||||
/** For code tokens: the raw content (not HTML-escaped). */
|
||||
content?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* A delimiter definition used by the tokenizer to recognize
|
||||
* opening and closing delimiter runs.
|
||||
*/
|
||||
export interface DelimiterDef {
|
||||
/** The delimiter string, e.g. '**', '*', '~~', '`'. */
|
||||
delimiter: string;
|
||||
/** The HTML tag name to emit, e.g. 'strong', 'em', 'del'. */
|
||||
htmlTag: string;
|
||||
/** Whether content inside this delimiter is parsed for further
|
||||
* inline markup. False for code spans. */
|
||||
recursive: boolean;
|
||||
/** Lower values are matched first. Ensures *** matches before **. */
|
||||
precedence: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Characters that count as punctuation for flanking delimiter rules.
|
||||
* A delimiter is left-flanking if preceded by whitespace/punctuation
|
||||
* and followed by non-whitespace. Right-flanking is the reverse.
|
||||
*/
|
||||
const PUNCTUATION = new Set(
|
||||
' \t\n.,;:!?\'"()[]{}/<>\\-~#@&^|*`_'.split('')
|
||||
);
|
||||
|
||||
/**
|
||||
* Characters that can be backslash-escaped in markdown.
|
||||
*/
|
||||
const ESCAPABLE = new Set(
|
||||
'\\`*_{}[]()#+-.!~|><'.split('')
|
||||
);
|
||||
|
||||
/**
|
||||
* Named HTML entities recognized by the tokenizer.
|
||||
*/
|
||||
const NAMED_ENTITIES: Record<string, string> = {
|
||||
'amp': '&',
|
||||
'lt': '<',
|
||||
'gt': '>',
|
||||
'quot': '"',
|
||||
'apos': "'",
|
||||
'nbsp': '\u00A0',
|
||||
};
|
||||
|
||||
/**
|
||||
* Scans markdown text into a stream of typed tokens. Handles
|
||||
* backslash escapes, entities, flanking rules, code spans, links,
|
||||
* autolinks, HTML tags, and hard line breaks.
|
||||
*
|
||||
* const tokenizer = new InlineTokenizer([
|
||||
* { delimiter: '**', htmlTag: 'strong', recursive: true, precedence: 40 },
|
||||
* { delimiter: '*', htmlTag: 'em', recursive: true, precedence: 50 },
|
||||
* ]);
|
||||
* const tokens = tokenizer.tokenize('**bold**');
|
||||
*/
|
||||
export class InlineTokenizer {
|
||||
private delimiters: DelimiterDef[];
|
||||
private codeSpansEnabled: boolean;
|
||||
|
||||
constructor(delimiters: DelimiterDef[], options?: { codeSpans?: boolean }) {
|
||||
this.codeSpansEnabled = options?.codeSpans !== false;
|
||||
// Sort by delimiter length descending so longer delimiters
|
||||
// are tried first (*** before ** before *)
|
||||
this.delimiters = [...delimiters].sort(
|
||||
(first, second) => second.delimiter.length - first.delimiter.length
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Tokenize a markdown string into an inline token stream.
|
||||
*
|
||||
* tokenizer.tokenize('hello **world**')
|
||||
* // [text "hello "] [open "**"] [text "world"] [close "**"]
|
||||
*/
|
||||
tokenize(source: string): InlineToken[] {
|
||||
const tokens: InlineToken[] = [];
|
||||
let position = 0;
|
||||
let textBuffer = '';
|
||||
|
||||
const flushText = () => {
|
||||
if (textBuffer.length > 0) {
|
||||
tokens.push({
|
||||
role: 'text',
|
||||
value: textBuffer,
|
||||
});
|
||||
textBuffer = '';
|
||||
}
|
||||
};
|
||||
|
||||
while (position < source.length) {
|
||||
const remaining = source.slice(position);
|
||||
|
||||
// Backslash escape: \X → literal X
|
||||
if (source[position] === '\\' && position + 1 < source.length) {
|
||||
const nextChar = source[position + 1];
|
||||
if (ESCAPABLE.has(nextChar)) {
|
||||
textBuffer += nextChar;
|
||||
position += 2;
|
||||
continue;
|
||||
}
|
||||
// \ before newline is a hard break
|
||||
if (nextChar === '\n') {
|
||||
flushText();
|
||||
tokens.push({ role: 'break', value: '<br>' });
|
||||
position += 2;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
// Hard line break: two+ trailing spaces before newline
|
||||
if (source[position] === ' ') {
|
||||
const spaceMatch = remaining.match(/^(?<spaces> {2,})\n/);
|
||||
if (spaceMatch?.groups) {
|
||||
flushText();
|
||||
tokens.push({ role: 'break', value: '<br>' });
|
||||
position += spaceMatch[0].length;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
// HTML entity resolution: &name; or &#digits; or &#xhex;
|
||||
if (source[position] === '&') {
|
||||
const resolved = this.resolveEntity(remaining);
|
||||
if (resolved) {
|
||||
textBuffer += resolved.character;
|
||||
position += resolved.length;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
// Code span: `content` — not parsed for further inline markup
|
||||
if (this.codeSpansEnabled && source[position] === '`') {
|
||||
const codeSpan = this.matchCodeSpan(source, position);
|
||||
if (codeSpan) {
|
||||
flushText();
|
||||
tokens.push({
|
||||
role: 'code',
|
||||
value: codeSpan.raw,
|
||||
content: codeSpan.content,
|
||||
});
|
||||
position += codeSpan.raw.length;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
// Link: [text](url) or [text](url "title")
|
||||
if (source[position] === '[') {
|
||||
const link = this.matchLink(source, position);
|
||||
if (link) {
|
||||
flushText();
|
||||
tokens.push({
|
||||
role: 'link',
|
||||
value: link.text,
|
||||
href: link.href,
|
||||
title: link.title,
|
||||
});
|
||||
position += link.length;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
// Autolink: <url>
|
||||
if (source[position] === '<') {
|
||||
const autolink = this.matchAutolink(remaining);
|
||||
if (autolink) {
|
||||
flushText();
|
||||
tokens.push({
|
||||
role: 'autolink',
|
||||
value: autolink.url,
|
||||
href: autolink.url,
|
||||
});
|
||||
position += autolink.length;
|
||||
continue;
|
||||
}
|
||||
// HTML tag passthrough
|
||||
const htmlTagMatch = this.matchHtmlTag(remaining);
|
||||
if (htmlTagMatch) {
|
||||
flushText();
|
||||
tokens.push({
|
||||
role: 'html',
|
||||
value: htmlTagMatch.tag,
|
||||
});
|
||||
position += htmlTagMatch.length;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
// Bare URL autolink: https://...
|
||||
if (remaining.startsWith('http://') || remaining.startsWith('https://')) {
|
||||
const bareUrl = this.matchBareUrl(remaining);
|
||||
if (bareUrl) {
|
||||
flushText();
|
||||
tokens.push({
|
||||
role: 'autolink',
|
||||
value: bareUrl.url,
|
||||
href: bareUrl.url,
|
||||
});
|
||||
position += bareUrl.length;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
// Delimiter: check each registered delimiter
|
||||
const delimiterMatch = this.matchDelimiter(source, position);
|
||||
if (delimiterMatch) {
|
||||
flushText();
|
||||
tokens.push(delimiterMatch.token);
|
||||
position += delimiterMatch.length;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Plain character
|
||||
textBuffer += source[position];
|
||||
position++;
|
||||
}
|
||||
|
||||
flushText();
|
||||
return tokens;
|
||||
}
|
||||
|
||||
/**
|
||||
* Try to resolve an HTML entity at the start of the string.
|
||||
* Returns the resolved character and the length consumed, or null.
|
||||
*/
|
||||
private resolveEntity(text: string): { character: string; length: number } | null {
|
||||
const namedPattern = /^&(?<name>[a-zA-Z]+);/;
|
||||
const numericPattern = /^&#(?<code>\d+);/;
|
||||
const hexPattern = /^&#x(?<hex>[0-9a-fA-F]+);/;
|
||||
|
||||
const named = text.match(namedPattern);
|
||||
if (named?.groups) {
|
||||
const resolved = NAMED_ENTITIES[named.groups.name.toLowerCase()];
|
||||
if (resolved) {
|
||||
return {
|
||||
character: resolved,
|
||||
length: named[0].length,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
const numeric = text.match(numericPattern);
|
||||
if (numeric?.groups) {
|
||||
return {
|
||||
character: String.fromCharCode(parseInt(numeric.groups.code, 10)),
|
||||
length: numeric[0].length,
|
||||
};
|
||||
}
|
||||
|
||||
const hex = text.match(hexPattern);
|
||||
if (hex?.groups) {
|
||||
return {
|
||||
character: String.fromCharCode(parseInt(hex.groups.hex, 16)),
|
||||
length: hex[0].length,
|
||||
};
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Match a code span starting at the given position.
|
||||
* Handles single backtick delimiters only (not multi-backtick).
|
||||
*/
|
||||
private matchCodeSpan(
|
||||
source: string,
|
||||
position: number,
|
||||
): { content: string; raw: string } | null {
|
||||
if (source[position] !== '`') {
|
||||
return null;
|
||||
}
|
||||
const closeIndex = source.indexOf('`', position + 1);
|
||||
if (closeIndex === -1) {
|
||||
return null;
|
||||
}
|
||||
const content = source.slice(position + 1, closeIndex);
|
||||
return {
|
||||
content,
|
||||
raw: source.slice(position, closeIndex + 1),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Match a markdown link [text](url) or [text](url "title")
|
||||
* starting at the given position. Disallows [ in link text
|
||||
* to prevent nested link ambiguity.
|
||||
*/
|
||||
private matchLink(
|
||||
source: string,
|
||||
position: number,
|
||||
): { text: string; href: string; title?: string; length: number } | null {
|
||||
const linkPattern = /^\[(?<text>[^\[\]]+)\]\((?<href>[^\s)]+)(?:\s+"(?<title>[^"]*)")?\)/;
|
||||
const match = source.slice(position).match(linkPattern);
|
||||
if (!match?.groups) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
text: match.groups.text,
|
||||
href: match.groups.href,
|
||||
title: match.groups.title,
|
||||
length: match[0].length,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Match an angle-bracket autolink <url> at the start of the string.
|
||||
*/
|
||||
private matchAutolink(text: string): { url: string; length: number } | null {
|
||||
const pattern = /^<(?<url>https?:\/\/[^\s>]+)>/;
|
||||
const match = text.match(pattern);
|
||||
if (!match?.groups) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
url: match.groups.url,
|
||||
length: match[0].length,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Match a bare URL (https://...) at the start of the string.
|
||||
*/
|
||||
private matchBareUrl(text: string): { url: string; length: number } | null {
|
||||
const pattern = /^https?:\/\/[^\s<>\x00]+/;
|
||||
const match = text.match(pattern);
|
||||
if (!match) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
url: match[0],
|
||||
length: match[0].length,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Match an HTML tag at the start of the string.
|
||||
*/
|
||||
private matchHtmlTag(text: string): { tag: string; length: number } | null {
|
||||
const pattern = /^<\/?[a-zA-Z][a-zA-Z0-9]*(?:\s+[^>]*)?\s*\/?>/;
|
||||
const match = text.match(pattern);
|
||||
if (!match) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
tag: match[0],
|
||||
length: match[0].length,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Try to match a delimiter at the given position. For runs of the
|
||||
* same character (e.g. *** = 3 asterisks), the run is split into
|
||||
* the longest registered delimiter that fits, then the remainder.
|
||||
* This handles cases like **bold***italic* where *** must split
|
||||
* into ** (close bold) + * (open italic).
|
||||
*/
|
||||
private matchDelimiter(
|
||||
source: string,
|
||||
position: number,
|
||||
): { token: InlineToken; length: number } | null {
|
||||
// Count the full run of the same character
|
||||
const runChar = source[position];
|
||||
let runLength = 0;
|
||||
while (position + runLength < source.length && source[position + runLength] === runChar) {
|
||||
runLength++;
|
||||
}
|
||||
if (runLength === 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
// Find registered delimiters that use this character
|
||||
const candidates = this.delimiters.filter(
|
||||
definition => definition.delimiter[0] === runChar
|
||||
);
|
||||
if (candidates.length === 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
// Try each candidate delimiter length (longest first, already sorted)
|
||||
for (const definition of candidates) {
|
||||
const delimiter = definition.delimiter;
|
||||
if (delimiter.length > runLength) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const charBefore = position > 0 ? source[position - 1] : '\n';
|
||||
const charAfter = source[position + delimiter.length];
|
||||
|
||||
const leftFlanking = (charBefore === undefined || PUNCTUATION.has(charBefore) || charBefore === '\n')
|
||||
&& charAfter !== undefined && charAfter !== ' ' && charAfter !== '\n' && charAfter !== '\t';
|
||||
|
||||
const rightFlanking = charBefore !== undefined && charBefore !== ' ' && charBefore !== '\n' && charBefore !== '\t'
|
||||
&& (charAfter === undefined || PUNCTUATION.has(charAfter) || charAfter === '\n');
|
||||
|
||||
if (leftFlanking) {
|
||||
return {
|
||||
token: {
|
||||
role: 'open',
|
||||
value: delimiter,
|
||||
delimiter,
|
||||
},
|
||||
length: delimiter.length,
|
||||
};
|
||||
}
|
||||
if (rightFlanking) {
|
||||
return {
|
||||
token: {
|
||||
role: 'close',
|
||||
value: delimiter,
|
||||
delimiter,
|
||||
},
|
||||
length: delimiter.length,
|
||||
};
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user