448 lines
15 KiB
TypeScript
448 lines
15 KiB
TypeScript
|
/*
|
||
|
|
* tokenizer.ts — Inline markdown tokenizer.
|
||
|
|
*
|
||
|
|
* Scans markdown text left-to-right producing a typed token stream.
|
||
|
|
* Tokens carry their semantic role (delimiter, text, code, link, etc.)
|
||
|
|
* so downstream consumers can make correct escaping and pairing
|
||
|
|
* decisions without regex heuristics.
|
||
|
|
*
|
||
|
|
* const tokenizer = new InlineTokenizer(delimiterDefs);
|
||
|
|
* const tokens = tokenizer.tokenize('hello **bold** end');
|
||
|
|
* // [text "hello "] [open "**"] [text "bold"] [close "**"] [text " end"]
|
||
|
|
*/
|
||
|
|
|
||
|
|
/**
|
||
|
|
* A single token in the inline token stream. The `role` field
|
||
|
|
* distinguishes structural markers from literal content, which
|
||
|
|
* is the key insight that makes round-trip escaping correct.
|
||
|
|
*/
|
||
|
|
export interface InlineToken {
|
||
|
|
role: 'text' | 'open' | 'close' | 'code' | 'link' | 'autolink' | 'html' | 'break';
|
||
|
|
value: string;
|
||
|
|
/** For link tokens: the href and optional title. */
|
||
|
|
href?: string;
|
||
|
|
title?: string;
|
||
|
|
/** For delimiter tokens: which delimiter this is (e.g. '**'). */
|
||
|
|
delimiter?: string;
|
||
|
|
/** For code tokens: the raw content (not HTML-escaped). */
|
||
|
|
content?: string;
|
||
|
|
}
|
||
|
|
|
||
|
|
/**
|
||
|
|
* A delimiter definition used by the tokenizer to recognize
|
||
|
|
* opening and closing delimiter runs.
|
||
|
|
*/
|
||
|
|
export interface DelimiterDef {
|
||
|
|
/** The delimiter string, e.g. '**', '*', '~~', '`'. */
|
||
|
|
delimiter: string;
|
||
|
|
/** The HTML tag name to emit, e.g. 'strong', 'em', 'del'. */
|
||
|
|
htmlTag: string;
|
||
|
|
/** Whether content inside this delimiter is parsed for further
|
||
|
|
* inline markup. False for code spans. */
|
||
|
|
recursive: boolean;
|
||
|
|
/** Lower values are matched first. Ensures *** matches before **. */
|
||
|
|
precedence: number;
|
||
|
|
}
|
||
|
|
|
||
|
|
/**
|
||
|
|
* Characters that count as punctuation for flanking delimiter rules.
|
||
|
|
* A delimiter is left-flanking if preceded by whitespace/punctuation
|
||
|
|
* and followed by non-whitespace. Right-flanking is the reverse.
|
||
|
|
*/
|
||
|
|
const PUNCTUATION = new Set(
|
||
|
|
' \t\n.,;:!?\'"()[]{}/<>\\-~#@&^|*`_'.split('')
|
||
|
|
);
|
||
|
|
|
||
|
|
/**
|
||
|
|
* Characters that can be backslash-escaped in markdown.
|
||
|
|
*/
|
||
|
|
const ESCAPABLE = new Set(
|
||
|
|
'\\`*_{}[]()#+-.!~|><'.split('')
|
||
|
|
);
|
||
|
|
|
||
|
|
/**
|
||
|
|
* Named HTML entities recognized by the tokenizer.
|
||
|
|
*/
|
||
|
|
const NAMED_ENTITIES: Record<string, string> = {
|
||
|
|
'amp': '&',
|
||
|
|
'lt': '<',
|
||
|
|
'gt': '>',
|
||
|
|
'quot': '"',
|
||
|
|
'apos': "'",
|
||
|
|
'nbsp': '\u00A0',
|
||
|
|
};
|
||
|
|
|
||
|
|
/**
|
||
|
|
* Scans markdown text into a stream of typed tokens. Handles
|
||
|
|
* backslash escapes, entities, flanking rules, code spans, links,
|
||
|
|
* autolinks, HTML tags, and hard line breaks.
|
||
|
|
*
|
||
|
|
* const tokenizer = new InlineTokenizer([
|
||
|
|
* { delimiter: '**', htmlTag: 'strong', recursive: true, precedence: 40 },
|
||
|
|
* { delimiter: '*', htmlTag: 'em', recursive: true, precedence: 50 },
|
||
|
|
* ]);
|
||
|
|
* const tokens = tokenizer.tokenize('**bold**');
|
||
|
|
*/
|
||
|
|
export class InlineTokenizer {
|
||
|
|
private delimiters: DelimiterDef[];
|
||
|
|
private codeSpansEnabled: boolean;
|
||
|
|
|
||
|
|
constructor(delimiters: DelimiterDef[], options?: { codeSpans?: boolean }) {
|
||
|
|
this.codeSpansEnabled = options?.codeSpans !== false;
|
||
|
|
// Sort by delimiter length descending so longer delimiters
|
||
|
|
// are tried first (*** before ** before *)
|
||
|
|
this.delimiters = [...delimiters].sort(
|
||
|
|
(first, second) => second.delimiter.length - first.delimiter.length
|
||
|
|
);
|
||
|
|
}
|
||
|
|
|
||
|
|
/**
|
||
|
|
* Tokenize a markdown string into an inline token stream.
|
||
|
|
*
|
||
|
|
* tokenizer.tokenize('hello **world**')
|
||
|
|
* // [text "hello "] [open "**"] [text "world"] [close "**"]
|
||
|
|
*/
|
||
|
|
tokenize(source: string): InlineToken[] {
|
||
|
|
const tokens: InlineToken[] = [];
|
||
|
|
let position = 0;
|
||
|
|
let textBuffer = '';
|
||
|
|
|
||
|
|
const flushText = () => {
|
||
|
|
if (textBuffer.length > 0) {
|
||
|
|
tokens.push({
|
||
|
|
role: 'text',
|
||
|
|
value: textBuffer,
|
||
|
|
});
|
||
|
|
textBuffer = '';
|
||
|
|
}
|
||
|
|
};
|
||
|
|
|
||
|
|
while (position < source.length) {
|
||
|
|
const remaining = source.slice(position);
|
||
|
|
|
||
|
|
// Backslash escape: \X → literal X
|
||
|
|
if (source[position] === '\\' && position + 1 < source.length) {
|
||
|
|
const nextChar = source[position + 1];
|
||
|
|
if (ESCAPABLE.has(nextChar)) {
|
||
|
|
textBuffer += nextChar;
|
||
|
|
position += 2;
|
||
|
|
continue;
|
||
|
|
}
|
||
|
|
// \ before newline is a hard break
|
||
|
|
if (nextChar === '\n') {
|
||
|
|
flushText();
|
||
|
|
tokens.push({ role: 'break', value: '<br>' });
|
||
|
|
position += 2;
|
||
|
|
continue;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
// Hard line break: two+ trailing spaces before newline
|
||
|
|
if (source[position] === ' ') {
|
||
|
|
const spaceMatch = remaining.match(/^(?<spaces> {2,})\n/);
|
||
|
|
if (spaceMatch?.groups) {
|
||
|
|
flushText();
|
||
|
|
tokens.push({ role: 'break', value: '<br>' });
|
||
|
|
position += spaceMatch[0].length;
|
||
|
|
continue;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
// HTML entity resolution: &name; or &#digits; or &#xhex;
|
||
|
|
if (source[position] === '&') {
|
||
|
|
const resolved = this.resolveEntity(remaining);
|
||
|
|
if (resolved) {
|
||
|
|
textBuffer += resolved.character;
|
||
|
|
position += resolved.length;
|
||
|
|
continue;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
// Code span: `content` — not parsed for further inline markup
|
||
|
|
if (this.codeSpansEnabled && source[position] === '`') {
|
||
|
|
const codeSpan = this.matchCodeSpan(source, position);
|
||
|
|
if (codeSpan) {
|
||
|
|
flushText();
|
||
|
|
tokens.push({
|
||
|
|
role: 'code',
|
||
|
|
value: codeSpan.raw,
|
||
|
|
content: codeSpan.content,
|
||
|
|
});
|
||
|
|
position += codeSpan.raw.length;
|
||
|
|
continue;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
// Link: [text](url) or [text](url "title")
|
||
|
|
if (source[position] === '[') {
|
||
|
|
const link = this.matchLink(source, position);
|
||
|
|
if (link) {
|
||
|
|
flushText();
|
||
|
|
tokens.push({
|
||
|
|
role: 'link',
|
||
|
|
value: link.text,
|
||
|
|
href: link.href,
|
||
|
|
title: link.title,
|
||
|
|
});
|
||
|
|
position += link.length;
|
||
|
|
continue;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
// Autolink: <url>
|
||
|
|
if (source[position] === '<') {
|
||
|
|
const autolink = this.matchAutolink(remaining);
|
||
|
|
if (autolink) {
|
||
|
|
flushText();
|
||
|
|
tokens.push({
|
||
|
|
role: 'autolink',
|
||
|
|
value: autolink.url,
|
||
|
|
href: autolink.url,
|
||
|
|
});
|
||
|
|
position += autolink.length;
|
||
|
|
continue;
|
||
|
|
}
|
||
|
|
// HTML tag passthrough
|
||
|
|
const htmlTagMatch = this.matchHtmlTag(remaining);
|
||
|
|
if (htmlTagMatch) {
|
||
|
|
flushText();
|
||
|
|
tokens.push({
|
||
|
|
role: 'html',
|
||
|
|
value: htmlTagMatch.tag,
|
||
|
|
});
|
||
|
|
position += htmlTagMatch.length;
|
||
|
|
continue;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
// Bare URL autolink: https://...
|
||
|
|
if (remaining.startsWith('http://') || remaining.startsWith('https://')) {
|
||
|
|
const bareUrl = this.matchBareUrl(remaining);
|
||
|
|
if (bareUrl) {
|
||
|
|
flushText();
|
||
|
|
tokens.push({
|
||
|
|
role: 'autolink',
|
||
|
|
value: bareUrl.url,
|
||
|
|
href: bareUrl.url,
|
||
|
|
});
|
||
|
|
position += bareUrl.length;
|
||
|
|
continue;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
// Delimiter: check each registered delimiter
|
||
|
|
const delimiterMatch = this.matchDelimiter(source, position);
|
||
|
|
if (delimiterMatch) {
|
||
|
|
flushText();
|
||
|
|
tokens.push(delimiterMatch.token);
|
||
|
|
position += delimiterMatch.length;
|
||
|
|
continue;
|
||
|
|
}
|
||
|
|
|
||
|
|
// Plain character
|
||
|
|
textBuffer += source[position];
|
||
|
|
position++;
|
||
|
|
}
|
||
|
|
|
||
|
|
flushText();
|
||
|
|
return tokens;
|
||
|
|
}
|
||
|
|
|
||
|
|
/**
|
||
|
|
* Try to resolve an HTML entity at the start of the string.
|
||
|
|
* Returns the resolved character and the length consumed, or null.
|
||
|
|
*/
|
||
|
|
private resolveEntity(text: string): { character: string; length: number } | null {
|
||
|
|
const namedPattern = /^&(?<name>[a-zA-Z]+);/;
|
||
|
|
const numericPattern = /^&#(?<code>\d+);/;
|
||
|
|
const hexPattern = /^&#x(?<hex>[0-9a-fA-F]+);/;
|
||
|
|
|
||
|
|
const named = text.match(namedPattern);
|
||
|
|
if (named?.groups) {
|
||
|
|
const resolved = NAMED_ENTITIES[named.groups.name.toLowerCase()];
|
||
|
|
if (resolved) {
|
||
|
|
return {
|
||
|
|
character: resolved,
|
||
|
|
length: named[0].length,
|
||
|
|
};
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
const numeric = text.match(numericPattern);
|
||
|
|
if (numeric?.groups) {
|
||
|
|
return {
|
||
|
|
character: String.fromCharCode(parseInt(numeric.groups.code, 10)),
|
||
|
|
length: numeric[0].length,
|
||
|
|
};
|
||
|
|
}
|
||
|
|
|
||
|
|
const hex = text.match(hexPattern);
|
||
|
|
if (hex?.groups) {
|
||
|
|
return {
|
||
|
|
character: String.fromCharCode(parseInt(hex.groups.hex, 16)),
|
||
|
|
length: hex[0].length,
|
||
|
|
};
|
||
|
|
}
|
||
|
|
|
||
|
|
return null;
|
||
|
|
}
|
||
|
|
|
||
|
|
/**
|
||
|
|
* Match a code span starting at the given position.
|
||
|
|
* Handles single backtick delimiters only (not multi-backtick).
|
||
|
|
*/
|
||
|
|
private matchCodeSpan(
|
||
|
|
source: string,
|
||
|
|
position: number,
|
||
|
|
): { content: string; raw: string } | null {
|
||
|
|
if (source[position] !== '`') {
|
||
|
|
return null;
|
||
|
|
}
|
||
|
|
const closeIndex = source.indexOf('`', position + 1);
|
||
|
|
if (closeIndex === -1) {
|
||
|
|
return null;
|
||
|
|
}
|
||
|
|
const content = source.slice(position + 1, closeIndex);
|
||
|
|
return {
|
||
|
|
content,
|
||
|
|
raw: source.slice(position, closeIndex + 1),
|
||
|
|
};
|
||
|
|
}
|
||
|
|
|
||
|
|
/**
|
||
|
|
* Match a markdown link [text](url) or [text](url "title")
|
||
|
|
* starting at the given position. Disallows [ in link text
|
||
|
|
* to prevent nested link ambiguity.
|
||
|
|
*/
|
||
|
|
private matchLink(
|
||
|
|
source: string,
|
||
|
|
position: number,
|
||
|
|
): { text: string; href: string; title?: string; length: number } | null {
|
||
|
|
const linkPattern = /^\[(?<text>[^\[\]]+)\]\((?<href>[^\s)]+)(?:\s+"(?<title>[^"]*)")?\)/;
|
||
|
|
const match = source.slice(position).match(linkPattern);
|
||
|
|
if (!match?.groups) {
|
||
|
|
return null;
|
||
|
|
}
|
||
|
|
return {
|
||
|
|
text: match.groups.text,
|
||
|
|
href: match.groups.href,
|
||
|
|
title: match.groups.title,
|
||
|
|
length: match[0].length,
|
||
|
|
};
|
||
|
|
}
|
||
|
|
|
||
|
|
/**
|
||
|
|
* Match an angle-bracket autolink <url> at the start of the string.
|
||
|
|
*/
|
||
|
|
private matchAutolink(text: string): { url: string; length: number } | null {
|
||
|
|
const pattern = /^<(?<url>https?:\/\/[^\s>]+)>/;
|
||
|
|
const match = text.match(pattern);
|
||
|
|
if (!match?.groups) {
|
||
|
|
return null;
|
||
|
|
}
|
||
|
|
return {
|
||
|
|
url: match.groups.url,
|
||
|
|
length: match[0].length,
|
||
|
|
};
|
||
|
|
}
|
||
|
|
|
||
|
|
/**
|
||
|
|
* Match a bare URL (https://...) at the start of the string.
|
||
|
|
*/
|
||
|
|
private matchBareUrl(text: string): { url: string; length: number } | null {
|
||
|
|
const pattern = /^https?:\/\/[^\s<>\x00]+/;
|
||
|
|
const match = text.match(pattern);
|
||
|
|
if (!match) {
|
||
|
|
return null;
|
||
|
|
}
|
||
|
|
return {
|
||
|
|
url: match[0],
|
||
|
|
length: match[0].length,
|
||
|
|
};
|
||
|
|
}
|
||
|
|
|
||
|
|
/**
|
||
|
|
* Match an HTML tag at the start of the string.
|
||
|
|
*/
|
||
|
|
private matchHtmlTag(text: string): { tag: string; length: number } | null {
|
||
|
|
const pattern = /^<\/?[a-zA-Z][a-zA-Z0-9]*(?:\s+[^>]*)?\s*\/?>/;
|
||
|
|
const match = text.match(pattern);
|
||
|
|
if (!match) {
|
||
|
|
return null;
|
||
|
|
}
|
||
|
|
return {
|
||
|
|
tag: match[0],
|
||
|
|
length: match[0].length,
|
||
|
|
};
|
||
|
|
}
|
||
|
|
|
||
|
|
/**
|
||
|
|
* Try to match a delimiter at the given position. For runs of the
|
||
|
|
* same character (e.g. *** = 3 asterisks), the run is split into
|
||
|
|
* the longest registered delimiter that fits, then the remainder.
|
||
|
|
* This handles cases like **bold***italic* where *** must split
|
||
|
|
* into ** (close bold) + * (open italic).
|
||
|
|
*/
|
||
|
|
private matchDelimiter(
|
||
|
|
source: string,
|
||
|
|
position: number,
|
||
|
|
): { token: InlineToken; length: number } | null {
|
||
|
|
// Count the full run of the same character
|
||
|
|
const runChar = source[position];
|
||
|
|
let runLength = 0;
|
||
|
|
while (position + runLength < source.length && source[position + runLength] === runChar) {
|
||
|
|
runLength++;
|
||
|
|
}
|
||
|
|
if (runLength === 0) {
|
||
|
|
return null;
|
||
|
|
}
|
||
|
|
|
||
|
|
// Find registered delimiters that use this character
|
||
|
|
const candidates = this.delimiters.filter(
|
||
|
|
definition => definition.delimiter[0] === runChar
|
||
|
|
);
|
||
|
|
if (candidates.length === 0) {
|
||
|
|
return null;
|
||
|
|
}
|
||
|
|
|
||
|
|
// Try each candidate delimiter length (longest first, already sorted)
|
||
|
|
for (const definition of candidates) {
|
||
|
|
const delimiter = definition.delimiter;
|
||
|
|
if (delimiter.length > runLength) {
|
||
|
|
continue;
|
||
|
|
}
|
||
|
|
|
||
|
|
const charBefore = position > 0 ? source[position - 1] : '\n';
|
||
|
|
const charAfter = source[position + delimiter.length];
|
||
|
|
|
||
|
|
const leftFlanking = (charBefore === undefined || PUNCTUATION.has(charBefore) || charBefore === '\n')
|
||
|
|
&& charAfter !== undefined && charAfter !== ' ' && charAfter !== '\n' && charAfter !== '\t';
|
||
|
|
|
||
|
|
const rightFlanking = charBefore !== undefined && charBefore !== ' ' && charBefore !== '\n' && charBefore !== '\t'
|
||
|
|
&& (charAfter === undefined || PUNCTUATION.has(charAfter) || charAfter === '\n');
|
||
|
|
|
||
|
|
if (leftFlanking) {
|
||
|
|
return {
|
||
|
|
token: {
|
||
|
|
role: 'open',
|
||
|
|
value: delimiter,
|
||
|
|
delimiter,
|
||
|
|
},
|
||
|
|
length: delimiter.length,
|
||
|
|
};
|
||
|
|
}
|
||
|
|
if (rightFlanking) {
|
||
|
|
return {
|
||
|
|
token: {
|
||
|
|
role: 'close',
|
||
|
|
value: delimiter,
|
||
|
|
delimiter,
|
||
|
|
},
|
||
|
|
length: delimiter.length,
|
||
|
|
};
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return null;
|
||
|
|
}
|
||
|
|
}
|