evilchili/ribbit
1
0
Fork 0
Code Issues 5 Pull Requests Actions Packages Projects Releases Wiki Activity

Reimplement as a tokenizer with GFM parity

Browse Source
This commit is contained in:
gsb
2026-04-30 07:09:19 +00:00
parent 005db2f431
commit d41716c8b2
28 changed files with 4798 additions and 1398 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/integration/test_fuzz.js
+29 -18
View File
@@ -34,8 +34,8 @@ function mulberry32(seed) {
/* ── Keystroke generation ── */
const PRINTABLE = 'abcdefghijklmnopqrstuvwxyz 0123456789.,!?';
const DELIMITERS = ['*', '**', '***', '`', '~~'];
const BLOCK_PREFIXES = ['# ', '## ', '### ', '- ', '1. ', '> ', '---'];
const DELIMITERS = ['*', '**', '***', '`', '~~', '_', '__', '___'];
const BLOCK_PREFIXES = ['# ', '## ', '### ', '- ', '+ ', '1. ', '> ', '---', '~~~'];
const SPECIAL_KEYS = [
{ name: 'Enter', keys: Key.ENTER, isSpecial: true },
{ name: 'Backspace', keys: Key.BACK_SPACE, isSpecial: true },
@@ -70,8 +70,14 @@ function generateSequence(random, length) {
} else if (roll < 0.94) {
/* repeated delimiter (stress test) */
const count = 2 + Math.floor(random() * 4);
const character = '*';
const delimiters = ['*', '_', '~'];
const character = delimiters[Math.floor(random() * delimiters.length)];
sequence.push({ name: character.repeat(count), keys: character.repeat(count) });
} else if (roll < 0.97) {
/* backslash sequences */
const escaped = ['\\*', '\\_', '\\`', '\\~', '\\\\', '\\'];
const fragment = escaped[Math.floor(random() * escaped.length)];
sequence.push({ name: fragment, keys: fragment });
} else {
/* angle bracket / HTML-like content */
const fragments = ['<', '>', '<div>', '</div>', '<b>', '&amp;'];
@@ -188,7 +194,8 @@ async function checkInvariants() {
var forbiddenRules = {
'STRONG': ['STRONG','B'], 'B': ['STRONG','B'],
'EM': ['EM','I'], 'I': ['EM','I'],
'CODE': ['CODE','STRONG','B','EM','I','A'],
'CODE': ['CODE','STRONG','B','EM','I','A','DEL'],
'DEL': ['DEL','S','STRIKE'], 'S': ['DEL','S','STRIKE'], 'STRIKE': ['DEL','S','STRIKE'],
'A': ['A'],
};
var allElements = editor.querySelectorAll('*');
@@ -212,28 +219,32 @@ async function checkInvariants() {
}
/* Invariant 6: rendered HTML is stable through markdown round-trip.
md → toHTML → toMarkdown → toHTML must produce the same HTML.
The markdown representation may change (e.g. ***** → ***) but
the rendered output must be identical.
md → toHTML → toMarkdown → toHTML must eventually stabilize.
The first round-trip may change the HTML (e.g. literal <strong>
in text becomes a real element via HTML passthrough, then
serializes as **). But the second round-trip must be stable.
Skip if there are speculative elements (in-progress editing). */
var hasSpeculative = editor.querySelector('[data-speculative]');
if (!hasSpeculative) {
try {
var md = window.__ribbitEditor.getMarkdown();
var converter = window.__ribbitEditor.converter;
// Two round-trips: allow the first to normalize, check
// that the second produces identical HTML
var html1 = converter.toHTML(md);
var md2 = converter.toMarkdown(html1);
var html2 = converter.toHTML(md2);
/* Compare the rendered HTML, not the markdown */
var div1 = document.createElement('div');
div1.innerHTML = html1;
var div2 = document.createElement('div');
div2.innerHTML = html2;
var text1 = div1.textContent.replace(/\s+/g, ' ').trim();
var text2 = div2.textContent.replace(/\s+/g, ' ').trim();
if (text1 !== text2) {
return 'Round-trip HTML mismatch:\n html1: "' + text1.slice(0, 80) +
'"\n html2: "' + text2.slice(0, 80) + '"';
var md3 = converter.toMarkdown(html2);
var html3 = converter.toHTML(md3);
var normalize = function(html) {
return html
.replace(/\s*id='[^']*'/g, '')
.replace(/\s+/g, ' ')
.trim();
};
if (normalize(html2) !== normalize(html3)) {
return 'Round-trip HTML not stable after 2 passes:\n pass2: "' + normalize(html2).slice(0, 80) +
'"\n pass3: "' + normalize(html3).slice(0, 80) + '"';
}
} catch (err) {
return 'Round-trip check threw: ' + err.message;
@@ -241,7 +252,7 @@ async function checkInvariants() {
}
/* Invariant 7: only valid inline elements inside block content */
var validInline = ['STRONG','B','EM','I','CODE','A','BR'];
var validInline = ['STRONG','B','EM','I','CODE','A','BR','DEL','S','STRIKE'];
var blocks = editor.querySelectorAll('p,h1,h2,h3,h4,h5,h6,li,blockquote,td,th');
for (var b = 0; b < blocks.length; b++) {
var inlineEls = blocks[b].querySelectorAll('*');