markdown 0.0.1-preview.1
CommonMark and GFM Markdown parser, AST, plugin system, and HTML renderer
Markdown (@zena-lang/markdown)#
A CommonMark and GitHub Flavored Markdown (GFM) parser, syntax tree, plugin
system, and HTML renderer written in Zena (zena-lang:markdown/<library>).
See docs/design/markdown.md for the design.
Usage#
Add it with zena pkg add zena-lang:markdown, then import its main
library:
import {
parse, renderHtml, markdownToHtml, gfmExtensions
} from 'zena-lang:markdown';
// Render CommonMark + GFM to safe HTML (raw HTML stripped, unsafe URLs blocked)
let html = markdownToHtml('# Hello\n\nSome ~~old~~ **new** text.', {
extensions: gfmExtensions(),
safe: true,
});
// Or parse to an AST first
let doc = parse('# Heading\n\nParagraph.', gfmExtensions());
let out = renderHtml(doc, {safe: true, extensions: gfmExtensions()});
Running Tests#
npm test -w @zena-lang/markdown
npm run test:quick -- packages/markdown/test
35.2 KiB
Raw
/**
* CommonMark Phase 2 inline parser with linked delimiter stack, bracket stack
* for links and images, and `DelimiterProcessor` / `InlineContentParser`
* plugin dispatch.
*/
import {FixedArray, GrowableArray} from 'zena:core';
import {StringBuilder, StringReader, isAsciiWhitespace, isAsciiPunctuation, isAsciiAlpha, isAsciiAlphanumeric, isUnicodeWhitespace, isUnicodePunctuation} from 'zena:string';
import {HashMap} from 'zena:collections';
import {Inline, LinkDef, Span, Text, CodeSpan, Emphasis, Strong, Link, Image, Autolink, HtmlInline, SoftBreak, HardBreak} from './ast.zena';
import {DelimiterProcessor, DelimiterRunInfo, InlineContentParser, InlineMatch, InlineParserRunner, MarkdownExtension} from './plugin.zena';
import {skipSpacesAndOneLineEnd, normalizeLinkLabel, tryDecodeEntity, scanLinkDestination, scanLinkTitle, scanLinkLabel} from './scanner.zena';
/**
* Internal doubly-linked list node holding an `Inline` during Phase 2 parsing
* so delimiter and link resolution can splice ranges in O(1) without
* invalidating references held by outer delimiters or brackets.
*/
final class InlineEntry {
var node: Inline;
var prev: InlineEntry | null = null;
var next: InlineEntry | null = null;
/** How deep `node` nests other inlines: 0 when it holds none. */
var depth = 0;
new(this.node) {}
}
/**
* How deep emphasis, links and images may nest. A run of delimiters or
* brackets that would nest deeper stays text: every walk over the tree
* recurses through the inlines, and a deeper one would overflow the stack.
*/
let MAX_INLINE_NESTING = 100;
/** The deepest of the entries from `first` up to `end`, not including it. */
function deepestBetween(first: InlineEntry | null, end: InlineEntry | null) {
var deepest = 0;
var curr = first;
while (curr != null && curr != end) {
if (curr.depth > deepest) {
deepest = curr.depth;
}
curr = curr.next;
}
return deepest;
}
/**
* An active run of delimiter characters (`*`, `_`, or a custom
* `DelimiterProcessor` character) on the delimiter stack.
*/
final class Delimiter {
entry: InlineEntry;
delimiterByte: i32;
var length: i32;
originalLength: i32;
canOpen: boolean;
canClose: boolean;
startOffset: i32;
var prev: Delimiter | null = null;
var next: Delimiter | null = null;
new(this.entry, this.delimiterByte, this.length, this.originalLength, this.canOpen, this.canClose, this.startOffset) {}
}
/**
* An open `[` or `![` bracket on the bracket stack waiting for a matching `]`.
*/
final class Bracket {
entry: InlineEntry;
index: i32;
startOffset: i32;
isImage: boolean;
previousDelimiter: Delimiter | null;
prev: Bracket | null;
var active: boolean = true;
var bracketAfter: boolean = false;
new(this.entry, this.index, this.startOffset, this.isImage, this.previousDelimiter, this.prev) {}
}
/**
* CommonMark Phase 2 inline parser.
*/
export final class InlineParser implements InlineParserRunner {
links: HashMap<String, LinkDef>;
#delimiterProcessors = new HashMap<i32, DelimiterProcessor>();
#inlineParsersByByte = new FixedArray<GrowableArray<InlineContentParser> | null>(
128,
null
);
#specialBytes = new FixedArray<boolean>(128, false);
var #firstEntry: InlineEntry | null = null;
var #lastEntry: InlineEntry | null = null;
var #lastDelimiter: Delimiter | null = null;
var #lastBracket: Bracket | null = null;
new(links: HashMap<String, LinkDef>, extensions: GrowableArray<MarkdownExtension>) : links = links {
// Mark built-in special ASCII bytes:
// \n (10), \r (13), ! (33), & (38), * (42), < (60), [ (91), \ (92), ] (93), _ (95), ` (96)
this.#specialBytes[0x0A] = true;
this.#specialBytes[0x0D] = true;
this.#specialBytes[0x21] = true;
this.#specialBytes[0x26] = true;
this.#specialBytes[0x2A] = true;
this.#specialBytes[0x3C] = true;
this.#specialBytes[0x5B] = true;
this.#specialBytes[0x5C] = true;
this.#specialBytes[0x5D] = true;
this.#specialBytes[0x5F] = true;
this.#specialBytes[0x60] = true;
for (let ext in extensions) {
for (let dp in ext.delimiterProcessors) {
let openByte = dp.openingByte();
let closeByte = dp.closingByte();
if (openByte >= 0 && openByte < 128) {
this.#delimiterProcessors[openByte] = dp;
this.#specialBytes[openByte] = true;
}
if (closeByte >= 0 && closeByte < 128) {
this.#delimiterProcessors[closeByte] = dp;
this.#specialBytes[closeByte] = true;
}
}
for (let ip in ext.inlineParsers) {
let triggers = ip.triggerBytes();
for (let b in triggers) {
if (b >= 0 && b < 128) {
var bucket = this.#inlineParsersByByte[b];
if (bucket == null) {
bucket = new GrowableArray<InlineContentParser>(2);
this.#inlineParsersByByte[b] = bucket;
}
bucket.push(ip);
this.#specialBytes[b] = true;
}
}
}
}
}
/**
* Parses `text` (located at `baseOffset` in the document) into a list of
* `Inline` AST nodes.
*/
parse(text: String, baseOffset: i32): GrowableArray<Inline> {
this.#firstEntry = null;
this.#lastEntry = null;
this.#lastDelimiter = null;
this.#lastBracket = null;
let r = new StringReader(text);
while (!r.isAtEnd) {
let i = r.mark();
let b = r.peek();
// 1. Check registered InlineContentParser plugins triggered by `b`
var pluginMatched = false;
if (b >= 0 && b < 128) {
let bucket = this.#inlineParsersByByte[b];
if (bucket != null) {
let prevCp = r.peekBefore();
for (let parser in bucket) {
let matchResult = parser.tryParse(
text,
baseOffset,
i,
prevCp,
this
);
if (matchResult != null && matchResult.nextIndex > i) {
this.#appendNode(matchResult.node);
r.reset(matchResult.nextIndex);
pluginMatched = true;
break;
}
}
}
}
if (pluginMatched) {
continue;
}
// 2. Built-in CommonMark inline constructs and registered DelimiterProcessors
if (b == 0x0A || b == 0x0D) {
// \n or \r
this.#parseNewline(r, baseOffset);
} else if (b == 0x5C) {
// \
this.#parseBackslash(r, baseOffset);
} else if (b == 0x60) {
// `
this.#parseBackticks(r, baseOffset);
} else if (b == 0x2A
|| b == 0x5F
|| b < 128 && this.#delimiterProcessors.has(b)) {
// *, _, or custom delimiter
this.#parseDelimiterRun(r, baseOffset, b);
} else if (b == 0x5B) {
// [
r.advance();
let startOff = baseOffset + i;
let entry = this.#appendNode(
new Text('[', new Span(startOff, baseOffset + r.mark()))
);
this.#addBracket(entry, i, startOff, false);
} else if (b == 0x21) {
// !
r.advance();
let bracketPos = r.mark();
if (r.matchCodePoint(0x5B)) {
// ![
let startOff = baseOffset + i;
let entry = this.#appendNode(
new Text('![', new Span(startOff, baseOffset + r.mark()))
);
this.#addBracket(entry, bracketPos, startOff, true);
} else {
let startOff = baseOffset + i;
this.#appendNode(
new Text('!', new Span(startOff, baseOffset + bracketPos))
);
}
} else if (b == 0x5D) {
// ]
this.#parseCloseBracket(r, baseOffset);
} else if (b == 0x3C) {
// <
if (!this.#parseAutolinkOrHtml(r, baseOffset)) {
r.advance();
let startOff = baseOffset + i;
this.#appendNode(
new Text('<', new Span(startOff, baseOffset + r.mark()))
);
}
} else if (b == 0x26) {
// &
let sb = new StringBuilder(8);
if (tryDecodeEntity(r, sb)) {
this.#appendNode(
new Text(
sb.toString(),
new Span(baseOffset + i, baseOffset + r.mark())
)
);
} else {
r.advance();
let startOff = baseOffset + i;
this.#appendNode(
new Text('&', new Span(startOff, baseOffset + r.mark()))
);
}
} else {
// Consume a run of plain text code points until the next special ASCII character
r.advance();
while (!r.isAtEnd) {
let nb = r.peek();
if (nb >= 0 && nb < 128 && this.#specialBytes[nb]) {
break;
}
r.advance();
}
let end = r.mark();
this.#appendNode(
new Text(
r.sliceRange(i, end),
new Span(baseOffset + i, baseOffset + end)
)
);
}
}
this.#processDelimiters(null);
return this.#collectAndMergeText();
}
#appendNode(node: Inline): InlineEntry {
let entry = new InlineEntry(node);
let last = this.#lastEntry;
if (last == null) {
this.#firstEntry = entry;
this.#lastEntry = entry;
} else {
last.next = entry;
entry.prev = last;
this.#lastEntry = entry;
}
return entry;
}
#unlinkEntry(entry: InlineEntry): void {
let prev = entry.prev;
let next = entry.next;
if (prev != null) {
prev.next = next;
} else {
this.#firstEntry = next;
}
if (next != null) {
next.prev = prev;
} else {
this.#lastEntry = prev;
}
entry.prev = null;
entry.next = null;
}
#insertEntryBefore(target: InlineEntry, newEntry: InlineEntry): void {
let prev = target.prev;
newEntry.prev = prev;
newEntry.next = target;
target.prev = newEntry;
if (prev != null) {
prev.next = newEntry;
} else {
this.#firstEntry = newEntry;
}
}
#parseNewline(r: StringReader, baseOffset: i32): void {
let index = r.mark();
if (r.matchCodePoint(0x0D)) {
r.matchCodePoint(0x0A);
} else {
r.advance();
}
let afterNewline = r.mark();
// Check trailing spaces in the previous Text node
var trailingSpaces = 0;
let last = this.#lastEntry;
if (last != null) {
if (let Text {literal} = last.node) {
let lr = new StringReader(literal);
var end = lr.remaining;
while (end > 0) {
lr.reset(end);
if (lr.peekBefore() == 0x20) {
trailingSpaces += 1;
end -= 1;
} else {
break;
}
}
if (trailingSpaces > 0) {
if (end == 0) {
this.#unlinkEntry(last);
} else {
last.node = new Text(
lr.sliceRange(0, end),
new Span(last.node.span.start, last.node.span.start + end)
);
}
}
}
}
let span = new Span(baseOffset + index, baseOffset + afterNewline);
if (trailingSpaces >= 2) {
this.#appendNode(new HardBreak(span));
} else {
this.#appendNode(new SoftBreak(span));
}
// Skip leading spaces on the next line
while (r.matchCodePoint(0x20)) {}
}
#parseBackslash(r: StringReader, baseOffset: i32): void {
let index = r.mark();
r.advance(); // consume '\'
if (!r.isAtEnd) {
let next = r.peek();
if (next == 0x0A || next == 0x0D) {
if (r.matchCodePoint(0x0D)) {
r.matchCodePoint(0x0A);
} else {
r.advance();
}
this.#appendNode(
new HardBreak(new Span(baseOffset + index, baseOffset + r.mark()))
);
while (r.matchCodePoint(0x20)) {}
return;
}
if (isAsciiPunctuation(next)) {
let charStart = r.mark();
r.advance();
let charEnd = r.mark();
this.#appendNode(
new Text(
r.sliceRange(charStart, charEnd),
new Span(baseOffset + index, baseOffset + charEnd)
)
);
return;
}
}
this.#appendNode(
new Text('\\', new Span(baseOffset + index, baseOffset + r.mark()))
);
}
#parseBackticks(r: StringReader, baseOffset: i32): void {
let index = r.mark();
var tickCount = 0;
while (r.matchCodePoint(0x60)) {
tickCount += 1;
}
let openEnd = r.mark();
while (!r.isAtEnd) {
if (r.peek() == 0x60) {
let closeStart = r.mark();
var closeCount = 0;
while (r.matchCodePoint(0x60)) {
closeCount += 1;
}
if (closeCount == tickCount) {
let closeEnd = r.mark();
// Normalize code span interior: convert line endings to spaces
let raw = r.sliceRange(openEnd, closeStart);
let rawReader = new StringReader(raw);
let sb = new StringBuilder();
var hasNonSpace = false;
while (!rawReader.isAtEnd) {
let cp = rawReader.advance();
if (cp == 0x0D) {
sb.appendByte(0x20);
rawReader.matchCodePoint(0x0A);
} else if (cp == 0x0A) {
sb.appendByte(0x20);
} else {
if (cp != 0x20) {
hasNonSpace = true;
}
sb.appendCodePoint(cp);
}
}
var normalized = sb.toString();
if (hasNonSpace
&& normalized.startsWith(' ')
&& normalized.endsWith(' ')) {
let nr = new StringReader(normalized);
let normEnd = nr.remaining;
if (normEnd >= 2) {
normalized = nr.sliceRange(1, normEnd - 1);
}
}
this.#appendNode(
new CodeSpan(
normalized,
new Span(baseOffset + index, baseOffset + closeEnd)
)
);
return;
}
} else {
r.advance();
}
}
// Unmatched backtick run becomes literal text
r.reset(openEnd);
this.#appendNode(
new Text(
r.sliceRange(index, openEnd),
new Span(baseOffset + index, baseOffset + openEnd)
)
);
}
#parseDelimiterRun(r: StringReader, baseOffset: i32, delimByte: i32): void {
let index = r.mark();
let prevCp = r.peekBefore();
let beforeCp = if (prevCp < 0) 0x20 else prevCp;
var count = 0;
while (r.matchCodePoint(delimByte)) {
count += 1;
}
let end = r.mark();
let nextCp = r.peek();
let afterCp = if (nextCp < 0) 0x20 else nextCp;
let beforeIsWhitespace = isUnicodeWhitespace(beforeCp);
let beforeIsPunctuation = isUnicodePunctuation(beforeCp);
let afterIsWhitespace = isUnicodeWhitespace(afterCp);
let afterIsPunctuation = isUnicodePunctuation(afterCp);
let leftFlanking = !afterIsWhitespace
&& (!afterIsPunctuation || beforeIsWhitespace || beforeIsPunctuation);
let rightFlanking = !beforeIsWhitespace
&& (!beforeIsPunctuation || afterIsWhitespace || afterIsPunctuation);
var canOpen = false;
var canClose = false;
if (delimByte == 0x5F) {
// '_'
canOpen = leftFlanking && (!rightFlanking || beforeIsPunctuation);
canClose = rightFlanking && (!leftFlanking || afterIsPunctuation);
} else {
canOpen = leftFlanking;
canClose = rightFlanking;
}
let startOff = baseOffset + index;
let entry = this.#appendNode(
new Text(r.sliceRange(index, end), new Span(startOff, baseOffset + end))
);
if (canOpen || canClose) {
let delim = new Delimiter(
entry,
delimByte,
count,
count,
canOpen,
canClose,
startOff
);
let prev = this.#lastDelimiter;
delim.prev = prev;
if (prev != null) {
prev.next = delim;
}
this.#lastDelimiter = delim;
}
}
#addBracket(entry: InlineEntry, index: i32, startOffset: i32, isImage: boolean): void {
let prev = this.#lastBracket;
if (prev != null) {
prev.bracketAfter = true;
}
this.#lastBracket = new Bracket(
entry,
index,
startOffset,
isImage,
this.#lastDelimiter,
prev
);
}
#removeLastBracket(): void {
let b = this.#lastBracket;
if (b != null) {
this.#lastBracket = b.prev;
}
}
#parseCloseBracket(r: StringReader, baseOffset: i32): void {
let index = r.mark();
r.advance(); // consume ']'
let afterClose = r.mark();
let opener = this.#lastBracket;
if (opener == null) {
this.#appendNode(
new Text(']', new Span(baseOffset + index, baseOffset + afterClose))
);
return;
}
if (!opener.active) {
this.#removeLastBracket();
this.#appendNode(
new Text(']', new Span(baseOffset + index, baseOffset + afterClose))
);
return;
}
var dest = '';
var title = '';
var nextPos = afterClose;
var matched = false;
// 1. Inline link/image: `[text](dest "title")`
if (r.matchCodePoint(0x28)) {
// '('
skipSpacesAndOneLineEnd(r);
if (r.matchCodePoint(0x29)) {
// empty `()`
matched = true;
nextPos = r.mark();
} else {
let (destOk, parsedDest) = scanLinkDestination(r);
if (destOk) {
let afterDest = r.mark();
skipSpacesAndOneLineEnd(r);
let p = r.mark();
if (r.matchCodePoint(0x29)) {
dest = parsedDest;
matched = true;
nextPos = r.mark();
} else if (p > afterDest) {
let (titleOk, parsedTitle) = scanLinkTitle(r);
if (titleOk) {
skipSpacesAndOneLineEnd(r);
if (r.matchCodePoint(0x29)) {
dest = parsedDest;
title = parsedTitle;
matched = true;
nextPos = r.mark();
}
}
}
}
}
if (!matched) {
r.reset(afterClose);
}
}
// 2. Reference link: full `[text][ref]`, collapsed `[text][]`, or shortcut `[text]`
if (!matched) {
var refLabel: String | null = null;
var refEnd = afterClose;
if (r.peek() == 0x5B) {
// '['
if (r.matchString('[]')) {
// Collapsed reference `[text][]`
if (!opener.bracketAfter && index - (opener.index + 1) <= 999) {
refLabel = r.sliceRange(opener.index + 1, index);
refEnd = r.mark();
}
} else {
let (labelOk, rawLabel) = scanLinkLabel(r);
if (labelOk) {
refLabel = rawLabel;
refEnd = r.mark();
}
}
} else if (!opener.bracketAfter && index - (opener.index + 1) <= 999) {
// Shortcut reference `[text]`
refLabel = r.sliceRange(opener.index + 1, index);
}
if (refLabel != null) {
let normalized = normalizeLinkLabel(refLabel);
if (!normalized.isEmpty && this.links.has(normalized)) {
let def = this.links[normalized];
dest = def.destination;
title = def.title;
matched = true;
nextPos = refEnd;
}
}
}
if (!matched) {
r.reset(afterClose);
this.#removeLastBracket();
this.#appendNode(
new Text(']', new Span(baseOffset + index, baseOffset + afterClose))
);
return;
}
// Resolve delimiters strictly inside the bracketed text
this.#processDelimiters(opener.previousDelimiter);
// Text inside it too deep to wrap again, as a bracket that matches
// nothing is.
let deepest = deepestBetween(opener.entry.next, null);
if (deepest >= MAX_INLINE_NESTING) {
r.reset(afterClose);
this.#removeLastBracket();
this.#appendNode(
new Text(']', new Span(baseOffset + index, baseOffset + afterClose))
);
return;
}
r.reset(nextPos);
// Collect children after opener.entry
let children = new GrowableArray<Inline>(4);
var curr = opener.entry.next;
while (curr != null) {
let nextEntry = curr.next;
children.push(curr.node);
this.#unlinkEntry(curr);
curr = nextEntry;
}
let span = new Span(opener.startOffset, baseOffset + nextPos);
opener.entry.depth = deepest + 1;
if (opener.isImage) {
opener.entry.node = new Image(dest, title, children, span);
} else {
opener.entry.node = new Link(dest, title, children, span);
// Deactivate earlier `[` link openers so links cannot contain links
var b = opener.prev;
while (b != null) {
if (!b.isImage) {
b.active = false;
}
b = b.prev;
}
}
this.#removeLastBracket();
}
#parseAutolinkOrHtml(r: StringReader, baseOffset: i32): boolean {
let index = r.mark();
if (!r.matchCodePoint(0x3C)) {
// '<'
return false;
}
let contentStart = r.mark();
if (r.isAtEnd) {
r.reset(index);
return false;
}
// 1. Check for `<...>`: URI autolink or email autolink
var hasSpaceOrAngle = false;
var colonPos = -1;
var atPos = -1;
while (!r.isAtEnd && r.peek() != 0x3E) {
let pos = r.mark();
let cp = r.advance();
if (cp <= 0x20 || cp == 0x3C) {
hasSpaceOrAngle = true;
break;
}
if (cp == 0x3A && colonPos < 0) {
colonPos = pos;
}
if (cp == 0x40 && atPos < 0) {
atPos = pos;
}
}
if (!hasSpaceOrAngle && r.peek() == 0x3E) {
let closeAngle = r.mark();
r.advance(); // consume '>'
let afterAngle = r.mark();
// Check URI autolink: scheme of 2..32 chars starting with letter followed by ':'
if (colonPos >= contentStart + 2 && colonPos - contentStart <= 32) {
r.reset(contentStart);
var validScheme = isAsciiAlpha(r.advance());
while (validScheme && r.mark() < colonPos) {
let sb = r.advance();
validScheme = isAsciiAlphanumeric(sb)
|| sb == 0x2B
|| sb == 0x2D
|| sb == 0x2E;
}
if (validScheme) {
r.reset(afterAngle);
let uri = r.sliceRange(contentStart, closeAngle);
this.#appendNode(
new Autolink(
uri,
uri,
false,
new Span(baseOffset + index, baseOffset + afterAngle)
)
);
return true;
}
}
// Check email autolink: local@domain
if (atPos > contentStart && atPos + 1 < closeAngle && colonPos < 0) {
r.reset(contentStart);
var validEmail = true;
while (r.mark() < atPos) {
let eb = r.advance();
let ok = isAsciiAlphanumeric(eb)
|| eb == 0x2E
|| eb == 0x2D
|| eb == 0x5F
|| eb == 0x2B;
if (!ok) {
validEmail = false;
break;
}
}
var hasDot = false;
var labelLen = 0;
r.reset(atPos + 1);
while (validEmail && r.mark() < closeAngle) {
let prev = r.peekBefore();
let db = r.advance();
if (db == 0x2E) {
if (labelLen == 0 || prev == 0x2D) {
validEmail = false;
}
hasDot = true;
labelLen = 0;
} else if (isAsciiAlphanumeric(db) || db == 0x2D && labelLen > 0) {
labelLen += 1;
} else {
validEmail = false;
}
}
r.reset(closeAngle);
if (validEmail && hasDot && labelLen > 0 && r.peekBefore() != 0x2D) {
r.reset(afterAngle);
let email = r.sliceRange(contentStart, closeAngle);
this.#appendNode(
new Autolink(
`mailto:${email}`,
email,
true,
new Span(baseOffset + index, baseOffset + afterAngle)
)
);
return true;
}
}
}
// 2. Check for inline HTML tag, comment, processing instruction, declaration, or CDATA
r.reset(index);
if (scanInlineHtmlTag(r)) {
let htmlEnd = r.mark();
this.#appendNode(
new HtmlInline(
r.sliceRange(index, htmlEnd),
new Span(baseOffset + index, baseOffset + htmlEnd)
)
);
return true;
}
r.reset(index);
return false;
}
#removeDelimiter(delim: Delimiter): void {
let prev = delim.prev;
let next = delim.next;
if (prev != null) {
prev.next = next;
}
if (next != null) {
next.prev = prev;
} else {
this.#lastDelimiter = prev;
}
}
#removeDelimitersBetween(opener: Delimiter, closer: Delimiter): void {
var d = closer.prev;
while (d != null && d != opener) {
let prev = d.prev;
this.#removeDelimiter(d);
d = prev;
}
}
#processDelimiters(stackBottom: Delimiter | null): void {
let openersBottom = new HashMap<i32, Delimiter | null>();
// Find the first closer above `stackBottom`
var closer = this.#lastDelimiter;
while (closer != null && closer.prev != stackBottom) {
closer = closer.prev;
}
while (closer != null) {
let c = closer;
if (!c.canClose) {
closer = c.next;
continue;
}
let delimByte = c.delimiterByte;
let customProc = if (this.#delimiterProcessors.has(
delimByte
)) this.#delimiterProcessors[delimByte] else null;
let minLen = if (customProc != null) customProc.minLength() else 1;
if (c.length < minLen) {
closer = c.next;
continue;
}
let lenKey = if (customProc
!= null) c.length % 16 else c.originalLength % 3;
let bottomKey = delimByte * 32 + (if (c.canOpen) 16 else 0) + lenKey;
let lowerBound = if (openersBottom.has(
bottomKey
)) openersBottom[bottomKey] else stackBottom;
var opener = c.prev;
var matchedOpener: Delimiter | null = null;
var usedDelims = 0;
var wrappedNode: Inline | null = null;
var wrappedDepth = 0;
while (opener != null && opener != stackBottom && opener != lowerBound) {
let o = opener;
if (o.canOpen && o.delimiterByte == delimByte && o.length >= minLen) {
// CommonMark Rule 9/10 interior multiple-of-3 rule for `*` and `_`
if (customProc == null
&& (o.canClose || c.canOpen)
&& c.originalLength % 3 != 0
&& (o.originalLength + c.originalLength) % 3 == 0) {
opener = o.prev;
continue;
}
// What it would wrap is too deep to wrap again, so the closer
// matches nothing, and its delimiters stay text.
let deepest = deepestBetween(o.entry.next, c.entry);
if (deepest >= MAX_INLINE_NESTING) {
break;
}
wrappedDepth = deepest + 1;
// Collect children strictly between `o.entry` and `c.entry`
let children = new GrowableArray<Inline>(4);
var curr = o.entry.next;
while (curr != null && curr != c.entry) {
children.push(curr.node);
curr = curr.next;
}
if (customProc != null) {
let openerInfo = new DelimiterRunInfo(
o.delimiterByte,
o.length,
o.originalLength,
o.canOpen,
o.canClose,
o.startOffset + (o.originalLength - o.length),
o.startOffset + o.originalLength
);
let closerInfo = new DelimiterRunInfo(
c.delimiterByte,
c.length,
c.originalLength,
c.canOpen,
c.canClose,
c.startOffset,
c.startOffset + c.length
);
let (used, customNode) = customProc.process(
openerInfo,
closerInfo,
children
);
if (used > 0 && customNode != null) {
usedDelims = used;
wrappedNode = customNode;
matchedOpener = o;
break;
}
} else {
usedDelims = if (o.length >= 2 && c.length >= 2) 2 else 1;
let spanStart = o.startOffset + (o.length - usedDelims);
let spanEnd = c.startOffset + usedDelims;
let span = new Span(spanStart, spanEnd);
wrappedNode = if (usedDelims
== 2) new Strong(
delimByte,
children,
span
) else new Emphasis(delimByte, children, span);
matchedOpener = o;
break;
}
}
opener = o.prev;
}
if (matchedOpener == null || wrappedNode == null || usedDelims <= 0) {
openersBottom[bottomKey] = c.prev;
if (!c.canOpen) {
this.#removeDelimiter(c);
}
closer = c.next;
continue;
}
let o = matchedOpener;
// Unlink the child entries that are now inside `wrappedNode`
var unlinkCurr = o.entry.next;
while (unlinkCurr != null && unlinkCurr != c.entry) {
let nextUnlink = unlinkCurr.next;
this.#unlinkEntry(unlinkCurr);
unlinkCurr = nextUnlink;
}
// Insert `wrappedNode` before `c.entry`
let wrapped = new InlineEntry(wrappedNode);
wrapped.depth = wrappedDepth;
this.#insertEntryBefore(c.entry, wrapped);
// Remove any delimiters strictly between `o` and `c`
this.#removeDelimitersBetween(o, c);
// Shrink `o` and `c`
o.length -= usedDelims;
c.length -= usedDelims;
if (o.length == 0) {
this.#unlinkEntry(o.entry);
this.#removeDelimiter(o);
} else {
o.entry.node = new Text(
repeatAsciiChar(o.delimiterByte, o.length),
new Span(o.startOffset, o.startOffset + o.length)
);
}
if (c.length == 0) {
let nextCloser = c.next;
this.#unlinkEntry(c.entry);
this.#removeDelimiter(c);
closer = nextCloser;
} else {
c.entry.node = new Text(
repeatAsciiChar(c.delimiterByte, c.length),
new Span(c.startOffset, c.startOffset + c.length)
);
}
}
// Clear remaining delimiters above `stackBottom`
while (this.#lastDelimiter != null && this.#lastDelimiter != stackBottom) {
let lastDelim = this.#lastDelimiter;
if (lastDelim != null) {
this.#removeDelimiter(lastDelim);
}
}
}
#collectAndMergeText(): GrowableArray<Inline> {
let result = new GrowableArray<Inline>(8);
var curr = this.#firstEntry;
while (curr != null) {
let c = curr;
let node = c.node;
var nextEntry = c.next;
if (let Text {literal} = node) {
if (!literal.isEmpty) {
if (nextEntry != null && nextEntry.node is Text) {
let sb = new StringBuilder();
sb.append(literal);
let start = node.span.start;
var end = node.span.end;
while (nextEntry != null) {
let ne = nextEntry;
if (let Text {literal as nextLit} = ne.node) {
sb.append(nextLit);
end = ne.node.span.end;
nextEntry = ne.next;
} else {
break;
}
}
result.push(new Text(sb.toString(), new Span(start, end)));
} else {
result.push(node);
}
}
} else {
result.push(node);
}
curr = nextEntry;
}
return result;
}
}
function repeatAsciiChar(ch: i32, count: i32) {
let sb = new StringBuilder(count);
var i = 0;
while (i < count) {
sb.appendByte(ch);
i += 1;
}
return sb.toString();
}
/**
* Scans a CommonMark inline HTML tag, comment, processing instruction,
* declaration, or CDATA section starting with `<` at the current position of
* `r`. Advances `r` past the closing `>` and returns `true` on match; resets
* `r` and returns `false` otherwise.
*/
function scanInlineHtmlTag(r: StringReader): boolean {
let saved = r.mark();
if (!r.matchCodePoint(0x3C)) {
// '<'
return false;
}
let second = r.peek();
// 1. Comment `<!-- ... -->`, declaration `<!A...>`, or CDATA `<![CDATA[...]]>`
if (second == 0x21) {
// '!'
r.advance();
if (r.matchString('--')) {
if (r.matchCodePoint(0x3E) || r.matchString('->')) {
return true;
}
while (!r.isAtEnd) {
if (r.matchString('-->')) {
return true;
}
r.advance();
}
r.reset(saved);
return false;
}
if (r.matchString('[CDATA[')) {
while (!r.isAtEnd) {
if (r.matchString(']]>')) {
return true;
}
r.advance();
}
r.reset(saved);
return false;
}
if (isAsciiAlpha(r.peek())) {
r.advance();
while (!r.isAtEnd && r.peek() != 0x3E) {
r.advance();
}
if (r.matchCodePoint(0x3E)) {
return true;
}
r.reset(saved);
return false;
}
r.reset(saved);
return false;
}
// 2. Processing instruction `<? ... ?>`
if (second == 0x3F) {
// '?'
r.advance();
while (!r.isAtEnd) {
if (r.matchString('?>')) {
return true;
}
r.advance();
}
r.reset(saved);
return false;
}
// 3. Closing tag `</tagname\s*>`
if (second == 0x2F) {
// '/'
r.advance();
if (!isAsciiAlpha(r.peek())) {
r.reset(saved);
return false;
}
r.advance();
r.skipWhile((cp) => isAsciiAlphanumeric(cp) || cp == 0x2D);
r.skipWhile((cp) => isAsciiWhitespace(cp));
if (r.matchCodePoint(0x3E)) {
return true;
}
r.reset(saved);
return false;
}
// 4. Opening tag `<tagname attr="val" ... /?>`
if (!isAsciiAlpha(second)) {
r.reset(saved);
return false;
}
r.advance();
r.skipWhile((cp) => isAsciiAlphanumeric(cp) || cp == 0x2D);
while (!r.isAtEnd) {
let wsStart = r.mark();
r.skipWhile((cp) => isAsciiWhitespace(cp));
if (r.isAtEnd) {
r.reset(saved);
return false;
}
if (r.matchCodePoint(0x3E)) {
// '>'
return true;
}
if (r.matchString('/>')) {
return true;
}
// Attribute must be preceded by whitespace
if (r.mark() == wsStart) {
r.reset(saved);
return false;
}
// Attribute name: `[a-zA-Z_:][a-zA-Z0-9_.:-]*`
let attrStart = r.peek();
if (!isAsciiAlpha(attrStart) && attrStart != 0x5F && attrStart != 0x3A) {
r.reset(saved);
return false;
}
r.advance();
r.skipWhile(
(ab) => isAsciiAlphanumeric(ab)
|| ab == 0x5F
|| ab == 0x2E
|| ab == 0x3A
|| ab == 0x2D
);
// Optional `= value`
let afterAttrName = r.mark();
r.skipWhile((cp) => isAsciiWhitespace(cp));
if (r.matchCodePoint(0x3D)) {
// '='
r.skipWhile((cp) => isAsciiWhitespace(cp));
if (r.isAtEnd) {
r.reset(saved);
return false;
}
let vb = r.peek();
if (vb == 0x22 || vb == 0x27) {
// "..." or '...'
let quote = r.advance();
while (!r.isAtEnd && r.peek() != quote) {
r.advance();
}
if (!r.matchCodePoint(quote)) {
r.reset(saved);
return false;
}
} else {
let valStart = r.mark();
r.skipWhile(
(ub) => !isAsciiWhitespace(ub)
&& ub != 0x22
&& ub != 0x27
&& ub != 0x3D
&& ub != 0x3C
&& ub != 0x3E
&& ub != 0x60
);
if (r.mark() == valStart) {
r.reset(saved);
return false;
}
}
} else {
r.reset(afterAttrName);
}
}
r.reset(saved);
return false;
}
| Version | Published | Dependencies | Approved by |
|---|---|---|---|
| 0.0.1-preview.1 | 2026-11-19 | none |
|