markdown 0.0.1-preview.1
CommonMark and GFM Markdown parser, AST, plugin system, and HTML renderer
Markdown (@zena-lang/markdown)#
A CommonMark and GitHub Flavored Markdown (GFM) parser, syntax tree, plugin
system, and HTML renderer written in Zena (zena-lang:markdown/<library>).
See docs/design/markdown.md for the design.
Usage#
Add it with zena pkg add zena-lang:markdown, then import its main
library:
import {
parse, renderHtml, markdownToHtml, gfmExtensions
} from 'zena-lang:markdown';
// Render CommonMark + GFM to safe HTML (raw HTML stripped, unsafe URLs blocked)
let html = markdownToHtml('# Hello\n\nSome ~~old~~ **new** text.', {
extensions: gfmExtensions(),
safe: true,
});
// Or parse to an AST first
let doc = parse('# Heading\n\nParagraph.', gfmExtensions());
let out = renderHtml(doc, {safe: true, extensions: gfmExtensions()});
Running Tests#
npm test -w @zena-lang/markdown
npm run test:quick -- packages/markdown/test
46.8 KiB
Raw
/**
* CommonMark Phase 1 block parser with open-block stack, lazy continuation,
* link reference definitions, and `BlockParserFactory` plugin dispatch.
*/
import {FixedArray, GrowableArray, Option, none} from 'zena:core';
import {StringBuilder, StringReader, isSpaceOrTab, isAsciiAlpha, isAsciiDigit, isAsciiAlphanumeric} from 'zena:string';
import {HashMap} from 'zena:collections';
import {Block, Document, Inline, LinkDef, Span, Paragraph, Heading, BlockQuote, ListBlock, ListItem, FencedCodeBlock, IndentedCodeBlock, HtmlBlock, ThematicBreak, Text} from './ast.zena';
import {AstTransformer, BlockContinue, BlockLineState, BlockParserFactory, CustomBlockParser, CustomBlockStart, InlineParserRunner, MarkdownExtension, MatchedBlockInfo} from './plugin.zena';
import {InlineParser} from './inline-parser.zena';
import {trimSpaces, normalizeLinkLabel, unescapeString, scanLinkDestination, scanLinkTitle, scanLinkLabel} from './scanner.zena';
/**
* How many blocks may be open at once, the document included: quotes,
* lists and list items nest no deeper. markdown-it's default preset allows
* the same depth.
*/
let MAX_BLOCK_NESTING = 100;
enum OpenBlockKind {
Doc,
Quote,
List,
Item,
Para,
HeadingBlock,
FencedCode,
IndentedCode,
Html,
BreakBlock,
Custom
}
/**
* State for an open block on the Phase 1 stack.
*/
final class OpenBlock {
kind: OpenBlockKind;
var startOffset: i32;
var endOffset: i32;
children = new GrowableArray<Block>(4);
// Paragraph / code / HTML accumulated lines
lines = new GrowableArray<String>(4);
lineOffsets = new GrowableArray<i32>(4);
// Heading
var headingLevel: i32 = 1;
var headingText: String = '';
var headingTextOffset: i32 = 0;
// Fenced code
var fenceByte: i32 = 0;
var fenceLength: i32 = 0;
var fenceIndent: i32 = 0;
var fenceInfo: String = '';
var fenceFirstLine: boolean = true;
// HTML block (types 1..7)
var htmlType: i32 = 0;
// List & ListItem
var listOrdered: boolean = false;
var listStartNumber: i32 = 1;
var listDelimiter: i32 = 0;
var listTight: boolean = true;
listItems = new GrowableArray<ListItem>(4);
var listHadBlankLine: boolean = false;
var itemContentIndent: i32 = 0;
var itemHadBlankLine: boolean = false;
var itemLineCount: i32 = 0;
// Custom plugin block
var customParser: CustomBlockParser | null = null;
var builtBlock: Block | null = null;
new(this.kind, this.startOffset) : endOffset = startOffset {}
}
/**
* Parses a Markdown document string into a `Document` AST.
*/
export function parse(source: String, extensions: GrowableArray<MarkdownExtension> = new GrowableArray<MarkdownExtension>(
0
)): Document {
let parser = new DocumentBlockParser(extensions);
return parser.parseDocument(source);
}
/**
* Phase 1 + Phase 2 + Phase 3 coordinator.
*/
export final class DocumentBlockParser {
#extensions: GrowableArray<MarkdownExtension>;
#factoriesByByte = new FixedArray<GrowableArray<BlockParserFactory> | null>(
128,
null
);
#wildcardFactories = new GrowableArray<BlockParserFactory>(2);
#transformers = new GrowableArray<AstTransformer>(4);
#links = new HashMap<String, LinkDef>();
#openBlocks = new GrowableArray<OpenBlock>(8);
#customHolders = new GrowableArray<OpenBlock>(4);
var #partiallyConsumedTab: boolean = false;
var #tabRemainingColumns: i32 = 0;
var #closingFenceConsumedLine: boolean = false;
new(extensions: GrowableArray<MarkdownExtension>) : #extensions = extensions {
for (let ext in extensions) {
for (let factory in ext.blockFactories) {
let triggers = factory.triggerBytes();
if (triggers.length == 0) {
this.#wildcardFactories.push(factory);
} else {
for (let b in triggers) {
if (b >= 0 && b < 128) {
var bucket = this.#factoriesByByte[b];
if (bucket == null) {
bucket = new GrowableArray<BlockParserFactory>(2);
this.#factoriesByByte[b] = bucket;
}
bucket.push(factory);
}
}
}
}
for (let transformer in ext.astTransformers) {
this.#transformers.push(transformer);
}
}
}
parseDocument(source: String): Document {
this.#links.clear();
this.#openBlocks.clear();
this.#customHolders.clear();
let docReader = new StringReader(source);
let len = docReader.remaining;
let root = new OpenBlock(OpenBlockKind.Doc, 0);
root.endOffset = len;
this.#openBlocks.push(root);
while (!docReader.isAtEnd) {
let pos = docReader.mark();
let rawLine = docReader.readLine() ?? '';
let lr = new StringReader(rawLine);
let lineEnd = pos + lr.remaining;
let line = replaceNullCodePoints(rawLine);
this.#processLine(source, line, pos, lineEnd);
}
// Finalize all remaining open blocks from tip down to root
while (this.#openBlocks.length > 1) {
this.#closeTipBlock(len);
}
let inlineParser = new InlineParser(this.#links, this.#extensions);
this.#parseBlockInlines(root.children, inlineParser);
for (let holder in this.#customHolders) {
let cp = holder.customParser;
if (cp != null) {
cp.parseInlines(inlineParser);
}
}
let doc = new Document(root.children, this.#links, new Span(0, len));
for (let transformer in this.#transformers) {
transformer.transform(doc, inlineParser);
}
return doc;
}
#processLine(source: String, line: String, lineStartOffset: i32, lineEndOffset: i32): void {
this.#partiallyConsumedTab = false;
this.#tabRemainingColumns = 0;
this.#closingFenceConsumedLine = false;
let state = new BlockLineState(
source,
line,
lineStartOffset,
0,
0,
0,
0,
0,
true
);
this.#findNextNonSpace(state);
// Step 1: Check continuation of open blocks (starting after Document at index 0)
var matchedCount = 1;
var i = 1;
while (i < this.#openBlocks.length) {
let block = this.#openBlocks[i];
if (!this.#continueOpenBlock(block, state)) {
break;
}
block.endOffset = lineEndOffset;
matchedCount += 1;
i += 1;
}
let allMatched = matchedCount == this.#openBlocks.length;
let lastMatched = this.#openBlocks[matchedCount - 1];
var currentTip = this.#openBlocks[this.#openBlocks.length - 1];
// Step 2: Try to open new child blocks unless the last matched block is a
// leaf block that consumes raw text (FencedCode, IndentedCode, Html, Custom)
let tryBlockStarts = lastMatched.kind != OpenBlockKind.FencedCode
&& lastMatched.kind != OpenBlockKind.IndentedCode
&& lastMatched.kind != OpenBlockKind.Html
&& lastMatched.kind != OpenBlockKind.Custom;
while (tryBlockStarts) {
this.#findNextNonSpace(state);
if (state.isBlank) {
this.#AdvanceToEnd(state);
break;
}
let matchedBlock = this.#openBlocks[matchedCount - 1];
// Past MAX_BLOCK_NESTING, a marker that would open another block is
// text: every walk over the tree recurses through the blocks, and a
// deeper document would overflow the stack.
let started = this.#openBlocks.length <= MAX_BLOCK_NESTING && this.#tryStartNewBlock(
state,
matchedBlock,
currentTip,
lineEndOffset
);
if (!started) {
this.#setNewIndex(state, state.nextNonSpaceIndex);
break;
}
currentTip = this.#openBlocks[this.#openBlocks.length - 1];
matchedCount = this.#openBlocks.length;
if (!this.#isContainerOrCustomizable(currentTip)) {
break;
}
}
// Step 3: Lazy continuation of an open paragraph
if (!allMatched
&& !state.isBlank
&& currentTip.kind == OpenBlockKind.Para) {
this.#addParagraphLine(currentTip, state, lineEndOffset);
return;
}
// Close unmatched blocks
while (this.#openBlocks.length > matchedCount) {
this.#closeTipBlock(lineStartOffset);
}
// If a non-blank line did not start a new list item in an open List block,
// close the List block now before adding paragraph text
this.#findNextNonSpace(state);
while (!state.isBlank
&& this.#openBlocks.length > 1
&& this.#openBlocks[this.#openBlocks.length - 1].kind
== OpenBlockKind.List) {
this.#closeTipBlock(lineStartOffset);
}
currentTip = this.#openBlocks[this.#openBlocks.length - 1];
this.#findNextNonSpace(state);
// Track blank lines on lists/items for tight vs. loose determination
if (state.isBlank && !this.#closingFenceConsumedLine) {
if (currentTip.kind == OpenBlockKind.List) {
if (currentTip.listItems.length > 0) {
currentTip.listHadBlankLine = true;
}
} else if (currentTip.kind == OpenBlockKind.Item) {
if (currentTip.children.length > 0 || currentTip.itemLineCount > 0) {
currentTip.itemHadBlankLine = true;
}
}
}
// Step 4: Add remaining line text to the active block
match (currentTip.kind) {
case OpenBlockKind.IndentedCode: {
this.#addIndentedCodeLine(currentTip, state, lineEndOffset);
}
case OpenBlockKind.FencedCode: {
this.#addFencedCodeLine(currentTip, state, lineEndOffset);
}
case OpenBlockKind.Html: {
this.#addHtmlBlockLine(currentTip, state, lineEndOffset);
}
case OpenBlockKind.Para: {
if (!state.isBlank) {
this.#addParagraphLine(currentTip, state, lineEndOffset);
};
}
case OpenBlockKind.Custom: {
let cp = currentTip.customParser;
if (cp != null && cp.acceptsLines() && !state.isBlank) {
let r = new StringReader(state.line);
r.reset(state.index);
cp.addLine(r.readToEnd(), state.lineStartOffset + state.index);
currentTip.endOffset = lineEndOffset;
};
}
case OpenBlockKind.HeadingBlock | OpenBlockKind.BreakBlock: {
// Single-line blocks close immediately at the end of the line
this.#closeTipBlock(lineEndOffset);
}
case _: {
if (!state.isBlank) {
let para = new OpenBlock(
OpenBlockKind.Para,
state.lineStartOffset + state.nextNonSpaceIndex
);
para.endOffset = lineEndOffset;
this.#openChildBlock(para, state.lineStartOffset);
this.#setNewIndex(state, state.nextNonSpaceIndex);
this.#addParagraphLine(para, state, lineEndOffset);
};
}
};
}
#isContainerOrCustomizable(block: OpenBlock): boolean {
return block.kind == OpenBlockKind.Doc
|| block.kind == OpenBlockKind.Quote
|| block.kind == OpenBlockKind.List
|| block.kind == OpenBlockKind.Item;
}
#continueOpenBlock(block: OpenBlock, state: BlockLineState): boolean {
this.#findNextNonSpace(state);
let r = new StringReader(state.line);
match (block.kind) {
case OpenBlockKind.Quote: {
r.reset(state.nextNonSpaceIndex);
if (state.indent <= 3 && r.peek() == 0x3E) {
// '>'
this.#setNewColumn(state, state.nextNonSpaceColumn + 1);
r.reset(state.index);
if (isSpaceOrTab(r.peek())) {
this.#setNewColumn(state, state.column + 1);
};
return true;
};
return false;
}
case OpenBlockKind.List: {
return true;
}
case OpenBlockKind.Item: {
if (state.isBlank) {
if (block.children.length == 0 && block.itemLineCount == 0) {
// An empty list item cannot continue past a blank line
return false;
};
this.#setNewIndex(state, state.nextNonSpaceIndex);
return true;
};
if (state.indent >= block.itemContentIndent) {
this.#setNewColumn(state, state.column + block.itemContentIndent);
return true;
};
return false;
}
case OpenBlockKind.IndentedCode: {
if (state.indent >= 4) {
this.#setNewColumn(state, state.column + 4);
return true;
};
if (state.isBlank) {
this.#setNewIndex(state, state.nextNonSpaceIndex);
return true;
};
return false;
}
case OpenBlockKind.FencedCode: {
r.reset(state.nextNonSpaceIndex);
if (state.indent <= 3 && r.peek() == block.fenceByte) {
var closeLen = 0;
while (r.matchCodePoint(block.fenceByte)) {
closeLen += 1;
}
if (closeLen >= block.fenceLength) {
r.skipSpaces();
if (r.isAtEnd) {
// Closing fence matched: close block and consume line
this.#closingFenceConsumedLine = true;
this.#closeTipBlock(state.lineStartOffset + r.mark());
this.#AdvanceToEnd(state);
return false;
};
};
};
// Strip up to `fenceIndent` leading spaces
var spaces = block.fenceIndent;
while (spaces > 0) {
r.reset(state.index);
if (r.peek() == 0x20) {
this.#setNewIndex(state, state.index + 1);
spaces -= 1;
} else {
break;
}
}
return true;
}
case OpenBlockKind.Html: {
if (state.isBlank && (block.htmlType == 6 || block.htmlType == 7)) {
return false;
};
return true;
}
case OpenBlockKind.Para: {
return !state.isBlank;
}
case OpenBlockKind.Custom: {
let cp = block.customParser;
if (cp == null) {
return false;
};
let cont = cp.tryContinue(state);
if (!cont.matched) {
return false;
};
if (cont.finalize) {
this.#closeTipBlock(state.lineStartOffset + r.remaining);
return false;
};
if (cont.newIndex >= 0) {
this.#setNewIndex(state, cont.newIndex);
} else if (cont.newColumn >= 0) {
this.#setNewColumn(state, cont.newColumn);
};
return true;
}
case _: {
return false;
}
};
}
#tryStartNewBlock(state: BlockLineState, matchedBlock: OpenBlock, currentTip: OpenBlock, lineEndOffset: i32): boolean {
let inParagraph = currentTip.kind == OpenBlockKind.Para;
let r = new StringReader(state.line);
r.reset(state.nextNonSpaceIndex);
let b = r.peek();
// 1. Custom `BlockParserFactory` plugins (allowing plugins like GFM tables
// to inspect `matchedInfo` and replace an open paragraph before Setext
// headings or thematic breaks claim a `---` line)
if (this.#tryCustomBlockFactories(state, currentTip, b, lineEndOffset)) {
return true;
}
if (state.indent >= 4) {
if (!inParagraph && !state.isBlank) {
this.#closeUnmatchedBeforeNewBlock(matchedBlock, state.lineStartOffset);
this.#setNewColumn(state, state.column + 4);
let codeBlock = new OpenBlock(
OpenBlockKind.IndentedCode,
state.lineStartOffset + state.index
);
codeBlock.endOffset = lineEndOffset;
this.#openChildBlock(codeBlock, state.lineStartOffset);
return true;
}
return false;
}
// 2. BlockQuote (`>`)
if (b == 0x3E) {
this.#closeUnmatchedBeforeNewBlock(matchedBlock, state.lineStartOffset);
let quote = new OpenBlock(
OpenBlockKind.Quote,
state.lineStartOffset + state.nextNonSpaceIndex
);
quote.endOffset = lineEndOffset;
this.#openChildBlock(quote, state.lineStartOffset);
this.#setNewColumn(state, state.nextNonSpaceColumn + 1);
r.reset(state.index);
if (isSpaceOrTab(r.peek())) {
this.#setNewColumn(state, state.column + 1);
}
return true;
}
// 3. ATX Heading (`#`..`######`)
if (b == 0x23) {
let heading = this.#tryParseAtxHeading(state, lineEndOffset);
if (heading != null) {
this.#closeUnmatchedBeforeNewBlock(matchedBlock, state.lineStartOffset);
this.#openChildBlock(heading, state.lineStartOffset);
this.#AdvanceToEnd(state);
return true;
}
}
// 4. Fenced Code Block (``` or ~~~)
if (b == 0x60 || b == 0x7E) {
let fenced = this.#tryParseFencedCode(state, lineEndOffset);
if (fenced != null) {
this.#closeUnmatchedBeforeNewBlock(matchedBlock, state.lineStartOffset);
this.#openChildBlock(fenced, state.lineStartOffset);
this.#AdvanceToEnd(state);
return true;
}
}
// 5. HTML Block (`<...`)
if (b == 0x3C) {
let htmlType = detectHtmlBlockStart(
state.line,
state.nextNonSpaceIndex,
inParagraph
);
if (htmlType > 0) {
this.#closeUnmatchedBeforeNewBlock(matchedBlock, state.lineStartOffset);
let htmlBlock = new OpenBlock(
OpenBlockKind.Html,
state.lineStartOffset + state.nextNonSpaceIndex
);
htmlBlock.htmlType = htmlType;
htmlBlock.endOffset = lineEndOffset;
this.#openChildBlock(htmlBlock, state.lineStartOffset);
return true;
}
}
// 6. Setext Heading underline (`===` or `---`) when tip is an open Paragraph
if (inParagraph && (b == 0x3D || b == 0x2D)) {
let level = matchSetextUnderline(state.line, state.nextNonSpaceIndex);
if (level > 0) {
// Strip leading link reference definitions from the paragraph first;
// only become a Setext heading if non-blank paragraph text remains!
this.#extractLinkDefinitionsFromParagraph(currentTip);
if (currentTip.lines.length > 0) {
let paraStart = currentTip.startOffset;
let text = joinParagraphLines(currentTip.lines);
// Close unmatched up to and including the open paragraph
while (this.#openBlocks.length > 1) {
let top = this.#openBlocks[this.#openBlocks.length - 1];
this.#openBlocks.pop();
if (top == currentTip) {
break;
}
}
let setext = new OpenBlock(OpenBlockKind.HeadingBlock, paraStart);
setext.headingLevel = level;
setext.headingText = text;
setext.headingTextOffset = paraStart;
setext.endOffset = lineEndOffset;
this.#openChildBlock(setext, paraStart);
this.#AdvanceToEnd(state);
return true;
}
}
}
// 7. Thematic Break (`***`, `---`, `___`)
if (b == 0x2A || b == 0x2D || b == 0x5F) {
if (isThematicBreak(state.line, state.nextNonSpaceIndex)) {
this.#closeUnmatchedBeforeNewBlock(matchedBlock, state.lineStartOffset);
let hr = new OpenBlock(
OpenBlockKind.BreakBlock,
state.lineStartOffset + state.nextNonSpaceIndex
);
hr.endOffset = lineEndOffset;
this.#openChildBlock(hr, state.lineStartOffset);
this.#AdvanceToEnd(state);
return true;
}
}
// 8. List Item (`-`, `+`, `*`, or `1.`, `1)`)
if (b == 0x2D || b == 0x2B || b == 0x2A || isAsciiDigit(b)) {
let inParaAtCurrentLevel = matchedBlock.kind == OpenBlockKind.Para;
let (listOk, ordered, startNum, delim, contentIndent, nextCol) = tryParseListMarker(
state,
inParaAtCurrentLevel
);
if (listOk) {
this.#closeUnmatchedBeforeNewBlock(
matchedBlock,
state.lineStartOffset,
true
);
this.#setNewColumn(state, nextCol);
// Ensure an open `ListBlock` with matching type/delimiter is on top
let top = this.#openBlocks[this.#openBlocks.length - 1];
if (top.kind != OpenBlockKind.List
|| top.listOrdered != ordered
|| top.listDelimiter != delim) {
if (top.kind == OpenBlockKind.List) {
this.#closeTipBlock(state.lineStartOffset);
}
let listBlock = new OpenBlock(
OpenBlockKind.List,
state.lineStartOffset + state.nextNonSpaceIndex
);
listBlock.listOrdered = ordered;
listBlock.listStartNumber = startNum;
listBlock.listDelimiter = delim;
listBlock.endOffset = lineEndOffset;
this.#openChildBlock(listBlock, state.lineStartOffset);
}
let listParent = this.#openBlocks[this.#openBlocks.length - 1];
if (listParent.listHadBlankLine && listParent.listItems.length > 0) {
listParent.listTight = false;
listParent.listHadBlankLine = false;
}
let item = new OpenBlock(
OpenBlockKind.Item,
state.lineStartOffset + state.nextNonSpaceIndex
);
item.itemContentIndent = contentIndent;
item.endOffset = lineEndOffset;
this.#openChildBlock(item, state.lineStartOffset);
return true;
}
}
return false;
}
#tryCustomBlockFactories(state: BlockLineState, currentTip: OpenBlock, firstByte: i32, lineEndOffset: i32): boolean {
var matchedInfo: MatchedBlockInfo | null = null;
if (firstByte >= 0 && firstByte < 128) {
let bucket = this.#factoriesByByte[firstByte];
if (bucket != null) {
matchedInfo = this.#buildMatchedBlockInfo(currentTip);
for (let factory in bucket) {
let res = factory.tryStart(state, matchedInfo);
if (res != null) {
this.#applyCustomBlockStart(res, state, currentTip, lineEndOffset);
return true;
}
}
}
}
if (this.#wildcardFactories.length > 0) {
if (matchedInfo == null) {
matchedInfo = this.#buildMatchedBlockInfo(currentTip);
}
for (let factory in this.#wildcardFactories) {
let res = factory.tryStart(state, matchedInfo);
if (res != null) {
this.#applyCustomBlockStart(res, state, currentTip, lineEndOffset);
return true;
}
}
}
return false;
}
#buildMatchedBlockInfo(currentTip: OpenBlock): MatchedBlockInfo {
if (currentTip.kind == OpenBlockKind.Para) {
this.#extractLinkDefinitionsFromParagraph(currentTip);
if (currentTip.lines.length > 0) {
return new MatchedBlockInfo(
true,
currentTip.lines,
currentTip.startOffset
);
}
}
return new MatchedBlockInfo(false, null, 0);
}
#applyCustomBlockStart(start: CustomBlockStart, state: BlockLineState, currentTip: OpenBlock, lineEndOffset: i32): void {
var startOff = state.lineStartOffset + state.nextNonSpaceIndex;
if (start.replaceParagraph && currentTip.kind == OpenBlockKind.Para) {
if (currentTip.lines.length > 1) {
let lastIdx = currentTip.lines.length - 1;
currentTip.lines.pop();
if (currentTip.lineOffsets.length > lastIdx) {
startOff = currentTip.lineOffsets.pop();
} else {
startOff = state.lineStartOffset;
}
currentTip.endOffset = startOff;
this.#closeTipBlock(startOff);
} else {
startOff = currentTip.startOffset;
this.#openBlocks.pop();
}
}
let customBlock = new OpenBlock(OpenBlockKind.Custom, startOff);
customBlock.customParser = start.parser;
customBlock.endOffset = lineEndOffset;
this.#openChildBlock(customBlock, state.lineStartOffset);
this.#customHolders.push(customBlock);
if (start.newIndex >= 0) {
this.#setNewIndex(state, start.newIndex);
} else if (start.newColumn >= 0) {
this.#setNewColumn(state, start.newColumn);
}
}
#closeUnmatchedBeforeNewBlock(matchedBlock: OpenBlock, closeOffset: i32, startingListItem: boolean = false): void {
while (this.#openBlocks.length > 1) {
let top = this.#openBlocks[this.#openBlocks.length - 1];
if (top == matchedBlock) {
if (top.kind == OpenBlockKind.Para) {
this.#closeTipBlock(closeOffset);
} else if (top.kind == OpenBlockKind.List && !startingListItem) {
this.#closeTipBlock(closeOffset);
}
break;
}
this.#closeTipBlock(closeOffset);
}
}
#openChildBlock(child: OpenBlock, offset: i32): void {
let parent = this.#openBlocks[this.#openBlocks.length - 1];
if (parent.kind == OpenBlockKind.Item && parent.itemHadBlankLine) {
parent.itemHadBlankLine = false;
// A blank line directly between two blocks in a list item makes the
// enclosing list loose
if (this.#openBlocks.length >= 2) {
let maybeList = this.#openBlocks[this.#openBlocks.length - 2];
if (maybeList.kind == OpenBlockKind.List) {
maybeList.listTight = false;
}
}
}
this.#openBlocks.push(child);
}
#closeTipBlock(closeOffset: i32): void {
let tip = this.#openBlocks.pop();
let parent = this.#openBlocks[this.#openBlocks.length - 1];
if (tip.endOffset > parent.endOffset && parent.kind != OpenBlockKind.Doc) {
parent.endOffset = tip.endOffset;
}
let span = new Span(tip.startOffset, tip.endOffset);
match (tip.kind) {
case OpenBlockKind.Para: {
this.#extractLinkDefinitionsFromParagraph(tip);
if (tip.lines.length > 0) {
let rawText = joinParagraphLines(tip.lines);
// Store rawText temporarily in a single Text inline; Phase 2 replaces
// `inlines` with the parsed inline nodes
let inlines = new GrowableArray<Inline>(1);
inlines.push(new Text(rawText, span));
parent.children.push(new Paragraph(inlines, null, span));
};
}
case OpenBlockKind.HeadingBlock: {
let inlines = new GrowableArray<Inline>(1);
inlines.push(
new Text(
tip.headingText,
new Span(tip.headingTextOffset, tip.endOffset)
)
);
parent.children.push(
new Heading(tip.headingLevel, inlines, null, span)
);
}
case OpenBlockKind.Quote: {
parent.children.push(new BlockQuote(tip.children, span));
}
case OpenBlockKind.List: {
if (tip.listHadBlankLine) {
if (parent.kind == OpenBlockKind.Item) {
parent.itemHadBlankLine = true;
} else if (parent.kind == OpenBlockKind.List) {
parent.listHadBlankLine = true;
};
};
parent.children.push(
new ListBlock(
tip.listOrdered,
tip.listStartNumber,
tip.listDelimiter,
tip.listTight,
tip.listItems,
span
)
);
}
case OpenBlockKind.Item: {
let itemNode = new ListItem(tip.children, none, span);
if (parent.kind == OpenBlockKind.List) {
if (tip.itemHadBlankLine) {
parent.listHadBlankLine = true;
};
parent.listItems.push(itemNode);
} else {
parent.children.push(itemNode);
};
}
case OpenBlockKind.FencedCode: {
let sb = new StringBuilder(64);
for (let line in tip.lines) {
sb.append(line);
sb.appendByte(0x0A);
}
parent.children.push(
new FencedCodeBlock(tip.fenceInfo, sb.toString(), span)
);
}
case OpenBlockKind.IndentedCode: {
// Strip leading and trailing blank lines per CommonMark 4.4
var start = 0;
var end = tip.lines.length;
while (start < end && isBlankString(tip.lines[start])) {
start += 1;
}
while (end > start && isBlankString(tip.lines[end - 1])) {
end -= 1;
}
if (end < tip.lines.length) {
if (parent.kind == OpenBlockKind.Item) {
parent.itemHadBlankLine = true;
} else if (parent.kind == OpenBlockKind.List) {
parent.listHadBlankLine = true;
};
};
let sb = new StringBuilder(64);
var i = start;
while (i < end) {
sb.append(tip.lines[i]);
sb.appendByte(0x0A);
i += 1;
}
parent.children.push(new IndentedCodeBlock(sb.toString(), span));
}
case OpenBlockKind.Html: {
let sb = new StringBuilder(64);
var i = 0;
while (i < tip.lines.length) {
if (i > 0) {
sb.appendByte(0x0A);
}
sb.append(tip.lines[i]);
i += 1;
}
parent.children.push(new HtmlBlock(sb.toString(), span));
}
case OpenBlockKind.BreakBlock: {
parent.children.push(new ThematicBreak(span));
}
case OpenBlockKind.Custom: {
let cp = tip.customParser;
if (cp != null) {
let built = cp.closeBlock(tip.endOffset);
tip.builtBlock = built;
parent.children.push(built);
};
}
case OpenBlockKind.Doc: {}
};
}
#addParagraphLine(para: OpenBlock, state: BlockLineState, lineEndOffset: i32): void {
let r = new StringReader(state.line);
r.reset(state.nextNonSpaceIndex);
para.lines.push(r.readToEnd());
para.lineOffsets.push(state.lineStartOffset + state.nextNonSpaceIndex);
para.endOffset = lineEndOffset;
this.#markEnclosingItemHasLine();
}
#addIndentedCodeLine(code: OpenBlock, state: BlockLineState, lineEndOffset: i32): void {
let content = this.#extractRemainingWithTabSpaces(state);
code.lines.push(content);
code.endOffset = lineEndOffset;
this.#markEnclosingItemHasLine();
}
#addFencedCodeLine(code: OpenBlock, state: BlockLineState, lineEndOffset: i32): void {
if (code.fenceFirstLine) {
code.fenceFirstLine = false;
return;
}
let content = this.#extractRemainingWithTabSpaces(state);
code.lines.push(content);
code.endOffset = lineEndOffset;
this.#markEnclosingItemHasLine();
}
#addHtmlBlockLine(html: OpenBlock, state: BlockLineState, lineEndOffset: i32): void {
let content = this.#extractRemainingWithTabSpaces(state);
html.lines.push(content);
html.endOffset = lineEndOffset;
this.#markEnclosingItemHasLine();
if (htmlBlockEndsOnLine(content, html.htmlType)) {
this.#closeTipBlock(lineEndOffset);
}
}
#markEnclosingItemHasLine(): void {
var i = this.#openBlocks.length - 1;
while (i >= 0) {
let b = this.#openBlocks[i];
if (b.kind == OpenBlockKind.Item) {
b.itemLineCount += 1;
return;
}
i -= 1;
}
}
#extractRemainingWithTabSpaces(state: BlockLineState): String {
let r = new StringReader(state.line);
r.reset(state.index);
let rest = r.readToEnd();
if (!this.#partiallyConsumedTab || this.#tabRemainingColumns <= 0) {
return rest;
}
let sb = new StringBuilder();
var s = 0;
while (s < this.#tabRemainingColumns) {
sb.appendByte(0x20);
s += 1;
}
sb.append(rest);
return sb.toString();
}
#extractLinkDefinitionsFromParagraph(para: OpenBlock): void {
if (para.lines.length == 0) {
return;
}
let combined = joinParagraphLines(para.lines);
let r = new StringReader(combined);
var consumedAny = false;
while (!r.isAtEnd) {
let defStart = r.mark();
let (ok, label, dest, title) = tryParseLinkReferenceDefinition(r);
if (!ok) {
r.reset(defStart);
break;
}
consumedAny = true;
let nextPos = r.mark();
if (!this.#links.has(label)) {
this.#links[label] = new LinkDef(
label,
dest,
title,
new Span(para.startOffset + defStart, para.startOffset + nextPos)
);
}
}
if (!consumedAny) {
return;
}
let pos = r.mark();
para.lines.clear();
if (!r.isAtEnd) {
// Split remaining lines back into `para.lines`
while (!r.isAtEnd) {
let line = r.readLine();
if (line != null) {
para.lines.push(line);
}
}
para.startOffset += pos;
}
}
#parseBlockInlines(blocks: GrowableArray<Block>, inlineParser: InlineParserRunner): void {
for (let block in blocks) {
match (block) {
case Paragraph as p: {
if (p.inlines.length == 1) {
if (let Text {literal, span} = p.inlines[0]) {
p.inlines = inlineParser.parse(literal, span.start);
};
};
}
case Heading as h: {
if (h.inlines.length == 1) {
if (let Text {literal, span} = h.inlines[0]) {
h.inlines = inlineParser.parse(literal, span.start);
};
};
}
case BlockQuote {blocks: let children}: {
this.#parseBlockInlines(children, inlineParser);
}
case ListBlock {items}: {
for (let item in items) {
this.#parseBlockInlines(item.blocks, inlineParser);
}
}
case ListItem {blocks: let children}: {
this.#parseBlockInlines(children, inlineParser);
}
case _: {}
};
}
}
#tryParseAtxHeading(state: BlockLineState, lineEndOffset: i32): OpenBlock | null {
let r = new StringReader(state.line);
let start = state.nextNonSpaceIndex;
r.reset(start);
var level = 0;
while (level < 7 && r.matchCodePoint(0x23)) {
level += 1;
}
if (level == 0 || level > 6) {
return null;
}
if (!r.isAtEnd && !isSpaceOrTab(r.peek())) {
return null;
}
// Trim leading spaces/tabs and optional closing `#` sequence
r.skipSpaces();
let contentStart = r.mark();
var contentEnd = contentStart + r.remaining;
while (contentEnd > contentStart) {
r.reset(contentEnd);
if (isSpaceOrTab(r.peekBefore())) {
contentEnd -= 1;
} else {
break;
}
}
var hashEnd = contentEnd;
while (hashEnd > contentStart) {
r.reset(hashEnd);
if (r.peekBefore() == 0x23) {
hashEnd -= 1;
} else {
break;
}
}
if (hashEnd < contentEnd) {
r.reset(hashEnd);
if (hashEnd == contentStart || isSpaceOrTab(r.peekBefore())) {
contentEnd = hashEnd;
while (contentEnd > contentStart) {
r.reset(contentEnd);
if (isSpaceOrTab(r.peekBefore())) {
contentEnd -= 1;
} else {
break;
}
}
}
}
let h = new OpenBlock(
OpenBlockKind.HeadingBlock,
state.lineStartOffset + start
);
h.headingLevel = level;
h.headingText = r.sliceRange(contentStart, contentEnd);
h.headingTextOffset = state.lineStartOffset + contentStart;
h.endOffset = lineEndOffset;
return h;
}
#tryParseFencedCode(state: BlockLineState, lineEndOffset: i32): OpenBlock | null {
let r = new StringReader(state.line);
let start = state.nextNonSpaceIndex;
r.reset(start);
let fenceByte = r.peek();
var fenceLen = 0;
while (r.matchCodePoint(fenceByte)) {
fenceLen += 1;
}
if (fenceLen < 3) {
return null;
}
let rest = r.readToEnd();
// Backtick fence info string cannot contain backticks
if (fenceByte == 0x60 && rest.contains('`')) {
return null;
}
let rawInfo = trimSpaces(rest);
let block = new OpenBlock(
OpenBlockKind.FencedCode,
state.lineStartOffset + start
);
block.fenceByte = fenceByte;
block.fenceLength = fenceLen;
block.fenceIndent = state.indent;
block.fenceInfo = unescapeString(rawInfo);
block.fenceFirstLine = true;
block.endOffset = lineEndOffset;
return block;
}
#findNextNonSpace(state: BlockLineState): void {
let r = new StringReader(state.line);
r.reset(state.index);
var cols = state.column;
if (this.#partiallyConsumedTab) {
cols += this.#tabRemainingColumns;
}
while (!r.isAtEnd) {
let b = r.peek();
if (b == 0x20) {
r.advance();
cols += 1;
} else if (b == 0x09) {
r.advance();
cols += 4 - cols % 4;
} else {
break;
}
}
state.nextNonSpaceIndex = r.mark();
state.nextNonSpaceColumn = cols;
state.indent = cols - state.column;
state.isBlank = r.isAtEnd;
}
#setNewIndex(state: BlockLineState, newIndex: i32): void {
if (newIndex >= state.nextNonSpaceIndex) {
state.index = state.nextNonSpaceIndex;
state.column = state.nextNonSpaceColumn;
}
let r = new StringReader(state.line);
r.reset(state.index);
while (r.mark() < newIndex && !r.isAtEnd) {
let b = r.advance();
if (b == 0x09) {
state.column += 4 - state.column % 4;
} else {
state.column += 1;
}
}
state.index = r.mark();
this.#partiallyConsumedTab = false;
this.#tabRemainingColumns = 0;
}
#setNewColumn(state: BlockLineState, newColumn: i32): void {
if (newColumn >= state.nextNonSpaceColumn) {
state.index = state.nextNonSpaceIndex;
state.column = state.nextNonSpaceColumn;
}
let r = new StringReader(state.line);
r.reset(state.index);
while (state.column < newColumn && !r.isAtEnd) {
let b = r.advance();
if (b == 0x09) {
let step = 4 - state.column % 4;
if (state.column + step > newColumn) {
this.#partiallyConsumedTab = true;
this.#tabRemainingColumns = state.column + step - newColumn;
state.column = newColumn;
state.index = r.mark();
return;
}
state.column += step;
} else {
state.column += 1;
}
}
state.index = r.mark();
this.#partiallyConsumedTab = false;
this.#tabRemainingColumns = 0;
}
#AdvanceToEnd(state: BlockLineState): void {
let r = new StringReader(state.line);
this.#setNewIndex(state, r.remaining);
}
}
function replaceNullCodePoints(s: String): String {
let r = new StringReader(s);
var hasNull = false;
while (!r.isAtEnd) {
if (r.advance() == 0) {
hasNull = true;
break;
}
}
if (!hasNull) {
return s;
}
r.reset(0);
let sb = new StringBuilder();
while (!r.isAtEnd) {
let cp = r.advance();
sb.appendCodePoint(if (cp == 0) 0xFFFD else cp);
}
return sb.toString();
}
function isBlankString(s: String): boolean {
let r = new StringReader(s);
r.skipSpaces();
return r.isAtEnd;
}
function joinParagraphLines(lines: GrowableArray<String>): String {
if (lines.length == 0) {
return '';
}
if (lines.length == 1) {
return trimSpaces(lines[0]);
}
let sb = new StringBuilder(64);
var i = 0;
while (i < lines.length) {
if (i > 0) {
sb.appendByte(0x0A);
}
// Strip leading spaces on paragraph continuation lines
let r = new StringReader(lines[i]);
r.skipSpaces();
sb.append(r.readToEnd());
i += 1;
}
return sb.toString();
}
/**
* Checks whether `line` at `start` is a Setext heading underline (`===` or `---`).
* Returns `1` for `=`, `2` for `-`, or `0` if not a Setext underline.
*/
function matchSetextUnderline(line: String, start: i32): i32 {
let r = new StringReader(line);
r.reset(start);
let marker = r.peek();
if (marker != 0x3D && marker != 0x2D) {
return 0;
}
while (r.matchCodePoint(marker)) {}
r.skipSpaces();
if (!r.isAtEnd) {
return 0;
}
return if (marker == 0x3D) 1 else 2;
}
/**
* Checks whether `line` at `start` is a CommonMark thematic break (`***`, `---`, `___`).
*/
function isThematicBreak(line: String, start: i32): boolean {
let r = new StringReader(line);
r.reset(start);
let marker = r.peek();
if (marker != 0x2A && marker != 0x2D && marker != 0x5F) {
return false;
}
var count = 0;
while (!r.isAtEnd) {
let b = r.advance();
if (b == marker) {
count += 1;
} else if (!isSpaceOrTab(b)) {
return false;
}
}
return count >= 3;
}
/**
* Attempts to parse a list marker (`-`, `+`, `*`, or `1.`, `1)`) at
* `state.nextNonSpaceIndex`.
* Returns `(matched, ordered, startNumber, delimiterByte, contentIndent, nextColumn)`.
*/
function tryParseListMarker(state: BlockLineState, inParagraph: boolean): inline (boolean, boolean, i32, i32, i32, i32) {
let r = new StringReader(state.line);
let start = state.nextNonSpaceIndex;
r.reset(start);
let b = r.peek();
var ordered = false;
var startNum = 1;
var delim = b;
if (b == 0x2D || b == 0x2B || b == 0x2A) {
ordered = false;
r.advance();
} else if (isAsciiDigit(b)) {
ordered = true;
startNum = 0;
var digits = 0;
while (digits < 9 && isAsciiDigit(r.peek())) {
startNum = startNum * 10 + (r.advance() - 0x30);
digits += 1;
}
let d = r.peek();
if (d != 0x2E && d != 0x29) {
// '.' or ')'
return (false, false, 1, 0, 0, 0);
}
delim = r.advance();
} else {
return (false, false, 1, 0, 0, 0);
}
let afterMarker = r.mark();
// Must be followed by end of line, space, or tab
if (!r.isAtEnd && !isSpaceOrTab(r.peek())) {
return (false, false, 1, 0, 0, 0);
}
let markerCols = afterMarker - start;
let markerEndCol = state.nextNonSpaceColumn + markerCols;
// Compute columns of whitespace after the marker
var col = markerEndCol;
while (!r.isAtEnd) {
let wb = r.peek();
if (wb == 0x20) {
col += 1;
r.advance();
} else if (wb == 0x09) {
col += 4 - col % 4;
r.advance();
} else {
break;
}
}
let blankAfterMarker = r.isAtEnd;
// Rule: interrupting a paragraph requires non-blank item content and, for
// ordered lists, a start number of 1
if (inParagraph) {
if (blankAfterMarker || ordered && startNum != 1) {
return (false, false, 1, 0, 0, 0);
}
}
let spacesAfter = col - markerEndCol;
let spacesConsumed = if (!blankAfterMarker
&& spacesAfter >= 1
&& spacesAfter <= 4) spacesAfter else 1;
let contentIndent = state.indent + markerCols + spacesConsumed;
let nextCol = markerEndCol + (if (blankAfterMarker) 0 else spacesConsumed);
return (true, ordered, startNum, delim, contentIndent, nextCol);
}
/**
* Attempts to parse a link reference definition `[label]: destination "title"`
* at the current position of `r`.
* Returns `(matched, normalizedLabel, destination, title)`.
*/
function tryParseLinkReferenceDefinition(r: StringReader): inline (boolean, String, String, String) {
let saved = r.mark();
r.skipSpaces();
if (r.peek() != 0x5B) {
// '['
r.reset(saved);
return (false, '', '', '');
}
let (labelOk, rawLabel) = scanLinkLabel(r);
if (!labelOk || !r.matchCodePoint(0x3A)) {
// ':'
r.reset(saved);
return (false, '', '', '');
}
let normalizedLabel = normalizeLinkLabel(rawLabel);
if (normalizedLabel.isEmpty) {
r.reset(saved);
return (false, '', '', '');
}
r.skipSpaces();
if (r.matchCodePoint(0x0A)) {
r.skipSpaces();
}
let (destOk, dest) = scanLinkDestination(r);
if (!destOk) {
r.reset(saved);
return (false, '', '', '');
}
let afterDest = r.mark();
r.skipSpaces();
let afterDestSpaces = r.mark();
var lineEndAfterDest = afterDestSpaces;
var validWithoutTitle = false;
if (r.isAtEnd) {
validWithoutTitle = true;
lineEndAfterDest = r.mark();
} else if (r.peek() == 0x0A) {
validWithoutTitle = true;
lineEndAfterDest = afterDestSpaces + 1;
}
// Try optional title on the same line or the next line
r.reset(afterDestSpaces);
if (r.matchCodePoint(0x0A)) {
r.skipSpaces();
}
let titleCandidatePos = r.mark();
if (titleCandidatePos > afterDest && !r.isAtEnd) {
let tb = r.peek();
if (tb == 0x22 || tb == 0x27 || tb == 0x28) {
let (titleOk, parsedTitle) = scanLinkTitle(r);
if (titleOk) {
r.skipSpaces();
if (r.isAtEnd) {
return (true, normalizedLabel, dest, parsedTitle);
}
if (r.matchCodePoint(0x0A)) {
return (true, normalizedLabel, dest, parsedTitle);
}
}
}
}
if (validWithoutTitle) {
r.reset(lineEndAfterDest);
return (true, normalizedLabel, dest, '');
}
r.reset(saved);
return (false, '', '', '');
}
/**
* Detects whether `line` at `start` starts a CommonMark HTML block of type 1..7.
* Returns `0` if none matches.
*/
function detectHtmlBlockStart(line: String, start: i32, inParagraph: boolean): i32 {
let r = new StringReader(line);
r.reset(start);
if (!r.matchCodePoint(0x3C) || r.isAtEnd) {
// '<'
return 0;
}
let second = r.peek();
if (second == 0x21) {
// '<!'
r.advance();
if (r.matchString('--')) {
return 2; // <!--
}
if (r.matchString('[CDATA[')) {
return 5; // <![CDATA[
}
if (isAsciiAlpha(r.peek())) {
return 4; // <!DOCTYPE ...
}
return 0;
}
if (second == 0x3F) {
return 3; // <?
}
let isClose = r.matchCodePoint(0x2F); // '/'
if (!isAsciiAlpha(r.peek())) {
return 0;
}
let nameStart = r.mark();
r.advance();
r.skipWhile((cp) => isAsciiAlphanumeric(cp) || cp == 0x2D);
let nameEnd = r.mark();
let tagName = r.sliceRange(nameStart, nameEnd).asciiLowerCase();
let nextByte = if (r.isAtEnd) 0x20 else r.peek();
// Type 1: <pre, <script, <style, <textarea
if (!isClose
&& (tagName == 'pre'
|| tagName == 'script'
|| tagName == 'style'
|| tagName == 'textarea')
&& (isSpaceOrTab(nextByte) || nextByte == 0x3E)) {
return 1;
}
// Type 6: standard HTML block tags
let afterOk = isSpaceOrTab(nextByte)
|| nextByte == 0x3E
|| r.startsWith('/>');
if (afterOk && isType6HtmlTag(tagName)) {
return 6;
}
// Type 7: complete open or close tag alone on the line (cannot interrupt a paragraph)
if (!inParagraph && afterOk) {
while (!r.isAtEnd && r.peek() != 0x3E) {
r.advance();
}
if (r.matchCodePoint(0x3E)) {
r.skipSpaces();
if (r.isAtEnd) {
return 7;
}
}
}
return 0;
}
function isType6HtmlTag(tag: String): boolean {
return tag == 'address'
|| tag == 'article'
|| tag == 'aside'
|| tag == 'base'
|| tag == 'basefont'
|| tag == 'blockquote'
|| tag == 'body'
|| tag == 'caption'
|| tag == 'center'
|| tag == 'col'
|| tag == 'colgroup'
|| tag == 'dd'
|| tag == 'details'
|| tag == 'dialog'
|| tag == 'dir'
|| tag == 'div'
|| tag == 'dl'
|| tag == 'dt'
|| tag == 'fieldset'
|| tag == 'figcaption'
|| tag == 'figure'
|| tag == 'footer'
|| tag == 'form'
|| tag == 'frame'
|| tag == 'frameset'
|| tag == 'h1'
|| tag == 'h2'
|| tag == 'h3'
|| tag == 'h4'
|| tag == 'h5'
|| tag == 'h6'
|| tag == 'head'
|| tag == 'header'
|| tag == 'hr'
|| tag == 'html'
|| tag == 'iframe'
|| tag == 'legend'
|| tag == 'li'
|| tag == 'link'
|| tag == 'main'
|| tag == 'menu'
|| tag == 'menuitem'
|| tag == 'nav'
|| tag == 'noframes'
|| tag == 'ol'
|| tag == 'optgroup'
|| tag == 'option'
|| tag == 'p'
|| tag == 'param'
|| tag == 'search'
|| tag == 'section'
|| tag == 'summary'
|| tag == 'table'
|| tag == 'tbody'
|| tag == 'td'
|| tag == 'tfoot'
|| tag == 'th'
|| tag == 'thead'
|| tag == 'title'
|| tag == 'tr'
|| tag == 'track'
|| tag == 'ul';
}
function htmlBlockEndsOnLine(line: String, htmlType: i32): boolean {
if (htmlType == 1) {
let lower = line.asciiLowerCase();
return lower.contains('</pre>')
|| lower.contains('</script>')
|| lower.contains('</style>')
|| lower.contains('</textarea>');
}
if (htmlType == 2) {
return line.contains('-->');
}
if (htmlType == 3) {
return line.contains('?>');
}
if (htmlType == 4) {
return line.contains('>');
}
if (htmlType == 5) {
return line.contains(']]>');
}
return false;
}
| Version | Published | Dependencies | Approved by |
|---|---|---|---|
| 0.0.1-preview.1 | 2026-11-19 | none |
|