/** * CommonMark Phase 1 block parser with open-block stack, lazy continuation, * link reference definitions, and `BlockParserFactory` plugin dispatch. */ import {FixedArray, GrowableArray, Option, none} from 'zena:core'; import {StringBuilder, StringReader, isSpaceOrTab, isAsciiAlpha, isAsciiDigit, isAsciiAlphanumeric} from 'zena:string'; import {HashMap} from 'zena:collections'; import {Block, Document, Inline, LinkDef, Span, Paragraph, Heading, BlockQuote, ListBlock, ListItem, FencedCodeBlock, IndentedCodeBlock, HtmlBlock, ThematicBreak, Text} from './ast.zena'; import {AstTransformer, BlockContinue, BlockLineState, BlockParserFactory, CustomBlockParser, CustomBlockStart, InlineParserRunner, MarkdownExtension, MatchedBlockInfo} from './plugin.zena'; import {InlineParser} from './inline-parser.zena'; import {trimSpaces, normalizeLinkLabel, unescapeString, scanLinkDestination, scanLinkTitle, scanLinkLabel} from './scanner.zena'; /** * How many blocks may be open at once, the document included: quotes, * lists and list items nest no deeper. markdown-it's default preset allows * the same depth. */ let MAX_BLOCK_NESTING = 100; enum OpenBlockKind { Doc, Quote, List, Item, Para, HeadingBlock, FencedCode, IndentedCode, Html, BreakBlock, Custom } /** * State for an open block on the Phase 1 stack. */ final class OpenBlock { kind: OpenBlockKind; var startOffset: i32; var endOffset: i32; children = new GrowableArray(4); // Paragraph / code / HTML accumulated lines lines = new GrowableArray(4); lineOffsets = new GrowableArray(4); // Heading var headingLevel: i32 = 1; var headingText: String = ''; var headingTextOffset: i32 = 0; // Fenced code var fenceByte: i32 = 0; var fenceLength: i32 = 0; var fenceIndent: i32 = 0; var fenceInfo: String = ''; var fenceFirstLine: boolean = true; // HTML block (types 1..7) var htmlType: i32 = 0; // List & ListItem var listOrdered: boolean = false; var listStartNumber: i32 = 1; var listDelimiter: i32 = 0; var listTight: boolean = true; listItems = new GrowableArray(4); var listHadBlankLine: boolean = false; var itemContentIndent: i32 = 0; var itemHadBlankLine: boolean = false; var itemLineCount: i32 = 0; // Custom plugin block var customParser: CustomBlockParser | null = null; var builtBlock: Block | null = null; new(this.kind, this.startOffset) : endOffset = startOffset {} } /** * Parses a Markdown document string into a `Document` AST. */ export function parse(source: String, extensions: GrowableArray = new GrowableArray( 0 )): Document { let parser = new DocumentBlockParser(extensions); return parser.parseDocument(source); } /** * Phase 1 + Phase 2 + Phase 3 coordinator. */ export final class DocumentBlockParser { #extensions: GrowableArray; #factoriesByByte = new FixedArray | null>( 128, null ); #wildcardFactories = new GrowableArray(2); #transformers = new GrowableArray(4); #links = new HashMap(); #openBlocks = new GrowableArray(8); #customHolders = new GrowableArray(4); var #partiallyConsumedTab: boolean = false; var #tabRemainingColumns: i32 = 0; var #closingFenceConsumedLine: boolean = false; new(extensions: GrowableArray) : #extensions = extensions { for (let ext in extensions) { for (let factory in ext.blockFactories) { let triggers = factory.triggerBytes(); if (triggers.length == 0) { this.#wildcardFactories.push(factory); } else { for (let b in triggers) { if (b >= 0 && b < 128) { var bucket = this.#factoriesByByte[b]; if (bucket == null) { bucket = new GrowableArray(2); this.#factoriesByByte[b] = bucket; } bucket.push(factory); } } } } for (let transformer in ext.astTransformers) { this.#transformers.push(transformer); } } } parseDocument(source: String): Document { this.#links.clear(); this.#openBlocks.clear(); this.#customHolders.clear(); let docReader = new StringReader(source); let len = docReader.remaining; let root = new OpenBlock(OpenBlockKind.Doc, 0); root.endOffset = len; this.#openBlocks.push(root); while (!docReader.isAtEnd) { let pos = docReader.mark(); let rawLine = docReader.readLine() ?? ''; let lr = new StringReader(rawLine); let lineEnd = pos + lr.remaining; let line = replaceNullCodePoints(rawLine); this.#processLine(source, line, pos, lineEnd); } // Finalize all remaining open blocks from tip down to root while (this.#openBlocks.length > 1) { this.#closeTipBlock(len); } let inlineParser = new InlineParser(this.#links, this.#extensions); this.#parseBlockInlines(root.children, inlineParser); for (let holder in this.#customHolders) { let cp = holder.customParser; if (cp != null) { cp.parseInlines(inlineParser); } } let doc = new Document(root.children, this.#links, new Span(0, len)); for (let transformer in this.#transformers) { transformer.transform(doc, inlineParser); } return doc; } #processLine(source: String, line: String, lineStartOffset: i32, lineEndOffset: i32): void { this.#partiallyConsumedTab = false; this.#tabRemainingColumns = 0; this.#closingFenceConsumedLine = false; let state = new BlockLineState( source, line, lineStartOffset, 0, 0, 0, 0, 0, true ); this.#findNextNonSpace(state); // Step 1: Check continuation of open blocks (starting after Document at index 0) var matchedCount = 1; var i = 1; while (i < this.#openBlocks.length) { let block = this.#openBlocks[i]; if (!this.#continueOpenBlock(block, state)) { break; } block.endOffset = lineEndOffset; matchedCount += 1; i += 1; } let allMatched = matchedCount == this.#openBlocks.length; let lastMatched = this.#openBlocks[matchedCount - 1]; var currentTip = this.#openBlocks[this.#openBlocks.length - 1]; // Step 2: Try to open new child blocks unless the last matched block is a // leaf block that consumes raw text (FencedCode, IndentedCode, Html, Custom) let tryBlockStarts = lastMatched.kind != OpenBlockKind.FencedCode && lastMatched.kind != OpenBlockKind.IndentedCode && lastMatched.kind != OpenBlockKind.Html && lastMatched.kind != OpenBlockKind.Custom; while (tryBlockStarts) { this.#findNextNonSpace(state); if (state.isBlank) { this.#AdvanceToEnd(state); break; } let matchedBlock = this.#openBlocks[matchedCount - 1]; // Past MAX_BLOCK_NESTING, a marker that would open another block is // text: every walk over the tree recurses through the blocks, and a // deeper document would overflow the stack. let started = this.#openBlocks.length <= MAX_BLOCK_NESTING && this.#tryStartNewBlock( state, matchedBlock, currentTip, lineEndOffset ); if (!started) { this.#setNewIndex(state, state.nextNonSpaceIndex); break; } currentTip = this.#openBlocks[this.#openBlocks.length - 1]; matchedCount = this.#openBlocks.length; if (!this.#isContainerOrCustomizable(currentTip)) { break; } } // Step 3: Lazy continuation of an open paragraph if (!allMatched && !state.isBlank && currentTip.kind == OpenBlockKind.Para) { this.#addParagraphLine(currentTip, state, lineEndOffset); return; } // Close unmatched blocks while (this.#openBlocks.length > matchedCount) { this.#closeTipBlock(lineStartOffset); } // If a non-blank line did not start a new list item in an open List block, // close the List block now before adding paragraph text this.#findNextNonSpace(state); while (!state.isBlank && this.#openBlocks.length > 1 && this.#openBlocks[this.#openBlocks.length - 1].kind == OpenBlockKind.List) { this.#closeTipBlock(lineStartOffset); } currentTip = this.#openBlocks[this.#openBlocks.length - 1]; this.#findNextNonSpace(state); // Track blank lines on lists/items for tight vs. loose determination if (state.isBlank && !this.#closingFenceConsumedLine) { if (currentTip.kind == OpenBlockKind.List) { if (currentTip.listItems.length > 0) { currentTip.listHadBlankLine = true; } } else if (currentTip.kind == OpenBlockKind.Item) { if (currentTip.children.length > 0 || currentTip.itemLineCount > 0) { currentTip.itemHadBlankLine = true; } } } // Step 4: Add remaining line text to the active block match (currentTip.kind) { case OpenBlockKind.IndentedCode: { this.#addIndentedCodeLine(currentTip, state, lineEndOffset); } case OpenBlockKind.FencedCode: { this.#addFencedCodeLine(currentTip, state, lineEndOffset); } case OpenBlockKind.Html: { this.#addHtmlBlockLine(currentTip, state, lineEndOffset); } case OpenBlockKind.Para: { if (!state.isBlank) { this.#addParagraphLine(currentTip, state, lineEndOffset); }; } case OpenBlockKind.Custom: { let cp = currentTip.customParser; if (cp != null && cp.acceptsLines() && !state.isBlank) { let r = new StringReader(state.line); r.reset(state.index); cp.addLine(r.readToEnd(), state.lineStartOffset + state.index); currentTip.endOffset = lineEndOffset; }; } case OpenBlockKind.HeadingBlock | OpenBlockKind.BreakBlock: { // Single-line blocks close immediately at the end of the line this.#closeTipBlock(lineEndOffset); } case _: { if (!state.isBlank) { let para = new OpenBlock( OpenBlockKind.Para, state.lineStartOffset + state.nextNonSpaceIndex ); para.endOffset = lineEndOffset; this.#openChildBlock(para, state.lineStartOffset); this.#setNewIndex(state, state.nextNonSpaceIndex); this.#addParagraphLine(para, state, lineEndOffset); }; } }; } #isContainerOrCustomizable(block: OpenBlock): boolean { return block.kind == OpenBlockKind.Doc || block.kind == OpenBlockKind.Quote || block.kind == OpenBlockKind.List || block.kind == OpenBlockKind.Item; } #continueOpenBlock(block: OpenBlock, state: BlockLineState): boolean { this.#findNextNonSpace(state); let r = new StringReader(state.line); match (block.kind) { case OpenBlockKind.Quote: { r.reset(state.nextNonSpaceIndex); if (state.indent <= 3 && r.peek() == 0x3E) { // '>' this.#setNewColumn(state, state.nextNonSpaceColumn + 1); r.reset(state.index); if (isSpaceOrTab(r.peek())) { this.#setNewColumn(state, state.column + 1); }; return true; }; return false; } case OpenBlockKind.List: { return true; } case OpenBlockKind.Item: { if (state.isBlank) { if (block.children.length == 0 && block.itemLineCount == 0) { // An empty list item cannot continue past a blank line return false; }; this.#setNewIndex(state, state.nextNonSpaceIndex); return true; }; if (state.indent >= block.itemContentIndent) { this.#setNewColumn(state, state.column + block.itemContentIndent); return true; }; return false; } case OpenBlockKind.IndentedCode: { if (state.indent >= 4) { this.#setNewColumn(state, state.column + 4); return true; }; if (state.isBlank) { this.#setNewIndex(state, state.nextNonSpaceIndex); return true; }; return false; } case OpenBlockKind.FencedCode: { r.reset(state.nextNonSpaceIndex); if (state.indent <= 3 && r.peek() == block.fenceByte) { var closeLen = 0; while (r.matchCodePoint(block.fenceByte)) { closeLen += 1; } if (closeLen >= block.fenceLength) { r.skipSpaces(); if (r.isAtEnd) { // Closing fence matched: close block and consume line this.#closingFenceConsumedLine = true; this.#closeTipBlock(state.lineStartOffset + r.mark()); this.#AdvanceToEnd(state); return false; }; }; }; // Strip up to `fenceIndent` leading spaces var spaces = block.fenceIndent; while (spaces > 0) { r.reset(state.index); if (r.peek() == 0x20) { this.#setNewIndex(state, state.index + 1); spaces -= 1; } else { break; } } return true; } case OpenBlockKind.Html: { if (state.isBlank && (block.htmlType == 6 || block.htmlType == 7)) { return false; }; return true; } case OpenBlockKind.Para: { return !state.isBlank; } case OpenBlockKind.Custom: { let cp = block.customParser; if (cp == null) { return false; }; let cont = cp.tryContinue(state); if (!cont.matched) { return false; }; if (cont.finalize) { this.#closeTipBlock(state.lineStartOffset + r.remaining); return false; }; if (cont.newIndex >= 0) { this.#setNewIndex(state, cont.newIndex); } else if (cont.newColumn >= 0) { this.#setNewColumn(state, cont.newColumn); }; return true; } case _: { return false; } }; } #tryStartNewBlock(state: BlockLineState, matchedBlock: OpenBlock, currentTip: OpenBlock, lineEndOffset: i32): boolean { let inParagraph = currentTip.kind == OpenBlockKind.Para; let r = new StringReader(state.line); r.reset(state.nextNonSpaceIndex); let b = r.peek(); // 1. Custom `BlockParserFactory` plugins (allowing plugins like GFM tables // to inspect `matchedInfo` and replace an open paragraph before Setext // headings or thematic breaks claim a `---` line) if (this.#tryCustomBlockFactories(state, currentTip, b, lineEndOffset)) { return true; } if (state.indent >= 4) { if (!inParagraph && !state.isBlank) { this.#closeUnmatchedBeforeNewBlock(matchedBlock, state.lineStartOffset); this.#setNewColumn(state, state.column + 4); let codeBlock = new OpenBlock( OpenBlockKind.IndentedCode, state.lineStartOffset + state.index ); codeBlock.endOffset = lineEndOffset; this.#openChildBlock(codeBlock, state.lineStartOffset); return true; } return false; } // 2. BlockQuote (`>`) if (b == 0x3E) { this.#closeUnmatchedBeforeNewBlock(matchedBlock, state.lineStartOffset); let quote = new OpenBlock( OpenBlockKind.Quote, state.lineStartOffset + state.nextNonSpaceIndex ); quote.endOffset = lineEndOffset; this.#openChildBlock(quote, state.lineStartOffset); this.#setNewColumn(state, state.nextNonSpaceColumn + 1); r.reset(state.index); if (isSpaceOrTab(r.peek())) { this.#setNewColumn(state, state.column + 1); } return true; } // 3. ATX Heading (`#`..`######`) if (b == 0x23) { let heading = this.#tryParseAtxHeading(state, lineEndOffset); if (heading != null) { this.#closeUnmatchedBeforeNewBlock(matchedBlock, state.lineStartOffset); this.#openChildBlock(heading, state.lineStartOffset); this.#AdvanceToEnd(state); return true; } } // 4. Fenced Code Block (``` or ~~~) if (b == 0x60 || b == 0x7E) { let fenced = this.#tryParseFencedCode(state, lineEndOffset); if (fenced != null) { this.#closeUnmatchedBeforeNewBlock(matchedBlock, state.lineStartOffset); this.#openChildBlock(fenced, state.lineStartOffset); this.#AdvanceToEnd(state); return true; } } // 5. HTML Block (`<...`) if (b == 0x3C) { let htmlType = detectHtmlBlockStart( state.line, state.nextNonSpaceIndex, inParagraph ); if (htmlType > 0) { this.#closeUnmatchedBeforeNewBlock(matchedBlock, state.lineStartOffset); let htmlBlock = new OpenBlock( OpenBlockKind.Html, state.lineStartOffset + state.nextNonSpaceIndex ); htmlBlock.htmlType = htmlType; htmlBlock.endOffset = lineEndOffset; this.#openChildBlock(htmlBlock, state.lineStartOffset); return true; } } // 6. Setext Heading underline (`===` or `---`) when tip is an open Paragraph if (inParagraph && (b == 0x3D || b == 0x2D)) { let level = matchSetextUnderline(state.line, state.nextNonSpaceIndex); if (level > 0) { // Strip leading link reference definitions from the paragraph first; // only become a Setext heading if non-blank paragraph text remains! this.#extractLinkDefinitionsFromParagraph(currentTip); if (currentTip.lines.length > 0) { let paraStart = currentTip.startOffset; let text = joinParagraphLines(currentTip.lines); // Close unmatched up to and including the open paragraph while (this.#openBlocks.length > 1) { let top = this.#openBlocks[this.#openBlocks.length - 1]; this.#openBlocks.pop(); if (top == currentTip) { break; } } let setext = new OpenBlock(OpenBlockKind.HeadingBlock, paraStart); setext.headingLevel = level; setext.headingText = text; setext.headingTextOffset = paraStart; setext.endOffset = lineEndOffset; this.#openChildBlock(setext, paraStart); this.#AdvanceToEnd(state); return true; } } } // 7. Thematic Break (`***`, `---`, `___`) if (b == 0x2A || b == 0x2D || b == 0x5F) { if (isThematicBreak(state.line, state.nextNonSpaceIndex)) { this.#closeUnmatchedBeforeNewBlock(matchedBlock, state.lineStartOffset); let hr = new OpenBlock( OpenBlockKind.BreakBlock, state.lineStartOffset + state.nextNonSpaceIndex ); hr.endOffset = lineEndOffset; this.#openChildBlock(hr, state.lineStartOffset); this.#AdvanceToEnd(state); return true; } } // 8. List Item (`-`, `+`, `*`, or `1.`, `1)`) if (b == 0x2D || b == 0x2B || b == 0x2A || isAsciiDigit(b)) { let inParaAtCurrentLevel = matchedBlock.kind == OpenBlockKind.Para; let (listOk, ordered, startNum, delim, contentIndent, nextCol) = tryParseListMarker( state, inParaAtCurrentLevel ); if (listOk) { this.#closeUnmatchedBeforeNewBlock( matchedBlock, state.lineStartOffset, true ); this.#setNewColumn(state, nextCol); // Ensure an open `ListBlock` with matching type/delimiter is on top let top = this.#openBlocks[this.#openBlocks.length - 1]; if (top.kind != OpenBlockKind.List || top.listOrdered != ordered || top.listDelimiter != delim) { if (top.kind == OpenBlockKind.List) { this.#closeTipBlock(state.lineStartOffset); } let listBlock = new OpenBlock( OpenBlockKind.List, state.lineStartOffset + state.nextNonSpaceIndex ); listBlock.listOrdered = ordered; listBlock.listStartNumber = startNum; listBlock.listDelimiter = delim; listBlock.endOffset = lineEndOffset; this.#openChildBlock(listBlock, state.lineStartOffset); } let listParent = this.#openBlocks[this.#openBlocks.length - 1]; if (listParent.listHadBlankLine && listParent.listItems.length > 0) { listParent.listTight = false; listParent.listHadBlankLine = false; } let item = new OpenBlock( OpenBlockKind.Item, state.lineStartOffset + state.nextNonSpaceIndex ); item.itemContentIndent = contentIndent; item.endOffset = lineEndOffset; this.#openChildBlock(item, state.lineStartOffset); return true; } } return false; } #tryCustomBlockFactories(state: BlockLineState, currentTip: OpenBlock, firstByte: i32, lineEndOffset: i32): boolean { var matchedInfo: MatchedBlockInfo | null = null; if (firstByte >= 0 && firstByte < 128) { let bucket = this.#factoriesByByte[firstByte]; if (bucket != null) { matchedInfo = this.#buildMatchedBlockInfo(currentTip); for (let factory in bucket) { let res = factory.tryStart(state, matchedInfo); if (res != null) { this.#applyCustomBlockStart(res, state, currentTip, lineEndOffset); return true; } } } } if (this.#wildcardFactories.length > 0) { if (matchedInfo == null) { matchedInfo = this.#buildMatchedBlockInfo(currentTip); } for (let factory in this.#wildcardFactories) { let res = factory.tryStart(state, matchedInfo); if (res != null) { this.#applyCustomBlockStart(res, state, currentTip, lineEndOffset); return true; } } } return false; } #buildMatchedBlockInfo(currentTip: OpenBlock): MatchedBlockInfo { if (currentTip.kind == OpenBlockKind.Para) { this.#extractLinkDefinitionsFromParagraph(currentTip); if (currentTip.lines.length > 0) { return new MatchedBlockInfo( true, currentTip.lines, currentTip.startOffset ); } } return new MatchedBlockInfo(false, null, 0); } #applyCustomBlockStart(start: CustomBlockStart, state: BlockLineState, currentTip: OpenBlock, lineEndOffset: i32): void { var startOff = state.lineStartOffset + state.nextNonSpaceIndex; if (start.replaceParagraph && currentTip.kind == OpenBlockKind.Para) { if (currentTip.lines.length > 1) { let lastIdx = currentTip.lines.length - 1; currentTip.lines.pop(); if (currentTip.lineOffsets.length > lastIdx) { startOff = currentTip.lineOffsets.pop(); } else { startOff = state.lineStartOffset; } currentTip.endOffset = startOff; this.#closeTipBlock(startOff); } else { startOff = currentTip.startOffset; this.#openBlocks.pop(); } } let customBlock = new OpenBlock(OpenBlockKind.Custom, startOff); customBlock.customParser = start.parser; customBlock.endOffset = lineEndOffset; this.#openChildBlock(customBlock, state.lineStartOffset); this.#customHolders.push(customBlock); if (start.newIndex >= 0) { this.#setNewIndex(state, start.newIndex); } else if (start.newColumn >= 0) { this.#setNewColumn(state, start.newColumn); } } #closeUnmatchedBeforeNewBlock(matchedBlock: OpenBlock, closeOffset: i32, startingListItem: boolean = false): void { while (this.#openBlocks.length > 1) { let top = this.#openBlocks[this.#openBlocks.length - 1]; if (top == matchedBlock) { if (top.kind == OpenBlockKind.Para) { this.#closeTipBlock(closeOffset); } else if (top.kind == OpenBlockKind.List && !startingListItem) { this.#closeTipBlock(closeOffset); } break; } this.#closeTipBlock(closeOffset); } } #openChildBlock(child: OpenBlock, offset: i32): void { let parent = this.#openBlocks[this.#openBlocks.length - 1]; if (parent.kind == OpenBlockKind.Item && parent.itemHadBlankLine) { parent.itemHadBlankLine = false; // A blank line directly between two blocks in a list item makes the // enclosing list loose if (this.#openBlocks.length >= 2) { let maybeList = this.#openBlocks[this.#openBlocks.length - 2]; if (maybeList.kind == OpenBlockKind.List) { maybeList.listTight = false; } } } this.#openBlocks.push(child); } #closeTipBlock(closeOffset: i32): void { let tip = this.#openBlocks.pop(); let parent = this.#openBlocks[this.#openBlocks.length - 1]; if (tip.endOffset > parent.endOffset && parent.kind != OpenBlockKind.Doc) { parent.endOffset = tip.endOffset; } let span = new Span(tip.startOffset, tip.endOffset); match (tip.kind) { case OpenBlockKind.Para: { this.#extractLinkDefinitionsFromParagraph(tip); if (tip.lines.length > 0) { let rawText = joinParagraphLines(tip.lines); // Store rawText temporarily in a single Text inline; Phase 2 replaces // `inlines` with the parsed inline nodes let inlines = new GrowableArray(1); inlines.push(new Text(rawText, span)); parent.children.push(new Paragraph(inlines, null, span)); }; } case OpenBlockKind.HeadingBlock: { let inlines = new GrowableArray(1); inlines.push( new Text( tip.headingText, new Span(tip.headingTextOffset, tip.endOffset) ) ); parent.children.push( new Heading(tip.headingLevel, inlines, null, span) ); } case OpenBlockKind.Quote: { parent.children.push(new BlockQuote(tip.children, span)); } case OpenBlockKind.List: { if (tip.listHadBlankLine) { if (parent.kind == OpenBlockKind.Item) { parent.itemHadBlankLine = true; } else if (parent.kind == OpenBlockKind.List) { parent.listHadBlankLine = true; }; }; parent.children.push( new ListBlock( tip.listOrdered, tip.listStartNumber, tip.listDelimiter, tip.listTight, tip.listItems, span ) ); } case OpenBlockKind.Item: { let itemNode = new ListItem(tip.children, none, span); if (parent.kind == OpenBlockKind.List) { if (tip.itemHadBlankLine) { parent.listHadBlankLine = true; }; parent.listItems.push(itemNode); } else { parent.children.push(itemNode); }; } case OpenBlockKind.FencedCode: { let sb = new StringBuilder(64); for (let line in tip.lines) { sb.append(line); sb.appendByte(0x0A); } parent.children.push( new FencedCodeBlock(tip.fenceInfo, sb.toString(), span) ); } case OpenBlockKind.IndentedCode: { // Strip leading and trailing blank lines per CommonMark 4.4 var start = 0; var end = tip.lines.length; while (start < end && isBlankString(tip.lines[start])) { start += 1; } while (end > start && isBlankString(tip.lines[end - 1])) { end -= 1; } if (end < tip.lines.length) { if (parent.kind == OpenBlockKind.Item) { parent.itemHadBlankLine = true; } else if (parent.kind == OpenBlockKind.List) { parent.listHadBlankLine = true; }; }; let sb = new StringBuilder(64); var i = start; while (i < end) { sb.append(tip.lines[i]); sb.appendByte(0x0A); i += 1; } parent.children.push(new IndentedCodeBlock(sb.toString(), span)); } case OpenBlockKind.Html: { let sb = new StringBuilder(64); var i = 0; while (i < tip.lines.length) { if (i > 0) { sb.appendByte(0x0A); } sb.append(tip.lines[i]); i += 1; } parent.children.push(new HtmlBlock(sb.toString(), span)); } case OpenBlockKind.BreakBlock: { parent.children.push(new ThematicBreak(span)); } case OpenBlockKind.Custom: { let cp = tip.customParser; if (cp != null) { let built = cp.closeBlock(tip.endOffset); tip.builtBlock = built; parent.children.push(built); }; } case OpenBlockKind.Doc: {} }; } #addParagraphLine(para: OpenBlock, state: BlockLineState, lineEndOffset: i32): void { let r = new StringReader(state.line); r.reset(state.nextNonSpaceIndex); para.lines.push(r.readToEnd()); para.lineOffsets.push(state.lineStartOffset + state.nextNonSpaceIndex); para.endOffset = lineEndOffset; this.#markEnclosingItemHasLine(); } #addIndentedCodeLine(code: OpenBlock, state: BlockLineState, lineEndOffset: i32): void { let content = this.#extractRemainingWithTabSpaces(state); code.lines.push(content); code.endOffset = lineEndOffset; this.#markEnclosingItemHasLine(); } #addFencedCodeLine(code: OpenBlock, state: BlockLineState, lineEndOffset: i32): void { if (code.fenceFirstLine) { code.fenceFirstLine = false; return; } let content = this.#extractRemainingWithTabSpaces(state); code.lines.push(content); code.endOffset = lineEndOffset; this.#markEnclosingItemHasLine(); } #addHtmlBlockLine(html: OpenBlock, state: BlockLineState, lineEndOffset: i32): void { let content = this.#extractRemainingWithTabSpaces(state); html.lines.push(content); html.endOffset = lineEndOffset; this.#markEnclosingItemHasLine(); if (htmlBlockEndsOnLine(content, html.htmlType)) { this.#closeTipBlock(lineEndOffset); } } #markEnclosingItemHasLine(): void { var i = this.#openBlocks.length - 1; while (i >= 0) { let b = this.#openBlocks[i]; if (b.kind == OpenBlockKind.Item) { b.itemLineCount += 1; return; } i -= 1; } } #extractRemainingWithTabSpaces(state: BlockLineState): String { let r = new StringReader(state.line); r.reset(state.index); let rest = r.readToEnd(); if (!this.#partiallyConsumedTab || this.#tabRemainingColumns <= 0) { return rest; } let sb = new StringBuilder(); var s = 0; while (s < this.#tabRemainingColumns) { sb.appendByte(0x20); s += 1; } sb.append(rest); return sb.toString(); } #extractLinkDefinitionsFromParagraph(para: OpenBlock): void { if (para.lines.length == 0) { return; } let combined = joinParagraphLines(para.lines); let r = new StringReader(combined); var consumedAny = false; while (!r.isAtEnd) { let defStart = r.mark(); let (ok, label, dest, title) = tryParseLinkReferenceDefinition(r); if (!ok) { r.reset(defStart); break; } consumedAny = true; let nextPos = r.mark(); if (!this.#links.has(label)) { this.#links[label] = new LinkDef( label, dest, title, new Span(para.startOffset + defStart, para.startOffset + nextPos) ); } } if (!consumedAny) { return; } let pos = r.mark(); para.lines.clear(); if (!r.isAtEnd) { // Split remaining lines back into `para.lines` while (!r.isAtEnd) { let line = r.readLine(); if (line != null) { para.lines.push(line); } } para.startOffset += pos; } } #parseBlockInlines(blocks: GrowableArray, inlineParser: InlineParserRunner): void { for (let block in blocks) { match (block) { case Paragraph as p: { if (p.inlines.length == 1) { if (let Text {literal, span} = p.inlines[0]) { p.inlines = inlineParser.parse(literal, span.start); }; }; } case Heading as h: { if (h.inlines.length == 1) { if (let Text {literal, span} = h.inlines[0]) { h.inlines = inlineParser.parse(literal, span.start); }; }; } case BlockQuote {blocks: let children}: { this.#parseBlockInlines(children, inlineParser); } case ListBlock {items}: { for (let item in items) { this.#parseBlockInlines(item.blocks, inlineParser); } } case ListItem {blocks: let children}: { this.#parseBlockInlines(children, inlineParser); } case _: {} }; } } #tryParseAtxHeading(state: BlockLineState, lineEndOffset: i32): OpenBlock | null { let r = new StringReader(state.line); let start = state.nextNonSpaceIndex; r.reset(start); var level = 0; while (level < 7 && r.matchCodePoint(0x23)) { level += 1; } if (level == 0 || level > 6) { return null; } if (!r.isAtEnd && !isSpaceOrTab(r.peek())) { return null; } // Trim leading spaces/tabs and optional closing `#` sequence r.skipSpaces(); let contentStart = r.mark(); var contentEnd = contentStart + r.remaining; while (contentEnd > contentStart) { r.reset(contentEnd); if (isSpaceOrTab(r.peekBefore())) { contentEnd -= 1; } else { break; } } var hashEnd = contentEnd; while (hashEnd > contentStart) { r.reset(hashEnd); if (r.peekBefore() == 0x23) { hashEnd -= 1; } else { break; } } if (hashEnd < contentEnd) { r.reset(hashEnd); if (hashEnd == contentStart || isSpaceOrTab(r.peekBefore())) { contentEnd = hashEnd; while (contentEnd > contentStart) { r.reset(contentEnd); if (isSpaceOrTab(r.peekBefore())) { contentEnd -= 1; } else { break; } } } } let h = new OpenBlock( OpenBlockKind.HeadingBlock, state.lineStartOffset + start ); h.headingLevel = level; h.headingText = r.sliceRange(contentStart, contentEnd); h.headingTextOffset = state.lineStartOffset + contentStart; h.endOffset = lineEndOffset; return h; } #tryParseFencedCode(state: BlockLineState, lineEndOffset: i32): OpenBlock | null { let r = new StringReader(state.line); let start = state.nextNonSpaceIndex; r.reset(start); let fenceByte = r.peek(); var fenceLen = 0; while (r.matchCodePoint(fenceByte)) { fenceLen += 1; } if (fenceLen < 3) { return null; } let rest = r.readToEnd(); // Backtick fence info string cannot contain backticks if (fenceByte == 0x60 && rest.contains('`')) { return null; } let rawInfo = trimSpaces(rest); let block = new OpenBlock( OpenBlockKind.FencedCode, state.lineStartOffset + start ); block.fenceByte = fenceByte; block.fenceLength = fenceLen; block.fenceIndent = state.indent; block.fenceInfo = unescapeString(rawInfo); block.fenceFirstLine = true; block.endOffset = lineEndOffset; return block; } #findNextNonSpace(state: BlockLineState): void { let r = new StringReader(state.line); r.reset(state.index); var cols = state.column; if (this.#partiallyConsumedTab) { cols += this.#tabRemainingColumns; } while (!r.isAtEnd) { let b = r.peek(); if (b == 0x20) { r.advance(); cols += 1; } else if (b == 0x09) { r.advance(); cols += 4 - cols % 4; } else { break; } } state.nextNonSpaceIndex = r.mark(); state.nextNonSpaceColumn = cols; state.indent = cols - state.column; state.isBlank = r.isAtEnd; } #setNewIndex(state: BlockLineState, newIndex: i32): void { if (newIndex >= state.nextNonSpaceIndex) { state.index = state.nextNonSpaceIndex; state.column = state.nextNonSpaceColumn; } let r = new StringReader(state.line); r.reset(state.index); while (r.mark() < newIndex && !r.isAtEnd) { let b = r.advance(); if (b == 0x09) { state.column += 4 - state.column % 4; } else { state.column += 1; } } state.index = r.mark(); this.#partiallyConsumedTab = false; this.#tabRemainingColumns = 0; } #setNewColumn(state: BlockLineState, newColumn: i32): void { if (newColumn >= state.nextNonSpaceColumn) { state.index = state.nextNonSpaceIndex; state.column = state.nextNonSpaceColumn; } let r = new StringReader(state.line); r.reset(state.index); while (state.column < newColumn && !r.isAtEnd) { let b = r.advance(); if (b == 0x09) { let step = 4 - state.column % 4; if (state.column + step > newColumn) { this.#partiallyConsumedTab = true; this.#tabRemainingColumns = state.column + step - newColumn; state.column = newColumn; state.index = r.mark(); return; } state.column += step; } else { state.column += 1; } } state.index = r.mark(); this.#partiallyConsumedTab = false; this.#tabRemainingColumns = 0; } #AdvanceToEnd(state: BlockLineState): void { let r = new StringReader(state.line); this.#setNewIndex(state, r.remaining); } } function replaceNullCodePoints(s: String): String { let r = new StringReader(s); var hasNull = false; while (!r.isAtEnd) { if (r.advance() == 0) { hasNull = true; break; } } if (!hasNull) { return s; } r.reset(0); let sb = new StringBuilder(); while (!r.isAtEnd) { let cp = r.advance(); sb.appendCodePoint(if (cp == 0) 0xFFFD else cp); } return sb.toString(); } function isBlankString(s: String): boolean { let r = new StringReader(s); r.skipSpaces(); return r.isAtEnd; } function joinParagraphLines(lines: GrowableArray): String { if (lines.length == 0) { return ''; } if (lines.length == 1) { return trimSpaces(lines[0]); } let sb = new StringBuilder(64); var i = 0; while (i < lines.length) { if (i > 0) { sb.appendByte(0x0A); } // Strip leading spaces on paragraph continuation lines let r = new StringReader(lines[i]); r.skipSpaces(); sb.append(r.readToEnd()); i += 1; } return sb.toString(); } /** * Checks whether `line` at `start` is a Setext heading underline (`===` or `---`). * Returns `1` for `=`, `2` for `-`, or `0` if not a Setext underline. */ function matchSetextUnderline(line: String, start: i32): i32 { let r = new StringReader(line); r.reset(start); let marker = r.peek(); if (marker != 0x3D && marker != 0x2D) { return 0; } while (r.matchCodePoint(marker)) {} r.skipSpaces(); if (!r.isAtEnd) { return 0; } return if (marker == 0x3D) 1 else 2; } /** * Checks whether `line` at `start` is a CommonMark thematic break (`***`, `---`, `___`). */ function isThematicBreak(line: String, start: i32): boolean { let r = new StringReader(line); r.reset(start); let marker = r.peek(); if (marker != 0x2A && marker != 0x2D && marker != 0x5F) { return false; } var count = 0; while (!r.isAtEnd) { let b = r.advance(); if (b == marker) { count += 1; } else if (!isSpaceOrTab(b)) { return false; } } return count >= 3; } /** * Attempts to parse a list marker (`-`, `+`, `*`, or `1.`, `1)`) at * `state.nextNonSpaceIndex`. * Returns `(matched, ordered, startNumber, delimiterByte, contentIndent, nextColumn)`. */ function tryParseListMarker(state: BlockLineState, inParagraph: boolean): inline (boolean, boolean, i32, i32, i32, i32) { let r = new StringReader(state.line); let start = state.nextNonSpaceIndex; r.reset(start); let b = r.peek(); var ordered = false; var startNum = 1; var delim = b; if (b == 0x2D || b == 0x2B || b == 0x2A) { ordered = false; r.advance(); } else if (isAsciiDigit(b)) { ordered = true; startNum = 0; var digits = 0; while (digits < 9 && isAsciiDigit(r.peek())) { startNum = startNum * 10 + (r.advance() - 0x30); digits += 1; } let d = r.peek(); if (d != 0x2E && d != 0x29) { // '.' or ')' return (false, false, 1, 0, 0, 0); } delim = r.advance(); } else { return (false, false, 1, 0, 0, 0); } let afterMarker = r.mark(); // Must be followed by end of line, space, or tab if (!r.isAtEnd && !isSpaceOrTab(r.peek())) { return (false, false, 1, 0, 0, 0); } let markerCols = afterMarker - start; let markerEndCol = state.nextNonSpaceColumn + markerCols; // Compute columns of whitespace after the marker var col = markerEndCol; while (!r.isAtEnd) { let wb = r.peek(); if (wb == 0x20) { col += 1; r.advance(); } else if (wb == 0x09) { col += 4 - col % 4; r.advance(); } else { break; } } let blankAfterMarker = r.isAtEnd; // Rule: interrupting a paragraph requires non-blank item content and, for // ordered lists, a start number of 1 if (inParagraph) { if (blankAfterMarker || ordered && startNum != 1) { return (false, false, 1, 0, 0, 0); } } let spacesAfter = col - markerEndCol; let spacesConsumed = if (!blankAfterMarker && spacesAfter >= 1 && spacesAfter <= 4) spacesAfter else 1; let contentIndent = state.indent + markerCols + spacesConsumed; let nextCol = markerEndCol + (if (blankAfterMarker) 0 else spacesConsumed); return (true, ordered, startNum, delim, contentIndent, nextCol); } /** * Attempts to parse a link reference definition `[label]: destination "title"` * at the current position of `r`. * Returns `(matched, normalizedLabel, destination, title)`. */ function tryParseLinkReferenceDefinition(r: StringReader): inline (boolean, String, String, String) { let saved = r.mark(); r.skipSpaces(); if (r.peek() != 0x5B) { // '[' r.reset(saved); return (false, '', '', ''); } let (labelOk, rawLabel) = scanLinkLabel(r); if (!labelOk || !r.matchCodePoint(0x3A)) { // ':' r.reset(saved); return (false, '', '', ''); } let normalizedLabel = normalizeLinkLabel(rawLabel); if (normalizedLabel.isEmpty) { r.reset(saved); return (false, '', '', ''); } r.skipSpaces(); if (r.matchCodePoint(0x0A)) { r.skipSpaces(); } let (destOk, dest) = scanLinkDestination(r); if (!destOk) { r.reset(saved); return (false, '', '', ''); } let afterDest = r.mark(); r.skipSpaces(); let afterDestSpaces = r.mark(); var lineEndAfterDest = afterDestSpaces; var validWithoutTitle = false; if (r.isAtEnd) { validWithoutTitle = true; lineEndAfterDest = r.mark(); } else if (r.peek() == 0x0A) { validWithoutTitle = true; lineEndAfterDest = afterDestSpaces + 1; } // Try optional title on the same line or the next line r.reset(afterDestSpaces); if (r.matchCodePoint(0x0A)) { r.skipSpaces(); } let titleCandidatePos = r.mark(); if (titleCandidatePos > afterDest && !r.isAtEnd) { let tb = r.peek(); if (tb == 0x22 || tb == 0x27 || tb == 0x28) { let (titleOk, parsedTitle) = scanLinkTitle(r); if (titleOk) { r.skipSpaces(); if (r.isAtEnd) { return (true, normalizedLabel, dest, parsedTitle); } if (r.matchCodePoint(0x0A)) { return (true, normalizedLabel, dest, parsedTitle); } } } } if (validWithoutTitle) { r.reset(lineEndAfterDest); return (true, normalizedLabel, dest, ''); } r.reset(saved); return (false, '', '', ''); } /** * Detects whether `line` at `start` starts a CommonMark HTML block of type 1..7. * Returns `0` if none matches. */ function detectHtmlBlockStart(line: String, start: i32, inParagraph: boolean): i32 { let r = new StringReader(line); r.reset(start); if (!r.matchCodePoint(0x3C) || r.isAtEnd) { // '<' return 0; } let second = r.peek(); if (second == 0x21) { // ' isAsciiAlphanumeric(cp) || cp == 0x2D); let nameEnd = r.mark(); let tagName = r.sliceRange(nameStart, nameEnd).asciiLowerCase(); let nextByte = if (r.isAtEnd) 0x20 else r.peek(); // Type 1: '); if (afterOk && isType6HtmlTag(tagName)) { return 6; } // Type 7: complete open or close tag alone on the line (cannot interrupt a paragraph) if (!inParagraph && afterOk) { while (!r.isAtEnd && r.peek() != 0x3E) { r.advance(); } if (r.matchCodePoint(0x3E)) { r.skipSpaces(); if (r.isAtEnd) { return 7; } } } return 0; } function isType6HtmlTag(tag: String): boolean { return tag == 'address' || tag == 'article' || tag == 'aside' || tag == 'base' || tag == 'basefont' || tag == 'blockquote' || tag == 'body' || tag == 'caption' || tag == 'center' || tag == 'col' || tag == 'colgroup' || tag == 'dd' || tag == 'details' || tag == 'dialog' || tag == 'dir' || tag == 'div' || tag == 'dl' || tag == 'dt' || tag == 'fieldset' || tag == 'figcaption' || tag == 'figure' || tag == 'footer' || tag == 'form' || tag == 'frame' || tag == 'frameset' || tag == 'h1' || tag == 'h2' || tag == 'h3' || tag == 'h4' || tag == 'h5' || tag == 'h6' || tag == 'head' || tag == 'header' || tag == 'hr' || tag == 'html' || tag == 'iframe' || tag == 'legend' || tag == 'li' || tag == 'link' || tag == 'main' || tag == 'menu' || tag == 'menuitem' || tag == 'nav' || tag == 'noframes' || tag == 'ol' || tag == 'optgroup' || tag == 'option' || tag == 'p' || tag == 'param' || tag == 'search' || tag == 'section' || tag == 'summary' || tag == 'table' || tag == 'tbody' || tag == 'td' || tag == 'tfoot' || tag == 'th' || tag == 'thead' || tag == 'title' || tag == 'tr' || tag == 'track' || tag == 'ul'; } function htmlBlockEndsOnLine(line: String, htmlType: i32): boolean { if (htmlType == 1) { let lower = line.asciiLowerCase(); return lower.contains('') || lower.contains('') || lower.contains('') || lower.contains(''); } if (htmlType == 2) { return line.contains('-->'); } if (htmlType == 3) { return line.contains('?>'); } if (htmlType == 4) { return line.contains('>'); } if (htmlType == 5) { return line.contains(']]>'); } return false; }