markdown 0.0.1-preview.1

CommonMark and GFM Markdown parser, AST, plugin system, and HTML renderer

Readme Files Dependencies Versions

Markdown (@zena-lang/markdown)#

A CommonMark and GitHub Flavored Markdown (GFM) parser, syntax tree, plugin system, and HTML renderer written in Zena (zena-lang:markdown/<library>). See docs/design/markdown.md for the design.

Usage#

Add it with zena pkg add zena-lang:markdown, then import its main library:

import {
  parse, renderHtml, markdownToHtml, gfmExtensions
} from 'zena-lang:markdown';

// Render CommonMark + GFM to safe HTML (raw HTML stripped, unsafe URLs blocked)
let html = markdownToHtml('# Hello\n\nSome ~~old~~ **new** text.', {
  extensions: gfmExtensions(),
  safe: true,
});

// Or parse to an AST first
let doc = parse('# Heading\n\nParagraph.', gfmExtensions());
let out = renderHtml(doc, {safe: true, extensions: gfmExtensions()});

Running Tests#

npm test -w @zena-lang/markdown
npm run test:quick -- packages/markdown/test
9.3 KiB Raw
/**
 * GFM extended autolink extension (`http://...`, `https://...`, `www....`).
 */

import {Array, GrowableArray} from 'zena:core';
import {StringReader, isAsciiAlphanumeric} from 'zena:string';
import {Autolink, Block, BlockQuote, Document, Emphasis, Heading, Inline, ListBlock, ListItem, Paragraph, Span, Strong, Text} from '../ast.zena';
import {AstTransformer, BlockParserFactory, DelimiterProcessor, InlineContentParser, InlineMatch, InlineParserRunner, MarkdownExtension, NodeRenderer} from '../plugin.zena';

final class GfmAutolinkParser implements InlineContentParser {
  triggerBytes(): Array<i32> {
    return [0x68, 0x48, 0x77, 0x57]; // 'h', 'H', 'w', 'W'
  }

  tryParse(source: String, baseOffset: i32, index: i32, precedingByte: i32, inlineParser: InlineParserRunner): InlineMatch | null {
    // Do not trigger in the middle of an identifier or path
    if (precedingByte >= 0
      && (isAsciiAlphanumeric(precedingByte)
        || precedingByte == 0x5F
        || precedingByte == 0x2F
        || precedingByte == 0x2E)) {
      return null;
    }

    let r = new StringReader(source);
    r.reset(index);
    var isWww = false;

    if (r.startsWithIgnoreAsciiCase('https://')) {
      r.skip(8);
    } else if (r.startsWithIgnoreAsciiCase('http://')) {
      r.skip(7);
    } else if (r.startsWithIgnoreAsciiCase('www.')) {
      r.skip(4);
      isWww = true;
    } else {
      return null;
    }

    let domainStart = r.mark();

    // Validate domain: non-empty, contains at least one '.', valid domain chars
    if (!scanAutolinkDomain(r, isWww)) {
      return null;
    }

    // Scan path/query/fragment up to whitespace or `<`
    while (!r.isAtEnd) {
      let cp = r.peek();
      if (cp <= 0x20 || cp == 0x3C || cp == 0x7F) {
        break;
      }
      r.advance();
    }

    // Apply GFM extended autolink trailing punctuation trimming
    let end = trimAutolinkTrailing(r, index, r.mark());
    if (end <= domainStart) {
      return null;
    }

    let label = r.sliceRange(index, end);
    let destination = if (isWww) `http://${label}` else label;
    let node = new Autolink(
      destination,
      label,
      false,
      new Span(baseOffset + index, baseOffset + end)
    );
    return new InlineMatch(node, end);
  }
}

function scanAutolinkDomain(r: StringReader, requireDot: boolean): boolean {
  let start = r.mark();
  var hasDot = false;
  var segLen = 0;
  var lastDotStart = -1;
  while (!r.isAtEnd) {
    let cp = r.peek();
    if (isAsciiAlphanumeric(cp) || cp == 0x2D || cp == 0x5F) {
      segLen += 1;
      r.advance();
    } else if (cp == 0x2E) {
      // '.'
      if (segLen == 0) {
        break;
      }
      hasDot = true;
      segLen = 0;
      lastDotStart = r.mark();
      r.advance();
    } else {
      break;
    }
  }
  if (segLen == 0 && lastDotStart >= start) {
    r.reset(lastDotStart);
  }
  return r.mark() != start && (!requireDot || hasDot);
}

function trimAutolinkTrailing(r: StringReader, start: i32, initialEnd: i32): i32 {
  var end = initialEnd;
  while (end > start) {
    r.reset(end);
    let last = r.peekBefore();
    // Always trim `? ! . , : * _ ~`
    if (last == 0x3F
      || last == 0x21
      || last == 0x2E
      || last == 0x2C
      || last == 0x3A
      || last == 0x2A
      || last == 0x5F
      || last == 0x7E) {
      end -= 1;
      continue;
    }
    // Trim `)` when closing parens outnumber opening parens
    if (last == 0x29) {
      var openCount = 0;
      var closeCount = 0;
      r.reset(start);
      while (r.mark() < end) {
        let cp = r.advance();
        if (cp == 0x28) {
          openCount += 1;
        } else if (cp == 0x29) {
          closeCount += 1;
        }
      }
      if (closeCount > openCount) {
        end -= 1;
        continue;
      }
      break;
    }
    // Trim `;` and preceding `&alnum+` entity reference
    if (last == 0x3B) {
      var k = end - 1;
      while (k > start) {
        r.reset(k);
        if (isAsciiAlphanumeric(r.peekBefore())) {
          k -= 1;
        } else {
          break;
        }
      }
      r.reset(k);
      if (k > start && k < end - 1 && r.peekBefore() == 0x26) {
        end = k - 1;
        continue;
      }
      end -= 1;
      continue;
    }
    break;
  }
  return end;
}

final class AutolinkEmailTransformer implements AstTransformer {
  transform(doc: Document, inlineParser: InlineParserRunner): void {
    this.#visitBlocks(doc.blocks);
  }

  #visitBlocks(blocks: GrowableArray<Block>): void {
    for (let block in blocks) {
      match (block) {
        case Paragraph as p: {
          p.inlines = this.#processInlines(p.inlines);
        }
        case Heading as h: {
          h.inlines = this.#processInlines(h.inlines);
        }
        case BlockQuote {blocks: let children}: {
          this.#visitBlocks(children);
        }
        case ListBlock {items}: {
          for (let item in items) {
            this.#visitBlocks(item.blocks);
          }
        }
        case ListItem {blocks: let children}: {
          this.#visitBlocks(children);
        }
        case _: {}
      };
    }
  }

  #processInlines(inlines: GrowableArray<Inline>): GrowableArray<Inline> {
    var hasAt = false;
    for (let node in inlines) {
      if (let Text {literal} = node) {
        if (literal.contains('@')) {
          hasAt = true;
          break;
        }
      } else if (node is Emphasis || node is Strong) {
        hasAt = true;
        break;
      }
    }
    if (!hasAt) {
      return inlines;
    }

    let out = new GrowableArray<Inline>(inlines.length);
    for (let node in inlines) {
      match (node) {
        case Text {literal, span}: {
          this.#splitEmailsInText(literal, span.start, out);
        }
        case Emphasis as em: {
          em.children = this.#processInlines(em.children);
          out.push(em);
        }
        case Strong as st: {
          st.children = this.#processInlines(st.children);
          out.push(st);
        }
        case _: {
          out.push(node);
        }
      };
    }
    return out;
  }

  #splitEmailsInText(text: String, baseOffset: i32, out: GrowableArray<Inline>): void {
    let r = new StringReader(text);
    var textStart = r.mark();
    while (!r.isAtEnd) {
      let atPos = r.mark();
      let cp = r.advance();
      if (cp == 0x40) {
        // '@'
        let afterAt = r.mark();
        let (matched, emailStart, emailEnd) = tryScanEmailAt(
          r,
          atPos,
          textStart
        );
        if (matched) {
          if (emailStart > textStart) {
            out.push(
              new Text(
                r.sliceRange(textStart, emailStart),
                new Span(baseOffset + textStart, baseOffset + emailStart)
              )
            );
          }
          let email = r.sliceRange(emailStart, emailEnd);
          out.push(
            new Autolink(
              `mailto:${email}`,
              email,
              true,
              new Span(baseOffset + emailStart, baseOffset + emailEnd)
            )
          );
          textStart = emailEnd;
          r.reset(emailEnd);
          continue;
        }
        r.reset(afterAt);
      }
    }
    let endPos = r.mark();
    if (textStart < endPos) {
      out.push(
        new Text(
          r.sliceRange(textStart, endPos),
          new Span(baseOffset + textStart, baseOffset + endPos)
        )
      );
    }
  }
}

function isEmailLocalByte(b: i32) {
  return isAsciiAlphanumeric(b)
    || b == 0x2E
    || b == 0x2D
    || b == 0x5F
    || b == 0x2B;
}

function tryScanEmailAt(r: StringReader, atIndex: i32, minStart: i32): inline (boolean, i32, i32) {
  var localStart = atIndex;
  while (localStart > minStart) {
    r.reset(localStart);
    if (isEmailLocalByte(r.peekBefore())) {
      localStart -= 1;
    } else {
      break;
    }
  }
  if (localStart == atIndex) {
    return (false, 0, 0);
  }
  if (localStart > 0) {
    r.reset(localStart);
    let prev = r.peekBefore();
    if (prev == 0x2F || prev == 0x3A) {
      return (false, 0, 0);
    }
  }

  r.reset(atIndex + 1);
  var hasDot = false;
  var labelLen = 0;
  var lastDotPos = -1;
  while (!r.isAtEnd) {
    let pos = r.mark();
    let db = r.peek();
    if (db == 0x2E) {
      let prev = r.peekBefore();
      if (labelLen == 0 || prev == 0x2D || prev == 0x5F) {
        break;
      }
      hasDot = true;
      lastDotPos = pos;
      labelLen = 0;
      r.advance();
    } else if (isAsciiAlphanumeric(db)
      || (db == 0x2D || db == 0x5F) && labelLen > 0) {
      labelLen += 1;
      r.advance();
    } else {
      break;
    }
  }
  var p = r.mark();
  // Strip trailing '.', '-', '_'
  while (p > atIndex + 1) {
    r.reset(p);
    let last = r.peekBefore();
    if (last == 0x2E || last == 0x2D || last == 0x5F) {
      p -= 1;
    } else {
      break;
    }
  }
  if (!hasDot || lastDotPos < atIndex + 2 || lastDotPos >= p - 1) {
    return (false, 0, 0);
  }
  return (true, localStart, p);
}

/**
 * Returns the `MarkdownExtension` for GFM extended autolinks (`http://`,
 * `https://`, `www.`, and bare emails).
 */
export function autolinkExtension(): MarkdownExtension {
  let parsers = new GrowableArray<InlineContentParser>(1);
  parsers.push(new GfmAutolinkParser());
  let transformers = new GrowableArray<AstTransformer>(1);
  transformers.push(new AutolinkEmailTransformer());
  return new MarkdownExtension(
    new GrowableArray<BlockParserFactory>(0),
    new GrowableArray<DelimiterProcessor>(0),
    parsers,
    transformers,
    new GrowableArray<NodeRenderer>(0)
  );
}
none
VersionPublishedDependenciesApproved by
0.0.1-preview.1 2026-11-19 none