markdown 0.0.1-preview.1

CommonMark and GFM Markdown parser, AST, plugin system, and HTML renderer

Readme Files Dependencies Versions

Markdown (@zena-lang/markdown)#

A CommonMark and GitHub Flavored Markdown (GFM) parser, syntax tree, plugin system, and HTML renderer written in Zena (zena-lang:markdown/<library>). See docs/design/markdown.md for the design.

Usage#

Add it with zena pkg add zena-lang:markdown, then import its main library:

import {
  parse, renderHtml, markdownToHtml, gfmExtensions
} from 'zena-lang:markdown';

// Render CommonMark + GFM to safe HTML (raw HTML stripped, unsafe URLs blocked)
let html = markdownToHtml('# Hello\n\nSome ~~old~~ **new** text.', {
  extensions: gfmExtensions(),
  safe: true,
});

// Or parse to an AST first
let doc = parse('# Heading\n\nParagraph.', gfmExtensions());
let out = renderHtml(doc, {safe: true, extensions: gfmExtensions()});

Running Tests#

npm test -w @zena-lang/markdown
npm run test:quick -- packages/markdown/test
10.6 KiB Raw
/**
 * CommonMark scanning utilities built on `StringReader`: link label
 * normalization, HTML character reference decoding, backslash unescaping, and
 * link destination/title/label scanners.
 */

import {StringBuilder, StringReader, asciiLowerByte, isSpaceOrTab, isAsciiWhitespace, isAsciiPunctuation, isAsciiAlpha, isAsciiDigit, isAsciiAlphanumeric, isAsciiHexDigit, hexDigitValue, isUnicodeWhitespace, isUnicodePunctuation} from 'zena:string';

/**
 * Strips leading and trailing spaces (`0x20`) and tabs (`0x09`).
 */
export function trimSpaces(s: String): String {
  let r = new StringReader(s);
  r.skipSpaces();
  let start = r.mark();
  var lastNonSpace = start;
  while (!r.isAtEnd) {
    let cp = r.advance();
    if (!isSpaceOrTab(cp)) {
      lastNonSpace = r.mark();
    }
  }
  return r.sliceRange(start, lastNonSpace);
}

/**
 * Advances `r` past spaces, tabs, and at most one line ending (`\r\n` or `\n`)
 * followed by spaces and tabs.
 */
export function skipSpacesAndOneLineEnd(r: StringReader): void {
  r.skipSpaces();
  if (r.matchCodePoint(0x0D)) {
    r.matchCodePoint(0x0A);
    r.skipSpaces();
  } else if (r.matchCodePoint(0x0A)) {
    r.skipSpaces();
  }
}

/**
 * Normalizes a link reference label: trims surrounding whitespace, collapses
 * internal whitespace runs to a single space (`0x20`), and folds case.
 * Returns an empty string if the label contains no non-whitespace characters.
 */
export function normalizeLinkLabel(raw: String): String {
  let trimmed = raw.trim();
  if (trimmed.isEmpty) {
    return '';
  }
  let r = new StringReader(trimmed);
  let sb = new StringBuilder();
  var inSpace = false;
  while (!r.isAtEnd) {
    let cp = r.advance();
    if (isAsciiWhitespace(cp)) {
      if (!inSpace) {
        sb.appendByte(0x20);
        inSpace = true;
      }
    } else {
      sb.appendCodePoint(asciiLowerByte(cp));
      inSpace = false;
    }
  }
  return sb.toString();
}

/**
 * Resolves common named HTML entities (`&name;`) to their string value,
 * or returns `null` when `name` is unrecognized.
 */
function lookupNamedEntity(name: String): String | null {
  if (name == 'amp' || name == 'AMP') {
    return '&';
  }
  if (name == 'lt' || name == 'LT') {
    return '<';
  }
  if (name == 'gt' || name == 'GT') {
    return '>';
  }
  if (name == 'quot' || name == 'QUOT') {
    return '"';
  }
  if (name == 'apos') {
    return '\'';
  }
  if (name == 'nbsp') {
    return String.fromCodePoint(0x00A0);
  }
  if (name == 'copy' || name == 'COPY') {
    return '©';
  }
  if (name == 'reg' || name == 'REG') {
    return '®';
  }
  if (name == 'trade' || name == 'TRADE') {
    return '™';
  }
  if (name == 'mdash') {
    return '—';
  }
  if (name == 'ndash') {
    return '–';
  }
  if (name == 'hellip') {
    return '…';
  }
  if (name == 'laquo') {
    return '«';
  }
  if (name == 'raquo') {
    return '»';
  }
  if (name == 'lsquo') {
    return '‘';
  }
  if (name == 'rsquo') {
    return '’';
  }
  if (name == 'ldquo') {
    return '“';
  }
  if (name == 'rdquo') {
    return '”';
  }
  if (name == 'bull') {
    return '•';
  }
  if (name == 'middot') {
    return '·';
  }
  if (name == 'sect') {
    return '§';
  }
  if (name == 'para') {
    return '¶';
  }
  if (name == 'deg') {
    return '°';
  }
  if (name == 'plusmn') {
    return '±';
  }
  if (name == 'times') {
    return '×';
  }
  if (name == 'divide') {
    return '÷';
  }
  if (name == 'euro') {
    return '€';
  }
  if (name == 'pound') {
    return '£';
  }
  if (name == 'yen') {
    return '¥';
  }
  if (name == 'cent') {
    return '¢';
  }
  if (name == 'larr') {
    return '←';
  }
  if (name == 'rarr') {
    return '→';
  }
  if (name == 'uarr') {
    return '↑';
  }
  if (name == 'darr') {
    return '↓';
  }
  if (name == 'harr') {
    return '↔';
  }
  if (name == 'ZeroWidthSpace') {
    return String.fromCodePoint(0x200B);
  }
  if (name == 'ouml') {
    return 'ö';
  }
  if (name == 'Ouml') {
    return 'Ö';
  }
  if (name == 'auml') {
    return 'ä';
  }
  if (name == 'Auml') {
    return 'Ä';
  }
  if (name == 'uuml') {
    return 'ü';
  }
  if (name == 'Uuml') {
    return 'Ü';
  }
  if (name == 'eacute') {
    return 'é';
  }
  if (name == 'Eacute') {
    return 'É';
  }
  if (name == 'frac12') {
    return '½';
  }
  if (name == 'frac14') {
    return '¼';
  }
  if (name == 'frac34') {
    return '¾';
  }
  return null;
}

/**
 * Attempts to decode an HTML character reference starting with `&` at the
 * current position of `r`. If valid, appends the decoded character(s) to `sb`,
 * advances `r` past the closing `;`, and returns `true`; otherwise resets `r`
 * and returns `false`.
 */
export function tryDecodeEntity(r: StringReader, sb: StringBuilder): boolean {
  let saved = r.mark();
  if (!r.matchCodePoint(0x26)) {
    // &
    return false;
  }
  if (r.matchCodePoint(0x23)) {
    // #
    let next = r.peek();
    if (next == 0x78 || next == 0x58) {
      // x or X
      r.advance();
      var count = 0;
      var value = 0;
      while (count < 6 && isAsciiHexDigit(r.peek())) {
        value = value << 4 | hexDigitValue(r.advance());
        count += 1;
      }
      if (count > 0 && r.matchCodePoint(0x3B)) {
        // ;
        sb.appendCodePoint(if (value <= 0) 0xFFFD else value);
        return true;
      }
      r.reset(saved);
      return false;
    }
    var count = 0;
    var value = 0;
    while (count < 7 && isAsciiDigit(r.peek())) {
      value = value * 10 + (r.advance() - 0x30);
      count += 1;
    }
    if (count > 0 && r.matchCodePoint(0x3B)) {
      // ;
      sb.appendCodePoint(if (value <= 0) 0xFFFD else value);
      return true;
    }
    r.reset(saved);
    return false;
  }
  if (isAsciiAlpha(r.peek())) {
    let nameStart = r.mark();
    var count = 0;
    while (count < 32 && isAsciiAlphanumeric(r.peek())) {
      r.advance();
      count += 1;
    }
    let nameEnd = r.mark();
    if (r.matchCodePoint(0x3B)) {
      // ;
      let name = r.sliceRange(nameStart, nameEnd);
      let decoded = lookupNamedEntity(name);
      if (decoded != null) {
        sb.append(decoded);
        return true;
      }
    }
  }
  r.reset(saved);
  return false;
}

/**
 * Unescapes CommonMark backslash escapes (`\!`, `\*`, etc.) and HTML character
 * references (`&amp;`, `&#35;`, `&#x23;`) in `s`.
 */
export function unescapeString(s: String): String {
  if (!s.contains('\\') && !s.contains('&')) {
    return s;
  }
  let r = new StringReader(s);
  let sb = new StringBuilder();
  var last = r.mark();
  while (!r.isAtEnd) {
    let pos = r.mark();
    let cp = r.peek();
    if (cp == 0x5C) {
      // \
      r.advance();
      if (!r.isAtEnd && isAsciiPunctuation(r.peek())) {
        if (pos > last) {
          sb.append(r.sliceRange(last, pos));
        }
        sb.appendCodePoint(r.advance());
        last = r.mark();
      }
    } else if (cp == 0x26) {
      // &
      if (pos > last) {
        sb.append(r.sliceRange(last, pos));
        last = pos;
      }
      if (tryDecodeEntity(r, sb)) {
        last = r.mark();
      } else {
        r.advance();
      }
    } else {
      r.advance();
    }
  }
  if (last < r.mark()) {
    sb.append(r.sliceFrom(last));
  }
  return sb.toString();
}

/**
 * Scans a CommonMark link destination at the current position of `r`.
 * Returns `(matched, unescapedDestination)`. On failure, `r` is reset to its
 * initial position.
 */
export function scanLinkDestination(r: StringReader): inline (boolean, String) {
  if (r.isAtEnd) {
    return (false, '');
  }
  let saved = r.mark();
  if (r.matchCodePoint(0x3C)) {
    // <...>
    let contentStart = r.mark();
    while (!r.isAtEnd) {
      let pos = r.mark();
      let cp = r.advance();
      if (cp == 0x0A || cp == 0x0D || cp == 0x3C) {
        r.reset(saved);
        return (false, '');
      }
      if (cp == 0x3E) {
        // >
        let raw = r.sliceRange(contentStart, pos);
        return (true, unescapeString(raw));
      }
      if (cp == 0x5C && !r.isAtEnd && isAsciiPunctuation(r.peek())) {
        r.advance();
      }
    }
    r.reset(saved);
    return (false, '');
  }

  var openParens = 0;
  while (!r.isAtEnd) {
    let cp = r.peek();
    if (cp <= 0x20 || cp == 0x7F) {
      break;
    }
    if (cp == 0x5C) {
      r.advance();
      if (!r.isAtEnd && isAsciiPunctuation(r.peek())) {
        r.advance();
      }
    } else if (cp == 0x28) {
      // (
      openParens += 1;
      if (openParens > 32) {
        r.reset(saved);
        return (false, '');
      }
      r.advance();
    } else if (cp == 0x29) {
      // )
      if (openParens == 0) {
        break;
      }
      openParens -= 1;
      r.advance();
    } else {
      r.advance();
    }
  }
  if (openParens != 0 || r.mark() == saved) {
    r.reset(saved);
    return (false, '');
  }
  return (true, unescapeString(r.sliceFrom(saved)));
}

/**
 * Scans a CommonMark link title (`"..."`, `'...'`, or `(...)`) at the current
 * position of `r`. Returns `(matched, unescapedTitle)`. On failure, `r` is
 * reset to its initial position.
 */
export function scanLinkTitle(r: StringReader): inline (boolean, String) {
  if (r.isAtEnd) {
    return (false, '');
  }
  let saved = r.mark();
  let opener = r.peek();
  var closer = 0;
  if (opener == 0x22) {
    // "
    closer = 0x22;
  } else if (opener == 0x27) {
    // '
    closer = 0x27;
  } else if (opener == 0x28) {
    // (
    closer = 0x29;
  } else {
    return (false, '');
  }

  r.advance();
  let contentStart = r.mark();
  while (!r.isAtEnd) {
    let pos = r.mark();
    let cp = r.advance();
    if (cp == closer) {
      let raw = r.sliceRange(contentStart, pos);
      return (true, unescapeString(raw));
    }
    if (opener == 0x28 && cp == 0x28) {
      // Unescaped '(' inside (...) title is disallowed
      r.reset(saved);
      return (false, '');
    }
    if (cp == 0x5C && !r.isAtEnd && isAsciiPunctuation(r.peek())) {
      r.advance();
    }
  }
  r.reset(saved);
  return (false, '');
}

/**
 * Scans a link label `[...]` starting at the current position of `r`.
 * Returns `(matched, rawInterior)`. On failure, `r` is reset to its initial
 * position.
 */
export function scanLinkLabel(r: StringReader): inline (boolean, String) {
  let saved = r.mark();
  if (!r.matchCodePoint(0x5B)) {
    // [
    return (false, '');
  }
  let contentStart = r.mark();
  var charCount = 0;
  while (!r.isAtEnd && charCount <= 999) {
    let pos = r.mark();
    let cp = r.advance();
    if (cp == 0x5D) {
      // ]
      return (true, r.sliceRange(contentStart, pos));
    }
    if (cp == 0x5B) {
      // unescaped [
      r.reset(saved);
      return (false, '');
    }
    if (cp == 0x5C && !r.isAtEnd && isAsciiPunctuation(r.peek())) {
      r.advance();
      charCount += 2;
    } else {
      charCount += 1;
    }
  }
  r.reset(saved);
  return (false, '');
}
none
VersionPublishedDependenciesApproved by
0.0.1-preview.1 2026-11-19 none