compiler 0.0.2-preview.1

The Zena self-hosted compiler

Readme Files Dependencies Versions

compiler#

The Zena compiler, written in Zena. It parses, type-checks, and compiles Zena source code to WebAssembly (Wasm GC) and WebAssembly Text (WAT).

Usage#

Parsing a Single File#

Use compiler/parser to parse Zena source text into an AST (Module):

import { parse } from 'compiler/parser';

let ast = parse('export let answer = 42;', 'example.zena');

Loading and Checking a Program#

Use compiler (createCompiler and checkCompilation) with a CompilerHost to resolve imports across a module graph and type-check a program:

import {
  createCompiler, checkCompilation, buildStdlibConfig
} from 'compiler';
import { FsCompilerHost } from 'compiler/fs-host';

let stdlibRoot = '/stdlib';
let host = new FsCompilerHost(stdlibRoot);
let packages = buildStdlibConfig(stdlibRoot, 'host');
let compiler = createCompiler(host, {
  stdlibRoot,
  packages,
});

let result = await compiler.compile({
  entry: '/workspace/main.zena',
  target: 'host',
});
let checked = checkCompilation(result);

Compiling a File to WebAssembly#

Use compiler/compile-file to compile an entry file on disk to a .wasm or .wat output file:

import { compileFile } from 'compiler/compile-file';

let ok = await compileFile({
  entry: 'src/main.zena',
  outPath: 'out/main.wasm',
  stdlibRoot: '/stdlib',
});

Libraries#

  • compiler / compiler/compiler — compiler entry point (Compiler, createCompiler, checkCompilation, ProgramCheckResult)
  • compiler/compile-file — end-to-end file compilation (compileFile, CompileFileOptions)
  • compiler/tokenizer — lexer and token definitions (tokenize, Token, TokenType)
  • compiler/parser — recursive-descent parser (parse, Parser)
  • compiler/ast — AST node definitions (Node, Module, Statement, Expression, TypeAnnotation)
  • compiler/visitor — AST traversal helpers (walk, Visitor)
  • compiler/scope — lexical scope and symbol resolution (ScopeBuilder, ScopeResult, Symbol)
  • compiler/checker — static type checker (checkModule, CheckResult)
  • compiler/semantic-model — semantic queries over a checked module (SemanticModel)
  • compiler/types — compiler type representations (Type, ClassType, FunctionType, typeToString)
  • compiler/diagnostics — compiler diagnostics and source spans (Diagnostic, DiagnosticCode, DiagnosticSeverity, SourceSpan)
  • compiler/package-manifest — zena-package.json manifest and workspace loader (loadWorkspace, loadPackageMap)
  • compiler/module-resolver — package and relative module specifier resolution (ModuleResolver, PackageConfig)
  • compiler/library-loader — module graph loader (LibraryLoader, CompilerHost, SourceFile)
  • compiler/codegen/binary-generator — WebAssembly GC binary emitter (BinaryGenerator)
  • compiler/codegen/wat-generator — WebAssembly Text (WAT) emitter (WatGenerator)

License#

MIT

13.6 KiB Raw
/**
 * Doc comment parsing.
 *
 * A doc comment is a block comment whose body starts with a star — the
 * slash-star-star form — attached as a leading comment to a
 * declaration. Line comments are never doc comments, which is the rule
 * the language server already applies to hover.
 *
 * The body is markdown once each line's leading star (and one following
 * space) is removed. Lines that start with `@` outside a fenced code
 * block open a block tag; everything before the first tag is the
 * description, and its first paragraph is the summary.
 *
 * Tools read doc comments through `parseLeadingDoc`, or through
 * `declarationDoc` and `docCommentAt`, which find the comments to parse:
 * zenadoc for API documentation, the language service for hover.
 *
 * See docs/design/zenadoc.md.
 */

import { Comment, Node, Module, ClassDeclaration, InterfaceDeclaration } from './ast.zena';
import { CommentMap } from './comment-attachment.zena';
import { StringBuilder } from 'zena:core';

// The ASCII bytes this file looks for. `i32` rather than an enum
// because they are compared against `String.getByteAt`, which returns
// one: an enum is nominal, so every comparison would read
// `b == Byte.Star as i32`. An enum would also claim these ten are the
// domain, and the domain is all 256.
let SPACE = 32;
let TAB = 9;
let CR = 13;
let LF = 10;
let HASH = 35;
let STAR = 42;
let DASH = 45;
let AT = 64;
let GT = 62;
let BACKTICK = 96;

/** A block tag such as `@param index the offset to read`. */
export final class DocTag(
  /** Tag name without the `@`, e.g. `param`. */
  name: String,

  /**
   * The subject the tag names, for tags that take one (`@param`,
   * `@throws`, `@typeParam`): the first whitespace-delimited word after
   * the tag. Empty for every other tag.
   */
  subject: String,

  /** Everything after the tag and its subject, with blank lines kept. */
  text: String
)

/** A parsed doc comment. */
export final class DocComment(
  /**
   * The description's first paragraph, on one line. Empty when the
   * description opens with something other than a paragraph, such as a
   * fenced code block.
   */
  summary: String,

  /** Everything before the first block tag, as markdown. */
  description: String,

  /** Block tags in source order. */
  tags: GrowableArray<DocTag>,

  /** The comment body with delimiters and star prefixes removed. */
  raw: String
) {
  /** The first tag with this name, or null. */
  firstTag(name: String): DocTag | null {
    for (let tag in this.tags) {
      if (tag.name == name) {
        return tag;
      }
    }
    return null;
  }

  /** Every tag with this name, in source order. */
  tagsNamed(name: String): GrowableArray<DocTag> {
    let found = new Array<DocTag>(2);
    for (let tag in this.tags) {
      if (tag.name == name) {
        found.push(tag);
      }
    }
    return found;
  }

  /**
   * The `@deprecated` reason, empty when the tag carries no text, and
   * null when the declaration is not deprecated.
   */
  deprecation(): String | null {
    let tag = this.firstTag('deprecated');
    if (tag == null) {
      return null;
    }
    return tag.text;
  }
}

/** Whether a comment is a doc comment. */
export function isDocComment(comment: Comment): boolean {
  return comment.isBlock && comment.value.length > 0 &&
      comment.value.getByteAt(0) == STAR;
}

function isSpace(b: i32) {
  return b == SPACE || b == TAB;
}

/** Leading and trailing spaces and tabs removed. */
export function trim(s: String): String {
  var start = 0;
  var end = s.length;
  while (start < end && isSpace(s.getByteAt(start))) {
    start += 1;
  }
  while (end > start && isSpace(s.getByteAt(end - 1))) {
    end -= 1;
  }
  return s.sliceBytes(start, end);
}

/** Trailing whitespace, including newlines, removed. */
function trimTrailingBlank(s: String) {
  var end = s.length;
  while (end > 0) {
    let b = s.getByteAt(end - 1);
    if (b == LF || b == CR || isSpace(b)) {
      end -= 1;
    } else {
      return s.sliceBytes(0, end);
    }
  }
  return s.sliceBytes(0, end);
}

/**
 * One line of a comment body with its star prefix removed.
 *
 * Indentation after the prefix is kept: it carries meaning in markdown
 * (nested lists, indented code), so only the decoration comes off.
 */
function stripPrefix(line: String) {
  var end = line.length;
  if (end > 0 && line.getByteAt(end - 1) == CR) {
    end -= 1;
  }
  var start = 0;
  while (start < end && isSpace(line.getByteAt(start))) {
    start += 1;
  }
  if (start < end && line.getByteAt(start) == STAR) {
    start += 1;
    if (start < end && line.getByteAt(start) == SPACE) {
      start += 1;
    }
  } else {
    // No star on this line: it is either the first line, which follows
    // the opening delimiter, or a continuation written without
    // decoration. Keep it as written.
    start = 0;
  }
  while (end > start && isSpace(line.getByteAt(end - 1))) {
    end -= 1;
  }
  return line.sliceBytes(start, end);
}

/** Splits on newlines, keeping empty lines. */
function splitLines(s: String) {
  let lines = new Array<String>(8);
  var start = 0;
  var i = 0;
  while (i < s.length) {
    if (s.getByteAt(i) == LF) {
      lines.push(s.sliceBytes(start, i));
      start = i + 1;
    }
    i += 1;
  }
  lines.push(s.sliceBytes(start, s.length));
  return lines;
}

/** Strips the star prefix from every line, then drops blank edges. */
function bodyLines(body: String) {
  let stripped = new Array<String>(8);
  for (let line in splitLines(body)) {
    stripped.push(stripPrefix(line));
  }
  var start = 0;
  var end = stripped.length;
  while (start < end && stripped[start].length == 0) {
    start += 1;
  }
  while (end > start && stripped[end - 1].length == 0) {
    end -= 1;
  }
  let out = new Array<String>(8);
  var i = start;
  while (i < end) {
    out.push(stripped[i]);
    i += 1;
  }
  return out;
}

/** Joins `lines[from..to)` with newlines, dropping trailing blanks. */
function joinLines(lines: GrowableArray<String>, from: i32, to: i32) {
  let sb = new StringBuilder(64);
  var i = from;
  while (i < to) {
    if (i > from) {
      sb.append('\n');
    }
    sb.append(lines[i]);
    i += 1;
  }
  return trimTrailingBlank(sb.toString());
}

/** Whether a line opens or closes a fenced code block. */
function isFence(line: String) {
  let t = trim(line);
  return t.length >= 3 && t.getByteAt(0) == BACKTICK &&
      t.getByteAt(1) == BACKTICK && t.getByteAt(2) == BACKTICK;
}

/**
 * The tag name a line opens, or null if it opens none.
 *
 * A tag line starts with `@` followed by at least one ASCII letter, so
 * an address in prose (`user@example.com`) never opens a tag: the line
 * would have to begin with it.
 */
function tagNameAt(line: String): String | null {
  let t = trim(line);
  if (t.length < 2 || t.getByteAt(0) != AT) {
    return null;
  }
  var i = 1;
  while (i < t.length) {
    let b = t.getByteAt(i);
    let isLetter = (b >= 65 && b <= 90) || (b >= 97 && b <= 122);
    let isDigitOrDash = (b >= 48 && b <= 57) || b == DASH;
    if (isLetter || (i > 1 && isDigitOrDash)) {
      i += 1;
    } else {
      break;
    }
  }
  if (i == 1) {
    return null;
  }
  return t.sliceBytes(1, i);
}

/** Tags whose first word after the tag names a subject. */
function takesSubject(name: String) {
  return name == 'param' || name == 'throws' || name == 'typeParam';
}

/** Index of the first space or tab, or the string's length. */
function firstSpace(s: String) {
  var i = 0;
  while (i < s.length) {
    if (isSpace(s.getByteAt(i))) {
      return i;
    }
    i += 1;
  }
  return i;
}

/**
 * The first paragraph of a description, on one line.
 *
 * A description opening with a fence, a list, a heading or a quote has
 * no summary: joining those onto one line produces nonsense, and a
 * consumer showing summaries is better off showing nothing.
 */
function summaryOf(description: String) {
  if (description.length == 0) {
    return '';
  }
  let lines = splitLines(description);
  let first = trim(lines[0]);
  if (first.length == 0) {
    return '';
  }
  let b = first.getByteAt(0);
  if (b == BACKTICK || b == DASH || b == STAR || b == HASH || b == GT) {
    return '';
  }
  let sb = new StringBuilder(64);
  var i = 0;
  while (i < lines.length) {
    let line = trim(lines[i]);
    if (line.length == 0) {
      break;
    }
    if (i > 0) {
      sb.append(' ');
    }
    sb.append(line);
    i += 1;
  }
  return sb.toString();
}

/** Index of the first line that opens a block tag, else `lines.length`. */
function firstTagLine(lines: GrowableArray<String>) {
  var inFence = false;
  var i = 0;
  while (i < lines.length) {
    if (isFence(lines[i])) {
      inFence = !inFence;
    } else if (!inFence && tagNameAt(lines[i]) != null) {
      return i;
    }
    i += 1;
  }
  return lines.length;
}

/**
 * Collects block tags from `start` to the end of the comment.
 *
 * A tag runs until the next tag line, so its text can span lines and
 * contain fenced code — which `@example` needs.
 */
function parseTags(lines: GrowableArray<String>, start: i32) {
  // Every line that opens a tag. A tag's text is the lines from its
  // opener up to the next one, so fenced code inside a tag is carried
  // along rather than ending it.
  let openers = new Array<i32>(2);
  var inFence = false;
  var i = start;
  while (i < lines.length) {
    if (isFence(lines[i])) {
      inFence = !inFence;
    } else if (!inFence && tagNameAt(lines[i]) != null) {
      openers.push(i);
    }
    i += 1;
  }

  let tags = new Array<DocTag>(openers.length);
  var t = 0;
  while (t < openers.length) {
    let from = openers[t];
    let to = if (t + 1 < openers.length) { openers[t + 1] } else { lines.length };
    let opener = tagNameAt(lines[from]);
    if (opener == null) {
      throw new Error('a line recorded as opening a tag does not');
    }
    let head = trim(lines[from]);
    let afterTag = trim(head.sliceBytes(opener.length + 1, head.length));
    var subject = '';
    var firstLine = afterTag;
    if (takesSubject(opener)) {
      let cut = firstSpace(afterTag);
      subject = afterTag.sliceBytes(0, cut);
      firstLine = trim(afterTag.sliceBytes(cut, afterTag.length));
    }
    // `@example` and friends carry their text on the following lines,
    // so the tag's own line is often empty and must not open the text
    // with a blank line.
    let sb = new StringBuilder(64);
    var wrote = firstLine.length > 0;
    if (wrote) {
      sb.append(firstLine);
    }
    var j = from + 1;
    while (j < to) {
      let text = lines[j];
      if (wrote) {
        sb.append('\n');
        sb.append(text);
      } else if (trim(text).length > 0) {
        sb.append(text);
        wrote = true;
      }
      j += 1;
    }
    tags.push(new DocTag(opener, subject, trimTrailingBlank(sb.toString())));
    t += 1;
  }
  return tags;
}

/**
 * Parses a comment body — the text between the delimiters, which is
 * what `Comment.value` holds.
 */
export function parseDocCommentBody(body: String): DocComment {
  let lines = bodyLines(body);
  let tagStart = firstTagLine(lines);
  let description = joinLines(lines, 0, tagStart);
  let tags = parseTags(lines, tagStart);
  let raw = joinLines(lines, 0, lines.length);
  return new DocComment(summaryOf(description), description, tags, raw);
}

/**
 * The doc comment for a node, given its leading comments, or null when
 * none of them is a doc comment.
 *
 * Several doc comments in a row are joined: the sequence reads as one
 * block of prose, so the closest one to the declaration is not
 * privileged.
 */
export function parseLeadingDoc(comments: GrowableArray<Comment> | null): DocComment | null {
  if (comments == null) {
    return null;
  }
  let all = comments;
  let sb = new StringBuilder(64);
  var found = false;
  for (let comment in all) {
    if (isDocComment(comment)) {
      if (found) {
        sb.append('\n');
      }
      sb.append(comment.value);
      found = true;
    }
  }
  if (!found) {
    return null;
  }
  return parseDocCommentBody(sb.toString());
}

/** Whether `offset` is inside `node`'s source span. */
function spans(node: Node, offset: i32) {
  let loc = node.loc;
  return loc != null && offset >= loc.start && offset < loc.end;
}

/** The member of a class or interface body around `offset`, or null. */
function memberAt(body: GrowableArray<Node>, offset: i32): Node | null {
  for (let member in body) {
    if (spans(member, offset)) {
      return member;
    }
  }
  return null;
}

/**
 * The doc comment on the declaration around `offset`, or null when that
 * declaration has none.
 *
 * The declaration is the class or interface member around the offset
 * when there is one, else the top-level statement around it. An
 * undocumented member has no doc comment even when its class has one.
 */
export function docCommentAt(ast: Module, offset: i32, comments: CommentMap): DocComment | null {
  for (let stmt in ast.body) {
    if (!spans(stmt, offset)) {
      continue;
    }
    let member = match (stmt) {
      case ClassDeclaration {body}: memberAt(body, offset)
      case InterfaceDeclaration {body}: memberAt(body, offset)
      case _: null
    };
    return parseLeadingDoc(comments.getLeading(member ?? stmt));
  }
  return null;
}

/**
 * The doc comment on a declaration, or null when it has none. `ast` is
 * the module the declaration is in, and `comments` that module's
 * comments.
 *
 * A name bound inside a statement, like the `x` in `let x = 1;`, has no
 * comments of its own. Its doc comment is the statement's, found by
 * position.
 */
export function declarationDoc(decl: Node, ast: Module, comments: CommentMap): DocComment | null {
  let own = comments.getLeading(decl);
  if (own != null) {
    return parseLeadingDoc(own);
  }
  let loc = decl.loc;
  if (loc == null) {
    return null;
  }
  return docCommentAt(ast, loc.start, comments);
}
VersionPublishedDependenciesApproved by
0.0.2-preview.1 2026-11-19