compiler 0.0.2-preview.1

The Zena self-hosted compiler

Readme Files Dependencies Versions

compiler#

The Zena compiler, written in Zena. It parses, type-checks, and compiles Zena source code to WebAssembly (Wasm GC) and WebAssembly Text (WAT).

Usage#

Parsing a Single File#

Use compiler/parser to parse Zena source text into an AST (Module):

import { parse } from 'compiler/parser';

let ast = parse('export let answer = 42;', 'example.zena');

Loading and Checking a Program#

Use compiler (createCompiler and checkCompilation) with a CompilerHost to resolve imports across a module graph and type-check a program:

import {
  createCompiler, checkCompilation, buildStdlibConfig
} from 'compiler';
import { FsCompilerHost } from 'compiler/fs-host';

let stdlibRoot = '/stdlib';
let host = new FsCompilerHost(stdlibRoot);
let packages = buildStdlibConfig(stdlibRoot, 'host');
let compiler = createCompiler(host, {
  stdlibRoot,
  packages,
});

let result = await compiler.compile({
  entry: '/workspace/main.zena',
  target: 'host',
});
let checked = checkCompilation(result);

Compiling a File to WebAssembly#

Use compiler/compile-file to compile an entry file on disk to a .wasm or .wat output file:

import { compileFile } from 'compiler/compile-file';

let ok = await compileFile({
  entry: 'src/main.zena',
  outPath: 'out/main.wasm',
  stdlibRoot: '/stdlib',
});

Libraries#

  • compiler / compiler/compiler — compiler entry point (Compiler, createCompiler, checkCompilation, ProgramCheckResult)
  • compiler/compile-file — end-to-end file compilation (compileFile, CompileFileOptions)
  • compiler/tokenizer — lexer and token definitions (tokenize, Token, TokenType)
  • compiler/parser — recursive-descent parser (parse, Parser)
  • compiler/ast — AST node definitions (Node, Module, Statement, Expression, TypeAnnotation)
  • compiler/visitor — AST traversal helpers (walk, Visitor)
  • compiler/scope — lexical scope and symbol resolution (ScopeBuilder, ScopeResult, Symbol)
  • compiler/checker — static type checker (checkModule, CheckResult)
  • compiler/semantic-model — semantic queries over a checked module (SemanticModel)
  • compiler/types — compiler type representations (Type, ClassType, FunctionType, typeToString)
  • compiler/diagnostics — compiler diagnostics and source spans (Diagnostic, DiagnosticCode, DiagnosticSeverity, SourceSpan)
  • compiler/package-manifest — zena-package.json manifest and workspace loader (loadWorkspace, loadPackageMap)
  • compiler/module-resolver — package and relative module specifier resolution (ModuleResolver, PackageConfig)
  • compiler/library-loader — module graph loader (LibraryLoader, CompilerHost, SourceFile)
  • compiler/codegen/binary-generator — WebAssembly GC binary emitter (BinaryGenerator)
  • compiler/codegen/wat-generator — WebAssembly Text (WAT) emitter (WatGenerator)

License#

MIT

17.0 KiB Raw
/**
 * Comment Attachment — assigns each comment to the nearest AST node.
 *
 * Implements Babel-style comment attachment: for every comment we find the
 * enclosing node list (module body, class body, block body) and decide:
 *   - Leading:   comment before the following node
 *   - Trailing:  comment on the same line as the preceding node's end
 *   - Inner:     comment inside an empty body (no preceding/following node)
 *
 * File-level comments: comments at the start of a file that are followed by
 * a blank line are attached as leading on the Module, not on the first
 * declaration. Comments without a blank line separator are leading on the
 * first declaration.
 *
 * The result is a CommentMap keyed by Node reference that any tool can query.
 */

import { NodeMap } from './node-map.zena';
import {
  Comment, Node, Module, Statement, Expression, BlockStatement, ClassDeclaration, InterfaceDeclaration, MixinDeclaration, EnumDeclaration, EnumMember, TypeAliasDeclaration, ImportDeclaration, ExportFromDeclaration, ImportSpecifier, VariableDeclaration, FunctionDeclaration, FunctionExpression, BinaryExpression, UnaryExpression, AssignmentExpression, CallExpression, MemberExpression, IndexExpression, NewExpression, ArrayLiteral, IfExpression, MatchExpression, MatchCase, AsExpression, IsExpression, SpreadElement, IfStatement, WhileStatement, ForStatement, ForInStatement, ReturnStatement, YieldStatement, ExpressionStatement, UsingStatement, ShieldedStatement, MethodDefinition, AccessorDeclaration, SetterInfo, FieldDefinition, SealedVariant, CaseClassParam, TypeAnnotation, NamedTypeAnnotation, UnionTypeAnnotation, NullableTypeAnnotation, FunctionTypeAnnotation, RecordTypeAnnotation, TupleTypeAnnotation, InlineTypeAnnotation, PropertySignature,
} from './ast.zena';
import { growable } from 'zena:core';

/**
 * A collection of comments that maps AST nodes to their attached comments
 */
export final class CommentMap {
  // Keyed by NodeId (via NodeMap): node ids are assigned during parse, and id
  // keys give identity semantics with a real hash — Node itself is not
  // Hashable.
  #leading = new NodeMap<GrowableArray<Comment>>();
  #trailing = new NodeMap<GrowableArray<Comment>>();
  #inner = new NodeMap<GrowableArray<Comment>>();

  addLeading(node: Node, comment: Comment) {
    if (!this.#leading.has(node)) {
      this.#leading[node] = new Array<Comment>(2);
    }
    this.#leading[node].push(comment);
  }

  addTrailing(node: Node, comment: Comment) {
    if (!this.#trailing.has(node)) {
      this.#trailing[node] = new Array<Comment>(2);
    }
    this.#trailing[node].push(comment);
  }

  addInner(node: Node, comment: Comment) {
    if (!this.#inner.has(node)) {
      this.#inner[node] = new Array<Comment>(2);
    }
    this.#inner[node].push(comment);
  }

  getLeading(node: Node): GrowableArray<Comment> | null {
    if (this.#leading.has(node)) {
      return this.#leading[node];
    }
    return null;
  }

  getTrailing(node: Node): GrowableArray<Comment> | null {
    if (this.#trailing.has(node)) {
      return this.#trailing[node];
    }
    return null;
  }

  getInner(node: Node): GrowableArray<Comment> | null {
    if (this.#inner.has(node)) {
      return this.#inner[node];
    }
    return null;
  }
}

function getNodeStart(node: Node): i32 {
  let loc = node.loc;
  if (loc != null) {
    return loc.start;
  }
  return 0 - 1;
}

function getNodeEnd(node: Node): i32 {
  let loc = node.loc;
  if (loc != null) {
    return loc.end;
  }
  return 0 - 1;
}

function getNodeEndLine(source: String, node: Node): i32 {
  let loc = node.loc;
  if (loc == null) { return 0 - 1; }
  let {end} = loc;
  var line = 1;
  var i = 0;
  while (i < end && i < source.length) {
    if (source.getByteAt(i) == 10) {
      line += 1;
    }
    i += 1;
  }
  return line;
}

/**
 * Check if there is a blank line (two newlines with only whitespace between)
 * in the source text between positions `from` (exclusive) and `to` (exclusive).
 */
function hasBlankLineBetween(source: String, from: i32, to: i32): boolean {
  var newlineCount = 0;
  var i = from;
  while (i < to && i < source.length) {
    let ch = source.getByteAt(i);
    if (ch == 10) { // \n
      newlineCount += 1;
      if (newlineCount >= 2) { return true; }
    } else if (ch != 13 && ch != 32 && ch != 9) { // not \r, space, tab
      // Non-whitespace resets the counter
      newlineCount = 0;
    }
    i += 1;
  }
  return false;
}

/** Convert Array<Statement> to Array<Node> (WASM-GC arrays are invariant). */
function statementsToNodes(stmts: GrowableArray<Statement>) {
  let result = new Array<Node>(stmts.length);
  var i = 0;
  while (i < stmts.length) {
    result.push(stmts[i]);
    i += 1;
  }
  return result;
}


/** Convert Array<EnumMember> to Array<Node> (WASM-GC arrays are invariant). */
function enumMembersToNodes(members: GrowableArray<EnumMember>) {
  let result = new Array<Node>(members.length);
  var i = 0;
  while (i < members.length) {
    result.push(members[i]);
    i += 1;
  }
  return result;
}

/**
 * Wrap a single node as a one-element Array<Node>.
 */
function singleNode(node: Node) {
  let arr = growable<Node>([node]);
  return arr;
}

/** Convert Array<Expression> to Array<Node>. */
function expressionsToNodes(exprs: GrowableArray<Expression>) {
  let result = new Array<Node>(exprs.length);
  var i = 0;
  while (i < exprs.length) {
    result.push(exprs[i]);
    i += 1;
  }
  return result;
}

/** Build a two-element Array<Node>. */
function twoNodes(a: Node, b: Node) {
  let arr = growable<Node>([a, b]);
  return arr;
}

/** Convert Array<TypeAnnotation> to Array<Node>. */
function typeAnnotationsToNodes(anns: GrowableArray<TypeAnnotation>) {
  let result = new Array<Node>(anns.length);
  var i = 0;
  while (i < anns.length) {
    result.push(anns[i]);
    i += 1;
  }
  return result;
}

/** Convert Array<ImportSpecifier> to Array<Node>. */
function importSpecifiersToNodes(specs: GrowableArray<ImportSpecifier>) {
  let result = new Array<Node>(specs.length);
  for (let s in specs) {
    result.push(s);
  }
  return result;
}

/**
 * Extract the child body (Array<Node>) from nodes that contain bodies.
 * Also handles variable declarations and function expressions so that
 * comments inside nested blocks (e.g. `let f = () => { ... };`)
 * are correctly attached.
 */
function getChildBody(node: Node): GrowableArray<Node> | null {
  return match (node) {
  case BlockStatement {body}: statementsToNodes(body)
  case ClassDeclaration {caseParams, sealedVariants, body}: {
    if (caseParams == null && sealedVariants == null) {
      body
    } else {
      let result = new Array<Node>(8);
      if (caseParams != null) {
        for (let cp in caseParams) {
          result.push(cp);
        }
      }
      if (sealedVariants != null) {
        var vi = 0;
        var bi = 0;
        while (vi < sealedVariants.length && bi < body.length) {
          if (getNodeStart(sealedVariants[vi]) < getNodeStart(body[bi])) {
            result.push(sealedVariants[vi]);
            vi += 1;
          } else {
            result.push(body[bi]);
            bi += 1;
          }
        }
        while (vi < sealedVariants.length) {
          result.push(sealedVariants[vi]);
          vi += 1;
        }
        while (bi < body.length) {
          result.push(body[bi]);
          bi += 1;
        }
      } else {
        for (let m in body) {
          result.push(m);
        }
      }
      result
    }
  }
  case SealedVariant {params}: {
    if (params == null || params.length == 0) {
      null
    } else {
      let result = new Array<Node>(params.length);
      for (let p in params) {
        result.push(p);
      }
      result
    }
  }
  case CaseClassParam {typeAnnotation, defaultValue}: {
    if (defaultValue == null) {
      singleNode(typeAnnotation)
    } else {
      twoNodes(typeAnnotation, defaultValue as Node)
    }
  }
  case InterfaceDeclaration {body}: body
  case MixinDeclaration {body}: body
  case EnumDeclaration {members}: enumMembersToNodes(members)
  case TypeAliasDeclaration {typeAnnotation}: {
    if (typeAnnotation == null) { null } else { singleNode(typeAnnotation as Node) }
  }
  case ImportDeclaration {imports}: {
    if (imports.length == 1 && imports[0].imported.name == '*') {
      null
    } else {
      importSpecifiersToNodes(imports)
    }
  }
  case ExportFromDeclaration {specifiers}: importSpecifiersToNodes(specifiers)
  // Statements that hold a body. Without these a comment written inside the
  // body is not descended into, and ends up attached outside the statement.
  case IfStatement {test, consequent, alternate}: {
    let result = growable<Node>([test, consequent]);
    if (alternate != null) { result.push(alternate as Node); }
    result
  }
  case WhileStatement {test, body}: twoNodes(test, body)
  case ForStatement {init, test, update, body}: {
    let result = new Array<Node>(4);
    if (init != null) { result.push(init as Node); }
    if (test != null) { result.push(test as Node); }
    if (update != null) { result.push(update as Node); }
    result.push(body);
    result
  }
  case ForInStatement {iterable, body}: twoNodes(iterable, body)
  case ReturnStatement {argument}: {
    if (argument == null) { null } else { singleNode(argument as Node) }
  }
  case YieldStatement {argument}: singleNode(argument)
  case ExpressionStatement {expression}: singleNode(expression)
  case UsingStatement {init}: singleNode(init)
  case ShieldedStatement {body}: singleNode(body)
  // Class members that hold a body, so a comment in a method or an accessor
  // stays in it rather than becoming a trailing comment on the member before.
  case MethodDefinition {returnType, body}: {
    let result = new Array<Node>(2);
    if (returnType != null) { result.push(returnType as Node); }
    if (body != null) { result.push(body); }
    if (result.length == 0) { null } else { result }
  }
  case AccessorDeclaration {getter, setter}: {
    let result = new Array<Node>(2);
    if (getter != null) { result.push(getter as Node); }
    if (setter != null) { result.push(setter as Node); }
    if (result.length == 0) { null } else { result }
  }
  case SetterInfo {body}: singleNode(body)
  case FieldDefinition {typeAnnotation, value}: {
    let result = new Array<Node>(2);
    if (typeAnnotation != null) { result.push(typeAnnotation as Node); }
    if (value != null) { result.push(value as Node); }
    if (result.length == 0) { null } else { result }
  }
  case MatchCase {body}: singleNode(body)
  case VariableDeclaration {typeAnnotation, init}: {
    let result = new Array<Node>(2);
    if (typeAnnotation != null) { result.push(typeAnnotation as Node); }
    if (init.loc != null) { result.push(init as Node); }
    if (result.length == 0) { null } else { result }
  }
  case FunctionDeclaration {func}: singleNode(func as Node)
  case FunctionExpression {returnType, body}: {
    if (returnType == null) {
      singleNode(body)
    } else {
      twoNodes(returnType as Node, body)
    }
  }
  case BinaryExpression {left, right}: twoNodes(left, right)
  case AssignmentExpression {left, value}: twoNodes(left, value)
  case UnaryExpression {argument}: singleNode(argument)
  case CallExpression {callee, args}: {
    let result = growable<Node>([callee]);
    var i = 0;
    while (i < args.length) {
      result.push(args[i]);
      i += 1;
    }
    result
  }
  case NewExpression {callee, args}: {
    let result = growable<Node>([callee]);
    var i = 0;
    while (i < args.length) {
      result.push(args[i]);
      i += 1;
    }
    result
  }
  case MemberExpression {object}: singleNode(object)
  case IndexExpression {object, index}: twoNodes(object, index)
  case ArrayLiteral {elements}: expressionsToNodes(elements)
  case IfExpression {test, consequent, alternate}: {
    let result = growable<Node>([test, consequent]);
    if (alternate != null) { result.push(alternate); }
    result
  }
  case MatchExpression {discriminant, cases}: {
    let result = growable<Node>([discriminant]);
    for (let c in cases) { result.push(c); }
    result
  }
  case AsExpression {expression}: singleNode(expression)
  case IsExpression {expression}: singleNode(expression)
  case SpreadElement {argument}: singleNode(argument)
  case RecordTypeAnnotation {properties, spreads}: {
    let result = new Array<Node>(properties.length);
    var si = 0;
    for (let p in properties) {
      if (spreads != null) {
        while (si < spreads.length && getNodeStart(spreads[si]) < getNodeStart(p)) {
          result.push(spreads[si]);
          si += 1;
        }
      }
      result.push(p);
    }
    if (spreads != null) {
      while (si < spreads.length) {
        result.push(spreads[si]);
        si += 1;
      }
    }
    result
  }
  case PropertySignature {typeAnnotation}: singleNode(typeAnnotation)
  case NamedTypeAnnotation {typeArguments}: {
    if (typeArguments == null || typeArguments.length == 0) {
      null
    } else {
      typeAnnotationsToNodes(typeArguments)
    }
  }
  case UnionTypeAnnotation {types}: typeAnnotationsToNodes(types)
  case NullableTypeAnnotation {typeAnnotation}: singleNode(typeAnnotation)
  case InlineTypeAnnotation {typeAnnotation}: singleNode(typeAnnotation)
  case TupleTypeAnnotation {elementTypes}: typeAnnotationsToNodes(elementTypes)
  case FunctionTypeAnnotation {params, returnType}: {
    let result = typeAnnotationsToNodes(params);
    result.push(returnType);
    result
  }
  case _: null
};
}

/**
 * Attach comments to nodes in a list. Recurses into sub-bodies.
 */
function attachToNodeList(
  comments: GrowableArray<Comment>,
  nodes: GrowableArray<Node>,
  source: String,
  map: CommentMap
): void {
  if (comments.length == 0) { return; }

  // For each comment, find where it belongs
  for (let comment in comments) {
    var attached = false;

    // Check if comment belongs inside a child node's body — if so, recurse
    for (let node in nodes) {
      let childBody = getChildBody(node);
      if (childBody != null) {
        let nStart = getNodeStart(node);
        let nEnd = getNodeEnd(node);
        if (nStart >= 0 && nEnd >= 0 && comment.start > nStart && comment.end < nEnd) {
          let singleComment = growable<Comment>([comment]);
          attachToNodeList(singleComment, childBody, source, map);
          // If the body was empty, the comment dangles — attach as inner
          if (childBody.length == 0) {
            map.addInner(node, comment);
          }
          attached = true;
          break;
        }
      }
    }

    if (!attached) {
      // Find preceding and following nodes
      var preceding: Node | null = null;
      var following: Node | null = null;

      for (let node in nodes) {
        let nEnd = getNodeEnd(node);
        let nStart = getNodeStart(node);
        if (nEnd >= 0 && nEnd <= comment.start) {
          preceding = node;
        }
        if (nStart >= 0 && nStart >= comment.end && following == null) {
          following = node;
        }
      }

      if (preceding != null && comment.line == getNodeEndLine(source, preceding)) {
        map.addTrailing(preceding, comment);
      } else if (following != null) {
        map.addLeading(following, comment);
      } else if (preceding != null) {
        map.addTrailing(preceding, comment);
      }
      // Note: inner comments on empty containers are handled above in the recurse branch
    }

  }
}

/**
 * Attach all comments from the module to their nearest AST nodes.
 * Returns a CommentMap keyed by Node reference.
 */
export function attachComments(module: Module): CommentMap {
  let map = new CommentMap();
  let {comments} = module;
  if (comments.length == 0) {
    return map;
  }
  let {source} = module;
  let nodes = statementsToNodes(module.body);

  // Empty module: all comments are file-level (leading on Module)
  if (nodes.length == 0) {
    var ei = 0;
    while (ei < comments.length) {
      map.addLeading(module, comments[ei]);
      ei += 1;
    }
    return map;
  }

  // Comments at the top of the file separated from the first declaration
  // (or the next comment group) by a blank line are file-level: leading on Module.
  let firstNodeStart = getNodeStart(nodes[0]);

  // Collect comments that appear before the first statement
  var prefixEnd = 0;
  while (prefixEnd < comments.length && comments[prefixEnd].start < firstNodeStart) {
    prefixEnd += 1;
  }

  // Find the split point: the last comment before a blank line gap
  var fileLevelEnd = 0;
  if (prefixEnd > 0) {
    var ci = 0;
    while (ci < prefixEnd) {
      let commentEnd = comments[ci].end;
      // What comes after this comment?
      let nextStart = if (ci + 1 < prefixEnd) {
        comments[ci + 1].start;
      } else {
        firstNodeStart;
      };
      if (hasBlankLineBetween(source, commentEnd, nextStart)) {
        fileLevelEnd = ci + 1;
      }
      ci += 1;
    }
  }

  // Attach file-level comments as leading on the Module node
  var fi = 0;
  while (fi < fileLevelEnd) {
    map.addLeading(module, comments[fi]);
    fi += 1;
  }

  // Build the remaining comments array (skip file-level ones)
  let remaining = new Array<Comment>(comments.length - fileLevelEnd);
  var ri = fileLevelEnd;
  while (ri < comments.length) {
    remaining.push(comments[ri]);
    ri += 1;
  }

  // Attach remaining comments to nodes in the body
  attachToNodeList(remaining, nodes, source, map);

  return map;
}
VersionPublishedDependenciesApproved by
0.0.2-preview.1 2026-11-19