compiler 0.0.2-preview.1

The Zena self-hosted compiler

Readme Files Dependencies Versions

compiler#

The Zena compiler, written in Zena. It parses, type-checks, and compiles Zena source code to WebAssembly (Wasm GC) and WebAssembly Text (WAT).

Usage#

Parsing a Single File#

Use compiler/parser to parse Zena source text into an AST (Module):

import { parse } from 'compiler/parser';

let ast = parse('export let answer = 42;', 'example.zena');

Loading and Checking a Program#

Use compiler (createCompiler and checkCompilation) with a CompilerHost to resolve imports across a module graph and type-check a program:

import {
  createCompiler, checkCompilation, buildStdlibConfig
} from 'compiler';
import { FsCompilerHost } from 'compiler/fs-host';

let stdlibRoot = '/stdlib';
let host = new FsCompilerHost(stdlibRoot);
let packages = buildStdlibConfig(stdlibRoot, 'host');
let compiler = createCompiler(host, {
  stdlibRoot,
  packages,
});

let result = await compiler.compile({
  entry: '/workspace/main.zena',
  target: 'host',
});
let checked = checkCompilation(result);

Compiling a File to WebAssembly#

Use compiler/compile-file to compile an entry file on disk to a .wasm or .wat output file:

import { compileFile } from 'compiler/compile-file';

let ok = await compileFile({
  entry: 'src/main.zena',
  outPath: 'out/main.wasm',
  stdlibRoot: '/stdlib',
});

Libraries#

  • compiler / compiler/compiler — compiler entry point (Compiler, createCompiler, checkCompilation, ProgramCheckResult)
  • compiler/compile-file — end-to-end file compilation (compileFile, CompileFileOptions)
  • compiler/tokenizer — lexer and token definitions (tokenize, Token, TokenType)
  • compiler/parser — recursive-descent parser (parse, Parser)
  • compiler/ast — AST node definitions (Node, Module, Statement, Expression, TypeAnnotation)
  • compiler/visitor — AST traversal helpers (walk, Visitor)
  • compiler/scope — lexical scope and symbol resolution (ScopeBuilder, ScopeResult, Symbol)
  • compiler/checker — static type checker (checkModule, CheckResult)
  • compiler/semantic-model — semantic queries over a checked module (SemanticModel)
  • compiler/types — compiler type representations (Type, ClassType, FunctionType, typeToString)
  • compiler/diagnostics — compiler diagnostics and source spans (Diagnostic, DiagnosticCode, DiagnosticSeverity, SourceSpan)
  • compiler/package-manifest — zena-package.json manifest and workspace loader (loadWorkspace, loadPackageMap)
  • compiler/module-resolver — package and relative module specifier resolution (ModuleResolver, PackageConfig)
  • compiler/library-loader — module graph loader (LibraryLoader, CompilerHost, SourceFile)
  • compiler/codegen/binary-generator — WebAssembly GC binary emitter (BinaryGenerator)
  • compiler/codegen/wat-generator — WebAssembly Text (WAT) emitter (WatGenerator)

License#

MIT

21.1 KiB Raw
/**
 * Library loading and caching for the Zena compiler.
 *
 * The LibraryLoader manages source loading, parsing, and caching. Each source
 * file is loaded exactly once and cached by its canonical path. This
 * guarantees identity: types from the same library source are always the
 * same object.
 *
 * The loader uses a ModuleResolver for import resolution and a CompilerHost
 * for file I/O.
 */

import { Set, HashMap } from 'zena:collections';
import {
  Module, ImportDeclaration, ExportAllDeclaration, ExportFromDeclaration, Statement, makeNodeIdGenerator, IdGenerator, NodeId,
} from './ast.zena';
import { parse, parseRecovering, ParseCompleteness } from './parser.zena';
import { primeKeywords } from './tokenizer.zena';
import { StringTable } from 'zena:core';
import { Diagnostic } from './diagnostics.zena';
import { ModuleResolver, normalizePath } from './module-resolver.zena';
import { ScopeResult } from './scope/scope-builder.zena';
import { performance } from './time.zena';

// ============================================================================
// Compiler Host
// ============================================================================

/**
 * Abstract interface for file I/O.
 *
 * The compiler host is the only point of contact between the compiler and
 * the outside world. Different implementations provide different backends:
 * - Filesystem (Node.js, WASI)
 * - In-memory (testing, browser playground)
 * - Network (future)
 *
 * Both methods are asynchronous, so a host may answer from the network
 * or from an editor, and a compile holds no task while it waits for a
 * read. A host that has its sources in memory declares the methods
 * `async` and returns directly. See docs/design/async-compiler.md.
 */
export interface CompilerHost {
  /**
   * Returns the source code of a file by its canonical path, or fails
   * when there is no such file.
   *
   * For stdlib modules, the path has the form "zena:module-name". For file
   * modules, the path is an absolute filesystem path.
   */
  readFile(path: String): Future<String>;

  /**
   * Returns the names of the entries in the directory at `path`, or `null` when
   * `path` is not a directory.
   *
   * This is mainly used to read a WIT-backed package's root, which may be a
   * directory of `.wit` files. A host that serves no such package can return
   * `null`.
   */
  listDir(path: String): Future<Array<String> | null>;
}

// ============================================================================
// Source File
// ============================================================================

/**
 * A loaded and parsed source file.
 *
 * SourceFiles are cached by canonical path. Requesting the same path
 * twice always returns the same instance.
 */
export class SourceFile {
  /** Canonical path identifying this library. */
  path: String;

  /** Whether this library is part of the standard library. */
  isStdlib: boolean;

  /** The package this library belongs to (empty string if none). */
  packageName: String;

  /** The original source code. */
  source: String;

  /** Parsed AST. */
  ast: Module;

  /**
   * Opaque version tag for cache invalidation.
   *
   * The compiler itself does not interpret this value. It is set by
   * the host (e.g., the language service passes the editor document
   * version) and compared by the loader to decide whether a cached
   * file is still fresh.
   */
  version: String;

  /**
   * Resolved import mappings.
   * Key: import specifier as written in source (e.g., "./utils", "zena:string")
   * Value: canonical resolved path
   */
  imports = new Map<String, String>();

  /**
   * Scope analysis result (populated after scope building).
   * Contains the scope tree, exports, unresolved names, and references.
   */
  var scopeResult: ScopeResult | null = null;

  /**
   * Errors the parser recovered from, empty unless the loader was told
   * to recover. A file lands here with a usable AST *and* real errors:
   * whoever reports diagnostics has to include these.
   */
  var parseErrors: GrowableArray<Diagnostic> = new Array<Diagnostic>(0);

  /**
   * What this file's AST is fit for. Ask this rather than inferring it
   * from `parseErrors` — the parser owns the answer, and it is the one
   * thing that changes as recovery grows to cover more of the grammar.
   */
  var completeness = ParseCompleteness.Complete;

  new(
    this.path: String,
    this.isStdlib: boolean,
    this.packageName: String,
    this.source: String,
    this.ast: Module,
    this.version: String
  );
}

/**
 * A module's top-level statements, preceded by the import declarations
 * its implicit prelude imports stand for — every import it makes.
 */
export function moduleStatements(lib: SourceFile): GrowableArray<Statement> {
  let statements = new Array<Statement>(lib.ast.body.length);
  let {scopeResult} = lib;
  if (scopeResult != null) {
    for (let decl in scopeResult.implicitImportDeclarations) {
      statements.push(decl);
    }
  }
  for (let stmt in lib.ast.body) {
    statements.push(stmt);
  }
  return statements;
}

// ============================================================================
// Library Graph
// ============================================================================

/** Result of topologically sorting source files by their dependencies. */
export class SourceFileGraph(
  /** Source files in topological order (dependencies before dependents). */
  files: GrowableArray<SourceFile>,

  /** True if a cycle was detected (libraries still returned, best-effort order). */
  hasCycle: boolean
)

// ============================================================================
// Library Loader
// ============================================================================

/**
 * Loads, parses, and caches Zena libraries.
 *
 * The LibraryLoader is responsible for:
 * - Loading library source code (via CompilerHost)
 * - Parsing source into ASTs
 * - Caching loaded libraries by path (ensuring identity)
 * - Recursively loading dependencies (via import resolution)
 *
 * Loading reads a library's imports together and parses them in the
 * order the imports are written: a library is parsed and cached, a read
 * of every import it names starts, and then each import is awaited and
 * loaded in turn. The parse order matters because every parse draws
 * node ids from this loader's one generator, and those ids reach the
 * emitted module; the reads' order does not, since a read changes
 * nothing until its turn.
 *
 * Usage:
 *   let loader = new LibraryLoader(host, resolver);
 *   let entry = await loader.load("/path/to/main.zena");
 *   let graph = loader.computeGraph([entry]);
 */
/** A fresh identifier table with the keywords already in it. */
function primedTable() {
  let table = new StringTable(4096);
  primeKeywords(table);
  return table;
}

export class LibraryLoader {
  #host: CompilerHost;
  #resolver: ModuleResolver;
  #cache: HashMap<String, SourceFile>;

  /**
   * The reads in flight, by path. A library's imports are all read at
   * once and parsed in source order, so a read is started when its
   * importer is parsed and finished when its turn comes; two importers
   * of one file share its read. Cleared when a load starts: a read a
   * failed load left here would serve the next load stale text.
   */
  #pending = new Map<String, Future<String>>();

  /**
   * Node ids must be unique across every file this loader has parsed —
   * they key the semantic model, so two files sharing one id means one
   * file's answers get served for the other's node. The loader owns the
   * generator because the loader owns the parse cache: the set of files
   * the ids have to be unique across is exactly the set it holds.
   *
   * It used to be a defaulted parameter on `load`, which made a
   * mismatched generator something a caller could supply by omission —
   * and `invalidate` and `refreshCache` both did, re-parsing from 1 and
   * colliding with everything already cached.
   */
  #nodeIdGenerator = makeNodeIdGenerator();
  /**
   * The identifier table every parse this loader drives shares. The
   * loader owns it for the same reason it owns the node id generator:
   * the useful scope for interning is exactly the set of files it holds,
   * so a name is copied once however many files mention it and every
   * identifier of one spelling is one `String`. Primed with the
   * keywords, so a lexed `return` interns to the very instance
   * `KEYWORDS` is keyed by.
   */
  #table = primedTable();

  /**
   * The generator every node this loader's modules contain draws its id
   * from, for nodes the compiler adds to them after parsing.
   */
  nodeIdGenerator: IdGenerator<NodeId> {
    get {
      return this.#nodeIdGenerator;
    }
  }

  var loadTime: f64 = 0.0;
  var parseTime: f64 = 0.0;

  /**
   * Whether to recover from the parse errors recovery covers, keeping a
   * partial AST, instead of letting them abort the load. The errors are
   * still reported — they land on `SourceFile.parseErrors`. Editors want
   * this; a build does not.
   */
  #recover: boolean;

  new(host: CompilerHost, resolver: ModuleResolver, recover: boolean = false)
    : #host = host, #resolver = resolver, #recover = recover, #cache = new Map<String, SourceFile>();

  /**
   * Load a library by its canonical path.
   *
   * If the library has already been loaded, returns the cached record.
   * Otherwise: loads source → parses → caches → recursively loads imports.
   *
   * If `version` is provided, the cache is checked for a matching version.
   * A cached file with a different version is invalidated and re-loaded.
   * When omitted (or null), the cache is trusted unconditionally.
   */
  async load(path: String, version: String | null = null): Future<SourceFile> {
    if (this.#cache.has(path)) {
      // Version check: if a version was provided and it matches, use cache.
      // If no version was provided, always trust the cache.
      if (version == null || this.#cache[path].version == version) {
        return this.#cache[path];
      }
      // Version mismatch — invalidate and re-load below.
      this.#cache.delete(path);
    }

    this.#pending.clear();
    let tLoad = performance.start("load");
    let source = await this.#host.readFile(path);
    this.loadTime = this.loadTime + performance.end(tLoad);
    return await this.#register(path, source, version, null);
  }

  /**
   * Load a module the compiler wrote itself, from its source rather
   * than from the host. `pinned` maps import specifiers the source
   * uses to paths that are already loaded, so a synthesized module can
   * import the entry it was written for without the resolver spelling
   * the entry's path a second way and loading it twice. Every other
   * import resolves as usual.
   */
  async loadSynthesized(
    path: String,
    source: String,
    pinned: HashMap<String, String>
  ): Future<SourceFile> {
    if (this.#cache.has(path)) {
      return this.#cache[path];
    }
    return await this.#register(path, source, null, pinned);
  }

  /** Parse `source` as the module at `path`, cache it, and load what it imports. */
  async #register(
    path: String,
    source: String,
    version: String | null,
    pinned: HashMap<String, String> | null
  ): Future<SourceFile> {
    let isStdlib = path.startsWith("zena:") || path.contains("stdlib");
    let packageName = this.#resolver.findPackageForPath(path);

    let tParse = performance.start("parse");
    var parseErrors = new Array<Diagnostic>(0);
    var completeness = ParseCompleteness.Complete;
    let ast = if (this.#recover) {
      let result = parseRecovering(source, path, this.#nodeIdGenerator, this.#table);
      parseErrors = result.diagnostics;
      completeness = result.completeness;
      result.ast
    } else {
      // Without recovery an incomplete parse throws, so reaching here at
      // all means the AST accounts for the whole file.
      parse(source, path, this.#nodeIdGenerator, this.#table)
    };
    this.parseTime = this.parseTime + performance.end(tParse);

    // Cache the record BEFORE resolving imports to handle circular imports.
    let record = new SourceFile(path, isStdlib, packageName, source, ast, version ?? "");
    record.parseErrors = parseErrors;
    record.completeness = completeness;
    this.#cache[path] = record;

    // Resolve every import, and start reading each one that is not
    // loaded yet, so the reads are in flight together. A failure to
    // resolve or load a dependency is left for type checking to report
    // as ModuleNotFound.
    let targets = new Array<String>();
    for (let stmt in ast.body) {
      let specifier = match (stmt) {
        case ImportDeclaration {moduleSpecifier}: moduleSpecifier.value
        case ExportAllDeclaration {moduleSpecifier}: moduleSpecifier.value
        case ExportFromDeclaration {moduleSpecifier}: moduleSpecifier.value
        case _: { continue; }
      };
      let target = this.#resolveImport(record, pinned, specifier);
      if (target != null && !this.#cache.has(target)) {
        targets.push(target);
        this.#startRead(target);
      }
    }

    // Then load them in source order. The order is what keeps node ids
    // where they were: each parse draws from the one generator, so
    // parsing in the order the reads completed would hand out
    // different ids on every run.
    for (let target in targets) {
      if (this.#cache.has(target)) {
        continue;
      }
      let tLoad = performance.start("load");
      let source = await this.#finishRead(target);
      this.loadTime = this.loadTime + performance.end(tLoad);
      if (source == null) {
        // No file there, so this is the missing-library case after all.
        continue;
      }
      // Parsing sits outside the read's catch. A dependency that does
      // not parse throws, and swallowing that would report the file as
      // missing at the import site — naming a file that is sitting right
      // there, and saying nothing about the error or its line.
      await this.#register(target, source, null, null);
    }

    return record;
  }

  /**
   * The path one of `record`'s imports names, recorded on the record;
   * null when there is nothing to load for it.
   */
  #resolveImport(
    record: SourceFile,
    pinned: HashMap<String, String> | null,
    specifier: String
  ): String | null {
    // An empty specifier is the parser's stand-in for one that has not
    // been typed yet (see #missingModuleSpecifier). There is nothing to
    // resolve, so the declaration contributes no dependency.
    if (specifier == "") {
      return null;
    }
    if (pinned != null && pinned.has(specifier)) {
      record.imports[specifier] = pinned[specifier];
      return null;
    }
    // Two failures here mean the import names nothing, and the checker
    // reports both against the import itself: a specifier that does not
    // resolve, and a path with no file behind it.
    try {
      let resolved = this.#resolver.resolve(specifier, record.path);
      record.imports[specifier] = resolved.path;
      return resolved.path;
    } catch (e) {
      // Nothing to load, and nothing more to say here.
      return null;
    }
  }

  /** Starts reading `path` unless a read of it is already in flight. */
  #startRead(path: String): void {
    if (this.#pending.has(path)) {
      return;
    }
    this.#pending[path] = this.#host.readFile(path);
  }

  /**
   * The source of `path`, from the read `#startRead` began, or from one
   * started now; null when the host has no such file.
   */
  async #finishRead(path: String): Future<String | null> {
    this.#startRead(path);
    let read = this.#pending[path];
    this.#pending.delete(path);
    try {
      return await read;
    } catch (e) {
      return null;
    }
  }

  /**
   * Delete a file from the loader cache (e.g. on editor document close/delete).
   *
   * Evicts the file from the cache and clears `scopeResult` on any cached file
   * that directly imports it so import wiring will fail or update on next compile.
   */
  delete(path: String): void {
    let norm = normalizePath(path);
    if (this.#cache.has(norm)) { this.#cache.delete(norm); }
    if (this.#cache.has(path)) { this.#cache.delete(path); }
    // The files that import this one keep their scope trees. Import
    // wiring is redone for every file on every compile (the compiler's
    // wire-to-fixpoint pass re-links each import specifier to the
    // exporter's current Symbol), so nothing about the importer's own
    // scope depends on the file that changed — and a rebuilt scope
    // would mint fresh Symbols, which a check result carried forward
    // for the importer could not answer for (ProgramCheckResult.scopes).
  }

  /**
   * Invalidate a single file by canonical path (push-based).
   *
   * Evicts the file from the cache, then re-loads it from the host so
   * it's ready for the next compile. Files that import it keep their
   * scope trees; the compiler re-wires every import on each compile.
   *
   * Use this in push-based environments (IDE, language service) where
   * the host knows exactly which file changed. For pull-based batch
   * compilation, use `refreshCache()` instead.
   */
  async invalidate(path: String): Future<void> {
    this.delete(path);

    // Re-load so the file is back in the cache with fresh content.
    await this.load(path);
  }

  /**
   * Refresh the loader cache by re-reading non-stdlib files from the host
   * (pull-based).
   *
   * For each cached user file, re-reads the source from the host and
   * compares it to the cached source. If the source changed, the cache
   * entry is removed and re-loaded.
   *
   * Use this in pull-based environments (CLI, batch compilation) where
   * no external notification of file changes is available. In push-based
   * environments (IDE), prefer `invalidate(path)` for targeted updates.
   */
  async refreshCache(): Future<void> {
    // Collect paths first — can't mutate the map during iteration.
    let paths = new Array<String>();
    let iter = this.#cache.keys();
    for (let path in iter) {
      paths.push(path);
    }

    // Phase 1: Detect changed files and remove from cache. Every file
    // is read at once, and compared in order.
    this.#pending.clear();
    let userPaths = new Array<String>();
    for (let path in paths) {
      if (!this.#cache[path].isStdlib) {
        userPaths.push(path);
        this.#startRead(path);
      }
    }
    let changedPaths = new Set<String>();
    for (let path in userPaths) {
      let cached = this.#cache[path];
      let currentSource = await this.#finishRead(path);
      if (currentSource != cached.source) {
        changedPaths.add(path);
        this.#cache.delete(path);
      }
    }

    if (changedPaths.size > 0) {
      // Files importing a changed one keep their scope trees, as in
      // `delete`: import wiring is redone every compile, and a fresh
      // scope would invalidate their carried check results.

      // Phase 2: Re-load evicted files so they're back in the cache
      // before computeGraph() tries to find them.
      for (let path in changedPaths) {
        await this.load(path);
      }
    }
  }

  /** Check if a library has already been loaded. */
  has(path: String): boolean {
    return this.#cache.has(path);
  }

  /** Get a previously loaded library without triggering a load. */
  get(path: String): SourceFile | null {
    if (this.#cache.has(path)) {
      return this.#cache[path];
    }
    return null;
  }

  /**
   * Compute the topological order of libraries starting from a root.
   *
   * Returns libraries in dependency order: if A imports B, then B appears
   * before A. This ordering is required for correct type checking (a module's
   * dependencies must be checked before the module itself).
   */
  computeGraph(entries: GrowableArray<SourceFile>): SourceFileGraph {
    let visited = new Map<String, i32>();
    let stack = new Map<String, i32>();
    let sorted = new Array<SourceFile>();
    var hasCycle = false;
    for (let entry in entries) {
      if (this.#topoVisit(entry, visited, stack, sorted)) {
        hasCycle = true;
      }
    }
    return new SourceFileGraph(sorted, hasCycle);
  }

  #topoVisit(
    lib: SourceFile,
    visited: HashMap<String, i32>,
    stack: HashMap<String, i32>,
    sorted: GrowableArray<SourceFile>
  ): boolean {
    if (visited.has(lib.path)) {
      return false;
    }

    if (stack.has(lib.path)) {
      return true;
    }

    stack[lib.path] = 1;

    var hasCycle = false;

    // Visit dependencies first. Implicit prelude imports precede
    // written imports, matching `moduleStatements` and `beginModule`.
    let {scopeResult} = lib;
    if (scopeResult != null) {
      for (let decl in scopeResult.implicitImportDeclarations) {
        let importPath = decl.moduleSpecifier.value;
        if (this.#cache.has(importPath)) {
          if (this.#topoVisit(this.#cache[importPath], visited, stack, sorted)) {
            hasCycle = true;
          }
        }
      }
    }
    let iter = lib.imports.keys();
    for (let specifier in iter) {
      let importPath = lib.imports[specifier];
      if (this.#cache.has(importPath)) {
        if (this.#topoVisit(this.#cache[importPath], visited, stack, sorted)) {
          hasCycle = true;
        }
      }
    }

    stack.delete(lib.path);
    visited[lib.path] = 1;
    sorted.push(lib);
    return hasCycle;
  }
}
VersionPublishedDependenciesApproved by
0.0.2-preview.1 2026-11-19