compiler 0.0.2-preview.1

The Zena self-hosted compiler

Readme Files Dependencies Versions

compiler#

The Zena compiler, written in Zena. It parses, type-checks, and compiles Zena source code to WebAssembly (Wasm GC) and WebAssembly Text (WAT).

Usage#

Parsing a Single File#

Use compiler/parser to parse Zena source text into an AST (Module):

import { parse } from 'compiler/parser';

let ast = parse('export let answer = 42;', 'example.zena');

Loading and Checking a Program#

Use compiler (createCompiler and checkCompilation) with a CompilerHost to resolve imports across a module graph and type-check a program:

import {
  createCompiler, checkCompilation, buildStdlibConfig
} from 'compiler';
import { FsCompilerHost } from 'compiler/fs-host';

let stdlibRoot = '/stdlib';
let host = new FsCompilerHost(stdlibRoot);
let packages = buildStdlibConfig(stdlibRoot, 'host');
let compiler = createCompiler(host, {
  stdlibRoot,
  packages,
});

let result = await compiler.compile({
  entry: '/workspace/main.zena',
  target: 'host',
});
let checked = checkCompilation(result);

Compiling a File to WebAssembly#

Use compiler/compile-file to compile an entry file on disk to a .wasm or .wat output file:

import { compileFile } from 'compiler/compile-file';

let ok = await compileFile({
  entry: 'src/main.zena',
  outPath: 'out/main.wasm',
  stdlibRoot: '/stdlib',
});

Libraries#

  • compiler / compiler/compiler — compiler entry point (Compiler, createCompiler, checkCompilation, ProgramCheckResult)
  • compiler/compile-file — end-to-end file compilation (compileFile, CompileFileOptions)
  • compiler/tokenizer — lexer and token definitions (tokenize, Token, TokenType)
  • compiler/parser — recursive-descent parser (parse, Parser)
  • compiler/ast — AST node definitions (Node, Module, Statement, Expression, TypeAnnotation)
  • compiler/visitor — AST traversal helpers (walk, Visitor)
  • compiler/scope — lexical scope and symbol resolution (ScopeBuilder, ScopeResult, Symbol)
  • compiler/checker — static type checker (checkModule, CheckResult)
  • compiler/semantic-model — semantic queries over a checked module (SemanticModel)
  • compiler/types — compiler type representations (Type, ClassType, FunctionType, typeToString)
  • compiler/diagnostics — compiler diagnostics and source spans (Diagnostic, DiagnosticCode, DiagnosticSeverity, SourceSpan)
  • compiler/package-manifest — zena-package.json manifest and workspace loader (loadWorkspace, loadPackageMap)
  • compiler/module-resolver — package and relative module specifier resolution (ModuleResolver, PackageConfig)
  • compiler/library-loader — module graph loader (LibraryLoader, CompilerHost, SourceFile)
  • compiler/codegen/binary-generator — WebAssembly GC binary emitter (BinaryGenerator)
  • compiler/codegen/wat-generator — WebAssembly Text (WAT) emitter (WatGenerator)

License#

MIT

22.6 KiB Raw
// The standard library's own package manifest, and the mapping from a
// stdlib module id back to the library that owns it.
//
// This lives apart from `package-manifest.zena`, which reads a workspace's
// `zena-package.json` files and therefore depends on `CompilerHost` and the
// loader. Nothing here needs either, so files as low in the import graph as
// `types.zena` can reach the stdlib layout without a cycle.

import { PackageConfig, ModuleExport } from './module-resolver.zena';
import { HashMap } from 'zena:collections';

// Build the standard library PackageConfig.
//
// The `stdlibRoot` is the absolute path to the stdlib's zena source
// directory (e.g., "/path/to/packages/stdlib/zena").
//
// This is kept separate from user packages because the stdlib is a
// language-level concept, not a user dependency. The compiler always
// includes it. Eventually, the stdlib source could be embedded in the
// compiler binary, eliminating the need for a path.
export function buildStdlibConfig(stdlibRoot: String): PackageConfig {
  let exports = new Map<String, ModuleExport>();

  // Public modules (no virtual mapping).
  // Keep in sync with packages/stdlib/stdlib-manifest.json.
  let publicModules = [
    "assert",
    "async",
    "encoding",
    "glob",
    "json",
    "math",
    "path",
    "regex",
    "signals",
    "task",
    "test"
  ];
  var i = 0;
  while (i < publicModules.length) {
    exports[publicModules[i]] = new ModuleExport(null);
    i += 1;
  }

  // Virtual module: console → per-target entry file. The implementation
  // files live under console/ and are not importable by name. Every
  // target has one, because `console` is in every target's prelude and
  // code that prints has to be portable.
  //
  // Virtual module: memory → the linear-memory allocator. One
  // implementation everywhere but the component target, where the
  // memory belongs to the runtime memory module and the allocator
  // delegates to its `realloc` — two allocators handing out blocks from
  // the same pages would corrupt each other, so the component entry
  // keeps no allocation state at all.
  let memoryVirtual: HashMap<String, String> = {
    "core" => "memory.zena",
    "js" => "memory.zena",
    "zena-js" => "memory.zena",
    "component" => "memory/component.zena",
    "zena-component" => "memory/component.zena",
  };
  exports["memory"] = new ModuleExport(memoryVirtual);

  let consoleVirtual = new Map<String, String>();
  consoleVirtual["js"] = "console/none.zena";
  consoleVirtual["zena-js"] = "console/js.zena";
  consoleVirtual["zena-component"] = "console/component.zena";
  // The component console prints over p3 stdio. `core` shares it: a
  // `core` build is a component's core module without the wrapper, and
  // the two must load the same prelude for the wrapper to embed exactly
  // the module the `core` build emits.
  consoleVirtual["component"] = "console/component.zena";
  consoleVirtual["core"] = "console/component.zena";
  exports["console"] = new ModuleExport(consoleVirtual);

  // Virtual module: wasi → the guest side of the Component Model. The
  // canonical ABI's string marshaling and allocator, the async-import
  // driver that turns the subtask protocol into a `Future`, and the
  // guest half of `stream<u8>`. Only a component has any of that
  // beneath it, so the component target resolves it, and `core` with
  // it: a `core` build is a component's core module without the
  // wrapper, and its prelude's console is the component's. The
  // component runtime's own memory entry sits beside these files
  // without being one of them.
  exports["wasi"] = new ModuleExport(null, "wasi/index.zena");

  // Virtual module: error-stack → the `Error` stack-trace hooks. The host
  // hooks trade in `anyref`/`externref`, which have no Component Model
  // representation, so a component (and a `core` or `js` module, which
  // assumes no Zena host) gets the null implementation instead. Every
  // module pulls this in through `zena:error`, so it is the one import
  // that a target cannot opt out of by writing less code — see
  // docs/design/component-emission.md, Part 2.
  let errorStackVirtual = new Map<String, String>();
  errorStackVirtual["core"] = "core/error/stack-none.zena";
  errorStackVirtual["js"] = "core/error/stack-none.zena";
  errorStackVirtual["zena-js"] = "core/error/stack-host.zena";
  // `zena-runtime` formats the backtrace over `zena:host/stack-trace`,
  // installed at run time by `zena:host-setup` (below); a portable
  // component has no host to ask.
  errorStackVirtual["component"] = "core/error/stack-none.zena";
  errorStackVirtual["zena-component"] = "core/error/stack-hooked.zena";
  // Internal: `zena:core` is the only thing that reaches it, through
  // `core/error.zena` and its own entry point. The manifest entry is
  // what picks the file per target, which is the only reason the name
  // exists at all.
  exports["error-stack"] = new ModuleExport(errorStackVirtual, null, true);

  // Virtual module: host-setup → what a program on `zena-runtime` needs
  // installed before it runs: the host as the source of `Error`'s stack
  // traces. A runtime module of the `zena-component` target
  // (`prelude.zena`'s `getTargetRuntimeModules`), whose `installHost`
  // the component entry calls before `main`.
  let hostSetupVirtual: HashMap<String, String> = {
    "zena-component" => "host-setup.zena",
  };
  exports["host-setup"] = new ModuleExport(hostSetupVirtual);

  // Virtual module: fs → the file system, over wasi:filesystem 0.3 on
  // both component targets. The 0.3 filesystem is asynchronous
  // throughout, and so is the API.
  let fsVirtual: HashMap<String, String> = {
    "component" => "fs/component.zena",
    "zena-component" => "fs/component.zena",
  };
  exports["fs"] = new ModuleExport(fsVirtual);

  // Virtual module: cli → arguments, environment and exit, over
  // wasi:cli 0.3.
  let cliVirtual: HashMap<String, String> = {
    "component" => "cli/component.zena",
    "zena-component" => "cli/component.zena",
  };
  exports["cli"] = new ModuleExport(cliVirtual);

  // Virtual module: process → host process spawning, on `zena-component`
  // only, which links it for trusted invocations; see the library header.
  let processVirtual: HashMap<String, String> = {
    "zena-component" => "process/component.zena",
  };
  exports["process"] = new ModuleExport(processVirtual);

  // Virtual module: wasm → running other Wasm modules, on `zena-component`
  // only: it needs the `zena:host/wasm` host imports. Granted with
  // process spawning and linked by the same hosts; see the library header.
  let wasmVirtual: HashMap<String, String> = {
    "zena-component" => "wasm/component.zena",
  };
  exports["wasm"] = new ModuleExport(wasmVirtual);

  // Virtual module: extensions → loading other components as long-lived
  // installations, over `zena:host/extensions`. On the `component`
  // target too, so a program built against a declared world can load
  // them where `zena-runtime` runs it.
  let extensionsVirtual: HashMap<String, String> = {
    "component" => "extensions/component.zena",
    "zena-component" => "extensions/component.zena",
  };
  exports["extensions"] = new ModuleExport(extensionsVirtual);

  // Virtual module: js → `@zena-lang/runtime` interop, currently the
  // host-async completion registry. Only `zena-js` has an embedder that
  // pushes completions back in through the module's exports; the
  // component targets' futures ride the canonical ABI's waitables instead.
  let jsVirtual: HashMap<String, String> = {"zena-js" => "js/index.zena"};
  exports["js"] = new ModuleExport(jsVirtual);

  // Modules: web-templates and web-elements → the Wasm Web UI protocol
  // (docs/design/wasm-web-ui.md). One source serves every target;
  // what differs per host is behind one internal virtual file each: the
  // DOM host's imports and string crossings on `js` and `zena-js`, and
  // their stand-ins elsewhere, which throw or do nothing. The `webui_*`
  // exports are shared files that only a JS-string compile roots.
  exports["web-templates"] = new ModuleExport(null, "web-templates/index.zena");
  let htmlHostVirtual: HashMap<String, String> = {
    "core" => "web-templates/host-none.zena",
    "js" => "web-templates/host-js.zena",
    "zena-js" => "web-templates/host-js.zena",
    "component" => "web-templates/host-none.zena",
    "zena-component" => "web-templates/host-none.zena",
  };
  exports["web-templates/host.zena"] = new ModuleExport(htmlHostVirtual, null, true);
  exports["web-elements"] = new ModuleExport(null, "web-elements/index.zena");
  let elementsHostVirtual: HashMap<String, String> = {
    "core" => "web-elements/host-none.zena",
    "js" => "web-elements/host-js.zena",
    "zena-js" => "web-elements/host-js.zena",
    "component" => "web-elements/host-none.zena",
    "zena-component" => "web-elements/host-none.zena",
  };
  exports["web-elements/host.zena"] = new ModuleExport(elementsHostVirtual, null, true);

  // Virtual module: fetch → HTTP requests. `zena-js` goes through
  // `@zena-lang/runtime`'s `fetch()` import; the component targets speak
  // wasi:http through the bindings synthesized from the stdlib's WIT.
  let fetchVirtual: HashMap<String, String> = {
    "zena-js" => "fetch/host.zena",
    "component" => "fetch/component.zena",
    "zena-component" => "fetch/component.zena",
  };
  exports["fetch"] = new ModuleExport(fetchVirtual);

  // Module: http → HTTP server and client support, Express-like App and WASI
  // HTTP handler.
  exports["http"] = new ModuleExport(null, "http/index.zena");
  // Module: crypto → SHA-256, HMAC, HKDF and ECDSA, implemented in private
  // files under crypto/.
  exports["crypto"] = new ModuleExport(null, "crypto/index.zena");


  // Virtual module: time → sleeping and the monotonic clock, over a
  // `@zena-lang/runtime` import on `zena-js` and wasi:clocks 0.3 on
  // component targets.
  let timeVirtual = new Map<String, String>();
  timeVirtual["zena-js"] = "time/host.zena";
  timeVirtual["component"] = "time/component.zena";
  timeVirtual["zena-component"] = "time/component.zena";
  exports["time"] = new ModuleExport(timeVirtual);

  // Virtual module: random → the host's cryptographically secure
  // generator. The public file is the same on every target that has a
  // generator; what differs is the internal `random/source.zena` it
  // imports. `core` and `js` assume no host generator, so importing
  // `zena:random` there reports the library unavailable on that target.
  let randomVirtual: HashMap<String, String> = {
    "zena-js" => "random/index.zena",
    "component" => "random/index.zena",
    "zena-component" => "random/index.zena",
  };
  exports["random"] = new ModuleExport(randomVirtual);
  let randomSourceVirtual: HashMap<String, String> = {
    "zena-js" => "random/source-js.zena",
    "component" => "random/source-wasi.zena",
    "zena-component" => "random/source-wasi.zena",
  };
  exports["random/source.zena"] = new ModuleExport(randomSourceVirtual, null, true);


  // Directory-based modules: entry file inside a library directory. The
  // sibling implementation files are private — reachable only via relative
  // imports from the library's own files.
  // The rendezvous stream is its own library, not part of zena:async: the
  // prelude names zena:async, so anything folded in is parsed and checked
  // by every compilation whether or not it is used.
  exports["stream"] = new ModuleExport(null, "stream/index.zena");
  // Its twenty-three members have no module name of their own: they are
  // private files of this library, reached through this entry point or by
  // relative import from inside it. `error-stack`, `string`,
  // `string-js-ops`, `string-builder`, `number-format` and `number-parse`
  // are the exceptions, and they are virtual — the manifest is what picks
  // a file per target, so a relative import would nail one of them.
  exports["core"] = new ModuleExport(null, "core/index.zena");
  let stringVirtual: HashMap<String, String> = {
    "core" => "core/string.zena",
    "js" => "core/string-js.zena",
    "zena-js" => "core/string-js.zena",
    "component" => "core/string.zena",
    "zena-component" => "core/string.zena",
  };
  exports["core/string.zena"] = new ModuleExport(stringVirtual, null, true);
  let stringJsOpsVirtual: HashMap<String, String> = {
    "js" => "core/string-js-pure.zena",
    "zena-js" => "core/string-js-host.zena",
  };
  exports["core/string-js-ops.zena"] =
      new ModuleExport(stringJsOpsVirtual, null, true);
  // A JS `String` is a JavaScript string, so its builder concatenates
  // JavaScript strings instead of copying bytes.
  let stringBuilderVirtual: HashMap<String, String> = {
    "core" => "core/string-builder.zena",
    "js" => "core/string-builder-js.zena",
    "zena-js" => "core/string-builder-js.zena",
    "component" => "core/string-builder.zena",
    "zena-component" => "core/string-builder.zena",
  };
  exports["core/string-builder.zena"] =
      new ModuleExport(stringBuilderVirtual, null, true);
  // `zena-js` delegates number formatting and parsing to `@zena-lang/runtime`;
  // `js` and the byte targets use the pure-Zena implementations.
  let numberFormatVirtual: HashMap<String, String> = {
    "core" => "core/number-format.zena",
    "js" => "core/number-format.zena",
    "zena-js" => "core/number-format-js.zena",
    "component" => "core/number-format.zena",
    "zena-component" => "core/number-format.zena",
  };
  exports["core/number-format.zena"] =
      new ModuleExport(numberFormatVirtual, null, true);
  let numberParseVirtual: HashMap<String, String> = {
    "core" => "core/number-parse.zena",
    "js" => "core/number-parse.zena",
    "zena-js" => "core/number-parse-js.zena",
    "component" => "core/number-parse.zena",
    "zena-component" => "core/number-parse.zena",
  };
  exports["core/number-parse.zena"] =
      new ModuleExport(numberParseVirtual, null, true);
  exports["url"] = new ModuleExport(null, "url/index.zena");
  // `URL` itself: the WHATWG parser in Zena on most targets (including
  // pure `js`), and `@zena-lang/runtime`'s host `URL` on `zena-js`.
  let urlVirtual: HashMap<String, String> = {
    "core" => "url/url.zena",
    "js" => "url/url.zena",
    "zena-js" => "url/url-js.zena",
    "component" => "url/url.zena",
    "zena-component" => "url/url.zena",
  };
  exports["url/url.zena"] = new ModuleExport(urlVirtual, null, true);
  exports["args"] = new ModuleExport(null, "args/index.zena");
  exports["bench"] = new ModuleExport(null, "bench/index.zena");
  exports["bigint"] = new ModuleExport(null, "bigint/index.zena");
  exports["collections"] = new ModuleExport(null, "collections/index.zena");
  exports["decimal"] = new ModuleExport(null, "decimal/index.zena");
  exports["simd"] = new ModuleExport(null, "simd/index.zena");
  exports["string"] = new ModuleExport(null, "string/index.zena");

  let config = new PackageConfig(stdlibRoot, exports);
  // `zena:host/*` WIT interfaces (`stack-trace`, `process`, `wasm`,
  // `extensions`, `http-server`, `secrets`, `terminal`) live under
  // `host-wit/` and resolve alongside the standard library's `.zena`
  // modules.
  config.witRoot = `${stdlibRoot}/host-wit`;
  return config;
}

// ============================================================================
// Module id → owning library
// ============================================================================

// Module id (without the "zena:" prefix) → the library that owns it.
//
// Built once. The stdlib layout is compiled into the compiler rather than
// read from the program being compiled, so the index is the same for every
// compilation and a long-lived compiler can keep it.
var libraryIndex: HashMap<String, String> | null = null;
var moduleNameIndex: HashMap<String, String> | null = null;

// Every id the manifest names outright: each library's own name, and each
// entry file a `path` or `virtual` mapping points at. A library's private
// files are not in here — `stdlibLibraryOf` attributes those by directory.
function buildLibraryIndex() {
  let index = new Map<String, String>();
  let moduleNames = new Map<String, String>();
  // The root is only used to resolve entry files on disk, which nothing
  // reading this index does.
  let config = buildStdlibConfig("");
  let exports = config.exports as HashMap<String, ModuleExport>;
  exports.forEach((name, entry) => {
    // An internal entry named for a file, such as "core/string.zena" or
    // "url/url.zena", belongs to the library of its directory.
    let owningLibrary = if (entry.internal) {
      firstSegment(name)
    } else {
      name
    };
    index[name] = owningLibrary;
    moduleNames[name] = name;
    let path = entry.path;
    if (path != null) {
      index[path as String] = owningLibrary;
      moduleNames[path as String] = name;
    }
    let virtual = entry.virtual;
    if (virtual != null) {
      (virtual as HashMap<String, String>).forEach(
        (target, entryPath) => {
          index[entryPath] = owningLibrary;
          moduleNames[entryPath] = name;
        });
    }
  });
  libraryIndex = index;
  moduleNameIndex = moduleNames;
  return index;
}

// The first path segment of `id`, or the whole id if it has no "/".
function firstSegment(id: String) {
  var i = 0;
  while (i < id.length) {
    if (id.getByteAt(i) == 47) { // 47 = '/'
      return id.sliceBytes(0, i);
    }
    i += 1;
  }
  return id;
}

/**
 * The stdlib library that owns `moduleId`, or null if no library does.
 *
 * `moduleId` is a canonical module id as the resolver produces it, in
 * either of its two forms: a name id for a library's default entry file
 * ("zena:string"), or a path id for every other stdlib file
 * ("zena:async/future.zena", "zena:console/host.zena"). Anything without
 * the "zena:" prefix is a user module and owned by no library, which is
 * the answer for most calls and is why it is the first thing tested.
 *
 * A path id the manifest names is owned by the library it is an entry for,
 * which is why "core/error/stack-host.zena" belongs to `error-stack` and not
 * to `core` despite sitting under `core/`. A path id the manifest does not name
 * is one of a library's private implementation files, and belongs to the
 * library its directory is named for.
 *
 * Every comparison is exact. The directory rule tests one path segment for
 * membership in the set of published libraries; nothing here matches a
 * prefix, suffix or substring, so a module cannot be attributed to a
 * library whose name it merely begins with.
 */
/**
 * The published module name whose entry file is `entryPath`, or null if
 * the manifest names no module for it.
 *
 * `entryPath` is relative to the stdlib source root — "component-stream.zena",
 * "console/host.zena" — which is the form a canonical path id carries after
 * its "zena:" prefix. Only entry files match: a library's private
 * implementation files have no module name, and this reports that rather
 * than inventing one from the file name.
 */
function getModuleNameIndex() {
  if (moduleNameIndex == null) {
    buildLibraryIndex();
  }
  let m = moduleNameIndex;
  if (m != null) {
    return m as HashMap<String, String>;
  }
  return new Map<String, String>();
}

export function stdlibModuleNameForEntryPath(entryPath: String): String | null {
  let index = getModuleNameIndex();
  if (let (true, name) = index.get(entryPath)) {
    return name;
  }
  return null;
}

export function stdlibLibraryOf(moduleId: String): String | null {
  if (!moduleId.startsWith("zena:")) {
    return null;
  }
  let index = if (libraryIndex != null) {
    libraryIndex as HashMap<String, String>
  } else {
    buildLibraryIndex()
  };
  let id = moduleId.sliceBytes(5, moduleId.length);
  if (let (true, library) = index.get(id)) {
    return library;
  }
  let dir = firstSegment(id);
  if (dir == id) {
    // No directory to fall back on: an unrecognized top-level id.
    return null;
  }
  if (let (true, library) = index.get(dir)) {
    // A private file of `library`, but only if the directory is the
    // library's own name — an entry path that happens to sit one level
    // down was already matched above.
    if (library == dir) {
      return library;
    }
  }
  return null;
}

/**
 * The library a library's files are moving into, for the length of the move.
 *
 * The counterpart of `movingTo` in the well-known registry, which does this
 * for a single declaration. A lookup that names a library instead — the
 * roots taken by `queueStdlibFunction` and `getStdlibFunc` — needs the same
 * grace for the same reason: the checked-in bootstrap answers with ITS
 * manifest, which does not name the new path until the reseed after the
 * files land, so until then `stdlibLibraryOf` attributes a moved file to
 * the directory it now sits in.
 *
 * An entry is one exact destination, not a relaxation. There is deliberately
 * no fallback to `core` for libraries with no entry here: that would let a
 * lookup that should have failed find something plausible in another
 * library. Each entry is deleted by the reseed after its files land.
 */
var libraryMoves: HashMap<String, String> | null = null;

function buildLibraryMoves() {
  // Empty between migrations, which is the resting state: every library is
  // where the manifest says it is, so a lookup naming one has exactly one
  // answer. An entry is added for the length of a move and deleted by the
  // reseed after its files land. See docs/design/stdlib-organization.md.
  let t = new Map<String, String>();
  libraryMoves = t;
  return t;
}

/**
 * Whether the library at `path` is a host boundary: a library whose
 * exports the host calls from outside the program, so nothing inside
 * the program references them and reachability roots them instead.
 * `zena:js` holds the host-async completion entry points (async.md §4);
 * the protocol files of `zena:web-templates` and `zena:web-elements`
 * hold the Wasm Web UI protocol's `webui_*` functions
 * (docs/design/wasm-web-ui.md), which only a js-target compile roots.
 */
export function isHostBoundaryLibrary(path: String): boolean {
  return path == "zena:js/index.zena" || isWebUiProtocolLibrary(path);
}

/**
 * Whether the library at `path` is one whose exported functions named
 * with the Wasm Web UI prefix are exported from the module under
 * their own names, on the js target.
 */
export function isWebUiProtocolLibrary(path: String): boolean {
  return path == "zena:web-templates/protocol.zena" || path == "zena:web-elements/protocol.zena";
}

/**
 * Whether a module owned by `owner` answers a lookup for `library` — the
 * same library, or the one it is moving into while it moves.
 */
export function stdlibLibraryAnswers(owner: String, library: String): boolean {
  if (owner == library) {
    return true;
  }
  let moves = if (libraryMoves != null) {
    libraryMoves as HashMap<String, String>
  } else {
    buildLibraryMoves()
  };
  if (let (true, destination) = moves.get(library)) {
    return owner == destination;
  }
  return false;
}
VersionPublishedDependenciesApproved by
0.0.2-preview.1 2026-11-19