compiler 0.0.2-preview.1

The Zena self-hosted compiler

Readme Files Dependencies Versions

compiler#

The Zena compiler, written in Zena. It parses, type-checks, and compiles Zena source code to WebAssembly (Wasm GC) and WebAssembly Text (WAT).

Usage#

Parsing a Single File#

Use compiler/parser to parse Zena source text into an AST (Module):

import { parse } from 'compiler/parser';

let ast = parse('export let answer = 42;', 'example.zena');

Loading and Checking a Program#

Use compiler (createCompiler and checkCompilation) with a CompilerHost to resolve imports across a module graph and type-check a program:

import {
  createCompiler, checkCompilation, buildStdlibConfig
} from 'compiler';
import { FsCompilerHost } from 'compiler/fs-host';

let stdlibRoot = '/stdlib';
let host = new FsCompilerHost(stdlibRoot);
let packages = buildStdlibConfig(stdlibRoot, 'host');
let compiler = createCompiler(host, {
  stdlibRoot,
  packages,
});

let result = await compiler.compile({
  entry: '/workspace/main.zena',
  target: 'host',
});
let checked = checkCompilation(result);

Compiling a File to WebAssembly#

Use compiler/compile-file to compile an entry file on disk to a .wasm or .wat output file:

import { compileFile } from 'compiler/compile-file';

let ok = await compileFile({
  entry: 'src/main.zena',
  outPath: 'out/main.wasm',
  stdlibRoot: '/stdlib',
});

Libraries#

  • compiler / compiler/compiler — compiler entry point (Compiler, createCompiler, checkCompilation, ProgramCheckResult)
  • compiler/compile-file — end-to-end file compilation (compileFile, CompileFileOptions)
  • compiler/tokenizer — lexer and token definitions (tokenize, Token, TokenType)
  • compiler/parser — recursive-descent parser (parse, Parser)
  • compiler/ast — AST node definitions (Node, Module, Statement, Expression, TypeAnnotation)
  • compiler/visitor — AST traversal helpers (walk, Visitor)
  • compiler/scope — lexical scope and symbol resolution (ScopeBuilder, ScopeResult, Symbol)
  • compiler/checker — static type checker (checkModule, CheckResult)
  • compiler/semantic-model — semantic queries over a checked module (SemanticModel)
  • compiler/types — compiler type representations (Type, ClassType, FunctionType, typeToString)
  • compiler/diagnostics — compiler diagnostics and source spans (Diagnostic, DiagnosticCode, DiagnosticSeverity, SourceSpan)
  • compiler/package-manifest — zena-package.json manifest and workspace loader (loadWorkspace, loadPackageMap)
  • compiler/module-resolver — package and relative module specifier resolution (ModuleResolver, PackageConfig)
  • compiler/library-loader — module graph loader (LibraryLoader, CompilerHost, SourceFile)
  • compiler/codegen/binary-generator — WebAssembly GC binary emitter (BinaryGenerator)
  • compiler/codegen/wat-generator — WebAssembly Text (WAT) emitter (WatGenerator)

License#

MIT

23.8 KiB Raw
// What a component wraps around the core module: which WIT interfaces it
// imports, which core exports it lifts, and the component-level types of
// both.
//
// Read off the same `WasmModule` the core module was emitted from, so
// the two cannot disagree about what exists: the import list is
// `functionImports` in emission order, and the export list is the
// program's own exports (`exportMap`) — the synthesized host-interop
// helpers ($stringCreate and friends, __zena_drain) stay core-module
// exports, because they exist for an embedder that instantiates the core
// module directly and have no component-level meaning.
//
// Types come from the *Zena* signature rather than the wasm one. A wasm
// `i32` is `s32`, `u32`, `u8`, `bool` and four more, and an import's
// declared type has to match what the host actually exports — so
// throwing away the Zena type and re-deriving it from the lowered form
// would be a guess. See docs/design/component-emission.md, Part 4.

import { WasmModule, WasmFunction } from './wasm-module.zena';
import {
  Type, FunctionType, TypeAliasType, EnumType, ClassType, I32Type, U32Type, I64Type,
  U64Type, F32Type, F64Type, U8Type, U16Type, I8Type, I16Type, BooleanType,
  VoidType, typeToString, baseType
} from '../types.zena';
import {
  ComponentShape, ComponentImportFunc, ComponentExportFunc, ComponentFuncType,
  ComponentCanonBuiltin, ComponentValType, noComponentResult, isKebabName,
  toKebabName, canonNamespace, isKnownCanonBuiltin, isTypedCanonBuiltin,
  componentEntryExport, ComponentDtorExport,
  componentCallbackExport, componentTypeNeedsMemory,
  componentSignatureNeedsMemory, componentRuntimeNamespace
} from './component-emitter.zena';
import {
  ImportTypeEncoder, canonBuiltinBaseName
} from './component-emitter.zena';

/** Look through type aliases (`type`, `distinct type`) to the type itself. */
/**
 * The type a value of `t` is represented as: through distinct aliases
 * and enums to the scalar under them, and from a literal type (an
 * unannotated `main` returning `false` is typed at the literal) to
 * its base.
 */
function underlying(t: Type): Type {
  return match (t) {
  case TypeAliasType as alias: underlying(alias.target)
  case EnumType as et: underlying(et.backingType)
  case _: baseType(t)
};
}

/**
 * Whether `t` is the standard library's `String`, compared by identity
 * against the class the module already found rather than by name. A
 * program is free to declare a `String` of its own, and it does not
 * cross the boundary as `string`.
 */
function isStringType(wasm: WasmModule, t: Type): boolean {
  let {stringClass} = wasm;
  if (stringClass == null) {
    return false;
  }
  let u = underlying(t);
  if (!(u is ClassType)) {
    return false;
  }
  return u.symbolId == stringClass.symbolId;
}

/**
 * The component-level type of a Zena parameter or result type, or -1 if
 * it has none.
 *
 * The flat scalars need nothing but their opcode. `String` is the one
 * type beyond them so far, and it is a different kind of thing: the
 * component-level value is bytes in linear memory, so a signature
 * carrying one also carries the canonical options and a synthesized
 * wrapper to do the copying. Everything else still waits on C3's type
 * encoder.
 */
export function componentTypeOf(wasm: WasmModule, t: Type): i32 {
  if (isStringType(wasm, t)) {
    return ComponentValType.String as i32;
  }
  return match (underlying(t)) {
    case I32Type: ComponentValType.S32 as i32
    case U32Type: ComponentValType.U32 as i32
    case I64Type: ComponentValType.S64 as i32
    case U64Type: ComponentValType.U64 as i32
    case F32Type: ComponentValType.F32 as i32
    case F64Type: ComponentValType.F64 as i32
    case U8Type: ComponentValType.U8 as i32
    case U16Type: ComponentValType.U16 as i32
    case I8Type: ComponentValType.S8 as i32
    case I16Type: ComponentValType.S16 as i32
    case BooleanType: ComponentValType.Bool as i32
    case _: -1
  };
}

/** The `FunctionType` a function was declared with. */
export function declaredTypeOf(func: WasmFunction, what: String): FunctionType {
  let {node} = func;
  if (node == null) {
    throw new Error(
      "The component target cannot " + what + " '" + nameOf(func) +
        "': it has no declaration to read a signature from."
    );
  }
  let t = func.model.getNodeType(node);
  if (t == null || !(t is FunctionType)) {
    throw new Error(
      "The component target cannot " + what + " '" + nameOf(func) +
        "': its declaration has no function type."
    );
  }
  return t;
}

function nameOf(func: WasmFunction): String {
  let n = func.name;
  if (n != null) {
    return n;
  }
  let m = func.methodName;
  if (m != null) {
    return m;
  }
  return "<unnamed>";
}

/**
 * A parameter's name at the component level. Zena identifiers are
 * camelCase and cannot contain `-`, while the Component Model's names
 * are kebab-case, so the identifier is converted. Where it has no valid
 * rendering, the position stands in: parameter names take no part in
 * import subtyping — a component importing `wait-for` with the
 * parameter named `how-longx` links against a host declaring
 * `how-long` — so the name is documentation, and only its *shape* is
 * load-bearing.
 */
function paramName(declared: FunctionType, index: i32) {
  let names = declared.parameterNames;
  if (names != null) {
    if (index < names.length) {
      let kebab = toKebabName(names[index]);
      if (kebab != "") {
        return kebab;
      }
    }
  }
  return "a" + i32ToString(index);
}

/** What the boundary carries so far, for the message that says it does not. */
function crossesToday() {
  return "Only u8-u64, i8-i64, f32, f64, boolean and String cross the boundary " +
    "today; richer types need C3's type encoder " +
    "(docs/design/component-emission.md).";
}

/**
 * A `string` can be lifted but not yet lowered, and the reason is
 * structural rather than a missing case.
 *
 * Lifting happens after instantiation, so an export takes its memory and
 * `realloc` from the program module's own exports. A lowered *import*
 * has to exist before the module is instantiated — it is one of the
 * module's imports — so its memory has to come from somewhere that
 * exists earlier, which means a second core module owning linear memory
 * and the allocator. See docs/design/component-emission.md 1.3; it is
 * what C3 builds for stdio.
 */
function checkMemoryDirection(
  mapped: i32,
  func: WasmFunction,
  what: String
) {
  if (what == "import" && componentTypeNeedsMemory(mapped)) {
    throw new Error(
      "The component target cannot import '" + nameOf(func) +
        "': it carries a string, and lowering one needs a memory that " +
        "exists before the program module is instantiated. Exports can " +
        "carry strings today; imports wait on the runtime memory module " +
        "(docs/design/component-emission.md 1.3)."
    );
  }
}

/**
 * The component-level signature of a function, or a loud failure.
 *
 * `isAsync` is part of the *type*, not just the lowering: the canonical
 * `async` option is only legal against a function type that is itself
 * async, which is the format saying the two cannot disagree.
 */
function signatureOf(
  wasm: WasmModule,
  func: WasmFunction,
  what: String,
  isAsync: boolean = false
) {
  let declared = declaredTypeOf(func, what);
  let paramTypes = new Array<i32>();
  let paramNames = new Array<String>();
  for (var i = 0; i < declared.parameters.length; i += 1) {
    let mapped = componentTypeOf(wasm, declared.parameters[i]);
    if (mapped < 0) {
      throw new Error(
        "The component target cannot " + what + " '" + nameOf(func) +
          "': parameter " + i32ToString(i) + " has type '" +
          typeToString(declared.parameters[i]) + "'. " + crossesToday()
      );
    }
    checkMemoryDirection(mapped, func, what);
    paramTypes.push(mapped);
    paramNames.push(paramName(declared, i));
  }
  var result = noComponentResult;
  if (isAsync) {
    // An async-lowered call returns a *subtask handle*, packed with a
    // status — that is the core signature the canonical ABI hands the
    // guest, and it has nothing to do with the WIT function's own
    // result. The declaration says `i32` because that is what the call
    // evaluates to in Zena; the interface type says nothing, because a
    // WIT function that returns something under an async lowering
    // delivers it through the subtask, which needs C6's futures.
    if (!(underlying(declared.returnType) is I32Type)) {
      throw new Error(
        "The async-lowered import '" + nameOf(func) + "' must be declared " +
          "to return i32, the packed subtask handle the canonical ABI " +
          "hands back. It returns '" + typeToString(declared.returnType) +
          "'."
      );
    }
    return new ComponentFuncType(paramTypes, paramNames, result, true);
  }
  if (!(underlying(declared.returnType) is VoidType)) {
    result = componentTypeOf(wasm, declared.returnType);
    if (result < 0) {
      throw new Error(
        "The component target cannot " + what + " '" + nameOf(func) +
          "': it returns '" + typeToString(declared.returnType) + "'. " +
          crossesToday()
      );
    }
    checkMemoryDirection(result, func, what);
  }
  return new ComponentFuncType(paramTypes, paramNames, result, isAsync);
}

/**
 * The shape of the component wrapping `wasm`'s core module.
 *
 * Throws rather than dropping anything it cannot represent. A dropped
 * import would produce a component that fails to instantiate against a
 * host that has the function, and a dropped export would produce one
 * silently missing the entry point the program wrote — both worse than
 * a compile error naming the function.
 */
export function buildComponentShape(
  wasm: WasmModule,
  importEncoder: ImportTypeEncoder | null
): ComponentShape {
  let shape = new ComponentShape();
  // On this target the program module never owns its memory: if it
  // touches linear memory at all, it imports the runtime memory
  // module's, and the emitter must embed that module first.
  shape.needsRuntimeMemory = wasm.hasMemory;

  for (let f in wasm.functionImports) {
    let module = f.importModule;
    let field = f.importName;
    if (module == null || field == null) {
      continue;
    }
    // The canonical builtins: functions the ABI itself provides, with
    // no interface to import them from and no type to declare. The core
    // module still imports them, so they need a namespace.
    if (module == canonNamespace) {
      if (!isKnownCanonBuiltin(field)) {
        throw new Error(
          "'" + field + "' is not a canonical builtin this compiler " +
            "can emit. Known: waitable-set.new, waitable-set.drop, " +
            "waitable.join, subtask.drop, task.return, resource.new, " +
            "resource.rep, resource.drop, and the stream.*/future.* " +
            "families (new, read, write, drop-readable, drop-writable)."
        );
      }
      if (isTypedCanonBuiltin(field)) {
        // Per-type: the canon entry's immediate is a component type
        // index, so the type is part of which builtin this is.
        if (f.importCanonType == "") {
          throw new Error(
            "The canonical builtin '" + field + "' is per-type: name " +
              "the type as a third @external option, as in " +
              "@external(\"canon\", \"" + field + "\", \"stream<u8>\")."
          );
        }
      } else if (f.importCanonType != "" &&
          !(canonBuiltinBaseName(field) == "task.return" &&
            f.importCanonType.startsWith("return:"))) {
        throw new Error(
          "The canonical builtin '" + field + "' takes no type option; " +
            "only the stream.* and future.* families are per-type, and " +
            "task.return takes a `return:` result type."
        );
      }
      var builtinResult = noComponentResult;
      var returnsFor = "";
      if (canonBuiltinBaseName(field) == "task.return" &&
          f.importCanonType.startsWith("return:")) {
        // A `return:<interface>#<type>` option: the result is a WIT
        // type the encoder writes, and the declaration's parameters
        // are its flattening, which the wrapper that declared it
        // spelled from the same WIT. The name after `#` is the export
        // this returns for: `task.return#describe` returns for the
        // world's `describe`.
        var hashAt = -1;
        for (var i = 0; i < field.length; i += 1) {
          if (field.getByteAt(i) == 35) { hashAt = i; } // '#'
        }
        if (hashAt < 0) {
          throw new Error(
            "A typed task.return is declared as 'task.return#<export>', " +
              "naming the export it returns for; '" + field + "' names " +
              "none.");
        }
        returnsFor = field.sliceBytes(hashAt + 1, field.length);
      } else if (canonBuiltinBaseName(field) == "task.return" &&
          f.node == null) {
        // The entry adapter's own `task.return#main`, for a
        // synchronous main: no declaration to read, and its type is
        // the entry's result (ir/component-adapters.zena).
        builtinResult = wasm.componentEntryResult;
      } else if (canonBuiltinBaseName(field) == "task.return") {
        // `task.return` carries the lifted task's result, so its type
        // comes from what the declaration takes, not from what it
        // returns. Declared more than once at different result types
        // (the driver's bare one, an entry wrapper's typed one), each
        // under its own `#name`, as the per-type builtins are.
        let declared = declaredTypeOf(f, "import");
        if (declared.parameters.length == 1) {
          builtinResult = componentTypeOf(wasm, declared.parameters[0]);
          if (builtinResult < 0) {
            throw new Error(
              "task.return takes '" +
                typeToString(declared.parameters[0]) +
                "', which is not a flat scalar."
            );
          }
        } else if (declared.parameters.length > 1) {
          throw new Error(
            "task.return takes at most one value, the lifted task's result."
          );
        }
      }
      // One canon entry per distinct builtin: two modules may both
      // declare `waitable-set.new`, and the synthesized instance
      // exports each name once. A name redeclared at a different type
      // cannot share that export, so it is an error rather than a
      // merge.
      var duplicate = false;
      for (let existing in shape.builtins) {
        if (existing.name != field) { continue; }
        if (existing.canonType != f.importCanonType ||
            existing.isAsync != f.importIsAsync) {
          throw new Error(
            "The canonical builtin '" + field + "' is declared twice " +
              "with different options, and the core module imports it " +
              "by one name. Import the declaration from the module " +
              "that already has it instead."
          );
        }
        duplicate = true;
      }
      if (!duplicate) {
        shape.builtins.push(
            new ComponentCanonBuiltin(field, builtinResult,
                f.importCanonType, f.importIsAsync, returnsFor));
      }
      continue;
    }
    // The runtime memory module's exports. Not a WIT interface: the
    // emitter satisfies this namespace from the runtime core instance
    // at instantiation, the same way `canon` never reaches the host.
    if (module == componentRuntimeNamespace) {
      continue;
    }
    if (!module.contains(":")) {
      throw new Error(
        "The component target cannot import '" + field + "' from '" +
          module + "': a component imports WIT interfaces, and '" +
          module + "' is a core-module import namespace. Name a WIT " +
          "interface, as in @external(\"wasi:clocks/monotonic-clock@0.3.0\", " +
          "\"now\")."
      );
    }
    // An interface whose WIT the encoder provides — one the compiler
    // carries source for, or any interface at all when a world was
    // declared with `--wit`. The Zena declaration is core-shaped
    // (handles and pointers as i32s) and says nothing about the WIT
    // types, so no signature is derived from it, and the field may be
    // a resource path like `[method]r.f`, which is not kebab-case and
    // does not have to be. Whether the field exists, and whether its
    // declaration agrees with the interface's canonical flattening, is
    // settled against the parsed WIT in the encoder's `encode`.
    if (importEncoder != null &&
        (importEncoder as ImportTypeEncoder).witProvided(module)) {
      let imp = new ComponentImportFunc(module, field,
          new ComponentFuncType(new Array<i32>(), new Array<String>(),
              noComponentResult, f.importIsAsync),
          f.importIsAsync);
      imp.witProvided = true;
      imp.coreParamCount = f.signature.params.length;
      imp.coreResultCount = f.signature.returns.length;
      shape.imports.push(imp);
      continue;
    }
    if (!isKebabName(field)) {
      throw new Error(
        "The component target cannot import '" + field + "' from '" +
          module + "': '" + field + "' is not a kebab-case name, " +
          "so no WIT interface can export it."
      );
    }
    shape.imports.push(
      new ComponentImportFunc(module, field,
          signatureOf(wasm, f, "import", f.importIsAsync), f.importIsAsync));
  }

  // The async entry, where the program links a driver that needs one.
  // It replaces `main` rather than joining it, and takes its name:
  // `main` on its own would return before the timers it armed have
  // fired, and lifting both would offer the host two entry points that
  // mean different things. So a program is always entered as `main`,
  // and whether that is the synchronous lift or the callback-lifted
  // entry is the compiler's business.
  // A declared world whose `main` is `async func` while the program's
  // `main` is an ordinary synchronous function gets both: the wrapper
  // synthesizer writes `main_export` (its skip fires only for an async
  // `main`), and the entry is built anyway because the program reached a
  // door into the async driver — `sleep` does, without any `await`. Both
  // claimed the name and the component failed to validate with "export
  // name `main` conflicts with previous name `main`". The wrapper is the
  // one that knows the WIT's signature and issues its typed return, so
  // the entry yields the name to it and stays in the module as the
  // driver's host.
  var wrappedMain = false;
  for (let asyncExport in wasm.componentAsyncExports) {
    if (asyncExport.exportName == "main") { wrappedMain = true; }
  }
  for (let syncExport in wasm.componentSyncExports) {
    if (syncExport.exportName == "main") { wrappedMain = true; }
  }

  let entry = wasm.componentEntryFunc;
  if (entry != null && !wrappedMain) {
    let callback = wasm.componentCallbackFunc;
    if (callback == null) {
      throw new Error(
        "The component target has an async entry with no callback to " +
        "re-enter it through."
      );
    }
    // Its result is what the program's main resolves to, returned
    // through the typed `task.return` the entry wrapper declares —
    // nothing, for a main that returns nothing.
    let entryResult = if (wasm.componentEntryResult >= 0) {
      wasm.componentEntryResult
    } else {
      noComponentResult
    };
    shape.exports.push(new ComponentExportFunc(
      componentEntryExport, "main",
      new ComponentFuncType(new Array<i32>(), new Array<String>(),
          entryResult, true),
      componentCallbackExport));
  }

  // The declared world's async exports, each lifted from the adapter
  // the compiler built around the wrapper module's function
  // (component-adapters.zena), with the callback like the entry. The
  // lifted type is the WIT's, which the encoder writes; the core-shaped
  // signature here carries nothing.
  for (let asyncExport in wasm.componentAsyncExports) {
    if (wasm.componentCallbackFunc == null) {
      throw new Error(
        "The component target has an async export with no callback to " +
        "re-enter it through."
      );
    }
    let adapterName = asyncExport.adapter.name;
    if (adapterName == null) {
      throw new Error("The async export adapter for '" +
          asyncExport.exportName + "' has no name to alias it by.");
    }
    let lifted = new ComponentExportFunc(
      adapterName as String, asyncExport.exportName,
      new ComponentFuncType(new Array<i32>(), new Array<String>(),
          noComponentResult, true),
      componentCallbackExport);
    lifted.witTyped = true;
    shape.exports.push(lifted);
  }

  // The declared world's wrapped synchronous exports: the wrapper's
  // core signature is the canonical flattening of the WIT's, which
  // the encoder types; a result past one core value comes back as the
  // address of a return area, and the lift then carries the memory
  // options and a `post-return`.
  for (let syncExport in wasm.componentSyncExports) {
    let lifted = new ComponentExportFunc(
      syncExport.coreName, syncExport.exportName,
      new ComponentFuncType(new Array<i32>(), new Array<String>(),
          noComponentResult, false));
    lifted.witTyped = true;
    shape.exports.push(lifted);
  }
  // The destructors of the resources the program provides: not
  // lifted; the emitter's shim table is filled from them.
  for (let dtor in wasm.componentResourceDtors) {
    shape.dtorExports.push(
        new ComponentDtorExport(dtor.resourceName, dtor.coreName));
  }

  let exportKeys = wasm.exportMap.keys();
  for (let key in exportKeys) {
    let f = wasm.exportMap[key];
    let coreName = key as String;
    if (entry != null && coreName == "main") {
      continue;
    }
    // The program's own function behind a wrapped export: the adapter
    // or wrapper above is what the host reaches, not the function.
    var wrapped = false;
    for (let asyncExport in wasm.componentAsyncExports) {
      if (asyncExport.exportName == toKebabName(coreName)) { wrapped = true; }
    }
    for (let syncExport in wasm.componentSyncExports) {
      if (syncExport.exportName == toKebabName(coreName)) { wrapped = true; }
    }
    if (wrapped) {
      continue;
    }
    let exportName = toKebabName(coreName);
    if (exportName == "") {
      throw new Error(
        "The component target cannot export '" + coreName +
          "': it has no kebab-case rendering, and every component export " +
          "name is kebab-case."
      );
    }
    let signature = signatureOf(wasm, f, "export");
    // An export carrying a string is lifted through its wrapper rather
    // than directly: the canonical ABI hands the guest a pointer and a
    // length, and the Zena function takes a String. The wrapper is what
    // the two have in common, and it is synthesized before layout —
    // absent here means the synth pass and this disagree about which
    // signatures carry one, which is a compiler bug rather than a
    // program error.
    var lifted = coreName;
    if (let (true, wrapper) = wasm.componentExportWrappers.get(key)) {
      let wrapperName = wrapper.name;
      if (wrapperName == null) {
        throw new Error("The component export wrapper for '" + coreName +
            "' has no name to alias it by.");
      }
      lifted = wrapperName;
    } else if (componentSignatureNeedsMemory(signature)) {
      throw new Error("The component export '" + coreName +
          "' carries a string with no wrapper synthesized for it.");
    }
    shape.exports.push(
      new ComponentExportFunc(lifted, exportName, signature));
  }

  // The last step: hand the finished import list to the WIT encoder.
  // This is also where WIT-provided imports are checked against their
  // interfaces and their lowerings marked for the memory options. The
  // encoder is injected by the driver (`wit-imports.zena`) so codegen
  // does not link the WIT parser; a driver that emits components
  // without providing one gets `emitComponent`'s loud error.
  if (importEncoder != null) {
    shape.encodedImports = (importEncoder as ImportTypeEncoder).encode(shape);
  }

  return shape;
}
VersionPublishedDependenciesApproved by
0.0.2-preview.1 2026-11-19