compiler 0.0.2-preview.1
The Zena self-hosted compiler
compiler#
The Zena compiler, written in Zena. It parses, type-checks, and compiles Zena source code to WebAssembly (Wasm GC) and WebAssembly Text (WAT).
Usage#
Parsing a Single File#
Use compiler/parser to parse Zena source text into an AST (Module):
import { parse } from 'compiler/parser';
let ast = parse('export let answer = 42;', 'example.zena');
Loading and Checking a Program#
Use compiler (createCompiler and checkCompilation) with a CompilerHost to resolve imports across a module graph and type-check a program:
import {
createCompiler, checkCompilation, buildStdlibConfig
} from 'compiler';
import { FsCompilerHost } from 'compiler/fs-host';
let stdlibRoot = '/stdlib';
let host = new FsCompilerHost(stdlibRoot);
let packages = buildStdlibConfig(stdlibRoot, 'host');
let compiler = createCompiler(host, {
stdlibRoot,
packages,
});
let result = await compiler.compile({
entry: '/workspace/main.zena',
target: 'host',
});
let checked = checkCompilation(result);
Compiling a File to WebAssembly#
Use compiler/compile-file to compile an entry file on disk to a .wasm or .wat output file:
import { compileFile } from 'compiler/compile-file';
let ok = await compileFile({
entry: 'src/main.zena',
outPath: 'out/main.wasm',
stdlibRoot: '/stdlib',
});
Libraries#
compiler/compiler/compiler— compiler entry point (Compiler,createCompiler,checkCompilation,ProgramCheckResult)compiler/compile-file— end-to-end file compilation (compileFile,CompileFileOptions)compiler/tokenizer— lexer and token definitions (tokenize,Token,TokenType)compiler/parser— recursive-descent parser (parse,Parser)compiler/ast— AST node definitions (Node,Module,Statement,Expression,TypeAnnotation)compiler/visitor— AST traversal helpers (walk,Visitor)compiler/scope— lexical scope and symbol resolution (ScopeBuilder,ScopeResult,Symbol)compiler/checker— static type checker (checkModule,CheckResult)compiler/semantic-model— semantic queries over a checked module (SemanticModel)compiler/types— compiler type representations (Type,ClassType,FunctionType,typeToString)compiler/diagnostics— compiler diagnostics and source spans (Diagnostic,DiagnosticCode,DiagnosticSeverity,SourceSpan)compiler/package-manifest—zena-package.jsonmanifest and workspace loader (loadWorkspace,loadPackageMap)compiler/module-resolver— package and relative module specifier resolution (ModuleResolver,PackageConfig)compiler/library-loader— module graph loader (LibraryLoader,CompilerHost,SourceFile)compiler/codegen/binary-generator— WebAssembly GC binary emitter (BinaryGenerator)compiler/codegen/wat-generator— WebAssembly Text (WAT) emitter (WatGenerator)
License#
MIT
22.6 KiB
Raw
// The standard library's own package manifest, and the mapping from a
// stdlib module id back to the library that owns it.
//
// This lives apart from `package-manifest.zena`, which reads a workspace's
// `zena-package.json` files and therefore depends on `CompilerHost` and the
// loader. Nothing here needs either, so files as low in the import graph as
// `types.zena` can reach the stdlib layout without a cycle.
import { PackageConfig, ModuleExport } from './module-resolver.zena';
import { HashMap } from 'zena:collections';
// Build the standard library PackageConfig.
//
// The `stdlibRoot` is the absolute path to the stdlib's zena source
// directory (e.g., "/path/to/packages/stdlib/zena").
//
// This is kept separate from user packages because the stdlib is a
// language-level concept, not a user dependency. The compiler always
// includes it. Eventually, the stdlib source could be embedded in the
// compiler binary, eliminating the need for a path.
export function buildStdlibConfig(stdlibRoot: String): PackageConfig {
let exports = new Map<String, ModuleExport>();
// Public modules (no virtual mapping).
// Keep in sync with packages/stdlib/stdlib-manifest.json.
let publicModules = [
"assert",
"async",
"encoding",
"glob",
"json",
"math",
"path",
"regex",
"signals",
"task",
"test"
];
var i = 0;
while (i < publicModules.length) {
exports[publicModules[i]] = new ModuleExport(null);
i += 1;
}
// Virtual module: console → per-target entry file. The implementation
// files live under console/ and are not importable by name. Every
// target has one, because `console` is in every target's prelude and
// code that prints has to be portable.
//
// Virtual module: memory → the linear-memory allocator. One
// implementation everywhere but the component target, where the
// memory belongs to the runtime memory module and the allocator
// delegates to its `realloc` — two allocators handing out blocks from
// the same pages would corrupt each other, so the component entry
// keeps no allocation state at all.
let memoryVirtual: HashMap<String, String> = {
"core" => "memory.zena",
"js" => "memory.zena",
"zena-js" => "memory.zena",
"component" => "memory/component.zena",
"zena-component" => "memory/component.zena",
};
exports["memory"] = new ModuleExport(memoryVirtual);
let consoleVirtual = new Map<String, String>();
consoleVirtual["js"] = "console/none.zena";
consoleVirtual["zena-js"] = "console/js.zena";
consoleVirtual["zena-component"] = "console/component.zena";
// The component console prints over p3 stdio. `core` shares it: a
// `core` build is a component's core module without the wrapper, and
// the two must load the same prelude for the wrapper to embed exactly
// the module the `core` build emits.
consoleVirtual["component"] = "console/component.zena";
consoleVirtual["core"] = "console/component.zena";
exports["console"] = new ModuleExport(consoleVirtual);
// Virtual module: wasi → the guest side of the Component Model. The
// canonical ABI's string marshaling and allocator, the async-import
// driver that turns the subtask protocol into a `Future`, and the
// guest half of `stream<u8>`. Only a component has any of that
// beneath it, so the component target resolves it, and `core` with
// it: a `core` build is a component's core module without the
// wrapper, and its prelude's console is the component's. The
// component runtime's own memory entry sits beside these files
// without being one of them.
exports["wasi"] = new ModuleExport(null, "wasi/index.zena");
// Virtual module: error-stack → the `Error` stack-trace hooks. The host
// hooks trade in `anyref`/`externref`, which have no Component Model
// representation, so a component (and a `core` or `js` module, which
// assumes no Zena host) gets the null implementation instead. Every
// module pulls this in through `zena:error`, so it is the one import
// that a target cannot opt out of by writing less code — see
// docs/design/component-emission.md, Part 2.
let errorStackVirtual = new Map<String, String>();
errorStackVirtual["core"] = "core/error/stack-none.zena";
errorStackVirtual["js"] = "core/error/stack-none.zena";
errorStackVirtual["zena-js"] = "core/error/stack-host.zena";
// `zena-runtime` formats the backtrace over `zena:host/stack-trace`,
// installed at run time by `zena:host-setup` (below); a portable
// component has no host to ask.
errorStackVirtual["component"] = "core/error/stack-none.zena";
errorStackVirtual["zena-component"] = "core/error/stack-hooked.zena";
// Internal: `zena:core` is the only thing that reaches it, through
// `core/error.zena` and its own entry point. The manifest entry is
// what picks the file per target, which is the only reason the name
// exists at all.
exports["error-stack"] = new ModuleExport(errorStackVirtual, null, true);
// Virtual module: host-setup → what a program on `zena-runtime` needs
// installed before it runs: the host as the source of `Error`'s stack
// traces. A runtime module of the `zena-component` target
// (`prelude.zena`'s `getTargetRuntimeModules`), whose `installHost`
// the component entry calls before `main`.
let hostSetupVirtual: HashMap<String, String> = {
"zena-component" => "host-setup.zena",
};
exports["host-setup"] = new ModuleExport(hostSetupVirtual);
// Virtual module: fs → the file system, over wasi:filesystem 0.3 on
// both component targets. The 0.3 filesystem is asynchronous
// throughout, and so is the API.
let fsVirtual: HashMap<String, String> = {
"component" => "fs/component.zena",
"zena-component" => "fs/component.zena",
};
exports["fs"] = new ModuleExport(fsVirtual);
// Virtual module: cli → arguments, environment and exit, over
// wasi:cli 0.3.
let cliVirtual: HashMap<String, String> = {
"component" => "cli/component.zena",
"zena-component" => "cli/component.zena",
};
exports["cli"] = new ModuleExport(cliVirtual);
// Virtual module: process → host process spawning, on `zena-component`
// only, which links it for trusted invocations; see the library header.
let processVirtual: HashMap<String, String> = {
"zena-component" => "process/component.zena",
};
exports["process"] = new ModuleExport(processVirtual);
// Virtual module: wasm → running other Wasm modules, on `zena-component`
// only: it needs the `zena:host/wasm` host imports. Granted with
// process spawning and linked by the same hosts; see the library header.
let wasmVirtual: HashMap<String, String> = {
"zena-component" => "wasm/component.zena",
};
exports["wasm"] = new ModuleExport(wasmVirtual);
// Virtual module: extensions → loading other components as long-lived
// installations, over `zena:host/extensions`. On the `component`
// target too, so a program built against a declared world can load
// them where `zena-runtime` runs it.
let extensionsVirtual: HashMap<String, String> = {
"component" => "extensions/component.zena",
"zena-component" => "extensions/component.zena",
};
exports["extensions"] = new ModuleExport(extensionsVirtual);
// Virtual module: js → `@zena-lang/runtime` interop, currently the
// host-async completion registry. Only `zena-js` has an embedder that
// pushes completions back in through the module's exports; the
// component targets' futures ride the canonical ABI's waitables instead.
let jsVirtual: HashMap<String, String> = {"zena-js" => "js/index.zena"};
exports["js"] = new ModuleExport(jsVirtual);
// Modules: web-templates and web-elements → the Wasm Web UI protocol
// (docs/design/wasm-web-ui.md). One source serves every target;
// what differs per host is behind one internal virtual file each: the
// DOM host's imports and string crossings on `js` and `zena-js`, and
// their stand-ins elsewhere, which throw or do nothing. The `webui_*`
// exports are shared files that only a JS-string compile roots.
exports["web-templates"] = new ModuleExport(null, "web-templates/index.zena");
let htmlHostVirtual: HashMap<String, String> = {
"core" => "web-templates/host-none.zena",
"js" => "web-templates/host-js.zena",
"zena-js" => "web-templates/host-js.zena",
"component" => "web-templates/host-none.zena",
"zena-component" => "web-templates/host-none.zena",
};
exports["web-templates/host.zena"] = new ModuleExport(htmlHostVirtual, null, true);
exports["web-elements"] = new ModuleExport(null, "web-elements/index.zena");
let elementsHostVirtual: HashMap<String, String> = {
"core" => "web-elements/host-none.zena",
"js" => "web-elements/host-js.zena",
"zena-js" => "web-elements/host-js.zena",
"component" => "web-elements/host-none.zena",
"zena-component" => "web-elements/host-none.zena",
};
exports["web-elements/host.zena"] = new ModuleExport(elementsHostVirtual, null, true);
// Virtual module: fetch → HTTP requests. `zena-js` goes through
// `@zena-lang/runtime`'s `fetch()` import; the component targets speak
// wasi:http through the bindings synthesized from the stdlib's WIT.
let fetchVirtual: HashMap<String, String> = {
"zena-js" => "fetch/host.zena",
"component" => "fetch/component.zena",
"zena-component" => "fetch/component.zena",
};
exports["fetch"] = new ModuleExport(fetchVirtual);
// Module: http → HTTP server and client support, Express-like App and WASI
// HTTP handler.
exports["http"] = new ModuleExport(null, "http/index.zena");
// Module: crypto → SHA-256, HMAC, HKDF and ECDSA, implemented in private
// files under crypto/.
exports["crypto"] = new ModuleExport(null, "crypto/index.zena");
// Virtual module: time → sleeping and the monotonic clock, over a
// `@zena-lang/runtime` import on `zena-js` and wasi:clocks 0.3 on
// component targets.
let timeVirtual = new Map<String, String>();
timeVirtual["zena-js"] = "time/host.zena";
timeVirtual["component"] = "time/component.zena";
timeVirtual["zena-component"] = "time/component.zena";
exports["time"] = new ModuleExport(timeVirtual);
// Virtual module: random → the host's cryptographically secure
// generator. The public file is the same on every target that has a
// generator; what differs is the internal `random/source.zena` it
// imports. `core` and `js` assume no host generator, so importing
// `zena:random` there reports the library unavailable on that target.
let randomVirtual: HashMap<String, String> = {
"zena-js" => "random/index.zena",
"component" => "random/index.zena",
"zena-component" => "random/index.zena",
};
exports["random"] = new ModuleExport(randomVirtual);
let randomSourceVirtual: HashMap<String, String> = {
"zena-js" => "random/source-js.zena",
"component" => "random/source-wasi.zena",
"zena-component" => "random/source-wasi.zena",
};
exports["random/source.zena"] = new ModuleExport(randomSourceVirtual, null, true);
// Directory-based modules: entry file inside a library directory. The
// sibling implementation files are private — reachable only via relative
// imports from the library's own files.
// The rendezvous stream is its own library, not part of zena:async: the
// prelude names zena:async, so anything folded in is parsed and checked
// by every compilation whether or not it is used.
exports["stream"] = new ModuleExport(null, "stream/index.zena");
// Its twenty-three members have no module name of their own: they are
// private files of this library, reached through this entry point or by
// relative import from inside it. `error-stack`, `string`,
// `string-js-ops`, `string-builder`, `number-format` and `number-parse`
// are the exceptions, and they are virtual — the manifest is what picks
// a file per target, so a relative import would nail one of them.
exports["core"] = new ModuleExport(null, "core/index.zena");
let stringVirtual: HashMap<String, String> = {
"core" => "core/string.zena",
"js" => "core/string-js.zena",
"zena-js" => "core/string-js.zena",
"component" => "core/string.zena",
"zena-component" => "core/string.zena",
};
exports["core/string.zena"] = new ModuleExport(stringVirtual, null, true);
let stringJsOpsVirtual: HashMap<String, String> = {
"js" => "core/string-js-pure.zena",
"zena-js" => "core/string-js-host.zena",
};
exports["core/string-js-ops.zena"] =
new ModuleExport(stringJsOpsVirtual, null, true);
// A JS `String` is a JavaScript string, so its builder concatenates
// JavaScript strings instead of copying bytes.
let stringBuilderVirtual: HashMap<String, String> = {
"core" => "core/string-builder.zena",
"js" => "core/string-builder-js.zena",
"zena-js" => "core/string-builder-js.zena",
"component" => "core/string-builder.zena",
"zena-component" => "core/string-builder.zena",
};
exports["core/string-builder.zena"] =
new ModuleExport(stringBuilderVirtual, null, true);
// `zena-js` delegates number formatting and parsing to `@zena-lang/runtime`;
// `js` and the byte targets use the pure-Zena implementations.
let numberFormatVirtual: HashMap<String, String> = {
"core" => "core/number-format.zena",
"js" => "core/number-format.zena",
"zena-js" => "core/number-format-js.zena",
"component" => "core/number-format.zena",
"zena-component" => "core/number-format.zena",
};
exports["core/number-format.zena"] =
new ModuleExport(numberFormatVirtual, null, true);
let numberParseVirtual: HashMap<String, String> = {
"core" => "core/number-parse.zena",
"js" => "core/number-parse.zena",
"zena-js" => "core/number-parse-js.zena",
"component" => "core/number-parse.zena",
"zena-component" => "core/number-parse.zena",
};
exports["core/number-parse.zena"] =
new ModuleExport(numberParseVirtual, null, true);
exports["url"] = new ModuleExport(null, "url/index.zena");
// `URL` itself: the WHATWG parser in Zena on most targets (including
// pure `js`), and `@zena-lang/runtime`'s host `URL` on `zena-js`.
let urlVirtual: HashMap<String, String> = {
"core" => "url/url.zena",
"js" => "url/url.zena",
"zena-js" => "url/url-js.zena",
"component" => "url/url.zena",
"zena-component" => "url/url.zena",
};
exports["url/url.zena"] = new ModuleExport(urlVirtual, null, true);
exports["args"] = new ModuleExport(null, "args/index.zena");
exports["bench"] = new ModuleExport(null, "bench/index.zena");
exports["bigint"] = new ModuleExport(null, "bigint/index.zena");
exports["collections"] = new ModuleExport(null, "collections/index.zena");
exports["decimal"] = new ModuleExport(null, "decimal/index.zena");
exports["simd"] = new ModuleExport(null, "simd/index.zena");
exports["string"] = new ModuleExport(null, "string/index.zena");
let config = new PackageConfig(stdlibRoot, exports);
// `zena:host/*` WIT interfaces (`stack-trace`, `process`, `wasm`,
// `extensions`, `http-server`, `secrets`, `terminal`) live under
// `host-wit/` and resolve alongside the standard library's `.zena`
// modules.
config.witRoot = `${stdlibRoot}/host-wit`;
return config;
}
// ============================================================================
// Module id → owning library
// ============================================================================
// Module id (without the "zena:" prefix) → the library that owns it.
//
// Built once. The stdlib layout is compiled into the compiler rather than
// read from the program being compiled, so the index is the same for every
// compilation and a long-lived compiler can keep it.
var libraryIndex: HashMap<String, String> | null = null;
var moduleNameIndex: HashMap<String, String> | null = null;
// Every id the manifest names outright: each library's own name, and each
// entry file a `path` or `virtual` mapping points at. A library's private
// files are not in here — `stdlibLibraryOf` attributes those by directory.
function buildLibraryIndex() {
let index = new Map<String, String>();
let moduleNames = new Map<String, String>();
// The root is only used to resolve entry files on disk, which nothing
// reading this index does.
let config = buildStdlibConfig("");
let exports = config.exports as HashMap<String, ModuleExport>;
exports.forEach((name, entry) => {
// An internal entry named for a file, such as "core/string.zena" or
// "url/url.zena", belongs to the library of its directory.
let owningLibrary = if (entry.internal) {
firstSegment(name)
} else {
name
};
index[name] = owningLibrary;
moduleNames[name] = name;
let path = entry.path;
if (path != null) {
index[path as String] = owningLibrary;
moduleNames[path as String] = name;
}
let virtual = entry.virtual;
if (virtual != null) {
(virtual as HashMap<String, String>).forEach(
(target, entryPath) => {
index[entryPath] = owningLibrary;
moduleNames[entryPath] = name;
});
}
});
libraryIndex = index;
moduleNameIndex = moduleNames;
return index;
}
// The first path segment of `id`, or the whole id if it has no "/".
function firstSegment(id: String) {
var i = 0;
while (i < id.length) {
if (id.getByteAt(i) == 47) { // 47 = '/'
return id.sliceBytes(0, i);
}
i += 1;
}
return id;
}
/**
* The stdlib library that owns `moduleId`, or null if no library does.
*
* `moduleId` is a canonical module id as the resolver produces it, in
* either of its two forms: a name id for a library's default entry file
* ("zena:string"), or a path id for every other stdlib file
* ("zena:async/future.zena", "zena:console/host.zena"). Anything without
* the "zena:" prefix is a user module and owned by no library, which is
* the answer for most calls and is why it is the first thing tested.
*
* A path id the manifest names is owned by the library it is an entry for,
* which is why "core/error/stack-host.zena" belongs to `error-stack` and not
* to `core` despite sitting under `core/`. A path id the manifest does not name
* is one of a library's private implementation files, and belongs to the
* library its directory is named for.
*
* Every comparison is exact. The directory rule tests one path segment for
* membership in the set of published libraries; nothing here matches a
* prefix, suffix or substring, so a module cannot be attributed to a
* library whose name it merely begins with.
*/
/**
* The published module name whose entry file is `entryPath`, or null if
* the manifest names no module for it.
*
* `entryPath` is relative to the stdlib source root — "component-stream.zena",
* "console/host.zena" — which is the form a canonical path id carries after
* its "zena:" prefix. Only entry files match: a library's private
* implementation files have no module name, and this reports that rather
* than inventing one from the file name.
*/
function getModuleNameIndex() {
if (moduleNameIndex == null) {
buildLibraryIndex();
}
let m = moduleNameIndex;
if (m != null) {
return m as HashMap<String, String>;
}
return new Map<String, String>();
}
export function stdlibModuleNameForEntryPath(entryPath: String): String | null {
let index = getModuleNameIndex();
if (let (true, name) = index.get(entryPath)) {
return name;
}
return null;
}
export function stdlibLibraryOf(moduleId: String): String | null {
if (!moduleId.startsWith("zena:")) {
return null;
}
let index = if (libraryIndex != null) {
libraryIndex as HashMap<String, String>
} else {
buildLibraryIndex()
};
let id = moduleId.sliceBytes(5, moduleId.length);
if (let (true, library) = index.get(id)) {
return library;
}
let dir = firstSegment(id);
if (dir == id) {
// No directory to fall back on: an unrecognized top-level id.
return null;
}
if (let (true, library) = index.get(dir)) {
// A private file of `library`, but only if the directory is the
// library's own name — an entry path that happens to sit one level
// down was already matched above.
if (library == dir) {
return library;
}
}
return null;
}
/**
* The library a library's files are moving into, for the length of the move.
*
* The counterpart of `movingTo` in the well-known registry, which does this
* for a single declaration. A lookup that names a library instead — the
* roots taken by `queueStdlibFunction` and `getStdlibFunc` — needs the same
* grace for the same reason: the checked-in bootstrap answers with ITS
* manifest, which does not name the new path until the reseed after the
* files land, so until then `stdlibLibraryOf` attributes a moved file to
* the directory it now sits in.
*
* An entry is one exact destination, not a relaxation. There is deliberately
* no fallback to `core` for libraries with no entry here: that would let a
* lookup that should have failed find something plausible in another
* library. Each entry is deleted by the reseed after its files land.
*/
var libraryMoves: HashMap<String, String> | null = null;
function buildLibraryMoves() {
// Empty between migrations, which is the resting state: every library is
// where the manifest says it is, so a lookup naming one has exactly one
// answer. An entry is added for the length of a move and deleted by the
// reseed after its files land. See docs/design/stdlib-organization.md.
let t = new Map<String, String>();
libraryMoves = t;
return t;
}
/**
* Whether the library at `path` is a host boundary: a library whose
* exports the host calls from outside the program, so nothing inside
* the program references them and reachability roots them instead.
* `zena:js` holds the host-async completion entry points (async.md §4);
* the protocol files of `zena:web-templates` and `zena:web-elements`
* hold the Wasm Web UI protocol's `webui_*` functions
* (docs/design/wasm-web-ui.md), which only a js-target compile roots.
*/
export function isHostBoundaryLibrary(path: String): boolean {
return path == "zena:js/index.zena" || isWebUiProtocolLibrary(path);
}
/**
* Whether the library at `path` is one whose exported functions named
* with the Wasm Web UI prefix are exported from the module under
* their own names, on the js target.
*/
export function isWebUiProtocolLibrary(path: String): boolean {
return path == "zena:web-templates/protocol.zena" || path == "zena:web-elements/protocol.zena";
}
/**
* Whether a module owned by `owner` answers a lookup for `library` — the
* same library, or the one it is moving into while it moves.
*/
export function stdlibLibraryAnswers(owner: String, library: String): boolean {
if (owner == library) {
return true;
}
let moves = if (libraryMoves != null) {
libraryMoves as HashMap<String, String>
} else {
buildLibraryMoves()
};
if (let (true, destination) = moves.get(library)) {
return owner == destination;
}
return false;
}
- zena-lang:wit-parser wit-parser.zena-lang.pkg.zena-lang.dev ~0.0.1-preview.1
| Version | Published | Dependencies | Approved by |
|---|---|---|---|
| 0.0.2-preview.1 | 2026-11-19 |
|
|