compiler 0.0.2-preview.1
The Zena self-hosted compiler
compiler#
The Zena compiler, written in Zena. It parses, type-checks, and compiles Zena source code to WebAssembly (Wasm GC) and WebAssembly Text (WAT).
Usage#
Parsing a Single File#
Use compiler/parser to parse Zena source text into an AST (Module):
import { parse } from 'compiler/parser';
let ast = parse('export let answer = 42;', 'example.zena');
Loading and Checking a Program#
Use compiler (createCompiler and checkCompilation) with a CompilerHost to resolve imports across a module graph and type-check a program:
import {
createCompiler, checkCompilation, buildStdlibConfig
} from 'compiler';
import { FsCompilerHost } from 'compiler/fs-host';
let stdlibRoot = '/stdlib';
let host = new FsCompilerHost(stdlibRoot);
let packages = buildStdlibConfig(stdlibRoot, 'host');
let compiler = createCompiler(host, {
stdlibRoot,
packages,
});
let result = await compiler.compile({
entry: '/workspace/main.zena',
target: 'host',
});
let checked = checkCompilation(result);
Compiling a File to WebAssembly#
Use compiler/compile-file to compile an entry file on disk to a .wasm or .wat output file:
import { compileFile } from 'compiler/compile-file';
let ok = await compileFile({
entry: 'src/main.zena',
outPath: 'out/main.wasm',
stdlibRoot: '/stdlib',
});
Libraries#
compiler/compiler/compiler— compiler entry point (Compiler,createCompiler,checkCompilation,ProgramCheckResult)compiler/compile-file— end-to-end file compilation (compileFile,CompileFileOptions)compiler/tokenizer— lexer and token definitions (tokenize,Token,TokenType)compiler/parser— recursive-descent parser (parse,Parser)compiler/ast— AST node definitions (Node,Module,Statement,Expression,TypeAnnotation)compiler/visitor— AST traversal helpers (walk,Visitor)compiler/scope— lexical scope and symbol resolution (ScopeBuilder,ScopeResult,Symbol)compiler/checker— static type checker (checkModule,CheckResult)compiler/semantic-model— semantic queries over a checked module (SemanticModel)compiler/types— compiler type representations (Type,ClassType,FunctionType,typeToString)compiler/diagnostics— compiler diagnostics and source spans (Diagnostic,DiagnosticCode,DiagnosticSeverity,SourceSpan)compiler/package-manifest—zena-package.jsonmanifest and workspace loader (loadWorkspace,loadPackageMap)compiler/module-resolver— package and relative module specifier resolution (ModuleResolver,PackageConfig)compiler/library-loader— module graph loader (LibraryLoader,CompilerHost,SourceFile)compiler/codegen/binary-generator— WebAssembly GC binary emitter (BinaryGenerator)compiler/codegen/wat-generator— WebAssembly Text (WAT) emitter (WatGenerator)
License#
MIT
21.1 KiB
Raw
/**
* Library loading and caching for the Zena compiler.
*
* The LibraryLoader manages source loading, parsing, and caching. Each source
* file is loaded exactly once and cached by its canonical path. This
* guarantees identity: types from the same library source are always the
* same object.
*
* The loader uses a ModuleResolver for import resolution and a CompilerHost
* for file I/O.
*/
import { Set, HashMap } from 'zena:collections';
import {
Module, ImportDeclaration, ExportAllDeclaration, ExportFromDeclaration, Statement, makeNodeIdGenerator, IdGenerator, NodeId,
} from './ast.zena';
import { parse, parseRecovering, ParseCompleteness } from './parser.zena';
import { primeKeywords } from './tokenizer.zena';
import { StringTable } from 'zena:core';
import { Diagnostic } from './diagnostics.zena';
import { ModuleResolver, normalizePath } from './module-resolver.zena';
import { ScopeResult } from './scope/scope-builder.zena';
import { performance } from './time.zena';
// ============================================================================
// Compiler Host
// ============================================================================
/**
* Abstract interface for file I/O.
*
* The compiler host is the only point of contact between the compiler and
* the outside world. Different implementations provide different backends:
* - Filesystem (Node.js, WASI)
* - In-memory (testing, browser playground)
* - Network (future)
*
* Both methods are asynchronous, so a host may answer from the network
* or from an editor, and a compile holds no task while it waits for a
* read. A host that has its sources in memory declares the methods
* `async` and returns directly. See docs/design/async-compiler.md.
*/
export interface CompilerHost {
/**
* Returns the source code of a file by its canonical path, or fails
* when there is no such file.
*
* For stdlib modules, the path has the form "zena:module-name". For file
* modules, the path is an absolute filesystem path.
*/
readFile(path: String): Future<String>;
/**
* Returns the names of the entries in the directory at `path`, or `null` when
* `path` is not a directory.
*
* This is mainly used to read a WIT-backed package's root, which may be a
* directory of `.wit` files. A host that serves no such package can return
* `null`.
*/
listDir(path: String): Future<Array<String> | null>;
}
// ============================================================================
// Source File
// ============================================================================
/**
* A loaded and parsed source file.
*
* SourceFiles are cached by canonical path. Requesting the same path
* twice always returns the same instance.
*/
export class SourceFile {
/** Canonical path identifying this library. */
path: String;
/** Whether this library is part of the standard library. */
isStdlib: boolean;
/** The package this library belongs to (empty string if none). */
packageName: String;
/** The original source code. */
source: String;
/** Parsed AST. */
ast: Module;
/**
* Opaque version tag for cache invalidation.
*
* The compiler itself does not interpret this value. It is set by
* the host (e.g., the language service passes the editor document
* version) and compared by the loader to decide whether a cached
* file is still fresh.
*/
version: String;
/**
* Resolved import mappings.
* Key: import specifier as written in source (e.g., "./utils", "zena:string")
* Value: canonical resolved path
*/
imports = new Map<String, String>();
/**
* Scope analysis result (populated after scope building).
* Contains the scope tree, exports, unresolved names, and references.
*/
var scopeResult: ScopeResult | null = null;
/**
* Errors the parser recovered from, empty unless the loader was told
* to recover. A file lands here with a usable AST *and* real errors:
* whoever reports diagnostics has to include these.
*/
var parseErrors: GrowableArray<Diagnostic> = new Array<Diagnostic>(0);
/**
* What this file's AST is fit for. Ask this rather than inferring it
* from `parseErrors` — the parser owns the answer, and it is the one
* thing that changes as recovery grows to cover more of the grammar.
*/
var completeness = ParseCompleteness.Complete;
new(
this.path: String,
this.isStdlib: boolean,
this.packageName: String,
this.source: String,
this.ast: Module,
this.version: String
);
}
/**
* A module's top-level statements, preceded by the import declarations
* its implicit prelude imports stand for — every import it makes.
*/
export function moduleStatements(lib: SourceFile): GrowableArray<Statement> {
let statements = new Array<Statement>(lib.ast.body.length);
let {scopeResult} = lib;
if (scopeResult != null) {
for (let decl in scopeResult.implicitImportDeclarations) {
statements.push(decl);
}
}
for (let stmt in lib.ast.body) {
statements.push(stmt);
}
return statements;
}
// ============================================================================
// Library Graph
// ============================================================================
/** Result of topologically sorting source files by their dependencies. */
export class SourceFileGraph(
/** Source files in topological order (dependencies before dependents). */
files: GrowableArray<SourceFile>,
/** True if a cycle was detected (libraries still returned, best-effort order). */
hasCycle: boolean
)
// ============================================================================
// Library Loader
// ============================================================================
/**
* Loads, parses, and caches Zena libraries.
*
* The LibraryLoader is responsible for:
* - Loading library source code (via CompilerHost)
* - Parsing source into ASTs
* - Caching loaded libraries by path (ensuring identity)
* - Recursively loading dependencies (via import resolution)
*
* Loading reads a library's imports together and parses them in the
* order the imports are written: a library is parsed and cached, a read
* of every import it names starts, and then each import is awaited and
* loaded in turn. The parse order matters because every parse draws
* node ids from this loader's one generator, and those ids reach the
* emitted module; the reads' order does not, since a read changes
* nothing until its turn.
*
* Usage:
* let loader = new LibraryLoader(host, resolver);
* let entry = await loader.load("/path/to/main.zena");
* let graph = loader.computeGraph([entry]);
*/
/** A fresh identifier table with the keywords already in it. */
function primedTable() {
let table = new StringTable(4096);
primeKeywords(table);
return table;
}
export class LibraryLoader {
#host: CompilerHost;
#resolver: ModuleResolver;
#cache: HashMap<String, SourceFile>;
/**
* The reads in flight, by path. A library's imports are all read at
* once and parsed in source order, so a read is started when its
* importer is parsed and finished when its turn comes; two importers
* of one file share its read. Cleared when a load starts: a read a
* failed load left here would serve the next load stale text.
*/
#pending = new Map<String, Future<String>>();
/**
* Node ids must be unique across every file this loader has parsed —
* they key the semantic model, so two files sharing one id means one
* file's answers get served for the other's node. The loader owns the
* generator because the loader owns the parse cache: the set of files
* the ids have to be unique across is exactly the set it holds.
*
* It used to be a defaulted parameter on `load`, which made a
* mismatched generator something a caller could supply by omission —
* and `invalidate` and `refreshCache` both did, re-parsing from 1 and
* colliding with everything already cached.
*/
#nodeIdGenerator = makeNodeIdGenerator();
/**
* The identifier table every parse this loader drives shares. The
* loader owns it for the same reason it owns the node id generator:
* the useful scope for interning is exactly the set of files it holds,
* so a name is copied once however many files mention it and every
* identifier of one spelling is one `String`. Primed with the
* keywords, so a lexed `return` interns to the very instance
* `KEYWORDS` is keyed by.
*/
#table = primedTable();
/**
* The generator every node this loader's modules contain draws its id
* from, for nodes the compiler adds to them after parsing.
*/
nodeIdGenerator: IdGenerator<NodeId> {
get {
return this.#nodeIdGenerator;
}
}
var loadTime: f64 = 0.0;
var parseTime: f64 = 0.0;
/**
* Whether to recover from the parse errors recovery covers, keeping a
* partial AST, instead of letting them abort the load. The errors are
* still reported — they land on `SourceFile.parseErrors`. Editors want
* this; a build does not.
*/
#recover: boolean;
new(host: CompilerHost, resolver: ModuleResolver, recover: boolean = false)
: #host = host, #resolver = resolver, #recover = recover, #cache = new Map<String, SourceFile>();
/**
* Load a library by its canonical path.
*
* If the library has already been loaded, returns the cached record.
* Otherwise: loads source → parses → caches → recursively loads imports.
*
* If `version` is provided, the cache is checked for a matching version.
* A cached file with a different version is invalidated and re-loaded.
* When omitted (or null), the cache is trusted unconditionally.
*/
async load(path: String, version: String | null = null): Future<SourceFile> {
if (this.#cache.has(path)) {
// Version check: if a version was provided and it matches, use cache.
// If no version was provided, always trust the cache.
if (version == null || this.#cache[path].version == version) {
return this.#cache[path];
}
// Version mismatch — invalidate and re-load below.
this.#cache.delete(path);
}
this.#pending.clear();
let tLoad = performance.start("load");
let source = await this.#host.readFile(path);
this.loadTime = this.loadTime + performance.end(tLoad);
return await this.#register(path, source, version, null);
}
/**
* Load a module the compiler wrote itself, from its source rather
* than from the host. `pinned` maps import specifiers the source
* uses to paths that are already loaded, so a synthesized module can
* import the entry it was written for without the resolver spelling
* the entry's path a second way and loading it twice. Every other
* import resolves as usual.
*/
async loadSynthesized(
path: String,
source: String,
pinned: HashMap<String, String>
): Future<SourceFile> {
if (this.#cache.has(path)) {
return this.#cache[path];
}
return await this.#register(path, source, null, pinned);
}
/** Parse `source` as the module at `path`, cache it, and load what it imports. */
async #register(
path: String,
source: String,
version: String | null,
pinned: HashMap<String, String> | null
): Future<SourceFile> {
let isStdlib = path.startsWith("zena:") || path.contains("stdlib");
let packageName = this.#resolver.findPackageForPath(path);
let tParse = performance.start("parse");
var parseErrors = new Array<Diagnostic>(0);
var completeness = ParseCompleteness.Complete;
let ast = if (this.#recover) {
let result = parseRecovering(source, path, this.#nodeIdGenerator, this.#table);
parseErrors = result.diagnostics;
completeness = result.completeness;
result.ast
} else {
// Without recovery an incomplete parse throws, so reaching here at
// all means the AST accounts for the whole file.
parse(source, path, this.#nodeIdGenerator, this.#table)
};
this.parseTime = this.parseTime + performance.end(tParse);
// Cache the record BEFORE resolving imports to handle circular imports.
let record = new SourceFile(path, isStdlib, packageName, source, ast, version ?? "");
record.parseErrors = parseErrors;
record.completeness = completeness;
this.#cache[path] = record;
// Resolve every import, and start reading each one that is not
// loaded yet, so the reads are in flight together. A failure to
// resolve or load a dependency is left for type checking to report
// as ModuleNotFound.
let targets = new Array<String>();
for (let stmt in ast.body) {
let specifier = match (stmt) {
case ImportDeclaration {moduleSpecifier}: moduleSpecifier.value
case ExportAllDeclaration {moduleSpecifier}: moduleSpecifier.value
case ExportFromDeclaration {moduleSpecifier}: moduleSpecifier.value
case _: { continue; }
};
let target = this.#resolveImport(record, pinned, specifier);
if (target != null && !this.#cache.has(target)) {
targets.push(target);
this.#startRead(target);
}
}
// Then load them in source order. The order is what keeps node ids
// where they were: each parse draws from the one generator, so
// parsing in the order the reads completed would hand out
// different ids on every run.
for (let target in targets) {
if (this.#cache.has(target)) {
continue;
}
let tLoad = performance.start("load");
let source = await this.#finishRead(target);
this.loadTime = this.loadTime + performance.end(tLoad);
if (source == null) {
// No file there, so this is the missing-library case after all.
continue;
}
// Parsing sits outside the read's catch. A dependency that does
// not parse throws, and swallowing that would report the file as
// missing at the import site — naming a file that is sitting right
// there, and saying nothing about the error or its line.
await this.#register(target, source, null, null);
}
return record;
}
/**
* The path one of `record`'s imports names, recorded on the record;
* null when there is nothing to load for it.
*/
#resolveImport(
record: SourceFile,
pinned: HashMap<String, String> | null,
specifier: String
): String | null {
// An empty specifier is the parser's stand-in for one that has not
// been typed yet (see #missingModuleSpecifier). There is nothing to
// resolve, so the declaration contributes no dependency.
if (specifier == "") {
return null;
}
if (pinned != null && pinned.has(specifier)) {
record.imports[specifier] = pinned[specifier];
return null;
}
// Two failures here mean the import names nothing, and the checker
// reports both against the import itself: a specifier that does not
// resolve, and a path with no file behind it.
try {
let resolved = this.#resolver.resolve(specifier, record.path);
record.imports[specifier] = resolved.path;
return resolved.path;
} catch (e) {
// Nothing to load, and nothing more to say here.
return null;
}
}
/** Starts reading `path` unless a read of it is already in flight. */
#startRead(path: String): void {
if (this.#pending.has(path)) {
return;
}
this.#pending[path] = this.#host.readFile(path);
}
/**
* The source of `path`, from the read `#startRead` began, or from one
* started now; null when the host has no such file.
*/
async #finishRead(path: String): Future<String | null> {
this.#startRead(path);
let read = this.#pending[path];
this.#pending.delete(path);
try {
return await read;
} catch (e) {
return null;
}
}
/**
* Delete a file from the loader cache (e.g. on editor document close/delete).
*
* Evicts the file from the cache and clears `scopeResult` on any cached file
* that directly imports it so import wiring will fail or update on next compile.
*/
delete(path: String): void {
let norm = normalizePath(path);
if (this.#cache.has(norm)) { this.#cache.delete(norm); }
if (this.#cache.has(path)) { this.#cache.delete(path); }
// The files that import this one keep their scope trees. Import
// wiring is redone for every file on every compile (the compiler's
// wire-to-fixpoint pass re-links each import specifier to the
// exporter's current Symbol), so nothing about the importer's own
// scope depends on the file that changed — and a rebuilt scope
// would mint fresh Symbols, which a check result carried forward
// for the importer could not answer for (ProgramCheckResult.scopes).
}
/**
* Invalidate a single file by canonical path (push-based).
*
* Evicts the file from the cache, then re-loads it from the host so
* it's ready for the next compile. Files that import it keep their
* scope trees; the compiler re-wires every import on each compile.
*
* Use this in push-based environments (IDE, language service) where
* the host knows exactly which file changed. For pull-based batch
* compilation, use `refreshCache()` instead.
*/
async invalidate(path: String): Future<void> {
this.delete(path);
// Re-load so the file is back in the cache with fresh content.
await this.load(path);
}
/**
* Refresh the loader cache by re-reading non-stdlib files from the host
* (pull-based).
*
* For each cached user file, re-reads the source from the host and
* compares it to the cached source. If the source changed, the cache
* entry is removed and re-loaded.
*
* Use this in pull-based environments (CLI, batch compilation) where
* no external notification of file changes is available. In push-based
* environments (IDE), prefer `invalidate(path)` for targeted updates.
*/
async refreshCache(): Future<void> {
// Collect paths first — can't mutate the map during iteration.
let paths = new Array<String>();
let iter = this.#cache.keys();
for (let path in iter) {
paths.push(path);
}
// Phase 1: Detect changed files and remove from cache. Every file
// is read at once, and compared in order.
this.#pending.clear();
let userPaths = new Array<String>();
for (let path in paths) {
if (!this.#cache[path].isStdlib) {
userPaths.push(path);
this.#startRead(path);
}
}
let changedPaths = new Set<String>();
for (let path in userPaths) {
let cached = this.#cache[path];
let currentSource = await this.#finishRead(path);
if (currentSource != cached.source) {
changedPaths.add(path);
this.#cache.delete(path);
}
}
if (changedPaths.size > 0) {
// Files importing a changed one keep their scope trees, as in
// `delete`: import wiring is redone every compile, and a fresh
// scope would invalidate their carried check results.
// Phase 2: Re-load evicted files so they're back in the cache
// before computeGraph() tries to find them.
for (let path in changedPaths) {
await this.load(path);
}
}
}
/** Check if a library has already been loaded. */
has(path: String): boolean {
return this.#cache.has(path);
}
/** Get a previously loaded library without triggering a load. */
get(path: String): SourceFile | null {
if (this.#cache.has(path)) {
return this.#cache[path];
}
return null;
}
/**
* Compute the topological order of libraries starting from a root.
*
* Returns libraries in dependency order: if A imports B, then B appears
* before A. This ordering is required for correct type checking (a module's
* dependencies must be checked before the module itself).
*/
computeGraph(entries: GrowableArray<SourceFile>): SourceFileGraph {
let visited = new Map<String, i32>();
let stack = new Map<String, i32>();
let sorted = new Array<SourceFile>();
var hasCycle = false;
for (let entry in entries) {
if (this.#topoVisit(entry, visited, stack, sorted)) {
hasCycle = true;
}
}
return new SourceFileGraph(sorted, hasCycle);
}
#topoVisit(
lib: SourceFile,
visited: HashMap<String, i32>,
stack: HashMap<String, i32>,
sorted: GrowableArray<SourceFile>
): boolean {
if (visited.has(lib.path)) {
return false;
}
if (stack.has(lib.path)) {
return true;
}
stack[lib.path] = 1;
var hasCycle = false;
// Visit dependencies first. Implicit prelude imports precede
// written imports, matching `moduleStatements` and `beginModule`.
let {scopeResult} = lib;
if (scopeResult != null) {
for (let decl in scopeResult.implicitImportDeclarations) {
let importPath = decl.moduleSpecifier.value;
if (this.#cache.has(importPath)) {
if (this.#topoVisit(this.#cache[importPath], visited, stack, sorted)) {
hasCycle = true;
}
}
}
}
let iter = lib.imports.keys();
for (let specifier in iter) {
let importPath = lib.imports[specifier];
if (this.#cache.has(importPath)) {
if (this.#topoVisit(this.#cache[importPath], visited, stack, sorted)) {
hasCycle = true;
}
}
}
stack.delete(lib.path);
visited[lib.path] = 1;
sorted.push(lib);
return hasCycle;
}
}
- zena-lang:wit-parser wit-parser.zena-lang.pkg.zena-lang.dev ~0.0.1-preview.1
| Version | Published | Dependencies | Approved by |
|---|---|---|---|
| 0.0.2-preview.1 | 2026-11-19 |
|
|