compiler 0.0.2-preview.1

The Zena self-hosted compiler

Readme Files Dependencies Versions

compiler#

The Zena compiler, written in Zena. It parses, type-checks, and compiles Zena source code to WebAssembly (Wasm GC) and WebAssembly Text (WAT).

Usage#

Parsing a Single File#

Use compiler/parser to parse Zena source text into an AST (Module):

import { parse } from 'compiler/parser';

let ast = parse('export let answer = 42;', 'example.zena');

Loading and Checking a Program#

Use compiler (createCompiler and checkCompilation) with a CompilerHost to resolve imports across a module graph and type-check a program:

import {
  createCompiler, checkCompilation, buildStdlibConfig
} from 'compiler';
import { FsCompilerHost } from 'compiler/fs-host';

let stdlibRoot = '/stdlib';
let host = new FsCompilerHost(stdlibRoot);
let packages = buildStdlibConfig(stdlibRoot, 'host');
let compiler = createCompiler(host, {
  stdlibRoot,
  packages,
});

let result = await compiler.compile({
  entry: '/workspace/main.zena',
  target: 'host',
});
let checked = checkCompilation(result);

Compiling a File to WebAssembly#

Use compiler/compile-file to compile an entry file on disk to a .wasm or .wat output file:

import { compileFile } from 'compiler/compile-file';

let ok = await compileFile({
  entry: 'src/main.zena',
  outPath: 'out/main.wasm',
  stdlibRoot: '/stdlib',
});

Libraries#

  • compiler / compiler/compiler — compiler entry point (Compiler, createCompiler, checkCompilation, ProgramCheckResult)
  • compiler/compile-file — end-to-end file compilation (compileFile, CompileFileOptions)
  • compiler/tokenizer — lexer and token definitions (tokenize, Token, TokenType)
  • compiler/parser — recursive-descent parser (parse, Parser)
  • compiler/ast — AST node definitions (Node, Module, Statement, Expression, TypeAnnotation)
  • compiler/visitor — AST traversal helpers (walk, Visitor)
  • compiler/scope — lexical scope and symbol resolution (ScopeBuilder, ScopeResult, Symbol)
  • compiler/checker — static type checker (checkModule, CheckResult)
  • compiler/semantic-model — semantic queries over a checked module (SemanticModel)
  • compiler/types — compiler type representations (Type, ClassType, FunctionType, typeToString)
  • compiler/diagnostics — compiler diagnostics and source spans (Diagnostic, DiagnosticCode, DiagnosticSeverity, SourceSpan)
  • compiler/package-manifest — zena-package.json manifest and workspace loader (loadWorkspace, loadPackageMap)
  • compiler/module-resolver — package and relative module specifier resolution (ModuleResolver, PackageConfig)
  • compiler/library-loader — module graph loader (LibraryLoader, CompilerHost, SourceFile)
  • compiler/codegen/binary-generator — WebAssembly GC binary emitter (BinaryGenerator)
  • compiler/codegen/wat-generator — WebAssembly Text (WAT) emitter (WatGenerator)

License#

MIT

16.5 KiB Raw
/**
 * Synthesizers for the named module-scaffold helpers streaming builds
 * without an AST: the host-interop string helpers ($stringCreate,
 * $stringSetByte, $stringGetByte, $stringGetLength, $stringHash) and
 * the internal string_hash_helper. Each
 * mirrors its streaming generator instruction for instruction — these
 * exist in EVERY module, so they were the structural blocker keeping
 * zir-strict red before any real construct was measured.
 */
import {
  ValType, ValTypeI32, ValTypeRef, ValTypeRefNull, ValTypeAnyRef, ValTypeExternRef,
} from '../wasm.zena';
import {
  WasmModule, WasmFunction, WasmStruct, WasmGlobal,
} from '../wasm-module.zena';
import { getClassKey } from '../codegen-utils.zena';
import { IrBody, IrOp, IrRefType, IrRefField, IrRefGlobal, IrRefFunc } from './ir.zena';
import { IrBuilder } from './builder.zena';
import { growable } from 'zena:core';

/** The String struct, its vtable global, and the field names by role. */
final class StringLayout {
  st: WasmStruct;
  vtable: WasmGlobal | null;
  data: String;
  start: String;
  end: String;
  encoding: String | null;
  hashCode: String | null;
  new(this.st, this.vtable, this.data, this.start, this.end,
      this.encoding, this.hashCode);
}

function stringLayoutOf(wasm: WasmModule) {
  let {stringClass} = wasm;
  if (stringClass == null) {
    throw new Error('string helper synthesis: String class not found');
  }
  let key = getClassKey(stringClass);
  var st: WasmStruct | null = null;
  var vt: WasmGlobal | null = null;
  if (let (true, info) = wasm.classInfos.get(key)) {
    st = info.structType;
    vt = info.vtableGlobal;
  }
  if (st == null) {
    throw new Error('string helper synthesis: String struct not found');
  }
  var data: String | null = null;
  var start: String | null = null;
  var end: String | null = null;
  var encoding: String | null = null;
  var hashCode: String | null = null;
  for (let f in st.fields) {
    let n = f.name;
    if (n == null) {
      continue;
    }
    if (n.endsWith('::#data')) {
      data = n;
    } else if (n.endsWith('::#start')) {
      start = n;
    } else if (n.endsWith('::#end')) {
      end = n;
    } else if (n.endsWith('::#encoding')) {
      encoding = n;
    } else if (n.endsWith('::#hashCode')) {
      hashCode = n;
    }
  }
  if (data == null || start == null || end == null) {
    throw new Error('string helper synthesis: String field layout');
  }
  return new StringLayout(st, vt, data, start,
      end, encoding, hashCode);
}

function fieldGet(b: IrBuilder, st: WasmStruct, name: String, recv: i32) {
  let idx = st.fieldIndexByName(name);
  if (idx < 0) {
    throw new Error('string helper synthesis: missing field ' + name);
  }
  return b.emit(IrOp.struct_get, st.fields[idx].type, recv,
      b.addRef(new IrRefField(st, name)));
}

function fieldSet(b: IrBuilder, st: WasmStruct, name: String, recv: i32, value: i32) {
  let pair = growable<i32>([recv, value]);
  b.emitVoid(IrOp.struct_set, b.pushExtra(pair), b.addRef(new IrRefField(st, name)));
}

function i32c(b: IrBuilder, v: i32) {
  return b.constI32(v, new ValTypeI32());
}

/** entry block with one param per signature slot, left current. */
function entryOf(b: IrBuilder, func: WasmFunction) {
  let entry = b.newBlock();
  let params = new Array<i32>();
  for (let vt in func.signature.params) {
    params.push(b.blockParam(entry, vt));
  }
  b.switchTo(entry);
  return params;
}

/**
 * The FNV-1a byte loop both hash helpers share: hashes
 * data[start..end) starting from the offset basis, returning the
 * running-hash value in the block the builder is left in.
 */
function fnvLoop(b: IrBuilder, data: i32, start: i32, end: i32) {
  let i32vt = new ValTypeI32() as ValType;
  let header = b.newBlock();
  let hParam = b.blockParam(header, i32vt);
  let iParam = b.blockParam(header, i32vt);
  let seed = growable<i32>([i32c(b, -2128831035), start]);
  b.br(header, seed);

  b.switchTo(header);
  let done = b.binary(IrOp.i32_ge_u, i32vt, iParam, end);
  let bodyB = b.newBlock();
  let exitB = b.newBlock();
  let hOut = b.blockParam(exitB, i32vt);
  let exitArgs = growable<i32>([hParam]);
  b.brIf(done, exitB, exitArgs, bodyB, new Array<i32>());

  b.switchTo(bodyB);
  let byte = b.emit(IrOp.array_get_u, i32vt, data, iParam);
  let xored = b.binary(IrOp.i32_xor, i32vt, hParam, byte);
  let mixed = b.binary(IrOp.i32_mul, i32vt, xored, i32c(b, 16777619));
  let one = i32c(b, 1);
  let next = b.binary(IrOp.i32_add, i32vt, iParam, one);
  let back = growable<i32>([mixed, next]);
  b.br(header, back);

  b.switchTo(exitB);
  return hOut;
}

/** $stringCreate(len) -> externref: fresh empty String of `len` bytes. */
function synthStringCreate(wasm: WasmModule, func: WasmFunction) {
  let lay = stringLayoutOf(wasm);
  if (lay.vtable == null || lay.encoding == null) {
    throw new Error('string helper synthesis: $stringCreate layout');
  }
  let byteArr = wasm.getByteArrayType();
  let b = new IrBuilder();
  let params = entryOf(b, func);
  let lenInit = growable<i32>([params[0], -1]);
  let bytes = b.emit(IrOp.array_new, new ValTypeRef(byteArr) as ValType,
      b.pushExtra(lenInit), b.addRef(new IrRefType(byteArr)));
  // Single-shot struct.new in field order, like string_from_shared:
  // String's fields are non-null, so there is no defaultable shell.
  let args = new Array<i32>();
  for (let f in lay.st.fields) {
    let fname = if (f.name != null) f.name as String else '';
    if (fname == '<vtable>') {
      let vtg = lay.vtable as WasmGlobal;
      args.push(b.emit(IrOp.global_get, vtg.type, -1,
          b.addRef(new IrRefGlobal(vtg))));
    } else if (fname == lay.data) {
      args.push(bytes);
    } else if (fname == lay.end) {
      args.push(params[0]);
    } else if (f.type is ValTypeI32) {
      // start, encoding, and any other counter default to 0.
      args.push(i32c(b, 0));
    } else {
      throw new Error('$stringCreate: unexpected String field ' + fname);
    }
  }
  let inst = b.emit(IrOp.struct_new, new ValTypeRef(lay.st) as ValType,
      b.pushArgList(args), b.addRef(new IrRefType(lay.st)));
  let ext = b.emit(IrOp.extern_convert_any, new ValTypeExternRef(), inst, -1);
  b.ret(ext);
  return b.finish();
}

/** externref param -> non-null String struct value. */
function externToString(b: IrBuilder, lay: StringLayout, ext: i32) {
  let anyv = b.emit(IrOp.any_convert_extern, new ValTypeAnyRef(), ext, -1);
  let vt = new ValTypeRef(lay.st) as ValType;
  return b.emit(IrOp.ref_cast, vt, anyv, b.internType(vt));
}

/** $stringSetByte(str externref, index, value). */
function synthStringSetByte(wasm: WasmModule, func: WasmFunction) {
  let lay = stringLayoutOf(wasm);
  let b = new IrBuilder();
  let params = entryOf(b, func);
  let s = externToString(b, lay, params[0]);
  let data = fieldGet(b, lay.st, lay.data, s);
  let start = fieldGet(b, lay.st, lay.start, s);
  let idx = b.binary(IrOp.i32_add, new ValTypeI32(), start, params[1]);
  let triple = growable<i32>([data, idx, params[2]]);
  b.emitVoid(IrOp.array_set, b.pushExtra(triple), -1);
  b.retVoid();
  return b.finish();
}

/** $stringGetByte(str externref, index) -> i32. */
function synthStringGetByte(wasm: WasmModule, func: WasmFunction) {
  let lay = stringLayoutOf(wasm);
  let b = new IrBuilder();
  let params = entryOf(b, func);
  let s = externToString(b, lay, params[0]);
  let data = fieldGet(b, lay.st, lay.data, s);
  let start = fieldGet(b, lay.st, lay.start, s);
  let idx = b.binary(IrOp.i32_add, new ValTypeI32(), start, params[1]);
  b.ret(b.emit(IrOp.array_get_u, new ValTypeI32(), data, idx));
  return b.finish();
}

/** $stringGetLength(str externref) -> i32 (#end - #start). */
function synthStringGetLength(wasm: WasmModule, func: WasmFunction) {
  let lay = stringLayoutOf(wasm);
  let b = new IrBuilder();
  let params = entryOf(b, func);
  let s = externToString(b, lay, params[0]);
  let end = fieldGet(b, lay.st, lay.end, s);
  let start = fieldGet(b, lay.st, lay.start, s);
  b.ret(b.binary(IrOp.i32_sub, new ValTypeI32(), end, start));
  return b.finish();
}

/** string_hash_helper(str anyref-ish) -> i32: plain FNV-1a, no memo. */
function synthStringHashHelper(wasm: WasmModule, func: WasmFunction) {
  let lay = stringLayoutOf(wasm);
  let b = new IrBuilder();
  let params = entryOf(b, func);
  let vt = new ValTypeRef(lay.st) as ValType;
  let s = b.emit(IrOp.ref_cast, vt, params[0], b.internType(vt));
  let data = fieldGet(b, lay.st, lay.data, s);
  let start = fieldGet(b, lay.st, lay.start, s);
  let end = fieldGet(b, lay.st, lay.end, s);
  b.ret(fnvLoop(b, data, start, end));
  return b.finish();
}

/**
 * $stringHash(s) -> i32: memoized FNV-1a — a cached non-zero
 * #hashCode returns directly; otherwise hash, coerce 0 to 1 (0 means
 * "uncomputed"), store the cache, return.
 */
function synthStringHash(wasm: WasmModule, func: WasmFunction) {
  let lay = stringLayoutOf(wasm);
  let hashField = lay.hashCode;
  if (hashField == null) {
    throw new Error('string helper synthesis: $stringHash layout');
  }
  let i32vt = new ValTypeI32() as ValType;
  let b = new IrBuilder();
  let params = entryOf(b, func);
  let s = params[0];
  let cached = fieldGet(b, lay.st, hashField, s);
  let missB = b.newBlock();
  let hitB = b.newBlock();
  b.brIf(cached, hitB, new Array<i32>(), missB, new Array<i32>());

  b.switchTo(hitB);
  b.ret(cached);

  b.switchTo(missB);
  let data = fieldGet(b, lay.st, lay.data, s);
  let start = fieldGet(b, lay.st, lay.start, s);
  let end = fieldGet(b, lay.st, lay.end, s);
  let h = fnvLoop(b, data, start, end);
  // h == 0 -> 1, so the cache sentinel stays unambiguous.
  let zeroB = b.newBlock();
  let storeB = b.newBlock();
  let hFinal = b.blockParam(storeB, i32vt);
  let hArgs = growable<i32>([h]);
  b.brIf(h, storeB, hArgs, zeroB, new Array<i32>());
  b.switchTo(zeroB);
  let oneArg = growable<i32>([i32c(b, 1)]);
  b.br(storeB, oneArg);
  b.switchTo(storeB);
  fieldSet(b, lay.st, hashField, s, hFinal);
  b.ret(hFinal);
  return b.finish();
}

/**
 * string_from_shared(offset, len): a String struct.new with every
 * field in declaration order — vtable global, #data = the shared
 * byte-array global, #start = offset, #end = offset + len, scalars
 * (#encoding, #hashCode) zero. Mirrors streaming; any field shape it
 * does not recognize is a layout change and fails loudly.
 */
function synthStringFromShared(wasm: WasmModule, func: WasmFunction) {
  let lay = stringLayoutOf(wasm);
  let shared = wasm.sharedStringsGlobal;
  if (shared == null) {
    throw new Error('string_from_shared: shared strings global missing');
  }
  let b = new IrBuilder();
  let params = entryOf(b, func);
  let args = new Array<i32>();
  for (let f in lay.st.fields) {
    let fname = if (f.name != null) f.name as String else '';
    if (fname == '<vtable>') {
      let vtg = lay.vtable;
      if (vtg == null) {
        throw new Error('string_from_shared: String vtable missing');
      }
      args.push(b.emit(IrOp.global_get, vtg.type, -1,
          b.addRef(new IrRefGlobal(vtg))));
    } else if (fname == lay.data) {
      // The shared-bytes global is nullable (initialized by __start);
      // the data field is not — re-assert on the way in.
      var data = b.emit(IrOp.global_get, shared.type, -1,
          b.addRef(new IrRefGlobal(shared)));
      if (shared.type is ValTypeRefNull && f.type is ValTypeRef) {
        data = b.emit(IrOp.ref_as_non_null, f.type, data, -1);
      }
      args.push(data);
    } else if (fname == lay.start) {
      args.push(params[0]);
    } else if (fname == lay.end) {
      args.push(b.binary(IrOp.i32_add, new ValTypeI32(), params[0], params[1]));
    } else if (f.type is ValTypeI32) {
      args.push(i32c(b, 0));
    } else {
      throw new Error('string_from_shared: unexpected String field ' + fname);
    }
  }
  let inst = b.emit(IrOp.struct_new, new ValTypeRef(lay.st) as ValType,
      b.pushArgList(args), b.addRef(new IrRefType(lay.st)));
  b.ret(inst);
  return b.finish();
}

/**
 * get_string_literal(id): the lazy literal cache — a non-null
 * array[id] returns directly; otherwise an id-dispatch chain calls
 * string_from_shared with the literal's (offset, length) in the
 * packed shared segment, stores it back, and returns it. Mirrors
 * streaming's if-chain shape.
 */
function synthGetStringLiteral(wasm: WasmModule, func: WasmFunction) {
  let arrGlobal = wasm.stringLiteralsArrayGlobal;
  let arrTypeOpt = wasm.stringArrayType;
  let sfs = wasm.stringFromSharedFunc;
  if (arrGlobal == null || arrTypeOpt == null || sfs == null) {
    throw new Error('get_string_literal: literal infrastructure missing');
  }
  if (sfs.signature.returns.length != 1) {
    throw new Error('get_string_literal: string_from_shared shape');
  }
  // string_from_shared is declared with a nullable String return, so
  // values it types are nullable; this function's own return is
  // non-null, so both return paths re-assert explicitly.
  let strVt = sfs.signature.returns[0];
  if (func.signature.returns.length != 1) {
    throw new Error('get_string_literal: signature shape');
  }
  let retVt = func.signature.returns[0];
  let i32vt = new ValTypeI32() as ValType;
  let b = new IrBuilder();
  let params = entryOf(b, func);
  let id = params[0];
  let arr = b.emit(IrOp.global_get, arrGlobal.type, -1,
      b.addRef(new IrRefGlobal(arrGlobal)));
  let cached = b.emit(IrOp.array_get, strVt, arr, id);
  let isNull = b.emit(IrOp.ref_is_null, i32vt, cached, -1);
  let missB = b.newBlock();
  let hitB = b.newBlock();
  b.brIf(isNull, missB, new Array<i32>(), hitB, new Array<i32>());
  b.switchTo(hitB);
  b.ret(b.emit(IrOp.ref_as_non_null, retVt, cached, -1));

  b.switchTo(missB);
  // ids are dense 0..n-1 in assignment order.
  let count = wasm.stringLiteralIds.size;
  let byId = new Array<String>(count);
  for (var k = 0; k < count; k += 1) {
    byId.push('');
  }
  for (let pair in wasm.stringLiteralIds) {
    byId[pair.value] = pair.key;
  }
  let storeB = b.newBlock();
  let fresh = b.blockParam(storeB, strVt);
  if (count == 0) {
    b.unreachable();
  } else {
    // Binary split over the dense id range rather than a linear else
    // chain: block nesting (and the emitter's per-block recursion)
    // stays logarithmic even with thousands of literals. Driven by an
    // explicit worklist of (lo, hi, block) triples so lowering itself
    // does not recurse either. A range of one id calls
    // string_from_shared unconditionally — callers only pass ids the
    // compiler assigned, matching streaming's untested trailing else.
    let work = growable<i32>([0, count, missB]);
    while (work.length > 0) {
      let blk = work.pop();
      let hi = work.pop();
      let lo = work.pop();
      b.switchTo(blk);
      if (hi - lo == 1) {
        let loc = if (let (true, l) = wasm.stringLiteralLocations.get(byId[lo])) l else null;
        if (loc == null) {
          throw new Error('get_string_literal: literal location missing');
        }
        let callArgs = growable<i32>([
          i32c(b, loc.offset),
          i32c(b, loc.length),
        ]);
        let v = b.callFunc(b.addRef(new IrRefFunc(sfs)), callArgs, strVt);
        let brArgs = growable<i32>([v]);
        b.br(storeB, brArgs);
      } else {
        let mid = lo + ((hi - lo) >> 1);
        let isLow = b.binary(IrOp.i32_lt_s, i32vt, id, i32c(b, mid));
        let lowB = b.newBlock();
        let highB = b.newBlock();
        b.brIf(isLow, lowB, new Array<i32>(), highB, new Array<i32>());
        work.push(lo);
        work.push(mid);
        work.push(lowB);
        work.push(mid);
        work.push(hi);
        work.push(highB);
      }
    }
  }
  b.switchTo(storeB);
  let arr2 = b.emit(IrOp.global_get, arrGlobal.type, -1,
      b.addRef(new IrRefGlobal(arrGlobal)));
  let triple = growable<i32>([arr2, id, fresh]);
  b.emitVoid(IrOp.array_set, b.pushExtra(triple), -1);
  b.ret(b.emit(IrOp.ref_as_non_null, retVt, fresh, -1));
  return b.finish();
}

/**
 * Routes a named scaffold helper to its synthesizer, or null when the
 * name is not one of the seven.
 */
export function synthesizeScaffold(wasm: WasmModule, func: WasmFunction): IrBody | null {
  if (func.node != null) {
    return null;
  }
  let {name} = func;
  if (name == null) {
    return null;
  }
  if (name == '$stringCreate') { return synthStringCreate(wasm, func); }
  if (name == '$stringSetByte') { return synthStringSetByte(wasm, func); }
  if (name == '$stringGetByte') { return synthStringGetByte(wasm, func); }
  if (name == '$stringGetLength') { return synthStringGetLength(wasm, func); }
  if (name == '$stringHash') { return synthStringHash(wasm, func); }
  if (name == 'string_hash_helper') { return synthStringHashHelper(wasm, func); }
  if (name == 'string_from_shared') { return synthStringFromShared(wasm, func); }
  if (name == 'get_string_literal') { return synthGetStringLiteral(wasm, func); }
  return null;
}
VersionPublishedDependenciesApproved by
0.0.2-preview.1 2026-11-19