/** * Library loading and caching for the Zena compiler. * * The LibraryLoader manages source loading, parsing, and caching. Each source * file is loaded exactly once and cached by its canonical path. This * guarantees identity: types from the same library source are always the * same object. * * The loader uses a ModuleResolver for import resolution and a CompilerHost * for file I/O. */ import { Set, HashMap } from 'zena:collections'; import { Module, ImportDeclaration, ExportAllDeclaration, ExportFromDeclaration, Statement, makeNodeIdGenerator, IdGenerator, NodeId, } from './ast.zena'; import { parse, parseRecovering, ParseCompleteness } from './parser.zena'; import { primeKeywords } from './tokenizer.zena'; import { StringTable } from 'zena:core'; import { Diagnostic } from './diagnostics.zena'; import { ModuleResolver, normalizePath } from './module-resolver.zena'; import { ScopeResult } from './scope/scope-builder.zena'; import { performance } from './time.zena'; // ============================================================================ // Compiler Host // ============================================================================ /** * Abstract interface for file I/O. * * The compiler host is the only point of contact between the compiler and * the outside world. Different implementations provide different backends: * - Filesystem (Node.js, WASI) * - In-memory (testing, browser playground) * - Network (future) * * Both methods are asynchronous, so a host may answer from the network * or from an editor, and a compile holds no task while it waits for a * read. A host that has its sources in memory declares the methods * `async` and returns directly. See docs/design/async-compiler.md. */ export interface CompilerHost { /** * Returns the source code of a file by its canonical path, or fails * when there is no such file. * * For stdlib modules, the path has the form "zena:module-name". For file * modules, the path is an absolute filesystem path. */ readFile(path: String): Future; /** * Returns the names of the entries in the directory at `path`, or `null` when * `path` is not a directory. * * This is mainly used to read a WIT-backed package's root, which may be a * directory of `.wit` files. A host that serves no such package can return * `null`. */ listDir(path: String): Future | null>; } // ============================================================================ // Source File // ============================================================================ /** * A loaded and parsed source file. * * SourceFiles are cached by canonical path. Requesting the same path * twice always returns the same instance. */ export class SourceFile { /** Canonical path identifying this library. */ path: String; /** Whether this library is part of the standard library. */ isStdlib: boolean; /** The package this library belongs to (empty string if none). */ packageName: String; /** The original source code. */ source: String; /** Parsed AST. */ ast: Module; /** * Opaque version tag for cache invalidation. * * The compiler itself does not interpret this value. It is set by * the host (e.g., the language service passes the editor document * version) and compared by the loader to decide whether a cached * file is still fresh. */ version: String; /** * Resolved import mappings. * Key: import specifier as written in source (e.g., "./utils", "zena:string") * Value: canonical resolved path */ imports = new Map(); /** * Scope analysis result (populated after scope building). * Contains the scope tree, exports, unresolved names, and references. */ var scopeResult: ScopeResult | null = null; /** * Errors the parser recovered from, empty unless the loader was told * to recover. A file lands here with a usable AST *and* real errors: * whoever reports diagnostics has to include these. */ var parseErrors: GrowableArray = new Array(0); /** * What this file's AST is fit for. Ask this rather than inferring it * from `parseErrors` — the parser owns the answer, and it is the one * thing that changes as recovery grows to cover more of the grammar. */ var completeness = ParseCompleteness.Complete; new( this.path: String, this.isStdlib: boolean, this.packageName: String, this.source: String, this.ast: Module, this.version: String ); } /** * A module's top-level statements, preceded by the import declarations * its implicit prelude imports stand for — every import it makes. */ export function moduleStatements(lib: SourceFile): GrowableArray { let statements = new Array(lib.ast.body.length); let {scopeResult} = lib; if (scopeResult != null) { for (let decl in scopeResult.implicitImportDeclarations) { statements.push(decl); } } for (let stmt in lib.ast.body) { statements.push(stmt); } return statements; } // ============================================================================ // Library Graph // ============================================================================ /** Result of topologically sorting source files by their dependencies. */ export class SourceFileGraph( /** Source files in topological order (dependencies before dependents). */ files: GrowableArray, /** True if a cycle was detected (libraries still returned, best-effort order). */ hasCycle: boolean ) // ============================================================================ // Library Loader // ============================================================================ /** * Loads, parses, and caches Zena libraries. * * The LibraryLoader is responsible for: * - Loading library source code (via CompilerHost) * - Parsing source into ASTs * - Caching loaded libraries by path (ensuring identity) * - Recursively loading dependencies (via import resolution) * * Loading reads a library's imports together and parses them in the * order the imports are written: a library is parsed and cached, a read * of every import it names starts, and then each import is awaited and * loaded in turn. The parse order matters because every parse draws * node ids from this loader's one generator, and those ids reach the * emitted module; the reads' order does not, since a read changes * nothing until its turn. * * Usage: * let loader = new LibraryLoader(host, resolver); * let entry = await loader.load("/path/to/main.zena"); * let graph = loader.computeGraph([entry]); */ /** A fresh identifier table with the keywords already in it. */ function primedTable() { let table = new StringTable(4096); primeKeywords(table); return table; } export class LibraryLoader { #host: CompilerHost; #resolver: ModuleResolver; #cache: HashMap; /** * The reads in flight, by path. A library's imports are all read at * once and parsed in source order, so a read is started when its * importer is parsed and finished when its turn comes; two importers * of one file share its read. Cleared when a load starts: a read a * failed load left here would serve the next load stale text. */ #pending = new Map>(); /** * Node ids must be unique across every file this loader has parsed — * they key the semantic model, so two files sharing one id means one * file's answers get served for the other's node. The loader owns the * generator because the loader owns the parse cache: the set of files * the ids have to be unique across is exactly the set it holds. * * It used to be a defaulted parameter on `load`, which made a * mismatched generator something a caller could supply by omission — * and `invalidate` and `refreshCache` both did, re-parsing from 1 and * colliding with everything already cached. */ #nodeIdGenerator = makeNodeIdGenerator(); /** * The identifier table every parse this loader drives shares. The * loader owns it for the same reason it owns the node id generator: * the useful scope for interning is exactly the set of files it holds, * so a name is copied once however many files mention it and every * identifier of one spelling is one `String`. Primed with the * keywords, so a lexed `return` interns to the very instance * `KEYWORDS` is keyed by. */ #table = primedTable(); /** * The generator every node this loader's modules contain draws its id * from, for nodes the compiler adds to them after parsing. */ nodeIdGenerator: IdGenerator { get { return this.#nodeIdGenerator; } } var loadTime: f64 = 0.0; var parseTime: f64 = 0.0; /** * Whether to recover from the parse errors recovery covers, keeping a * partial AST, instead of letting them abort the load. The errors are * still reported — they land on `SourceFile.parseErrors`. Editors want * this; a build does not. */ #recover: boolean; new(host: CompilerHost, resolver: ModuleResolver, recover: boolean = false) : #host = host, #resolver = resolver, #recover = recover, #cache = new Map(); /** * Load a library by its canonical path. * * If the library has already been loaded, returns the cached record. * Otherwise: loads source → parses → caches → recursively loads imports. * * If `version` is provided, the cache is checked for a matching version. * A cached file with a different version is invalidated and re-loaded. * When omitted (or null), the cache is trusted unconditionally. */ async load(path: String, version: String | null = null): Future { if (this.#cache.has(path)) { // Version check: if a version was provided and it matches, use cache. // If no version was provided, always trust the cache. if (version == null || this.#cache[path].version == version) { return this.#cache[path]; } // Version mismatch — invalidate and re-load below. this.#cache.delete(path); } this.#pending.clear(); let tLoad = performance.start("load"); let source = await this.#host.readFile(path); this.loadTime = this.loadTime + performance.end(tLoad); return await this.#register(path, source, version, null); } /** * Load a module the compiler wrote itself, from its source rather * than from the host. `pinned` maps import specifiers the source * uses to paths that are already loaded, so a synthesized module can * import the entry it was written for without the resolver spelling * the entry's path a second way and loading it twice. Every other * import resolves as usual. */ async loadSynthesized( path: String, source: String, pinned: HashMap ): Future { if (this.#cache.has(path)) { return this.#cache[path]; } return await this.#register(path, source, null, pinned); } /** Parse `source` as the module at `path`, cache it, and load what it imports. */ async #register( path: String, source: String, version: String | null, pinned: HashMap | null ): Future { let isStdlib = path.startsWith("zena:") || path.contains("stdlib"); let packageName = this.#resolver.findPackageForPath(path); let tParse = performance.start("parse"); var parseErrors = new Array(0); var completeness = ParseCompleteness.Complete; let ast = if (this.#recover) { let result = parseRecovering(source, path, this.#nodeIdGenerator, this.#table); parseErrors = result.diagnostics; completeness = result.completeness; result.ast } else { // Without recovery an incomplete parse throws, so reaching here at // all means the AST accounts for the whole file. parse(source, path, this.#nodeIdGenerator, this.#table) }; this.parseTime = this.parseTime + performance.end(tParse); // Cache the record BEFORE resolving imports to handle circular imports. let record = new SourceFile(path, isStdlib, packageName, source, ast, version ?? ""); record.parseErrors = parseErrors; record.completeness = completeness; this.#cache[path] = record; // Resolve every import, and start reading each one that is not // loaded yet, so the reads are in flight together. A failure to // resolve or load a dependency is left for type checking to report // as ModuleNotFound. let targets = new Array(); for (let stmt in ast.body) { let specifier = match (stmt) { case ImportDeclaration {moduleSpecifier}: moduleSpecifier.value case ExportAllDeclaration {moduleSpecifier}: moduleSpecifier.value case ExportFromDeclaration {moduleSpecifier}: moduleSpecifier.value case _: { continue; } }; let target = this.#resolveImport(record, pinned, specifier); if (target != null && !this.#cache.has(target)) { targets.push(target); this.#startRead(target); } } // Then load them in source order. The order is what keeps node ids // where they were: each parse draws from the one generator, so // parsing in the order the reads completed would hand out // different ids on every run. for (let target in targets) { if (this.#cache.has(target)) { continue; } let tLoad = performance.start("load"); let source = await this.#finishRead(target); this.loadTime = this.loadTime + performance.end(tLoad); if (source == null) { // No file there, so this is the missing-library case after all. continue; } // Parsing sits outside the read's catch. A dependency that does // not parse throws, and swallowing that would report the file as // missing at the import site — naming a file that is sitting right // there, and saying nothing about the error or its line. await this.#register(target, source, null, null); } return record; } /** * The path one of `record`'s imports names, recorded on the record; * null when there is nothing to load for it. */ #resolveImport( record: SourceFile, pinned: HashMap | null, specifier: String ): String | null { // An empty specifier is the parser's stand-in for one that has not // been typed yet (see #missingModuleSpecifier). There is nothing to // resolve, so the declaration contributes no dependency. if (specifier == "") { return null; } if (pinned != null && pinned.has(specifier)) { record.imports[specifier] = pinned[specifier]; return null; } // Two failures here mean the import names nothing, and the checker // reports both against the import itself: a specifier that does not // resolve, and a path with no file behind it. try { let resolved = this.#resolver.resolve(specifier, record.path); record.imports[specifier] = resolved.path; return resolved.path; } catch (e) { // Nothing to load, and nothing more to say here. return null; } } /** Starts reading `path` unless a read of it is already in flight. */ #startRead(path: String): void { if (this.#pending.has(path)) { return; } this.#pending[path] = this.#host.readFile(path); } /** * The source of `path`, from the read `#startRead` began, or from one * started now; null when the host has no such file. */ async #finishRead(path: String): Future { this.#startRead(path); let read = this.#pending[path]; this.#pending.delete(path); try { return await read; } catch (e) { return null; } } /** * Delete a file from the loader cache (e.g. on editor document close/delete). * * Evicts the file from the cache and clears `scopeResult` on any cached file * that directly imports it so import wiring will fail or update on next compile. */ delete(path: String): void { let norm = normalizePath(path); if (this.#cache.has(norm)) { this.#cache.delete(norm); } if (this.#cache.has(path)) { this.#cache.delete(path); } // The files that import this one keep their scope trees. Import // wiring is redone for every file on every compile (the compiler's // wire-to-fixpoint pass re-links each import specifier to the // exporter's current Symbol), so nothing about the importer's own // scope depends on the file that changed — and a rebuilt scope // would mint fresh Symbols, which a check result carried forward // for the importer could not answer for (ProgramCheckResult.scopes). } /** * Invalidate a single file by canonical path (push-based). * * Evicts the file from the cache, then re-loads it from the host so * it's ready for the next compile. Files that import it keep their * scope trees; the compiler re-wires every import on each compile. * * Use this in push-based environments (IDE, language service) where * the host knows exactly which file changed. For pull-based batch * compilation, use `refreshCache()` instead. */ async invalidate(path: String): Future { this.delete(path); // Re-load so the file is back in the cache with fresh content. await this.load(path); } /** * Refresh the loader cache by re-reading non-stdlib files from the host * (pull-based). * * For each cached user file, re-reads the source from the host and * compares it to the cached source. If the source changed, the cache * entry is removed and re-loaded. * * Use this in pull-based environments (CLI, batch compilation) where * no external notification of file changes is available. In push-based * environments (IDE), prefer `invalidate(path)` for targeted updates. */ async refreshCache(): Future { // Collect paths first — can't mutate the map during iteration. let paths = new Array(); let iter = this.#cache.keys(); for (let path in iter) { paths.push(path); } // Phase 1: Detect changed files and remove from cache. Every file // is read at once, and compared in order. this.#pending.clear(); let userPaths = new Array(); for (let path in paths) { if (!this.#cache[path].isStdlib) { userPaths.push(path); this.#startRead(path); } } let changedPaths = new Set(); for (let path in userPaths) { let cached = this.#cache[path]; let currentSource = await this.#finishRead(path); if (currentSource != cached.source) { changedPaths.add(path); this.#cache.delete(path); } } if (changedPaths.size > 0) { // Files importing a changed one keep their scope trees, as in // `delete`: import wiring is redone every compile, and a fresh // scope would invalidate their carried check results. // Phase 2: Re-load evicted files so they're back in the cache // before computeGraph() tries to find them. for (let path in changedPaths) { await this.load(path); } } } /** Check if a library has already been loaded. */ has(path: String): boolean { return this.#cache.has(path); } /** Get a previously loaded library without triggering a load. */ get(path: String): SourceFile | null { if (this.#cache.has(path)) { return this.#cache[path]; } return null; } /** * Compute the topological order of libraries starting from a root. * * Returns libraries in dependency order: if A imports B, then B appears * before A. This ordering is required for correct type checking (a module's * dependencies must be checked before the module itself). */ computeGraph(entries: GrowableArray): SourceFileGraph { let visited = new Map(); let stack = new Map(); let sorted = new Array(); var hasCycle = false; for (let entry in entries) { if (this.#topoVisit(entry, visited, stack, sorted)) { hasCycle = true; } } return new SourceFileGraph(sorted, hasCycle); } #topoVisit( lib: SourceFile, visited: HashMap, stack: HashMap, sorted: GrowableArray ): boolean { if (visited.has(lib.path)) { return false; } if (stack.has(lib.path)) { return true; } stack[lib.path] = 1; var hasCycle = false; // Visit dependencies first. Implicit prelude imports precede // written imports, matching `moduleStatements` and `beginModule`. let {scopeResult} = lib; if (scopeResult != null) { for (let decl in scopeResult.implicitImportDeclarations) { let importPath = decl.moduleSpecifier.value; if (this.#cache.has(importPath)) { if (this.#topoVisit(this.#cache[importPath], visited, stack, sorted)) { hasCycle = true; } } } } let iter = lib.imports.keys(); for (let specifier in iter) { let importPath = lib.imports[specifier]; if (this.#cache.has(importPath)) { if (this.#topoVisit(this.#cache[importPath], visited, stack, sorted)) { hasCycle = true; } } } stack.delete(lib.path); visited[lib.path] = 1; sorted.push(lib); return hasCycle; } }