/** * Documenting a whole package. * * Every module the package exports is compiled and checked, then walked * into the API model. Modules are compiled in batches through one * `Compiler` each: a compiler accumulates per-entry state that is never * collected, so one compiler for a whole package gets slower than * several — the measurements are in packages/compiler/CONTEXT.md, * and 16 is where the win has already been taken. * * Type errors do not stop extraction. They are returned alongside the * model, for the caller to report and to exit non-zero on: a package * that does not check is exactly when someone wants to read its docs. */ import { readFile } from 'zena:fs'; import { createCompiler, checkCompilation, CompilerHost, PackageConfig, ModuleExport, ProgramCheckResult, Diagnostic, DiagnosticSeverity, } from 'compiler/compiler'; import { stdlibIdToRelativeFile } from 'compiler/module-resolver'; import { isWitModulePath, synthesizeWitModule } from 'compiler/wit-module-synth'; import { directoryNames } from 'compiler/fs-host'; import { Package } from './package.zena'; import { join2 } from 'zena:path'; import { extractModule, ExtractOptions } from './extract.zena'; import { resolveReexports, WalkCache } from './reexports.zena'; import { inheritMembers } from './inherit.zena'; import { resolveLinks } from './publish.zena'; import { Naming, moduleId } from './ids.zena'; import { ModelLinker } from './links.zena'; import { DocPackage } from './model.zena'; /** The standard library's package name; the compiler loads it itself. */ let STDLIB_PACKAGE = 'zena'; /** How many entry points share one compiler. */ let ENTRIES_PER_COMPILER = 16; /** What to document, and where the standard library lives. */ export final class DocumentOptions { /** Document non-exported declarations and `#private` members. */ var includePrivate: boolean = false; /** The stdlib source root, for resolving `zena:` imports. */ var stdlibRoot: String = 'packages/stdlib/zena'; /** The compilation target, which decides virtual modules' entry files. */ var target: String = 'zena-component'; } /** A package's documentation, and what went wrong producing it. */ export final class DocumentResult { docs: DocPackage; /** Every diagnostic the checker reported, across all modules. */ var diagnostics = new Array(0); /** * Modules that could not be compiled at all, as `: `. * A module that resolves to a file that is not there, or that names a * package the compiler does not know, fails here rather than * producing diagnostics. */ var failures = new Array(0); new(this.docs); /** Whether any module failed or reported an error. */ hasErrors(): boolean { if (this.failures.length > 0) { return true; } for (let diagnostic in this.diagnostics) { if (diagnostic.severity == DiagnosticSeverity.Error) { return true; } } return false; } } /** Reads a package's files, and the stdlib's, off disk. */ final class FsHost implements CompilerHost { #stdlibRoot: String; new(stdlibRoot: String) : #stdlibRoot = stdlibRoot; async readFile(path: String): Future { // A WIT-backed package's module has no file: the source is // synthesized from the package's WIT. if (isWitModulePath(path)) { return await synthesizeWitModule(this, path); } if (path.startsWith('zena:')) { return await readFile(join2(this.#stdlibRoot, stdlibIdToRelativeFile(path))); } return await readFile(path); } async listDir(path: String): Future | null> { return await directoryNames(path); } } /** Documents every module a package exports. */ export async function documentPackage( pkg: Package, options: DocumentOptions ): Future { let docs = new DocPackage(pkg.name, pkg.root); docs.version = pkg.version; let out = new DocumentResult(docs); let extractOptions = new ExtractOptions(); extractOptions.includePrivate = options.includePrivate; // Shared across batches: a file is walked into the model once per run, // whichever module first needs it. let cache = new WalkCache(); var i = 0; while (i < pkg.modules.length) { let end = if (i + ENTRIES_PER_COMPILER < pkg.modules.length) { i + ENTRIES_PER_COMPILER } else { pkg.modules.length }; await documentBatch(pkg, options, extractOptions, i, end, out, cache); i = end; } // Both wait until every module has been walked: a link resolves to // the module that declares a type, and the page hosting it may be // another module entirely. resolveLinks(docs); inheritMembers(docs); return out; } /** One batch of modules, through one compiler. */ async function documentBatch( pkg: Package, options: DocumentOptions, extractOptions: ExtractOptions, from: i32, to: i32, out: DocumentResult, cache: WalkCache ): Future { let host = new FsHost(options.stdlibRoot); let packages = new Map(); if (pkg.name != STDLIB_PACKAGE) { // The compiler must resolve `pkg:module` to the same entry file // zenadoc chose, so it gets the module list zenadoc computed rather // than re-deriving one — otherwise a module whose manifest entry is // `nested/widgets.zena` resolves to `widgets.zena` and is missing. let exports = new Map(); for (let module in pkg.modules) { exports[module.name] = new ModuleExport(null, module.file); } packages[pkg.name] = new PackageConfig(pkg.root, exports); } let compiler = createCompiler(host, {stdlibRoot: options.stdlibRoot, packages}); let target = compiler.forTarget(options.target); // Threaded across the batch's entry points so that a file whose source // has not changed — the whole standard library, on every entry after // the first — carries its CheckResult forward instead of being checked // again. Parsing is already shared: the loader caches a SourceFile per // path, and extraction walks that same AST. var previous: ProgramCheckResult | null = null; var i = from; while (i < to) { let module = pkg.modules[i]; try { let entry = target.resolve(moduleId(pkg.name, module.name), ''); let result = await compiler.compile({entry: entry.path, target: options.target}); let checkResults = checkCompilation(result, previous); previous = checkResults; for (let diagnostic in checkResults.entry.diagnostics) { out.diagnostics.push(diagnostic); } let checked = checkResults.results[result.entry.path]; let model = if (checked == null) { null } else { checked.model }; let naming = new Naming(pkg.name, pkg.root); let documented = extractModule(moduleId(pkg.name, module.name), module.name, module.file, result.entry.ast, model, extractOptions, new ModelLinker(model, naming)); // Through the same naming as a link, or the two spell the same // module differently and the alias never matches. documented.canonicalId = naming.idOf(result.entry.path); // A module's own walk is what a facade re-exporting it should copy // from, so it goes in the cache before any re-export is followed. cache.put(result.entry.path, documented); resolveReexports(documented, pkg.name, pkg.root, target, result, checkResults, extractOptions, cache); out.docs.modules.push(documented); } catch (e) { let reason = if (e is Error) { e.message } else { 'unknown failure' }; out.failures.push(`${module.name}: ${reason}`); } i += 1; } }