/** * The language service: what an editor asks about Zena source, answered * from the compiler's own analysis. Diagnostics, hover, go to definition, * completions and document symbols. * * It keeps the documents an editor has open and serves them to the * compiler in place of the files on disk, and keeps the compiler between * requests so that unchanged files are not checked again. * * Two programs use it: `lsp.zena`, the module the VS Code extension and * the playground load, and `zena lsp`, the language server in the zena * command. Each passes in how to read a file that is not open. * * Each feature is in a module of its own: `hover.zena`, * `completion.zena` and `symbols.zena`, and `compile.zena` generates code * for the playground. This one keeps the compiler and the analysis they * share, and finds definitions. */ import { Compiler, CompilerTarget, createCompiler, checkCompilation, ProgramCheckResult, findNodeAtOffset, resolveMemberDefinition, CompilationResult, CheckResult, } from 'compiler/compiler'; import { Module, ImportDeclaration, ExportAllDeclaration, ExportFromDeclaration, } from 'compiler/ast'; import { parse as parseSource, parseRecovering, ParseCompleteness, } from 'compiler/parser'; import { PackageConfig } from 'compiler/module-resolver'; import { Diagnostic, DiagnosticCode, SourceSpan, DiagnosticSeverity, SyntaxError, spanOf } from 'compiler/diagnostics'; import { Symbol, ScopeResult } from 'compiler/scope'; import { Program, CompilationUnit } from 'compiler/program'; import { DocumentHost, DirReader, FileReader, normalizePath } from './host.zena'; import { FileAnalysis, symbolUnder } from './analysis.zena'; import { HoverResult, hoverAt } from './hover.zena'; import { CompletionItem, memberCompletions, scopeCompletions } from './completion.zena'; import { DocumentSymbol, documentSymbols } from './symbols.zena'; import { CheckedFile } from 'lint/checked'; import { NamedRule, proposeEdits, messageOf } from 'lint/rule'; import { allRules } from 'lint/rules'; import { Edit, applyEdits } from 'lint/source'; import { growable } from 'zena:core'; import { Completer } from 'zena:async'; export { Edit } from 'lint/source'; export { DirReader, FileReader } from './host.zena'; export { HoverResult } from './hover.zena'; export { CompletionItem, CompletionItemKind } from './completion.zena'; export { DocumentSymbol, SymbolKind } from './symbols.zena'; /** * The 0-based byte offset of a 1-based line and column in `source`, the * end of the source for a column past the last line's end, and 0 for a * line past the end. */ function offsetOf(source: String, line: i32, column: i32): i32 { var currentLine = 1; var currentColumn = 1; var i = 0; while (i < source.length) { if (currentLine == line && currentColumn == column) { return i; } if (source.getByteAt(i) == 10) { currentLine += 1; currentColumn = 1; } else { currentColumn += 1; } i += 1; } return if (currentLine == line) { i } else { 0 }; } /** * A diagnostic for a compile that threw instead of reporting. The * message's first line may start with `file:line:column:` or * `file:line:column - Error: `, which becomes the diagnostic's location. */ function diagnosticFromError(message: String, source: String) { let firstLine = message.split('\n')[0]; let parts = firstLine.split(':'); if (parts.length >= 4) { let file = parts[0]; if (let (true, line) = parseI32(parts[1])) { let colPart = parts[2]; let colText = if (colPart.endsWith(' - Error')) { colPart.sliceBytes(0, colPart.length - 8) } else { colPart }; if (let (true, column) = parseI32(colText)) { let location = new SourceSpan(file, offsetOf(source, line, column), 1, line, column); let prefixLength = file.length + parts[1].length + colPart.length + 4; let text = if (firstLine.length > prefixLength) { firstLine.sliceBytes(prefixLength, firstLine.length) } else { message }; return new Diagnostic(DiagnosticCode.UnexpectedToken, text, DiagnosticSeverity.Error, location); } } } return new Diagnostic(DiagnosticCode.UnexpectedToken, message, DiagnosticSeverity.Error, null); } /** * If `entry` is a stdlib module that only compiles on a JS target (`zena-js` * or `js`), returns that target name; otherwise null. */ function stdlibJsTarget(entry: String): String | null { if (!entry.startsWith('zena:')) { return null; } if (entry == 'zena:core/string-js-pure.zena') { return 'js'; } if ( entry == 'zena:js' || entry.startsWith('zena:js/') || entry == 'zena:console/js.zena' || entry == 'zena:console/host.zena' || entry == 'zena:core/js-string-builtins.zena' || entry == 'zena:core/error/stack-host.zena' || entry == 'zena:fetch/host.zena' || entry == 'zena:time/host.zena' || entry.endsWith('-js.zena') || entry.endsWith('-js-host.zena') ) { return 'zena-js'; } return null; } function isComponentOnlyModule(spec: String) { return spec == 'zena:fs' || spec == 'zena:cli' || spec == 'zena:process' || spec == 'zena:wasm' || spec == 'zena:wasi' || spec == 'zena:extensions' || spec == 'zena:host-setup'; } function moduleSpecOfStatement(stmt: anyref): String | null { return match (stmt) { case ImportDeclaration {moduleSpecifier}: moduleSpecifier.value case ExportAllDeclaration {moduleSpecifier}: moduleSpecifier.value case ExportFromDeclaration {moduleSpecifier}: moduleSpecifier.value case _: null }; } function astNeedsZenaJs(ast: Module): (boolean, boolean) { var hasZenaJs = false; var hasComponentOnly = false; for (let stmt in ast.body) { let spec = moduleSpecOfStatement(stmt); if (spec == null) { continue; } if (spec == 'zena:js') { hasZenaJs = true; } else if (isComponentOnlyModule(spec)) { hasComponentOnly = true; } } return (hasZenaJs, hasComponentOnly); } class TargetState { targetName: String; var checked: ProgramCheckResult | null = null; scopes = new Map(); compilations = new Map(); new(this.targetName); } /** * Stateful language service backed by a persistent Compiler. * * Owns the compiler host, compiler instance, and cached analysis * results. The compiler's LibraryLoader caches parsed stdlib files * across calls so they aren't re-parsed on every keystroke. * * `check` and `checkProgram` compile, which awaits the host, and the * compiler runs one compile at a time. A caller may ask again before the * last answer is in — an editor on every keystroke — so the two queue: * each waits for the one before it to settle, and the open-document map * is updated only once its turn comes, so a compile in flight reads the * documents it started with. The queries (`getDefinition`, `getHover`, * `getCompletions`, `lint`) read the last check that finished. */ export class LanguageService { #host: DocumentHost; var #compiler: Compiler; #makeCompiler: (() => Compiler) | null; /** Default target for files that do not require a specific JS target. */ #targetName: String; /** Per-target analysis state (`checked`, `scopes`, `compilations`). */ #states = new Map(); /** Target name under which each checked entry (`#keyOf(path)`) was analyzed. */ #entryTargets = new Map(); /** Stdlib modules whose open editor buffer differs from what was first loaded. */ #dirtyStdlib = new Map(); /** The last compile queued, which the next waits for; null when none is in flight. */ var #tail: Future | null = null; new( host: DocumentHost, compiler: Compiler, targetName: String, makeCompiler: (() => Compiler) | null = null, ) : #host = host, #compiler = compiler, #makeCompiler = makeCompiler, #targetName = targetName; /** * Takes this call's turn: waits for the compile before it, if any, and * returns the completer the caller settles when it is done, so the * next call can take its turn. The completer is always settled, so the * wait never fails. */ async #takeTurn(): Future> { let previous = this.#tail; let done = new Completer(); this.#tail = done.future; if (previous != null) { await previous; } return done; } /** Releases the turn `#takeTurn` gave, so the next queued call runs. */ #endTurn(done: Completer): void { if (this.#tail == done.future) { this.#tail = null; } done.complete(); } targetName: String { get { return this.#targetName; } } /** The compiler's library set for this service's target. */ #target: CompilerTarget { get { return this.#compiler.forTarget(this.#targetName); } } /** Compile `entry` on this service's target. */ async #compile(entry: String): Future { return await this.#compiler.compile({entry, target: this.#targetName}); } preloadStdlib(): void { // Best-effort preload: intentionally no-op to avoid polluting analysis // state with a synthetic core root cycle. } #keyOf(path: String): String { return normalizePath(this.#host.canonicalizePath(path)); } #stateFor(target: String): TargetState { if (let (true, existing) = this.#states.get(target)) { return existing; } let created = new TargetState(target); this.#states[target] = created; return created; } #targetNameOf(key: String): String { if (let (true, entryTarget) = this.#entryTargets.get(key)) { return entryTarget; } let jsStdlib = stdlibJsTarget(key); if (jsStdlib != null) { return jsStdlib; } if (let (true, primary) = this.#states.get(this.#targetName)) { if (primary.scopes.has(key) || primary.compilations.has(key)) { return this.#targetName; } } for (let tName in this.#states.keys()) { let st = this.#states[tName]; if (st.scopes.has(key) || st.compilations.has(key)) { return tName; } } return this.#targetName; } #inferTarget(entry: String, source: String): String { if (this.#targetName == 'js' || this.#targetName == 'zena-js') { return this.#targetName; } let jsStdlib = stdlibJsTarget(entry); if (jsStdlib != null) { return jsStdlib; } if (source.contains('zena:js')) { let parsed = try { parseRecovering(source, entry).ast } catch (e) { null }; if (parsed != null) { let (hasZenaJs, hasComponentOnly) = astNeedsZenaJs(parsed); if (hasZenaJs && !hasComponentOnly) { return 'zena-js'; } } } return this.#targetName; } #findLoadedSource(key: String): String | null { for (let tName in this.#compiler.compiledTargets) { let sf = this.#compiler.forTarget(tName).getSourceFile(key); if (sf != null) { return sf.source; } } return null; } #resetCompiler(): void { let make = this.#makeCompiler; if (make != null) { this.#compiler = make(); this.#states.clear(); this.#entryTargets.clear(); } } /** * Register or update an open document's source in the host and compiler cache. */ openDocument(path: String, source: String): void { let normalized = normalizePath(path); let key = this.#keyOf(path); let existingSource = this.#findLoadedSource(key); this.#host.setDocument(path, source); this.#host.setDocument(normalized, source); if (key.startsWith('zena:')) { if (existingSource != null && existingSource != source) { this.#dirtyStdlib[key] = true; this.#resetCompiler(); } return; } if (existingSource != null && existingSource == source) { return; } this.#compiler.delete(path); this.#compiler.delete(normalized); } /** * Remove an open document, evicting it from compiler cache and analysis state. */ closeDocument(path: String): void { let normalized = normalizePath(path); let key = this.#keyOf(path); this.#host.deleteDocument(path); this.#host.deleteDocument(normalized); this.#entryTargets.delete(key); for (let tName in this.#states.keys()) { let st = this.#states[tName]; st.compilations.delete(key); if (!key.startsWith('zena:')) { st.scopes.delete(key); } } if (key.startsWith('zena:')) { if (this.#dirtyStdlib.has(key)) { this.#dirtyStdlib.delete(key); this.#resetCompiler(); } return; } this.#compiler.delete(path); this.#compiler.delete(normalized); } /** * Parse, scope-analyze, and type-check a source string with full * cross-module import resolution. * * Updates the open documents map for `path` so that any module * importing this file sees the editor's version, then compiles * starting from `path` as the root of the dependency tree. */ async check(path: String, source: String): Future> { let entry = this.#keyOf(path); let target = this.#inferTarget(entry, source); let diagnostics = await this.checkForTarget(path, source, target); if (target != 'zena-js' && target != 'js') { let state = this.#stateFor(target); if (let (true, compilation) = state.compilations.get(entry)) { var transitiveZenaJs = false; var transitiveComponentOnly = false; for (let file in compilation.files) { if (file.isStdlib) { continue; } let (hasJs, hasComp) = astNeedsZenaJs(file.ast); if (hasJs) { transitiveZenaJs = true; } if (hasComp) { transitiveComponentOnly = true; } } if (transitiveZenaJs && !transitiveComponentOnly) { return await this.checkForTarget(path, source, 'zena-js'); } } } return diagnostics; } #remapEntryDiagnostic(diag: Diagnostic, entry: String, fsPath: String): Diagnostic { if (entry == fsPath) { return diag; } let loc = diag.location; if (loc == null || loc.file != entry) { return diag; } let remapped = new SourceSpan(fsPath, loc.start, loc.length, loc.line, loc.column); return new Diagnostic(diag.code, diag.message, diag.severity, remapped); } /** * Preserves still-valid CheckResults from earlier checks of other open * entry files on the same `target` so hovering in an inactive split pane * or switching tabs does not lose already-checked modules. */ #mergeCarried(targetName: String, previous: ProgramCheckResult | null, current: ProgramCheckResult): void { if (previous == null || previous.sharedState != current.sharedState) { return; } let target = this.#compiler.forTarget(targetName); for (let prevPath in previous.results.keys()) { if (current.results.has(prevPath)) { continue; } let sf = target.getSourceFile(prevPath); if (sf == null) { continue; } if (!previous.sources.has(prevPath) || previous.sources[prevPath] != sf.source) { continue; } if (!previous.asts.has(prevPath) || previous.asts[prevPath] != sf.ast) { continue; } current.results[prevPath] = previous.results[prevPath]; current.sources[prevPath] = sf.source; current.asts[prevPath] = sf.ast; if (previous.exportSignatures.has(prevPath)) { current.exportSignatures[prevPath] = previous.exportSignatures[prevPath]; } if (previous.scopes.has(prevPath)) { current.scopes[prevPath] = previous.scopes[prevPath]; } } } /** * Like `check`, but against an explicit `target` instead of the * service's default target. */ async checkForTarget(path: String, source: String, target: String): Future> { let turn = await this.#takeTurn(); try { this.openDocument(path, source); let normalized = normalizePath(path); let entry = this.#keyOf(path); let result = await this.#compiler.compile({entry, target}); let state = this.#stateFor(target); this.#entryTargets[entry] = target; // Scopes for every file the compile reached, so that definition // and hover work in a dependency the editor navigates into. for (let file in result.files) { let scopes = file.scopeResult; if (scopes != null) { state.scopes[normalizePath(file.path)] = scopes; } } state.compilations[entry] = result; // Type-check all modules, carrying forward unchanged results on `target`. let previous = state.checked; let checked = checkCompilation(result, previous); this.#mergeCarried(target, previous, checked); state.checked = checked; // Recovering from a parse error does not excuse it. The file was // analyzed anyway, so report what the parser had to work around // ahead of what the checker went on to find. let diagnostics = new Array(checked.diagnostics.length); for (let file in result.files) { for (let parseError in file.parseErrors) { diagnostics.push(this.#remapEntryDiagnostic(parseError, entry, normalized)); } } for (let diagnostic in checked.diagnostics) { diagnostics.push(this.#remapEntryDiagnostic(diagnostic, entry, normalized)); } return diagnostics; } catch (e) { if (e is SyntaxError) { return growable([e.diagnostic]); } return growable([ diagnosticFromError(e.message, source), ]); } finally { this.#endTurn(turn); } } /** * A document checked for running: the program to generate code from, * or null when it has errors. `compile.zena` generates the code. */ async checkProgram(path: String, source: String): Future { let target = this.#inferTarget(this.#keyOf(path), source); return await this.checkProgramForTarget(path, source, target); } /** * Like `checkProgram`, but against an explicit `target`. */ async checkProgramForTarget(path: String, source: String, target: String): Future { let turn = await this.#takeTurn(); try { this.#host.setDocument(path, source); await this.#compiler.invalidate(path); let compiled = await this.#compiler.compile({entry: path, target}); // A recovered AST holds stand-ins where the user has not finished // typing, so it must not reach codegen. Check here rather than // below: parse errors are not in the checker's diagnostics, so the // error scan that follows would report none. for (let file in compiled.files) { if (file.completeness != ParseCompleteness.Complete) { return null; } } let state = this.#stateFor(target); let previous = state.checked; let checked = checkCompilation(compiled, previous); this.#mergeCarried(target, previous, checked); state.checked = checked; for (let diagnostic in checked.diagnostics) { if (diagnostic.severity == DiagnosticSeverity.Error) { return null; } } let units = new Array(); var entryUnit: CompilationUnit | null = null; for (let file in compiled.files) { if (let (true, res) = checked.results.get(file.path)) { let unit = {ast: file.ast, model: res.model}; units.push(unit); if (file.path == compiled.entry.path || file.ast.path == compiled.entry.ast.path) { entryUnit = unit; } } else { console.error(`File not in checkResult: ${file.path}`); } } if (entryUnit == null && units.length > 0) { entryUnit = units[units.length - 1]; } return new Program(units, entryUnit); } finally { this.#endTurn(turn); } } /** * What the lint rules find in a checked document: the edits of one * pass, in order and not overlapping, each the fix for its finding. * Empty until `check` has seen the document, or when it has errors — * a rule reads types the checker could not give. */ lint(path: String): GrowableArray { let key = this.#keyOf(path); let state = this.#stateFor(this.#targetNameOf(key)); let checked = state.checked; var compilation: CompilationResult | null = null; if (let (true, c) = state.compilations.get(key)) { compilation = c; } if (compilation == null || checked == null) { return new GrowableArray(0); } let entry = compilation.entry; var result: CheckResult | null = null; if (let (true, r) = checked.results.get(entry.path)) { result = r; } if (result == null) { return new GrowableArray(0); } for (let diagnostic in result.diagnostics) { if (diagnostic.severity == DiagnosticSeverity.Error) { return new GrowableArray(0); } } for (let file in compilation.files) { if (file.completeness != ParseCompleteness.Complete) { return new GrowableArray(0); } } let file = new CheckedFile(entry.path, entry.source, entry.ast, result.model, result.diagnostics); file.isStdlib = entry.path.startsWith('zena:'); return proposeEdits(file, this.#rules); } /** The document's text with every lint finding fixed. */ fixAll(path: String, source: String): String { return applyEdits(source, this.lint(path)); } /** The one-line message of the rule a finding came from. */ lintMessage(finding: Edit): String { return messageOf(finding, this.#rules); } #rules: GrowableArray = allRules(); #scopesOf(key: String, targetName: String): ScopeResult | null { let state = this.#stateFor(targetName); if (let (true, scopes) = state.scopes.get(key)) { return scopes; } return null; } /** * The analysis of a checked file: the entry of a compile, or a file one * reached, which an editor opens by going to a definition in it. */ #analysis(key: String, targetName: String): FileAnalysis | null { let state = this.#stateFor(targetName); let checked = state.checked; if (checked == null) { return null; } let scopes = this.#scopesOf(key, targetName); if (let (true, compilation) = state.compilations.get(key)) { if (let (true, result) = checked.results.get(compilation.entry.path)) { return {ast: compilation.entry.ast, model: result.model, scopes: scopes}; } } if (let (true, result) = checked.results.get(key)) { let source = this.#compiler.forTarget(targetName).getSourceFile(key); if (source != null) { return {ast: source.ast, model: result.model, scopes: scopes}; } } return null; } /** * Where a symbol is declared. An import leads to the declaration it * names, in the module that declares it. That is only sometimes the * module the import names: `import { Error } from 'zena:core'` names * the library's entry point, and the declaration is in a file behind * it. */ #declarationSpan(sym: Symbol, key: String): SourceSpan | null { let target = sym.resolveTarget(); let decl = target.declaration; if (decl == null) { return null; } let loc = decl.loc; if (loc == null) { return null; } let rawFile = if (target.modulePath.length > 0) { target.modulePath } else { key }; if (rawFile.startsWith('wit:')) { return null; } let file = this.#host.resolveToFilesystemPath(rawFile); // Line and column come from the location's own path (the module it // was parsed under); the span names the filesystem path the editor // wants. let span = spanOf(loc, file); return new SourceSpan(file, span.start, span.length, span.line, span.column); } /** The definition of what is at a byte offset in a checked file. */ getDefinition(path: String, offset: i32): SourceSpan | null { let key = this.#keyOf(path); let targetName = this.#targetNameOf(key); let scopes = this.#scopesOf(key, targetName); if (scopes == null) { return null; } let file = this.#analysis(key, targetName); let node = if (file == null) { null } else { findNodeAtOffset(file.ast, offset) }; let sym = symbolUnder(scopes, node, offset); if (sym != null) { return this.#declarationSpan(sym, key); } // A member access, like the `x` in `p.x`, has no symbol of its own: // the member is found through the object's type. if (file == null) { return null; } let memberSpan = resolveMemberDefinition( file.ast, offset, file.model, scopes, this.#compiler.forTarget(targetName), ); if (memberSpan == null || memberSpan.file.startsWith('wit:')) { return null; } let memberFile = this.#host.resolveToFilesystemPath(memberSpan.file); return new SourceSpan(memberFile, memberSpan.start, memberSpan.length, memberSpan.line, memberSpan.column); } /** Hover for a byte offset in a checked file, or null for none. */ getHover(path: String, offset: i32): HoverResult | null { let key = this.#keyOf(path); let targetName = this.#targetNameOf(key); let checked = this.#stateFor(targetName).checked; let file = this.#analysis(key, targetName); if (checked == null || file == null) { return null; } return hoverAt(file, offset, checked, this.#compiler.forTarget(targetName)); } /** Completions at a byte offset. */ getCompletions(path: String, offset: i32): Array { let key = this.#keyOf(path); let targetName = this.#targetNameOf(key); let checked = this.#stateFor(targetName).checked; let file = this.#analysis(key, targetName); if (checked != null && file != null) { let members = memberCompletions(file, offset, checked); if (members != null) { return members; } } return scopeCompletions(this.#scopesOf(key, targetName)); } /** * The outline of a document: its top-level declarations, and in a test * file its suites and tests. It parses `source` itself, so it works on * a document that has not been checked. */ getDocumentSymbols(path: String, source: String): Array { let symbols = try { documentSymbols(parseSource(source, path)) } catch (e) { new Array() }; return symbols; } } /** * Create a LanguageService with the standard pipeline. `readFile` reads * the files that are not open, the standard library's included; * `target` decides which of the standard library's per-target modules * the analysis sees; and `packages` is the package map imports like * `compiler/parser` resolve through, none if null. */ export function createLanguageService( stdlibRoot: String, readFile: FileReader, target: String = 'js', packageMap: Map | null = null, listDir: DirReader | null = null, ): LanguageService { let host = new DocumentHost(stdlibRoot, readFile, listDir); let packages = packageMap ?? new Map(); // Source in an editor is source mid-edit: recover from what can be // recovered from, so an unfinished import still yields an AST to // answer completions and symbols against. The errors are still // reported — check() folds them in with the rest. // Not `warnUnnecessaryCasts`: that flag also turns on the checker's // adaptation warnings ("boxed to pass as"), which valid code gets, so // the lint rule for unnecessary casts has nothing to read here and is a // command-line rule only. let makeCompiler = () => createCompiler(host, { stdlibRoot, packages, recoverFromParseErrors: true, }); let ls = new LanguageService(host, makeCompiler(), target, makeCompiler); ls.preloadStdlib(); return ls; }