compiler 0.0.2-preview.1
The Zena self-hosted compiler
compiler#
The Zena compiler, written in Zena. It parses, type-checks, and compiles Zena source code to WebAssembly (Wasm GC) and WebAssembly Text (WAT).
Usage#
Parsing a Single File#
Use compiler/parser to parse Zena source text into an AST (Module):
import { parse } from 'compiler/parser';
let ast = parse('export let answer = 42;', 'example.zena');
Loading and Checking a Program#
Use compiler (createCompiler and checkCompilation) with a CompilerHost to resolve imports across a module graph and type-check a program:
import {
createCompiler, checkCompilation, buildStdlibConfig
} from 'compiler';
import { FsCompilerHost } from 'compiler/fs-host';
let stdlibRoot = '/stdlib';
let host = new FsCompilerHost(stdlibRoot);
let packages = buildStdlibConfig(stdlibRoot, 'host');
let compiler = createCompiler(host, {
stdlibRoot,
packages,
});
let result = await compiler.compile({
entry: '/workspace/main.zena',
target: 'host',
});
let checked = checkCompilation(result);
Compiling a File to WebAssembly#
Use compiler/compile-file to compile an entry file on disk to a .wasm or .wat output file:
import { compileFile } from 'compiler/compile-file';
let ok = await compileFile({
entry: 'src/main.zena',
outPath: 'out/main.wasm',
stdlibRoot: '/stdlib',
});
Libraries#
compiler/compiler/compiler— compiler entry point (Compiler,createCompiler,checkCompilation,ProgramCheckResult)compiler/compile-file— end-to-end file compilation (compileFile,CompileFileOptions)compiler/tokenizer— lexer and token definitions (tokenize,Token,TokenType)compiler/parser— recursive-descent parser (parse,Parser)compiler/ast— AST node definitions (Node,Module,Statement,Expression,TypeAnnotation)compiler/visitor— AST traversal helpers (walk,Visitor)compiler/scope— lexical scope and symbol resolution (ScopeBuilder,ScopeResult,Symbol)compiler/checker— static type checker (checkModule,CheckResult)compiler/semantic-model— semantic queries over a checked module (SemanticModel)compiler/types— compiler type representations (Type,ClassType,FunctionType,typeToString)compiler/diagnostics— compiler diagnostics and source spans (Diagnostic,DiagnosticCode,DiagnosticSeverity,SourceSpan)compiler/package-manifest—zena-package.jsonmanifest and workspace loader (loadWorkspace,loadPackageMap)compiler/module-resolver— package and relative module specifier resolution (ModuleResolver,PackageConfig)compiler/library-loader— module graph loader (LibraryLoader,CompilerHost,SourceFile)compiler/codegen/binary-generator— WebAssembly GC binary emitter (BinaryGenerator)compiler/codegen/wat-generator— WebAssembly Text (WAT) emitter (WatGenerator)
License#
MIT
13.6 KiB
Raw
/**
* Doc comment parsing.
*
* A doc comment is a block comment whose body starts with a star — the
* slash-star-star form — attached as a leading comment to a
* declaration. Line comments are never doc comments, which is the rule
* the language server already applies to hover.
*
* The body is markdown once each line's leading star (and one following
* space) is removed. Lines that start with `@` outside a fenced code
* block open a block tag; everything before the first tag is the
* description, and its first paragraph is the summary.
*
* Tools read doc comments through `parseLeadingDoc`, or through
* `declarationDoc` and `docCommentAt`, which find the comments to parse:
* zenadoc for API documentation, the language service for hover.
*
* See docs/design/zenadoc.md.
*/
import { Comment, Node, Module, ClassDeclaration, InterfaceDeclaration } from './ast.zena';
import { CommentMap } from './comment-attachment.zena';
import { StringBuilder } from 'zena:core';
// The ASCII bytes this file looks for. `i32` rather than an enum
// because they are compared against `String.getByteAt`, which returns
// one: an enum is nominal, so every comparison would read
// `b == Byte.Star as i32`. An enum would also claim these ten are the
// domain, and the domain is all 256.
let SPACE = 32;
let TAB = 9;
let CR = 13;
let LF = 10;
let HASH = 35;
let STAR = 42;
let DASH = 45;
let AT = 64;
let GT = 62;
let BACKTICK = 96;
/** A block tag such as `@param index the offset to read`. */
export final class DocTag(
/** Tag name without the `@`, e.g. `param`. */
name: String,
/**
* The subject the tag names, for tags that take one (`@param`,
* `@throws`, `@typeParam`): the first whitespace-delimited word after
* the tag. Empty for every other tag.
*/
subject: String,
/** Everything after the tag and its subject, with blank lines kept. */
text: String
)
/** A parsed doc comment. */
export final class DocComment(
/**
* The description's first paragraph, on one line. Empty when the
* description opens with something other than a paragraph, such as a
* fenced code block.
*/
summary: String,
/** Everything before the first block tag, as markdown. */
description: String,
/** Block tags in source order. */
tags: GrowableArray<DocTag>,
/** The comment body with delimiters and star prefixes removed. */
raw: String
) {
/** The first tag with this name, or null. */
firstTag(name: String): DocTag | null {
for (let tag in this.tags) {
if (tag.name == name) {
return tag;
}
}
return null;
}
/** Every tag with this name, in source order. */
tagsNamed(name: String): GrowableArray<DocTag> {
let found = new Array<DocTag>(2);
for (let tag in this.tags) {
if (tag.name == name) {
found.push(tag);
}
}
return found;
}
/**
* The `@deprecated` reason, empty when the tag carries no text, and
* null when the declaration is not deprecated.
*/
deprecation(): String | null {
let tag = this.firstTag('deprecated');
if (tag == null) {
return null;
}
return tag.text;
}
}
/** Whether a comment is a doc comment. */
export function isDocComment(comment: Comment): boolean {
return comment.isBlock && comment.value.length > 0 &&
comment.value.getByteAt(0) == STAR;
}
function isSpace(b: i32) {
return b == SPACE || b == TAB;
}
/** Leading and trailing spaces and tabs removed. */
export function trim(s: String): String {
var start = 0;
var end = s.length;
while (start < end && isSpace(s.getByteAt(start))) {
start += 1;
}
while (end > start && isSpace(s.getByteAt(end - 1))) {
end -= 1;
}
return s.sliceBytes(start, end);
}
/** Trailing whitespace, including newlines, removed. */
function trimTrailingBlank(s: String) {
var end = s.length;
while (end > 0) {
let b = s.getByteAt(end - 1);
if (b == LF || b == CR || isSpace(b)) {
end -= 1;
} else {
return s.sliceBytes(0, end);
}
}
return s.sliceBytes(0, end);
}
/**
* One line of a comment body with its star prefix removed.
*
* Indentation after the prefix is kept: it carries meaning in markdown
* (nested lists, indented code), so only the decoration comes off.
*/
function stripPrefix(line: String) {
var end = line.length;
if (end > 0 && line.getByteAt(end - 1) == CR) {
end -= 1;
}
var start = 0;
while (start < end && isSpace(line.getByteAt(start))) {
start += 1;
}
if (start < end && line.getByteAt(start) == STAR) {
start += 1;
if (start < end && line.getByteAt(start) == SPACE) {
start += 1;
}
} else {
// No star on this line: it is either the first line, which follows
// the opening delimiter, or a continuation written without
// decoration. Keep it as written.
start = 0;
}
while (end > start && isSpace(line.getByteAt(end - 1))) {
end -= 1;
}
return line.sliceBytes(start, end);
}
/** Splits on newlines, keeping empty lines. */
function splitLines(s: String) {
let lines = new Array<String>(8);
var start = 0;
var i = 0;
while (i < s.length) {
if (s.getByteAt(i) == LF) {
lines.push(s.sliceBytes(start, i));
start = i + 1;
}
i += 1;
}
lines.push(s.sliceBytes(start, s.length));
return lines;
}
/** Strips the star prefix from every line, then drops blank edges. */
function bodyLines(body: String) {
let stripped = new Array<String>(8);
for (let line in splitLines(body)) {
stripped.push(stripPrefix(line));
}
var start = 0;
var end = stripped.length;
while (start < end && stripped[start].length == 0) {
start += 1;
}
while (end > start && stripped[end - 1].length == 0) {
end -= 1;
}
let out = new Array<String>(8);
var i = start;
while (i < end) {
out.push(stripped[i]);
i += 1;
}
return out;
}
/** Joins `lines[from..to)` with newlines, dropping trailing blanks. */
function joinLines(lines: GrowableArray<String>, from: i32, to: i32) {
let sb = new StringBuilder(64);
var i = from;
while (i < to) {
if (i > from) {
sb.append('\n');
}
sb.append(lines[i]);
i += 1;
}
return trimTrailingBlank(sb.toString());
}
/** Whether a line opens or closes a fenced code block. */
function isFence(line: String) {
let t = trim(line);
return t.length >= 3 && t.getByteAt(0) == BACKTICK &&
t.getByteAt(1) == BACKTICK && t.getByteAt(2) == BACKTICK;
}
/**
* The tag name a line opens, or null if it opens none.
*
* A tag line starts with `@` followed by at least one ASCII letter, so
* an address in prose (`user@example.com`) never opens a tag: the line
* would have to begin with it.
*/
function tagNameAt(line: String): String | null {
let t = trim(line);
if (t.length < 2 || t.getByteAt(0) != AT) {
return null;
}
var i = 1;
while (i < t.length) {
let b = t.getByteAt(i);
let isLetter = (b >= 65 && b <= 90) || (b >= 97 && b <= 122);
let isDigitOrDash = (b >= 48 && b <= 57) || b == DASH;
if (isLetter || (i > 1 && isDigitOrDash)) {
i += 1;
} else {
break;
}
}
if (i == 1) {
return null;
}
return t.sliceBytes(1, i);
}
/** Tags whose first word after the tag names a subject. */
function takesSubject(name: String) {
return name == 'param' || name == 'throws' || name == 'typeParam';
}
/** Index of the first space or tab, or the string's length. */
function firstSpace(s: String) {
var i = 0;
while (i < s.length) {
if (isSpace(s.getByteAt(i))) {
return i;
}
i += 1;
}
return i;
}
/**
* The first paragraph of a description, on one line.
*
* A description opening with a fence, a list, a heading or a quote has
* no summary: joining those onto one line produces nonsense, and a
* consumer showing summaries is better off showing nothing.
*/
function summaryOf(description: String) {
if (description.length == 0) {
return '';
}
let lines = splitLines(description);
let first = trim(lines[0]);
if (first.length == 0) {
return '';
}
let b = first.getByteAt(0);
if (b == BACKTICK || b == DASH || b == STAR || b == HASH || b == GT) {
return '';
}
let sb = new StringBuilder(64);
var i = 0;
while (i < lines.length) {
let line = trim(lines[i]);
if (line.length == 0) {
break;
}
if (i > 0) {
sb.append(' ');
}
sb.append(line);
i += 1;
}
return sb.toString();
}
/** Index of the first line that opens a block tag, else `lines.length`. */
function firstTagLine(lines: GrowableArray<String>) {
var inFence = false;
var i = 0;
while (i < lines.length) {
if (isFence(lines[i])) {
inFence = !inFence;
} else if (!inFence && tagNameAt(lines[i]) != null) {
return i;
}
i += 1;
}
return lines.length;
}
/**
* Collects block tags from `start` to the end of the comment.
*
* A tag runs until the next tag line, so its text can span lines and
* contain fenced code — which `@example` needs.
*/
function parseTags(lines: GrowableArray<String>, start: i32) {
// Every line that opens a tag. A tag's text is the lines from its
// opener up to the next one, so fenced code inside a tag is carried
// along rather than ending it.
let openers = new Array<i32>(2);
var inFence = false;
var i = start;
while (i < lines.length) {
if (isFence(lines[i])) {
inFence = !inFence;
} else if (!inFence && tagNameAt(lines[i]) != null) {
openers.push(i);
}
i += 1;
}
let tags = new Array<DocTag>(openers.length);
var t = 0;
while (t < openers.length) {
let from = openers[t];
let to = if (t + 1 < openers.length) { openers[t + 1] } else { lines.length };
let opener = tagNameAt(lines[from]);
if (opener == null) {
throw new Error('a line recorded as opening a tag does not');
}
let head = trim(lines[from]);
let afterTag = trim(head.sliceBytes(opener.length + 1, head.length));
var subject = '';
var firstLine = afterTag;
if (takesSubject(opener)) {
let cut = firstSpace(afterTag);
subject = afterTag.sliceBytes(0, cut);
firstLine = trim(afterTag.sliceBytes(cut, afterTag.length));
}
// `@example` and friends carry their text on the following lines,
// so the tag's own line is often empty and must not open the text
// with a blank line.
let sb = new StringBuilder(64);
var wrote = firstLine.length > 0;
if (wrote) {
sb.append(firstLine);
}
var j = from + 1;
while (j < to) {
let text = lines[j];
if (wrote) {
sb.append('\n');
sb.append(text);
} else if (trim(text).length > 0) {
sb.append(text);
wrote = true;
}
j += 1;
}
tags.push(new DocTag(opener, subject, trimTrailingBlank(sb.toString())));
t += 1;
}
return tags;
}
/**
* Parses a comment body — the text between the delimiters, which is
* what `Comment.value` holds.
*/
export function parseDocCommentBody(body: String): DocComment {
let lines = bodyLines(body);
let tagStart = firstTagLine(lines);
let description = joinLines(lines, 0, tagStart);
let tags = parseTags(lines, tagStart);
let raw = joinLines(lines, 0, lines.length);
return new DocComment(summaryOf(description), description, tags, raw);
}
/**
* The doc comment for a node, given its leading comments, or null when
* none of them is a doc comment.
*
* Several doc comments in a row are joined: the sequence reads as one
* block of prose, so the closest one to the declaration is not
* privileged.
*/
export function parseLeadingDoc(comments: GrowableArray<Comment> | null): DocComment | null {
if (comments == null) {
return null;
}
let all = comments;
let sb = new StringBuilder(64);
var found = false;
for (let comment in all) {
if (isDocComment(comment)) {
if (found) {
sb.append('\n');
}
sb.append(comment.value);
found = true;
}
}
if (!found) {
return null;
}
return parseDocCommentBody(sb.toString());
}
/** Whether `offset` is inside `node`'s source span. */
function spans(node: Node, offset: i32) {
let loc = node.loc;
return loc != null && offset >= loc.start && offset < loc.end;
}
/** The member of a class or interface body around `offset`, or null. */
function memberAt(body: GrowableArray<Node>, offset: i32): Node | null {
for (let member in body) {
if (spans(member, offset)) {
return member;
}
}
return null;
}
/**
* The doc comment on the declaration around `offset`, or null when that
* declaration has none.
*
* The declaration is the class or interface member around the offset
* when there is one, else the top-level statement around it. An
* undocumented member has no doc comment even when its class has one.
*/
export function docCommentAt(ast: Module, offset: i32, comments: CommentMap): DocComment | null {
for (let stmt in ast.body) {
if (!spans(stmt, offset)) {
continue;
}
let member = match (stmt) {
case ClassDeclaration {body}: memberAt(body, offset)
case InterfaceDeclaration {body}: memberAt(body, offset)
case _: null
};
return parseLeadingDoc(comments.getLeading(member ?? stmt));
}
return null;
}
/**
* The doc comment on a declaration, or null when it has none. `ast` is
* the module the declaration is in, and `comments` that module's
* comments.
*
* A name bound inside a statement, like the `x` in `let x = 1;`, has no
* comments of its own. Its doc comment is the statement's, found by
* position.
*/
export function declarationDoc(decl: Node, ast: Module, comments: CommentMap): DocComment | null {
let own = comments.getLeading(decl);
if (own != null) {
return parseLeadingDoc(own);
}
let loc = decl.loc;
if (loc == null) {
return null;
}
return docCommentAt(ast, loc.start, comments);
}
- zena-lang:wit-parser wit-parser.zena-lang.pkg.zena-lang.dev ~0.0.1-preview.1
| Version | Published | Dependencies | Approved by |
|---|---|---|---|
| 0.0.2-preview.1 | 2026-11-19 |
|
|