compiler 0.0.2-preview.1
The Zena self-hosted compiler
compiler#
The Zena compiler, written in Zena. It parses, type-checks, and compiles Zena source code to WebAssembly (Wasm GC) and WebAssembly Text (WAT).
Usage#
Parsing a Single File#
Use compiler/parser to parse Zena source text into an AST (Module):
import { parse } from 'compiler/parser';
let ast = parse('export let answer = 42;', 'example.zena');
Loading and Checking a Program#
Use compiler (createCompiler and checkCompilation) with a CompilerHost to resolve imports across a module graph and type-check a program:
import {
createCompiler, checkCompilation, buildStdlibConfig
} from 'compiler';
import { FsCompilerHost } from 'compiler/fs-host';
let stdlibRoot = '/stdlib';
let host = new FsCompilerHost(stdlibRoot);
let packages = buildStdlibConfig(stdlibRoot, 'host');
let compiler = createCompiler(host, {
stdlibRoot,
packages,
});
let result = await compiler.compile({
entry: '/workspace/main.zena',
target: 'host',
});
let checked = checkCompilation(result);
Compiling a File to WebAssembly#
Use compiler/compile-file to compile an entry file on disk to a .wasm or .wat output file:
import { compileFile } from 'compiler/compile-file';
let ok = await compileFile({
entry: 'src/main.zena',
outPath: 'out/main.wasm',
stdlibRoot: '/stdlib',
});
Libraries#
compiler/compiler/compiler— compiler entry point (Compiler,createCompiler,checkCompilation,ProgramCheckResult)compiler/compile-file— end-to-end file compilation (compileFile,CompileFileOptions)compiler/tokenizer— lexer and token definitions (tokenize,Token,TokenType)compiler/parser— recursive-descent parser (parse,Parser)compiler/ast— AST node definitions (Node,Module,Statement,Expression,TypeAnnotation)compiler/visitor— AST traversal helpers (walk,Visitor)compiler/scope— lexical scope and symbol resolution (ScopeBuilder,ScopeResult,Symbol)compiler/checker— static type checker (checkModule,CheckResult)compiler/semantic-model— semantic queries over a checked module (SemanticModel)compiler/types— compiler type representations (Type,ClassType,FunctionType,typeToString)compiler/diagnostics— compiler diagnostics and source spans (Diagnostic,DiagnosticCode,DiagnosticSeverity,SourceSpan)compiler/package-manifest—zena-package.jsonmanifest and workspace loader (loadWorkspace,loadPackageMap)compiler/module-resolver— package and relative module specifier resolution (ModuleResolver,PackageConfig)compiler/library-loader— module graph loader (LibraryLoader,CompilerHost,SourceFile)compiler/codegen/binary-generator— WebAssembly GC binary emitter (BinaryGenerator)compiler/codegen/wat-generator— WebAssembly Text (WAT) emitter (WatGenerator)
License#
MIT
30.1 KiB
Raw
import { StringReader, StringBuilder, StringTable } from 'zena:core';
import { Comment } from './ast.zena';
import { HashMap } from 'zena:collections';
// ============================================================================
// Token Types
// ============================================================================
export enum TokenType {
// Keywords
Let,
Var,
Class,
Import,
Export,
Return,
If,
Else,
While,
For,
Break,
Continue,
True,
False,
Null,
New,
This,
Extends,
Interface,
Implements,
Final,
Super,
Mixin,
Decorator,
With,
On,
Abstract,
Operator,
Declare,
Function,
From,
Type,
Distinct,
Opaque,
As,
Is,
Extension,
Static,
Throw,
Try,
Catch,
Finally,
Match,
Case,
Symbol,
Enum,
In,
Inline,
Sealed,
Gen,
Yield,
Async,
Await,
Using,
// Identifiers & Literals
Identifier,
Number,
String,
// Template literals
NoSubstitutionTemplate,
TemplateHead,
TemplateMiddle,
TemplateTail,
// Operators
Equals,
EqualsEquals,
EqualsEqualsEquals,
Bang,
BangEquals,
BangEqualsEquals,
Less,
LessEquals,
LessLess,
Greater,
GreaterEquals,
GreaterGreater,
GreaterGreaterGreater,
Arrow,
Plus,
PlusEquals,
Minus,
MinusEquals,
Star,
StarEquals,
StarStar,
StarStarEquals,
Slash,
SlashEquals,
Percent,
PercentEquals,
Pipe,
PipePipe,
PipeGreater,
Ampersand,
AmpersandAmpersand,
Caret,
Question,
QuestionQuestion,
QuestionQuestionEquals,
QuestionDot,
QuestionLBracket,
QuestionLParen,
// Punctuation
LParen,
RParen,
LBrace,
RBrace,
LBracket,
RBracket,
Colon,
Semi,
Comma,
Dot,
DotDot,
DotDotDot,
Hash,
At,
Dollar,
EOF,
Unknown
}
// ============================================================================
// Token
// ============================================================================
export final class Token {
let kind: TokenType;
let value: String;
let rawValue: String | null;
let line: i32;
let column: i32;
let start: i32;
let end: i32;
new(t: TokenType, v: String, l: i32, c: i32, s: i32, e: i32, r: String | null) : kind = t, value = v, rawValue = r, line = l, column = c, start = s, end = e;
}
// ============================================================================
// Keywords
// ============================================================================
let KEYWORDS: HashMap<String, TokenType> = {
'let' => TokenType.Let,
'var' => TokenType.Var,
'class' => TokenType.Class,
'final' => TokenType.Final,
'import' => TokenType.Import,
'export' => TokenType.Export,
'return' => TokenType.Return,
'if' => TokenType.If,
'else' => TokenType.Else,
'while' => TokenType.While,
'for' => TokenType.For,
'break' => TokenType.Break,
'continue' => TokenType.Continue,
'true' => TokenType.True,
'false' => TokenType.False,
'null' => TokenType.Null,
'new' => TokenType.New,
'this' => TokenType.This,
'extends' => TokenType.Extends,
'interface' => TokenType.Interface,
'implements' => TokenType.Implements,
'super' => TokenType.Super,
'mixin' => TokenType.Mixin,
'decorator' => TokenType.Decorator,
'with' => TokenType.With,
'on' => TokenType.On,
'abstract' => TokenType.Abstract,
'operator' => TokenType.Operator,
'declare' => TokenType.Declare,
'function' => TokenType.Function,
'from' => TokenType.From,
'type' => TokenType.Type,
'distinct' => TokenType.Distinct,
'opaque' => TokenType.Opaque,
'as' => TokenType.As,
'is' => TokenType.Is,
'extension' => TokenType.Extension,
'static' => TokenType.Static,
'throw' => TokenType.Throw,
'try' => TokenType.Try,
'catch' => TokenType.Catch,
'finally' => TokenType.Finally,
'match' => TokenType.Match,
'case' => TokenType.Case,
'symbol' => TokenType.Symbol,
'enum' => TokenType.Enum,
'in' => TokenType.In,
'inline' => TokenType.Inline,
'sealed' => TokenType.Sealed,
'gen' => TokenType.Gen,
'yield' => TokenType.Yield,
// Reserved for async functions (concurrency.md); not yet parsed.
'async' => TokenType.Async,
'await' => TokenType.Await,
// `using` is reserved rather than contextual, which is what lets the
// unbound form `using acquire(lock);` be unambiguous — see
// docs/design/ownership.md §`using`.
'using' => TokenType.Using
};
// ============================================================================
// Character helpers
// ============================================================================
function isWhitespace(c: i32) {
// space=32, tab=9, newline=10, carriage return=13
return c == 32 || c == 9 || c == 10 || c == 13;
}
function isDigit(c: i32) {
// '0'=48, '9'=57
return c >= 48 && c <= 57;
}
function isHexDigit(c: i32) {
// '0'-'9'=48-57, 'a'-'f'=97-102, 'A'-'F'=65-70
return (c >= 48 && c <= 57) || (c >= 97 && c <= 102) || (c >= 65 && c <= 70);
}
function isAlpha(c: i32) {
// 'a'-'z'=97-122, 'A'-'Z'=65-90
return (c >= 97 && c <= 122) || (c >= 65 && c <= 90);
}
function isIdentifierStart(c: i32) {
// letters, underscore=95, $=36
return isAlpha(c) || c == 95 || c == 36;
}
export function isIdentifierPart(c: i32): boolean {
return isAlpha(c) || isDigit(c) || c == 95 || c == 36;
}
// ============================================================================
// Keyword recognition
// ============================================================================
/** One row of the keyword table: the canonical text and what it scans as. */
export final class Keyword(text: String, type: TokenType)
/**
* Keywords bucketed by byte length. A scanned word is compared only
* against keywords that could possibly match it, and a word longer than
* the longest keyword is rejected by the length alone.
*/
var keywordBuckets: GrowableArray<GrowableArray<Keyword>> | null = null;
function buildKeywordBuckets() {
let texts = new Array<String>();
var longest = 0;
for (let text in KEYWORDS.keys()) {
texts.push(text);
if (text.length > longest) {
longest = text.length;
}
}
let buckets = new Array<GrowableArray<Keyword>>();
var i = 0;
while (i <= longest) {
buckets.push(new Array<Keyword>());
i += 1;
}
// Bucket order follows the map's iteration order, which is not fixed.
// It does not need to be: a word matches at most one keyword, so the
// order candidates are tried in cannot change the answer.
for (let text in texts) {
if (let (true, type) = KEYWORDS.get(text)) {
buckets[text.length].push(new Keyword(text, type));
}
}
return buckets;
}
/**
* The keyword a byte range spells, or null if it spells an identifier.
*
* This reads the source bytes in place: recognizing `let` must not cost
* a `String` that the token type makes redundant. In the compiler's own
* sources one scanned word in three is a keyword, and materializing
* each one allocated a wrapper nothing ever read.
*
* Bucketing by length first is what keeps the comparison cheap. Hashing
* every byte to look the word up — which is what a map keyed by String
* does — costs more than the compare that has to follow it anyway.
*/
export function matchKeyword(source: String, start: i32, length: i32): Keyword | null {
var buckets = keywordBuckets;
if (buckets == null) {
buckets = buildKeywordBuckets();
keywordBuckets = buckets;
}
if (length >= buckets.length) {
return null;
}
let first = source.getByteAt(start);
for (let candidate in buckets[length]) {
let {text} = candidate;
if (text.getByteAt(0) != first) {
continue;
}
var i = 1;
var same = true;
while (i < length) {
if (source.getByteAt(start + i) != text.getByteAt(i)) {
same = false;
break;
}
i += 1;
}
if (same) {
return candidate;
}
}
return null;
}
// ============================================================================
// Tokenizer
// ============================================================================
export final class Tokenizer {
let #reader: StringReader;
let #tokens: GrowableArray<Token>;
let #comments: GrowableArray<Comment>;
let #templateStack: GrowableArray<i32>;
var #line: i32;
var #column: i32;
var #lineStart: i32;
new(source: String, table: StringTable | null = null)
: #reader = new StringReader(source, table),
#tokens = new Array<Token>(64),
#comments = new Array<Comment>(8),
#templateStack = new Array<i32>(8),
#line = 1,
#column = 1,
#lineStart = 0
{ }
/** Returns all comments collected during tokenization. */
getComments(): GrowableArray<Comment> {
return this.#comments;
}
// ============================================================================
// Cursor helpers
// ============================================================================
#peek(): i32 {
return this.#reader.peekByte();
}
#peekNext(): i32 {
return this.#reader.peekByteAt(1);
}
#advance(): i32 {
let c = this.#reader.advanceByte();
if (c == 10) {
// newline
this.#line += 1;
this.#lineStart = this.#reader.position;
this.#column = 1;
} else {
this.#column += 1;
}
return c;
}
#currentColumn(): i32 {
return this.#reader.position - this.#lineStart + 1;
}
#addToken(type: TokenType, value: String, startPos: i32, startCol: i32): void {
let token = new Token(type, value, this.#line, startCol, startPos, this.#reader.position, null);
this.#tokens.push(token);
}
#addTemplateToken(type: TokenType, cooked: String, raw: String, startPos: i32, startCol: i32): void {
let token = new Token(type, cooked, this.#line, startCol, startPos, this.#reader.position, raw);
this.#tokens.push(token);
}
// ============================================================================
// Main tokenize entry point
// ============================================================================
tokenize(): GrowableArray<Token> {
while (!this.#reader.isAtEnd) {
this.#scanToken();
}
// Add EOF token
let pos = this.#reader.position;
this.#addToken(TokenType.EOF, '', pos, this.#currentColumn());
return this.#tokens;
}
#scanToken(): void {
let startPos = this.#reader.position;
let startCol = this.#currentColumn();
let c = this.#peek();
// Whitespace
if (isWhitespace(c)) {
this.#advance();
return;
}
// Numbers
if (isDigit(c)) {
this.#scanNumber(startPos, startCol);
return;
}
// Identifiers & Keywords
if (isIdentifierStart(c)) {
this.#scanIdentifier(startPos, startCol);
return;
}
// Strings
if (c == 39 || c == 34) {
// single quote=39, double quote=34
this.#scanString(startPos, startCol);
return;
}
// Template literals
if (c == 96) {
// backtick=96
this.#advance();
this.#scanTemplatePart(false, startPos, startCol);
return;
}
// Operators and punctuation
this.#scanOperator(startPos, startCol);
}
// ============================================================================
// Number scanning
// ============================================================================
#scanNumber(startPos: i32, startCol: i32): void {
let start = this.#reader.mark();
let c = this.#peek();
let next = this.#peekNext();
// Radix prefix: 0x/0X, 0b/0B, 0o/0O
if (c == 48 &&
(next == 120 || next == 88 ||
next == 98 || next == 66 ||
next == 111 || next == 79)) {
this.#advance(); // '0'
this.#advance(); // 'x'/'X'/'b'/'B'/'o'/'O'
while (!this.#reader.isAtEnd && isIdentifierPart(this.#peek())) {
this.#advance();
}
} else {
// Decimal integer digits and '_' separators
while (!this.#reader.isAtEnd && (isDigit(this.#peek()) || this.#peek() == 95)) {
this.#advance();
}
// Fractional part (including malformed '3._14' so the parser diagnoses '_' next to '.')
if (this.#peek() == 46) {
let afterDot = this.#peekNext();
let dotFollowedByDigitOrSepDigit = isDigit(afterDot) ||
(afterDot == 95 &&
this.#reader.position + 2 < this.#reader.source.length &&
isDigit(this.#reader.source.getByteAt(this.#reader.position + 2)));
if (dotFollowedByDigitOrSepDigit) {
this.#advance(); // '.'
while (!this.#reader.isAtEnd && (isDigit(this.#peek()) || this.#peek() == 95)) {
this.#advance();
}
}
}
// Any trailing suffix or malformed identifier tail
while (!this.#reader.isAtEnd && isIdentifierPart(this.#peek())) {
this.#advance();
}
}
let raw = this.#reader.source.sliceBytes(start, this.#reader.position);
this.#addToken(TokenType.Number, raw, startPos, startCol);
}
// ============================================================================
// Identifier scanning
// ============================================================================
#scanIdentifier(startPos: i32, startCol: i32): void {
let start = this.#reader.mark();
while (!this.#reader.isAtEnd && isIdentifierPart(this.#peek())) {
this.#advance();
}
// Classify before materializing: a keyword's text is already in the
// table, so a hit reuses that instance and allocates nothing. Only
// an identifier needs a slice of its own.
let keyword = matchKeyword(
this.#reader.source, start, this.#reader.position - start);
if (keyword != null) {
this.#addToken(keyword.type, keyword.text, startPos, startCol);
} else {
// Interned when the reader has a table: a repeat identifier costs a
// probe against the source bytes and no allocation; the first
// occurrence of a spelling is copied once into the table.
this.#addToken(
TokenType.Identifier, this.#reader.internFrom(start), startPos, startCol);
}
}
// ============================================================================
// String scanning
// ============================================================================
#scanString(startPos: i32, startCol: i32): void {
let quote = this.#advance();
let builder = new StringBuilder();
// Plain characters are copied a run at a time, as a slice of the
// source: offsets are bytes on the byte targets and UTF-16 code units
// on js, and a slice keeps every character on both. Copying a byte at
// a time kept only the low byte of each code unit on js (#431).
let source = this.#reader.source;
var runStart = this.#reader.position;
while (!this.#reader.isAtEnd && this.#peek() != quote) {
if (this.#peek() == 92) {
// backslash=92
builder.appendSlice(source, runStart, this.#reader.position);
this.#advance();
if (this.#reader.isAtEnd) {
break;
}
let escaped = this.#advance();
// 'n'=110, 'r'=114, 't'=116, '\\'=92, '\''=39, '"'=34, '0'=48, '\n'=10, '\r'=13
if (escaped == 110) {
builder.appendByte(10); // newline
} else if (escaped == 114) {
builder.appendByte(13); // carriage return
} else if (escaped == 116) {
builder.appendByte(9); // tab
} else if (escaped == 92) {
builder.appendByte(92); // backslash
} else if (escaped == 39) {
builder.appendByte(39); // single quote
} else if (escaped == 34) {
builder.appendByte(34); // double quote
} else if (escaped == 48) {
builder.appendByte(0); // null
} else if (escaped == 10) {
// Line continuation (LF) - omit from string
} else if (escaped == 13) {
// Line continuation (CR or CRLF) - omit from string
if (this.#peek() == 10) {
this.#advance();
}
} else {
// Unknown escape - keep as-is
builder.appendByte(92);
builder.appendByte(escaped);
}
runStart = this.#reader.position;
} else {
this.#advance();
}
}
builder.appendSlice(source, runStart, this.#reader.position);
if (!this.#reader.isAtEnd) {
this.#advance(); // closing quote
}
this.#addToken(TokenType.String, builder.toString(), startPos, startCol);
}
// ============================================================================
// Template literal scanning
// ============================================================================
#scanTemplatePart(isMiddleOrTail: boolean, startPos: i32, startCol: i32): void {
let cooked = new StringBuilder();
// As in `#scanString`, plain characters are copied a run at a time.
// The raw string is the part's source text exactly, so it is one slice.
let source = this.#reader.source;
let partStart = this.#reader.position;
var runStart = partStart;
while (!this.#reader.isAtEnd) {
let c = this.#peek();
// End of template
if (c == 96) {
// backtick=96
cooked.appendSlice(source, runStart, this.#reader.position);
let raw = source.sliceBytes(partStart, this.#reader.position);
this.#advance();
let tokenType = if (isMiddleOrTail) {
TokenType.TemplateTail
} else {
TokenType.NoSubstitutionTemplate
};
this.#addTemplateToken(tokenType, cooked.toString(), raw, startPos, startCol);
return;
}
// Template expression start
if (c == 36 && this.#peekNext() == 123) {
// '$'=36, '{'=123
cooked.appendSlice(source, runStart, this.#reader.position);
let raw = source.sliceBytes(partStart, this.#reader.position);
this.#advance(); // $
this.#advance(); // {
this.#templateStack.push(1); // start tracking braces
let tokenType = if (isMiddleOrTail) {
TokenType.TemplateMiddle
} else {
TokenType.TemplateHead
};
this.#addTemplateToken(tokenType, cooked.toString(), raw, startPos, startCol);
return;
}
// Escape sequence
if (c == 92) {
// backslash=92
cooked.appendSlice(source, runStart, this.#reader.position);
this.#advance();
this.#processTemplateEscape(cooked);
runStart = this.#reader.position;
} else {
this.#advance();
}
}
// Reached end without closing
cooked.appendSlice(source, runStart, this.#reader.position);
let tokenType = if (isMiddleOrTail) {
TokenType.TemplateTail
} else {
TokenType.NoSubstitutionTemplate
};
this.#addTemplateToken(tokenType, cooked.toString(),
source.sliceBytes(partStart, this.#reader.position), startPos, startCol);
}
#processTemplateEscape(cooked: StringBuilder): void {
if (this.#reader.isAtEnd) {
return;
}
let escaped = this.#advance();
// 'n'=110, 'r'=114, 't'=116, '\\'=92, '`'=96, '$'=36, '0'=48, '\n'=10, '\r'=13
if (escaped == 110) {
cooked.appendByte(10); // newline
} else if (escaped == 114) {
cooked.appendByte(13); // carriage return
} else if (escaped == 116) {
cooked.appendByte(9); // tab
} else if (escaped == 92) {
cooked.appendByte(92); // backslash
} else if (escaped == 96) {
cooked.appendByte(96); // backtick
} else if (escaped == 36) {
cooked.appendByte(36); // dollar
} else if (escaped == 48) {
cooked.appendByte(0); // null
} else if (escaped == 10) {
// Line continuation (LF) - omitted from cooked
} else if (escaped == 13) {
// Line continuation (CR or CRLF)
if (this.#peek() == 10) {
this.#advance();
}
// omitted from cooked
} else {
// Unknown escape - keep as-is
cooked.appendByte(92);
cooked.appendByte(escaped);
}
}
// ============================================================================
// Operator scanning
// ============================================================================
#scanOperator(startPos: i32, startCol: i32): void {
let c = this.#advance();
// '='=61, '>'=62
if (c == 61) {
if (this.#peek() == 62) {
this.#advance();
this.#addToken(TokenType.Arrow, '=>', startPos, startCol);
} else if (this.#peek() == 61) {
this.#advance();
if (this.#peek() == 61) {
this.#advance();
this.#addToken(TokenType.EqualsEqualsEquals, '===', startPos, startCol);
} else {
this.#addToken(TokenType.EqualsEquals, '==', startPos, startCol);
}
} else {
this.#addToken(TokenType.Equals, '=', startPos, startCol);
}
return;
}
// '!'=33
if (c == 33) {
if (this.#peek() == 61) {
this.#advance();
if (this.#peek() == 61) {
this.#advance();
this.#addToken(TokenType.BangEqualsEquals, '!==', startPos, startCol);
} else {
this.#addToken(TokenType.BangEquals, '!=', startPos, startCol);
}
} else {
this.#addToken(TokenType.Bang, '!', startPos, startCol);
}
return;
}
// '<'=60
if (c == 60) {
if (this.#peek() == 61) {
this.#advance();
this.#addToken(TokenType.LessEquals, '<=', startPos, startCol);
} else if (this.#peek() == 60) {
this.#advance();
this.#addToken(TokenType.LessLess, '<<', startPos, startCol);
} else {
this.#addToken(TokenType.Less, '<', startPos, startCol);
}
return;
}
// '>'=62
if (c == 62) {
if (this.#peek() == 61) {
this.#advance();
this.#addToken(TokenType.GreaterEquals, '>=', startPos, startCol);
} else if (this.#peek() == 62) {
this.#advance();
if (this.#peek() == 62) {
this.#advance();
this.#addToken(TokenType.GreaterGreaterGreater, '>>>', startPos, startCol);
} else {
this.#addToken(TokenType.GreaterGreater, '>>', startPos, startCol);
}
} else {
this.#addToken(TokenType.Greater, '>', startPos, startCol);
}
return;
}
// '+'=43
if (c == 43) {
if (this.#peek() == 61) {
this.#advance();
this.#addToken(TokenType.PlusEquals, '+=', startPos, startCol);
} else {
this.#addToken(TokenType.Plus, '+', startPos, startCol);
}
return;
}
// '-'=45
if (c == 45) {
if (this.#peek() == 61) {
this.#advance();
this.#addToken(TokenType.MinusEquals, '-=', startPos, startCol);
} else {
this.#addToken(TokenType.Minus, '-', startPos, startCol);
}
return;
}
// '*'=42
if (c == 42) {
if (this.#peek() == 61) {
this.#advance();
this.#addToken(TokenType.StarEquals, '*=', startPos, startCol);
} else if (this.#peek() == 42) {
this.#advance();
if (this.#peek() == 61) {
this.#advance();
this.#addToken(TokenType.StarStarEquals, '**=', startPos, startCol);
} else {
this.#addToken(TokenType.StarStar, '**', startPos, startCol);
}
} else {
this.#addToken(TokenType.Star, '*', startPos, startCol);
}
return;
}
// '/'=47
if (c == 47) {
if (this.#peek() == 47) {
// Single-line comment — collect text after //
let commentStart = startPos;
let commentLine = this.#line;
let commentCol = startCol;
while (!this.#reader.isAtEnd && this.#peek() != 10) {
this.#advance();
}
let commentEnd = this.#reader.position;
// Extract text after '//' (startPos is at first '/', startPos+2 skips '//')
let value = this.#reader.source.sliceBytes(startPos + 2, commentEnd);
this.#comments.push(new Comment(value, false, commentStart, commentEnd, commentLine, commentCol));
return;
} else if (this.#peek() == 42) {
// Multi-line comment — collect text between /* and */
let commentStart = startPos;
let commentLine = this.#line;
let commentCol = startCol;
this.#advance(); // consume '*'
while (!this.#reader.isAtEnd) {
if (this.#peek() == 42 && this.#peekNext() == 47) {
this.#advance(); // consume '*'
this.#advance(); // consume '/'
break;
}
this.#advance();
}
let commentEnd = this.#reader.position;
// Extract text between '/*' and '*/' (startPos+2 to commentEnd-2)
var innerEnd = commentEnd - 2;
if (innerEnd < startPos + 2) { innerEnd = startPos + 2; }
let value = this.#reader.source.sliceBytes(startPos + 2, innerEnd);
this.#comments.push(new Comment(value, true, commentStart, commentEnd, commentLine, commentCol));
return;
} else if (this.#peek() == 61) {
this.#advance();
this.#addToken(TokenType.SlashEquals, '/=', startPos, startCol);
} else {
this.#addToken(TokenType.Slash, '/', startPos, startCol);
}
return;
}
// '%'=37
if (c == 37) {
if (this.#peek() == 61) {
this.#advance();
this.#addToken(TokenType.PercentEquals, '%=', startPos, startCol);
} else {
this.#addToken(TokenType.Percent, '%', startPos, startCol);
}
return;
}
// '|'=124
if (c == 124) {
if (this.#peek() == 124) {
this.#advance();
this.#addToken(TokenType.PipePipe, '||', startPos, startCol);
} else if (this.#peek() == 62) {
this.#advance();
this.#addToken(TokenType.PipeGreater, '|>', startPos, startCol);
} else {
this.#addToken(TokenType.Pipe, '|', startPos, startCol);
}
return;
}
// '&'=38
if (c == 38) {
if (this.#peek() == 38) {
this.#advance();
this.#addToken(TokenType.AmpersandAmpersand, '&&', startPos, startCol);
} else {
this.#addToken(TokenType.Ampersand, '&', startPos, startCol);
}
return;
}
// '^'=94
if (c == 94) {
this.#addToken(TokenType.Caret, '^', startPos, startCol);
return;
}
// '?'=63
if (c == 63) {
if (this.#peek() == 63) {
this.#advance();
if (this.#peek() == 61) {
this.#advance();
this.#addToken(TokenType.QuestionQuestionEquals, '??=', startPos, startCol);
} else {
this.#addToken(TokenType.QuestionQuestion, '??', startPos, startCol);
}
} else if (this.#peek() == 46) {
// '.'=46
this.#advance();
this.#addToken(TokenType.QuestionDot, '?.', startPos, startCol);
} else if (this.#peek() == 91) {
// '['=91
this.#advance();
this.#addToken(TokenType.QuestionLBracket, '?[', startPos, startCol);
} else if (this.#peek() == 40) {
// '('=40
this.#advance();
this.#addToken(TokenType.QuestionLParen, '?(', startPos, startCol);
} else {
this.#addToken(TokenType.Question, '?', startPos, startCol);
}
return;
}
// '('=40
if (c == 40) {
this.#addToken(TokenType.LParen, '(', startPos, startCol);
return;
}
// ')'=41
if (c == 41) {
this.#addToken(TokenType.RParen, ')', startPos, startCol);
return;
}
// '{'=123
if (c == 123) {
// Track brace depth for template expressions
if (this.#templateStack.length > 0) {
let lastIdx = this.#templateStack.length - 1;
let depth = this.#templateStack[lastIdx];
this.#templateStack[lastIdx] = depth + 1;
}
this.#addToken(TokenType.LBrace, '{', startPos, startCol);
return;
}
// '}'=125
if (c == 125) {
// Check if this closes a template expression
if (this.#templateStack.length > 0) {
let lastIdx = this.#templateStack.length - 1;
let depth = this.#templateStack[lastIdx];
if (depth == 1) {
this.#templateStack.pop();
// Continue scanning template
this.#scanTemplatePart(true, startPos, startCol);
return;
} else {
this.#templateStack[lastIdx] = depth - 1;
}
}
this.#addToken(TokenType.RBrace, '}', startPos, startCol);
return;
}
// '['=91
if (c == 91) {
this.#addToken(TokenType.LBracket, '[', startPos, startCol);
return;
}
// ']'=93
if (c == 93) {
this.#addToken(TokenType.RBracket, ']', startPos, startCol);
return;
}
// ':'=58
if (c == 58) {
this.#addToken(TokenType.Colon, ':', startPos, startCol);
return;
}
// ';'=59
if (c == 59) {
this.#addToken(TokenType.Semi, ';', startPos, startCol);
return;
}
// ','=44
if (c == 44) {
this.#addToken(TokenType.Comma, ',', startPos, startCol);
return;
}
// '.'=46
if (c == 46) {
if (this.#peek() == 46 && this.#peekNext() == 46) {
this.#advance();
this.#advance();
this.#addToken(TokenType.DotDotDot, '...', startPos, startCol);
} else if (this.#peek() == 46) {
this.#advance();
this.#addToken(TokenType.DotDot, '..', startPos, startCol);
} else {
this.#addToken(TokenType.Dot, '.', startPos, startCol);
}
return;
}
// '#'=35
if (c == 35) {
this.#addToken(TokenType.Hash, '#', startPos, startCol);
return;
}
// '@'=64
if (c == 64) {
this.#addToken(TokenType.At, '@', startPos, startCol);
return;
}
// Unknown character
let unknown = new StringBuilder();
unknown.appendByte(c);
this.#addToken(TokenType.Unknown, unknown.toString(), startPos, startCol);
}
}
// ============================================================================
// Convenience function
// ============================================================================
export function tokenize(source: String, table: StringTable | null = null): GrowableArray<Token> {
let tokenizer = new Tokenizer(source, table);
return tokenizer.tokenize();
}
/**
* Primes `table` with Zena's keywords, so a keyword lexed out of source
* interns to the very String instance `KEYWORDS` is keyed by. The keyword
* lookup in `#scanIdentifier` then hits `==`'s reference-equal fast path
* against a key whose hash was cached the first time it was probed.
*
* Call this once per table, not once per tokenizer: re-priming is harmless
* but it is a lookup per keyword per file.
*/
export function primeKeywords(table: StringTable): void {
for (let keyword in KEYWORDS.keys()) {
table.prime(keyword);
}
}
- zena-lang:wit-parser wit-parser.zena-lang.pkg.zena-lang.dev ~0.0.1-preview.1
| Version | Published | Dependencies | Approved by |
|---|---|---|---|
| 0.0.2-preview.1 | 2026-11-19 |
|
|