/** * WIT Lexer * * Tokenizes WIT (WebAssembly Interface Types) source text. * Based on the canonical implementation in wasm-tools: * https://github.com/bytecodealliance/wasm-tools/blob/main/crates/wit-parser/src/ast/lex.rs */ import {StringBuilder, StringReader} from 'zena:core'; import {TokenType, Span, Token, keywordToToken} from './token.zena'; // ============================================================================ // Character Classification // ============================================================================ /** Check if a byte is ASCII whitespace (space, tab, newline, carriage return). */ function isWhitespace(b: i32) { return b == 32 || b == 9 || b == 10 || b == 13; } /** Check if a byte is an ASCII digit (0-9). */ function isDigit(b: i32) { return b >= 48 && b <= 57; // '0' to '9' } /** Check if a byte is an ASCII hex digit (0-9, a-f, A-F). */ function isHexDigit(b: i32) { return isDigit(b) || (b >= 65 && b <= 70) || (b >= 97 && b <= 102); } /** Check if a byte is a valid start of an identifier (a-z, A-Z). */ function isIdStart(b: i32) { return (b >= 65 && b <= 90) || (b >= 97 && b <= 122); // A-Z or a-z } /** Check if a byte is valid within an identifier (a-z, A-Z, 0-9, -). */ function isIdContinue(b: i32) { return isIdStart(b) || isDigit(b) || b == 45; // also '-' (kebab-case) } // ============================================================================ // Lexer Error // ============================================================================ /** * Error produced during lexing. */ export class LexError extends Error { span: Span; new(message: String, span: Span) : span = span, super(message); } // ============================================================================ // Lexer // ============================================================================ /** * A lexer (tokenizer) for WIT source text. * * Usage: * ```zena * let lexer = new Lexer(source); * while (true) { * let token = lexer.next(); * if (token.type == TokenType.Eof) { break; } * // process token * } * ``` */ export class Lexer { #reader: StringReader; /** Comment content accumulated before items (used for docs). */ var #precedingComments: StringBuilder | null = null; /** * Create a new lexer for the given source text. */ new(source: String) : #reader = new StringReader(source); /** * Get and clear any comments accumulated before the current position. * Returns null if no comments were accumulated. */ takePrecedingComments(): String | null { if (this.#precedingComments == null) { return null; } let sb = this.#precedingComments as StringBuilder; let result = sb.toString(); this.#precedingComments = null; return result; } // -------------------------------------------------------------------------- // Core Lexer Methods // -------------------------------------------------------------------------- /** * Get the next token from the source. */ next(): Token { // Skip whitespace and comments this.#skipTrivia(); let start = this.#reader.position; // Check for end of input if (this.#isEof()) { return new Token(TokenType.Eof, new Span(start, start)); } let b = this.#peek(); // Single-character tokens if (b == 61) { return this.#single(TokenType.Equals); } // = if (b == 44) { return this.#single(TokenType.Comma); } // , if (b == 58) { return this.#single(TokenType.Colon); } // : if (b == 46) { return this.#single(TokenType.Period); } // . if (b == 59) { return this.#single(TokenType.Semicolon); } // ; if (b == 40) { return this.#single(TokenType.LeftParen); } // ( if (b == 41) { return this.#single(TokenType.RightParen); } // ) if (b == 123) { return this.#single(TokenType.LeftBrace); } // { if (b == 125) { return this.#single(TokenType.RightBrace); } // } if (b == 60) { return this.#single(TokenType.LessThan); } // < if (b == 62) { return this.#single(TokenType.GreaterThan); } // > if (b == 42) { return this.#single(TokenType.Star); } // * if (b == 64) { return this.#single(TokenType.At); } // @ if (b == 43) { return this.#single(TokenType.Plus); } // + if (b == 95) { return this.#single(TokenType.Underscore); } // _ // -> (right arrow) if (b == 45) { this.#advance(); if (!this.#isEof() && this.#peek() == 62) { // > this.#advance(); return new Token(TokenType.RArrow, new Span(start, this.#reader.position)); } return new Token(TokenType.Minus, new Span(start, this.#reader.position)); } // / or /// doc comment if (b == 47) { // Check for /// (doc comment) if (this.#peekAt(1) == 47 && this.#peekAt(2) == 47) { // Skip the /// this.#advance(); this.#advance(); this.#advance(); // Skip optional space after /// if (!this.#isEof() && this.#peek() == 32) { this.#advance(); } // Skip until newline or EOF while (!this.#isEof()) { let c = this.#peek(); if (c == 10) { // \n this.#advance(); break; } this.#advance(); } return new Token(TokenType.DocComment, new Span(start, this.#reader.position)); } return this.#single(TokenType.Slash); } // %identifier (explicit identifier) if (b == 37) { // % this.#advance(); return this.#lexExplicitId(start); } // Integer literals if (isDigit(b)) { return this.#lexInteger(start); } // Identifiers and keywords if (isIdStart(b)) { return this.#lexIdOrKeyword(start); } // Unknown character this.#advance(); throw new LexError("unexpected character", new Span(start, this.#reader.position)); } /** * Peek at the current token without consuming it. * Note: This re-lexes; for efficiency, callers should cache the result. */ peek(): Token { let savedPos = this.#reader.position; let token = this.next(); this.#reader.reset(savedPos); return token; } /** * Expect the next token to be of a specific type. * Throws LexError if it doesn't match. */ expect(expected: TokenType): Token { let token = this.next(); if (token.type != expected) { throw new LexError( "unexpected token", token.span ); } return token; } /** * Get the source text for a span. */ getSpanText(span: Span): String { return this.#reader.sliceRange(span.start, span.end); } // -------------------------------------------------------------------------- // Private Helpers // -------------------------------------------------------------------------- /** Check if we're at end of file. */ #isEof(): boolean { return this.#reader.isAtEnd; } /** Peek at current byte without advancing. */ #peek(): i32 { return this.#reader.peekByte(); } /** Peek at byte at given offset from current position. */ #peekAt(offset: i32): i32 { return this.#reader.peekByteAt(offset); } /** Advance position by one byte. */ #advance(): void { this.#reader.advanceByte(); } /** Create a single-character token. */ #single(type: TokenType): Token { let start = this.#reader.position; this.#advance(); return new Token(type, new Span(start, this.#reader.position)); } /** Skip whitespace and comments. */ #skipTrivia(): void { while (!this.#isEof()) { let b = this.#peek(); // Whitespace if (isWhitespace(b)) { this.#advance(); continue; } // Single-line comment: // ... (but NOT doc comments: ///) if (b == 47 && this.#peekAt(1) == 47 && this.#peekAt(2) != 47) { this.#captureLineComment(); continue; } // Block comment: /* ... */ if (b == 47 && this.#peekAt(1) == 42) { this.#captureBlockComment(); continue; } // Not trivia break; } } /** Capture a single-line comment content (// ...). */ #captureLineComment(): void { // Skip the // this.#advance(); this.#advance(); let contentStart = this.#reader.position; // Capture everything until newline while (!this.#isEof()) { let b = this.#peek(); if (b == 10) { // \n break; } this.#advance(); } let contentEnd = this.#reader.position; // Skip the newline if present if (!this.#isEof() && this.#peek() == 10) { this.#advance(); } // Accumulate the comment content if (this.#precedingComments == null) { this.#precedingComments = new StringBuilder(); } let sb = this.#precedingComments as StringBuilder; if (sb.length > 0) { sb.append("\n"); } // Get comment content let rawContent = this.#reader.sliceRange(contentStart, contentEnd); sb.append(rawContent); } /** Capture a block comment content (supports nesting). */ #captureBlockComment(): void { // Skip just the opening '/' - we keep the '*' as part of the content this.#advance(); let contentStart = this.#reader.position; // Skip the '*' this.#advance(); // Track nesting depth (we're already inside one comment) var depth = 1; // Skip until all nested comments are closed or EOF while (!this.#isEof() && depth > 0) { let b = this.#peek(); let next = this.#peekAt(1); // Check for nested opening /* if (b == 47 && next == 42) { this.#advance(); this.#advance(); depth += 1; continue; } // Check for closing */ if (b == 42 && next == 47) { this.#advance(); this.#advance(); depth -= 1; continue; } this.#advance(); } // contentEnd is the current position (includes the final */) let contentEnd = this.#reader.position; // Accumulate the comment content if (this.#precedingComments == null) { this.#precedingComments = new StringBuilder(); } let sb = this.#precedingComments as StringBuilder; if (sb.length > 0) { sb.append("\n"); } // Get block comment content (including leading * and trailing */) let content = this.#reader.sliceRange(contentStart, contentEnd); sb.append(content); } /** Lex an identifier or keyword. */ #lexIdOrKeyword(start: i32): Token { // Consume identifier characters while (!this.#isEof() && isIdContinue(this.#peek())) { this.#advance(); } let span = new Span(start, this.#reader.position); let text = this.getSpanText(span); // Check if it's a keyword let tokenType = keywordToToken(text); return new Token(tokenType, span); } /** Lex an explicit identifier (%foo). */ #lexExplicitId(start: i32): Token { // We've already consumed the % // Now consume the identifier if (this.#isEof() || !isIdStart(this.#peek())) { throw new LexError("expected identifier after '%'", new Span(start, this.#reader.position)); } while (!this.#isEof() && isIdContinue(this.#peek())) { this.#advance(); } return new Token(TokenType.ExplicitId, new Span(start, this.#reader.position)); } /** Lex an integer literal. */ #lexInteger(start: i32): Token { // Check for hex literal (0x...) if (this.#peek() == 48 && this.#peekAt(1) == 120) { // 0x this.#advance(); // 0 this.#advance(); // x if (this.#isEof() || !isHexDigit(this.#peek())) { throw new LexError("expected hex digit after '0x'", new Span(start, this.#reader.position)); } while (!this.#isEof() && isHexDigit(this.#peek())) { this.#advance(); } } else { // Decimal literal while (!this.#isEof() && isDigit(this.#peek())) { this.#advance(); } } return new Token(TokenType.Integer, new Span(start, this.#reader.position)); } }