import { StringReader, StringBuilder, StringTable } from 'zena:core'; import { Comment } from './ast.zena'; import { HashMap } from 'zena:collections'; // ============================================================================ // Token Types // ============================================================================ export enum TokenType { // Keywords Let, Var, Class, Import, Export, Return, If, Else, While, For, Break, Continue, True, False, Null, New, This, Extends, Interface, Implements, Final, Super, Mixin, Decorator, With, On, Abstract, Operator, Declare, Function, From, Type, Distinct, Opaque, As, Is, Extension, Static, Throw, Try, Catch, Finally, Match, Case, Symbol, Enum, In, Inline, Sealed, Gen, Yield, Async, Await, Using, // Identifiers & Literals Identifier, Number, String, // Template literals NoSubstitutionTemplate, TemplateHead, TemplateMiddle, TemplateTail, // Operators Equals, EqualsEquals, EqualsEqualsEquals, Bang, BangEquals, BangEqualsEquals, Less, LessEquals, LessLess, Greater, GreaterEquals, GreaterGreater, GreaterGreaterGreater, Arrow, Plus, PlusEquals, Minus, MinusEquals, Star, StarEquals, StarStar, StarStarEquals, Slash, SlashEquals, Percent, PercentEquals, Pipe, PipePipe, PipeGreater, Ampersand, AmpersandAmpersand, Caret, Question, QuestionQuestion, QuestionQuestionEquals, QuestionDot, QuestionLBracket, QuestionLParen, // Punctuation LParen, RParen, LBrace, RBrace, LBracket, RBracket, Colon, Semi, Comma, Dot, DotDot, DotDotDot, Hash, At, Dollar, EOF, Unknown } // ============================================================================ // Token // ============================================================================ export final class Token { let kind: TokenType; let value: String; let rawValue: String | null; let line: i32; let column: i32; let start: i32; let end: i32; new(t: TokenType, v: String, l: i32, c: i32, s: i32, e: i32, r: String | null) : kind = t, value = v, rawValue = r, line = l, column = c, start = s, end = e; } // ============================================================================ // Keywords // ============================================================================ let KEYWORDS: HashMap = { 'let' => TokenType.Let, 'var' => TokenType.Var, 'class' => TokenType.Class, 'final' => TokenType.Final, 'import' => TokenType.Import, 'export' => TokenType.Export, 'return' => TokenType.Return, 'if' => TokenType.If, 'else' => TokenType.Else, 'while' => TokenType.While, 'for' => TokenType.For, 'break' => TokenType.Break, 'continue' => TokenType.Continue, 'true' => TokenType.True, 'false' => TokenType.False, 'null' => TokenType.Null, 'new' => TokenType.New, 'this' => TokenType.This, 'extends' => TokenType.Extends, 'interface' => TokenType.Interface, 'implements' => TokenType.Implements, 'super' => TokenType.Super, 'mixin' => TokenType.Mixin, 'decorator' => TokenType.Decorator, 'with' => TokenType.With, 'on' => TokenType.On, 'abstract' => TokenType.Abstract, 'operator' => TokenType.Operator, 'declare' => TokenType.Declare, 'function' => TokenType.Function, 'from' => TokenType.From, 'type' => TokenType.Type, 'distinct' => TokenType.Distinct, 'opaque' => TokenType.Opaque, 'as' => TokenType.As, 'is' => TokenType.Is, 'extension' => TokenType.Extension, 'static' => TokenType.Static, 'throw' => TokenType.Throw, 'try' => TokenType.Try, 'catch' => TokenType.Catch, 'finally' => TokenType.Finally, 'match' => TokenType.Match, 'case' => TokenType.Case, 'symbol' => TokenType.Symbol, 'enum' => TokenType.Enum, 'in' => TokenType.In, 'inline' => TokenType.Inline, 'sealed' => TokenType.Sealed, 'gen' => TokenType.Gen, 'yield' => TokenType.Yield, // Reserved for async functions (concurrency.md); not yet parsed. 'async' => TokenType.Async, 'await' => TokenType.Await, // `using` is reserved rather than contextual, which is what lets the // unbound form `using acquire(lock);` be unambiguous — see // docs/design/ownership.md §`using`. 'using' => TokenType.Using }; // ============================================================================ // Character helpers // ============================================================================ function isWhitespace(c: i32) { // space=32, tab=9, newline=10, carriage return=13 return c == 32 || c == 9 || c == 10 || c == 13; } function isDigit(c: i32) { // '0'=48, '9'=57 return c >= 48 && c <= 57; } function isHexDigit(c: i32) { // '0'-'9'=48-57, 'a'-'f'=97-102, 'A'-'F'=65-70 return (c >= 48 && c <= 57) || (c >= 97 && c <= 102) || (c >= 65 && c <= 70); } function isAlpha(c: i32) { // 'a'-'z'=97-122, 'A'-'Z'=65-90 return (c >= 97 && c <= 122) || (c >= 65 && c <= 90); } function isIdentifierStart(c: i32) { // letters, underscore=95, $=36 return isAlpha(c) || c == 95 || c == 36; } export function isIdentifierPart(c: i32): boolean { return isAlpha(c) || isDigit(c) || c == 95 || c == 36; } // ============================================================================ // Keyword recognition // ============================================================================ /** One row of the keyword table: the canonical text and what it scans as. */ export final class Keyword(text: String, type: TokenType) /** * Keywords bucketed by byte length. A scanned word is compared only * against keywords that could possibly match it, and a word longer than * the longest keyword is rejected by the length alone. */ var keywordBuckets: GrowableArray> | null = null; function buildKeywordBuckets() { let texts = new Array(); var longest = 0; for (let text in KEYWORDS.keys()) { texts.push(text); if (text.length > longest) { longest = text.length; } } let buckets = new Array>(); var i = 0; while (i <= longest) { buckets.push(new Array()); i += 1; } // Bucket order follows the map's iteration order, which is not fixed. // It does not need to be: a word matches at most one keyword, so the // order candidates are tried in cannot change the answer. for (let text in texts) { if (let (true, type) = KEYWORDS.get(text)) { buckets[text.length].push(new Keyword(text, type)); } } return buckets; } /** * The keyword a byte range spells, or null if it spells an identifier. * * This reads the source bytes in place: recognizing `let` must not cost * a `String` that the token type makes redundant. In the compiler's own * sources one scanned word in three is a keyword, and materializing * each one allocated a wrapper nothing ever read. * * Bucketing by length first is what keeps the comparison cheap. Hashing * every byte to look the word up — which is what a map keyed by String * does — costs more than the compare that has to follow it anyway. */ export function matchKeyword(source: String, start: i32, length: i32): Keyword | null { var buckets = keywordBuckets; if (buckets == null) { buckets = buildKeywordBuckets(); keywordBuckets = buckets; } if (length >= buckets.length) { return null; } let first = source.getByteAt(start); for (let candidate in buckets[length]) { let {text} = candidate; if (text.getByteAt(0) != first) { continue; } var i = 1; var same = true; while (i < length) { if (source.getByteAt(start + i) != text.getByteAt(i)) { same = false; break; } i += 1; } if (same) { return candidate; } } return null; } // ============================================================================ // Tokenizer // ============================================================================ export final class Tokenizer { let #reader: StringReader; let #tokens: GrowableArray; let #comments: GrowableArray; let #templateStack: GrowableArray; var #line: i32; var #column: i32; var #lineStart: i32; new(source: String, table: StringTable | null = null) : #reader = new StringReader(source, table), #tokens = new Array(64), #comments = new Array(8), #templateStack = new Array(8), #line = 1, #column = 1, #lineStart = 0 { } /** Returns all comments collected during tokenization. */ getComments(): GrowableArray { return this.#comments; } // ============================================================================ // Cursor helpers // ============================================================================ #peek(): i32 { return this.#reader.peekByte(); } #peekNext(): i32 { return this.#reader.peekByteAt(1); } #advance(): i32 { let c = this.#reader.advanceByte(); if (c == 10) { // newline this.#line += 1; this.#lineStart = this.#reader.position; this.#column = 1; } else { this.#column += 1; } return c; } #currentColumn(): i32 { return this.#reader.position - this.#lineStart + 1; } #addToken(type: TokenType, value: String, startPos: i32, startCol: i32): void { let token = new Token(type, value, this.#line, startCol, startPos, this.#reader.position, null); this.#tokens.push(token); } #addTemplateToken(type: TokenType, cooked: String, raw: String, startPos: i32, startCol: i32): void { let token = new Token(type, cooked, this.#line, startCol, startPos, this.#reader.position, raw); this.#tokens.push(token); } // ============================================================================ // Main tokenize entry point // ============================================================================ tokenize(): GrowableArray { while (!this.#reader.isAtEnd) { this.#scanToken(); } // Add EOF token let pos = this.#reader.position; this.#addToken(TokenType.EOF, '', pos, this.#currentColumn()); return this.#tokens; } #scanToken(): void { let startPos = this.#reader.position; let startCol = this.#currentColumn(); let c = this.#peek(); // Whitespace if (isWhitespace(c)) { this.#advance(); return; } // Numbers if (isDigit(c)) { this.#scanNumber(startPos, startCol); return; } // Identifiers & Keywords if (isIdentifierStart(c)) { this.#scanIdentifier(startPos, startCol); return; } // Strings if (c == 39 || c == 34) { // single quote=39, double quote=34 this.#scanString(startPos, startCol); return; } // Template literals if (c == 96) { // backtick=96 this.#advance(); this.#scanTemplatePart(false, startPos, startCol); return; } // Operators and punctuation this.#scanOperator(startPos, startCol); } // ============================================================================ // Number scanning // ============================================================================ #scanNumber(startPos: i32, startCol: i32): void { let start = this.#reader.mark(); let c = this.#peek(); let next = this.#peekNext(); // Radix prefix: 0x/0X, 0b/0B, 0o/0O if (c == 48 && (next == 120 || next == 88 || next == 98 || next == 66 || next == 111 || next == 79)) { this.#advance(); // '0' this.#advance(); // 'x'/'X'/'b'/'B'/'o'/'O' while (!this.#reader.isAtEnd && isIdentifierPart(this.#peek())) { this.#advance(); } } else { // Decimal integer digits and '_' separators while (!this.#reader.isAtEnd && (isDigit(this.#peek()) || this.#peek() == 95)) { this.#advance(); } // Fractional part (including malformed '3._14' so the parser diagnoses '_' next to '.') if (this.#peek() == 46) { let afterDot = this.#peekNext(); let dotFollowedByDigitOrSepDigit = isDigit(afterDot) || (afterDot == 95 && this.#reader.position + 2 < this.#reader.source.length && isDigit(this.#reader.source.getByteAt(this.#reader.position + 2))); if (dotFollowedByDigitOrSepDigit) { this.#advance(); // '.' while (!this.#reader.isAtEnd && (isDigit(this.#peek()) || this.#peek() == 95)) { this.#advance(); } } } // Any trailing suffix or malformed identifier tail while (!this.#reader.isAtEnd && isIdentifierPart(this.#peek())) { this.#advance(); } } let raw = this.#reader.source.sliceBytes(start, this.#reader.position); this.#addToken(TokenType.Number, raw, startPos, startCol); } // ============================================================================ // Identifier scanning // ============================================================================ #scanIdentifier(startPos: i32, startCol: i32): void { let start = this.#reader.mark(); while (!this.#reader.isAtEnd && isIdentifierPart(this.#peek())) { this.#advance(); } // Classify before materializing: a keyword's text is already in the // table, so a hit reuses that instance and allocates nothing. Only // an identifier needs a slice of its own. let keyword = matchKeyword( this.#reader.source, start, this.#reader.position - start); if (keyword != null) { this.#addToken(keyword.type, keyword.text, startPos, startCol); } else { // Interned when the reader has a table: a repeat identifier costs a // probe against the source bytes and no allocation; the first // occurrence of a spelling is copied once into the table. this.#addToken( TokenType.Identifier, this.#reader.internFrom(start), startPos, startCol); } } // ============================================================================ // String scanning // ============================================================================ #scanString(startPos: i32, startCol: i32): void { let quote = this.#advance(); let builder = new StringBuilder(); // Plain characters are copied a run at a time, as a slice of the // source: offsets are bytes on the byte targets and UTF-16 code units // on js, and a slice keeps every character on both. Copying a byte at // a time kept only the low byte of each code unit on js (#431). let source = this.#reader.source; var runStart = this.#reader.position; while (!this.#reader.isAtEnd && this.#peek() != quote) { if (this.#peek() == 92) { // backslash=92 builder.appendSlice(source, runStart, this.#reader.position); this.#advance(); if (this.#reader.isAtEnd) { break; } let escaped = this.#advance(); // 'n'=110, 'r'=114, 't'=116, '\\'=92, '\''=39, '"'=34, '0'=48, '\n'=10, '\r'=13 if (escaped == 110) { builder.appendByte(10); // newline } else if (escaped == 114) { builder.appendByte(13); // carriage return } else if (escaped == 116) { builder.appendByte(9); // tab } else if (escaped == 92) { builder.appendByte(92); // backslash } else if (escaped == 39) { builder.appendByte(39); // single quote } else if (escaped == 34) { builder.appendByte(34); // double quote } else if (escaped == 48) { builder.appendByte(0); // null } else if (escaped == 10) { // Line continuation (LF) - omit from string } else if (escaped == 13) { // Line continuation (CR or CRLF) - omit from string if (this.#peek() == 10) { this.#advance(); } } else { // Unknown escape - keep as-is builder.appendByte(92); builder.appendByte(escaped); } runStart = this.#reader.position; } else { this.#advance(); } } builder.appendSlice(source, runStart, this.#reader.position); if (!this.#reader.isAtEnd) { this.#advance(); // closing quote } this.#addToken(TokenType.String, builder.toString(), startPos, startCol); } // ============================================================================ // Template literal scanning // ============================================================================ #scanTemplatePart(isMiddleOrTail: boolean, startPos: i32, startCol: i32): void { let cooked = new StringBuilder(); // As in `#scanString`, plain characters are copied a run at a time. // The raw string is the part's source text exactly, so it is one slice. let source = this.#reader.source; let partStart = this.#reader.position; var runStart = partStart; while (!this.#reader.isAtEnd) { let c = this.#peek(); // End of template if (c == 96) { // backtick=96 cooked.appendSlice(source, runStart, this.#reader.position); let raw = source.sliceBytes(partStart, this.#reader.position); this.#advance(); let tokenType = if (isMiddleOrTail) { TokenType.TemplateTail } else { TokenType.NoSubstitutionTemplate }; this.#addTemplateToken(tokenType, cooked.toString(), raw, startPos, startCol); return; } // Template expression start if (c == 36 && this.#peekNext() == 123) { // '$'=36, '{'=123 cooked.appendSlice(source, runStart, this.#reader.position); let raw = source.sliceBytes(partStart, this.#reader.position); this.#advance(); // $ this.#advance(); // { this.#templateStack.push(1); // start tracking braces let tokenType = if (isMiddleOrTail) { TokenType.TemplateMiddle } else { TokenType.TemplateHead }; this.#addTemplateToken(tokenType, cooked.toString(), raw, startPos, startCol); return; } // Escape sequence if (c == 92) { // backslash=92 cooked.appendSlice(source, runStart, this.#reader.position); this.#advance(); this.#processTemplateEscape(cooked); runStart = this.#reader.position; } else { this.#advance(); } } // Reached end without closing cooked.appendSlice(source, runStart, this.#reader.position); let tokenType = if (isMiddleOrTail) { TokenType.TemplateTail } else { TokenType.NoSubstitutionTemplate }; this.#addTemplateToken(tokenType, cooked.toString(), source.sliceBytes(partStart, this.#reader.position), startPos, startCol); } #processTemplateEscape(cooked: StringBuilder): void { if (this.#reader.isAtEnd) { return; } let escaped = this.#advance(); // 'n'=110, 'r'=114, 't'=116, '\\'=92, '`'=96, '$'=36, '0'=48, '\n'=10, '\r'=13 if (escaped == 110) { cooked.appendByte(10); // newline } else if (escaped == 114) { cooked.appendByte(13); // carriage return } else if (escaped == 116) { cooked.appendByte(9); // tab } else if (escaped == 92) { cooked.appendByte(92); // backslash } else if (escaped == 96) { cooked.appendByte(96); // backtick } else if (escaped == 36) { cooked.appendByte(36); // dollar } else if (escaped == 48) { cooked.appendByte(0); // null } else if (escaped == 10) { // Line continuation (LF) - omitted from cooked } else if (escaped == 13) { // Line continuation (CR or CRLF) if (this.#peek() == 10) { this.#advance(); } // omitted from cooked } else { // Unknown escape - keep as-is cooked.appendByte(92); cooked.appendByte(escaped); } } // ============================================================================ // Operator scanning // ============================================================================ #scanOperator(startPos: i32, startCol: i32): void { let c = this.#advance(); // '='=61, '>'=62 if (c == 61) { if (this.#peek() == 62) { this.#advance(); this.#addToken(TokenType.Arrow, '=>', startPos, startCol); } else if (this.#peek() == 61) { this.#advance(); if (this.#peek() == 61) { this.#advance(); this.#addToken(TokenType.EqualsEqualsEquals, '===', startPos, startCol); } else { this.#addToken(TokenType.EqualsEquals, '==', startPos, startCol); } } else { this.#addToken(TokenType.Equals, '=', startPos, startCol); } return; } // '!'=33 if (c == 33) { if (this.#peek() == 61) { this.#advance(); if (this.#peek() == 61) { this.#advance(); this.#addToken(TokenType.BangEqualsEquals, '!==', startPos, startCol); } else { this.#addToken(TokenType.BangEquals, '!=', startPos, startCol); } } else { this.#addToken(TokenType.Bang, '!', startPos, startCol); } return; } // '<'=60 if (c == 60) { if (this.#peek() == 61) { this.#advance(); this.#addToken(TokenType.LessEquals, '<=', startPos, startCol); } else if (this.#peek() == 60) { this.#advance(); this.#addToken(TokenType.LessLess, '<<', startPos, startCol); } else { this.#addToken(TokenType.Less, '<', startPos, startCol); } return; } // '>'=62 if (c == 62) { if (this.#peek() == 61) { this.#advance(); this.#addToken(TokenType.GreaterEquals, '>=', startPos, startCol); } else if (this.#peek() == 62) { this.#advance(); if (this.#peek() == 62) { this.#advance(); this.#addToken(TokenType.GreaterGreaterGreater, '>>>', startPos, startCol); } else { this.#addToken(TokenType.GreaterGreater, '>>', startPos, startCol); } } else { this.#addToken(TokenType.Greater, '>', startPos, startCol); } return; } // '+'=43 if (c == 43) { if (this.#peek() == 61) { this.#advance(); this.#addToken(TokenType.PlusEquals, '+=', startPos, startCol); } else { this.#addToken(TokenType.Plus, '+', startPos, startCol); } return; } // '-'=45 if (c == 45) { if (this.#peek() == 61) { this.#advance(); this.#addToken(TokenType.MinusEquals, '-=', startPos, startCol); } else { this.#addToken(TokenType.Minus, '-', startPos, startCol); } return; } // '*'=42 if (c == 42) { if (this.#peek() == 61) { this.#advance(); this.#addToken(TokenType.StarEquals, '*=', startPos, startCol); } else if (this.#peek() == 42) { this.#advance(); if (this.#peek() == 61) { this.#advance(); this.#addToken(TokenType.StarStarEquals, '**=', startPos, startCol); } else { this.#addToken(TokenType.StarStar, '**', startPos, startCol); } } else { this.#addToken(TokenType.Star, '*', startPos, startCol); } return; } // '/'=47 if (c == 47) { if (this.#peek() == 47) { // Single-line comment — collect text after // let commentStart = startPos; let commentLine = this.#line; let commentCol = startCol; while (!this.#reader.isAtEnd && this.#peek() != 10) { this.#advance(); } let commentEnd = this.#reader.position; // Extract text after '//' (startPos is at first '/', startPos+2 skips '//') let value = this.#reader.source.sliceBytes(startPos + 2, commentEnd); this.#comments.push(new Comment(value, false, commentStart, commentEnd, commentLine, commentCol)); return; } else if (this.#peek() == 42) { // Multi-line comment — collect text between /* and */ let commentStart = startPos; let commentLine = this.#line; let commentCol = startCol; this.#advance(); // consume '*' while (!this.#reader.isAtEnd) { if (this.#peek() == 42 && this.#peekNext() == 47) { this.#advance(); // consume '*' this.#advance(); // consume '/' break; } this.#advance(); } let commentEnd = this.#reader.position; // Extract text between '/*' and '*/' (startPos+2 to commentEnd-2) var innerEnd = commentEnd - 2; if (innerEnd < startPos + 2) { innerEnd = startPos + 2; } let value = this.#reader.source.sliceBytes(startPos + 2, innerEnd); this.#comments.push(new Comment(value, true, commentStart, commentEnd, commentLine, commentCol)); return; } else if (this.#peek() == 61) { this.#advance(); this.#addToken(TokenType.SlashEquals, '/=', startPos, startCol); } else { this.#addToken(TokenType.Slash, '/', startPos, startCol); } return; } // '%'=37 if (c == 37) { if (this.#peek() == 61) { this.#advance(); this.#addToken(TokenType.PercentEquals, '%=', startPos, startCol); } else { this.#addToken(TokenType.Percent, '%', startPos, startCol); } return; } // '|'=124 if (c == 124) { if (this.#peek() == 124) { this.#advance(); this.#addToken(TokenType.PipePipe, '||', startPos, startCol); } else if (this.#peek() == 62) { this.#advance(); this.#addToken(TokenType.PipeGreater, '|>', startPos, startCol); } else { this.#addToken(TokenType.Pipe, '|', startPos, startCol); } return; } // '&'=38 if (c == 38) { if (this.#peek() == 38) { this.#advance(); this.#addToken(TokenType.AmpersandAmpersand, '&&', startPos, startCol); } else { this.#addToken(TokenType.Ampersand, '&', startPos, startCol); } return; } // '^'=94 if (c == 94) { this.#addToken(TokenType.Caret, '^', startPos, startCol); return; } // '?'=63 if (c == 63) { if (this.#peek() == 63) { this.#advance(); if (this.#peek() == 61) { this.#advance(); this.#addToken(TokenType.QuestionQuestionEquals, '??=', startPos, startCol); } else { this.#addToken(TokenType.QuestionQuestion, '??', startPos, startCol); } } else if (this.#peek() == 46) { // '.'=46 this.#advance(); this.#addToken(TokenType.QuestionDot, '?.', startPos, startCol); } else if (this.#peek() == 91) { // '['=91 this.#advance(); this.#addToken(TokenType.QuestionLBracket, '?[', startPos, startCol); } else if (this.#peek() == 40) { // '('=40 this.#advance(); this.#addToken(TokenType.QuestionLParen, '?(', startPos, startCol); } else { this.#addToken(TokenType.Question, '?', startPos, startCol); } return; } // '('=40 if (c == 40) { this.#addToken(TokenType.LParen, '(', startPos, startCol); return; } // ')'=41 if (c == 41) { this.#addToken(TokenType.RParen, ')', startPos, startCol); return; } // '{'=123 if (c == 123) { // Track brace depth for template expressions if (this.#templateStack.length > 0) { let lastIdx = this.#templateStack.length - 1; let depth = this.#templateStack[lastIdx]; this.#templateStack[lastIdx] = depth + 1; } this.#addToken(TokenType.LBrace, '{', startPos, startCol); return; } // '}'=125 if (c == 125) { // Check if this closes a template expression if (this.#templateStack.length > 0) { let lastIdx = this.#templateStack.length - 1; let depth = this.#templateStack[lastIdx]; if (depth == 1) { this.#templateStack.pop(); // Continue scanning template this.#scanTemplatePart(true, startPos, startCol); return; } else { this.#templateStack[lastIdx] = depth - 1; } } this.#addToken(TokenType.RBrace, '}', startPos, startCol); return; } // '['=91 if (c == 91) { this.#addToken(TokenType.LBracket, '[', startPos, startCol); return; } // ']'=93 if (c == 93) { this.#addToken(TokenType.RBracket, ']', startPos, startCol); return; } // ':'=58 if (c == 58) { this.#addToken(TokenType.Colon, ':', startPos, startCol); return; } // ';'=59 if (c == 59) { this.#addToken(TokenType.Semi, ';', startPos, startCol); return; } // ','=44 if (c == 44) { this.#addToken(TokenType.Comma, ',', startPos, startCol); return; } // '.'=46 if (c == 46) { if (this.#peek() == 46 && this.#peekNext() == 46) { this.#advance(); this.#advance(); this.#addToken(TokenType.DotDotDot, '...', startPos, startCol); } else if (this.#peek() == 46) { this.#advance(); this.#addToken(TokenType.DotDot, '..', startPos, startCol); } else { this.#addToken(TokenType.Dot, '.', startPos, startCol); } return; } // '#'=35 if (c == 35) { this.#addToken(TokenType.Hash, '#', startPos, startCol); return; } // '@'=64 if (c == 64) { this.#addToken(TokenType.At, '@', startPos, startCol); return; } // Unknown character let unknown = new StringBuilder(); unknown.appendByte(c); this.#addToken(TokenType.Unknown, unknown.toString(), startPos, startCol); } } // ============================================================================ // Convenience function // ============================================================================ export function tokenize(source: String, table: StringTable | null = null): GrowableArray { let tokenizer = new Tokenizer(source, table); return tokenizer.tokenize(); } /** * Primes `table` with Zena's keywords, so a keyword lexed out of source * interns to the very String instance `KEYWORDS` is keyed by. The keyword * lookup in `#scanIdentifier` then hits `==`'s reference-equal fast path * against a key whose hash was cached the first time it was probed. * * Call this once per table, not once per tokenizer: re-priming is harmless * but it is a lookup per keyword per file. */ export function primeKeywords(table: StringTable): void { for (let keyword in KEYWORDS.keys()) { table.prime(keyword); } }