wit-parser 0.0.1-preview.1

A WIT (WebAssembly Interface Types) parser, resolver and component encoder

Readme Files Dependencies Versions

wit-parser#

A WebAssembly Interface Types (WIT) parser, semantic resolver, .wit.json serializer, and component type encoder for Zena.

Installation#

zena pkg add zena-lang:wit-parser

Example#

import { parse } from 'zena-lang:wit-parser';
import { AstItemTag } from 'zena-lang:wit-parser/parser';

let source = `
package example:greeter@1.0.0;

interface greet {
  record greeting { text: string, formal: bool }
  hello: func(name: string) -> greeting;
}

interface farewell {
  use greet.{greeting};
  goodbye: func(name: string) -> greeting;
}

world greeter {
  export greet;
  export farewell;
}
`;

// Parses syntax and resolves names and cross-interface `use` references.
// Throws `ParseError` or `ResolveError` if the document is invalid.
let doc = parse(source);

var interfaces = 0;
var worlds = 0;
for (let item in doc.items) {
  if (item.tag == AstItemTag.Interface) {
    interfaces += 1;
  } else if (item.tag == AstItemTag.World) {
    worlds += 1;
  }
}

Libraries#

zena-lang:wit-parser#

The primary entry point for parsing, resolving, and serializing WIT source text.

Function Description
parse(source) Parses and resolves source into a validated Ast. Throws ParseError or ResolveError.
parseSyntax(source) Parses source without resolving names, for single-file syntax tooling such as formatters.
resolve(ast) Resolves names and validates an Ast in place.
toJson(ast, packageOrder?) Serializes a resolved Ast to the .wit.json format emitted by wasm-tools component wit --json.

zena-lang:wit-parser also re-exports Ast, ParseError, and ResolveError. Both error types carry a Span with byte offsets into the source text.

zena-lang:wit-parser/parser#

The AST node declarations (Ast, AstItem, AstItemTag, InterfaceDef, WorldDef, TypeDef, Type, NamedFunc, ResourceFunc, Use, Include, Docs, Attribute, PackageName, NestedPackage, and related types) and Parser.

zena-lang:wit-parser/document#

Utilities for assembling multi-file WIT directories into a single source document while tracking file boundaries for error reporting:

  • WitFile(name, source) and joinWitDocument(files): concatenates .wit files in canonical name order and records where each file starts.
  • WitDocument.locate(offset): maps a byte offset back to a WitLocation (file, 1-based line, 1-based column).
  • witDiagnostic(doc, error): formats a LexError, ParseError, or ResolveError with its filename, line, column, source line, and caret underline.
  • findInterface(ast, namespace, interfacePath): looks up an interface (pkg/iface) in a parsed document.

zena-lang:wit-parser/component-encoder#

Encodes a resolved WIT world into WebAssembly Component Model binary type and import sections:

  • encodeTypesOnlyComponent(ast, worldName): encodes the named world as a standalone types-only component binary.
  • ComponentTypeEncoder: emits component type, alias, import, and export sections and canonical ABI metadata (WitFuncMeta, WitFlatMeta).
  • expandedWorldItems(ast, world) and worldNames(ast): expands world includes and transitive use dependencies.

zena-lang:wit-parser/lexer and zena-lang:wit-parser/token#

Low-level tokenization: Lexer, LexError, Token, TokenType, and Span.

zena-lang:wit-parser/resolver#

Semantic validation and name resolution: Resolver and ResolveError.

11.8 KiB Raw
/**
 * WIT Lexer
 *
 * Tokenizes WIT (WebAssembly Interface Types) source text.
 * Based on the canonical implementation in wasm-tools:
 * https://github.com/bytecodealliance/wasm-tools/blob/main/crates/wit-parser/src/ast/lex.rs
 */

import {StringBuilder, StringReader} from 'zena:core';
import {TokenType, Span, Token, keywordToToken} from './token.zena';

// ============================================================================
// Character Classification
// ============================================================================

/** Check if a byte is ASCII whitespace (space, tab, newline, carriage return). */
function isWhitespace(b: i32) {
  return b == 32 || b == 9 || b == 10 || b == 13;
}

/** Check if a byte is an ASCII digit (0-9). */
function isDigit(b: i32) {
  return b >= 48 && b <= 57;  // '0' to '9'
}

/** Check if a byte is an ASCII hex digit (0-9, a-f, A-F). */
function isHexDigit(b: i32) {
  return isDigit(b) || (b >= 65 && b <= 70) || (b >= 97 && b <= 102);
}

/** Check if a byte is a valid start of an identifier (a-z, A-Z). */
function isIdStart(b: i32) {
  return (b >= 65 && b <= 90) || (b >= 97 && b <= 122);  // A-Z or a-z
}

/** Check if a byte is valid within an identifier (a-z, A-Z, 0-9, -). */
function isIdContinue(b: i32) {
  return isIdStart(b) || isDigit(b) || b == 45;  // also '-' (kebab-case)
}

// ============================================================================
// Lexer Error
// ============================================================================

/**
 * Error produced during lexing.
 */
export class LexError extends Error {
  span: Span;

  new(message: String, span: Span) : span = span, super(message);
}

// ============================================================================
// Lexer
// ============================================================================

/**
 * A lexer (tokenizer) for WIT source text.
 *
 * Usage:
 * ```zena
 * let lexer = new Lexer(source);
 * while (true) {
 *   let token = lexer.next();
 *   if (token.type == TokenType.Eof) { break; }
 *   // process token
 * }
 * ```
 */
export class Lexer {
  #reader: StringReader;
  
  /** Comment content accumulated before items (used for docs). */
  var #precedingComments: StringBuilder | null = null;

  /**
   * Create a new lexer for the given source text.
   */
  new(source: String) : #reader = new StringReader(source);
  
  /**
   * Get and clear any comments accumulated before the current position.
   * Returns null if no comments were accumulated.
   */
  takePrecedingComments(): String | null {
    if (this.#precedingComments == null) {
      return null;
    }
    let sb = this.#precedingComments as StringBuilder;
    let result = sb.toString();
    this.#precedingComments = null;
    return result;
  }

  // --------------------------------------------------------------------------
  // Core Lexer Methods
  // --------------------------------------------------------------------------

  /**
   * Get the next token from the source.
   */
  next(): Token {
    // Skip whitespace and comments
    this.#skipTrivia();

    let start = this.#reader.position;

    // Check for end of input
    if (this.#isEof()) {
      return new Token(TokenType.Eof, new Span(start, start));
    }

    let b = this.#peek();

    // Single-character tokens
    if (b == 61) { return this.#single(TokenType.Equals); }      // =
    if (b == 44) { return this.#single(TokenType.Comma); }       // ,
    if (b == 58) { return this.#single(TokenType.Colon); }       // :
    if (b == 46) { return this.#single(TokenType.Period); }      // .
    if (b == 59) { return this.#single(TokenType.Semicolon); }   // ;
    if (b == 40) { return this.#single(TokenType.LeftParen); }   // (
    if (b == 41) { return this.#single(TokenType.RightParen); }  // )
    if (b == 123) { return this.#single(TokenType.LeftBrace); }  // {
    if (b == 125) { return this.#single(TokenType.RightBrace); } // }
    if (b == 60) { return this.#single(TokenType.LessThan); }    // <
    if (b == 62) { return this.#single(TokenType.GreaterThan); } // >
    if (b == 42) { return this.#single(TokenType.Star); }        // *
    if (b == 64) { return this.#single(TokenType.At); }          // @
    if (b == 43) { return this.#single(TokenType.Plus); }        // +
    if (b == 95) { return this.#single(TokenType.Underscore); }  // _

    // -> (right arrow)
    if (b == 45) {
      this.#advance();
      if (!this.#isEof() && this.#peek() == 62) {  // >
        this.#advance();
        return new Token(TokenType.RArrow, new Span(start, this.#reader.position));
      }
      return new Token(TokenType.Minus, new Span(start, this.#reader.position));
    }

    // / or /// doc comment
    if (b == 47) {
      // Check for /// (doc comment)
      if (this.#peekAt(1) == 47 && this.#peekAt(2) == 47) {
        // Skip the ///
        this.#advance();
        this.#advance();
        this.#advance();
        // Skip optional space after ///
        if (!this.#isEof() && this.#peek() == 32) {
          this.#advance();
        }
        // Skip until newline or EOF
        while (!this.#isEof()) {
          let c = this.#peek();
          if (c == 10) {  // \n
            this.#advance();
            break;
          }
          this.#advance();
        }
        return new Token(TokenType.DocComment, new Span(start, this.#reader.position));
      }
      return this.#single(TokenType.Slash);
    }

    // %identifier (explicit identifier)
    if (b == 37) {  // %
      this.#advance();
      return this.#lexExplicitId(start);
    }

    // Integer literals
    if (isDigit(b)) {
      return this.#lexInteger(start);
    }

    // Identifiers and keywords
    if (isIdStart(b)) {
      return this.#lexIdOrKeyword(start);
    }

    // Unknown character
    this.#advance();
    throw new LexError("unexpected character", new Span(start, this.#reader.position));
  }

  /**
   * Peek at the current token without consuming it.
   * Note: This re-lexes; for efficiency, callers should cache the result.
   */
  peek(): Token {
    let savedPos = this.#reader.position;
    let token = this.next();
    this.#reader.reset(savedPos);
    return token;
  }

  /**
   * Expect the next token to be of a specific type.
   * Throws LexError if it doesn't match.
   */
  expect(expected: TokenType): Token {
    let token = this.next();
    if (token.type != expected) {
      throw new LexError(
        "unexpected token",
        token.span
      );
    }
    return token;
  }

  /**
   * Get the source text for a span.
   */
  getSpanText(span: Span): String {
    return this.#reader.sliceRange(span.start, span.end);
  }

  // --------------------------------------------------------------------------
  // Private Helpers
  // --------------------------------------------------------------------------

  /** Check if we're at end of file. */
  #isEof(): boolean {
    return this.#reader.isAtEnd;
  }

  /** Peek at current byte without advancing. */
  #peek(): i32 {
    return this.#reader.peekByte();
  }

  /** Peek at byte at given offset from current position. */
  #peekAt(offset: i32): i32 {
    return this.#reader.peekByteAt(offset);
  }

  /** Advance position by one byte. */
  #advance(): void {
    this.#reader.advanceByte();
  }

  /** Create a single-character token. */
  #single(type: TokenType): Token {
    let start = this.#reader.position;
    this.#advance();
    return new Token(type, new Span(start, this.#reader.position));
  }

  /** Skip whitespace and comments. */
  #skipTrivia(): void {
    while (!this.#isEof()) {
      let b = this.#peek();

      // Whitespace
      if (isWhitespace(b)) {
        this.#advance();
        continue;
      }

      // Single-line comment: // ... (but NOT doc comments: ///)
      if (b == 47 && this.#peekAt(1) == 47 && this.#peekAt(2) != 47) {
        this.#captureLineComment();
        continue;
      }

      // Block comment: /* ... */
      if (b == 47 && this.#peekAt(1) == 42) {
        this.#captureBlockComment();
        continue;
      }

      // Not trivia
      break;
    }
  }

  /** Capture a single-line comment content (// ...). */
  #captureLineComment(): void {
    // Skip the //
    this.#advance();
    this.#advance();
    
    let contentStart = this.#reader.position;
    
    // Capture everything until newline
    while (!this.#isEof()) {
      let b = this.#peek();
      if (b == 10) {  // \n
        break;
      }
      this.#advance();
    }
    let contentEnd = this.#reader.position;
    
    // Skip the newline if present
    if (!this.#isEof() && this.#peek() == 10) {
      this.#advance();
    }
    
    // Accumulate the comment content
    if (this.#precedingComments == null) {
      this.#precedingComments = new StringBuilder();
    }
    let sb = this.#precedingComments as StringBuilder;
    if (sb.length > 0) {
      sb.append("\n");
    }
    // Get comment content
    let rawContent = this.#reader.sliceRange(contentStart, contentEnd);
    sb.append(rawContent);
  }

  /** Capture a block comment content (supports nesting). */
  #captureBlockComment(): void {
    // Skip just the opening '/' - we keep the '*' as part of the content
    this.#advance();
    
    let contentStart = this.#reader.position;

    // Skip the '*'
    this.#advance();

    // Track nesting depth (we're already inside one comment)
    var depth = 1;

    // Skip until all nested comments are closed or EOF
    while (!this.#isEof() && depth > 0) {
      let b = this.#peek();
      let next = this.#peekAt(1);
      
      // Check for nested opening /*
      if (b == 47 && next == 42) {
        this.#advance();
        this.#advance();
        depth += 1;
        continue;
      }
      
      // Check for closing */
      if (b == 42 && next == 47) {
        this.#advance();
        this.#advance();
        depth -= 1;
        continue;
      }
      
      this.#advance();
    }
    
    // contentEnd is the current position (includes the final */)
    let contentEnd = this.#reader.position;
    
    // Accumulate the comment content
    if (this.#precedingComments == null) {
      this.#precedingComments = new StringBuilder();
    }
    let sb = this.#precedingComments as StringBuilder;
    if (sb.length > 0) {
      sb.append("\n");
    }
    // Get block comment content (including leading * and trailing */)
    let content = this.#reader.sliceRange(contentStart, contentEnd);
    sb.append(content);
  }

  /** Lex an identifier or keyword. */
  #lexIdOrKeyword(start: i32): Token {
    // Consume identifier characters
    while (!this.#isEof() && isIdContinue(this.#peek())) {
      this.#advance();
    }

    let span = new Span(start, this.#reader.position);
    let text = this.getSpanText(span);

    // Check if it's a keyword
    let tokenType = keywordToToken(text);
    return new Token(tokenType, span);
  }

  /** Lex an explicit identifier (%foo). */
  #lexExplicitId(start: i32): Token {
    // We've already consumed the %
    // Now consume the identifier
    if (this.#isEof() || !isIdStart(this.#peek())) {
      throw new LexError("expected identifier after '%'", new Span(start, this.#reader.position));
    }

    while (!this.#isEof() && isIdContinue(this.#peek())) {
      this.#advance();
    }

    return new Token(TokenType.ExplicitId, new Span(start, this.#reader.position));
  }

  /** Lex an integer literal. */
  #lexInteger(start: i32): Token {
    // Check for hex literal (0x...)
    if (this.#peek() == 48 && this.#peekAt(1) == 120) {  // 0x
      this.#advance();  // 0
      this.#advance();  // x
      
      if (this.#isEof() || !isHexDigit(this.#peek())) {
        throw new LexError("expected hex digit after '0x'", new Span(start, this.#reader.position));
      }
      
      while (!this.#isEof() && isHexDigit(this.#peek())) {
        this.#advance();
      }
    } else {
      // Decimal literal
      while (!this.#isEof() && isDigit(this.#peek())) {
        this.#advance();
      }
    }

    return new Token(TokenType.Integer, new Span(start, this.#reader.position));
  }
}
none
VersionPublishedDependenciesApproved by
0.0.1-preview.1 2026-11-19 none