protobuf 0.0.1-preview.1

Protocol Buffers and gRPC

Readme Files Dependencies Versions

protobuf#

A Protocol Buffers (.proto) code generator, wire-format runtime, and gRPC / gRPC-fallback transports for Zena.

Installation#

zena pkg add @zena/protobuf

Generating Code#

protobuf/generator compiles .proto schemas into Zena source code with no I/O of its own—you supply the entry file paths and a callback that reads imported .proto files:

import { generate } from 'protobuf/generator';

let source = await generate({
  files: ['service.proto'],
  read: async (path) => readProtoFile(path),
});

Options (GenerateOptions)#

  • files: Array<String> — entry .proto files to generate from.
  • read: (path: String) => Future<String | null> — returns the .proto text for path, or null if not found.
  • roots: Array<String> (optional) — generate only the named messages, enums, or services (by full name) and the types they transitively reference. When omitted or empty, everything declared in files is generated.
  • runtimeImport: String (optional) — module specifier that generated code imports runtime helpers from (defaults to 'protobuf/runtime').

Using Generated Code#

For each message M, the generator emits a view class M (which reads an encoded message in place without eagerly allocating child objects) and a record type MInit, which M.of encodes:

import { HashMap } from 'zena:collections';

// Decoding wire bytes
let doc = Document.decode(bytes);
let title = doc.fields.getOr('title', Value.of({}));
let text = match (title.valueType) {
  case ValueStringValue {value}: value
  case _: ''
};

// Encoding a message
let fields = new HashMap<String, Value>();
fields['title'] = Value.of({stringValue: 'Hello'});
let encoded = Document.of({name, fields}).protoView.toByteArray();

A message-typed field accepts the message's class, so nested messages are built inside out with M.of, and a message read from one response can be passed into another message unchanged without re-encoding.

Services and Transports#

For each service in a .proto file, the generator emits a client class over ProtoTransport (protobuf/runtime). Two transports speak over wasi:http, both supporting unary and server-streaming RPCs (and GrpcTransport also supporting client-streaming RPCs):

  • protobuf/grpc (GrpcTransport): standard gRPC over HTTP/2 (https:// via ALPN or cleartext HTTP/2 for http://).
  • protobuf/fallback (FallbackTransport): Google's gRPC HTTP/1.1 $rpc fallback protocol (binary protobuf posted to /$rpc/<service>/<method>), usable on hosts whose outgoing HTTP is HTTP/1.1 only.
import { GrpcTransport } from 'protobuf/grpc';

let transport = new GrpcTransport('https://example.googleapis.com', credentials);
let client = new MyServiceClient(transport);
let response = await client.getItem({name: 'items/1'});

credentials is any Credentials (protobuf/runtime), which supplies the authorization header before each call. Generated clients also populate the x-goog-request-params routing header from google.api.http and google.api.routing options (AIP-4222). Failed calls throw RpcError (protobuf/runtime) carrying the google.rpc.Code.

Libraries#

  • protobuf/runtime — Wire-format views and writers (ProtoView, ProtoWriter, ProtoList, ProtoMap, ProtoBytes, ProtoStream), ProtoTransport, Credentials, RpcMetadata, RpcMethod, and RpcError.
  • protobuf/grpc — GrpcTransport over HTTP/2.
  • protobuf/fallback — FallbackTransport over Google's HTTP/1.1 $rpc fallback protocol.
  • protobuf/generator — generate and GenerateOptions for turning .proto files into Zena source.
  • protobuf/parser and protobuf/lexer — .proto syntax tree and tokenizer (parseProto, Lexer).
  • protobuf/schema and protobuf/emitter — Schema resolution and Zena code emission.
  • protobuf/text — Protobuf text format helpers.

License#

MIT

9.5 KiB Raw
/**
 * Tokenizes `.proto` source text.
 *
 * The token set follows the protobuf language specification
 * (https://protobuf.dev/reference/protobuf/proto3-spec/): identifiers,
 * integer and float literals, string literals, and single-character
 * symbols. Keywords are identifiers; the parser tells them apart by
 * position, as protoc does, so `message message = 1;` stays legal.
 *
 * Each token carries the comment written directly above it, which the
 * generator copies into the generated declaration's doc comment.
 */

import {StringBuilder, StringReader} from 'zena:core';

/** The kind of a token. */
export enum TokenKind {
  Ident,
  Int,
  Float,
  Str,
  Symbol,
  Eof
}

/**
 * One token. `text` is the identifier, the literal as written, the decoded
 * contents of a string literal, or the symbol character.
 */
export final class Token {
  kind: TokenKind;
  text: String;
  line: i32;
  column: i32;
  /** The comment block that ends on the line above this token, if any. */
  comment: String | null;
  new(this.kind, this.text, this.line, this.column, this.comment);
}

/** A malformed token, with the position it starts at. */
export class LexError extends Error {
  line: i32;
  column: i32;
  new(message: String, line: i32, column: i32)
    : line = line, column = column, super(`${line}:${column}: ${message}`);
}

function isDigit(b: i32) {
  return b >= 0x30 && b <= 0x39;
}

function isHexDigit(b: i32) {
  return isDigit(b) || (b >= 0x41 && b <= 0x46) || (b >= 0x61 && b <= 0x66);
}

function isLetter(b: i32) {
  return (b >= 0x41 && b <= 0x5A) || (b >= 0x61 && b <= 0x7A) || b == 0x5F;
}

function hexValue(b: i32) {
  return if (isDigit(b)) b - 0x30 else if (b >= 0x61) b - 0x61 + 10 else b - 0x41 + 10;
}

/** Produces the tokens of one `.proto` file, one per call to `next`. */
export final class Lexer {
  #reader: StringReader;
  var #line: i32 = 1;
  var #lineStart: i32 = 0;

  new(source: String) : #reader = new StringReader(source);

  /** The next token; `Eof` at the end of input, and on every call after. */
  next(): Token {
    let comment = this.#skipTrivia();
    let reader = this.#reader;
    let line = this.#line;
    let column = reader.position - this.#lineStart + 1;
    if (reader.isAtEnd) {
      return new Token(TokenKind.Eof, '', line, column, comment);
    }
    let start = reader.position;
    let b = reader.peekByte();
    if (isLetter(b)) {
      while (!reader.isAtEnd && (isLetter(reader.peekByte()) || isDigit(reader.peekByte()))) {
        reader.advanceByte();
      }
      return new Token(TokenKind.Ident, reader.sliceFrom(start), line, column, comment);
    }
    if (isDigit(b) || (b == 0x2E && isDigit(reader.peekByteAt(1)))) {
      return this.#number(line, column, comment);
    }
    if (b == 0x22 || b == 0x27) {
      return new Token(TokenKind.Str, this.#string(b, line, column), line, column, comment);
    }
    reader.advanceByte();
    return new Token(TokenKind.Symbol, reader.sliceFrom(start), line, column, comment);
  }

  /**
   * Skips whitespace and comments, and returns the comment block that ends
   * on the line before the next token. A blank line between a comment and
   * the token detaches the comment, as in protoc.
   */
  #skipTrivia(): String | null {
    let reader = this.#reader;
    var comment: StringBuilder | null = null;
    var commentEndLine = -1;
    while (!reader.isAtEnd) {
      let b = reader.peekByte();
      if (b == 0x0A) {
        reader.advanceByte();
        this.#line += 1;
        this.#lineStart = reader.position;
        continue;
      }
      if (b == 0x20 || b == 0x09 || b == 0x0D || b == 0x0C || b == 0x0B) {
        reader.advanceByte();
        continue;
      }
      if (b == 0x2F && reader.peekByteAt(1) == 0x2F) {
        let commentLine = this.#line;
        reader.skipBytes(2);
        let start = reader.position;
        while (!reader.isAtEnd && reader.peekByte() != 0x0A) {
          reader.advanceByte();
        }
        let text = stripLeadingSpace(reader.sliceFrom(start));
        if (comment == null || commentEndLine < commentLine - 1) {
          comment = new StringBuilder();
        } else {
          comment.appendByte(0x0A);
        }
        comment.append(text);
        commentEndLine = commentLine;
        continue;
      }
      if (b == 0x2F && reader.peekByteAt(1) == 0x2A) {
        let startLine = this.#line;
        let startColumn = reader.position - this.#lineStart + 1;
        reader.skipBytes(2);
        let start = reader.position;
        var closed = false;
        while (!reader.isAtEnd) {
          let c = reader.peekByte();
          if (c == 0x2A && reader.peekByteAt(1) == 0x2F) {
            closed = true;
            break;
          }
          if (c == 0x0A) {
            this.#line += 1;
            this.#lineStart = reader.position + 1;
          }
          reader.advanceByte();
        }
        if (!closed) {
          throw new LexError('unterminated block comment', startLine, startColumn);
        }
        let text = blockCommentText(reader.sliceFrom(start));
        reader.skipBytes(2);
        comment = new StringBuilder();
        comment.append(text);
        commentEndLine = this.#line;
        continue;
      }
      break;
    }
    if (comment == null || commentEndLine < this.#line - 1) {
      return null;
    }
    return comment.toString();
  }

  #number(line: i32, column: i32, comment: String | null): Token {
    let reader = this.#reader;
    let start = reader.position;
    if (reader.peekByte() == 0x30 && (reader.peekByteAt(1) == 0x78 || reader.peekByteAt(1) == 0x58)) {
      reader.skipBytes(2);
      if (!isHexDigit(reader.peekByte())) {
        throw new LexError('expected a hex digit', line, column);
      }
      while (!reader.isAtEnd && isHexDigit(reader.peekByte())) {
        reader.advanceByte();
      }
      return new Token(TokenKind.Int, reader.sliceFrom(start), line, column, comment);
    }
    var isFloat = false;
    while (!reader.isAtEnd && isDigit(reader.peekByte())) {
      reader.advanceByte();
    }
    if (reader.peekByte() == 0x2E) {
      isFloat = true;
      reader.advanceByte();
      while (!reader.isAtEnd && isDigit(reader.peekByte())) {
        reader.advanceByte();
      }
    }
    let e = reader.peekByte();
    if (e == 0x65 || e == 0x45) {
      isFloat = true;
      reader.advanceByte();
      let sign = reader.peekByte();
      if (sign == 0x2B || sign == 0x2D) {
        reader.advanceByte();
      }
      if (!isDigit(reader.peekByte())) {
        throw new LexError('expected an exponent', line, column);
      }
      while (!reader.isAtEnd && isDigit(reader.peekByte())) {
        reader.advanceByte();
      }
    }
    if (!reader.isAtEnd && isLetter(reader.peekByte())) {
      throw new LexError('a number may not be followed by a letter', line, column);
    }
    let kind = if (isFloat) TokenKind.Float else TokenKind.Int;
    return new Token(kind, reader.sliceFrom(start), line, column, comment);
  }

  /** Reads a string literal and returns its decoded contents. */
  #string(quote: i32, line: i32, column: i32): String {
    let reader = this.#reader;
    reader.advanceByte();
    let out = new StringBuilder();
    while (true) {
      if (reader.isAtEnd || reader.peekByte() == 0x0A) {
        throw new LexError('unterminated string literal', line, column);
      }
      let b = reader.advanceByte();
      if (b == quote) {
        break;
      }
      if (b != 0x5C) {
        out.appendByte(b);
        continue;
      }
      let e = reader.advanceByte();
      if (e == 0x6E) {
        out.appendByte(0x0A);
      } else if (e == 0x74) {
        out.appendByte(0x09);
      } else if (e == 0x72) {
        out.appendByte(0x0D);
      } else if (e == 0x61) {
        out.appendByte(0x07);
      } else if (e == 0x62) {
        out.appendByte(0x08);
      } else if (e == 0x66) {
        out.appendByte(0x0C);
      } else if (e == 0x76) {
        out.appendByte(0x0B);
      } else if (e == 0x78 || e == 0x58) {
        var value = 0;
        var digits = 0;
        while (digits < 2 && isHexDigit(reader.peekByte())) {
          value = value * 16 + hexValue(reader.advanceByte());
          digits += 1;
        }
        if (digits == 0) {
          throw new LexError('expected a hex digit after \\x', line, column);
        }
        out.appendByte(value);
      } else if (e >= 0x30 && e <= 0x37) {
        var value = e - 0x30;
        var digits = 1;
        while (digits < 3 && reader.peekByte() >= 0x30 && reader.peekByte() <= 0x37) {
          value = value * 8 + (reader.advanceByte() - 0x30);
          digits += 1;
        }
        out.appendByte(value & 0xFF);
      } else if (e == 0x5C || e == 0x27 || e == 0x22 || e == 0x3F) {
        out.appendByte(e);
      } else {
        throw new LexError('unknown escape sequence', line, column);
      }
    }
    return out.toString();
  }
}

/** Drops the one space conventionally written after `//`. */
function stripLeadingSpace(s: String): String {
  return if (s.length > 0 && s.getByteAt(0) == 0x20) s.sliceBytes(1, s.length) else s;
}

/**
 * The text of a block comment: each line with its leading whitespace and
 * `*` decoration removed, and blank first and last lines dropped.
 */
function blockCommentText(body: String): String {
  let lines = body.split('\n');
  let out = new StringBuilder();
  var first = true;
  for (var i = 0; i < lines.length; i += 1) {
    var line = lines[i].trim();
    if (line.startsWith('*')) {
      line = stripLeadingSpace(line.sliceBytes(1, line.length));
    }
    if ((i == 0 || i == lines.length - 1) && line.length == 0) {
      continue;
    }
    if (!first) {
      out.appendByte(0x0A);
    }
    out.append(line);
    first = false;
  }
  return out.toString();
}
none
VersionPublishedDependenciesApproved by
0.0.1-preview.1 2026-11-19 none