protobuf 0.0.1-preview.1

Protocol Buffers and gRPC

Readme Files Dependencies Versions

protobuf#

A Protocol Buffers (.proto) code generator, wire-format runtime, and gRPC / gRPC-fallback transports for Zena.

Installation#

zena pkg add @zena/protobuf

Generating Code#

protobuf/generator compiles .proto schemas into Zena source code with no I/O of its own—you supply the entry file paths and a callback that reads imported .proto files:

import { generate } from 'protobuf/generator';

let source = await generate({
  files: ['service.proto'],
  read: async (path) => readProtoFile(path),
});

Options (GenerateOptions)#

  • files: Array<String> — entry .proto files to generate from.
  • read: (path: String) => Future<String | null> — returns the .proto text for path, or null if not found.
  • roots: Array<String> (optional) — generate only the named messages, enums, or services (by full name) and the types they transitively reference. When omitted or empty, everything declared in files is generated.
  • runtimeImport: String (optional) — module specifier that generated code imports runtime helpers from (defaults to 'protobuf/runtime').

Using Generated Code#

For each message M, the generator emits a view class M (which reads an encoded message in place without eagerly allocating child objects) and a record type MInit, which M.of encodes:

import { HashMap } from 'zena:collections';

// Decoding wire bytes
let doc = Document.decode(bytes);
let title = doc.fields.getOr('title', Value.of({}));
let text = match (title.valueType) {
  case ValueStringValue {value}: value
  case _: ''
};

// Encoding a message
let fields = new HashMap<String, Value>();
fields['title'] = Value.of({stringValue: 'Hello'});
let encoded = Document.of({name, fields}).protoView.toByteArray();

A message-typed field accepts the message's class, so nested messages are built inside out with M.of, and a message read from one response can be passed into another message unchanged without re-encoding.

Services and Transports#

For each service in a .proto file, the generator emits a client class over ProtoTransport (protobuf/runtime). Two transports speak over wasi:http, both supporting unary and server-streaming RPCs (and GrpcTransport also supporting client-streaming RPCs):

  • protobuf/grpc (GrpcTransport): standard gRPC over HTTP/2 (https:// via ALPN or cleartext HTTP/2 for http://).
  • protobuf/fallback (FallbackTransport): Google's gRPC HTTP/1.1 $rpc fallback protocol (binary protobuf posted to /$rpc/<service>/<method>), usable on hosts whose outgoing HTTP is HTTP/1.1 only.
import { GrpcTransport } from 'protobuf/grpc';

let transport = new GrpcTransport('https://example.googleapis.com', credentials);
let client = new MyServiceClient(transport);
let response = await client.getItem({name: 'items/1'});

credentials is any Credentials (protobuf/runtime), which supplies the authorization header before each call. Generated clients also populate the x-goog-request-params routing header from google.api.http and google.api.routing options (AIP-4222). Failed calls throw RpcError (protobuf/runtime) carrying the google.rpc.Code.

Libraries#

  • protobuf/runtime — Wire-format views and writers (ProtoView, ProtoWriter, ProtoList, ProtoMap, ProtoBytes, ProtoStream), ProtoTransport, Credentials, RpcMetadata, RpcMethod, and RpcError.
  • protobuf/grpc — GrpcTransport over HTTP/2.
  • protobuf/fallback — FallbackTransport over Google's HTTP/1.1 $rpc fallback protocol.
  • protobuf/generator — generate and GenerateOptions for turning .proto files into Zena source.
  • protobuf/parser and protobuf/lexer — .proto syntax tree and tokenizer (parseProto, Lexer).
  • protobuf/schema and protobuf/emitter — Schema resolution and Zena code emission.
  • protobuf/text — Protobuf text format helpers.

License#

MIT

9.5 KiB Raw
/**
 * A `ProtoTransport` that speaks Google's gRPC fallback protocol
 * (https://googleapis.github.io/HowToRPC.html) over `wasi:http`, for hosts
 * whose outgoing HTTP is HTTP/1.1 only, such as `wasmtime serve`.
 *
 * A call is a `POST` of the binary request to
 * `<origin>/$rpc/<service>/<method>` with `Content-Type:
 * application/x-protobuf`.
 *
 * - A unary call answers 200 with the binary response, or another status
 *   with a binary `google.rpc.Status`.
 * - A server-streaming call answers with a `google.rpc.StreamBody`: each
 *   response is one occurrence of field 1, the final status is field 2,
 *   and field 15 carries keep-alive padding. The body is read as it
 *   arrives, so each response is available as soon as its bytes are.
 *
 * Firestore answers both shapes on `firestore.googleapis.com`; the
 * streaming one was confirmed on `RunQuery` (docs/design/protobuf.md).
 * This module imports `wasi:http`, so it compiles for component targets
 * only. Generated code does not import it.
 */

import { AsyncIterator, AsyncStep } from 'zena:async';
import { HashMap } from 'zena:collections';
import { Option, newByteArray, copyBytes, readULEB128_64, Result } from 'zena:core';
import { Stream, readAllBytes } from 'zena:stream';
import { ErrorCode } from 'wasi:http/types';
import { Credentials, ProtoError, ProtoTransport, ProtoView, RpcError, RpcMetadata, RpcMethod } from './runtime.zena';
import { Exchange, HeaderList, Origin, callHeaders, post } from './http.zena';

export { RpcError } from './runtime.zena';

/** `google.rpc.Code` values this module produces itself. */
let CODE_UNKNOWN = 2;
let CODE_UNIMPLEMENTED = 12;

/**
 * The `google.rpc.Code` for an HTTP status, as Google APIs map them, for
 * an error response without a `google.rpc.Status` body.
 */
export function codeForHttpStatus(status: i32): i32 {
  if (status == 400) { return 3; }
  if (status == 401) { return 16; }
  if (status == 403) { return 7; }
  if (status == 404) { return 5; }
  if (status == 409) { return 10; }
  if (status == 429) { return 8; }
  if (status == 499) { return 1; }
  if (status == 501) { return CODE_UNIMPLEMENTED; }
  if (status == 503) { return 14; }
  if (status == 504) { return 4; }
  if (status >= 500) { return 13; }
  return CODE_UNKNOWN;
}

/**
 * Calls RPCs on one server with the fallback protocol. `origin` is the
 * scheme and host, such as `https://firestore.googleapis.com`. When
 * `credentials` is given, every call asks it for an `authorization`
 * header.
 */
export final class FallbackTransport implements ProtoTransport {
  #origin: Origin;
  #credentials: Credentials | null;
  #headers: HashMap<String, String> = new HashMap<String, String>();

  new(origin: String, credentials: Credentials | null = null)
    : #origin = Origin.parse(origin), #credentials = credentials;

  /**
   * Sends `value` as header `name` on every later call, replacing any
   * earlier value: `authorization: Bearer <token>`, for one.
   */
  setHeader(name: String, value: String): void {
    this.#headers[name] = value;
  }

  async unary(method: RpcMethod, request: ProtoView, metadata: RpcMetadata): Future<ProtoView> {
    let exchange = await this.#post(method, request, metadata);
    let bytes = await readAllBytes(exchange.body);
    exchange.finish();
    let body = ProtoView.of(bytes as ByteArray);
    if (exchange.status == 200) {
      return body;
    }
    throw errorFor(exchange, body);
  }

  serverStream(method: RpcMethod, request: ProtoView, metadata: RpcMetadata): AsyncIterator<ProtoView> {
    return new StreamBodyReader(this.#post(method, request, metadata));
  }

  /** Fails with UNIMPLEMENTED: the fallback protocol has no streamed requests. */
  async clientStream(method: RpcMethod, requests: Array<ProtoView>, metadata: RpcMetadata): Future<ProtoView> {
    throw new RpcError(CODE_UNIMPLEMENTED,
        `${method.service}/${method.name}: the gRPC fallback protocol cannot stream requests`, ProtoView.empty());
  }

  async #post(method: RpcMethod, request: ProtoView, metadata: RpcMetadata): Future<Exchange> {
    let headers = await callHeaders('application/x-protobuf', this.#headers, metadata, this.#credentials);
    return await post(this.#origin, `/$rpc/${method.service}/${method.name}`, headers, request.toByteArray(), method);
  }
}

/**
 * The error a non-200 response stands for: its `google.rpc.Status` when it
 * carries one, otherwise the code for its HTTP status.
 */
function errorFor(exchange: Exchange, body: ProtoView): RpcError {
  if (exchange.contentTypeIs('application/x-protobuf')) {
    let status = try {
      new RpcError(body.int32(1), body.string(2), body)
    } catch (e) {
      null
    };
    if (status != null && status.code != 0) {
      return status;
    }
  }
  return new RpcError(
    codeForHttpStatus(exchange.status),
    `the server answered with HTTP ${exchange.status}`,
    ProtoView.empty()
  );
}

/**
 * The responses in `body`, an encoded `google.rpc.StreamBody` arriving as a
 * stream, each available as soon as its bytes are; the final status throws
 * an `RpcError` unless it is OK. `FallbackTransport.serverStream` reads
 * response bodies with it.
 */
export function readStreamBody(body: Stream<u8>): AsyncIterator<ProtoView> {
  return new StreamBodyReader(Future.of(Exchange.of(200, new GrowableArray<(String, FixedArray<u8>)>(), body,
      Future.of<Result<Option<HeaderList>, ErrorCode>>((true, none)))));
}

/** The field numbers of `google.rpc.StreamBody`. */
let STREAM_MESSAGE = 1;
let STREAM_STATUS = 2;

/**
 * Reads the responses out of a `google.rpc.StreamBody` as its bytes
 * arrive. A response is handed out once all of its bytes are buffered; the
 * final status ends the stream, and throws an `RpcError` unless it is OK.
 */
final class StreamBodyReader implements AsyncIterator<ProtoView> {
  #pending: Future<Exchange>;
  var #exchange: Exchange | null = null;
  /** Bytes received and not yet consumed: `#buffer[#start..#end]`. */
  var #buffer: ByteArray = newByteArray(4096);
  var #start: i32 = 0;
  var #end: i32 = 0;
  #chunk: FixedArray<u8> = new FixedArray<u8>(4096, 0);
  /** Set once the status arrived and was OK: nothing more is handed out. */
  var #finished: boolean = false;

  new(pending: Future<Exchange>) : #pending = pending;

  next(): AsyncStep<ProtoView> {
    if (this.#finished) {
      return (0, _, _);
    }
    if (this.#exchange != null) {
      let message = this.#take();
      if (message != null) {
        return (1, message, _);
      }
      if (this.#finished) {
        return (0, _, _);
      }
    }
    return (2, _, this.#pull());
  }

  /** The next response, reading more of the body until one is complete. */
  async #pull(): Future<Option<ProtoView>> {
    if (this.#exchange == null) {
      this.#exchange = await this.#pending;
    }
    let exchange = this.#exchange as Exchange;
    while (true) {
      let message = this.#take();
      if (message != null) {
        return some(message);
      }
      if (this.#finished) {
        return none;
      }
      let n = await exchange.body.read(this.#chunk);
      if (n == 0) {
        exchange.finish();
        this.#finished = true;
        if (this.#start != this.#end) {
          throw new ProtoError('the response stream ended inside a message');
        }
        if (exchange.status != 200) {
          throw errorFor(exchange, ProtoView.empty());
        }
        return none;
      }
      this.#append(n);
    }
    return none;
  }

  /**
   * The next complete response in the buffer, or null when the buffer
   * holds none. Consumes keep-alive padding and the status on the way;
   * a status that is not OK throws.
   */
  #take(): ProtoView | null {
    while (this.#start < this.#end) {
      let (tag, afterTag) = readULEB128_64(this.#buffer, this.#start, this.#end);
      if (afterTag < 0) {
        return this.#incomplete();
      }
      let (length, afterLength) = readULEB128_64(this.#buffer, afterTag, this.#end);
      if (afterLength < 0) {
        return this.#incomplete();
      }
      if ((tag & 7) != 2) {
        throw new ProtoError('a response stream field is not length-delimited');
      }
      if (length > ((this.#end - afterLength) as i64)) {
        return null;
      }
      let valueEnd = afterLength + (length as i32);
      let field = (tag >>> 3) as i32;
      let value = newByteArray(valueEnd - afterLength);
      copyBytes(value, 0, this.#buffer, afterLength, valueEnd - afterLength);
      this.#start = valueEnd;
      if (field == STREAM_MESSAGE) {
        return ProtoView.of(value);
      }
      if (field == STREAM_STATUS) {
        let status = ProtoView.of(value);
        this.#finished = true;
        if (status.int32(1) != 0) {
          throw new RpcError(status.int32(1), status.string(2), status);
        }
        return null;
      }
    }
    return null;
  }

  /** Null for a varint cut off by the end of the buffer; throws for one that cannot end. */
  #incomplete(): ProtoView | null {
    if (this.#end - this.#start >= 20) {
      throw new ProtoError('a response stream field has a malformed header');
    }
    return null;
  }

  /** Appends the first `n` bytes of `#chunk` to the buffer, compacting or growing it. */
  #append(n: i32): void {
    let live = this.#end - this.#start;
    if (this.#end + n > this.#buffer.length) {
      var size = this.#buffer.length;
      while (live + n > size) {
        size = size * 2;
      }
      let next = if (size == this.#buffer.length) this.#buffer else newByteArray(size);
      copyBytes(next, 0, this.#buffer, this.#start, live);
      this.#buffer = next;
      this.#start = 0;
      this.#end = live;
    }
    for (var i = 0; i < n; i += 1) {
      this.#buffer[this.#end + i] = this.#chunk[i];
    }
    this.#end += n;
  }
}
none
VersionPublishedDependenciesApproved by
0.0.1-preview.1 2026-11-19 none