protobuf 0.0.1-preview.1

Protocol Buffers and gRPC

Readme Files Dependencies Versions

protobuf#

A Protocol Buffers (.proto) code generator, wire-format runtime, and gRPC / gRPC-fallback transports for Zena.

Installation#

zena pkg add @zena/protobuf

Generating Code#

protobuf/generator compiles .proto schemas into Zena source code with no I/O of its own—you supply the entry file paths and a callback that reads imported .proto files:

import { generate } from 'protobuf/generator';

let source = await generate({
  files: ['service.proto'],
  read: async (path) => readProtoFile(path),
});

Options (GenerateOptions)#

  • files: Array<String> — entry .proto files to generate from.
  • read: (path: String) => Future<String | null> — returns the .proto text for path, or null if not found.
  • roots: Array<String> (optional) — generate only the named messages, enums, or services (by full name) and the types they transitively reference. When omitted or empty, everything declared in files is generated.
  • runtimeImport: String (optional) — module specifier that generated code imports runtime helpers from (defaults to 'protobuf/runtime').

Using Generated Code#

For each message M, the generator emits a view class M (which reads an encoded message in place without eagerly allocating child objects) and a record type MInit, which M.of encodes:

import { HashMap } from 'zena:collections';

// Decoding wire bytes
let doc = Document.decode(bytes);
let title = doc.fields.getOr('title', Value.of({}));
let text = match (title.valueType) {
  case ValueStringValue {value}: value
  case _: ''
};

// Encoding a message
let fields = new HashMap<String, Value>();
fields['title'] = Value.of({stringValue: 'Hello'});
let encoded = Document.of({name, fields}).protoView.toByteArray();

A message-typed field accepts the message's class, so nested messages are built inside out with M.of, and a message read from one response can be passed into another message unchanged without re-encoding.

Services and Transports#

For each service in a .proto file, the generator emits a client class over ProtoTransport (protobuf/runtime). Two transports speak over wasi:http, both supporting unary and server-streaming RPCs (and GrpcTransport also supporting client-streaming RPCs):

  • protobuf/grpc (GrpcTransport): standard gRPC over HTTP/2 (https:// via ALPN or cleartext HTTP/2 for http://).
  • protobuf/fallback (FallbackTransport): Google's gRPC HTTP/1.1 $rpc fallback protocol (binary protobuf posted to /$rpc/<service>/<method>), usable on hosts whose outgoing HTTP is HTTP/1.1 only.
import { GrpcTransport } from 'protobuf/grpc';

let transport = new GrpcTransport('https://example.googleapis.com', credentials);
let client = new MyServiceClient(transport);
let response = await client.getItem({name: 'items/1'});

credentials is any Credentials (protobuf/runtime), which supplies the authorization header before each call. Generated clients also populate the x-goog-request-params routing header from google.api.http and google.api.routing options (AIP-4222). Failed calls throw RpcError (protobuf/runtime) carrying the google.rpc.Code.

Libraries#

  • protobuf/runtime — Wire-format views and writers (ProtoView, ProtoWriter, ProtoList, ProtoMap, ProtoBytes, ProtoStream), ProtoTransport, Credentials, RpcMetadata, RpcMethod, and RpcError.
  • protobuf/grpc — GrpcTransport over HTTP/2.
  • protobuf/fallback — FallbackTransport over Google's HTTP/1.1 $rpc fallback protocol.
  • protobuf/generator — generate and GenerateOptions for turning .proto files into Zena source.
  • protobuf/parser and protobuf/lexer — .proto syntax tree and tokenizer (parseProto, Lexer).
  • protobuf/schema and protobuf/emitter — Schema resolution and Zena code emission.
  • protobuf/text — Protobuf text format helpers.

License#

MIT

15.4 KiB Raw
/**
 * A `ProtoTransport` that speaks gRPC over `wasi:http`
 * (https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-HTTP2.md).
 *
 * A call is a `POST` to `<origin>/<service>/<method>` with
 * `content-type: application/grpc` and `te: trailers`. Each message, the
 * request and every response, is framed by a byte that says whether it is
 * compressed and a 4-byte big-endian length. The call's status arrives in
 * the `grpc-status`, `grpc-message` and `grpc-status-details-bin`
 * trailers, or, when a call fails before any response, in the headers of
 * a response with no body.
 *
 * gRPC needs HTTP/2 and trailers. Under `zena` and `zena-run`,
 * `https://` origins negotiate HTTP/2, and `http://` gRPC requests use
 * cleartext HTTP/2 (h2c), which is how a local server or the Firestore
 * emulator is reached. Stock `wasmtime serve` speaks HTTP/1.1 only, so
 * calls through it fail; `FallbackTransport` is the transport there.
 *
 * Requests are sent uncompressed, and no compression is offered for
 * responses. This module imports `wasi:http`, so it compiles for
 * component targets only.
 */

import { AsyncIterator, AsyncStep } from 'zena:async';
import { HashMap } from 'zena:collections';
import { Option, Some, copyBytes, newByteArray } from 'zena:core';
import { Stream } from 'zena:stream';
import { div } from 'zena:math';
import { Duration } from 'zena:time';
import { percentDecode } from 'zena:url';
import { Credentials, ProtoError, ProtoTransport, ProtoView, RpcError, RpcMetadata, RpcMethod } from './runtime.zena';
import {
  Exchange, HeaderList, Origin, Trailers, callHeaders, hasHeader, headerValue, post
} from './http.zena';

export { RpcError } from './runtime.zena';

/** `google.rpc.Code` values this module produces itself. */
let CODE_UNKNOWN = 2;
let CODE_INTERNAL = 13;
let CODE_UNAVAILABLE = 14;

/** The length of a message's frame header: a compressed flag and a 4-byte length. */
let FRAME_HEADER = 5;

/**
 * The `google.rpc.Code` for the HTTP status of a response that is not
 * gRPC, as the gRPC specification maps them (doc/http-grpc-status-mapping.md).
 */
export function codeForGrpcHttpStatus(status: i32): i32 {
  if (status == 400) { return CODE_INTERNAL; }
  if (status == 401) { return 16; }
  if (status == 403) { return 7; }
  if (status == 404) { return 12; }
  if (status == 429 || status == 502 || status == 503 || status == 504) { return CODE_UNAVAILABLE; }
  return CODE_UNKNOWN;
}

/**
 * Calls RPCs on one server with gRPC. `origin` is the scheme and host,
 * such as `https://firestore.googleapis.com` or `http://localhost:8080`.
 * When `credentials` is given, every call asks it for an `authorization`
 * header.
 */
export final class GrpcTransport implements ProtoTransport {
  #origin: Origin;
  #credentials: Credentials | null;
  #headers: HashMap<String, String> = new HashMap<String, String>();
  /** Each call's deadline after it starts, in nanoseconds, or -1 for none. */
  var #timeout: i64 = -1;

  new(origin: String, credentials: Credentials | null = null)
    : #origin = Origin.parse(origin), #credentials = credentials {
    this.#headers['te'] = 'trailers';
  }

  /**
   * Gives every later call a deadline `timeout` after it starts. The
   * server ends a call still running then with DEADLINE_EXCEEDED, from
   * the `grpc-timeout` header; and a call whose response has not started
   * by then fails with DEADLINE_EXCEEDED here, should the server not
   * answer at all.
   */
  setTimeout(timeout: Duration): void {
    let nanos = timeout as i64;
    this.#timeout = if (nanos < 0) 0 as i64 else nanos;
    this.#headers['grpc-timeout'] = grpcTimeout(this.#timeout);
  }

  /**
   * Sends `value` as header `name` on every later call, replacing any
   * earlier value: `authorization: Bearer <token>`, for one.
   */
  setHeader(name: String, value: String): void {
    this.#headers[name] = value;
  }

  async unary(method: RpcMethod, request: ProtoView, metadata: RpcMetadata): Future<ProtoView> {
    return await singleMessage(method, new GrpcResponseReader(this.#post(method, frame(request), metadata)));
  }

  serverStream(method: RpcMethod, request: ProtoView, metadata: RpcMetadata): AsyncIterator<ProtoView> {
    return new GrpcResponseReader(this.#post(method, frame(request), metadata));
  }

  /** Sends `requests` as one body of consecutive frames. */
  async clientStream(method: RpcMethod, requests: Array<ProtoView>, metadata: RpcMetadata): Future<ProtoView> {
    return await singleMessage(method, new GrpcResponseReader(this.#post(method, frames(requests), metadata)));
  }

  async #post(method: RpcMethod, body: ByteArray, metadata: RpcMetadata): Future<Exchange> {
    let headers = await callHeaders('application/grpc', this.#headers, metadata, this.#credentials);
    return await post(this.#origin, `/${method.service}/${method.name}`, headers, body, method, this.#timeout);
  }
}

/**
 * `nanos` as a `grpc-timeout` value: at most eight digits and a unit,
 * the finest unit that fits, rounded up.
 */
export function grpcTimeout(nanos: i64): String {
  let units = ['n', 'u', 'm', 'S', 'M', 'H'];
  let sizes: Array<i64> = [1, 1000, 1000000, 1000000000, 60000000000, 3600000000000];
  for (var i = 0; i < units.length; i += 1) {
    let value = div(nanos + sizes[i] - 1, sizes[i]);
    if (value <= 99999999) {
      return `${value}${units[i]}`;
    }
  }
  return '99999999H';
}

/** `message` framed for a gRPC body: uncompressed, with its length. */
export function frame(message: ProtoView): ByteArray {
  let bytes = message.toByteArray();
  let n = bytes.length;
  let out = newByteArray(FRAME_HEADER + n);
  out[0] = 0 as u8;
  out[1] = ((n >>> 24) & 0xFF) as u8;
  out[2] = ((n >>> 16) & 0xFF) as u8;
  out[3] = ((n >>> 8) & 0xFF) as u8;
  out[4] = (n & 0xFF) as u8;
  copyBytes(out, FRAME_HEADER, bytes, 0, n);
  return out;
}

/** `messages` framed one after another, for a client-streaming call's body. */
export function frames(messages: Array<ProtoView>): ByteArray {
  let parts = new GrowableArray<ByteArray>();
  var total = 0;
  for (let message in messages) {
    let part = frame(message);
    parts.push(part);
    total += part.length;
  }
  let out = newByteArray(total);
  var at = 0;
  for (let part in parts) {
    copyBytes(out, at, part, 0, part.length);
    at += part.length;
  }
  return out;
}

/**
 * The one message of a unary call's responses, read to the end so the
 * call's status is checked. No message, or more than one, is INTERNAL.
 */
export async function singleMessage(method: RpcMethod, reader: AsyncIterator<ProtoView>): Future<ProtoView> {
  let first = await nextMessage(reader);
  if (first is Some<ProtoView>) {
    if (await nextMessage(reader) is Some<ProtoView>) {
      throw new RpcError(CODE_INTERNAL, `${method.service}/${method.name}: more than one response to a unary call`, ProtoView.empty());
    }
    return first.value;
  }
  throw new RpcError(CODE_INTERNAL, `${method.service}/${method.name}: no response to a unary call`, ProtoView.empty());
}

/** The next element of `reader`, waiting for it if it is still coming. */
async function nextMessage(reader: AsyncIterator<ProtoView>): Future<Option<ProtoView>> {
  match (reader.next()) {
    case (1, let view, _): {
      return some(view);
    }
    case (2, _, let pending): {
      return await pending;
    }
    case _: {
      return none;
    }
  }
}

/**
 * The responses of a gRPC call whose response has HTTP status `status`,
 * `headers`, `body` and `trailers`, each available as soon as its bytes
 * are; the call's status ends the stream, and throws an `RpcError` unless
 * it is OK. `GrpcTransport` reads responses with it.
 */
export function readGrpcResponse(status: i32, headers: HeaderList, body: Stream<u8>, trailers: Trailers): AsyncIterator<ProtoView> {
  return new GrpcResponseReader(Future.of(Exchange.of(status, headers, body, trailers)));
}

/**
 * The error, or null for OK, that the gRPC status in `fields` stands for:
 * `grpc-status`, `grpc-message` (percent-encoded) and
 * `grpc-status-details-bin` (a base64 `google.rpc.Status`). A missing or
 * malformed `grpc-status` is UNKNOWN.
 */
export function statusError(fields: HeaderList): RpcError | null {
  let codeText = headerValue(fields, 'grpc-status');
  var code = if (codeText == null) -1 else parseCode(codeText);
  if (code == 0) {
    return null;
  }
  let message = headerValue(fields, 'grpc-message');
  var text = if (message == null) '' else percentDecode(message);
  if (code < 0) {
    code = CODE_UNKNOWN;
    if (text.length == 0) {
      text = if (codeText == null) 'the response has no grpc-status'
        else `the response has a malformed grpc-status: ${codeText}`;
    }
  }
  var details = ProtoView.empty();
  let encoded = headerValue(fields, 'grpc-status-details-bin');
  if (encoded != null) {
    let decoded = decodeBase64(encoded);
    if (decoded != null) {
      details = ProtoView.of(decoded as ByteArray);
    }
  }
  return new RpcError(code, text, details);
}

/** A non-negative decimal status code, or -1. */
function parseCode(text: String): i32 {
  if (text.length == 0 || text.length > 4) {
    return -1;
  }
  var code = 0;
  for (var i = 0; i < text.length; i += 1) {
    let b = text.getByteAt(i);
    if (b < 0x30 || b > 0x39) {
      return -1;
    }
    code = code * 10 + (b - 0x30);
  }
  return code;
}

/** The value of base64 digit `b` in either alphabet, or -1. */
function base64Digit(b: i32): i32 {
  if (b >= 0x41 && b <= 0x5A) { return b - 0x41; }
  if (b >= 0x61 && b <= 0x7A) { return b - 0x61 + 26; }
  if (b >= 0x30 && b <= 0x39) { return b - 0x30 + 52; }
  if (b == 0x2B || b == 0x2D) { return 62; }
  if (b == 0x2F || b == 0x5F) { return 63; }
  return -1;
}

/**
 * `text` decoded as base64, padded or not, in the standard or the URL
 * alphabet; null when it is malformed.
 */
export function decodeBase64(text: String): ByteArray | null {
  var end = text.length;
  while (end > 0 && text.getByteAt(end - 1) == 0x3D) {
    end -= 1;
  }
  if (end % 4 == 1) {
    return null;
  }
  let out = newByteArray((end * 3) >> 2);
  var bits = 0;
  var count = 0;
  var o = 0;
  for (var i = 0; i < end; i += 1) {
    let digit = base64Digit(text.getByteAt(i));
    if (digit < 0) {
      return null;
    }
    bits = (bits << 6) | digit;
    count += 6;
    if (count >= 8) {
      count -= 8;
      out[o] = ((bits >>> count) & 0xFF) as u8;
      o += 1;
    }
  }
  return out;
}

/**
 * Reads the messages out of a gRPC response body as its bytes arrive. A
 * message is handed out once all of its bytes are buffered. When the body
 * ends, the status in the trailers ends the stream, and throws an
 * `RpcError` unless it is OK.
 */
final class GrpcResponseReader implements AsyncIterator<ProtoView> {
  #pending: Future<Exchange>;
  var #exchange: Exchange | null = null;
  /** Bytes received and not yet consumed: `#buffer[#start..#end]`. */
  var #buffer: ByteArray = newByteArray(4096);
  var #start: i32 = 0;
  var #end: i32 = 0;
  #chunk: FixedArray<u8> = new FixedArray<u8>(4096, 0);
  /** Set once the stream has ended: nothing more is handed out. */
  var #finished: boolean = false;

  new(pending: Future<Exchange>) : #pending = pending;

  next(): AsyncStep<ProtoView> {
    if (this.#finished) {
      return (0, _, _);
    }
    if (this.#exchange != null) {
      let message = this.#take();
      if (message != null) {
        return (1, message, _);
      }
    }
    return (2, _, this.#pull());
  }

  /** The next message, reading more of the body until one is complete. */
  async #pull(): Future<Option<ProtoView>> {
    if (this.#exchange == null) {
      let exchange = await this.#pending;
      this.#exchange = exchange;
      await this.#checkHead(exchange);
      if (this.#finished) {
        return none;
      }
    }
    let exchange = this.#exchange as Exchange;
    while (true) {
      let message = this.#take();
      if (message != null) {
        return some(message);
      }
      let n = await exchange.body.read(this.#chunk);
      if (n == 0) {
        exchange.finish();
        this.#finished = true;
        if (this.#start != this.#end) {
          throw new RpcError(CODE_INTERNAL, 'the response ended inside a message', ProtoView.empty());
        }
        let error = await this.#trailerStatus(exchange);
        if (error != null) {
          throw error;
        }
        return none;
      }
      this.#append(n);
    }
    return none;
  }

  /**
   * Checks the response head. A status in the headers is a response with
   * no messages (trailers-only): it ends the stream, and throws unless
   * OK. A response that is not gRPC throws.
   */
  async #checkHead(exchange: Exchange): Future<void> {
    if (hasHeader(exchange.headers, 'grpc-status')) {
      await this.#drain(exchange);
      let error = statusError(exchange.headers);
      if (error != null) {
        throw error;
      }
      return;
    }
    if (exchange.status != 200) {
      await this.#drain(exchange);
      throw new RpcError(
        codeForGrpcHttpStatus(exchange.status),
        `the server answered with HTTP ${exchange.status}`,
        ProtoView.empty()
      );
    }
    if (!exchange.contentTypeIs('application/grpc')) {
      await this.#drain(exchange);
      let contentType = headerValue(exchange.headers, 'content-type');
      throw new RpcError(
        CODE_UNKNOWN,
        `the server answered with content-type ${contentType ?? '(none)'}`,
        ProtoView.empty()
      );
    }
  }

  /** Reads and drops the rest of the body, and ends the stream. */
  async #drain(exchange: Exchange): Future<void> {
    this.#finished = true;
    while (await exchange.body.read(this.#chunk) > 0) {
    }
    exchange.finish();
  }

  /** The error the trailers' status stands for, or null for OK. */
  async #trailerStatus(exchange: Exchange): Future<RpcError | null> {
    let trailers = await exchange.trailers;
    if (let (true, fields) = trailers) {
      if (let Some {value} = fields) {
        return statusError(value);
      }
      return new RpcError(CODE_INTERNAL, 'the response has no trailers', ProtoView.empty());
    }
    return new RpcError(CODE_UNAVAILABLE, 'the connection failed before the call\'s status arrived', ProtoView.empty());
  }

  /** The next complete message in the buffer, or null when the buffer holds none. */
  #take(): ProtoView | null {
    if (this.#end - this.#start < FRAME_HEADER) {
      return null;
    }
    let b = this.#buffer;
    let s = this.#start;
    if (b[s] != (0 as u8)) {
      throw new RpcError(CODE_INTERNAL, 'the server sent a compressed message, and no compression was offered', ProtoView.empty());
    }
    let length = ((b[s + 1] as i32) << 24) | ((b[s + 2] as i32) << 16) |
        ((b[s + 3] as i32) << 8) | (b[s + 4] as i32);
    if (length < 0) {
      throw new RpcError(CODE_INTERNAL, 'the server sent a message over 2 GiB', ProtoView.empty());
    }
    if (this.#end - s - FRAME_HEADER < length) {
      return null;
    }
    let value = newByteArray(length);
    copyBytes(value, 0, b, s + FRAME_HEADER, length);
    this.#start = s + FRAME_HEADER + length;
    return ProtoView.of(value);
  }

  /** Appends the first `n` bytes of `#chunk` to the buffer, compacting or growing it. */
  #append(n: i32): void {
    let live = this.#end - this.#start;
    if (this.#end + n > this.#buffer.length) {
      var size = this.#buffer.length;
      while (live + n > size) {
        size = size * 2;
      }
      let next = if (size == this.#buffer.length) this.#buffer else newByteArray(size);
      copyBytes(next, 0, this.#buffer, this.#start, live);
      this.#buffer = next;
      this.#start = 0;
      this.#end = live;
    }
    for (var i = 0; i < n; i += 1) {
      this.#buffer[this.#end + i] = this.#chunk[i];
    }
    this.#end += n;
  }
}
none
VersionPublishedDependenciesApproved by
0.0.1-preview.1 2026-11-19 none