protobuf 0.0.1-preview.1
Protocol Buffers and gRPC
protobuf#
A Protocol Buffers (.proto) code generator, wire-format runtime, and gRPC / gRPC-fallback transports for Zena.
Installation#
zena pkg add @zena/protobuf
Generating Code#
protobuf/generator compiles .proto schemas into Zena source code with no I/O of its own—you supply the entry file paths and a callback that reads imported .proto files:
import { generate } from 'protobuf/generator';
let source = await generate({
files: ['service.proto'],
read: async (path) => readProtoFile(path),
});
Options (GenerateOptions)#
files:Array<String>— entry.protofiles to generate from.read:(path: String) => Future<String | null>— returns the.prototext forpath, ornullif not found.roots:Array<String>(optional) — generate only the named messages, enums, or services (by full name) and the types they transitively reference. When omitted or empty, everything declared infilesis generated.runtimeImport:String(optional) — module specifier that generated code imports runtime helpers from (defaults to'protobuf/runtime').
Using Generated Code#
For each message M, the generator emits a view class M (which reads an encoded message in place without eagerly allocating child objects) and a record type MInit, which M.of encodes:
import { HashMap } from 'zena:collections';
// Decoding wire bytes
let doc = Document.decode(bytes);
let title = doc.fields.getOr('title', Value.of({}));
let text = match (title.valueType) {
case ValueStringValue {value}: value
case _: ''
};
// Encoding a message
let fields = new HashMap<String, Value>();
fields['title'] = Value.of({stringValue: 'Hello'});
let encoded = Document.of({name, fields}).protoView.toByteArray();
A message-typed field accepts the message's class, so nested messages are built inside out with M.of, and a message read from one response can be passed into another message unchanged without re-encoding.
Services and Transports#
For each service in a .proto file, the generator emits a client class over ProtoTransport (protobuf/runtime). Two transports speak over wasi:http, both supporting unary and server-streaming RPCs (and GrpcTransport also supporting client-streaming RPCs):
protobuf/grpc(GrpcTransport): standard gRPC over HTTP/2 (https://via ALPN or cleartext HTTP/2 forhttp://).protobuf/fallback(FallbackTransport): Google's gRPC HTTP/1.1$rpcfallback protocol (binary protobuf posted to/$rpc/<service>/<method>), usable on hosts whose outgoing HTTP is HTTP/1.1 only.
import { GrpcTransport } from 'protobuf/grpc';
let transport = new GrpcTransport('https://example.googleapis.com', credentials);
let client = new MyServiceClient(transport);
let response = await client.getItem({name: 'items/1'});
credentials is any Credentials (protobuf/runtime), which supplies the authorization header before each call. Generated clients also populate the x-goog-request-params routing header from google.api.http and google.api.routing options (AIP-4222). Failed calls throw RpcError (protobuf/runtime) carrying the google.rpc.Code.
Libraries#
protobuf/runtime— Wire-format views and writers (ProtoView,ProtoWriter,ProtoList,ProtoMap,ProtoBytes,ProtoStream),ProtoTransport,Credentials,RpcMetadata,RpcMethod, andRpcError.protobuf/grpc—GrpcTransportover HTTP/2.protobuf/fallback—FallbackTransportover Google's HTTP/1.1$rpcfallback protocol.protobuf/generator—generateandGenerateOptionsfor turning.protofiles into Zena source.protobuf/parserandprotobuf/lexer—.protosyntax tree and tokenizer (parseProto,Lexer).protobuf/schemaandprotobuf/emitter— Schema resolution and Zena code emission.protobuf/text— Protobuf text format helpers.
License#
MIT
/**
* A `ProtoTransport` that speaks Google's gRPC fallback protocol
* (https://googleapis.github.io/HowToRPC.html) over `wasi:http`, for hosts
* whose outgoing HTTP is HTTP/1.1 only, such as `wasmtime serve`.
*
* A call is a `POST` of the binary request to
* `<origin>/$rpc/<service>/<method>` with `Content-Type:
* application/x-protobuf`.
*
* - A unary call answers 200 with the binary response, or another status
* with a binary `google.rpc.Status`.
* - A server-streaming call answers with a `google.rpc.StreamBody`: each
* response is one occurrence of field 1, the final status is field 2,
* and field 15 carries keep-alive padding. The body is read as it
* arrives, so each response is available as soon as its bytes are.
*
* Firestore answers both shapes on `firestore.googleapis.com`; the
* streaming one was confirmed on `RunQuery` (docs/design/protobuf.md).
* This module imports `wasi:http`, so it compiles for component targets
* only. Generated code does not import it.
*/
import { AsyncIterator, AsyncStep } from 'zena:async';
import { HashMap } from 'zena:collections';
import { Option, newByteArray, copyBytes, readULEB128_64, Result } from 'zena:core';
import { Stream, readAllBytes } from 'zena:stream';
import { ErrorCode } from 'wasi:http/types';
import { Credentials, ProtoError, ProtoTransport, ProtoView, RpcError, RpcMetadata, RpcMethod } from './runtime.zena';
import { Exchange, HeaderList, Origin, callHeaders, post } from './http.zena';
export { RpcError } from './runtime.zena';
/** `google.rpc.Code` values this module produces itself. */
let CODE_UNKNOWN = 2;
let CODE_UNIMPLEMENTED = 12;
/**
* The `google.rpc.Code` for an HTTP status, as Google APIs map them, for
* an error response without a `google.rpc.Status` body.
*/
export function codeForHttpStatus(status: i32): i32 {
if (status == 400) { return 3; }
if (status == 401) { return 16; }
if (status == 403) { return 7; }
if (status == 404) { return 5; }
if (status == 409) { return 10; }
if (status == 429) { return 8; }
if (status == 499) { return 1; }
if (status == 501) { return CODE_UNIMPLEMENTED; }
if (status == 503) { return 14; }
if (status == 504) { return 4; }
if (status >= 500) { return 13; }
return CODE_UNKNOWN;
}
/**
* Calls RPCs on one server with the fallback protocol. `origin` is the
* scheme and host, such as `https://firestore.googleapis.com`. When
* `credentials` is given, every call asks it for an `authorization`
* header.
*/
export final class FallbackTransport implements ProtoTransport {
#origin: Origin;
#credentials: Credentials | null;
#headers: HashMap<String, String> = new HashMap<String, String>();
new(origin: String, credentials: Credentials | null = null)
: #origin = Origin.parse(origin), #credentials = credentials;
/**
* Sends `value` as header `name` on every later call, replacing any
* earlier value: `authorization: Bearer <token>`, for one.
*/
setHeader(name: String, value: String): void {
this.#headers[name] = value;
}
async unary(method: RpcMethod, request: ProtoView, metadata: RpcMetadata): Future<ProtoView> {
let exchange = await this.#post(method, request, metadata);
let bytes = await readAllBytes(exchange.body);
exchange.finish();
let body = ProtoView.of(bytes as ByteArray);
if (exchange.status == 200) {
return body;
}
throw errorFor(exchange, body);
}
serverStream(method: RpcMethod, request: ProtoView, metadata: RpcMetadata): AsyncIterator<ProtoView> {
return new StreamBodyReader(this.#post(method, request, metadata));
}
/** Fails with UNIMPLEMENTED: the fallback protocol has no streamed requests. */
async clientStream(method: RpcMethod, requests: Array<ProtoView>, metadata: RpcMetadata): Future<ProtoView> {
throw new RpcError(CODE_UNIMPLEMENTED,
`${method.service}/${method.name}: the gRPC fallback protocol cannot stream requests`, ProtoView.empty());
}
async #post(method: RpcMethod, request: ProtoView, metadata: RpcMetadata): Future<Exchange> {
let headers = await callHeaders('application/x-protobuf', this.#headers, metadata, this.#credentials);
return await post(this.#origin, `/$rpc/${method.service}/${method.name}`, headers, request.toByteArray(), method);
}
}
/**
* The error a non-200 response stands for: its `google.rpc.Status` when it
* carries one, otherwise the code for its HTTP status.
*/
function errorFor(exchange: Exchange, body: ProtoView): RpcError {
if (exchange.contentTypeIs('application/x-protobuf')) {
let status = try {
new RpcError(body.int32(1), body.string(2), body)
} catch (e) {
null
};
if (status != null && status.code != 0) {
return status;
}
}
return new RpcError(
codeForHttpStatus(exchange.status),
`the server answered with HTTP ${exchange.status}`,
ProtoView.empty()
);
}
/**
* The responses in `body`, an encoded `google.rpc.StreamBody` arriving as a
* stream, each available as soon as its bytes are; the final status throws
* an `RpcError` unless it is OK. `FallbackTransport.serverStream` reads
* response bodies with it.
*/
export function readStreamBody(body: Stream<u8>): AsyncIterator<ProtoView> {
return new StreamBodyReader(Future.of(Exchange.of(200, new GrowableArray<(String, FixedArray<u8>)>(), body,
Future.of<Result<Option<HeaderList>, ErrorCode>>((true, none)))));
}
/** The field numbers of `google.rpc.StreamBody`. */
let STREAM_MESSAGE = 1;
let STREAM_STATUS = 2;
/**
* Reads the responses out of a `google.rpc.StreamBody` as its bytes
* arrive. A response is handed out once all of its bytes are buffered; the
* final status ends the stream, and throws an `RpcError` unless it is OK.
*/
final class StreamBodyReader implements AsyncIterator<ProtoView> {
#pending: Future<Exchange>;
var #exchange: Exchange | null = null;
/** Bytes received and not yet consumed: `#buffer[#start..#end]`. */
var #buffer: ByteArray = newByteArray(4096);
var #start: i32 = 0;
var #end: i32 = 0;
#chunk: FixedArray<u8> = new FixedArray<u8>(4096, 0);
/** Set once the status arrived and was OK: nothing more is handed out. */
var #finished: boolean = false;
new(pending: Future<Exchange>) : #pending = pending;
next(): AsyncStep<ProtoView> {
if (this.#finished) {
return (0, _, _);
}
if (this.#exchange != null) {
let message = this.#take();
if (message != null) {
return (1, message, _);
}
if (this.#finished) {
return (0, _, _);
}
}
return (2, _, this.#pull());
}
/** The next response, reading more of the body until one is complete. */
async #pull(): Future<Option<ProtoView>> {
if (this.#exchange == null) {
this.#exchange = await this.#pending;
}
let exchange = this.#exchange as Exchange;
while (true) {
let message = this.#take();
if (message != null) {
return some(message);
}
if (this.#finished) {
return none;
}
let n = await exchange.body.read(this.#chunk);
if (n == 0) {
exchange.finish();
this.#finished = true;
if (this.#start != this.#end) {
throw new ProtoError('the response stream ended inside a message');
}
if (exchange.status != 200) {
throw errorFor(exchange, ProtoView.empty());
}
return none;
}
this.#append(n);
}
return none;
}
/**
* The next complete response in the buffer, or null when the buffer
* holds none. Consumes keep-alive padding and the status on the way;
* a status that is not OK throws.
*/
#take(): ProtoView | null {
while (this.#start < this.#end) {
let (tag, afterTag) = readULEB128_64(this.#buffer, this.#start, this.#end);
if (afterTag < 0) {
return this.#incomplete();
}
let (length, afterLength) = readULEB128_64(this.#buffer, afterTag, this.#end);
if (afterLength < 0) {
return this.#incomplete();
}
if ((tag & 7) != 2) {
throw new ProtoError('a response stream field is not length-delimited');
}
if (length > ((this.#end - afterLength) as i64)) {
return null;
}
let valueEnd = afterLength + (length as i32);
let field = (tag >>> 3) as i32;
let value = newByteArray(valueEnd - afterLength);
copyBytes(value, 0, this.#buffer, afterLength, valueEnd - afterLength);
this.#start = valueEnd;
if (field == STREAM_MESSAGE) {
return ProtoView.of(value);
}
if (field == STREAM_STATUS) {
let status = ProtoView.of(value);
this.#finished = true;
if (status.int32(1) != 0) {
throw new RpcError(status.int32(1), status.string(2), status);
}
return null;
}
}
return null;
}
/** Null for a varint cut off by the end of the buffer; throws for one that cannot end. */
#incomplete(): ProtoView | null {
if (this.#end - this.#start >= 20) {
throw new ProtoError('a response stream field has a malformed header');
}
return null;
}
/** Appends the first `n` bytes of `#chunk` to the buffer, compacting or growing it. */
#append(n: i32): void {
let live = this.#end - this.#start;
if (this.#end + n > this.#buffer.length) {
var size = this.#buffer.length;
while (live + n > size) {
size = size * 2;
}
let next = if (size == this.#buffer.length) this.#buffer else newByteArray(size);
copyBytes(next, 0, this.#buffer, this.#start, live);
this.#buffer = next;
this.#start = 0;
this.#end = live;
}
for (var i = 0; i < n; i += 1) {
this.#buffer[this.#end + i] = this.#chunk[i];
}
this.#end += n;
}
}
| Version | Published | Dependencies | Approved by |
|---|---|---|---|
| 0.0.1-preview.1 | 2026-11-19 | none |
|