protobuf 0.0.1-preview.1

Protocol Buffers and gRPC

Readme Files Dependencies Versions

protobuf#

A Protocol Buffers (.proto) code generator, wire-format runtime, and gRPC / gRPC-fallback transports for Zena.

Installation#

zena pkg add @zena/protobuf

Generating Code#

protobuf/generator compiles .proto schemas into Zena source code with no I/O of its own—you supply the entry file paths and a callback that reads imported .proto files:

import { generate } from 'protobuf/generator';

let source = await generate({
  files: ['service.proto'],
  read: async (path) => readProtoFile(path),
});

Options (GenerateOptions)#

  • files: Array<String> — entry .proto files to generate from.
  • read: (path: String) => Future<String | null> — returns the .proto text for path, or null if not found.
  • roots: Array<String> (optional) — generate only the named messages, enums, or services (by full name) and the types they transitively reference. When omitted or empty, everything declared in files is generated.
  • runtimeImport: String (optional) — module specifier that generated code imports runtime helpers from (defaults to 'protobuf/runtime').

Using Generated Code#

For each message M, the generator emits a view class M (which reads an encoded message in place without eagerly allocating child objects) and a record type MInit, which M.of encodes:

import { HashMap } from 'zena:collections';

// Decoding wire bytes
let doc = Document.decode(bytes);
let title = doc.fields.getOr('title', Value.of({}));
let text = match (title.valueType) {
  case ValueStringValue {value}: value
  case _: ''
};

// Encoding a message
let fields = new HashMap<String, Value>();
fields['title'] = Value.of({stringValue: 'Hello'});
let encoded = Document.of({name, fields}).protoView.toByteArray();

A message-typed field accepts the message's class, so nested messages are built inside out with M.of, and a message read from one response can be passed into another message unchanged without re-encoding.

Services and Transports#

For each service in a .proto file, the generator emits a client class over ProtoTransport (protobuf/runtime). Two transports speak over wasi:http, both supporting unary and server-streaming RPCs (and GrpcTransport also supporting client-streaming RPCs):

  • protobuf/grpc (GrpcTransport): standard gRPC over HTTP/2 (https:// via ALPN or cleartext HTTP/2 for http://).
  • protobuf/fallback (FallbackTransport): Google's gRPC HTTP/1.1 $rpc fallback protocol (binary protobuf posted to /$rpc/<service>/<method>), usable on hosts whose outgoing HTTP is HTTP/1.1 only.
import { GrpcTransport } from 'protobuf/grpc';

let transport = new GrpcTransport('https://example.googleapis.com', credentials);
let client = new MyServiceClient(transport);
let response = await client.getItem({name: 'items/1'});

credentials is any Credentials (protobuf/runtime), which supplies the authorization header before each call. Generated clients also populate the x-goog-request-params routing header from google.api.http and google.api.routing options (AIP-4222). Failed calls throw RpcError (protobuf/runtime) carrying the google.rpc.Code.

Libraries#

  • protobuf/runtime — Wire-format views and writers (ProtoView, ProtoWriter, ProtoList, ProtoMap, ProtoBytes, ProtoStream), ProtoTransport, Credentials, RpcMetadata, RpcMethod, and RpcError.
  • protobuf/grpc — GrpcTransport over HTTP/2.
  • protobuf/fallback — FallbackTransport over Google's HTTP/1.1 $rpc fallback protocol.
  • protobuf/generator — generate and GenerateOptions for turning .proto files into Zena source.
  • protobuf/parser and protobuf/lexer — .proto syntax tree and tokenizer (parseProto, Lexer).
  • protobuf/schema and protobuf/emitter — Schema resolution and Zena code emission.
  • protobuf/text — Protobuf text format helpers.

License#

MIT

15.6 KiB Raw
/**
 * Loads `.proto` files with their imports, resolves type references, and
 * computes which declarations a set of roots reaches.
 *
 * Nothing here does I/O: files are read through a function the caller
 * passes in, which returns null for a file it cannot find. A missing
 * import is an error only if a reached declaration refers to a type it
 * would have declared, so a file can import `google/api/annotations.proto`
 * for its options without that file being present.
 */

import {Future} from 'zena:async';
import {HashMap, HashSet} from 'zena:collections';
import {joinStrings} from './text.zena';
import {
  EnumDecl,
  FieldDecl,
  FieldLabel,
  MessageDecl,
  MethodDecl,
  OptionDecl,
  OptionIdent,
  OptionList,
  OptionValue,
  ProtoFile,
  ServiceDecl,
  parseProto
} from './parser.zena';

/** A schema that does not resolve: an unknown type, a missing file, a bad map key. */
export class SchemaError extends Error {
  new(message: String) : super(message);
}

/** The scalar types, by their `.proto` names. */
export enum Scalar {
  Double,
  Float,
  Int32,
  Int64,
  Uint32,
  Uint64,
  Sint32,
  Sint64,
  Fixed32,
  Fixed64,
  Sfixed32,
  Sfixed64,
  Bool,
  String,
  Bytes
}

/** The scalar a `.proto` type name denotes, or -1 when it names a message or enum. */
export function scalarNamed(name: String): i32 {
  if (name == 'double') { return Scalar.Double as i32; }
  if (name == 'float') { return Scalar.Float as i32; }
  if (name == 'int32') { return Scalar.Int32 as i32; }
  if (name == 'int64') { return Scalar.Int64 as i32; }
  if (name == 'uint32') { return Scalar.Uint32 as i32; }
  if (name == 'uint64') { return Scalar.Uint64 as i32; }
  if (name == 'sint32') { return Scalar.Sint32 as i32; }
  if (name == 'sint64') { return Scalar.Sint64 as i32; }
  if (name == 'fixed32') { return Scalar.Fixed32 as i32; }
  if (name == 'fixed64') { return Scalar.Fixed64 as i32; }
  if (name == 'sfixed32') { return Scalar.Sfixed32 as i32; }
  if (name == 'sfixed64') { return Scalar.Sfixed64 as i32; }
  if (name == 'bool') { return Scalar.Bool as i32; }
  if (name == 'string') { return Scalar.String as i32; }
  if (name == 'bytes') { return Scalar.Bytes as i32; }
  return -1;
}

/** A message declaration with its place in the schema. */
export final class MessageInfo {
  decl: MessageDecl;
  /** The fully qualified name, without a leading dot: `google.firestore.v1.Document`. */
  fullName: String;
  file: ProtoFile;
  /** The fields, resolved, in declaration order. */
  fields: GrowableArray<FieldInfo> = new GrowableArray<FieldInfo>();
  /** Whether the generator emits this message. */
  var reached: boolean = false;
  /** The Zena name of the generated class. */
  var zenaName: String = '';
  new(this.decl, this.fullName, this.file);
}

/** An enum declaration with its place in the schema. */
export final class EnumInfo {
  decl: EnumDecl;
  fullName: String;
  file: ProtoFile;
  var reached: boolean = false;
  var zenaName: String = '';
  new(this.decl, this.fullName, this.file);
}

/** A service declaration with its methods' types resolved. */
export final class ServiceInfo {
  decl: ServiceDecl;
  fullName: String;
  file: ProtoFile;
  methods: GrowableArray<MethodInfo> = new GrowableArray<MethodInfo>();
  new(this.decl, this.fullName, this.file);
}

/** One RPC, with its request and response messages. */
export final class MethodInfo {
  decl: MethodDecl;
  input: MessageInfo;
  output: MessageInfo;
  new(this.decl, this.input, this.output);
}

/**
 * What a field holds: a scalar, a message or an enum. A map field's
 * value type is described here and its key in `FieldInfo.mapKey`.
 */
export final class FieldType {
  /** A `Scalar` value, or -1 for a message or enum. */
  scalar: i32;
  message: MessageInfo | null;
  enumInfo: EnumInfo | null;
  new(this.scalar, this.message, this.enumInfo);
}

/** A field with its type resolved and its options interpreted. */
export final class FieldInfo {
  decl: FieldDecl;
  type: FieldType;
  /** For a map field, the key's `Scalar`; otherwise -1. */
  mapKey: i32;
  /** Whether reading the field can tell an absent value from a default one. */
  explicitPresence: boolean;
  /** Marked `(google.api.field_behavior) = REQUIRED`: the server rejects a request without it. */
  required: boolean;
  /** Marked `(google.api.field_behavior) = OUTPUT_ONLY`: the server sets it and ignores it in requests. */
  outputOnly: boolean;
  /** Written as one packed run, for a repeated numeric field. */
  packed: boolean;
  new(
    this.decl,
    this.type,
    this.mapKey,
    this.explicitPresence,
    this.required,
    this.outputOnly,
    this.packed
  );

  isRepeated: boolean {
    get { return this.decl.label == FieldLabel.Repeated; }
  }

  isMap: boolean {
    get { return this.mapKey >= 0; }
  }
}

/** Every file loaded, and every declaration in them by full name. */
export final class Schema {
  files: GrowableArray<ProtoFile> = new GrowableArray<ProtoFile>();
  /** The files named by the caller, as opposed to their imports. */
  rootFiles: GrowableArray<ProtoFile> = new GrowableArray<ProtoFile>();
  messages: HashMap<String, MessageInfo> = new HashMap<String, MessageInfo>();
  enums: HashMap<String, EnumInfo> = new HashMap<String, EnumInfo>();
  /** Messages and enums in the order they were declared, across files. */
  messageOrder: GrowableArray<MessageInfo> = new GrowableArray<MessageInfo>();
  enumOrder: GrowableArray<EnumInfo> = new GrowableArray<EnumInfo>();
  services: GrowableArray<ServiceInfo> = new GrowableArray<ServiceInfo>();
  /** Imports that could not be read, as `importer: path`. */
  missing: GrowableArray<String> = new GrowableArray<String>();
}

/**
 * Loads `paths` and everything they import, and resolves every type
 * reference in them. `read` returns a file's source, or null.
 */
export async function loadSchema(
  paths: Array<String>,
  read: (path: String) => Future<String | null>
): Future<Schema> {
  let schema = new Schema();
  let loaded = new HashSet<String>();
  /** For each queued import, the file that imported it first; roots have none. */
  let importers = new HashMap<String, String>();
  let queue = new GrowableArray<String>();
  for (let path in paths) {
    queue.push(path);
  }
  var next = 0;
  while (next < queue.length) {
    let path = queue[next];
    let isRoot = next < paths.length;
    next += 1;
    if (loaded.has(path)) {
      continue;
    }
    loaded.add(path);
    let source = await read(path);
    if (source == null) {
      if (isRoot) {
        throw new SchemaError(`cannot read ${path}`);
      }
      schema.missing.push(`${importers.getOr(path, '?')}: ${path}`);
      continue;
    }
    let file = parseProto(path, source);
    schema.files.push(file);
    if (isRoot) {
      schema.rootFiles.push(file);
    }
    for (let imp in file.imports) {
      if (!loaded.has(imp.path) && !importers.has(imp.path)) {
        importers[imp.path] = path;
        queue.push(imp.path);
      }
    }
  }
  for (let file in schema.files) {
    let prefix = file.package;
    for (let message in file.messages) {
      indexMessage(schema, file, message, prefix);
    }
    for (let e in file.enums) {
      indexEnum(schema, file, e, prefix);
    }
  }
  for (let message in schema.messageOrder) {
    resolveFields(schema, message);
  }
  for (let file in schema.files) {
    for (let service in file.services) {
      let fullName = qualify(file.package, service.name);
      let info = new ServiceInfo(service, fullName, file);
      for (let method in service.methods) {
        let input = resolveMessage(schema, method.inputType, file.package, file, method.name);
        let output = resolveMessage(schema, method.outputType, file.package, file, method.name);
        info.methods.push(new MethodInfo(method, input, output));
      }
      schema.services.push(info);
    }
  }
  return schema;
}

function qualify(scope: String, name: String): String {
  return if (scope.length == 0) name else `${scope}.${name}`;
}

function indexMessage(schema: Schema, file: ProtoFile, message: MessageDecl, scope: String): void {
  let fullName = qualify(scope, message.name);
  let info = new MessageInfo(message, fullName, file);
  schema.messages[fullName] = info;
  schema.messageOrder.push(info);
  for (let nested in message.messages) {
    indexMessage(schema, file, nested, fullName);
  }
  for (let e in message.enums) {
    indexEnum(schema, file, e, fullName);
  }
}

function indexEnum(schema: Schema, file: ProtoFile, e: EnumDecl, scope: String): void {
  let fullName = qualify(scope, e.name);
  let info = new EnumInfo(e, fullName, file);
  schema.enums[fullName] = info;
  schema.enumOrder.push(info);
}

/** Whether an option list sets `(google.api.field_behavior)` to `behavior`. */
function hasFieldBehavior(options: GrowableArray<OptionDecl>, behavior: String): boolean {
  for (let option in options) {
    if (option.name != '(google.api.field_behavior)') {
      continue;
    }
    if (optionNames(option.value, behavior)) {
      return true;
    }
  }
  return false;
}

/** Whether `value` is the identifier `name`, or a list containing it. */
function optionNames(value: OptionValue, name: String): boolean {
  if (value is OptionIdent) {
    return value.name == name;
  }
  if (value is OptionList) {
    for (let item in value.values) {
      if (optionNames(item, name)) {
        return true;
      }
    }
  }
  return false;
}

/** The identifier an option is set to, or null when the option is absent or not an identifier. */
function optionIdent(options: GrowableArray<OptionDecl>, name: String): String | null {
  for (let option in options) {
    if (option.name == name && option.value is OptionIdent) {
      return (option.value as OptionIdent).name;
    }
  }
  return null;
}

function isPackable(scalar: i32) {
  return scalar >= 0 && scalar != (Scalar.String as i32) && scalar != (Scalar.Bytes as i32);
}

function resolveFields(schema: Schema, message: MessageInfo): void {
  let file = message.file;
  for (let field in message.decl.fields) {
    let where = `${message.fullName}.${field.name}`;
    let type = resolveFieldType(schema, field.typeName, message.fullName, file, where);
    var mapKey = -1;
    if (field.mapKey != null) {
      mapKey = scalarNamed(field.mapKey as String);
      if (mapKey < 0 || mapKey == (Scalar.Double as i32) || mapKey == (Scalar.Float as i32) ||
          mapKey == (Scalar.Bytes as i32)) {
        throw new SchemaError(`${file.path}:${field.line}: ${where} has an invalid map key type`);
      }
    }
    let label = field.label;
    let explicitPresence = label == FieldLabel.Optional || field.oneofIndex >= 0 ||
      (label == FieldLabel.None && file.syntax == 'proto2');
    let packedOption = optionIdent(field.options, 'packed');
    let numeric = isPackable(type.scalar) || type.enumInfo != null;
    let packed = label == FieldLabel.Repeated && mapKey < 0 && numeric &&
      (if (packedOption == null) file.syntax == 'proto3' else packedOption == 'true');
    message.fields.push(new FieldInfo(
      field,
      type,
      mapKey,
      explicitPresence,
      field.oneofIndex < 0 && label != FieldLabel.Repeated &&
        hasFieldBehavior(field.options, 'REQUIRED'),
      hasFieldBehavior(field.options, 'OUTPUT_ONLY'),
      packed
    ));
  }
}

function resolveFieldType(
  schema: Schema,
  name: String,
  scope: String,
  file: ProtoFile,
  where: String
): FieldType {
  let scalar = scalarNamed(name);
  if (scalar >= 0) {
    return new FieldType(scalar, null, null);
  }
  let fullName = lookup(schema, name, scope);
  if (fullName == null) {
    throw unknownType(schema, name, file, where);
  }
  let found = fullName;
  if (let (true, message) = schema.messages.get(found)) {
    return new FieldType(-1, message, null);
  }
  if (let (true, e) = schema.enums.get(found)) {
    return new FieldType(-1, null, e);
  }
  throw unknownType(schema, name, file, where);
}

function resolveMessage(
  schema: Schema,
  name: String,
  scope: String,
  file: ProtoFile,
  where: String
): MessageInfo {
  let fullName = lookup(schema, name, scope);
  if (fullName != null) {
    if (let (true, message) = schema.messages.get(fullName)) {
      return message;
    }
  }
  throw unknownType(schema, name, file, where);
}

function unknownType(schema: Schema, name: String, file: ProtoFile, where: String): SchemaError {
  var message = `${file.path}: ${where} refers to unknown type ${name}`;
  if (schema.missing.length > 0) {
    message = `${message} (imports not found: ${joinStrings(schema.missing, ', ')})`;
  }
  return new SchemaError(message);
}

/**
 * The full name `name` refers to from inside `scope`, or null. A leading
 * dot makes the name absolute. Otherwise the innermost enclosing scope
 * that declares it wins, as in C++ name lookup: from scope `a.b.C`, the
 * name `D.E` is tried as `a.b.C.D.E`, `a.b.D.E`, `a.D.E` and `D.E`.
 */
function lookup(schema: Schema, name: String, scope: String): String | null {
  if (name.startsWith('.')) {
    let absolute = name.sliceBytes(1, name.length);
    return if (declares(schema, absolute)) absolute else null;
  }
  var current = scope;
  while (true) {
    let candidate = qualify(current, name);
    if (declares(schema, candidate)) {
      return candidate;
    }
    if (current.length == 0) {
      return null;
    }
    current = parentScope(current);
  }
  return null;
}

function declares(schema: Schema, fullName: String): boolean {
  return schema.messages.has(fullName) || schema.enums.has(fullName);
}

/** `a.b.c` without its last component; '' for a single component. */
function parentScope(scope: String): String {
  var i = scope.length - 1;
  while (i >= 0) {
    if (scope.getByteAt(i) == 0x2E) {
      return scope.sliceBytes(0, i);
    }
    i -= 1;
  }
  return '';
}

/**
 * Marks what the generator emits. With no `roots`, that is every service,
 * message and enum declared in the root files, nested ones included. With
 * `roots`, it is the messages, enums and services with those full names.
 * Either way, every message and enum a marked declaration refers to is
 * marked too.
 */
export function markReached(schema: Schema, roots: Array<String>): GrowableArray<ServiceInfo> {
  let services = new GrowableArray<ServiceInfo>();
  let work = new GrowableArray<MessageInfo>();
  if (roots.length == 0) {
    let rootPaths = new HashSet<String>();
    for (let file in schema.rootFiles) {
      rootPaths.add(file.path);
    }
    for (let message in schema.messageOrder) {
      if (rootPaths.has(message.file.path)) {
        work.push(message);
      }
    }
    for (let e in schema.enumOrder) {
      if (rootPaths.has(e.file.path)) {
        e.reached = true;
      }
    }
    for (let service in schema.services) {
      if (rootPaths.has(service.file.path)) {
        services.push(service);
      }
    }
  } else {
    for (let root in roots) {
      var found = false;
      if (let (true, message) = schema.messages.get(root)) {
        work.push(message);
        found = true;
      }
      if (let (true, e) = schema.enums.get(root)) {
        e.reached = true;
        found = true;
      }
      for (let service in schema.services) {
        if (service.fullName == root) {
          services.push(service);
          found = true;
        }
      }
      if (!found) {
        throw new SchemaError(`no message, enum or service is named ${root}`);
      }
    }
  }
  for (let service in services) {
    for (let method in service.methods) {
      work.push(method.input);
      work.push(method.output);
    }
  }
  while (work.length > 0) {
    let message = work.pop();
    if (message.reached) {
      continue;
    }
    message.reached = true;
    for (let field in message.fields) {
      if (field.type.message != null) {
        work.push(field.type.message as MessageInfo);
      }
      if (field.type.enumInfo != null) {
        (field.type.enumInfo as EnumInfo).reached = true;
      }
    }
  }
  return services;
}
none
VersionPublishedDependenciesApproved by
0.0.1-preview.1 2026-11-19 none