workflow 0.0.1-preview.1
Incremental workflow engine: steps, dependencies, fingerprints, caching, and services
workflow#
An incremental workflow execution engine for Zena: dependency graphs, content fingerprints, cache keys with early cutoff, fine-grained per-output invalidation, and on-demand service lifecycles.
workflow performs no I/O of its own. Hosts supply leaf fingerprints, a digest function, a step cache, and step callbacks; the engine schedules work in dependency order with bounded parallelism and returns the updated state.
Installation#
zena pkg add zena-lang:workflow
Example#
import { NullCache } from 'zena-lang:workflow/cache';
import { Engine, StepSource, Succeeded } from 'zena-lang:workflow/engine';
import { Fingerprint } from 'zena-lang:workflow/fingerprint';
import { Graph } from 'zena-lang:workflow/graph';
import { OutputEntry, OutputManifest } from 'zena-lang:workflow/manifest';
import { EngineState } from 'zena-lang:workflow/state';
import { LeafInput, StepKey, StepOutputs, TaskStep } from 'zena-lang:workflow/step';
let compile = new TaskStep(
new StepKey('compile'),
new Fingerprint('cmd:compile-v1'),
[new LeafInput('src/main.zena')],
async (ctx) => {
return new OutputManifest([
new OutputEntry('out/main.wasm', new Fingerprint('wasm-hash-1')),
]);
},
);
let bundle = new TaskStep(
new StepKey('bundle'),
new Fingerprint('cmd:bundle-v1'),
[new StepOutputs(new StepKey('compile'))],
async (ctx) => {
return new OutputManifest([
new OutputEntry('dist/app.wasm', new Fingerprint('bundle-hash-1')),
]);
},
);
let graph = new Graph([compile, bundle]);
let leaves = {'src/main.zena' => new Fingerprint('src-hash-1')};
let engine = new Engine((data) => new Fingerprint( sha256(data) ), new NullCache());
// First run executes both steps in topological order.
let first = await engine.run(graph, leaves, EngineState.empty());
// Second run with identical inputs and prior state is a no-op (StepSource.Fresh).
let second = await engine.run(graph, leaves, first.state);
Core Concepts#
- Steps and inputs (
workflow/step):TaskStep: runs to completion and returns anOutputManifest.ServiceStep: starts on demand only when a dependent step actually executes, and stops when its last dependent settles (or stays pinned across watch runs in aServicePool).LeafInput(name): a named external input resolved from theleavesmap passed toEngine.run.LeafSet(name, resolve): a lazy input set resolved after upstream dependencies finish (for example, expanding globs over generated files).StepOutputs(step): a dependency on another step whose output manifest (or service identity) enters this step's cache key. Because dependents key on upstream output fingerprints rather than upstream input fingerprints, an upstream step that reruns and produces identical outputs leaves downstream dependents fresh (early cutoff).StepOrder(step): an ordering-only dependency (like Wireit's"cascade": false) that waits for the upstream step without folding its outputs into the dependent's cache key.
- Validated graph (
workflow/graph):Graphchecks that step keys are unique, all referenced steps exist, no step lists duplicate inputs, and the dependency graph is acyclic (reporting the cycle path inWorkflowError), producing a deterministic topological order. - Caching and freshness (
workflow/engine,workflow/cache,workflow/state):- If a step's
CacheKeymatches theStepRecordin the previousEngineState(and optionalRunOptions.verifyOutputssucceeds), the step isStepSource.Freshwithout consulting the cache or starting services. - Otherwise
StepCache.getis checked (StepSource.Cache), falling back to executing the step callback (StepSource.Ran) under the run's parallelism limit.
- If a step's
- Incremental per-output deltas (
workflow/delta,workflow/manifest):- Each
OutputEntryin anOutputManifestcan recorddependsOn(the namespaced inputsleaf:...,step:...it was derived from). - When a step reruns with the same
configfingerprint,RunContextprovidesdelta(changed,added,removed),previous(OutputManifest), andstaleOutputs. The engine verifies before caching that any output carried over as non-stale kept its previous fingerprint.
- Each
Libraries#
workflow/engine—Engine,RunOptions,RunResult,StepOutcome(Succeeded,Failed,Skipped),StepSource,ServicePool, andPinnedService.workflow/step—Step,TaskStep,ServiceStep,StepKey,InputRef(LeafInput,LeafSet,StepOutputs,StepOrder),RunContext,ServiceContext, andServiceHandle.workflow/graph—Graphvalidation and deterministic topological sorting.workflow/manifest—OutputManifestandOutputEntry.workflow/state—EngineStateandStepRecordfor persisting prior-run state across runs.workflow/cache—StepCacheinterface andNullCache.workflow/fingerprint—Fingerprint,CacheKey, andDigest.workflow/canonical— Canonical length-prefixed encodings (cacheKeyFor,manifestFingerprint,ResolvedInput,compareStrings,sortBy).workflow/delta—InputDelta,computeDelta, andstaleOutputNames.workflow/errors—WorkflowError.
License#
MIT
27.4 KiB
Raw
import {Completer} from 'zena:async';
import {HashSet, OrderedHashMap} from 'zena:collections';
import {StepCache} from './cache.zena';
import {ResolvedInput, cacheKeyFor, manifestFingerprint} from './canonical.zena';
import {InputDelta, computeDelta, staleOutputNames} from './delta.zena';
import {WorkflowError} from './errors.zena';
import {Digest, Fingerprint} from './fingerprint.zena';
import {Graph} from './graph.zena';
import {Semaphore} from './internal/semaphore.zena';
import {OutputManifest} from './manifest.zena';
import {EngineState, StepRecord} from './state.zena';
import {
LeafInput,
LeafSet,
RunContext,
ServiceContext,
ServiceHandle,
ServiceStep,
Step,
StepKey,
StepOrder,
StepOutputs,
TaskStep,
} from './step.zena';
/** How a succeeded step's manifest was obtained. */
export enum StepSource {
/** The previous run recorded the same cache key; nothing was done. */
Fresh,
/** Restored from the cache. */
Cache,
/** The run callback executed. */
Ran,
}
/** The result of one step in a run. */
export sealed class StepOutcome {
case Succeeded, Failed, Skipped
}
/**
* For a task, the manifest it produced. For a service, an empty
* manifest with source `Ran` if it started (and stopped cleanly) or
* `Fresh` if no dependent ever demanded it.
*/
export final class Succeeded(manifest: OutputManifest, source: StepSource)
extends StepOutcome
export final class Failed(error: Error) extends StepOutcome
/** Not run because a dependency failed or was skipped. */
export final class Skipped(reason: String) extends StepOutcome
/** A running service held in a `ServicePool` across watch iterations. */
export class PinnedService(identity: Fingerprint, handle: ServiceHandle)
/**
* Holds running services across multiple `Engine.run` invocations (such
* as watch iterations) so services whose identity fingerprint has not
* changed stay alive instead of restarting on every rebuild.
*
* Services are recorded in startup order and stopped in reverse order
* by `stopAll()`.
*/
export class ServicePool {
#entries = new OrderedHashMap<String, PinnedService>();
size: i32 {
get {
return this.#entries.size;
}
}
has(key: String): boolean {
return this.#entries.has(key);
}
get(key: String): PinnedService? {
if (let (true, entry) = this.#entries.get(key)) {
return entry;
}
return null;
}
set(key: String, entry: PinnedService): void {
this.#entries[key] = entry;
}
take(key: String): PinnedService? {
if (let (true, entry) = this.#entries.get(key)) {
this.#entries.delete(key);
return entry;
}
return null;
}
keys(): Array<String> {
let result = new GrowableArray<String>();
this.#entries.forEach((key, _) => {
result.push(key);
});
return result;
}
/** Stops all remaining pinned services in reverse startup order. */
async stopAll(): Future<void> {
let allKeys = this.keys();
var i = allKeys.length - 1;
while (i >= 0) {
let key = allKeys[i];
let entry = this.take(key);
if (entry != null) {
try {
await entry.handle.stop();
} catch (_) {}
}
i -= 1;
}
}
}
/** Options for one engine run. */
export class RunOptions {
/** Maximum number of steps in their cache-or-run section at once. */
parallelism: i32;
/**
* Bypass freshness and `StepCache.get` so every reachable task step
* executes its run callback, still storing new outputs in the cache
* and returned `EngineState`.
*/
force: boolean;
/**
* Optional host check run before declaring a key-matched step `Fresh`
* (for example, checking that recorded output files still exist on
* disk). Returning `false` falls through to cache restore or execution
* while keeping `prior` for `RunContext`.
*/
verifyOutputs: ((step: StepKey, manifest: OutputManifest) => Future<boolean>)?;
/**
* Optional pool of persistent services across watch runs. When set,
* services whose identity fingerprint matches a live handle in the
* pool are reused without restarting, and newly started services stay
* alive in the pool when the run ends.
*/
servicePool: ServicePool?;
/**
* Optional callback invoked immediately before a task step executes
* its `run` callback or a service step executes its `start` callback.
*/
onStepStart: ((step: StepKey) => void)?;
/**
* Optional callback invoked as soon as a step's outcome is recorded.
*/
onStepFinish: ((step: StepKey, outcome: StepOutcome) => void)?;
new(
parallelism: i32 = 8,
force: boolean = false,
verifyOutputs: ((step: StepKey, manifest: OutputManifest) => Future<boolean>)? = null,
servicePool: ServicePool? = null,
onStepStart: ((step: StepKey) => void)? = null,
onStepFinish: ((step: StepKey, outcome: StepOutcome) => void)? = null,
) : parallelism = parallelism,
force = force,
verifyOutputs = verifyOutputs,
servicePool = servicePool,
onStepStart = onStepStart,
onStepFinish = onStepFinish;
}
/** The result of an engine run. */
export class RunResult {
#outcomes: HashMap<String, StepOutcome>;
/** State for the host to persist and pass to the next run. */
state: EngineState;
new(outcomes: HashMap<String, StepOutcome>, this.state)
: #outcomes = outcomes;
outcomeOf(key: StepKey): StepOutcome? {
if (let (true, outcome) = this.#outcomes.get(key.value)) {
return outcome;
}
return null;
}
allSucceeded: boolean {
get {
var ok = true;
this.#outcomes.forEach((key, outcome) => {
if (!(outcome is Succeeded)) {
ok = false;
}
});
return ok;
}
}
}
/**
* Engine-side bookkeeping for one service across a run.
*
* Holds the step base-typed and the start callback copied out, so that
* after classification no code touches ServiceStep members: in a graph
* with no services ServiceStep is never instantiated, and calls to its
* accessors are miscompiled rather than pruned today
* ("function index < 0 ... isReached=false").
*/
class ServiceRuntime {
step: Step;
start: (ctx: ServiceContext) => Future<ServiceHandle>;
/** Direct dependents that have not yet released the service. */
var pending: i32 = 0;
var started = false;
var stopping = false;
var reusedFromPool = false;
var wasInPool = false;
var identityFp: Fingerprint? = null;
var failure: Error? = null;
var handle: ServiceHandle? = null;
/** Settles when the identity fingerprint (or a failure) is known. */
identityDone = new Completer<void>();
/** Settles when the service is ready, or failed to start. */
ready = new Completer<void>();
stopped = new Completer<void>();
new(this.step, this.start);
}
/** Mutable bookkeeping shared across one run's step tasks. */
class RunState {
leaves: Map<String, Fingerprint>;
previous: EngineState;
next: EngineState;
outcomes: HashMap<String, StepOutcome>;
done: HashMap<String, Completer<void>>;
semaphore: Semaphore;
options: RunOptions;
taskList = new GrowableArray<TaskStep>();
serviceList = new GrowableArray<ServiceRuntime>();
services = new HashMap<String, ServiceRuntime>();
var remaining: i32 = 0;
allDone = new Completer<void>();
new(
this.leaves,
this.previous,
this.next,
this.outcomes,
this.done,
this.semaphore,
this.options,
);
}
/**
* Drives a graph of steps. For each task, in dependency order: if the
* previous run recorded the same cache key the task is fresh and
* nothing happens; otherwise the cache is consulted; otherwise the run
* callback executes and the result is recorded. A failed step fails
* alone; its dependents are skipped and everything else continues.
*
* A service contributes an identity fingerprint (a digest of its
* config and resolved inputs) to its dependents' cache keys. It starts
* only when a dependent actually executes, dependents wait for its
* ready signal, and it stops when its last dependent settles — so a
* fully fresh build never starts a service.
*/
export class Engine {
#digest: Digest;
#cache: StepCache;
new(digest: Digest, cache: StepCache) : #digest = digest, #cache = cache;
/**
* Runs the graph against a snapshot of leaf fingerprints. `previous`
* is the state returned by the last run, or `EngineState.empty()`.
*/
async run(
graph: Graph,
leaves: Map<String, Fingerprint>,
previous: EngineState,
options: RunOptions? = null,
): Future<RunResult> {
let opts = options ?? new RunOptions();
let rs = new RunState(
leaves,
previous,
new EngineState(),
new HashMap<String, StepOutcome>(),
new HashMap<String, Completer<void>>(),
new Semaphore(opts.parallelism),
opts,
);
this.#classify(graph, rs);
let pool = opts.servicePool;
if (pool != null) {
let poolKeys = pool.keys();
var i = poolKeys.length - 1;
while (i >= 0) {
let key = poolKeys[i];
if (!rs.services.has(key)) {
let evicted = pool.take(key);
if (evicted != null) {
try {
await evicted.handle.stop();
} catch (_) {}
}
}
i -= 1;
}
}
rs.remaining = rs.taskList.length;
// A countdown completer joins the step tasks. TaskGroup.spawn and
// Future.all over collected Future<void>s both miscompile today
// (a spawn closure calling methods on `this` loses members on a
// specialized Engine; GrowableArray<Future<void>> breaks ZIR array
// lowering), and calling the async method starts it just the same.
for (let svc in rs.serviceList) {
this.#serviceMain(svc, rs);
}
for (let task in rs.taskList) {
this.#runStep(task, rs);
}
if (rs.taskList.length == 0) {
rs.allDone.complete();
}
await rs.allDone.future;
// Services whose dependents were all services that never started
// (or that had no dependents at all) are stopped here; everything
// else was stopped by its last release.
for (let svc in rs.serviceList) {
this.#maybeStop(svc, rs);
}
for (let svc in rs.serviceList) {
await svc.stopped.future;
}
return new RunResult(rs.outcomes, rs.next);
}
// Narrowing a sealed class (`is` or `match`) inside an async method
// miscompiles today ("member not found" on a specialized Engine), so
// step and input discrimination lives in the synchronous helpers below
// and the async methods stay narrowing-free.
/** Splits the graph into tasks and services and counts service holds. */
#classify(graph: Graph, rs: RunState): void {
for (let step in graph.steps) {
match (step) {
case TaskStep as task: {
rs.taskList.push(task);
rs.done[task.key.value] = new Completer<void>();
}
case ServiceStep as svc: {
let runtime = new ServiceRuntime(svc, svc.start);
rs.serviceList.push(runtime);
rs.services[svc.key.value] = runtime;
}
}
}
for (let step in graph.steps) {
for (let input in step.inputs) {
match (input) {
case StepOutputs as dep: {
if (let (true, svc) = rs.services.get(dep.step.value)) {
svc.pending += 1;
}
}
case StepOrder as dep: {
if (let (true, svc) = rs.services.get(dep.step.value)) {
svc.pending += 1;
}
}
case _: {}
}
}
}
}
/** The done-signals of the step's task dependencies. */
#depCompleters(step: Step, rs: RunState): GrowableArray<Completer<void>> {
let completers = new GrowableArray<Completer<void>>();
for (let input in step.inputs) {
match (input) {
case StepOutputs as dep: {
if (let (true, completer) = rs.done.get(dep.step.value)) {
completers.push(completer);
}
}
case StepOrder as dep: {
if (let (true, completer) = rs.done.get(dep.step.value)) {
completers.push(completer);
}
}
case _: {}
}
}
return completers;
}
/** The step's service dependencies. */
#serviceDeps(step: Step, rs: RunState): GrowableArray<ServiceRuntime> {
let deps = new GrowableArray<ServiceRuntime>();
for (let input in step.inputs) {
match (input) {
case StepOutputs as dep: {
if (let (true, svc) = rs.services.get(dep.step.value)) {
deps.push(svc);
}
}
case StepOrder as dep: {
if (let (true, svc) = rs.services.get(dep.step.value)) {
deps.push(svc);
}
}
case _: {}
}
}
return deps;
}
/** Callbacks for the step's lazy leaf sets, extracted without async narrowing. */
#leafSetResolvers(
step: Step,
): GrowableArray<() => Future<Array<(String, Fingerprint)>>> {
let resolvers = new GrowableArray<() => Future<Array<(String, Fingerprint)>>>();
for (let input in step.inputs) {
match (input) {
case LeafSet as leafSet: {
resolvers.push(leafSet.resolve);
}
case _: {}
}
}
return resolvers;
}
/**
* Resolves the step's static inputs to fingerprints, filling `resolved`.
* Returns an outcome to short-circuit with (unknown leaf, unsucceeded
* dependency), or null to proceed. Leaf, task, and service namespaces
* are disjoint in the encoding, so names cannot collide across kinds.
*/
#resolveInputs(
step: Step,
rs: RunState,
resolved: GrowableArray<ResolvedInput>,
): StepOutcome? {
for (let input in step.inputs) {
match (input) {
case LeafInput as leaf: {
if (let (true, fingerprint) = rs.leaves.get(leaf.name)) {
resolved.push(new ResolvedInput('leaf:' + leaf.name, fingerprint));
} else {
return new Failed(new WorkflowError(
`step "${step.key.value}" reads unknown leaf input "${leaf.name}"`,
));
}
}
case LeafSet: {}
case StepOutputs as dep: {
let depKey = dep.step.value;
if (let (true, svc) = rs.services.get(depKey)) {
let identity = svc.identityFp;
if (identity == null) {
return new Skipped(`service "${depKey}" could not be resolved`);
}
resolved.push(new ResolvedInput('service:' + depKey, identity));
} else {
var manifest: OutputManifest? = null;
if (let (true, depOutcome) = rs.outcomes.get(depKey)) {
if (depOutcome is Succeeded) {
manifest = depOutcome.manifest;
}
}
if (manifest == null) {
return new Skipped(`dependency "${depKey}" did not succeed`);
}
resolved.push(new ResolvedInput(
'step:' + depKey,
manifestFingerprint(manifest, this.#digest),
));
}
}
case StepOrder as dep: {
let depKey = dep.step.value;
if (let (true, svc) = rs.services.get(depKey)) {
if (svc.identityFp == null) {
return new Skipped(`service "${depKey}" could not be resolved`);
}
} else {
var succeeded = false;
if (let (true, depOutcome) = rs.outcomes.get(depKey)) {
if (depOutcome is Succeeded) {
succeeded = true;
}
}
if (!succeeded) {
return new Skipped(`dependency "${depKey}" did not succeed`);
}
}
}
}
}
return null;
}
/** Releases the step's service holds, stopping services that hit zero. */
#releaseServices(step: Step, rs: RunState): void {
for (let svc in this.#serviceDeps(step, rs)) {
svc.pending -= 1;
this.#maybeStop(svc, rs);
}
}
#maybeStop(svc: ServiceRuntime, rs: RunState): void {
if (svc.stopping || svc.pending > 0) {
return;
}
svc.stopping = true;
this.#stopService(svc, rs);
}
async #runStep(step: TaskStep, rs: RunState): Future<void> {
let outcome = await this.#outcomeOf(step, rs);
rs.outcomes[step.key.value] = outcome;
let onFinish = rs.options.onStepFinish;
if (onFinish != null) {
onFinish(step.key, outcome);
}
if (let (true, completer) = rs.done.get(step.key.value)) {
completer.complete();
}
this.#releaseServices(step, rs);
rs.remaining -= 1;
if (rs.remaining == 0) {
rs.allDone.complete();
}
}
async #outcomeOf(step: TaskStep, rs: RunState): Future<StepOutcome> {
try {
return await this.#execute(step, rs);
} catch (e) {
return new Failed(e);
}
}
async #execute(step: TaskStep, rs: RunState): Future<StepOutcome> {
// Wait for every dependency to settle: tasks complete, services
// resolve their identity (they are not started for this).
for (let completer in this.#depCompleters(step, rs)) {
await completer.future;
}
for (let svc in this.#serviceDeps(step, rs)) {
await svc.identityDone.future;
}
let resolved = new GrowableArray<ResolvedInput>();
let shortCircuit = this.#resolveInputs(step, rs, resolved);
if (shortCircuit != null) {
return shortCircuit;
}
for (let resolve in this.#leafSetResolvers(step)) {
for (let (name, fp) in await resolve()) {
resolved.push(new ResolvedInput('leaf:' + name, fp));
}
}
let cacheKey = cacheKeyFor(step.config, resolved, this.#digest);
// Fresh: the previous run recorded the same key, and any host output
// check confirms the recorded outputs are still intact.
let prior = rs.previous.recordOf(step.key);
if (!rs.options.force && prior != null && prior.cacheKey.value == cacheKey.value) {
let verify = rs.options.verifyOutputs;
let intact = if (verify != null) await verify(step.key, prior.manifest) else true;
if (intact) {
rs.next.setRecord(step.key, prior);
return new Succeeded(prior.manifest, StepSource.Fresh);
}
}
await rs.semaphore.acquire();
try {
if (!rs.options.force) {
let cached = await this.#cache.get(step.key, cacheKey);
if (cached != null) {
rs.next.setRecord(
step.key,
new StepRecord(step.config, cacheKey, cached, resolved),
);
return new Succeeded(cached, StepSource.Cache);
}
}
// Only now is the step certain to execute, so only now are its
// services demanded.
let serviceFailure = await this.#acquireServices(step, rs);
if (serviceFailure != null) {
return serviceFailure;
}
let onStart = rs.options.onStepStart;
if (onStart != null) {
onStart(step.key);
}
let ctx = this.#runContext(step, prior, resolved);
let manifest = await step.run(ctx);
// Fingerprinting validates duplicate output names loudly here, at
// the producer, instead of at the first dependent; the carry-over
// check catches a step whose output changed although its records
// claim no input it depends on did. Both run before the result
// can reach the cache.
manifestFingerprint(manifest, this.#digest);
this.#verifyCarryOver(step, ctx, manifest);
await this.#cache.put(step.key, cacheKey, manifest);
rs.next.setRecord(
step.key,
new StepRecord(step.config, cacheKey, manifest, resolved),
);
return new Succeeded(manifest, StepSource.Ran);
} finally {
rs.semaphore.release();
}
}
/**
* Builds the run callback's context. The delta is only offered when
* the previous record exists and was made with the same config
* fingerprint; otherwise the step runs from scratch.
*/
#runContext(
step: TaskStep,
prior: StepRecord?,
resolved: GrowableArray<ResolvedInput>,
): RunContext {
if (prior == null || prior.config.value != step.config.value) {
return new RunContext(null, null, null, resolved);
}
let delta = computeDelta(prior.inputs, resolved);
return new RunContext(
delta,
prior.manifest,
staleOutputNames(prior.manifest, delta),
resolved,
);
}
/**
* Verifies that every previous output the delta left valid came back
* unchanged. A violation means the step's dependency records are
* wrong, and caching the result would corrupt later runs.
*/
#verifyCarryOver(
step: TaskStep,
ctx: RunContext,
manifest: OutputManifest,
): void {
let staleList = ctx.staleOutputs;
let previous = ctx.previous;
if (staleList == null || previous == null) {
return;
}
let stale = new HashSet<String>();
for (let name in staleList) {
stale.add(name);
}
let produced = new HashMap<String, Fingerprint>();
for (let entry in manifest.outputs) {
produced[entry.name] = entry.fingerprint;
}
for (let entry in previous.outputs) {
if (stale.has(entry.name)) {
continue;
}
if (let (true, fp) = produced.get(entry.name)) {
if (fp.value != entry.fingerprint.value) {
throw new WorkflowError(
`step "${step.key.value}" changed output "${entry.name}" although its dependency records report it unaffected`,
);
}
} else {
throw new WorkflowError(
`step "${step.key.value}" dropped output "${entry.name}" although its dependency records report it unaffected`,
);
}
}
}
/**
* Computes a service's identity fingerprint once its dependencies
* have settled. When a `ServicePool` is active, reuses a matching
* live handle or evicts stale ones in reverse dependency order.
*/
async #serviceMain(svc: ServiceRuntime, rs: RunState): Future<void> {
for (let completer in this.#depCompleters(svc.step, rs)) {
await completer.future;
}
for (let dep in this.#serviceDeps(svc.step, rs)) {
await dep.identityDone.future;
}
let resolved = new GrowableArray<ResolvedInput>();
let shortCircuit = this.#resolveInputs(svc.step, rs, resolved);
if (shortCircuit == null) {
try {
for (let resolve in this.#leafSetResolvers(svc.step)) {
for (let (name, fp) in await resolve()) {
resolved.push(new ResolvedInput('leaf:' + name, fp));
}
}
svc.identityFp = new Fingerprint(
cacheKeyFor(svc.step.config, resolved, this.#digest).value,
);
} catch (e) {
svc.failure = e;
}
} else {
svc.failure = new WorkflowError(
`service "${svc.step.key.value}" has unresolved inputs`,
);
}
let pool = rs.options.servicePool;
if (pool != null) {
let existing = pool.get(svc.step.key.value);
if (existing != null) {
let newFp = svc.identityFp;
if (
!rs.options.force &&
svc.failure == null &&
newFp != null &&
existing.identity.value == newFp.value &&
existing.handle.isAlive() &&
this.#allServiceDepsReused(svc, rs)
) {
svc.reusedFromPool = true;
svc.started = true;
svc.handle = existing.handle;
svc.ready.complete();
} else {
await this.#evictFromPool(svc, rs);
}
}
if (
svc.failure == null &&
!svc.started &&
(svc.pending == 0 || svc.wasInPool)
) {
svc.started = true;
this.#startService(svc, rs);
}
}
svc.identityDone.complete();
}
/** Whether every service that `svc` depends on was reused from the pool. */
#allServiceDepsReused(svc: ServiceRuntime, rs: RunState): boolean {
for (let dep in this.#serviceDeps(svc.step, rs)) {
if (!dep.reusedFromPool) {
return false;
}
}
return true;
}
/** Services that directly depend on `target`. */
#serviceConsumers(
target: ServiceRuntime,
rs: RunState,
): GrowableArray<ServiceRuntime> {
let consumers = new GrowableArray<ServiceRuntime>();
let targetKey = target.step.key.value;
for (let candidate in rs.serviceList) {
for (let dep in this.#serviceDeps(candidate.step, rs)) {
if (dep.step.key.value == targetKey) {
consumers.push(candidate);
break;
}
}
}
return consumers;
}
/**
* Stops and removes `svc` from the service pool, after recursively
* evicting any pooled services that depend on it so children stop
* before parents.
*/
async #evictFromPool(svc: ServiceRuntime, rs: RunState): Future<void> {
let pool = rs.options.servicePool;
if (pool == null) {
return;
}
for (let consumer in this.#serviceConsumers(svc, rs)) {
await this.#evictFromPool(consumer, rs);
}
let evicted = pool.take(svc.step.key.value);
if (evicted != null) {
svc.wasInPool = true;
try {
await evicted.handle.stop();
} catch (_) {}
}
}
/**
* Ensures every service the step depends on is started and ready.
* Returns a failure outcome if one could not start, else null.
*/
async #acquireServices(step: Step, rs: RunState): Future<StepOutcome?> {
for (let svc in this.#serviceDeps(step, rs)) {
if (!svc.started) {
svc.started = true;
this.#startService(svc, rs);
}
await svc.ready.future;
let failure = svc.failure;
if (failure != null) {
return new Failed(new WorkflowError(
`service "${svc.step.key.value}" failed to start: ` + failure.message,
));
}
}
return null;
}
async #startService(svc: ServiceRuntime, rs: RunState): Future<void> {
// A service's own service dependencies must be ready first.
let depFailure = await this.#acquireServices(svc.step, rs);
if (depFailure != null) {
svc.failure = new WorkflowError(
`service "${svc.step.key.value}" could not start its dependencies`,
);
svc.ready.complete();
return;
}
try {
let onStart = rs.options.onStepStart;
if (onStart != null) {
onStart(svc.step.key);
}
let handle = await svc.start(new ServiceContext());
svc.handle = handle;
let pool = rs.options.servicePool;
let identity = svc.identityFp;
if (pool != null && identity != null) {
pool.set(svc.step.key.value, new PinnedService(identity, handle));
}
} catch (e) {
svc.failure = e;
}
svc.ready.complete();
}
async #stopService(svc: ServiceRuntime, rs: RunState): Future<void> {
await svc.identityDone.future;
if (svc.started) {
await svc.ready.future;
}
let handle = svc.handle;
let pool = rs.options.servicePool;
if (pool != null) {
if (handle != null && (svc.failure != null || !handle.isAlive())) {
pool.take(svc.step.key.value);
try {
await handle.stop();
} catch (e) {
if (svc.failure == null) {
svc.failure = e;
}
}
}
} else if (handle != null) {
try {
await handle.stop();
} catch (e) {
if (svc.failure == null) {
svc.failure = e;
}
}
}
let failure = svc.failure;
let outcome: StepOutcome = if (failure != null)
new Failed(failure)
else
new Succeeded(
OutputManifest.empty(),
if (svc.started && !svc.reusedFromPool) StepSource.Ran else StepSource.Fresh,
);
rs.outcomes[svc.step.key.value] = outcome;
let onFinish = rs.options.onStepFinish;
if (onFinish != null) {
onFinish(svc.step.key, outcome);
}
// A stopped service no longer holds its own service dependencies.
this.#releaseServices(svc.step, rs);
svc.stopped.complete();
}
}
| Version | Published | Dependencies | Approved by |
|---|---|---|---|
| 0.0.1-preview.1 | 2026-11-19 | none |
|