import {Completer} from 'zena:async'; import {HashSet, OrderedHashMap} from 'zena:collections'; import {StepCache} from './cache.zena'; import {ResolvedInput, cacheKeyFor, manifestFingerprint} from './canonical.zena'; import {InputDelta, computeDelta, staleOutputNames} from './delta.zena'; import {WorkflowError} from './errors.zena'; import {Digest, Fingerprint} from './fingerprint.zena'; import {Graph} from './graph.zena'; import {Semaphore} from './internal/semaphore.zena'; import {OutputManifest} from './manifest.zena'; import {EngineState, StepRecord} from './state.zena'; import { LeafInput, LeafSet, RunContext, ServiceContext, ServiceHandle, ServiceStep, Step, StepKey, StepOrder, StepOutputs, TaskStep, } from './step.zena'; /** How a succeeded step's manifest was obtained. */ export enum StepSource { /** The previous run recorded the same cache key; nothing was done. */ Fresh, /** Restored from the cache. */ Cache, /** The run callback executed. */ Ran, } /** The result of one step in a run. */ export sealed class StepOutcome { case Succeeded, Failed, Skipped } /** * For a task, the manifest it produced. For a service, an empty * manifest with source `Ran` if it started (and stopped cleanly) or * `Fresh` if no dependent ever demanded it. */ export final class Succeeded(manifest: OutputManifest, source: StepSource) extends StepOutcome export final class Failed(error: Error) extends StepOutcome /** Not run because a dependency failed or was skipped. */ export final class Skipped(reason: String) extends StepOutcome /** A running service held in a `ServicePool` across watch iterations. */ export class PinnedService(identity: Fingerprint, handle: ServiceHandle) /** * Holds running services across multiple `Engine.run` invocations (such * as watch iterations) so services whose identity fingerprint has not * changed stay alive instead of restarting on every rebuild. * * Services are recorded in startup order and stopped in reverse order * by `stopAll()`. */ export class ServicePool { #entries = new OrderedHashMap(); size: i32 { get { return this.#entries.size; } } has(key: String): boolean { return this.#entries.has(key); } get(key: String): PinnedService? { if (let (true, entry) = this.#entries.get(key)) { return entry; } return null; } set(key: String, entry: PinnedService): void { this.#entries[key] = entry; } take(key: String): PinnedService? { if (let (true, entry) = this.#entries.get(key)) { this.#entries.delete(key); return entry; } return null; } keys(): Array { let result = new GrowableArray(); this.#entries.forEach((key, _) => { result.push(key); }); return result; } /** Stops all remaining pinned services in reverse startup order. */ async stopAll(): Future { let allKeys = this.keys(); var i = allKeys.length - 1; while (i >= 0) { let key = allKeys[i]; let entry = this.take(key); if (entry != null) { try { await entry.handle.stop(); } catch (_) {} } i -= 1; } } } /** Options for one engine run. */ export class RunOptions { /** Maximum number of steps in their cache-or-run section at once. */ parallelism: i32; /** * Bypass freshness and `StepCache.get` so every reachable task step * executes its run callback, still storing new outputs in the cache * and returned `EngineState`. */ force: boolean; /** * Optional host check run before declaring a key-matched step `Fresh` * (for example, checking that recorded output files still exist on * disk). Returning `false` falls through to cache restore or execution * while keeping `prior` for `RunContext`. */ verifyOutputs: ((step: StepKey, manifest: OutputManifest) => Future)?; /** * Optional pool of persistent services across watch runs. When set, * services whose identity fingerprint matches a live handle in the * pool are reused without restarting, and newly started services stay * alive in the pool when the run ends. */ servicePool: ServicePool?; /** * Optional callback invoked immediately before a task step executes * its `run` callback or a service step executes its `start` callback. */ onStepStart: ((step: StepKey) => void)?; /** * Optional callback invoked as soon as a step's outcome is recorded. */ onStepFinish: ((step: StepKey, outcome: StepOutcome) => void)?; new( parallelism: i32 = 8, force: boolean = false, verifyOutputs: ((step: StepKey, manifest: OutputManifest) => Future)? = null, servicePool: ServicePool? = null, onStepStart: ((step: StepKey) => void)? = null, onStepFinish: ((step: StepKey, outcome: StepOutcome) => void)? = null, ) : parallelism = parallelism, force = force, verifyOutputs = verifyOutputs, servicePool = servicePool, onStepStart = onStepStart, onStepFinish = onStepFinish; } /** The result of an engine run. */ export class RunResult { #outcomes: HashMap; /** State for the host to persist and pass to the next run. */ state: EngineState; new(outcomes: HashMap, this.state) : #outcomes = outcomes; outcomeOf(key: StepKey): StepOutcome? { if (let (true, outcome) = this.#outcomes.get(key.value)) { return outcome; } return null; } allSucceeded: boolean { get { var ok = true; this.#outcomes.forEach((key, outcome) => { if (!(outcome is Succeeded)) { ok = false; } }); return ok; } } } /** * Engine-side bookkeeping for one service across a run. * * Holds the step base-typed and the start callback copied out, so that * after classification no code touches ServiceStep members: in a graph * with no services ServiceStep is never instantiated, and calls to its * accessors are miscompiled rather than pruned today * ("function index < 0 ... isReached=false"). */ class ServiceRuntime { step: Step; start: (ctx: ServiceContext) => Future; /** Direct dependents that have not yet released the service. */ var pending: i32 = 0; var started = false; var stopping = false; var reusedFromPool = false; var wasInPool = false; var identityFp: Fingerprint? = null; var failure: Error? = null; var handle: ServiceHandle? = null; /** Settles when the identity fingerprint (or a failure) is known. */ identityDone = new Completer(); /** Settles when the service is ready, or failed to start. */ ready = new Completer(); stopped = new Completer(); new(this.step, this.start); } /** Mutable bookkeeping shared across one run's step tasks. */ class RunState { leaves: Map; previous: EngineState; next: EngineState; outcomes: HashMap; done: HashMap>; semaphore: Semaphore; options: RunOptions; taskList = new GrowableArray(); serviceList = new GrowableArray(); services = new HashMap(); var remaining: i32 = 0; allDone = new Completer(); new( this.leaves, this.previous, this.next, this.outcomes, this.done, this.semaphore, this.options, ); } /** * Drives a graph of steps. For each task, in dependency order: if the * previous run recorded the same cache key the task is fresh and * nothing happens; otherwise the cache is consulted; otherwise the run * callback executes and the result is recorded. A failed step fails * alone; its dependents are skipped and everything else continues. * * A service contributes an identity fingerprint (a digest of its * config and resolved inputs) to its dependents' cache keys. It starts * only when a dependent actually executes, dependents wait for its * ready signal, and it stops when its last dependent settles — so a * fully fresh build never starts a service. */ export class Engine { #digest: Digest; #cache: StepCache; new(digest: Digest, cache: StepCache) : #digest = digest, #cache = cache; /** * Runs the graph against a snapshot of leaf fingerprints. `previous` * is the state returned by the last run, or `EngineState.empty()`. */ async run( graph: Graph, leaves: Map, previous: EngineState, options: RunOptions? = null, ): Future { let opts = options ?? new RunOptions(); let rs = new RunState( leaves, previous, new EngineState(), new HashMap(), new HashMap>(), new Semaphore(opts.parallelism), opts, ); this.#classify(graph, rs); let pool = opts.servicePool; if (pool != null) { let poolKeys = pool.keys(); var i = poolKeys.length - 1; while (i >= 0) { let key = poolKeys[i]; if (!rs.services.has(key)) { let evicted = pool.take(key); if (evicted != null) { try { await evicted.handle.stop(); } catch (_) {} } } i -= 1; } } rs.remaining = rs.taskList.length; // A countdown completer joins the step tasks. TaskGroup.spawn and // Future.all over collected Futures both miscompile today // (a spawn closure calling methods on `this` loses members on a // specialized Engine; GrowableArray> breaks ZIR array // lowering), and calling the async method starts it just the same. for (let svc in rs.serviceList) { this.#serviceMain(svc, rs); } for (let task in rs.taskList) { this.#runStep(task, rs); } if (rs.taskList.length == 0) { rs.allDone.complete(); } await rs.allDone.future; // Services whose dependents were all services that never started // (or that had no dependents at all) are stopped here; everything // else was stopped by its last release. for (let svc in rs.serviceList) { this.#maybeStop(svc, rs); } for (let svc in rs.serviceList) { await svc.stopped.future; } return new RunResult(rs.outcomes, rs.next); } // Narrowing a sealed class (`is` or `match`) inside an async method // miscompiles today ("member not found" on a specialized Engine), so // step and input discrimination lives in the synchronous helpers below // and the async methods stay narrowing-free. /** Splits the graph into tasks and services and counts service holds. */ #classify(graph: Graph, rs: RunState): void { for (let step in graph.steps) { match (step) { case TaskStep as task: { rs.taskList.push(task); rs.done[task.key.value] = new Completer(); } case ServiceStep as svc: { let runtime = new ServiceRuntime(svc, svc.start); rs.serviceList.push(runtime); rs.services[svc.key.value] = runtime; } } } for (let step in graph.steps) { for (let input in step.inputs) { match (input) { case StepOutputs as dep: { if (let (true, svc) = rs.services.get(dep.step.value)) { svc.pending += 1; } } case StepOrder as dep: { if (let (true, svc) = rs.services.get(dep.step.value)) { svc.pending += 1; } } case _: {} } } } } /** The done-signals of the step's task dependencies. */ #depCompleters(step: Step, rs: RunState): GrowableArray> { let completers = new GrowableArray>(); for (let input in step.inputs) { match (input) { case StepOutputs as dep: { if (let (true, completer) = rs.done.get(dep.step.value)) { completers.push(completer); } } case StepOrder as dep: { if (let (true, completer) = rs.done.get(dep.step.value)) { completers.push(completer); } } case _: {} } } return completers; } /** The step's service dependencies. */ #serviceDeps(step: Step, rs: RunState): GrowableArray { let deps = new GrowableArray(); for (let input in step.inputs) { match (input) { case StepOutputs as dep: { if (let (true, svc) = rs.services.get(dep.step.value)) { deps.push(svc); } } case StepOrder as dep: { if (let (true, svc) = rs.services.get(dep.step.value)) { deps.push(svc); } } case _: {} } } return deps; } /** Callbacks for the step's lazy leaf sets, extracted without async narrowing. */ #leafSetResolvers( step: Step, ): GrowableArray<() => Future>> { let resolvers = new GrowableArray<() => Future>>(); for (let input in step.inputs) { match (input) { case LeafSet as leafSet: { resolvers.push(leafSet.resolve); } case _: {} } } return resolvers; } /** * Resolves the step's static inputs to fingerprints, filling `resolved`. * Returns an outcome to short-circuit with (unknown leaf, unsucceeded * dependency), or null to proceed. Leaf, task, and service namespaces * are disjoint in the encoding, so names cannot collide across kinds. */ #resolveInputs( step: Step, rs: RunState, resolved: GrowableArray, ): StepOutcome? { for (let input in step.inputs) { match (input) { case LeafInput as leaf: { if (let (true, fingerprint) = rs.leaves.get(leaf.name)) { resolved.push(new ResolvedInput('leaf:' + leaf.name, fingerprint)); } else { return new Failed(new WorkflowError( `step "${step.key.value}" reads unknown leaf input "${leaf.name}"`, )); } } case LeafSet: {} case StepOutputs as dep: { let depKey = dep.step.value; if (let (true, svc) = rs.services.get(depKey)) { let identity = svc.identityFp; if (identity == null) { return new Skipped(`service "${depKey}" could not be resolved`); } resolved.push(new ResolvedInput('service:' + depKey, identity)); } else { var manifest: OutputManifest? = null; if (let (true, depOutcome) = rs.outcomes.get(depKey)) { if (depOutcome is Succeeded) { manifest = depOutcome.manifest; } } if (manifest == null) { return new Skipped(`dependency "${depKey}" did not succeed`); } resolved.push(new ResolvedInput( 'step:' + depKey, manifestFingerprint(manifest, this.#digest), )); } } case StepOrder as dep: { let depKey = dep.step.value; if (let (true, svc) = rs.services.get(depKey)) { if (svc.identityFp == null) { return new Skipped(`service "${depKey}" could not be resolved`); } } else { var succeeded = false; if (let (true, depOutcome) = rs.outcomes.get(depKey)) { if (depOutcome is Succeeded) { succeeded = true; } } if (!succeeded) { return new Skipped(`dependency "${depKey}" did not succeed`); } } } } } return null; } /** Releases the step's service holds, stopping services that hit zero. */ #releaseServices(step: Step, rs: RunState): void { for (let svc in this.#serviceDeps(step, rs)) { svc.pending -= 1; this.#maybeStop(svc, rs); } } #maybeStop(svc: ServiceRuntime, rs: RunState): void { if (svc.stopping || svc.pending > 0) { return; } svc.stopping = true; this.#stopService(svc, rs); } async #runStep(step: TaskStep, rs: RunState): Future { let outcome = await this.#outcomeOf(step, rs); rs.outcomes[step.key.value] = outcome; let onFinish = rs.options.onStepFinish; if (onFinish != null) { onFinish(step.key, outcome); } if (let (true, completer) = rs.done.get(step.key.value)) { completer.complete(); } this.#releaseServices(step, rs); rs.remaining -= 1; if (rs.remaining == 0) { rs.allDone.complete(); } } async #outcomeOf(step: TaskStep, rs: RunState): Future { try { return await this.#execute(step, rs); } catch (e) { return new Failed(e); } } async #execute(step: TaskStep, rs: RunState): Future { // Wait for every dependency to settle: tasks complete, services // resolve their identity (they are not started for this). for (let completer in this.#depCompleters(step, rs)) { await completer.future; } for (let svc in this.#serviceDeps(step, rs)) { await svc.identityDone.future; } let resolved = new GrowableArray(); let shortCircuit = this.#resolveInputs(step, rs, resolved); if (shortCircuit != null) { return shortCircuit; } for (let resolve in this.#leafSetResolvers(step)) { for (let (name, fp) in await resolve()) { resolved.push(new ResolvedInput('leaf:' + name, fp)); } } let cacheKey = cacheKeyFor(step.config, resolved, this.#digest); // Fresh: the previous run recorded the same key, and any host output // check confirms the recorded outputs are still intact. let prior = rs.previous.recordOf(step.key); if (!rs.options.force && prior != null && prior.cacheKey.value == cacheKey.value) { let verify = rs.options.verifyOutputs; let intact = if (verify != null) await verify(step.key, prior.manifest) else true; if (intact) { rs.next.setRecord(step.key, prior); return new Succeeded(prior.manifest, StepSource.Fresh); } } await rs.semaphore.acquire(); try { if (!rs.options.force) { let cached = await this.#cache.get(step.key, cacheKey); if (cached != null) { rs.next.setRecord( step.key, new StepRecord(step.config, cacheKey, cached, resolved), ); return new Succeeded(cached, StepSource.Cache); } } // Only now is the step certain to execute, so only now are its // services demanded. let serviceFailure = await this.#acquireServices(step, rs); if (serviceFailure != null) { return serviceFailure; } let onStart = rs.options.onStepStart; if (onStart != null) { onStart(step.key); } let ctx = this.#runContext(step, prior, resolved); let manifest = await step.run(ctx); // Fingerprinting validates duplicate output names loudly here, at // the producer, instead of at the first dependent; the carry-over // check catches a step whose output changed although its records // claim no input it depends on did. Both run before the result // can reach the cache. manifestFingerprint(manifest, this.#digest); this.#verifyCarryOver(step, ctx, manifest); await this.#cache.put(step.key, cacheKey, manifest); rs.next.setRecord( step.key, new StepRecord(step.config, cacheKey, manifest, resolved), ); return new Succeeded(manifest, StepSource.Ran); } finally { rs.semaphore.release(); } } /** * Builds the run callback's context. The delta is only offered when * the previous record exists and was made with the same config * fingerprint; otherwise the step runs from scratch. */ #runContext( step: TaskStep, prior: StepRecord?, resolved: GrowableArray, ): RunContext { if (prior == null || prior.config.value != step.config.value) { return new RunContext(null, null, null, resolved); } let delta = computeDelta(prior.inputs, resolved); return new RunContext( delta, prior.manifest, staleOutputNames(prior.manifest, delta), resolved, ); } /** * Verifies that every previous output the delta left valid came back * unchanged. A violation means the step's dependency records are * wrong, and caching the result would corrupt later runs. */ #verifyCarryOver( step: TaskStep, ctx: RunContext, manifest: OutputManifest, ): void { let staleList = ctx.staleOutputs; let previous = ctx.previous; if (staleList == null || previous == null) { return; } let stale = new HashSet(); for (let name in staleList) { stale.add(name); } let produced = new HashMap(); for (let entry in manifest.outputs) { produced[entry.name] = entry.fingerprint; } for (let entry in previous.outputs) { if (stale.has(entry.name)) { continue; } if (let (true, fp) = produced.get(entry.name)) { if (fp.value != entry.fingerprint.value) { throw new WorkflowError( `step "${step.key.value}" changed output "${entry.name}" although its dependency records report it unaffected`, ); } } else { throw new WorkflowError( `step "${step.key.value}" dropped output "${entry.name}" although its dependency records report it unaffected`, ); } } } /** * Computes a service's identity fingerprint once its dependencies * have settled. When a `ServicePool` is active, reuses a matching * live handle or evicts stale ones in reverse dependency order. */ async #serviceMain(svc: ServiceRuntime, rs: RunState): Future { for (let completer in this.#depCompleters(svc.step, rs)) { await completer.future; } for (let dep in this.#serviceDeps(svc.step, rs)) { await dep.identityDone.future; } let resolved = new GrowableArray(); let shortCircuit = this.#resolveInputs(svc.step, rs, resolved); if (shortCircuit == null) { try { for (let resolve in this.#leafSetResolvers(svc.step)) { for (let (name, fp) in await resolve()) { resolved.push(new ResolvedInput('leaf:' + name, fp)); } } svc.identityFp = new Fingerprint( cacheKeyFor(svc.step.config, resolved, this.#digest).value, ); } catch (e) { svc.failure = e; } } else { svc.failure = new WorkflowError( `service "${svc.step.key.value}" has unresolved inputs`, ); } let pool = rs.options.servicePool; if (pool != null) { let existing = pool.get(svc.step.key.value); if (existing != null) { let newFp = svc.identityFp; if ( !rs.options.force && svc.failure == null && newFp != null && existing.identity.value == newFp.value && existing.handle.isAlive() && this.#allServiceDepsReused(svc, rs) ) { svc.reusedFromPool = true; svc.started = true; svc.handle = existing.handle; svc.ready.complete(); } else { await this.#evictFromPool(svc, rs); } } if ( svc.failure == null && !svc.started && (svc.pending == 0 || svc.wasInPool) ) { svc.started = true; this.#startService(svc, rs); } } svc.identityDone.complete(); } /** Whether every service that `svc` depends on was reused from the pool. */ #allServiceDepsReused(svc: ServiceRuntime, rs: RunState): boolean { for (let dep in this.#serviceDeps(svc.step, rs)) { if (!dep.reusedFromPool) { return false; } } return true; } /** Services that directly depend on `target`. */ #serviceConsumers( target: ServiceRuntime, rs: RunState, ): GrowableArray { let consumers = new GrowableArray(); let targetKey = target.step.key.value; for (let candidate in rs.serviceList) { for (let dep in this.#serviceDeps(candidate.step, rs)) { if (dep.step.key.value == targetKey) { consumers.push(candidate); break; } } } return consumers; } /** * Stops and removes `svc` from the service pool, after recursively * evicting any pooled services that depend on it so children stop * before parents. */ async #evictFromPool(svc: ServiceRuntime, rs: RunState): Future { let pool = rs.options.servicePool; if (pool == null) { return; } for (let consumer in this.#serviceConsumers(svc, rs)) { await this.#evictFromPool(consumer, rs); } let evicted = pool.take(svc.step.key.value); if (evicted != null) { svc.wasInPool = true; try { await evicted.handle.stop(); } catch (_) {} } } /** * Ensures every service the step depends on is started and ready. * Returns a failure outcome if one could not start, else null. */ async #acquireServices(step: Step, rs: RunState): Future { for (let svc in this.#serviceDeps(step, rs)) { if (!svc.started) { svc.started = true; this.#startService(svc, rs); } await svc.ready.future; let failure = svc.failure; if (failure != null) { return new Failed(new WorkflowError( `service "${svc.step.key.value}" failed to start: ` + failure.message, )); } } return null; } async #startService(svc: ServiceRuntime, rs: RunState): Future { // A service's own service dependencies must be ready first. let depFailure = await this.#acquireServices(svc.step, rs); if (depFailure != null) { svc.failure = new WorkflowError( `service "${svc.step.key.value}" could not start its dependencies`, ); svc.ready.complete(); return; } try { let onStart = rs.options.onStepStart; if (onStart != null) { onStart(svc.step.key); } let handle = await svc.start(new ServiceContext()); svc.handle = handle; let pool = rs.options.servicePool; let identity = svc.identityFp; if (pool != null && identity != null) { pool.set(svc.step.key.value, new PinnedService(identity, handle)); } } catch (e) { svc.failure = e; } svc.ready.complete(); } async #stopService(svc: ServiceRuntime, rs: RunState): Future { await svc.identityDone.future; if (svc.started) { await svc.ready.future; } let handle = svc.handle; let pool = rs.options.servicePool; if (pool != null) { if (handle != null && (svc.failure != null || !handle.isAlive())) { pool.take(svc.step.key.value); try { await handle.stop(); } catch (e) { if (svc.failure == null) { svc.failure = e; } } } } else if (handle != null) { try { await handle.stop(); } catch (e) { if (svc.failure == null) { svc.failure = e; } } } let failure = svc.failure; let outcome: StepOutcome = if (failure != null) new Failed(failure) else new Succeeded( OutputManifest.empty(), if (svc.started && !svc.reusedFromPool) StepSource.Ran else StepSource.Fresh, ); rs.outcomes[svc.step.key.value] = outcome; let onFinish = rs.options.onStepFinish; if (onFinish != null) { onFinish(svc.step.key, outcome); } // A stopped service no longer holds its own service dependencies. this.#releaseServices(svc.step, rs); svc.stopped.complete(); } }