/** * The async split pass (async.md §3). * * Runs between RTA and layout() (`prepareAsyncFunctions`), alongside * the generator split pass whose machinery it shares (generators.zena * exports the raw IrBody helpers, liveness, and edge rerouting). Every * reached `async` function's body is lowered to presplit ZIR — an * ordinary CFG with await_ suspension terminators — and rewritten into * two finished IrBodies: * * - the RAMP (the async function itself): allocate the frame, spill * the arguments, allocate the frame's `Future`, then — eager * start, §1.1 — call step() straight away so the body runs * synchronously up to the first await, and return the future. * - step(): the state machine, the same dispatcher-loop shape as a * generator's next(), with three differences (§3): * 1. Suspensions park rather than yield: `frame.$state = pc`, * register the frame on the awaited future as a `Task`, * and plain-`return`. **An await allocates nothing** — the * frame IS the waiter (it implements `Task` and subscribes * itself), a settle allocates nothing (waiters pull), and an * already-settled await is one queue hop, no allocation. The * only allocation in the picture is the CALL's own * frame-plus-future. Keep it that way: a wrapper minted here, * per await or per settle, would tax every suspension in * every program. * 2. Resume values are not passed in. The awaited future is an * ordinary value live across the suspension, so it already has * a frame field; the resumed segment reads the settled result * back off it with `valueOrThrow()` (emitted during presplit * lowering). A rejected future therefore resurfaces as an * ordinary throw at the resume point. * 3. Returns complete the frame's own future instead of yielding, * and the whole machine sits inside two nested try_br regions: * the error tag's catch fails that future, and outside it the * cancellation tag's catch completes it as cancelled * (docs/design/cancellation.md). No suspension edge crosses * either region — suspends *return* out of them — so this does * not hit the region-surgery problem user-level try/await * would. Each suspension point is a checkpoint: entering an * await and the head of each resume block test the frame's * scope, and the frame binds to the current scope in the ramp. * * $state encoding matches generators: 0 = START, 1 = RUNNING, * 2 = DONE, 3+k = suspended at await k, with await-containing loop * headers taking pc values above the suspend range. RUNNING is never a * dispatch target, so a re-entrant step() traps on the dispatcher's * fall-through rather than resuming a frame twice. * * Deviation from §3 worth naming: the design has the frame *be* its * `Future` (one allocation). Here the frame *holds* one and * *implements* `Task`. Making a synthesized struct a true * subclass of the specialized `Future` means inheriting its vtable * and layout through RTA; the extra allocation buys that complexity * back, and the observable semantics are identical. */ import { Set, HashMap, HashSet } from 'zena:collections'; import { ValType, ValTypeI32, ValTypeRef, ValTypeRefNull, WasmType, refTargetOf, } from '../wasm.zena'; import { WasmModule, WasmStruct, WasmField, WasmFunction, WasmGlobal, WasmTag, WasmSignature, VTableInit, ClassCodegenInfo, } from '../wasm-module.zena'; import { IrOp, IrBody, IrBlock, IrRefField, IrRefType, IrRefGlobal, IrRefFunc, IrRefTag, isIrTerminator, } from './ir.zena'; import { IrBuilder, valTypesStrictlyEqual } from './builder.zena'; import { IrCfg } from './cfg.zena'; import { FunctionLowerer } from './lowering.zena'; import { truncate, replaceContents, internVt, addRawRef, rawInst, placeInst, rawParam, rawDefaultOf, computeLiveIn, weakenToNullable, loopContainsSuspend, rerouteEdges, TryRegion, dispatchRegionsOf, addRegionMergeTargets, spillCatchLiveIns, varFieldsOf, rewriteVarsToFrame, iteratorInstantiationOf, isExplosionForwarder, StepValue, stepValueOf, rawValueLanes, rawStepResults, } from './generators.zena'; import { ReachabilityAnalysis } from '../reachability/analysis.zena'; import { getInterfaceMemberSignature, getClassTypeParameters, substituteTypeParamsInCodegen, asyncIteratorPendingFuture, } from '../type-mapping.zena'; import { ClassKey, ExportKey, getClassKey, getClassMethodKey, getInterfaceKey, getClassInterfaceKey, getClassInterfaceVTableName, getInterfaceTrampolineName, pendingSomeClass, pendingNoneClass, } from '../codegen-utils.zena'; import { Type, InterfaceType, FunctionType, ClassType, MethodInfo, isScopedType, scopedInner, } from '../../types.zena'; import { FunctionExpression, MethodDefinition, Node } from '../../ast.zena'; import { targetEmitsComponent } from '../../targets.zena'; import { growable } from 'zena:core'; let STATE_RUNNING = 1; let STATE_DONE = 2; let STATE_FIRST_SUSPEND = 3; /** The finished bodies for one async function, keyed by WasmFunction uid. */ final class AsyncArtifacts(rampBody: IrBody, stepBody: IrBody) /** * The cancellation machinery an async function's lowering uses, * present only when the program can cause a cancellation (the * whole-program gate, decided by the reachability pass before * anything is queued): the scope accessor, CancelScope's symbol-keyed * methods, the frame future's cancelled-completion, and the tag. Null * means the program cannot cancel, and async code is emitted exactly * as it was before cancellation existed — no scope field, no * checkpoints, no wake registration, no second capture region. */ final class ScopeMachinery( currentScopeFn: WasmFunction, scopeVt: ValType, installFn: WasmFunction, checkpointFn: WasmFunction, registerFn: WasmFunction, cancelFn: WasmFunction, tag: WasmTag) /** * Prep-time registry: prepareAsyncFunctions fills it, the per-function * loop reads it back through lowerAsyncRamp/lowerAsyncStep. Cleared at * the start of every prepare so repeated compiles in one process stay * independent. */ var rampBodies = new Map(); var stepBodies = new Map(); /** Bodies of the per-frame run() interposer that swaps the current scope * scope around an executor-driven step (docs/design/cancellation.md), * by uid. */ var stepOuterBodies = new Map(); /** Bodies of synthesized async-`main` export wrappers, by uid. */ var mainWrapperBodies = new Map(); /** True if this WasmFunction carries a non-generator `async` AST node. */ export function isAsyncFunction(func: WasmFunction): boolean { let {node} = func; if (node == null || isExplosionForwarder(func)) { return false; } return match (node) { case FunctionExpression {isAsync, isGenerator}: isAsync && !isGenerator case MethodDefinition {isAsync, isGenerator}: isAsync && !isGenerator case _: false }; } /** True if this WasmFunction carries an `async gen` AST node. */ export function isAsyncGeneratorFunction(func: WasmFunction): boolean { let {node} = func; if (node == null || isExplosionForwarder(func)) { return false; } return match (node) { case FunctionExpression {isAsync, isGenerator}: isAsync && isGenerator case MethodDefinition {isAsync, isGenerator}: isAsync && isGenerator case _: false }; } function funcDisplayName(func: WasmFunction) { if (func.name != null) { return func.name as String; } return 'async function'; } /** * The `Future` instantiation an async function returns, from its * checked FunctionType with class/function type arguments substituted * (the memoized instantiation object, so class keys match the ones * RTA specialized). */ function futureInstantiationOf(wasm: WasmModule, func: WasmFunction) { let t = func.model.getNodeType(func.node as Node); if (t == null || !(t is FunctionType)) { throw new Error('async function has no checked function type: ' + funcDisplayName(func)); } var ret = t.returnType; // A borrow-holding async function returns Scoped>; the // frame it lowers to settles the Future behind the wrapper. if (isScopedType(ret)) { let inner = scopedInner(ret); if (inner != null) { ret = inner; } } let {classType} = func; if (classType != null) { let ct = classType; if (ct.typeArguments != null) { ret = substituteTypeParamsInCodegen(wasm, getClassTypeParameters(ct), ct.typeArguments, ret); } } if (func.typeParameters != null && func.typeArguments != null) { ret = substituteTypeParamsInCodegen(wasm, func.typeParameters, func.typeArguments, ret); } if (!(ret is ClassType)) { throw new Error('async function return type is not Future: ' + funcDisplayName(func)); } return ret; } /** * The `Task` interface, read off `Future.subscribe`'s * parameter. Deriving it from the future rather than looking it up by * name keeps the pass free of a well-known-type registry: every async * function already resolves its own `Future`. */ function resumableInterfaceOf(futureType: ClassType) { if (!futureType.members.has(':subscribe')) { throw new Error('zena:async Future has no subscribe member'); } let mi = futureType.members[':subscribe']; if (!(mi is MethodInfo)) { throw new Error('subscribe is not a method'); } let ft = mi.type; if (!(ft is FunctionType)) { throw new Error('subscribe has no function type'); } let params = ft.parameters; if (params.length != 1 || !(params[0] is InterfaceType)) { throw new Error('subscribe does not take a Task'); } return params[0] as InterfaceType; } /** A method of the given specialized class, or a loud failure. */ function methodOf( wasm: WasmModule, cKey: ClassKey, name: String, who: String ): WasmFunction { let key = getClassMethodKey(cKey, name); if (let (true, f) = wasm.classMethodMap.get(key)) { return f; } throw new Error('async lowering: ' + (cKey as String) + '.' + name + ' was not reached (needed by ' + who + ')'); } /** The class key of the scope struct behind `currentScope`'s return * type, for looking up CancelScope's symbol-keyed methods. */ function scopeClassKeyOf(scopeVt: ValType, who: String) { let target = refTargetOf(scopeVt); if (target == null || !(target is WasmStruct)) { throw new Error(`async lowering: scope type is not a class (${who})`); } let ct = (target as WasmStruct).classType; if (ct == null) { throw new Error(`async lowering: scope struct has no class type (${who})`); } return getClassKey(ct as ClassType); } /** A `zena:async` module function RTA queued, or a loud miss. */ function stdlibFn( wasm: WasmModule, name: String, who: String ) { let f = wasm.getStdlibFunc('async', name); if (f == null) { throw new Error('async lowering: zena:async ' + name + ' was not reached (needed by ' + who + ')'); } return f as WasmFunction; } // --------------------------------------------------------------------------- // The pass driver. // --------------------------------------------------------------------------- /** * Called by ModuleGenerator between RTA and wasm.layout(): everything * an async function needs (frame struct, step() function, Task * trampoline and vtable global) must exist before indices lock. */ export function prepareAsyncFunctions( wasm: WasmModule, pass: ReachabilityAnalysis ): void { rampBodies = new Map(); stepBodies = new Map(); stepOuterBodies = new Map(); mainWrapperBodies = new Map(); // Index loop: prepareOne appends step/trampoline functions. var i = 0; while (i < wasm.functions.length) { let func = wasm.functions[i]; i += 1; if (func.typeParameters != null && func.typeArguments == null) { // Generic template; its specializations are processed instead. continue; } if (isAsyncGeneratorFunction(func)) { prepareOneAsyncGen(wasm, pass, func, i - 1); continue; } if (!isAsyncFunction(func)) { continue; } prepareOne(wasm, pass, func, i - 1); } prepareAsyncMain(wasm, pass); } /** * Level 0 (async.md §4): an async `main` is exported through a wrapper * that starts it, drains the microtask queue to empty, and unwraps the * result — so a program whose futures all complete internally runs to * completion on any host that can call one export, with no host event * loop and no changes to zena. * * If the queue empties with main's future still pending, nothing can * ever complete it. `valueOrThrow()` throws there, naming the * deadlock. */ function prepareAsyncMain(wasm: WasmModule, pass: ReachabilityAnalysis): void { if (targetEmitsComponent(wasm.target)) { return; } let key = 'main' as ExportKey; if (!wasm.exportMap.has(key)) { return; } let ramp = wasm.exportMap[key]; if (!ramp.isAsyncRamp || isAsyncGeneratorFunction(ramp)) { return; } let futureType = futureInstantiationOf(wasm, ramp); let fKey = getClassKey(futureType); let valueOrThrowFn = methodOf(wasm, fKey, ':valueOrThrow', 'async main'); let drainFn = wasm.getStdlibFunc('async', 'drainMicrotasks'); if (drainFn == null) { throw new Error('async main: zena:async drainMicrotasks was not reached'); } let wrapperSig = wasm.getSignature(ramp.signature.params, valueOrThrowFn.signature.returns); let wrapper = new WasmFunction(wrapperSig, null, ramp.model); wrapper.name = 'main$asyncEntry'; wrapper.isAsyncMainWrapper = true; wasm.addFunction(wrapper); pass.markFunctionReached(wrapper); let b = new IrBuilder(); let entry = b.newBlock(); let params = new Array(); for (let vt in ramp.signature.params) { params.push(b.blockParam(entry, vt)); } b.switchTo(entry); let futVt = if (ramp.signature.returns.length == 1) { ramp.signature.returns[0] } else { throw new Error('async main: ramp does not return its future') }; let fut = b.callFunc(b.addRef(new IrRefFunc(ramp)), params, futVt); b.callFunc(b.addRef(new IrRefFunc(drainFn)), new Array(0), null); let unwrapArgs = growable([fut]); if (valueOrThrowFn.signature.returns.length == 0) { b.callFunc(b.addRef(new IrRefFunc(valueOrThrowFn)), unwrapArgs, null); b.retVoid(); } else { let value = b.callFunc(b.addRef(new IrRefFunc(valueOrThrowFn)), unwrapArgs, valueOrThrowFn.signature.returns[0]); b.ret(value); } mainWrapperBodies[wrapper.uid] = b.finish(); // The export now names the wrapper; the ramp stays an ordinary // internal function that other code can still call directly. wasm.exportMap[key] = wrapper; buildHostEntry(wasm, pass, ramp, valueOrThrowFn, drainFn, futVtOf(ramp)); } /** The future type the ramp returns. */ function futVtOf(ramp: WasmFunction): ValType { if (ramp.signature.returns.length != 1) { throw new Error('async main: ramp does not return its future'); } return ramp.signature.returns[0]; } /** * The split entry a host that cannot wait needs (async.md §4). * * `main` above starts the body, drains, and unwraps in one call, which * works wherever the clock can genuinely wait — the drain simply never * unwinds, so by the time it returns the future has settled. A host * that cannot wait unwinds the drain early, and unwrapping there would * hit `valueOrThrow`'s deadlock case even though the program is fine * and merely waiting on a timer. * * So such a host uses two calls instead of one: `$asyncMainStart` runs * the body as far as it can and parks the future in a global, then — * after `$asyncDrain` pings have carried it to completion — * `$asyncMainResult` unwraps it. Neither needs a branch, which is why * this is two functions rather than one with an `isCompleted` test. * * `main` is left exactly as it was, so WASI and zena are unaffected * and a standalone `wasmtime --invoke main` still runs to completion. */ function buildHostEntry( wasm: WasmModule, pass: ReachabilityAnalysis, ramp: WasmFunction, valueOrThrowFn: WasmFunction, drainFn: WasmFunction, futVt: ValType ): void { // The in-flight future, so the two halves can find each other. let slot = new WasmGlobal('$asyncMainFuture', '$asyncMainFuture', futVt, true, null, null, ramp.model); wasm.globals.push(slot); let slotRef = new IrRefGlobal(slot); // $asyncMainStart(params...) -> void let startSig = wasm.getSignature(ramp.signature.params, new Array(0)); let startFn = new WasmFunction(startSig, null, ramp.model); startFn.name = 'main$asyncStart'; startFn.isAsyncMainWrapper = true; wasm.addFunction(startFn); pass.markFunctionReached(startFn); let sb = new IrBuilder(); let sEntry = sb.newBlock(); let sParams = new Array(); for (let vt in ramp.signature.params) { sParams.push(sb.blockParam(sEntry, vt)); } sb.switchTo(sEntry); let started = sb.callFunc(sb.addRef(new IrRefFunc(ramp)), sParams, futVt); sb.emitVoid(IrOp.global_set, started, sb.addRef(slotRef)); sb.callFunc(sb.addRef(new IrRefFunc(drainFn)), new Array(0), null); sb.retVoid(); mainWrapperBodies[startFn.uid] = sb.finish(); wasm.asyncMainStartFunc = startFn; // $asyncMainResult() -> T let resultSig = wasm.getSignature(new Array(0), valueOrThrowFn.signature.returns); let resultFn = new WasmFunction(resultSig, null, ramp.model); resultFn.name = 'main$asyncResult'; resultFn.isAsyncMainWrapper = true; wasm.addFunction(resultFn); pass.markFunctionReached(resultFn); let rb = new IrBuilder(); let rEntry = rb.newBlock(); rb.switchTo(rEntry); // A mutable reference global is emitted nullable (it needs a null // initializer), so the load is nullable and the receiver slot of // valueOrThrow is not. let weak = rb.emit(IrOp.global_get, weakenToNullable(futVt), -1, rb.addRef(slotRef)); let stored = rb.emit(IrOp.ref_as_non_null, futVt, weak, -1); let rArgs = growable([stored]); if (valueOrThrowFn.signature.returns.length == 0) { rb.callFunc(rb.addRef(new IrRefFunc(valueOrThrowFn)), rArgs, null); rb.retVoid(); } else { let v = rb.callFunc(rb.addRef(new IrRefFunc(valueOrThrowFn)), rArgs, valueOrThrowFn.signature.returns[0]); rb.ret(v); } mainWrapperBodies[resultFn.uid] = rb.finish(); wasm.asyncMainResultFunc = resultFn; } export function lowerAsyncMainWrapper( wasm: WasmModule, func: WasmFunction ): IrBody { if (let (true, body) = mainWrapperBodies.get(func.uid)) { return body; } throw new Error('async main wrapper was not prepared'); } export function lowerAsyncRamp(wasm: WasmModule, func: WasmFunction): IrBody { if (let (true, art) = rampBodies.get(func.uid)) { return art.rampBody; } throw new Error('async function was not prepared (split pass did not run): ' + funcDisplayName(func)); } export function lowerAsyncStep(wasm: WasmModule, func: WasmFunction): IrBody { if (let (true, art) = stepBodies.get(func.uid)) { return art.stepBody; } if (let (true, outer) = stepOuterBodies.get(func.uid)) { return outer; } throw new Error('async step() was not prepared: ' + funcDisplayName(func)); } function prepareOne( wasm: WasmModule, pass: ReachabilityAnalysis, func: WasmFunction, funcIndex: i32 ): void { // Async closures need no special handling, for the same reason // generator closures do not: the context struct arrives as param 0 // of the ramp and is spilled to a frame field like any other param. let futureType = futureInstantiationOf(wasm, func); pass.typeLowerer.discoverType(futureType as Type, func.model); let fKey = getClassKey(futureType); let futureInfo = if (let (true, info) = wasm.classInfos.get(fKey)) info else null; if (futureInfo == null) { throw new Error('async Future instantiation was not materialized: ' + funcDisplayName(func)); } let futureStruct = futureInfo.structType; let name = funcDisplayName(func); let completeFn = methodOf(wasm, fKey, ':complete', name); let failFn = methodOf(wasm, fKey, ':fail', name); let adoptFn = methodOf(wasm, fKey, ':adoptFrom', name); let futureCtor = methodOf(wasm, fKey, '', name); // The cancellation machinery, present only when the program can // cause a scope cancellation (docs/design/cancellation.md). Keyed // on the flag rather than the tag: generator disposal mints the // tag alone, and the scope methods resolved here exist only when // RTA queued them under the flag. var sm: ScopeMachinery | null = null; if (wasm.scopeCancellationActive) { let currentScopeFn = stdlibFn(wasm, 'currentScope', name); if (currentScopeFn.signature.returns.length != 1) { throw new Error('async lowering: currentScope does not return a scope'); } let scopeVt = currentScopeFn.signature.returns[0]; // The scope's own operations are symbol-keyed methods on // CancelScope, so nothing here is callable from user code. let scopeKey = scopeClassKeyOf(scopeVt, name); sm = new ScopeMachinery( currentScopeFn, scopeVt, methodOf(wasm, scopeKey, ':install', name), methodOf(wasm, scopeKey, ':checkpoint', name), methodOf(wasm, scopeKey, ':registerTask', name), methodOf(wasm, fKey, ':cancelComplete', name), wasm.cancellationTag as WasmTag); } // `complete(this, value: T)` is where the value type comes from. // `Future` (async.md §8 question 1) needs nothing special here: // `void` is a zero-width type argument, so `complete` specializes to // the receiver alone and valueVt falls out null — exactly the "no // value slot" case presplitAsyncValueVt already describes. let valueVt = if (completeFn.signature.params.length > 1) { completeFn.signature.params[1] } else { null }; if (wasm.exceptionTag == null || wasm.exceptionPayload == null) { // step()'s failure-capture region needs the exception infra; the // checker marks every async function as requiring it. throw new Error('async lowering: exception infrastructure missing for ' + name); } // Presplit lowering: the body as an ordinary CFG with await_ // terminators. A bail inside an async body is a hard error // (ZirUnsupported propagates; there is no fallback backend). let fl = new FunctionLowerer(wasm, func); fl.presplitAsync = true; fl.presplitAsyncValueVt = valueVt; let presplit = fl.lower(); let subscribeOf = fl.awaitSubscribeTargets; let subscribeCalleeOf = fl.awaitSubscribeCallees; let hopTerms = fl.awaitHopTerms; let shieldedAwaits = fl.shieldedAwaits; // A union await's bare arm parks the frame directly on the queue // (async.md §1); the stdlib entry for that is only linked when a // program contains one. var scheduleFn: WasmFunction | null = null; if (hopTerms.size > 0) { let scheduleOpt = wasm.getStdlibFunc('async', 'scheduleTask'); if (scheduleOpt == null) { throw new Error('async lowering: zena:async scheduleTask was not ' + 'reached (needed by a union await in ' + name + ')'); } scheduleFn = scheduleOpt; } let cfg = new IrCfg(presplit); // Suspension analysis on the pristine CFG. let awaitBlocks = new Array(); let resumeOf = new Array(); let shieldedResumes = new Set(); // For a shielded resume, the awaited future's value id: a cancel // WAKES parked frames, and with the checkpoint masked the resume // must re-park when the wake was spurious (the future still // pending) rather than read a result that is not there. Hop // resumes carry a plain value and need no guard. let shieldedResumeFuture = new Map(); var bi = 0; while (bi < presplit.blocks.length) { let insts = presplit.blocks[bi].insts; if (insts.length > 0) { let term = insts[insts.length - 1]; if (presplit.op(term) == IrOp.await_) { awaitBlocks.push(bi); let resumeTarget = presplit.extra[presplit.successorRecord(term, 0)]; resumeOf.push(resumeTarget); if (shieldedAwaits.has(term)) { shieldedResumes.add(resumeTarget); if (!hopTerms.has(term)) { shieldedResumeFuture[resumeTarget] = presplit.a[term]; } } } } bi += 1; } var shieldFn: WasmFunction | null = null; if (shieldedResumes.size > 0) { // `shielded` in the source enables the whole-program gate, so the // machinery is present whenever a shielded resume exists. if (sm == null) { throw new Error('async lowering: shielded region without the ' + 'cancellation machinery in ' + name); } let scopeKey = scopeClassKeyOf((sm as ScopeMachinery).scopeVt, name); shieldFn = methodOf(wasm, scopeKey, ':shieldScope', name); } // pc assignment: resume blocks get their suspend-state value; // await-containing loop headers get values above the suspend range. let pcOfBlock = new Map(); var ai = 0; while (ai < resumeOf.length) { pcOfBlock[resumeOf[ai]] = STATE_FIRST_SUSPEND + ai; ai += 1; } var nextPc = STATE_FIRST_SUSPEND + resumeOf.length; var hb = 0; while (hb < presplit.blocks.length) { if (cfg.isReachable(hb) && !pcOfBlock.has(hb) && hb != 0 && loopContainsSuspend(presplit, cfg, hb, awaitBlocks)) { pcOfBlock[hb] = nextPc; nextPc += 1; } hb += 1; } nextPc = addRegionMergeTargets(presplit, cfg, pcOfBlock, nextPc); // Which user try regions enclose each dispatch target (async.md §6). let dispatchRegions = dispatchRegionsOf(presplit, cfg, pcOfBlock); let liveIn = computeLiveIn(presplit, cfg); // Spill set: everything live into a dispatch target, every dispatch // target's block params, and every function parameter. let spilled = new Set(); let entryParamIds = new Array(); for (let p in presplit.blocks[0].params) { entryParamIds.push(p); spilled.add(p); } let dTargets = growable([0]); var db = 0; while (db < presplit.blocks.length) { if (pcOfBlock.has(db)) { dTargets.push(db); for (let v in liveIn[db]) { spilled.add(v); } for (let p in presplit.blocks[db].params) { spilled.add(p); } } db += 1; } for (let v in liveIn[0]) { spilled.add(v); } spillCatchLiveIns(liveIn, dispatchRegions, spilled); // The frame struct: $state, the frame's own future, then one // nullable-weakened field per spilled value, ascending value id. let fields = growable([ new WasmField('$state', new ValTypeI32(), true), new WasmField('$future', new ValTypeRefNull(futureStruct), true), ]); if (sm != null) { // The scope the frame bound to at creation; checkpoints test it. fields.push(new WasmField('$scope', weakenToNullable((sm as ScopeMachinery).scopeVt), true)); } var vid = 0; while (vid < presplit.instCount()) { if (spilled.has(vid)) { let vt = presplit.resultType(vid); if (vt == null) { throw new Error('async spill of valueless instruction'); } fields.push(new WasmField('$v' + i32ToString(vid), weakenToNullable(vt), true)); } vid += 1; } varFieldsOf(presplit, fields); let frameName = `${name}$${i32ToString(funcIndex)}$AsyncFrame`; let frame = new WasmStruct(frameName, fields); wasm.types.push(frame); pass.markTypeReached(frame); // step(): (ref null $Frame) -> (). let stepParams = growable([new ValTypeRefNull(frame)]); let stepSig = wasm.getSignature(stepParams, new Array(0)); let stepFunc = new WasmFunction(stepSig, null, func.model); stepFunc.name = frameName + '.step'; stepFunc.isAsyncStep = true; wasm.addFunction(stepFunc); pass.markFunctionReached(stepFunc); // run(): the Task entry the executor calls. With the machinery // present it installs the frame's scope around the step, so work // the step runs synchronously — new async calls, explicit polls — // sees the frame's scope rather than whatever ran last. The ramp // calls step() directly instead: during the eager first segment the // current scope IS the caller's scope, which is what the frame just // bound to. step() never unwinds (its tags are captured), so the // linear install/step/restore shape needs no protection region. // With no machinery the executor calls step() itself. var runTarget = stepFunc; if (sm != null) { let m = sm as ScopeMachinery; let stepOuterFunc = new WasmFunction(stepSig, null, func.model); stepOuterFunc.name = frameName + '.run'; stepOuterFunc.isAsyncStep = true; wasm.addFunction(stepOuterFunc); pass.markFunctionReached(stepOuterFunc); let ob = new IrBuilder(); let oEntry = ob.newBlock(); let oFrame = ob.blockParam(oEntry, new ValTypeRefNull(frame)); ob.switchTo(oEntry); let scopeWeak = ob.emit(IrOp.struct_get, weakenToNullable(m.scopeVt), oFrame, ob.addRef(new IrRefField(frame, '$scope'))); let scope = ob.emit(IrOp.ref_as_non_null, m.scopeVt, scopeWeak, -1); let swapArgs = growable([scope]); let prev = ob.callFunc(ob.addRef(new IrRefFunc(m.installFn)), swapArgs, m.scopeVt); let stepArgs = growable([oFrame]); ob.callFunc(ob.addRef(new IrRefFunc(stepFunc)), stepArgs, null); let restoreArgs = growable([prev]); let ignored = ob.callFunc(ob.addRef(new IrRefFunc(m.installFn)), restoreArgs, m.scopeVt); ob.retVoid(); stepOuterBodies[stepOuterFunc.uid] = ob.finish(); runTarget = stepOuterFunc; } // The frame implements Task: run() steps the machine. One // vtable per frame, whatever the function awaits. let iface = resumableInterfaceOf(futureType); pass.typeLowerer.discoverType(iface as Type, func.model); let iKey = getInterfaceKey(iface); if (!wasm.interfaceStructs.has(iKey) || !wasm.interfaceVTableStructs.has(iKey)) { throw new Error('async Task interface was not materialized: ' + name); } let fatPtr = wasm.interfaceStructs[iKey]; let vst = wasm.interfaceVTableStructs[iKey]; let slotSig = getInterfaceMemberSignature(wasm, iface, 'run'); let frameKey = frameName as ClassKey; let methods = new Array(); for (let field in vst.fields) { if ((field.name as String) != 'run') { throw new Error('unexpected Task vtable member: ' + (field.name as String)); } let tramp = new WasmFunction(slotSig, null, func.model); tramp.name = getInterfaceTrampolineName(frameKey, iKey, 'run'); tramp.interfaceTrampolineTarget = runTarget; wasm.addFunction(tramp); pass.markFunctionReached(tramp); methods.push(tramp); } let vtableInit = new VTableInit(vst, methods); let viName = getClassInterfaceVTableName(frameKey, iKey); let vtableGlobal = new WasmGlobal( viName, viName, new ValTypeRef(vst), false, null, vtableInit, func.model ); wasm.globals.push(vtableGlobal); wasm.classInterfaceVTables[getClassInterfaceKey(frameKey, iKey)] = vtableGlobal; wasm.usedInterfaceAdaptations.add(getClassInterfaceKey(frameKey, iKey)); // The two bodies. let rampBody = buildRamp(func, frame, futureInfo, futureCtor, stepFunc, entryParamIds, sm, fatPtr, vtableGlobal); let stepBody = buildStepBody(presplit, cfg, frame, futureStruct, spilled, pcOfBlock, dTargets, subscribeOf, subscribeCalleeOf, hopTerms, scheduleFn, completeFn, adoptFn, failFn, sm, resumeOf, shieldedAwaits, shieldedResumes, shieldedResumeFuture, shieldFn, valueVt, fatPtr, vtableGlobal, wasm.exceptionTag as WasmTag, wasm.exceptionPayload as WasmGlobal, dispatchRegions); let artifacts = new AsyncArtifacts(rampBody, stepBody); func.isAsyncRamp = true; rampBodies[func.uid] = artifacts; stepBodies[stepFunc.uid] = artifacts; } /** * ramp: allocate the frame, spill the arguments, allocate the frame's * future, then run the first segment eagerly (§1.1) and hand back the * future. */ function buildRamp( func: WasmFunction, frame: WasmStruct, futureInfo: ClassCodegenInfo, futureCtor: WasmFunction, stepFunc: WasmFunction, entryParamIds: GrowableArray, sm: ScopeMachinery | null, fatPtr: WasmStruct, vtableGlobal: WasmGlobal ): IrBody { let b = new IrBuilder(); let entry = b.newBlock(); let params = new Array(); for (let vt in func.signature.params) { params.push(b.blockParam(entry, vt)); } b.switchTo(entry); let frameRef = b.addRef(new IrRefType(frame)); let inst = b.emit(IrOp.struct_new_default, new ValTypeRef(frame), -1, frameRef); // $state defaults to 0 == START via struct.new_default. var i = 0; while (i < params.length) { let fieldRef = b.addRef(new IrRefField(frame, '$v' + i32ToString(entryParamIds[i]))); let pair = growable([inst, params[i]]); b.emitVoid(IrOp.struct_set, b.pushExtra(pair), fieldRef); i += 1; } // The frame's own Future: one factory call, the same shape // `new Future()` lowers to. let futureStruct = futureInfo.structType; let futVt: ValType = new ValTypeRef(futureStruct); let fut = b.callFunc(b.addRef(new IrRefFunc(futureCtor)), new Array(), futVt); let futField = b.addRef(new IrRefField(frame, '$future')); let futPair = growable([inst, fut]); b.emitVoid(IrOp.struct_set, b.pushExtra(futPair), futField); if (sm != null) { let m = sm as ScopeMachinery; // Bind to the current scope, once. Eager start does the // inheritance: this ramp runs synchronously inside its caller, so // the current scope here is the caller's. After this, nothing // consults the current scope for this frame — delivery is a field // test. Then register on the scope's wake list, as the Task the // executor runs: cancelling the scope schedules the frame, so // delivery does not wait on a future that may never settle. let scope = b.callFunc(b.addRef(new IrRefFunc(m.currentScopeFn)), new Array(), m.scopeVt); let scopeField = b.addRef(new IrRefField(frame, '$scope')); let scopePair = growable([inst, scope]); b.emitVoid(IrOp.struct_set, b.pushExtra(scopePair), scopeField); let packed = b.emit(IrOp.iface_pack, new ValTypeRef(fatPtr), inst, b.addRef(new IrRefGlobal(vtableGlobal))); let bindArgs = growable([scope, packed]); b.callFunc(b.addRef(new IrRefFunc(m.registerFn)), bindArgs, null); } // Eager start: the body runs synchronously up to the first await. let stepArgs = growable([inst]); b.callFunc(b.addRef(new IrRefFunc(stepFunc)), stepArgs, null); b.ret(fut); return b.finish(); } // --------------------------------------------------------------------------- // The state-machine transform: mutates the presplit body in place into // the step() body. // --------------------------------------------------------------------------- function buildStepBody( body: IrBody, cfg: IrCfg, frame: WasmStruct, futureStruct: WasmStruct, spilled: HashSet, pcOfBlockIn: HashMap, dTargetsIn: GrowableArray, subscribeOf: HashMap, subscribeCalleeOf: HashMap, hopTerms: HashSet, scheduleFn: WasmFunction | null, completeFn: WasmFunction, adoptFn: WasmFunction, failFn: WasmFunction, sm: ScopeMachinery | null, resumeTargets: GrowableArray, shieldedAwaits: HashSet, shieldedResumes: HashSet, shieldedResumeFuture: HashMap, shieldFn: WasmFunction | null, valueVt: ValType | null, fatPtr: WasmStruct, vtableGlobal: WasmGlobal, exceptionTag: WasmTag, exceptionPayload: WasmGlobal, dispatchRegions: HashMap> ): IrBody { let origCount = body.blocks.length; let stateRef = addRawRef(body, new IrRefField(frame, '$state')); let futureRef = addRawRef(body, new IrRefField(frame, '$future')); let scopeFieldRef = if (sm != null) { addRawRef(body, new IrRefField(frame, '$scope')) } else { -1 }; let i32Vt: ValType = new ValTypeI32(); // Snapshot the dispatch targets' params (cleared below; edges into // them turn into frame stores). let dParams = new Map>(); for (let d in dTargetsIn) { let copy = new Array(); for (let p in body.blocks[d].params) { copy.push(p); } dParams[d] = copy; } // 1. Evacuate the old entry into a fresh block E; block 0 becomes // the prologue head (the function's real entry: one frame param). let entryCopy = new IrBlock(); let oldEntry = body.blocks[0]; body.blocks.push(entryCopy); let entryCopyId = body.blocks.length - 1; for (let p in oldEntry.params) { entryCopy.params.push(p); body.instBlock[p] = entryCopyId; } for (let inst in oldEntry.insts) { entryCopy.insts.push(inst); body.instBlock[inst] = entryCopyId; } body.blocks[0] = new IrBlock(); // Dispatch-target bookkeeping with entry remapped to its new id. let pcOfBlock: HashMap = {entryCopyId => 0}; for (let d in dTargetsIn) { if (d == 0) { continue; } pcOfBlock[d] = pcOfBlockIn[d]; } let dParamsById = new Map>(); for (let d in dTargetsIn) { let mapped = if (d == 0) { entryCopyId } else { d }; dParamsById[mapped] = dParams[d]; } // 2. Prologue: the failure-capture region, the DONE guard, and the // dispatcher loop. let frameParam = rawParam(body, 0, new ValTypeRefNull(frame)); let newBlockId = () => { body.blocks.push(new IrBlock()); return body.blocks.length - 1; }; let tryBody = newBlockId(); let catchBlock = newBlockId(); let doneBlock = newBlockId(); let enterBlock = newBlockId(); let trapBlock = newBlockId(); let dispatcher = newBlockId(); var errRegion = -1; var cancelCatch = -1; if (sm != null) { errRegion = newBlockId(); cancelCatch = newBlockId(); } let emitIn = (blockId: i32, op: IrOp, typeIdx: i32, a: i32, b: i32) => { let inst = rawInst(body, op, typeIdx, a, b); placeInst(body, blockId, inst); return inst; }; let brTo = (fromBlock: i32, target: i32, args: GrowableArray) => { let idx = body.extra.length; body.extra.push(target); body.extra.push(args.length); for (let v in args) { body.extra.push(v); } let inst = rawInst(body, IrOp.br, -1, idx, -1); placeInst(body, fromBlock, inst); }; let brIfTo = (fromBlock: i32, cond: i32, thenTarget: i32, elseTarget: i32) => { let idx = body.extra.length; body.extra.push(thenTarget); body.extra.push(0); body.extra.push(elseTarget); body.extra.push(0); let inst = rawInst(body, IrOp.br_if, -1, cond, idx); placeInst(body, fromBlock, inst); }; let storeState = (blockId: i32, valueInst: i32) => { let idx = body.extra.length; body.extra.push(frameParam); body.extra.push(valueInst); let inst = rawInst(body, IrOp.struct_set, -1, idx, stateRef); placeInst(body, blockId, inst); }; let retVoidIn = (blockId: i32) => { let inst = rawInst(body, IrOp.ret, -1, -1, -1); placeInst(body, blockId, inst); }; /** frame.$future, re-asserted non-null for the call boundary. */ let loadFuture = (blockId: i32) => { let weak = emitIn(blockId, IrOp.struct_get, internVt(body, new ValTypeRefNull(futureStruct)), frameParam, futureRef); return emitIn(blockId, IrOp.ref_as_non_null, internVt(body, new ValTypeRef(futureStruct)), weak, -1); }; let callIn = (blockId: i32, target: WasmFunction, args: GrowableArray) => { let idx = body.extra.length; body.extra.push(args.length); for (let v in args) { body.extra.push(v); } let inst = rawInst(body, IrOp.call, -1, idx, addRawRef(body, new IrRefFunc(target))); placeInst(body, blockId, inst); }; let i32Idx = internVt(body, i32Vt); // b0: enter the capture regions. With the cancellation machinery // present, the cancellation region sits outside the failure region, // matching finalizer regions' nesting; without it there is only the // failure region, the shape async had before cancellation existed. // Every suspension and every completion RETURNS from inside them, so // the regions stay properly nested (§3) — nothing branches out. if (sm != null) { let cancelIdx = body.extra.length; body.extra.push(errRegion); body.extra.push(0); body.extra.push(cancelCatch); body.extra.push(0); let cancelInst = rawInst(body, IrOp.try_br, -1, addRawRef(body, new IrRefTag((sm as ScopeMachinery).tag)), cancelIdx); placeInst(body, 0, cancelInst); } let tryIdx = body.extra.length; body.extra.push(tryBody); body.extra.push(0); body.extra.push(catchBlock); body.extra.push(0); let tryInst = rawInst(body, IrOp.try_br, -1, addRawRef(body, new IrRefTag(exceptionTag)), tryIdx); placeInst(body, if (sm != null) errRegion else 0, tryInst); // tryBody: state = frame.$state; already DONE ? return : enter. let stateVal = emitIn(tryBody, IrOp.struct_get, i32Idx, frameParam, stateRef); let doneConst = emitIn(tryBody, IrOp.const_i32, i32Idx, STATE_DONE, -1); let isDone = emitIn(tryBody, IrOp.i32_eq, i32Idx, stateVal, doneConst); brIfTo(tryBody, isDone, doneBlock, enterBlock); // done: the future has already settled; nothing left to run. retVoidIn(doneBlock); // enter: $state = RUNNING, then dispatch on the state we loaded. // RUNNING is never a dispatch pc, so a re-entrant step() falls // through the chain into the trap instead of resuming twice. let runningConst = emitIn(enterBlock, IrOp.const_i32, i32Idx, STATE_RUNNING, -1); storeState(enterBlock, runningConst); let dispatchArgs = growable([stateVal]); brTo(enterBlock, dispatcher, dispatchArgs); // trap: an impossible state. let trapInst = rawInst(body, IrOp.unreachable, -1, -1, -1); placeInst(body, trapBlock, trapInst); // catch: the body threw. Fail the frame's own future with the caught // error and finish — this is the whole of async error handling (§3). let payloadVt = exceptionPayload.type; var payload = emitIn(catchBlock, IrOp.global_get, internVt(body, payloadVt), -1, addRawRef(body, new IrRefGlobal(exceptionPayload))); if (payloadVt is ValTypeRefNull) { let target = refTargetOf(payloadVt) as WasmType; payload = emitIn(catchBlock, IrOp.ref_as_non_null, internVt(body, new ValTypeRef(target)), payload, -1); } let doneC = emitIn(catchBlock, IrOp.const_i32, i32Idx, STATE_DONE, -1); storeState(catchBlock, doneC); let failFut = loadFuture(catchBlock); let failArgs = growable([failFut, payload]); callIn(catchBlock, failFn, failArgs); retVoidIn(catchBlock); // cancel catch: a cancellation unwound the body (its finalizers have // already run — their regions rethrow on this tag). Complete the // frame as cancelled and finish; the unwind continues to the frame's // awaiters when their resumes observe the cancelled future. No // payload: nothing may observe a cancellation. if (sm != null) { let cDoneC = emitIn(cancelCatch, IrOp.const_i32, i32Idx, STATE_DONE, -1); storeState(cancelCatch, cDoneC); let cancelFut = loadFuture(cancelCatch); let cancelArgs = growable([cancelFut]); callIn(cancelCatch, (sm as ScopeMachinery).cancelFn, cancelArgs); retVoidIn(cancelCatch); } /** * One checkpoint: frame.$scope handed to the stdlib's `checkpoint`, * which raises on the cancellation channel when the scope is * cancelled. Emitted on entering an await (before parking) and at * the head of every resume block — the two halves of a suspension * point, and the only places cancellation is delivered. The entry * segment gets none: a task in a just-cancelled scope still runs its * ramp, because code between suspensions is never interrupted. */ let checkpointInto = (blockId: i32, out: GrowableArray) => { if (sm == null) { // Nothing in the program can cancel: suspension points carry no // checkpoint at all. return; } let m = sm as ScopeMachinery; let weak = rawInst(body, IrOp.struct_get, internVt(body, weakenToNullable(m.scopeVt)), frameParam, scopeFieldRef); body.instBlock[weak] = blockId; out.push(weak); let strong = rawInst(body, IrOp.ref_as_non_null, internVt(body, m.scopeVt), weak, -1); body.instBlock[strong] = blockId; out.push(strong); let callIdx = body.extra.length; body.extra.push(1); body.extra.push(strong); let call = rawInst(body, IrOp.call, -1, callIdx, addRawRef(body, new IrRefFunc(m.checkpointFn))); body.instBlock[call] = blockId; out.push(call); }; /** * Re-enters the user `try` regions enclosing a dispatch target, and * returns the block the dispatcher should branch to (async.md §6). * * The dispatcher lives outside every user region, so branching * straight at the target would leave the segment it heads * unprotected — the region silently shrinks to exclude it, and a * failure raised there sails past the handler the user wrote. * Instead each enclosing region is re-entered through a fresh * `try_br` whose catch is that region's own handler, innermost last, * so the segment runs at the same protection depth the source put it * at. * * Applies to every dispatch target, not only resume blocks. A * suspending loop's header is a dispatch target too, and its * protection is not implied by the resume blocks inside it: a resume * block's fresh try_br covers what that block dominates, which is the * rest of that iteration after the await, but not the stretch from * the header down to the await. * * This is §6's construction with one simplification: §6 threads the * pc through the region's ORIGINAL try_br into a small inner * dispatcher. Re-entering through fresh try_brs needs no inner * dispatcher, no pc threading across a region edge, and no splitting * of the block that holds the original try_br — and the region it * builds is exactly the same one, because a region's extent is the * dominator subtree of its body target and a dispatch target inside a * try body dominates precisely the part of that body it heads. */ let reenterRegions = (dispatchTarget: i32) => { let regions = if (let (true, rs) = dispatchRegions.get(dispatchTarget)) rs else new Array(0); var target = dispatchTarget; // Innermost first, so the outermost try_br ends up on the outside. var ri = regions.length - 1; while (ri >= 0) { let r = regions[ri]; let t = newBlockId(); let idx = body.extra.length; body.extra.push(target); body.extra.push(0); body.extra.push(r.catchTarget); body.extra.push(0); let inst = rawInst(body, IrOp.try_br, -1, r.tagRef, idx); placeInst(body, t, inst); // The fresh region ends where the original did: without this, a // join dominated by the dispatch target would be placed inside // the re-entered try_table (issue #426). if (r.joinTarget >= 0) { body.tryJoin[inst] = r.joinTarget; } target = t; ri -= 1; } return target; }; // dispatcher(pc): br_if chain over the dispatch targets, ascending // pc. The final fall-through is an invalid state — trap. let pcParam = rawParam(body, dispatcher, i32Vt); let chainTargets = new Array(); let chainPcs = new Array(); chainTargets.push(entryCopyId); chainPcs.push(0); var scanPc = STATE_FIRST_SUSPEND; var remaining = pcOfBlock.size - 1; while (remaining > 0) { var found = -1; var fb = 0; while (fb < body.blocks.length) { if (pcOfBlock.has(fb) && pcOfBlock[fb] == scanPc) { found = fb; break; } fb += 1; } if (found >= 0) { chainTargets.push(reenterRegions(found)); chainPcs.push(scanPc); remaining -= 1; } scanPc += 1; } var chainBlock = dispatcher; var ci = 0; while (ci < chainTargets.length) { let cmpConst = emitIn(chainBlock, IrOp.const_i32, i32Idx, chainPcs[ci], -1); let cmp = emitIn(chainBlock, IrOp.i32_eq, i32Idx, pcParam, cmpConst); let elseBlock = if (ci == chainTargets.length - 1) { trapBlock } else { newBlockId() }; brIfTo(chainBlock, cmp, chainTargets[ci], elseBlock); chainBlock = elseBlock; ci += 1; } // The body blocks the remaining passes operate on. let bodyBlocks = growable([entryCopyId]); var obi = 1; while (obi < origCount) { bodyBlocks.push(obi); obi += 1; } // 3. Reroute every edge into a dispatch target through the // dispatcher, converting its branch args into frame stores. for (let blockId in bodyBlocks) { rerouteEdges(body, blockId, pcOfBlock, dParamsById, dispatcher, frameParam, frame, i32Idx, entryCopyId); } for (let d in dTargetsIn) { let mapped = if (d == 0) { entryCopyId } else { d }; truncate(body.blocks[mapped].params); } // 4. Store every spilled definition to its frame field. let skipRemap = new Set(); for (let blockId in bodyBlocks) { let block = body.blocks[blockId]; let rebuilt = new Array(); for (let p in block.params) { if (spilled.has(p)) { let idx = body.extra.length; body.extra.push(frameParam); body.extra.push(p); let st = rawInst(body, IrOp.struct_set, -1, idx, addRawRef(body, new IrRefField(frame, '$v' + i32ToString(p)))); skipRemap.add(st); rebuilt.push(st); body.instBlock[st] = blockId; } } for (let inst in block.insts) { rebuilt.push(inst); if (spilled.has(inst) && !isIrTerminator(body.op(inst))) { let idx = body.extra.length; body.extra.push(frameParam); body.extra.push(inst); let st = rawInst(body, IrOp.struct_set, -1, idx, addRawRef(body, new IrRefField(frame, '$v' + i32ToString(inst)))); skipRemap.add(st); rebuilt.push(st); body.instBlock[st] = blockId; } } replaceContents(block.insts, rebuilt); } // 5. Load every use of a spilled value from its frame field, // immediately before the use. for (let blockId in bodyBlocks) { let block = body.blocks[blockId]; let rebuilt = new Array(); let ops = new Array(); for (let inst in block.insts) { if (skipRemap.has(inst)) { rebuilt.push(inst); continue; } truncate(ops); body.appendValueOperands(inst, ops); let loadOf = new Map(); for (let v in ops) { if (spilled.has(v) && !loadOf.has(v)) { let vt = body.resultType(v) as ValType; let weak = weakenToNullable(vt); var load = rawInst(body, IrOp.struct_get, internVt(body, weak), frameParam, addRawRef(body, new IrRefField(frame, '$v' + i32ToString(v)))); rebuilt.push(load); body.instBlock[load] = blockId; if (!valTypesStrictlyEqual(weak, vt)) { load = rawInst(body, IrOp.ref_as_non_null, internVt(body, vt), load, -1); rebuilt.push(load); body.instBlock[load] = blockId; } loadOf[v] = load; } } if (loadOf.size > 0) { body.rewriteValueOperands(inst, (v) => { if (let (true, l) = loadOf.get(v)) { return l; } return v; }); } rebuilt.push(inst); } replaceContents(block.insts, rebuilt); } // 6. Rewrite suspensions and completions. for (let blockId in bodyBlocks) { let block = body.blocks[blockId]; if (block.insts.length == 0) { continue; } let term = block.insts[block.insts.length - 1]; let op = body.op(term); if (op == IrOp.await_) { let resumeTarget = body.extra[body.successorRecord(term, 0)]; let pc = pcOfBlock[resumeTarget]; let futureVal = body.a[term]; let isHop = hopTerms.has(term); let subscribeOpt = if (let (true, s) = subscribeOf.get(term)) s else null; if (subscribeOpt == null && !isHop) { throw new Error('async split: await site without a subscribe target'); } let rebuilt = new Array(block.insts.length - 1); var k = 0; while (k < block.insts.length - 1) { rebuilt.push(block.insts[k]); k += 1; } // Entering an await is a checkpoint: a cancelled scope raises // here, before the frame parks, and the unwind runs the user's // finalizers on its way to the cancellation capture. Awaits // inside a `shielded` region get none — the mask IS the shield. if (!shieldedAwaits.has(term)) { checkpointInto(blockId, rebuilt); } // $state = the resume pc. let pcConst = rawInst(body, IrOp.const_i32, i32Idx, pc, -1); rebuilt.push(pcConst); body.instBlock[pcConst] = blockId; let stIdx = body.extra.length; body.extra.push(frameParam); body.extra.push(pcConst); let st = rawInst(body, IrOp.struct_set, -1, stIdx, stateRef); rebuilt.push(st); body.instBlock[st] = blockId; // Park this frame as a Task: on the awaited future (the library // schedules it when the future settles — or immediately, if it // already had), or, for a union await's bare arm, directly on // the microtask queue — the value needs no completion, only the // one hop the always-async rule requires. let self = rawInst(body, IrOp.ref_as_non_null, internVt(body, new ValTypeRef(frame)), frameParam, -1); rebuilt.push(self); body.instBlock[self] = blockId; let packed = rawInst(body, IrOp.iface_pack, internVt(body, new ValTypeRef(fatPtr)), self, addRawRef(body, new IrRefGlobal(vtableGlobal))); rebuilt.push(packed); body.instBlock[packed] = blockId; let callIdx = body.extra.length; if (isHop) { body.extra.push(1); body.extra.push(packed); } else { body.extra.push(2); body.extra.push(futureVal); body.extra.push(packed); } let callee = if (isHop) { scheduleFn as WasmFunction } else { subscribeOpt as WasmFunction }; let slotCallee = if (let (true, c) = subscribeCalleeOf.get(term)) c else -1; let call = if (!isHop && slotCallee >= 0) { // A subclass overrides `subscribe`: call_ref through the // vtable slot the presplit lowering loaded (still in this // block — it precedes the terminator), so the override runs. let refIdx = body.extra.length; body.extra.push(slotCallee); body.extra.push(2); body.extra.push(futureVal); body.extra.push(packed); rawInst(body, IrOp.call_ref, -1, refIdx, -1) } else { rawInst(body, IrOp.call, -1, callIdx, addRawRef(body, new IrRefFunc(callee))) }; rebuilt.push(call); body.instBlock[call] = blockId; // The suspension itself: a plain return out of the region. body.opcode[term] = IrOp.ret as i32; body.a[term] = -1; body.b[term] = -1; rebuilt.push(term); replaceContents(block.insts, rebuilt); } else if (op == IrOp.ret) { // A return (or falling off a Future body) completes the // frame's own future and finishes the machine. let retVal = body.a[term]; let rebuilt = new Array(block.insts.length - 1); var k = 0; while (k < block.insts.length - 1) { rebuilt.push(block.insts[k]); k += 1; } let dC = rawInst(body, IrOp.const_i32, i32Idx, STATE_DONE, -1); rebuilt.push(dC); body.instBlock[dC] = blockId; let stIdx = body.extra.length; body.extra.push(frameParam); body.extra.push(dC); let st = rawInst(body, IrOp.struct_set, -1, stIdx, stateRef); rebuilt.push(st); body.instBlock[st] = blockId; let weakFut = rawInst(body, IrOp.struct_get, internVt(body, new ValTypeRefNull(futureStruct)), frameParam, futureRef); rebuilt.push(weakFut); body.instBlock[weakFut] = blockId; let fut = rawInst(body, IrOp.ref_as_non_null, internVt(body, new ValTypeRef(futureStruct)), weakFut, -1); rebuilt.push(fut); body.instBlock[fut] = blockId; let adopts = body.adoptReturns.has(term); let callIdx = body.extra.length; if (adopts) { // `tail return f()`: settle the frame's future FROM the // callee's (async.md §8) — the value is the future itself, // present at every T. if (retVal < 0) { throw new Error('async split: adopting return without a future'); } body.extra.push(2); body.extra.push(fut); body.extra.push(retVal); } else if (valueVt == null) { body.extra.push(1); body.extra.push(fut); } else { if (retVal < 0) { throw new Error('async split: valued future completed without a value'); } body.extra.push(2); body.extra.push(fut); body.extra.push(retVal); } let call = rawInst(body, IrOp.call, -1, callIdx, addRawRef(body, new IrRefFunc( if (adopts) adoptFn else completeFn))); rebuilt.push(call); body.instBlock[call] = blockId; body.a[term] = -1; body.b[term] = -1; rebuilt.push(term); replaceContents(block.insts, rebuilt); } } // 6b. Resuming from an await is the other half of the checkpoint: // test the frame's scope at the head of every resume block, before // the resumed segment reads the awaited future. The raise lands // inside the re-entered user regions, so finalizers run. The // awaited-future-was-cancelled case needs nothing here: the resume // reads the future with valueOrThrow, which raises on the // cancellation channel for a cancelled future. for (let rt in resumeTargets) { let block = body.blocks[rt]; let rebuilt = new Array(); if (shieldedResumes.has(rt)) { // Resuming inside a `shielded` region: no checkpoint, and the // current scope — which the run() interposer set to the frame's scope — // is re-swapped to the shield scope, so work created after the // resume still binds outside every cancellation domain. let m = sm as ScopeMachinery; let sf = shieldFn as WasmFunction; let scopeWeak = rawInst(body, IrOp.struct_get, internVt(body, weakenToNullable(m.scopeVt)), frameParam, scopeFieldRef); body.instBlock[scopeWeak] = rt; rebuilt.push(scopeWeak); let scopeVal = rawInst(body, IrOp.ref_as_non_null, internVt(body, m.scopeVt), scopeWeak, -1); body.instBlock[scopeVal] = rt; rebuilt.push(scopeVal); let scIdx = body.extra.length; body.extra.push(1); body.extra.push(scopeVal); let sc = rawInst(body, IrOp.call, internVt(body, m.scopeVt), scIdx, addRawRef(body, new IrRefFunc(sf))); body.instBlock[sc] = rt; rebuilt.push(sc); let swIdx = body.extra.length; body.extra.push(1); body.extra.push(sc); let sw = rawInst(body, IrOp.call, internVt(body, m.scopeVt), swIdx, addRawRef(body, new IrRefFunc(m.installFn))); body.instBlock[sw] = rt; rebuilt.push(sw); if (let (true, fvId) = shieldedResumeFuture.get(rt)) { // A cancel WAKES parked frames, and here the checkpoint is // masked — so a spurious wake would fall through to reading a // future that has not settled. Guard: still pending means // re-park (the frame stays subscribed; the real resume comes // when the future settles). The rest of the resume moves to a // continuation block the guard branches to. let contB = newBlockId(); // The moved tail still carries var ops; step 7's // frame-rewrite walks bodyBlocks, so the new block joins it. bodyBlocks.push(contB); let cont = body.blocks[contB]; for (let inst in block.insts) { cont.insts.push(inst); body.instBlock[inst] = contB; } truncate(block.insts); let reparkB = newBlockId(); let pcC = emitIn(reparkB, IrOp.const_i32, i32Idx, pcOfBlock[rt], -1); storeState(reparkB, pcC); retVoidIn(reparkB); let fvVt = body.resultType(fvId) as ValType; let weakVt = weakenToNullable(fvVt); var fv = rawInst(body, IrOp.struct_get, internVt(body, weakVt), frameParam, addRawRef(body, new IrRefField(frame, '$v' + i32ToString(fvId)))); body.instBlock[fv] = rt; rebuilt.push(fv); if (!valTypesStrictlyEqual(weakVt, fvVt)) { fv = rawInst(body, IrOp.ref_as_non_null, internVt(body, fvVt), fv, -1); body.instBlock[fv] = rt; rebuilt.push(fv); } let futStructOpt = refTargetOf(fvVt); if (futStructOpt == null || !(futStructOpt is WasmStruct)) { throw new Error('shielded resume guard: awaited value is not ' + 'a future struct'); } let futStruct = futStructOpt as WasmStruct; // Future's `#state` field (stored under its class-scoped key, // `::#state`), FutureState.Pending == 0 — pinned by // a comment on the declaration in zena:async. var stateField: String | null = null; for (let f in futStruct.fields) { let fn = f.name; if (fn != null) { let n = fn as String; if (n == '#state' || (n.length > 8 && n.sliceBytes(n.length - 8, n.length) == '::#state')) { stateField = n; } } } if (stateField == null) { throw new Error('shielded resume guard: no #state field on ' + 'the awaited future struct'); } let st = rawInst(body, IrOp.struct_get, i32Idx, fv, addRawRef(body, new IrRefField(futStruct, stateField as String))); body.instBlock[st] = rt; rebuilt.push(st); let zero = rawInst(body, IrOp.const_i32, i32Idx, 0, -1); body.instBlock[zero] = rt; rebuilt.push(zero); let isPending = rawInst(body, IrOp.i32_eq, i32Idx, st, zero); body.instBlock[isPending] = rt; rebuilt.push(isPending); replaceContents(block.insts, rebuilt); brIfTo(rt, isPending, reparkB, contB); continue; } } else { checkpointInto(rt, rebuilt); } for (let inst in block.insts) { rebuilt.push(inst); } replaceContents(block.insts, rebuilt); } // 7. Mutable variables move into the frame; their wasm locals would // not survive a suspension. rewriteVarsToFrame(body, bodyBlocks, frame, frameParam); return body; } // --------------------------------------------------------------------------- // Async generators (`async gen` -> `AsyncIterator`). // --------------------------------------------------------------------------- function prepareOneAsyncGen( wasm: WasmModule, pass: ReachabilityAnalysis, func: WasmFunction, funcIndex: i32 ): void { let name = funcDisplayName(func); let iterIface = iteratorInstantiationOf(wasm, func); pass.typeLowerer.discoverType(iterIface as Type, func.model); let iterIKey = getInterfaceKey(iterIface); if (!wasm.interfaceStructs.has(iterIKey) || !wasm.interfaceVTableStructs.has(iterIKey)) { throw new Error('async generator AsyncIterator was not materialized: ' + name); } let iterFatPtr = wasm.interfaceStructs[iterIKey]; let iterVst = wasm.interfaceVTableStructs[iterIKey]; let slotSig = getInterfaceMemberSignature(wasm, iterIface, 'next'); // (discriminant, value, future), with no value result for a // zero-width value and one per field for an unboxed tuple. let stepValue = stepValueOf(wasm, iterIface, func); let valueLanes = slotSig.returns.length - 2; if (valueLanes < 0 || valueLanes > stepValue.lanes() || (stepValue.struct != null && valueLanes != stepValue.lanes()) || !(slotSig.returns[0] is ValTypeI32)) { throw new Error('unexpected AsyncIterator.next result shape for ' + name); } let futureOpt = asyncIteratorPendingFuture(wasm, iterIface); if (futureOpt == null) { throw new Error('async generator AsyncIterator has no Pending future: ' + name); } let futureType = futureOpt as ClassType; pass.typeLowerer.discoverType(futureType as Type, func.model); let fKey = getClassKey(futureType); let futureInfo = if (let (true, info) = wasm.classInfos.get(fKey)) info else null; if (futureInfo == null) { throw new Error('async generator Future instantiation was not materialized: ' + name); } let futureStruct = futureInfo.structType; let completeFn = methodOf(wasm, fKey, ':complete', name); let failFn = methodOf(wasm, fKey, ':fail', name); let futureCtor = methodOf(wasm, fKey, '', name); let someOpt = pendingSomeClass(futureType); let noneOpt = pendingNoneClass(futureType); if (someOpt == null || noneOpt == null) { throw new Error('async generator Option variants missing for ' + name); } let someCt = someOpt as ClassType; let noneCt = noneOpt as ClassType; pass.typeLowerer.discoverType(someCt as Type, func.model); pass.typeLowerer.discoverType(noneCt as Type, func.model); let someCtor = methodOf(wasm, getClassKey(someCt), '', name); let noneCtor = methodOf(wasm, getClassKey(noneCt), '', name); var sm: ScopeMachinery | null = null; var childScopeCtor: WasmFunction | null = null; var cancelScopeFn: WasmFunction | null = null; if (wasm.scopeCancellationActive) { let currentScopeFn = stdlibFn(wasm, 'currentScope', name); if (currentScopeFn.signature.returns.length != 1) { throw new Error('async lowering: currentScope does not return a scope'); } let scopeVt = currentScopeFn.signature.returns[0]; let scopeKey = scopeClassKeyOf(scopeVt, name); sm = new ScopeMachinery( currentScopeFn, scopeVt, methodOf(wasm, scopeKey, ':install', name), methodOf(wasm, scopeKey, ':checkpoint', name), methodOf(wasm, scopeKey, ':registerTask', name), methodOf(wasm, fKey, ':cancelComplete', name), wasm.cancellationTag as WasmTag); childScopeCtor = methodOf(wasm, scopeKey, '.:childScope', name); cancelScopeFn = methodOf(wasm, scopeKey, ':cancelScope', name); } if (wasm.exceptionTag == null || wasm.exceptionPayload == null) { throw new Error('async generator lowering: exception infrastructure missing for ' + name); } let yieldVt = if (someCtor.signature.params.length == 1) { someCtor.signature.params[0] } else if (slotSig.returns.length >= 3) { slotSig.returns[1] } else { null }; let fl = new FunctionLowerer(wasm, func); fl.presplitGenerator = true; fl.presplitYieldVt = yieldVt; fl.presplitAsync = true; fl.presplitAsyncValueVt = null; let presplit = fl.lower(); let subscribeOf = fl.awaitSubscribeTargets; let subscribeCalleeOf = fl.awaitSubscribeCallees; let hopTerms = fl.awaitHopTerms; let shieldedAwaits = fl.shieldedAwaits; var scheduleFn: WasmFunction | null = null; if (hopTerms.size > 0) { let scheduleOpt = wasm.getStdlibFunc('async', 'scheduleTask'); if (scheduleOpt == null) { throw new Error('async lowering: zena:async scheduleTask was not ' + 'reached (needed by a union await in ' + name + ')'); } scheduleFn = scheduleOpt; } let cfg = new IrCfg(presplit); let suspendBlocks = new Array(); let resumeOf = new Array(); let shieldedResumes = new Set(); let shieldedResumeFuture = new Map(); var bi = 0; while (bi < presplit.blocks.length) { let insts = presplit.blocks[bi].insts; if (insts.length > 0) { let term = insts[insts.length - 1]; let op = presplit.op(term); if (op == IrOp.yield_ || op == IrOp.await_) { suspendBlocks.push(bi); let resumeTarget = presplit.extra[presplit.successorRecord(term, 0)]; resumeOf.push(resumeTarget); if (op == IrOp.await_ && shieldedAwaits.has(term)) { shieldedResumes.add(resumeTarget); if (!hopTerms.has(term)) { shieldedResumeFuture[resumeTarget] = presplit.a[term]; } } } } bi += 1; } var shieldFn: WasmFunction | null = null; if (shieldedResumes.size > 0) { if (sm == null) { throw new Error('async lowering: shielded region without the ' + 'cancellation machinery in ' + name); } let scopeKey = scopeClassKeyOf((sm as ScopeMachinery).scopeVt, name); shieldFn = methodOf(wasm, scopeKey, ':shieldScope', name); } let pcOfBlock = new Map(); var si = 0; while (si < resumeOf.length) { pcOfBlock[resumeOf[si]] = STATE_FIRST_SUSPEND + si; si += 1; } var nextPc = STATE_FIRST_SUSPEND + resumeOf.length; var hb = 0; while (hb < presplit.blocks.length) { if (cfg.isReachable(hb) && !pcOfBlock.has(hb) && hb != 0 && loopContainsSuspend(presplit, cfg, hb, suspendBlocks)) { pcOfBlock[hb] = nextPc; nextPc += 1; } hb += 1; } nextPc = addRegionMergeTargets(presplit, cfg, pcOfBlock, nextPc); let dispatchRegions = dispatchRegionsOf(presplit, cfg, pcOfBlock); let liveIn = computeLiveIn(presplit, cfg); let spilled = new Set(); let entryParamIds = new Array(); for (let p in presplit.blocks[0].params) { entryParamIds.push(p); spilled.add(p); } let dTargets = growable([0]); var db = 0; while (db < presplit.blocks.length) { if (pcOfBlock.has(db)) { dTargets.push(db); for (let v in liveIn[db]) { spilled.add(v); } for (let p in presplit.blocks[db].params) { spilled.add(p); } } db += 1; } for (let v in liveIn[0]) { spilled.add(v); } spillCatchLiveIns(liveIn, dispatchRegions, spilled); let disposal = wasm.generatorDisposalActive; let fields = growable([ new WasmField('$state', new ValTypeI32(), true), ]); if (disposal) { fields.push(new WasmField('$disposing', new ValTypeI32(), true)); } fields.push(new WasmField('$future', new ValTypeRefNull(futureStruct), true)); if (sm != null) { fields.push(new WasmField('$scope', weakenToNullable((sm as ScopeMachinery).scopeVt), true)); } var vid = 0; while (vid < presplit.instCount()) { if (spilled.has(vid)) { let vt = presplit.resultType(vid); if (vt == null) { throw new Error('async generator spill of valueless instruction'); } fields.push(new WasmField('$v' + i32ToString(vid), weakenToNullable(vt), true)); } vid += 1; } varFieldsOf(presplit, fields); let frameName = `${name}$${i32ToString(funcIndex)}$AsyncGenFrame`; let frame = new WasmStruct(frameName, fields); if (disposal) { frame.superStruct = wasm.generatorFrameSuper; } wasm.types.push(frame); pass.markTypeReached(frame); // next(): (ref null $Frame) -> AsyncIterator.next's scalarized results. let nextParams = growable([new ValTypeRefNull(frame)]); let nextSig = wasm.getSignature(nextParams, slotSig.returns); let nextFunc = new WasmFunction(nextSig, null, func.model); nextFunc.name = frameName + '.next'; nextFunc.isAsyncStep = true; wasm.addFunction(nextFunc); pass.markFunctionReached(nextFunc); // AsyncIterator vtable global for the frame. let frameKey = frameName as ClassKey; let iterMethods = new Array(); for (let field in iterVst.fields) { if ((field.name as String) != 'next') { throw new Error('unexpected AsyncIterator vtable member: ' + (field.name as String)); } let tramp = new WasmFunction(slotSig, null, func.model); tramp.name = getInterfaceTrampolineName(frameKey, iterIKey, 'next'); tramp.interfaceTrampolineTarget = nextFunc; wasm.addFunction(tramp); pass.markFunctionReached(tramp); iterMethods.push(tramp); } let iterVtableInit = new VTableInit(iterVst, iterMethods); let iterViName = getClassInterfaceVTableName(frameKey, iterIKey); let iterVtableGlobal = new WasmGlobal( iterViName, iterViName, new ValTypeRef(iterVst), false, null, iterVtableInit, func.model ); wasm.globals.push(iterVtableGlobal); wasm.classInterfaceVTables[getClassInterfaceKey(frameKey, iterIKey)] = iterVtableGlobal; wasm.usedInterfaceAdaptations.add(getClassInterfaceKey(frameKey, iterIKey)); // step(): (ref null $Frame) -> (). The Waiter entry called when an // awaited future inside the async generator settles (or its scope is // cancelled). Delegates to nextFunc and drops the returned Step. let stepParams = growable([new ValTypeRefNull(frame)]); let stepSig = wasm.getSignature(stepParams, new Array(0)); let stepFunc = new WasmFunction(stepSig, null, func.model); stepFunc.name = frameName + '.step'; stepFunc.isAsyncStep = true; wasm.addFunction(stepFunc); pass.markFunctionReached(stepFunc); let sb = new IrBuilder(); let sEntry = sb.newBlock(); let sFrame = sb.blockParam(sEntry, new ValTypeRefNull(frame)); sb.switchTo(sEntry); let sArgs = growable([sFrame]); sb.callFuncMulti(sb.addRef(new IrRefFunc(nextFunc)), sArgs, slotSig.returns); sb.retVoid(); stepOuterBodies[stepFunc.uid] = sb.finish(); let taskIface = resumableInterfaceOf(futureType); pass.typeLowerer.discoverType(taskIface as Type, func.model); let taskIKey = getInterfaceKey(taskIface); if (!wasm.interfaceStructs.has(taskIKey) || !wasm.interfaceVTableStructs.has(taskIKey)) { throw new Error('async Task interface was not materialized: ' + name); } let taskFatPtr = wasm.interfaceStructs[taskIKey]; let taskVst = wasm.interfaceVTableStructs[taskIKey]; let taskSlotSig = getInterfaceMemberSignature(wasm, taskIface, 'run'); let taskMethods = new Array(); for (let field in taskVst.fields) { if ((field.name as String) != 'run') { throw new Error('unexpected Task vtable member: ' + (field.name as String)); } let tramp = new WasmFunction(taskSlotSig, null, func.model); tramp.name = getInterfaceTrampolineName(frameKey, taskIKey, 'run'); tramp.interfaceTrampolineTarget = stepFunc; wasm.addFunction(tramp); pass.markFunctionReached(tramp); taskMethods.push(tramp); } let taskVtableInit = new VTableInit(taskVst, taskMethods); let taskViName = getClassInterfaceVTableName(frameKey, taskIKey); let taskVtableGlobal = new WasmGlobal( taskViName, taskViName, new ValTypeRef(taskVst), false, null, taskVtableInit, func.model ); wasm.globals.push(taskVtableGlobal); wasm.classInterfaceVTables[getClassInterfaceKey(frameKey, taskIKey)] = taskVtableGlobal; wasm.usedInterfaceAdaptations.add(getClassInterfaceKey(frameKey, taskIKey)); let rampBody = buildAsyncGenRamp(func, frame, entryParamIds, sm, childScopeCtor, iterFatPtr, iterVtableGlobal, taskFatPtr, taskVtableGlobal); let nextBody = buildAsyncGenNextBody(presplit, cfg, frame, futureStruct, spilled, pcOfBlock, dTargets, slotSig, stepValue, subscribeOf, subscribeCalleeOf, hopTerms, scheduleFn, completeFn, failFn, futureCtor, someCtor, noneCtor, sm, cancelScopeFn, disposal, if (disposal) wasm.cancellationTag else null, resumeOf, shieldedAwaits, shieldedResumes, shieldedResumeFuture, shieldFn, taskFatPtr, taskVtableGlobal, wasm.exceptionTag as WasmTag, wasm.exceptionPayload as WasmGlobal, dispatchRegions); let artifacts = new AsyncArtifacts(rampBody, nextBody); func.isAsyncRamp = true; rampBodies[func.uid] = artifacts; stepBodies[nextFunc.uid] = artifacts; } function buildAsyncGenRamp( func: WasmFunction, frame: WasmStruct, entryParamIds: GrowableArray, sm: ScopeMachinery | null, childScopeCtor: WasmFunction | null, iterFatPtr: WasmStruct, iterVtableGlobal: WasmGlobal, taskFatPtr: WasmStruct, taskVtableGlobal: WasmGlobal ): IrBody { let b = new IrBuilder(); let entry = b.newBlock(); let params = new Array(); for (let vt in func.signature.params) { params.push(b.blockParam(entry, vt)); } b.switchTo(entry); let frameRef = b.addRef(new IrRefType(frame)); let inst = b.emit(IrOp.struct_new_default, new ValTypeRef(frame), -1, frameRef); var i = 0; while (i < params.length) { let fieldRef = b.addRef(new IrRefField(frame, '$v' + i32ToString(entryParamIds[i]))); let pair = growable([inst, params[i]]); b.emitVoid(IrOp.struct_set, b.pushExtra(pair), fieldRef); i += 1; } if (sm != null) { let m = sm as ScopeMachinery; let ctor = childScopeCtor as WasmFunction; let scope = b.callFunc(b.addRef(new IrRefFunc(ctor)), new Array(), m.scopeVt); let scopeField = b.addRef(new IrRefField(frame, '$scope')); let scopePair = growable([inst, scope]); b.emitVoid(IrOp.struct_set, b.pushExtra(scopePair), scopeField); let packedTask = b.emit(IrOp.iface_pack, new ValTypeRef(taskFatPtr), inst, b.addRef(new IrRefGlobal(taskVtableGlobal))); let bindArgs = growable([scope, packedTask]); b.callFunc(b.addRef(new IrRefFunc(m.registerFn)), bindArgs, null); } let packedIter = b.emit(IrOp.iface_pack, new ValTypeRef(iterFatPtr), inst, b.addRef(new IrRefGlobal(iterVtableGlobal))); b.ret(packedIter); return b.finish(); } function buildAsyncGenNextBody( body: IrBody, cfg: IrCfg, frame: WasmStruct, futureStruct: WasmStruct, spilled: HashSet, pcOfBlockIn: HashMap, dTargetsIn: GrowableArray, slotSig: WasmSignature, stepValue: StepValue, subscribeOf: HashMap, subscribeCalleeOf: HashMap, hopTerms: HashSet, scheduleFn: WasmFunction | null, completeFn: WasmFunction, failFn: WasmFunction, futureCtor: WasmFunction, someCtor: WasmFunction, noneCtor: WasmFunction, sm: ScopeMachinery | null, cancelScopeFn: WasmFunction | null, disposal: boolean, disposalTag: WasmTag | null, resumeTargets: GrowableArray, shieldedAwaits: HashSet, shieldedResumes: HashSet, shieldedResumeFuture: HashMap, shieldFn: WasmFunction | null, taskFatPtr: WasmStruct, taskVtableGlobal: WasmGlobal, exceptionTag: WasmTag, exceptionPayload: WasmGlobal, dispatchRegions: HashMap> ): IrBody { let origCount = body.blocks.length; let stateRef = addRawRef(body, new IrRefField(frame, '$state')); let futureRef = addRawRef(body, new IrRefField(frame, '$future')); let disposingRef = if (disposal) { addRawRef(body, new IrRefField(frame, '$disposing')) } else { -1 }; let scopeFieldRef = if (sm != null) { addRawRef(body, new IrRefField(frame, '$scope')) } else { -1 }; let cancelTagOpt = if (sm != null) { (sm as ScopeMachinery).tag } else { disposalTag }; let hasCancelRegion = cancelTagOpt != null; let cancelTagRef = if (cancelTagOpt != null) { addRawRef(body, new IrRefTag(cancelTagOpt as WasmTag)) } else { -1 }; let i32Vt: ValType = new ValTypeI32(); let dParams = new Map>(); for (let d in dTargetsIn) { let copy = new Array(); for (let p in body.blocks[d].params) { copy.push(p); } dParams[d] = copy; } // 1. Evacuate the old entry into a fresh block E; block 0 becomes // the prologue head. let entryCopy = new IrBlock(); let oldEntry = body.blocks[0]; body.blocks.push(entryCopy); let entryCopyId = body.blocks.length - 1; for (let p in oldEntry.params) { entryCopy.params.push(p); body.instBlock[p] = entryCopyId; } for (let inst in oldEntry.insts) { entryCopy.insts.push(inst); body.instBlock[inst] = entryCopyId; } body.blocks[0] = new IrBlock(); let pcOfBlock: HashMap = {entryCopyId => 0}; for (let d in dTargetsIn) { if (d == 0) { continue; } pcOfBlock[d] = pcOfBlockIn[d]; } let dParamsById = new Map>(); for (let d in dTargetsIn) { let mapped = if (d == 0) { entryCopyId } else { d }; dParamsById[mapped] = dParams[d]; } // 2. Prologue: install scope, enter capture regions, check state, dispatch. let frameParam = rawParam(body, 0, new ValTypeRefNull(frame)); let newBlockId = () => { body.blocks.push(new IrBlock()); return body.blocks.length - 1; }; let tryBody = newBlockId(); let catchBlock = newBlockId(); let doneBlock = newBlockId(); let runCheckBlock = newBlockId(); let enterBlock = newBlockId(); let trapBlock = newBlockId(); let dispatcher = newBlockId(); var errRegion = -1; var cancelCatch = -1; if (hasCancelRegion) { errRegion = newBlockId(); cancelCatch = newBlockId(); } let emitIn = (blockId: i32, op: IrOp, typeIdx: i32, a: i32, b: i32) => { let inst = rawInst(body, op, typeIdx, a, b); placeInst(body, blockId, inst); return inst; }; let brTo = (fromBlock: i32, target: i32, args: GrowableArray) => { let idx = body.extra.length; body.extra.push(target); body.extra.push(args.length); for (let v in args) { body.extra.push(v); } let inst = rawInst(body, IrOp.br, -1, idx, -1); placeInst(body, fromBlock, inst); }; let brIfTo = (fromBlock: i32, cond: i32, thenTarget: i32, elseTarget: i32) => { let idx = body.extra.length; body.extra.push(thenTarget); body.extra.push(0); body.extra.push(elseTarget); body.extra.push(0); let inst = rawInst(body, IrOp.br_if, -1, cond, idx); placeInst(body, fromBlock, inst); }; let storeState = (blockId: i32, valueInst: i32) => { let idx = body.extra.length; body.extra.push(frameParam); body.extra.push(valueInst); let inst = rawInst(body, IrOp.struct_set, -1, idx, stateRef); placeInst(body, blockId, inst); }; let clearFutureIn = (blockId: i32) => { let nullFut = emitIn(blockId, IrOp.ref_null, internVt(body, new ValTypeRefNull(futureStruct)), -1, -1); let idx = body.extra.length; body.extra.push(frameParam); body.extra.push(nullFut); let inst = rawInst(body, IrOp.struct_set, -1, idx, futureRef); placeInst(body, blockId, inst); }; let loadFuture = (blockId: i32) => { let weak = emitIn(blockId, IrOp.struct_get, internVt(body, new ValTypeRefNull(futureStruct)), frameParam, futureRef); return emitIn(blockId, IrOp.ref_as_non_null, internVt(body, new ValTypeRef(futureStruct)), weak, -1); }; let callIn = (blockId: i32, target: WasmFunction, args: GrowableArray) => { let idx = body.extra.length; body.extra.push(args.length); for (let v in args) { body.extra.push(v); } let inst = rawInst(body, IrOp.call, -1, idx, addRawRef(body, new IrRefFunc(target))); placeInst(body, blockId, inst); }; let callValIn = ( blockId: i32, target: WasmFunction, args: GrowableArray, retVt: ValType ) => { let idx = body.extra.length; body.extra.push(args.length); for (let v in args) { body.extra.push(v); } let inst = rawInst(body, IrOp.call, internVt(body, retVt), idx, addRawRef(body, new IrRefFunc(target))); placeInst(body, blockId, inst); return inst; }; let i32Idx = internVt(body, i32Vt); var entryScope = -1; var prevScope = -1; if (sm != null) { let m = sm as ScopeMachinery; let scopeWeak = emitIn(0, IrOp.struct_get, internVt(body, weakenToNullable(m.scopeVt)), frameParam, scopeFieldRef); entryScope = emitIn(0, IrOp.ref_as_non_null, internVt(body, m.scopeVt), scopeWeak, -1); let swapArgs = growable([entryScope]); prevScope = callValIn(0, m.installFn, swapArgs, m.scopeVt); } let restoreScopeIn = (blockId: i32) => { if (sm == null) { return; } let m = sm as ScopeMachinery; let restoreArgs = growable([prevScope]); let ignored = callValIn(blockId, m.installFn, restoreArgs, m.scopeVt); }; // The value results of a step, placed in `blockId`: none for a // zero-width value, else `value` (its fields when the step unboxes // it), or defaults when `value` is -1. let valueLanesIn = (blockId: i32, value: i32) => { if (slotSig.returns.length == 2) { return new Array(); } let lanes = rawValueLanes(body, slotSig.returns, stepValue, value); for (let lane in lanes) { if (body.instBlock[lane] < 0) { placeInst(body, blockId, lane); } } return lanes; }; let retStepIn = ( blockId: i32, discVal: i32, lanes: GrowableArray, futInst: i32 ) => { restoreScopeIn(blockId); let extraIdx = rawStepResults(body, discVal, lanes, futInst); let inst = rawInst(body, IrOp.ret_multi, -1, extraIdx, -1); placeInst(body, blockId, inst); }; let retDoneIn = (blockId: i32) => { let disc = emitIn(blockId, IrOp.const_i32, i32Idx, 0, -1); let dfltLanes = valueLanesIn(blockId, -1); let dfltFut = rawDefaultOf(body, slotSig.returns[slotSig.returns.length - 1]); placeInst(body, blockId, dfltFut); retStepIn(blockId, disc, dfltLanes, dfltFut); }; if (hasCancelRegion) { let cancelIdx = body.extra.length; body.extra.push(errRegion); body.extra.push(0); body.extra.push(cancelCatch); body.extra.push(0); let cancelInst = rawInst(body, IrOp.try_br, -1, cancelTagRef, cancelIdx); placeInst(body, 0, cancelInst); } let tryIdx = body.extra.length; body.extra.push(tryBody); body.extra.push(0); body.extra.push(catchBlock); body.extra.push(0); let tryInst = rawInst(body, IrOp.try_br, -1, addRawRef(body, new IrRefTag(exceptionTag)), tryIdx); placeInst(body, if (hasCancelRegion) errRegion else 0, tryInst); // tryBody: state = frame.$state; state == DONE ? done : runCheck. let stateVal = emitIn(tryBody, IrOp.struct_get, i32Idx, frameParam, stateRef); let doneConst = emitIn(tryBody, IrOp.const_i32, i32Idx, STATE_DONE, -1); let isDone = emitIn(tryBody, IrOp.i32_eq, i32Idx, stateVal, doneConst); brIfTo(tryBody, isDone, doneBlock, runCheckBlock); retDoneIn(doneBlock); // runCheck: state == RUNNING ? trap : enter. let runConst = emitIn(runCheckBlock, IrOp.const_i32, i32Idx, STATE_RUNNING, -1); let isRunning = emitIn(runCheckBlock, IrOp.i32_eq, i32Idx, stateVal, runConst); brIfTo(runCheckBlock, isRunning, trapBlock, enterBlock); let trapInst = rawInst(body, IrOp.unreachable, -1, -1, -1); placeInst(body, trapBlock, trapInst); // enter: if $disposing is set and scope machinery is active, cancel // the generator's child scope so in-flight work is cancelled and the // resume checkpoint raises on the cancellation channel. var doEnterB = enterBlock; if (disposal && sm != null) { let disposeCancelB = newBlockId(); doEnterB = newBlockId(); let dv = emitIn(enterBlock, IrOp.struct_get, i32Idx, frameParam, disposingRef); brIfTo(enterBlock, dv, disposeCancelB, doEnterB); let cancelScopeArgs = growable([entryScope]); callIn(disposeCancelB, cancelScopeFn as WasmFunction, cancelScopeArgs); brTo(disposeCancelB, doEnterB, new Array(0)); } let runningConst = emitIn(doEnterB, IrOp.const_i32, i32Idx, STATE_RUNNING, -1); storeState(doEnterB, runningConst); let dispatchArgs = growable([stateVal]); brTo(doEnterB, dispatcher, dispatchArgs); // catch: the body threw. If $future is null, rethrow synchronously; // otherwise fail the pending step future and return Done. let payloadVt = exceptionPayload.type; var payload = emitIn(catchBlock, IrOp.global_get, internVt(body, payloadVt), -1, addRawRef(body, new IrRefGlobal(exceptionPayload))); if (payloadVt is ValTypeRefNull) { let target = refTargetOf(payloadVt) as WasmType; payload = emitIn(catchBlock, IrOp.ref_as_non_null, internVt(body, new ValTypeRef(target)), payload, -1); } let doneC = emitIn(catchBlock, IrOp.const_i32, i32Idx, STATE_DONE, -1); storeState(catchBlock, doneC); let weakErrFut = emitIn(catchBlock, IrOp.struct_get, internVt(body, new ValTypeRefNull(futureStruct)), frameParam, futureRef); let isSyncErr = emitIn(catchBlock, IrOp.ref_is_null, i32Idx, weakErrFut, -1); let syncErrB = newBlockId(); let asyncErrB = newBlockId(); brIfTo(catchBlock, isSyncErr, syncErrB, asyncErrB); restoreScopeIn(syncErrB); let rethrowErrIdx = body.extra.length; body.extra.push(0); let rethrowErr = rawInst(body, IrOp.throw, -1, rethrowErrIdx, addRawRef(body, new IrRefTag(exceptionTag))); placeInst(body, syncErrB, rethrowErr); let failFut = emitIn(asyncErrB, IrOp.ref_as_non_null, internVt(body, new ValTypeRef(futureStruct)), weakErrFut, -1); clearFutureIn(asyncErrB); let failArgs = growable([failFut, payload]); callIn(asyncErrB, failFn, failArgs); retDoneIn(asyncErrB); // cancel catch: a cancellation or disposal unwound the body. if (hasCancelRegion) { let cDoneC = emitIn(cancelCatch, IrOp.const_i32, i32Idx, STATE_DONE, -1); storeState(cancelCatch, cDoneC); let weakCFut = emitIn(cancelCatch, IrOp.struct_get, internVt(body, new ValTypeRefNull(futureStruct)), frameParam, futureRef); let isSyncC = emitIn(cancelCatch, IrOp.ref_is_null, i32Idx, weakCFut, -1); let syncCancelB = newBlockId(); let asyncCancelB = newBlockId(); brIfTo(cancelCatch, isSyncC, syncCancelB, asyncCancelB); let cancelFut = emitIn(asyncCancelB, IrOp.ref_as_non_null, internVt(body, new ValTypeRef(futureStruct)), weakCFut, -1); clearFutureIn(asyncCancelB); if (sm != null) { let cancelArgs = growable([cancelFut]); callIn(asyncCancelB, (sm as ScopeMachinery).cancelFn, cancelArgs); } retDoneIn(asyncCancelB); if (disposal) { let dv = emitIn(syncCancelB, IrOp.struct_get, i32Idx, frameParam, disposingRef); let disposeDoneB = newBlockId(); let rethrowCancelB = newBlockId(); brIfTo(syncCancelB, dv, disposeDoneB, rethrowCancelB); retDoneIn(disposeDoneB); restoreScopeIn(rethrowCancelB); let rtIdx = body.extra.length; body.extra.push(0); let rtInst = rawInst(body, IrOp.throw, -1, rtIdx, cancelTagRef); placeInst(body, rethrowCancelB, rtInst); } else { restoreScopeIn(syncCancelB); let rtIdx = body.extra.length; body.extra.push(0); let rtInst = rawInst(body, IrOp.throw, -1, rtIdx, cancelTagRef); placeInst(body, syncCancelB, rtInst); } } let checkpointInto = (blockId: i32, out: GrowableArray) => { if (sm == null) { return; } let m = sm as ScopeMachinery; let weak = rawInst(body, IrOp.struct_get, internVt(body, weakenToNullable(m.scopeVt)), frameParam, scopeFieldRef); body.instBlock[weak] = blockId; out.push(weak); let strong = rawInst(body, IrOp.ref_as_non_null, internVt(body, m.scopeVt), weak, -1); body.instBlock[strong] = blockId; out.push(strong); let callIdx = body.extra.length; body.extra.push(1); body.extra.push(strong); let call = rawInst(body, IrOp.call, -1, callIdx, addRawRef(body, new IrRefFunc(m.checkpointFn))); body.instBlock[call] = blockId; out.push(call); }; let reenterRegions = (dispatchTarget: i32) => { let regions = if (let (true, rs) = dispatchRegions.get(dispatchTarget)) rs else new Array(0); var target = dispatchTarget; var ri = regions.length - 1; while (ri >= 0) { let r = regions[ri]; let t = newBlockId(); let idx = body.extra.length; body.extra.push(target); body.extra.push(0); body.extra.push(r.catchTarget); body.extra.push(0); let inst = rawInst(body, IrOp.try_br, -1, r.tagRef, idx); placeInst(body, t, inst); if (r.joinTarget >= 0) { body.tryJoin[inst] = r.joinTarget; } target = t; ri -= 1; } return target; }; let pcParam = rawParam(body, dispatcher, i32Vt); let chainTargets = new Array(); let chainPcs = new Array(); chainTargets.push(reenterRegions(entryCopyId)); chainPcs.push(0); var scanPc = STATE_FIRST_SUSPEND; var remaining = pcOfBlock.size - 1; while (remaining > 0) { var found = -1; var fb = 0; while (fb < body.blocks.length) { if (pcOfBlock.has(fb) && pcOfBlock[fb] == scanPc) { found = fb; break; } fb += 1; } if (found >= 0) { chainTargets.push(reenterRegions(found)); chainPcs.push(scanPc); remaining -= 1; } scanPc += 1; } var chainBlock = dispatcher; var ci = 0; while (ci < chainTargets.length) { let cmpConst = emitIn(chainBlock, IrOp.const_i32, i32Idx, chainPcs[ci], -1); let cmp = emitIn(chainBlock, IrOp.i32_eq, i32Idx, pcParam, cmpConst); let elseBlock = if (ci == chainTargets.length - 1) { trapBlock } else { newBlockId() }; brIfTo(chainBlock, cmp, chainTargets[ci], elseBlock); chainBlock = elseBlock; ci += 1; } let bodyBlocks = growable([entryCopyId]); var obi = 1; while (obi < origCount) { bodyBlocks.push(obi); obi += 1; } // 3. Reroute every edge into a dispatch target through the dispatcher. for (let blockId in bodyBlocks) { rerouteEdges(body, blockId, pcOfBlock, dParamsById, dispatcher, frameParam, frame, i32Idx, entryCopyId); } for (let d in dTargetsIn) { let mapped = if (d == 0) { entryCopyId } else { d }; truncate(body.blocks[mapped].params); } // 4. Store every spilled definition to its frame field. let skipRemap = new Set(); for (let blockId in bodyBlocks) { let block = body.blocks[blockId]; let rebuilt = new Array(); for (let p in block.params) { if (spilled.has(p)) { let idx = body.extra.length; body.extra.push(frameParam); body.extra.push(p); let st = rawInst(body, IrOp.struct_set, -1, idx, addRawRef(body, new IrRefField(frame, '$v' + i32ToString(p)))); skipRemap.add(st); rebuilt.push(st); body.instBlock[st] = blockId; } } for (let inst in block.insts) { rebuilt.push(inst); if (spilled.has(inst) && !isIrTerminator(body.op(inst))) { let idx = body.extra.length; body.extra.push(frameParam); body.extra.push(inst); let st = rawInst(body, IrOp.struct_set, -1, idx, addRawRef(body, new IrRefField(frame, '$v' + i32ToString(inst)))); skipRemap.add(st); rebuilt.push(st); body.instBlock[st] = blockId; } } replaceContents(block.insts, rebuilt); } // 5. Load every use of a spilled value from its frame field. for (let blockId in bodyBlocks) { let block = body.blocks[blockId]; let rebuilt = new Array(); let ops = new Array(); for (let inst in block.insts) { if (skipRemap.has(inst)) { rebuilt.push(inst); continue; } truncate(ops); body.appendValueOperands(inst, ops); let loadOf = new Map(); for (let v in ops) { if (spilled.has(v) && !loadOf.has(v)) { let vt = body.resultType(v) as ValType; let weak = weakenToNullable(vt); var load = rawInst(body, IrOp.struct_get, internVt(body, weak), frameParam, addRawRef(body, new IrRefField(frame, '$v' + i32ToString(v)))); rebuilt.push(load); body.instBlock[load] = blockId; if (!valTypesStrictlyEqual(weak, vt)) { load = rawInst(body, IrOp.ref_as_non_null, internVt(body, vt), load, -1); rebuilt.push(load); body.instBlock[load] = blockId; } loadOf[v] = load; } } if (loadOf.size > 0) { body.rewriteValueOperands(inst, (v) => { if (let (true, l) = loadOf.get(v)) { return l; } return v; }); } rebuilt.push(inst); } replaceContents(block.insts, rebuilt); } // 6. Rewrite yield_, await_, and ret terminators. for (let blockId in bodyBlocks) { let block = body.blocks[blockId]; if (block.insts.length == 0) { continue; } let term = block.insts[block.insts.length - 1]; let op = body.op(term); if (op == IrOp.yield_) { let resumeTarget = body.extra[body.successorRecord(term, 0)]; let pc = pcOfBlock[resumeTarget]; let yieldVal = body.a[term]; let rebuilt = new Array(block.insts.length - 1); var k = 0; while (k < block.insts.length - 1) { rebuilt.push(block.insts[k]); k += 1; } let pcConst = rawInst(body, IrOp.const_i32, i32Idx, pc, -1); rebuilt.push(pcConst); body.instBlock[pcConst] = blockId; let stIdx = body.extra.length; body.extra.push(frameParam); body.extra.push(pcConst); let st = rawInst(body, IrOp.struct_set, -1, stIdx, stateRef); rebuilt.push(st); body.instBlock[st] = blockId; let weakFut = rawInst(body, IrOp.struct_get, internVt(body, new ValTypeRefNull(futureStruct)), frameParam, futureRef); rebuilt.push(weakFut); body.instBlock[weakFut] = blockId; let isSync = rawInst(body, IrOp.ref_is_null, i32Idx, weakFut, -1); rebuilt.push(isSync); body.instBlock[isSync] = blockId; replaceContents(block.insts, rebuilt); let syncYieldB = newBlockId(); let asyncYieldB = newBlockId(); brIfTo(blockId, isSync, syncYieldB, asyncYieldB); let readyDisc = emitIn(syncYieldB, IrOp.const_i32, i32Idx, 1, -1); let readyLanes = valueLanesIn(syncYieldB, yieldVal); let nullFut = rawDefaultOf(body, slotSig.returns[slotSig.returns.length - 1]); placeInst(body, syncYieldB, nullFut); retStepIn(syncYieldB, readyDisc, readyLanes, nullFut); let fut = emitIn(asyncYieldB, IrOp.ref_as_non_null, internVt(body, new ValTypeRef(futureStruct)), weakFut, -1); clearFutureIn(asyncYieldB); let someArgs = new Array(); if (someCtor.signature.params.length == 1) { var yv = yieldVal; let yvVt = body.resultType(yv) as ValType; let paramVt = someCtor.signature.params[0]; if (yvVt is ValTypeRefNull && paramVt is ValTypeRef) { yv = emitIn(asyncYieldB, IrOp.ref_as_non_null, internVt(body, paramVt), yv, -1); } someArgs.push(yv); } let someVal = callValIn(asyncYieldB, someCtor, someArgs, someCtor.signature.returns[0]); let compArgs = growable([fut, someVal]); callIn(asyncYieldB, completeFn, compArgs); retDoneIn(asyncYieldB); } else if (op == IrOp.await_) { let resumeTarget = body.extra[body.successorRecord(term, 0)]; let pc = pcOfBlock[resumeTarget]; let futureVal = body.a[term]; let isHop = hopTerms.has(term); let subscribeOpt = if (let (true, s) = subscribeOf.get(term)) s else null; if (subscribeOpt == null && !isHop) { throw new Error('async generator split: await site without a subscribe target'); } let rebuilt = new Array(block.insts.length - 1); var k = 0; while (k < block.insts.length - 1) { rebuilt.push(block.insts[k]); k += 1; } if (!shieldedAwaits.has(term)) { checkpointInto(blockId, rebuilt); } let pcConst = rawInst(body, IrOp.const_i32, i32Idx, pc, -1); rebuilt.push(pcConst); body.instBlock[pcConst] = blockId; let stIdx = body.extra.length; body.extra.push(frameParam); body.extra.push(pcConst); let st = rawInst(body, IrOp.struct_set, -1, stIdx, stateRef); rebuilt.push(st); body.instBlock[st] = blockId; let weakFut = rawInst(body, IrOp.struct_get, internVt(body, new ValTypeRefNull(futureStruct)), frameParam, futureRef); rebuilt.push(weakFut); body.instBlock[weakFut] = blockId; let needAlloc = rawInst(body, IrOp.ref_is_null, i32Idx, weakFut, -1); rebuilt.push(needAlloc); body.instBlock[needAlloc] = blockId; replaceContents(block.insts, rebuilt); let allocFutB = newBlockId(); let parkB = newBlockId(); brIfTo(blockId, needAlloc, allocFutB, parkB); let newFut = callValIn(allocFutB, futureCtor, new Array(0), new ValTypeRef(futureStruct)); let setFutIdx = body.extra.length; body.extra.push(frameParam); body.extra.push(newFut); let setFut = rawInst(body, IrOp.struct_set, -1, setFutIdx, futureRef); placeInst(body, allocFutB, setFut); brTo(allocFutB, parkB, new Array(0)); let self = emitIn(parkB, IrOp.ref_as_non_null, internVt(body, new ValTypeRef(frame)), frameParam, -1); let packed = emitIn(parkB, IrOp.iface_pack, internVt(body, new ValTypeRef(taskFatPtr)), self, addRawRef(body, new IrRefGlobal(taskVtableGlobal))); let callIdx = body.extra.length; if (isHop) { body.extra.push(1); body.extra.push(packed); } else { body.extra.push(2); body.extra.push(futureVal); body.extra.push(packed); } let callee = if (isHop) { scheduleFn as WasmFunction } else { subscribeOpt as WasmFunction }; let slotCallee = if (let (true, c) = subscribeCalleeOf.get(term)) c else -1; let subCall = if (!isHop && slotCallee >= 0) { let refIdx = body.extra.length; body.extra.push(slotCallee); body.extra.push(2); body.extra.push(futureVal); body.extra.push(packed); rawInst(body, IrOp.call_ref, -1, refIdx, -1) } else { rawInst(body, IrOp.call, -1, callIdx, addRawRef(body, new IrRefFunc(callee))) }; placeInst(body, parkB, subCall); let pendingDisc = emitIn(parkB, IrOp.const_i32, i32Idx, 2, -1); let pendingLanes = valueLanesIn(parkB, -1); let pendingFut = loadFuture(parkB); retStepIn(parkB, pendingDisc, pendingLanes, pendingFut); } else if (op == IrOp.ret) { let rebuilt = new Array(block.insts.length - 1); var k = 0; while (k < block.insts.length - 1) { rebuilt.push(block.insts[k]); k += 1; } let dC = rawInst(body, IrOp.const_i32, i32Idx, STATE_DONE, -1); rebuilt.push(dC); body.instBlock[dC] = blockId; let stIdx = body.extra.length; body.extra.push(frameParam); body.extra.push(dC); let st = rawInst(body, IrOp.struct_set, -1, stIdx, stateRef); rebuilt.push(st); body.instBlock[st] = blockId; let weakFut = rawInst(body, IrOp.struct_get, internVt(body, new ValTypeRefNull(futureStruct)), frameParam, futureRef); rebuilt.push(weakFut); body.instBlock[weakFut] = blockId; let isSync = rawInst(body, IrOp.ref_is_null, i32Idx, weakFut, -1); rebuilt.push(isSync); body.instBlock[isSync] = blockId; replaceContents(block.insts, rebuilt); let syncRetB = newBlockId(); let asyncRetB = newBlockId(); brIfTo(blockId, isSync, syncRetB, asyncRetB); retDoneIn(syncRetB); let fut = emitIn(asyncRetB, IrOp.ref_as_non_null, internVt(body, new ValTypeRef(futureStruct)), weakFut, -1); clearFutureIn(asyncRetB); let noneVal = callValIn(asyncRetB, noneCtor, new Array(0), noneCtor.signature.returns[0]); let compArgs = growable([fut, noneVal]); callIn(asyncRetB, completeFn, compArgs); retDoneIn(asyncRetB); } } // 6b. Resume checkpoints (both yield and await resumes). for (let rt in resumeTargets) { let block = body.blocks[rt]; let rebuilt = new Array(); if (shieldedResumes.has(rt)) { let m = sm as ScopeMachinery; let sf = shieldFn as WasmFunction; let scopeWeak = rawInst(body, IrOp.struct_get, internVt(body, weakenToNullable(m.scopeVt)), frameParam, scopeFieldRef); body.instBlock[scopeWeak] = rt; rebuilt.push(scopeWeak); let scopeVal = rawInst(body, IrOp.ref_as_non_null, internVt(body, m.scopeVt), scopeWeak, -1); body.instBlock[scopeVal] = rt; rebuilt.push(scopeVal); let scIdx = body.extra.length; body.extra.push(1); body.extra.push(scopeVal); let sc = rawInst(body, IrOp.call, internVt(body, m.scopeVt), scIdx, addRawRef(body, new IrRefFunc(sf))); body.instBlock[sc] = rt; rebuilt.push(sc); let swIdx = body.extra.length; body.extra.push(1); body.extra.push(sc); let sw = rawInst(body, IrOp.call, internVt(body, m.scopeVt), swIdx, addRawRef(body, new IrRefFunc(m.installFn))); body.instBlock[sw] = rt; rebuilt.push(sw); if (let (true, fvId) = shieldedResumeFuture.get(rt)) { let contB = newBlockId(); bodyBlocks.push(contB); let cont = body.blocks[contB]; for (let inst in block.insts) { cont.insts.push(inst); body.instBlock[inst] = contB; } truncate(block.insts); let reparkB = newBlockId(); let pcC = emitIn(reparkB, IrOp.const_i32, i32Idx, pcOfBlock[rt], -1); storeState(reparkB, pcC); retDoneIn(reparkB); let fvVt = body.resultType(fvId) as ValType; let weakVt = weakenToNullable(fvVt); var fv = rawInst(body, IrOp.struct_get, internVt(body, weakVt), frameParam, addRawRef(body, new IrRefField(frame, '$v' + i32ToString(fvId)))); body.instBlock[fv] = rt; rebuilt.push(fv); if (!valTypesStrictlyEqual(weakVt, fvVt)) { fv = rawInst(body, IrOp.ref_as_non_null, internVt(body, fvVt), fv, -1); body.instBlock[fv] = rt; rebuilt.push(fv); } let futStructOpt = refTargetOf(fvVt); if (futStructOpt == null || !(futStructOpt is WasmStruct)) { throw new Error('shielded resume guard: awaited value is not ' + 'a future struct'); } let futStruct = futStructOpt as WasmStruct; var stateField: String | null = null; for (let f in futStruct.fields) { let fn = f.name; if (fn != null) { let n = fn as String; if (n == '#state' || (n.length > 8 && n.sliceBytes(n.length - 8, n.length) == '::#state')) { stateField = n; } } } if (stateField == null) { throw new Error('shielded resume guard: no #state field on ' + 'the awaited future struct'); } let st = rawInst(body, IrOp.struct_get, i32Idx, fv, addRawRef(body, new IrRefField(futStruct, stateField as String))); body.instBlock[st] = rt; rebuilt.push(st); let zero = rawInst(body, IrOp.const_i32, i32Idx, 0, -1); body.instBlock[zero] = rt; rebuilt.push(zero); let isPending = rawInst(body, IrOp.i32_eq, i32Idx, st, zero); body.instBlock[isPending] = rt; rebuilt.push(isPending); replaceContents(block.insts, rebuilt); brIfTo(rt, isPending, reparkB, contB); continue; } } else if (sm != null) { checkpointInto(rt, rebuilt); } else if (disposal) { let contB = newBlockId(); bodyBlocks.push(contB); let cont = body.blocks[contB]; for (let inst in block.insts) { cont.insts.push(inst); body.instBlock[inst] = contB; } truncate(block.insts); let raiseB = newBlockId(); let rIdx = body.extra.length; body.extra.push(0); let rInst = rawInst(body, IrOp.throw, -1, rIdx, cancelTagRef); placeInst(body, raiseB, rInst); let dv = emitIn(rt, IrOp.struct_get, i32Idx, frameParam, disposingRef); brIfTo(rt, dv, raiseB, contB); continue; } for (let inst in block.insts) { rebuilt.push(inst); } replaceContents(block.insts, rebuilt); } // 7. Mutable variables move into the frame. rewriteVarsToFrame(body, bodyBlocks, frame, frameParam); return body; }