From 77cbbb54fbaeb5018047d350fb92bc78a92f5763 Mon Sep 17 00:00:00 2001 From: Matt Hargett Date: Sat, 26 Sep 2026 16:26:43 -0700 Subject: [PATCH 01/10] perf: skip unused value-stack lanes when entering a function Entering a function reserved and zero-filled all three value-stack lanes even when the function never touches the 64- or 128-bit lane, which is the common case. Skip a lane whose locals and operand-stack depth are both zero. A single-result return now moves the result down and truncates once, instead of popping, truncating and pushing it back. --- .../src/interpreter/stack/value_stack.rs | 26 +++++++++++++------ 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/crates/tinywasm/src/interpreter/stack/value_stack.rs b/crates/tinywasm/src/interpreter/stack/value_stack.rs index 1c40e31..3a9936e 100644 --- a/crates/tinywasm/src/interpreter/stack/value_stack.rs +++ b/crates/tinywasm/src/interpreter/stack/value_stack.rs @@ -135,12 +135,15 @@ impl Stack { self.data.truncate(n); } + /// Moves the top value down to index `n` and drops everything above it (a function's result + /// replacing its frame). #[inline(always)] pub(crate) fn truncate_to_one_tail(&mut self, n: usize) { - debug_assert!(n < self.data.len()); - let last = self.pop(); - self.data.truncate(n); - self.data.push(last); + let len = self.data.len(); + debug_assert!(n < len); + let last = self.data[len.wrapping_sub(1)]; + self.data[n] = last; + self.data.truncate(n + 1); } /// Enters a function: turns its parameters into the first locals, zeroes the rest, and reserves @@ -276,10 +279,17 @@ impl ValueStack { ) -> Result { let locals_base32 = self.stack_32.enter_locals(params.c32 as usize, locals.c32 as usize, max_stack.c32 as usize)?; - let locals_base64 = - self.stack_64.enter_locals(params.c64 as usize, locals.c64 as usize, max_stack.c64 as usize)?; - let locals_base128 = - self.stack_128.enter_locals(params.c128 as usize, locals.c128 as usize, max_stack.c128 as usize)?; + // Most functions use only the 32-bit lane; an unused lane needs no reservation. + let locals_base64 = if locals.c64 | max_stack.c64 == 0 { + self.stack_64.len() as u32 + } else { + self.stack_64.enter_locals(params.c64 as usize, locals.c64 as usize, max_stack.c64 as usize)? + }; + let locals_base128 = if locals.c128 | max_stack.c128 == 0 { + self.stack_128.len() as u32 + } else { + self.stack_128.enter_locals(params.c128 as usize, locals.c128 as usize, max_stack.c128 as usize)? + }; Ok(StackBase { s32: locals_base32, s64: locals_base64, s128: locals_base128 }) } From bae7a71d48b1db1a4dca51121c9029b403f80593 Mon Sep 17 00:00:00 2001 From: Matt Hargett Date: Sat, 26 Sep 2026 16:26:43 -0700 Subject: [PATCH 02/10] perf: borrow the executing function from its module instance Every call and return between two functions cloned the callee's `Shared` and dropped the previous one, two refcount updates each way, and looked the function up in the store. A module instance now keeps its own functions, which the store allocates contiguously. The interpreter holds the instance for the whole run, so the executor borrows the executing function and its module from it: a call or return within the instance switches a reference, and a direct call to one of the module's own functions skips the address table, the host check and the owner check. Execution that continues in another instance's function (a call through an import, table or reference, a return, or an exception unwinding into it) ends the run, and `InterpreterRuntime` resumes that frame with an executor for its instance. Fuel and time budgets carry over, so a run suspends at the same points as before. --- crates/tinywasm/src/instance.rs | 26 +- .../src/interpreter/executor/dispatch.rs | 53 ++-- .../interpreter/executor/dispatch_become.rs | 102 +++++--- .../tinywasm/src/interpreter/executor/mod.rs | 247 ++++++++++++------ crates/tinywasm/src/interpreter/mod.rs | 46 +++- crates/tinywasm/src/store/mod.rs | 4 +- crates/tinywasm/tests/cross_instance_calls.rs | 173 ++++++++++++ 7 files changed, 502 insertions(+), 149 deletions(-) create mode 100644 crates/tinywasm/tests/cross_instance_calls.rs diff --git a/crates/tinywasm/src/instance.rs b/crates/tinywasm/src/instance.rs index f92765d..45666a6 100644 --- a/crates/tinywasm/src/instance.rs +++ b/crates/tinywasm/src/instance.rs @@ -60,6 +60,11 @@ struct ModuleInstanceInner { id: ModuleInstanceId, type_addrs: Box<[TypeAddr]>, func_addrs: Box<[FuncAddr]>, + /// The module's own (non-imported) functions, in index order. The store allocates them + /// contiguously from `local_func_base`, so an executor can borrow any of them for a whole run. + local_funcs: Box<[Shared]>, + local_func_base: FuncAddr, + imported_funcs: u32, table_addrs: Box<[TableAddr]>, mem_addrs: Box<[MemAddr]>, #[cfg(feature = "std")] @@ -84,6 +89,20 @@ impl ModuleInstance { self.0.func_addrs[addr as usize] } + /// The store address and body of the module's own function `idx`, or `None` for an import. + #[inline(always)] + pub(crate) fn local_func_by_index(&self, idx: FuncAddr) -> Option<(FuncAddr, &WasmFunction)> { + let local = idx.wrapping_sub(self.0.imported_funcs); + let func = self.0.local_funcs.get(local as usize)?; + Some((self.0.local_func_base + local, func)) + } + + /// The body of the function at store address `addr`, if this instance owns it. + #[inline(always)] + pub(crate) fn local_func(&self, addr: FuncAddr) -> Option<&WasmFunction> { + self.0.local_funcs.get(addr.wrapping_sub(self.0.local_func_base) as usize).map(|func| &**func) + } + /// resolve a table address to the global store address #[inline] pub(crate) fn resolve_table_addr(&self, addr: TableAddr) -> TableAddr { @@ -211,7 +230,9 @@ impl ModuleInstance { let id = store.next_module_instance_id(); let mut addrs = crate::imports::ResolvedImports::new(store, module, &type_addrs, imports)?; let imported_funcs = addrs.funcs.len(); - addrs.funcs.extend(store.init_funcs(&module.funcs, id, &module.func_type_idxs[imported_funcs..], &type_addrs)); + let local_funcs = store.init_funcs(&module.funcs, id, &module.func_type_idxs[imported_funcs..], &type_addrs); + let local_func_base = local_funcs.start; + addrs.funcs.extend(local_funcs); addrs.tags.extend(store.init_tags(&module.tags, &type_addrs)); let limiter = store.engine.config().resource_limiter.clone(); if !module.skip_local_memory_allocation { @@ -238,6 +259,9 @@ impl ModuleInstance { id, type_addrs, func_addrs: addrs.funcs.into_boxed_slice(), + local_funcs: module.funcs.clone(), + local_func_base, + imported_funcs: imported_funcs as u32, table_addrs: addrs.tables.into_boxed_slice(), mem_addrs: addrs.memories.into_boxed_slice(), #[cfg(feature = "std")] diff --git a/crates/tinywasm/src/interpreter/executor/dispatch.rs b/crates/tinywasm/src/interpreter/executor/dispatch.rs index ac52047..0a2c4fe 100644 --- a/crates/tinywasm/src/interpreter/executor/dispatch.rs +++ b/crates/tinywasm/src/interpreter/executor/dispatch.rs @@ -24,67 +24,76 @@ macro_rules! define_stable_dispatch { }; } -impl Executor<'_> { +impl Executor<'_, '_> { instruction_handlers!(define_stable_dispatch); + /// Runs until the call completes (`None`) or continues in another module instance's frame. #[inline(always)] - pub(crate) fn run_to_completion(mut self) -> Result<()> { + pub(crate) fn run_to_completion(mut self) -> Result> { let mut instr_ptr = self.cf.instr_ptr; loop { match Self::exec_step(&mut self, instr_ptr)?.next_instr_ptr() { Some(next_instr_ptr) => instr_ptr = next_instr_ptr, - None => return cold!(Ok(())), + None => return cold!(Ok(self.left())), } } } + /// Runs `chunk_left` instructions, then checkpoint by checkpoint until `time_budget` has + /// elapsed since `start`. #[cfg(feature = "std")] #[inline(always)] - pub(crate) fn run_with_time_budget(mut self, time_budget: core::time::Duration) -> Result { - use crate::std::time::Instant; - - if time_budget.is_zero() { - return Ok(ExecState::Suspended(self.cf)); - } - let start = Instant::now(); + pub(crate) fn run_with_time_budget( + mut self, + start: crate::std::time::Instant, + time_budget: core::time::Duration, + mut chunk_left: u32, + ) -> Result { let mut instr_ptr = self.cf.instr_ptr; loop { - for _ in 0..CHECKPOINT_INTERVAL { + while chunk_left != 0 { + chunk_left -= 1; match Self::exec_step(&mut self, instr_ptr)?.next_instr_ptr() { Some(next_instr_ptr) => instr_ptr = next_instr_ptr, - None => return Ok(ExecState::Completed), + None => return Ok(self.run_end(chunk_left)), } } + chunk_left = CHECKPOINT_INTERVAL; if start.elapsed() >= time_budget { self.cf.instr_ptr = instr_ptr; - return Ok(ExecState::Suspended(self.cf)); + return Ok(RunEnd::State(ExecState::Suspended(self.cf))); } } } + /// Runs `chunk_left` instructions, then checkpoint by checkpoint until the store's fuel is out. #[inline(always)] - pub(crate) fn run_with_fuel(mut self, fuel: u32) -> Result { + pub(crate) fn run_with_fuel(mut self, mut chunk_left: u32) -> Result { self.fuel_metered = true; - self.store.execution_fuel = fuel; - if self.store.execution_fuel == 0 { - return Ok(ExecState::Suspended(self.cf)); - } - let mut instr_ptr = self.cf.instr_ptr; loop { - for _ in 0..CHECKPOINT_INTERVAL { + while chunk_left != 0 { + chunk_left -= 1; match Self::exec_step(&mut self, instr_ptr)?.next_instr_ptr() { Some(next_instr_ptr) => instr_ptr = next_instr_ptr, - None => return Ok(ExecState::Completed), + None => return Ok(self.run_end(chunk_left)), } } + chunk_left = CHECKPOINT_INTERVAL; self.store.execution_fuel = self.store.execution_fuel.saturating_sub(CHECKPOINT_INTERVAL); if self.store.execution_fuel == 0 { self.cf.instr_ptr = instr_ptr; - return Ok(ExecState::Suspended(self.cf)); + return Ok(RunEnd::State(ExecState::Suspended(self.cf))); } } } + + fn run_end(&self, chunk_left: u32) -> RunEnd { + match self.left() { + Some(frame) => RunEnd::Left(frame, chunk_left), + None => RunEnd::State(ExecState::Completed), + } + } } diff --git a/crates/tinywasm/src/interpreter/executor/dispatch_become.rs b/crates/tinywasm/src/interpreter/executor/dispatch_become.rs index f790d63..03ec2b8 100644 --- a/crates/tinywasm/src/interpreter/executor/dispatch_become.rs +++ b/crates/tinywasm/src/interpreter/executor/dispatch_become.rs @@ -3,8 +3,10 @@ use super::*; struct Unbudgeted; struct Bounded; -type UnbudgetedHandler = for<'store> fn(&mut Executor<'store>, &[Instruction], usize, Instruction) -> ExecResult<()>; -type BoundedHandler = for<'store> fn(&mut Executor<'store>, usize, Instruction, u32) -> ExecResult<()>; +type UnbudgetedHandler = + for<'store, 'module> fn(&mut Executor<'store, 'module>, &[Instruction], usize, Instruction) -> ExecResult<()>; +type BoundedHandler = + for<'store, 'module> fn(&mut Executor<'store, 'module>, usize, Instruction, u32) -> ExecResult<()>; macro_rules! define_unbudgeted_tail_dispatch { ($executor:ident, $instr_ptr:ident, $dispatch_next:ident, $dispatch_flow:ident; @@ -22,7 +24,7 @@ macro_rules! define_unbudgeted_tail_dispatch { $( #[allow(non_snake_case, unreachable_code, unused_imports, unused_macros, unused_variables)] fn $variant( - $executor: &mut Executor<'_>, + $executor: &mut Executor<'_, '_>, instructions: &[Instruction], $instr_ptr: usize, instruction: Instruction, @@ -45,7 +47,12 @@ macro_rules! define_unbudgeted_tail_dispatch { $executor.cf.instr_ptr = next_instr_ptr; return Ok(()); }, - ExecFlow::Complete => return cold!({ $executor.completed = true; Ok(()) }), + ExecFlow::Complete => return cold!({ + if !$executor.left { + $executor.completed = true; + } + Ok(()) + }), } }}; } @@ -79,7 +86,7 @@ macro_rules! define_bounded_tail_dispatch { $( #[allow(non_snake_case, unreachable_code, unused_imports, unused_macros, unused_variables)] fn $variant( - $executor: &mut Executor<'_>, + $executor: &mut Executor<'_, '_>, $instr_ptr: usize, instruction: Instruction, instructions_until_checkpoint: u32, @@ -106,7 +113,11 @@ macro_rules! define_bounded_tail_dispatch { match $flow.next_instr_ptr() { Some(next_instr_ptr) => $dispatch_next!(next_instr_ptr), None => return cold!({ - $executor.completed = true; + if $executor.left { + $executor.chunk_left = instructions_until_checkpoint; + } else { + $executor.completed = true; + } Ok(()) }), } @@ -134,13 +145,13 @@ impl Unbudgeted { #[cold] #[inline(never)] - fn handler_mismatch(_: &mut Executor<'_>, _: &[Instruction], _: usize, _: Instruction) -> ExecResult<()> { + fn handler_mismatch(_: &mut Executor<'_, '_>, _: &[Instruction], _: usize, _: Instruction) -> ExecResult<()> { unreachable!("instruction handler mismatch") } #[cold] #[inline(never)] - fn invalid_instr_ptr(_: &mut Executor<'_>, _: &[Instruction], instr_ptr: usize, _: Instruction) -> ExecResult<()> { + fn invalid_instr_ptr(_: &mut Executor<'_, '_>, _: &[Instruction], instr_ptr: usize, _: Instruction) -> ExecResult<()> { unreachable!("instruction pointer {instr_ptr} out of range, this is a bug") } } @@ -152,79 +163,92 @@ impl Bounded { #[cold] #[inline(never)] - fn handler_mismatch(_: &mut Executor<'_>, _: usize, _: Instruction, _: u32) -> ExecResult<()> { + fn handler_mismatch(_: &mut Executor<'_, '_>, _: usize, _: Instruction, _: u32) -> ExecResult<()> { unreachable!("instruction handler mismatch") } #[cold] #[inline(never)] - fn invalid_instr_ptr(_: &mut Executor<'_>, instr_ptr: usize, _: Instruction, _: u32) -> ExecResult<()> { + fn invalid_instr_ptr(_: &mut Executor<'_, '_>, instr_ptr: usize, _: Instruction, _: u32) -> ExecResult<()> { unreachable!("instruction pointer {instr_ptr} out of range, this is a bug") } + /// Runs up to `chunk_left` (at least 1) instructions from `executor.cf`. #[inline(always)] - fn run(executor: &mut Executor<'_>) -> ExecResult<()> { + fn run(executor: &mut Executor<'_, '_>, chunk_left: u32) -> ExecResult<()> { let instr_ptr = executor.cf.instr_ptr; let instruction = executor.func.instructions[instr_ptr]; let handler = Self::handler_for(instruction.opcode()); - handler(executor, instr_ptr, instruction, CHECKPOINT_INTERVAL - 1) + handler(executor, instr_ptr, instruction, chunk_left - 1) } } -impl<'store> Executor<'store> { +impl Executor<'_, '_> { + /// Runs until the call completes (`None`) or continues in another module instance's frame. #[inline(always)] - pub(crate) fn run_to_completion(mut self) -> Result<()> { + pub(crate) fn run_to_completion(mut self) -> Result> { loop { - let func = self.func.clone(); + let func = self.func; let instructions = &func.instructions; let instr_ptr = self.cf.instr_ptr; let instruction = instructions[instr_ptr]; let handler = Unbudgeted::handler_for(instruction.opcode()); handler(&mut self, instructions, instr_ptr, instruction)?; - if self.completed { - return Ok(()); + if self.completed || self.left { + return Ok(self.left()); } } } + /// Runs `chunk_left` instructions, then checkpoint by checkpoint until `time_budget` has + /// elapsed since `start`. #[cfg(feature = "std")] #[inline(always)] - pub(crate) fn run_with_time_budget(mut self, time_budget: core::time::Duration) -> Result { - use crate::std::time::Instant; - - if time_budget.is_zero() { - return Ok(ExecState::Suspended(self.cf)); - } - let start = Instant::now(); - + pub(crate) fn run_with_time_budget( + mut self, + start: crate::std::time::Instant, + time_budget: core::time::Duration, + mut chunk_left: u32, + ) -> Result { loop { - Bounded::run(&mut self)?; - if self.completed { - return cold!(Ok(ExecState::Completed)); + if chunk_left != 0 { + Bounded::run(&mut self, chunk_left)?; + if let Some(end) = self.run_end() { + return Ok(end); + } } + chunk_left = CHECKPOINT_INTERVAL; if start.elapsed() >= time_budget { - return cold!(Ok(ExecState::Suspended(self.cf))); + return cold!(Ok(RunEnd::State(ExecState::Suspended(self.cf)))); } } } + /// Runs `chunk_left` instructions, then checkpoint by checkpoint until the store's fuel is out. #[inline(always)] - pub(crate) fn run_with_fuel(mut self, fuel: u32) -> Result { + pub(crate) fn run_with_fuel(mut self, mut chunk_left: u32) -> Result { self.fuel_metered = true; - self.store.execution_fuel = fuel; - if self.store.execution_fuel == 0 { - return Ok(ExecState::Suspended(self.cf)); - } - loop { - Bounded::run(&mut self)?; - if self.completed { - return cold!(Ok(ExecState::Completed)); + if chunk_left != 0 { + Bounded::run(&mut self, chunk_left)?; + if let Some(end) = self.run_end() { + return Ok(end); + } } + chunk_left = CHECKPOINT_INTERVAL; self.store.execution_fuel = self.store.execution_fuel.saturating_sub(CHECKPOINT_INTERVAL); if self.store.execution_fuel == 0 { - return cold!(Ok(ExecState::Suspended(self.cf))); + return cold!(Ok(RunEnd::State(ExecState::Suspended(self.cf)))); } } } + + /// How a bounded chain that stopped ended, unless it stopped at a checkpoint. + #[inline(always)] + fn run_end(&self) -> Option { + if self.completed { + return cold!(Some(RunEnd::State(ExecState::Completed))); + } + self.left().map(|frame| RunEnd::Left(frame, self.chunk_left)) + } } diff --git a/crates/tinywasm/src/interpreter/executor/mod.rs b/crates/tinywasm/src/interpreter/executor/mod.rs index decc92b..33f88b6 100644 --- a/crates/tinywasm/src/interpreter/executor/mod.rs +++ b/crates/tinywasm/src/interpreter/executor/mod.rs @@ -26,7 +26,7 @@ mod dispatch; #[cfg(feature = "nightly-tail-calls")] mod dispatch_become; -const CHECKPOINT_INTERVAL: u32 = 128; +pub(crate) const CHECKPOINT_INTERVAL: u32 = 128; const FUEL_COST_CALL_TOTAL: u32 = 5; struct ExecError(Box); @@ -71,49 +71,80 @@ impl ExecFlow { } } -pub(crate) struct Executor<'store> { +/// Runs frames of one module instance. The executing function is borrowed from the instance, which +/// the caller keeps alive for the whole run, so calls and returns within the instance switch a +/// reference instead of cloning a `Shared` handle. Execution that continues in a frame of another +/// instance ends the run with [`Executor::left`] set; [`InterpreterRuntime`](super::InterpreterRuntime) +/// resumes that frame with an executor for its instance. +pub(crate) struct Executor<'store, 'module> { cf: CallFrame, - func: Shared, - module: ModuleInstance, + func: &'module WasmFunction, + module: &'module ModuleInstance, store: &'store mut Store, call_stack_base: u32, mem0: MemAddr, fuel_metered: bool, + /// Execution continues in `cf`, which belongs to another module instance. + left: bool, + /// Budgeted runs: instructions left before the next checkpoint when the run left. + #[cfg(feature = "nightly-tail-calls")] + chunk_left: u32, #[cfg(feature = "nightly-tail-calls")] completed: bool, } -impl<'store> Executor<'store> { - pub(crate) fn new(store: &'store mut Store, cf: CallFrame, call_stack_base: u32) -> Self { - let wasm_func = store.state.funcs.wasm(cf.func_addr); - let module = store.get_module_instance(wasm_func.owner).expect("invalid module instance").clone(); - let mem0 = module.mem0_addr(); +/// How a budgeted run of one executor ended. +pub(crate) enum RunEnd { + State(ExecState), + /// Execution continues in this frame of another module instance, with this many instructions + /// left before the next checkpoint. + Left(CallFrame, u32), +} + +impl<'store, 'module> Executor<'store, 'module> { + /// `module` must own the function of `cf`. + pub(crate) fn new( + store: &'store mut Store, + module: &'module ModuleInstance, + cf: CallFrame, + call_stack_base: u32, + ) -> Self { + let func = module.local_func(cf.func_addr).expect("call frame from another module instance"); Self { module, cf, - func: wasm_func.func.clone(), + func, store, call_stack_base, - mem0, + mem0: module.mem0_addr(), fuel_metered: false, + left: false, + #[cfg(feature = "nightly-tail-calls")] + chunk_left: 0, #[cfg(feature = "nightly-tail-calls")] completed: false, } } + /// The frame to resume in another module instance, if execution left this one. + #[inline(always)] + pub(crate) fn left(&self) -> Option { + self.left.then_some(self.cf) + } + + /// Ends the run because `self.cf` belongs to another module instance. + #[cold] + fn leave(&mut self) -> ExecFlow { + self.left = true; + ExecFlow::Complete + } + /// Resolves a module-local memory index to its store address, caching the common memory-0 case. #[inline(always)] fn mem_addr(&self, idx: MemAddr) -> MemAddr { if idx == 0 { self.mem0 } else { self.module.resolve_mem_addr(idx) } } - /// Switches the executor to another module, keeping the cached memory-0 address in sync. - #[inline] - fn set_module(&mut self, owner: ModuleInstanceId) { - self.module = self.store.get_module_instance(owner).expect("invalid module instance").clone(); - self.mem0 = self.module.mem0_addr(); - } - #[inline(always)] fn charge_call_fuel(&mut self, total_fuel_cost: u32) { if self.fuel_metered { @@ -577,8 +608,13 @@ impl<'store> Executor<'store> { } } - fn matching_catch(&self, protected_ip: usize, tag_addr: TagAddr) -> Option { - let handlers = &self.func.data.exception_handlers; + fn matching_catch( + func: &WasmFunction, + module: &ModuleInstance, + protected_ip: usize, + tag_addr: TagAddr, + ) -> Option { + let handlers = &func.data.exception_handlers; let end = handlers.partition_point(|handler| handler.start_ip as usize <= protected_ip); handlers[..end] .iter() @@ -586,35 +622,27 @@ impl<'store> Executor<'store> { .filter(|handler| protected_ip < handler.end_ip as usize) .flat_map(|handler| handler.catches.iter().copied()) .find(|catch| match catch { - ExceptionCatch::Tag { tag, .. } => self.module.resolve_tag_addr(*tag) == tag_addr, + ExceptionCatch::Tag { tag, .. } => module.resolve_tag_addr(*tag) == tag_addr, ExceptionCatch::All { .. } => true, }) } - #[inline(always)] - fn switch_to_frame(&mut self, frame: CallFrame) { - let previous = core::mem::replace(&mut self.cf, frame); - if previous.func_addr == self.cf.func_addr { - return; - } - - let wasm_func = self.store.state.funcs.wasm(self.cf.func_addr); - if !Shared::ptr_eq(&self.func, &wasm_func.func) { - self.func = wasm_func.func.clone(); - } - if wasm_func.owner != self.module.id() { - self.set_module(wasm_func.owner); - } - } - + /// Unwinds to the innermost handler that catches `exception`. Returns the flow to its landing + /// pad, or `None` if no frame of this run catches it. fn dispatch_exception(&mut self, exception: ValueRef, mut protected_ip: usize) -> Result, Trap> { let object = self.store.state.gc.get(exception).ok_or(Trap::InvalidReference)?; let crate::store::GcObjectKind::Exception(tag_addr) = object.kind else { return Err(Trap::InvalidReference); }; - let mut switched = false; + let entry_func_addr = self.cf.func_addr; + // The function and instance of the frame being searched while it belongs to another instance. + let mut foreign: Option<(Shared, ModuleInstance)> = None; loop { - if let Some(catch) = self.matching_catch(protected_ip, tag_addr) { + let catch = match &foreign { + None => Self::matching_catch(self.func, self.module, protected_ip, tag_addr), + Some((func, module)) => Self::matching_catch(func, module, protected_ip, tag_addr), + }; + if let Some(catch) = catch { let (landing_pad, base, with_ref, include_payload) = match catch { ExceptionCatch::Tag { landing_pad, base, with_ref, .. } => (landing_pad, base, with_ref, true), ExceptionCatch::All { landing_pad, base, with_ref } => (landing_pad, base, with_ref, false), @@ -636,10 +664,15 @@ impl<'store> Executor<'store> { if with_ref { ValueRef::stack_push(&mut self.store.value_stack, exception); } - return Ok(Some(if switched { - ExecFlow::Switch(landing_pad as usize) + if foreign.is_some() { + self.cf.instr_ptr = landing_pad as usize; + return Ok(Some(self.leave())); + } + let landing_pad = landing_pad as usize; + return Ok(Some(if self.cf.func_addr == entry_func_addr { + ExecFlow::Next(landing_pad) } else { - ExecFlow::Next(landing_pad as usize) + ExecFlow::Switch(landing_pad) })); } @@ -647,8 +680,21 @@ impl<'store> Executor<'store> { let Some(caller) = self.store.call_stack.pop_frame(self.call_stack_base) else { return Ok(None); }; - switched = true; - self.switch_to_frame(caller); + if caller.func_addr != self.cf.func_addr { + let module = self.module; + match module.local_func(caller.func_addr) { + Some(func) => { + self.func = func; + foreign = None; + } + None => { + let wasm_func = self.store.state.funcs.wasm(caller.func_addr); + let owner = self.store.get_module_instance(wasm_func.owner).expect("invalid module instance"); + foreign = Some((wasm_func.func.clone(), owner.clone())); + } + } + } + self.cf = caller; protected_ip = self.cf.instr_ptr.checked_sub(1).expect("invalid caller IP"); } } @@ -695,27 +741,34 @@ impl<'store> Executor<'store> { fn exec_call_direct(&mut self, v: u32, return_instr_ptr: usize) -> ExecResult { self.charge_call_fuel(FUEL_COST_CALL_TOTAL); - let addr = self.module.resolve_func_addr(v); - if self.store.state.funcs.is_host(addr) { - let host_func = self.store.state.funcs.host(addr); - let type_addr = host_func.type_addr; - let host_func = host_func.func.clone(); - self.exec_call_host::(host_func, type_addr, return_instr_ptr) - } else { - self.exec_call_wasm::(addr, return_instr_ptr) + let module = self.module; + match module.local_func_by_index(v) { + Some((addr, func)) => self.exec_call_local::(addr, func, return_instr_ptr), + None => self.exec_call_import::(v, return_instr_ptr), } } fn exec_return_call_direct(&mut self, v: u32) -> ExecResult { self.charge_call_fuel(FUEL_COST_CALL_TOTAL); + let module = self.module; + match module.local_func_by_index(v) { + Some((addr, func)) => self.exec_call_local::(addr, func, 0), + None => self.exec_call_import::(v, 0), + } + } + + /// Calls an imported function: a host function or another instance's Wasm function. + #[inline(always)] + fn exec_call_import(&mut self, v: u32, return_instr_ptr: usize) -> ExecResult { let addr = self.module.resolve_func_addr(v); if self.store.state.funcs.is_host(addr) { let host_func = self.store.state.funcs.host(addr); let type_addr = host_func.type_addr; let host_func = host_func.func.clone(); - self.exec_call_host::(host_func, type_addr, 0) + self.exec_call_host::(host_func, type_addr, return_instr_ptr) } else { - self.exec_call_wasm::(addr, 0) + // An instance cannot import its own functions. + self.exec_call_foreign::(addr, return_instr_ptr) } } @@ -787,35 +840,65 @@ impl<'store> Executor<'store> { self.exec_call_wasm::(func_addr, return_instr_ptr) } + /// Calls the Wasm function at store address `func_addr`. #[inline(always)] fn exec_call_wasm( &mut self, func_addr: FuncAddr, return_instr_ptr: usize, ) -> ExecResult { - let wasm_func = self.store.state.funcs.wasm(func_addr); - let (params, locals, max_stack, owner, next_func) = { - let next_func = (!Shared::ptr_eq(&self.func, &wasm_func.func)).then(|| wasm_func.func.clone()); - (wasm_func.func.params, wasm_func.func.locals, wasm_func.func.max_stack, wasm_func.owner, next_func) - }; - if TAIL { - self.store.value_stack.truncate_keep_counts(self.cf.locals_base, params); + let module = self.module; + match module.local_func(func_addr) { + Some(func) => self.exec_call_local::(func_addr, func, return_instr_ptr), + None => self.exec_call_foreign::(func_addr, return_instr_ptr), } - let locals_base = self.store.value_stack.enter_locals(¶ms, &locals, &max_stack)?; - if TAIL { - self.cf = CallFrame::new(func_addr, locals_base, locals); - } else { - self.store.call_stack.push(self.cf, return_instr_ptr)?; - self.cf = CallFrame::new(func_addr, locals_base, locals); + } + + /// Enters `func`, which this run's module instance owns. + #[inline(always)] + fn exec_call_local( + &mut self, + func_addr: FuncAddr, + func: &'module WasmFunction, + return_instr_ptr: usize, + ) -> ExecResult { + self.enter_frame::(func_addr, func, return_instr_ptr)?; + if core::ptr::eq(self.func, func) { + return Ok(ExecFlow::Next(0)); } - let switched = next_func.is_some(); - if let Some(next_func) = next_func { - self.func = next_func; + self.func = func; + Ok(ExecFlow::Switch(0)) + } + + /// Enters a function of another module instance, which ends this run. + #[inline(never)] + fn exec_call_foreign( + &mut self, + func_addr: FuncAddr, + return_instr_ptr: usize, + ) -> ExecResult { + let func = self.store.state.funcs.wasm(func_addr).func.clone(); + self.enter_frame::(func_addr, &func, return_instr_ptr)?; + Ok(self.leave()) + } + + /// Sets up the frame of a call to `func`, replacing the current one for a tail call. + #[inline(always)] + fn enter_frame( + &mut self, + func_addr: FuncAddr, + func: &WasmFunction, + return_instr_ptr: usize, + ) -> ExecResult<()> { + if TAIL { + self.store.value_stack.truncate_keep_counts(self.cf.locals_base, func.params); } - if owner != self.module.id() { - self.set_module(owner); + let locals_base = self.store.value_stack.enter_locals(&func.params, &func.locals, &func.max_stack)?; + if !TAIL { + self.store.call_stack.push(self.cf, return_instr_ptr)?; } - Ok(if switched { ExecFlow::Switch(0) } else { ExecFlow::Next(0) }) + self.cf = CallFrame::new(func_addr, locals_base, func.locals); + Ok(()) } fn exec_call_ref(&mut self, type_addr: u32, return_instr_ptr: usize) -> ExecResult { @@ -839,12 +922,18 @@ impl<'store> Executor<'store> { return ExecFlow::Complete; }; let instr_ptr = caller.instr_ptr; - if caller.func_addr == self.cf.func_addr { - self.cf = caller; - ExecFlow::Next(instr_ptr) - } else { - self.switch_to_frame(caller); - ExecFlow::Switch(instr_ptr) + let switched = caller.func_addr != self.cf.func_addr; + self.cf = caller; + if !switched { + return ExecFlow::Next(instr_ptr); + } + let module = self.module; + match module.local_func(caller.func_addr) { + Some(func) => { + self.func = func; + ExecFlow::Switch(instr_ptr) + } + None => self.leave(), } } diff --git a/crates/tinywasm/src/interpreter/mod.rs b/crates/tinywasm/src/interpreter/mod.rs index 0f7ae20..3400a94 100644 --- a/crates/tinywasm/src/interpreter/mod.rs +++ b/crates/tinywasm/src/interpreter/mod.rs @@ -26,20 +26,54 @@ pub(crate) enum ExecState { pub(crate) struct InterpreterRuntime; impl InterpreterRuntime { - pub(crate) fn exec(store: &mut Store, cf: CallFrame, call_stack_base: u32) -> Result<()> { - executor::Executor::new(store, cf, call_stack_base).run_to_completion() + pub(crate) fn exec(store: &mut Store, mut cf: CallFrame, call_stack_base: u32) -> Result<()> { + loop { + let module = Self::frame_module(store, &cf); + match executor::Executor::new(store, &module, cf, call_stack_base).run_to_completion()? { + None => return Ok(()), + Some(frame) => cf = frame, + } + } } - pub(crate) fn exec_with_fuel(store: &mut Store, cf: CallFrame, fuel: u32) -> Result { - executor::Executor::new(store, cf, 0).run_with_fuel(fuel) + pub(crate) fn exec_with_fuel(store: &mut Store, mut cf: CallFrame, fuel: u32) -> Result { + store.execution_fuel = fuel; + if fuel == 0 { + return Ok(ExecState::Suspended(cf)); + } + let mut chunk_left = executor::CHECKPOINT_INTERVAL; + loop { + let module = Self::frame_module(store, &cf); + match executor::Executor::new(store, &module, cf, 0).run_with_fuel(chunk_left)? { + executor::RunEnd::State(state) => return Ok(state), + executor::RunEnd::Left(frame, left) => (cf, chunk_left) = (frame, left), + } + } } #[cfg(feature = "std")] pub(crate) fn exec_with_time_budget( store: &mut Store, - cf: CallFrame, + mut cf: CallFrame, time_budget: core::time::Duration, ) -> Result { - executor::Executor::new(store, cf, 0).run_with_time_budget(time_budget) + if time_budget.is_zero() { + return Ok(ExecState::Suspended(cf)); + } + let start = crate::std::time::Instant::now(); + let mut chunk_left = executor::CHECKPOINT_INTERVAL; + loop { + let module = Self::frame_module(store, &cf); + match executor::Executor::new(store, &module, cf, 0).run_with_time_budget(start, time_budget, chunk_left)? { + executor::RunEnd::State(state) => return Ok(state), + executor::RunEnd::Left(frame, left) => (cf, chunk_left) = (frame, left), + } + } + } + + /// The module instance that owns the function of `cf`. + fn frame_module(store: &Store, cf: &CallFrame) -> crate::ModuleInstance { + let owner = store.state.funcs.wasm(cf.func_addr).owner; + store.get_module_instance(owner).expect("invalid module instance").clone() } } diff --git a/crates/tinywasm/src/store/mod.rs b/crates/tinywasm/src/store/mod.rs index 7bbab95..4e348fc 100644 --- a/crates/tinywasm/src/store/mod.rs +++ b/crates/tinywasm/src/store/mod.rs @@ -568,14 +568,14 @@ impl Store { result } - /// Add functions to the store, returning their addresses in the store + /// Add functions to the store, returning their (contiguous) addresses in the store pub(crate) fn init_funcs( &mut self, funcs: &[Shared], owner: ModuleInstanceId, module_type_idxs: &[TypeAddr], type_addrs: &[TypeAddr], - ) -> impl ExactSizeIterator { + ) -> core::ops::Range { debug_assert_eq!(funcs.len(), module_type_idxs.len()); let funcs = funcs.iter().cloned().zip(module_type_idxs).map(|(func, &type_idx)| (type_addrs[type_idx as usize], func)); diff --git a/crates/tinywasm/tests/cross_instance_calls.rs b/crates/tinywasm/tests/cross_instance_calls.rs new file mode 100644 index 0000000..1a3054f --- /dev/null +++ b/crates/tinywasm/tests/cross_instance_calls.rs @@ -0,0 +1,173 @@ +//! Calls, returns and exceptions that move between module instances. The interpreter runs one +//! instance's frames at a time and hands over to the other instance at each crossing. + +use tinywasm::{ExecProgress, Imports, ModuleInstance, Store}; + +type TestResult = Result<(), Box>; + +const MODULE_A: &str = r#" + (module + (type $i2i (func (param i32) (result i32))) + (tag $e (export "e") (param i32)) + (table $t (export "t") 4 funcref) + (func $add1 (export "add1") (param i32) (result i32) + (i32.add (local.get 0) (i32.const 1))) + (func (export "throw") (param i32) + (throw $e (local.get 0))) + ;; Calls table slot 0, which the other instance fills. + (func (export "apply") (param i32) (result i32) + (i32.add (call_indirect $t (type $i2i) (local.get 0) (i32.const 0)) (i32.const 100))) + ;; Same, catching `$e` from the callee. + (func (export "apply_catching") (param i32) (result i32) + (block $caught (result i32) + (try_table (result i32) (catch $e $caught) + (call_indirect $t (type $i2i) (local.get 0) (i32.const 0))) + (return)) + (i32.add (i32.const 1000))) + (elem (table $t) (i32.const 1) func $add1)) +"#; + +const MODULE_B: &str = r#" + (module + (type $i2i (func (param i32) (result i32))) + (import "a" "e" (tag $e (param i32))) + (import "a" "t" (table $t 4 funcref)) + (import "a" "add1" (func $add1 (param i32) (result i32))) + (import "a" "throw" (func $throw (param i32))) + (import "a" "apply" (func $apply (param i32) (result i32))) + (import "a" "apply_catching" (func $apply_catching (param i32) (result i32))) + (func $double (param i32) (result i32) + (i32.mul (local.get 0) (i32.const 2))) + (func $throws (param i32) (result i32) + (throw $e (local.get 0))) + (elem declare func $double $throws) + + (func (export "direct") (param i32) (result i32) + (call $add1 (call $add1 (local.get 0)))) + (func (export "tail") (param i32) (result i32) + (return_call $add1 (local.get 0))) + (func (export "indirect") (param i32) (result i32) + (call_indirect $t (type $i2i) (local.get 0) (i32.const 1))) + ;; B -> A -> B -> A -> B + (func (export "callback") (param i32) (result i32) + (table.set $t (i32.const 0) (ref.func $double)) + (call $apply (local.get 0))) + (func (export "catch_from_import") (param i32) (result i32) + (block $caught (result i32) + (try_table (catch $e $caught) + (call $throw (local.get 0))) + (i32.const -1))) + ;; Thrown in B, unwinds through A, caught in B. + (func (export "catch_through_import") (param i32) (result i32) + (table.set $t (i32.const 0) (ref.func $throws)) + (block $caught (result i32) + (try_table (result i32) (catch $e $caught) + (call $apply (local.get 0))) + (return)) + (i32.add (i32.const 2000))) + ;; Thrown in B, caught in A. + (func (export "caught_in_import") (param i32) (result i32) + (table.set $t (i32.const 0) (ref.func $throws)) + (call $apply_catching (local.get 0))) + ;; Four crossings per iteration. + (func (export "crossings") (param $n i32) (result i32) (local $sum i32) + (table.set $t (i32.const 0) (ref.func $double)) + (loop $next + (local.set $sum (i32.add (local.get $sum) (call $apply (local.get $n)))) + (local.set $n (i32.sub (local.get $n) (i32.const 1))) + (br_if $next (local.get $n))) + (local.get $sum))) +"#; + +fn instantiate(store: &mut Store) -> Result> { + let a = tinywasm::parse_bytes(&wat::parse_str(MODULE_A)?)?; + let b = tinywasm::parse_bytes(&wat::parse_str(MODULE_B)?)?; + let a = ModuleInstance::instantiate(store, &a, None)?; + let mut imports = Imports::new(); + imports.link_module("a", a)?; + Ok(ModuleInstance::instantiate(store, &b, Some(&imports))?) +} + +fn call( + store: &mut Store, + instance: &ModuleInstance, + name: &str, + arg: i32, +) -> Result> { + Ok(instance.func::(store, name)?.call(store, arg)?) +} + +#[test] +fn calls_and_returns_cross_instances() -> TestResult { + let mut store = Store::default(); + let b = instantiate(&mut store)?; + assert_eq!(call(&mut store, &b, "direct", 5)?, 7); + assert_eq!(call(&mut store, &b, "tail", 5)?, 6); + assert_eq!(call(&mut store, &b, "indirect", 5)?, 6); + assert_eq!(call(&mut store, &b, "callback", 5)?, 110); + Ok(()) +} + +#[test] +fn exceptions_cross_instances() -> TestResult { + let mut store = Store::default(); + let b = instantiate(&mut store)?; + assert_eq!(call(&mut store, &b, "catch_from_import", 7)?, 7); + assert_eq!(call(&mut store, &b, "catch_through_import", 9)?, 2009); + assert_eq!(call(&mut store, &b, "caught_in_import", 9)?, 1009); + // The instances keep working after each unwind. + assert_eq!(call(&mut store, &b, "callback", 5)?, 110); + Ok(()) +} + +#[test] +fn budgeted_runs_cross_instances() -> TestResult { + const N: i32 = 1000; + const EXPECTED: i32 = N * (N + 1) + 100 * N; + + let mut store = Store::default(); + let b = instantiate(&mut store)?; + assert_eq!(call(&mut store, &b, "crossings", N)?, EXPECTED); + + for fuel in [1, 3, 64, 1000] { + let mut store = Store::default(); + let b = instantiate(&mut store)?; + let func = b.func::(&store, "crossings")?; + let mut exec = func.call_resumable(&mut store, N)?; + let mut suspensions = 0; + let result = loop { + match exec.resume_with_fuel(fuel)? { + ExecProgress::Completed(value) => break value, + ExecProgress::Suspended => suspensions += 1, + } + }; + assert_eq!(result, EXPECTED, "fuel {fuel}"); + assert!(suspensions > 0, "fuel {fuel}"); + } + + #[cfg(feature = "std")] + { + let mut store = Store::default(); + let b = instantiate(&mut store)?; + let func = b.func::(&store, "crossings")?; + let mut exec = func.call_resumable(&mut store, N)?; + let result = loop { + match exec.resume_with_time_budget(std::time::Duration::from_micros(10))? { + ExecProgress::Completed(value) => break value, + ExecProgress::Suspended => {} + } + }; + assert_eq!(result, EXPECTED); + + // An effectively unlimited budget completes in one resume. + let mut store = Store::default(); + let b = instantiate(&mut store)?; + let func = b.func::(&store, "crossings")?; + let mut exec = func.call_resumable(&mut store, N)?; + match exec.resume_with_time_budget(std::time::Duration::MAX)? { + ExecProgress::Completed(value) => assert_eq!(value, EXPECTED), + ExecProgress::Suspended => panic!("suspended with an unlimited time budget"), + } + } + Ok(()) +} From 1b509551dbe3cdf3beaa7a856035952a380e68c0 Mon Sep 17 00:00:00 2001 From: Matt Hargett Date: Sat, 3 Oct 2026 12:13:42 -0700 Subject: [PATCH 03/10] perf: keep the instruction slice in a local in the match dispatch loop The match loop spilled the executor's function and reloaded it, then the instruction slice's pointer and length, on every step. The executing function is borrowed from the module instance, not from the executor, so the loop can hold its instructions in a local and reload them only when a step switches functions. --- .../src/interpreter/executor/dispatch.rs | 44 ++++++++++++++++--- .../tinywasm/src/interpreter/executor/mod.rs | 1 + 2 files changed, 39 insertions(+), 6 deletions(-) diff --git a/crates/tinywasm/src/interpreter/executor/dispatch.rs b/crates/tinywasm/src/interpreter/executor/dispatch.rs index 0a2c4fe..08477f7 100644 --- a/crates/tinywasm/src/interpreter/executor/dispatch.rs +++ b/crates/tinywasm/src/interpreter/executor/dispatch.rs @@ -4,7 +4,11 @@ macro_rules! define_stable_dispatch { ($executor:ident, $instr_ptr:ident, $dispatch_next:ident, $dispatch_flow:ident; $($variant:ident $(($($arg:pat),*))? $({ $($field:ident),* })? => $body:expr),* $(,)?) => { #[inline(always)] - fn exec_step($executor: &mut Self, $instr_ptr: usize) -> ExecResult { + fn exec_step( + $executor: &mut Self, + instructions: &[tinywasm_types::Instruction], + $instr_ptr: usize, + ) -> ExecResult { macro_rules! $dispatch_next { ($next_instr_ptr:expr) => {{ return Ok(ExecFlow::Next($next_instr_ptr)); @@ -16,7 +20,7 @@ macro_rules! define_stable_dispatch { }}; } use tinywasm_types::Instruction::*; - match &$executor.func.instructions[$instr_ptr] { + match &instructions[$instr_ptr] { $($variant $(($($arg),*))? $({ $($field),* })? => $body,)* } Ok(ExecFlow::Next($instr_ptr + 1)) @@ -24,15 +28,41 @@ macro_rules! define_stable_dispatch { }; } -impl Executor<'_, '_> { +impl<'module> Executor<'_, 'module> { instruction_handlers!(define_stable_dispatch); + /// The executing function's instructions. They are borrowed from the module instance, not from + /// the executor, so the dispatch loop keeps them in a local across steps. + #[inline(always)] + fn instructions(&self) -> &'module [tinywasm_types::Instruction] { + let func: &'module WasmFunction = self.func; + &func.instructions + } + + /// One step from `instr_ptr`. A step that switches functions reloads `instructions`. + #[inline(always)] + fn step( + &mut self, + instructions: &mut &'module [tinywasm_types::Instruction], + instr_ptr: usize, + ) -> ExecResult> { + Ok(match Self::exec_step(self, instructions, instr_ptr)? { + ExecFlow::Next(next_instr_ptr) => Some(next_instr_ptr), + ExecFlow::Switch(next_instr_ptr) => { + *instructions = self.instructions(); + Some(next_instr_ptr) + } + ExecFlow::Complete => None, + }) + } + /// Runs until the call completes (`None`) or continues in another module instance's frame. #[inline(always)] pub(crate) fn run_to_completion(mut self) -> Result> { let mut instr_ptr = self.cf.instr_ptr; + let mut instructions = self.instructions(); loop { - match Self::exec_step(&mut self, instr_ptr)?.next_instr_ptr() { + match self.step(&mut instructions, instr_ptr)? { Some(next_instr_ptr) => instr_ptr = next_instr_ptr, None => return cold!(Ok(self.left())), } @@ -50,10 +80,11 @@ impl Executor<'_, '_> { mut chunk_left: u32, ) -> Result { let mut instr_ptr = self.cf.instr_ptr; + let mut instructions = self.instructions(); loop { while chunk_left != 0 { chunk_left -= 1; - match Self::exec_step(&mut self, instr_ptr)?.next_instr_ptr() { + match self.step(&mut instructions, instr_ptr)? { Some(next_instr_ptr) => instr_ptr = next_instr_ptr, None => return Ok(self.run_end(chunk_left)), } @@ -72,10 +103,11 @@ impl Executor<'_, '_> { pub(crate) fn run_with_fuel(mut self, mut chunk_left: u32) -> Result { self.fuel_metered = true; let mut instr_ptr = self.cf.instr_ptr; + let mut instructions = self.instructions(); loop { while chunk_left != 0 { chunk_left -= 1; - match Self::exec_step(&mut self, instr_ptr)?.next_instr_ptr() { + match self.step(&mut instructions, instr_ptr)? { Some(next_instr_ptr) => instr_ptr = next_instr_ptr, None => return Ok(self.run_end(chunk_left)), } diff --git a/crates/tinywasm/src/interpreter/executor/mod.rs b/crates/tinywasm/src/interpreter/executor/mod.rs index 33f88b6..932a2b3 100644 --- a/crates/tinywasm/src/interpreter/executor/mod.rs +++ b/crates/tinywasm/src/interpreter/executor/mod.rs @@ -62,6 +62,7 @@ enum ExecFlow { } impl ExecFlow { + #[cfg(feature = "nightly-tail-calls")] #[inline(always)] fn next_instr_ptr(self) -> Option { match self { From 97fc951b418fd262b158048497e612de234cd93a Mon Sep 17 00:00:00 2001 From: Matt Hargett Date: Sat, 3 Oct 2026 16:19:30 -0700 Subject: [PATCH 04/10] perf: hold the module in the instance instead of copying its functions The instance kept its own copy of the module's function list for the executor to borrow from, which cost an allocation and a reference count per function on every instantiation, and as many again on drop. Hold the module instead, which costs one reference count per instance. --- crates/tinywasm/src/instance.rs | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/crates/tinywasm/src/instance.rs b/crates/tinywasm/src/instance.rs index 45666a6..b0b0b24 100644 --- a/crates/tinywasm/src/instance.rs +++ b/crates/tinywasm/src/instance.rs @@ -60,9 +60,10 @@ struct ModuleInstanceInner { id: ModuleInstanceId, type_addrs: Box<[TypeAddr]>, func_addrs: Box<[FuncAddr]>, - /// The module's own (non-imported) functions, in index order. The store allocates them - /// contiguously from `local_func_base`, so an executor can borrow any of them for a whole run. - local_funcs: Box<[Shared]>, + /// The module, whose functions are the instance's own (non-imported) ones in index order. The + /// store allocates them contiguously from `local_func_base`, so an executor can borrow any of + /// them for a whole run. Holding the module costs one reference count per instance. + module: Module, local_func_base: FuncAddr, imported_funcs: u32, table_addrs: Box<[TableAddr]>, @@ -93,14 +94,14 @@ impl ModuleInstance { #[inline(always)] pub(crate) fn local_func_by_index(&self, idx: FuncAddr) -> Option<(FuncAddr, &WasmFunction)> { let local = idx.wrapping_sub(self.0.imported_funcs); - let func = self.0.local_funcs.get(local as usize)?; + let func = self.0.module.funcs.get(local as usize)?; Some((self.0.local_func_base + local, func)) } /// The body of the function at store address `addr`, if this instance owns it. #[inline(always)] pub(crate) fn local_func(&self, addr: FuncAddr) -> Option<&WasmFunction> { - self.0.local_funcs.get(addr.wrapping_sub(self.0.local_func_base) as usize).map(|func| &**func) + self.0.module.funcs.get(addr.wrapping_sub(self.0.local_func_base) as usize).map(|func| &**func) } /// resolve a table address to the global store address @@ -259,7 +260,7 @@ impl ModuleInstance { id, type_addrs, func_addrs: addrs.funcs.into_boxed_slice(), - local_funcs: module.funcs.clone(), + module: module.clone(), local_func_base, imported_funcs: imported_funcs as u32, table_addrs: addrs.tables.into_boxed_slice(), From 756f9375407fb2b49a5ed7e8a3bdf344c54e777a Mon Sep 17 00:00:00 2001 From: Matt Hargett Date: Tue, 29 Sep 2026 19:14:08 -0700 Subject: [PATCH 05/10] perf: pass the tail-call handlers' Instruction in a register on arm64_32 On arm64_32 (watchOS) the Rust ABI passes the 8-byte Instruction by reference, so every dispatch stores it and the next handler loads it back. The C convention passes it in a register, and `C-unwind` still lets a host function's panic unwind. Other targets keep the Rust ABI. --- .../interpreter/executor/dispatch_become.rs | 81 +++++++++++++------ 1 file changed, 57 insertions(+), 24 deletions(-) diff --git a/crates/tinywasm/src/interpreter/executor/dispatch_become.rs b/crates/tinywasm/src/interpreter/executor/dispatch_become.rs index 03ec2b8..c1c62fd 100644 --- a/crates/tinywasm/src/interpreter/executor/dispatch_become.rs +++ b/crates/tinywasm/src/interpreter/executor/dispatch_become.rs @@ -3,10 +3,35 @@ use super::*; struct Unbudgeted; struct Bounded; -type UnbudgetedHandler = - for<'store, 'module> fn(&mut Executor<'store, 'module>, &[Instruction], usize, Instruction) -> ExecResult<()>; -type BoundedHandler = - for<'store, 'module> fn(&mut Executor<'store, 'module>, usize, Instruction, u32) -> ExecResult<()>; +// Between two handlers only what the calling convention passes in registers stays out of memory. +// On arm64_32 (watchOS) the Rust ABI passes an aggregate larger than a pointer, the 8-byte +// `Instruction`, by reference, so every dispatch would store it and the next handler load it back. +// The C convention passes it in a register; `C-unwind` still lets a host function's panic unwind. +// Elsewhere the handlers keep the Rust ABI. +macro_rules! handler_fn { + ($(#[$meta:meta])* fn $($rest:tt)*) => { + #[cfg(all(target_arch = "aarch64", target_pointer_width = "32"))] + #[allow(improper_ctypes_definitions)] + $(#[$meta])* extern "C-unwind" fn $($rest)* + #[cfg(not(all(target_arch = "aarch64", target_pointer_width = "32")))] + $(#[$meta])* fn $($rest)* + }; +} + +macro_rules! handler_types { + ($($abi:literal)?) => { + // Both sides are Rust, so the C convention's view of these types need not be FFI-safe. + #[allow(improper_ctypes_definitions)] + type UnbudgetedHandler = + for<'store, 'module> $(extern $abi)? fn(&mut Executor<'store, 'module>, &[Instruction], usize, Instruction) -> ExecResult<()>; + #[allow(improper_ctypes_definitions)] + type BoundedHandler = for<'store, 'module> $(extern $abi)? fn(&mut Executor<'store, 'module>, usize, Instruction, u32) -> ExecResult<()>; + }; +} +#[cfg(all(target_arch = "aarch64", target_pointer_width = "32"))] +handler_types!("C-unwind"); +#[cfg(not(all(target_arch = "aarch64", target_pointer_width = "32")))] +handler_types!(); macro_rules! define_unbudgeted_tail_dispatch { ($executor:ident, $instr_ptr:ident, $dispatch_next:ident, $dispatch_flow:ident; @@ -21,7 +46,7 @@ macro_rules! define_unbudgeted_tail_dispatch { HANDLERS[opcode as usize] } - $( + $(handler_fn! { #[allow(non_snake_case, unreachable_code, unused_imports, unused_macros, unused_variables)] fn $variant( $executor: &mut Executor<'_, '_>, @@ -66,7 +91,7 @@ macro_rules! define_unbudgeted_tail_dispatch { $body; $dispatch_next!($instr_ptr + 1) } - )* + })* }; } @@ -83,7 +108,7 @@ macro_rules! define_bounded_tail_dispatch { HANDLERS[opcode as usize] } - $( + $(handler_fn! { #[allow(non_snake_case, unreachable_code, unused_imports, unused_macros, unused_variables)] fn $variant( $executor: &mut Executor<'_, '_>, @@ -133,7 +158,7 @@ macro_rules! define_bounded_tail_dispatch { $body; $dispatch_next!($instr_ptr + 1) } - )* + })* }; } @@ -143,16 +168,20 @@ impl Unbudgeted { // The handlers tail-call these cold paths instead of calling them: a call would make every // handler save a stack frame. - #[cold] - #[inline(never)] - fn handler_mismatch(_: &mut Executor<'_, '_>, _: &[Instruction], _: usize, _: Instruction) -> ExecResult<()> { - unreachable!("instruction handler mismatch") + handler_fn! { + #[cold] + #[inline(never)] + fn handler_mismatch(_: &mut Executor<'_, '_>, _: &[Instruction], _: usize, _: Instruction) -> ExecResult<()> { + unreachable!("instruction handler mismatch") + } } - #[cold] - #[inline(never)] - fn invalid_instr_ptr(_: &mut Executor<'_, '_>, _: &[Instruction], instr_ptr: usize, _: Instruction) -> ExecResult<()> { - unreachable!("instruction pointer {instr_ptr} out of range, this is a bug") + handler_fn! { + #[cold] + #[inline(never)] + fn invalid_instr_ptr(_: &mut Executor<'_, '_>, _: &[Instruction], instr_ptr: usize, _: Instruction) -> ExecResult<()> { + unreachable!("instruction pointer {instr_ptr} out of range, this is a bug") + } } } @@ -161,16 +190,20 @@ impl Bounded { // Tail-called like `Unbudgeted`'s. - #[cold] - #[inline(never)] - fn handler_mismatch(_: &mut Executor<'_, '_>, _: usize, _: Instruction, _: u32) -> ExecResult<()> { - unreachable!("instruction handler mismatch") + handler_fn! { + #[cold] + #[inline(never)] + fn handler_mismatch(_: &mut Executor<'_, '_>, _: usize, _: Instruction, _: u32) -> ExecResult<()> { + unreachable!("instruction handler mismatch") + } } - #[cold] - #[inline(never)] - fn invalid_instr_ptr(_: &mut Executor<'_, '_>, instr_ptr: usize, _: Instruction, _: u32) -> ExecResult<()> { - unreachable!("instruction pointer {instr_ptr} out of range, this is a bug") + handler_fn! { + #[cold] + #[inline(never)] + fn invalid_instr_ptr(_: &mut Executor<'_, '_>, instr_ptr: usize, _: Instruction, _: u32) -> ExecResult<()> { + unreachable!("instruction pointer {instr_ptr} out of range, this is a bug") + } } /// Runs up to `chunk_left` (at least 1) instructions from `executor.cf`. From d3c345c5e38364942a5e56bb37928c33bae711cc Mon Sep 17 00:00:00 2001 From: Matt Hargett Date: Tue, 29 Sep 2026 13:23:15 -0700 Subject: [PATCH 06/10] perf: keep each value stack's height outside its Vec The height of a value-stack lane is its own field, over a Vec that holds every slot the lane has reached. Slots above the height hold stale values that are written before they are read again. The tail-call handlers can then pass the height between them in a register and write it back without unsafe code. Entering a function writes its locals, and its operand-stack reservation if that is at most 64 slots, so later entries at that height compare once. A larger reservation stays capacity, and a push writes each slot the first time it reaches it, so a function whose deep branch rarely runs keeps no pages for it; once the locals have slots, later entries compare and go on. With 64 stores of a module whose untaken branch declares a 30000-deep i64 stack, each store keeps next's footprint (35 KB on macOS, 16-19 KB on glibc) instead of 273 KB and 250 KB. v128.any_true and i8x16.all_true test the vector with integer arithmetic: LLVM scalarized their iterator forms once pushes gained that path. Only builds with debug assertions check indices against the height. CI also runs the tests optimized with them, keeping the release profile's wrapping arithmetic. Entering a function zeroes one or two locals with direct stores, and only the ones in between with memset. --- .github/workflows/test.yaml | 5 + .../src/interpreter/simd/instructions.rs | 8 +- .../src/interpreter/stack/value_stack.rs | 234 ++++++++++++++---- 3 files changed, 200 insertions(+), 47 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 6be4ae5..8a94aa2 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -46,6 +46,11 @@ jobs: name: "Linux x86 (nightly tail calls)" target: x86_64-unknown-linux-gnu args: "--features tinywasm/nightly-tail-calls" + - os: ubuntu-26.04 + rust: nightly + name: "Linux x86 (nightly tail calls, release with debug assertions)" + target: x86_64-unknown-linux-gnu + args: "--release --config profile.release.debug-assertions=true --features tinywasm/nightly-tail-calls" - os: ubuntu-26.04 rust: stable name: "Linux x86 (stable, no default features)" diff --git a/crates/tinywasm/src/interpreter/simd/instructions.rs b/crates/tinywasm/src/interpreter/simd/instructions.rs index 1ad4c25..a3bc078 100644 --- a/crates/tinywasm/src/interpreter/simd/instructions.rs +++ b/crates/tinywasm/src/interpreter/simd/instructions.rs @@ -30,7 +30,7 @@ impl Value128 { pub(crate) fn v128_any_true(self) -> bool { simd_impl! { wasm => { wasm::v128_any_true(self.to_wasm_v128()) } - generic => { self.0.iter().any(|&b| b != 0) } + generic => { u128::from_ne_bytes(self.0) != 0 } } } @@ -498,7 +498,11 @@ impl Value128 { "f64x2.extract_lane" => extract_lane_f64(self, lane: u8) -> f64 => f64::from_bits(self.extract_lane_i64(lane) as u64); // Truth checks, popcount, and averaging - "i8x16.all_true" => i8x16_all_true(self) -> bool => self.0.iter().all(|&b| b != 0); + "i8x16.all_true" => i8x16_all_true(self) -> bool => { + // No byte is zero: the classic zero-byte test on the whole vector. + let v = u128::from_ne_bytes(self.0); + v.wrapping_sub(u128::from_ne_bytes([1; 16])) & !v & u128::from_ne_bytes([0x80; 16]) == 0 + }; "i16x8.all_true" => i16x8_all_true(self) -> bool => self.as_i16x8().iter().all(|&x| x != 0); "i32x4.all_true" => i32x4_all_true(self) -> bool => self.as_i32x4().iter().all(|&x| x != 0); "i64x2.all_true" => i64x2_all_true(self) -> bool => self.as_i64x2().iter().all(|&x| x != 0); diff --git a/crates/tinywasm/src/interpreter/stack/value_stack.rs b/crates/tinywasm/src/interpreter/stack/value_stack.rs index 3a9936e..457194e 100644 --- a/crates/tinywasm/src/interpreter/stack/value_stack.rs +++ b/crates/tinywasm/src/interpreter/stack/value_stack.rs @@ -18,31 +18,59 @@ pub(crate) struct ValueStack { } #[cfg_attr(feature = "debug", derive(Debug))] +/// One value lane: a stack of `len` values over `data`. +/// +/// `data` holds every slot the stack has reached and only grows. The slots above `len` hold stale +/// values that are written before they are read again. A large operand-stack reservation stays +/// capacity, so its slots are written, and their pages touched, only when a push first reaches them. +/// Keeping the height outside the `Vec` lets the tail-call handlers carry it in a register and +/// write it back without `unsafe`. pub(crate) struct Stack { data: Vec, + len: usize, max_size: usize, dynamic: bool, } +/// The most slots [`Stack::enter_locals`] writes ahead of the pushes that reach them. +const WRITTEN_RESERVATION: usize = 64; + impl Stack { pub(crate) fn new(config: StackConfig) -> Self { - Self { data: Vec::with_capacity(config.initial_size), max_size: config.max_size, dynamic: config.dynamic } + Self { + data: Vec::with_capacity(config.initial_size), + len: 0, + max_size: config.max_size, + dynamic: config.dynamic, + } } pub(crate) fn clear(&mut self) { - self.data.clear(); + self.len = 0; } #[inline(always)] pub(crate) fn len(&self) -> usize { - self.data.len() + self.len } /// Pushes a value inside a function body. `enter_locals` reserved the function's whole operand - /// stack, so the stack is never full here. After this check `Vec::push` cannot reach its own - /// growth path, so the instruction handlers make no calls. + /// stack, so the stack is never full here, and the handlers make no calls. #[inline(always)] pub(crate) fn push(&mut self, value: T) { + let len = self.len; + match self.data.get_mut(len) { + Some(slot) => *slot = value, + None => self.push_first(value), + } + self.len = len + 1; + } + + /// Adds the first slot at a height the stack has not reached before, within the reserved + /// capacity: it never reallocates. On the tail-call build this makes no call. + #[inline(always)] + fn push_first(&mut self, value: T) { + core::hint::cold_path(); if self.data.len() == self.data.capacity() { crate::invariant_violated("value stack push beyond the function's reservation"); } @@ -53,10 +81,12 @@ impl Stack { /// covers, so a dynamic stack grows here if needed. #[inline(always)] pub(crate) fn push_or_grow(&mut self, value: T) -> Result<(), Trap> { - if self.data.len() == self.data.capacity() { - return self.push_grow(value); + let len = self.len; + match self.data.get_mut(len) { + Some(slot) => *slot = value, + None => return self.push_grow(value), } - self.data.push(value); + self.len = len + 1; Ok(()) } @@ -69,33 +99,38 @@ impl Stack { #[cold] #[inline(never)] fn push_grow(&mut self, value: T) -> Result<(), Trap> { - // Check the limit only at capacity to avoid an extra hot-path check. Vec growth may - // intentionally overshoot max_size. - if !self.dynamic || self.data.len() >= self.max_size { + // Only the reserved capacity is free to use. Past it, check the limit, which Vec growth + // may intentionally overshoot. + if self.data.len() == self.data.capacity() && (!self.dynamic || self.data.len() >= self.max_size) { return Err(Trap::ValueStackOverflow); } self.data.push(value); + self.len = self.data.len(); Ok(()) } + /// Pops the top value. On an empty stack the index wraps around, so the bounds check in + /// [`Self::get`] also catches an underflow. #[inline(always)] pub(crate) fn pop(&mut self) -> T { - match self.data.pop() { - Some(value) => value, - None => crate::invariant_violated("value stack underflow"), - } + let index = self.len.wrapping_sub(1); + let value = *self.get(index); + self.len = index; + value } #[inline(always)] pub(crate) fn last(&self) -> &T { - match self.data.last() { - Some(value) => value, - None => crate::invariant_violated("value stack underflow"), - } + self.get(self.len.wrapping_sub(1)) } + /// The slot at `index`, which validation keeps below the height. The check is against the + /// slots in use at any height, which is what memory safety needs. Builds with debug assertions + /// also check the height, and CI runs the tests optimized with them, with the release profile's + /// wrapping arithmetic. #[inline(always)] pub(crate) fn get(&self, index: usize) -> &T { + debug_assert!(index < self.len); match self.data.get(index) { Some(value) => value, None => crate::invariant_violated("value stack index out of range"), @@ -104,6 +139,7 @@ impl Stack { #[inline(always)] pub(crate) fn set(&mut self, index: usize, value: T) { + debug_assert!(index < self.len); match self.data.get_mut(index) { Some(slot) => *slot = value, None => crate::invariant_violated("value stack index out of range"), @@ -118,32 +154,36 @@ impl Stack { #[inline(always)] pub(crate) fn truncate_keep(&mut self, n: usize, end_keep: usize) { - let len = self.data.len(); + let len = self.len; debug_assert!(n <= len); if n >= len { return; } let keep = len.wrapping_sub(n).min(end_keep); - self.data.copy_within(len.wrapping_sub(keep)..len, n); - self.data.truncate(n.wrapping_add(keep)); + if keep != 0 { + // Copying to the start of the values above `n` leaves copy_within no check to fail. + match self.data.get_mut(n..len) { + Some(above) => above.copy_within(above.len().wrapping_sub(keep).., 0), + None => crate::invariant_violated("value stack index out of range"), + } + } + self.len = n.wrapping_add(keep); } #[inline(always)] pub(crate) fn truncate_to(&mut self, n: usize) { - debug_assert!(n <= self.data.len()); - self.data.truncate(n); + debug_assert!(n <= self.len); + self.len = n; } /// Moves the top value down to index `n` and drops everything above it (a function's result /// replacing its frame). #[inline(always)] pub(crate) fn truncate_to_one_tail(&mut self, n: usize) { - let len = self.data.len(); - debug_assert!(n < len); - let last = self.data[len.wrapping_sub(1)]; - self.data[n] = last; - self.data.truncate(n + 1); + debug_assert!(n < self.len); + self.copy(self.len.wrapping_sub(1), n); + self.len = n + 1; } /// Enters a function: turns its parameters into the first locals, zeroes the rest, and reserves @@ -157,36 +197,80 @@ impl Stack { max_stack: usize, ) -> Result { debug_assert!(param_count <= local_count); - debug_assert!(param_count <= self.data.len()); + debug_assert!(param_count <= self.len); - let len = self.data.len(); + let len = self.len; let start = len - param_count; let end = start + local_count; let reserve = end + max_stack; - if reserve > self.data.capacity() { + // A reservation larger than `WRITTEN_RESERVATION` stays capacity, so once the locals have + // slots and the capacity covers it, entries skip `reserve_slots`. + let len_slots = self.data.len(); + if reserve > len_slots + && (end > len_slots || reserve > self.data.capacity() || reserve - len_slots <= WRITTEN_RESERVATION) + { core::hint::cold_path(); - if reserve > self.max_size || !self.dynamic { - return Err(Trap::ValueStackOverflow); - } - let cap = self.data.capacity(); - let target = reserve.max(cap.max(1).saturating_mul(2)).min(self.max_size); - if self.data.try_reserve(target - len).is_err() { - return Err(Trap::ValueStackOverflow); - } + self.reserve_slots(end, reserve)?; } - self.data.resize(end, T::default()); + // Most functions have no or few locals in a lane. Store the first and last directly and + // fill only what lies between: `fill` becomes a memset call even for one value. + if end > len { + match self.data.get_mut(len..end) { + Some([a]) => *a = T::default(), + Some([a, middle @ .., b]) => { + if !middle.is_empty() { + middle.fill(T::default()); + } + (*a, *b) = (T::default(), T::default()); + } + _ => crate::invariant_violated("value stack index out of range"), + } + } + self.len = end; Ok(start as u32) } + /// Makes slots for a function's locals and reserves its operand stack when they reach past the + /// slots the stack has had. A reservation of up to [`WRITTEN_RESERVATION`] slots is written out + /// with the locals, so later entries at this height skip this. A larger one stays capacity and + /// its slots are written as they are pushed, so a function whose deep branch rarely runs does + /// not keep pages for it. + #[cold] + #[inline(never)] + fn reserve_slots(&mut self, end: usize, reserve: usize) -> Result<(), Trap> { + if reserve > self.data.capacity() { + self.grow_to(reserve)?; + } + let slots = if reserve - self.data.len() <= WRITTEN_RESERVATION { reserve } else { end }; + if slots > self.data.len() { + self.data.resize(slots, T::default()); + } + Ok(()) + } + + /// Makes room for `reserve` slots, growing the allocation if a dynamic stack allows it. + #[cold] + #[inline(never)] + fn grow_to(&mut self, reserve: usize) -> Result<(), Trap> { + if reserve > self.max_size || !self.dynamic { + return Err(Trap::ValueStackOverflow); + } + let target = reserve.max(self.data.capacity().max(1).saturating_mul(2)).min(self.max_size); + if self.data.try_reserve(target - self.data.len()).is_err() { + return Err(Trap::ValueStackOverflow); + } + Ok(()) + } + #[inline(always)] pub(crate) fn select_many(&mut self, count: usize, condition: bool) { if count == 0 { return; } - let len = self.data.len(); + let len = self.len; let needed = count.wrapping_mul(2); if len < needed { @@ -196,10 +280,13 @@ impl Stack { if !condition { let dst = len.wrapping_sub(needed); let src = len.wrapping_sub(count); - self.data.copy_within(src..len, dst); + match self.data.get_mut(..len) { + Some(live) => live.copy_within(src..len, dst), + None => crate::invariant_violated("value stack index out of range"), + } } - self.data.truncate(len.wrapping_sub(count)); + self.len = len.wrapping_sub(count); } } @@ -208,7 +295,7 @@ impl<'a, T: Copy + Default> IntoIterator for &'a Stack { type IntoIter = core::slice::Iter<'a, T>; fn into_iter(self) -> Self::IntoIter { - self.data.iter() + self.data[..self.len].iter() } } impl ValueStack { @@ -327,3 +414,60 @@ impl ValueStack { } } } + +#[cfg(test)] +mod tests { + use super::*; + + /// A stack of height 2 whose third slot still holds a popped value. + #[cfg(debug_assertions)] + fn popped() -> Stack { + let mut stack = Stack::new(StackConfig::fixed(8)); + for value in [1, 2, 3] { + stack.push_or_grow(value).unwrap(); + } + assert_eq!(stack.pop(), 3); + stack + } + + #[test] + #[cfg(debug_assertions)] + #[should_panic] + fn get_above_the_height() { + popped().get(2); + } + + #[test] + #[cfg(debug_assertions)] + #[should_panic] + fn set_above_the_height() { + popped().set(2, 0); + } + + /// A large operand-stack reservation is not written: only the locals and the values pushed + /// become slots. A small one is written with the locals. + #[test] + fn large_reservations_are_written_as_they_are_pushed() { + let mut stack = Stack::::new(StackConfig::fixed(1024)); + assert!(matches!(stack.enter_locals(0, 2, 500), Ok(0))); + assert_eq!(stack.data.len(), 2); + for value in 0..500 { + stack.push(value); + } + assert_eq!((stack.len(), stack.data.len()), (502, 502)); + + // Below the slots the stack already has, entering a function writes only its locals. + stack.truncate_to(2); + stack.push(7); + assert!(matches!(stack.enter_locals(0, 2, 10), Ok(3))); + assert_eq!((stack.len(), stack.data.len()), (5, 502)); + + stack.truncate_to(0); + for value in 0..500 { + stack.push_or_grow(value).unwrap(); + } + assert!(matches!(stack.enter_locals(0, 1, 8), Ok(500))); + assert_eq!(stack.data.len(), 509); + assert!(matches!(stack.enter_locals(0, 2, 1000), Err(Trap::ValueStackOverflow))); + } +} From a875964903bc7604bd749a12a29751ed77eb829f Mon Sep 17 00:00:00 2001 From: Matt Hargett Date: Tue, 29 Sep 2026 13:33:05 -0700 Subject: [PATCH 07/10] perf: hold the value stack in the executor while it runs The executor takes the store's value stack when it is created and gives it back when it is dropped, and around host calls, which read their arguments from the store and may call back into wasm. Every handler receives the executor as a `&mut`, so a stack access reached through it needs one load fewer (the store pointer), and the compiler knows that no store to a slot or to memory changes a stack's height, so it can keep the heights in registers within a handler. --- .../src/interpreter/executor/instructions.rs | 86 +-- .../tinywasm/src/interpreter/executor/mod.rs | 489 ++++++++++-------- .../tinywasm/src/interpreter/num_helpers.rs | 4 +- .../src/interpreter/stack/value_stack.rs | 8 + 4 files changed, 313 insertions(+), 274 deletions(-) diff --git a/crates/tinywasm/src/interpreter/executor/instructions.rs b/crates/tinywasm/src/interpreter/executor/instructions.rs index 22892eb..15932fa 100644 --- a/crates/tinywasm/src/interpreter/executor/instructions.rs +++ b/crates/tinywasm/src/interpreter/executor/instructions.rs @@ -6,14 +6,14 @@ macro_rules! exec_op { <$ty>::stack_push(value_stack, $expr?); Ok(()) } - exec_binary_fallible(&mut $executor.store.value_stack)?; + exec_binary_fallible(&mut $executor.value_stack)?; }}; ($executor:ident; unary $from:ty => $to:ty, |$v:ident| $expr:expr) => {{ fn exec_unary(value_stack: &mut ValueStack) { let $v = <$from>::stack_pop(value_stack); <$to>::stack_push(value_stack, $expr); } - exec_unary(&mut $executor.store.value_stack); + exec_unary(&mut $executor.value_stack); }}; ($executor:ident; binary $from:ty => $to:ty, |$lhs:ident, $rhs:ident| $expr:expr) => {{ exec_op!($executor; binary $from, $from => $to, |$lhs, $rhs| $expr) @@ -24,7 +24,7 @@ macro_rules! exec_op { let $lhs = <$lhs_ty>::stack_pop(value_stack); <$res>::stack_push(value_stack, $expr); } - exec_binary(&mut $executor.store.value_stack); + exec_binary(&mut $executor.value_stack); }}; ($executor:ident; ternary $from:ty => $to:ty, |$a:ident, $b:ident, $c:ident| $expr:expr) => {{ fn exec_ternary(value_stack: &mut ValueStack) { @@ -33,7 +33,7 @@ macro_rules! exec_op { let $a = <$from>::stack_pop(value_stack); <$to>::stack_push(value_stack, $expr); } - exec_ternary(&mut $executor.store.value_stack); + exec_ternary(&mut $executor.value_stack); }}; } @@ -43,15 +43,15 @@ macro_rules! instruction_handlers { ($emit:ident) => { $emit! { executor, instr_ptr, dispatch_next, dispatch_flow; Unreachable => { return cold!(Err(Trap::Unreachable.into())); }, - Drop32 => { _ = Value32::stack_pop(&mut executor.store.value_stack)}, - Drop64 => { _ = Value64::stack_pop(&mut executor.store.value_stack)}, - Drop128 => { _ = Value128::stack_pop(&mut executor.store.value_stack)}, - Select32 => Value32::stack_select(&mut executor.store.value_stack), - Select64 => Value64::stack_select(&mut executor.store.value_stack), - Select128 => Value128::stack_select(&mut executor.store.value_stack), + Drop32 => { _ = Value32::stack_pop(&mut executor.value_stack)}, + Drop64 => { _ = Value64::stack_pop(&mut executor.value_stack)}, + Drop128 => { _ = Value128::stack_pop(&mut executor.value_stack)}, + Select32 => Value32::stack_select(&mut executor.value_stack), + Select64 => Value64::stack_select(&mut executor.value_stack), + Select128 => Value128::stack_select(&mut executor.value_stack), SelectStore32(idx) => executor.exec_select_store::(idx.resolve(&executor.func.data))?, SelectStore64(idx) => executor.exec_select_store::(idx.resolve(&executor.func.data))?, - SelectMulti(counts) => executor.store.value_stack.select_multi(*counts), + SelectMulti(counts) => executor.value_stack.select_multi(*counts), Call(v) => dispatch_flow!(executor.exec_call_direct(*v, instr_ptr + 1)?), CallSelf => { executor.exec_call_self(instr_ptr + 1)?; dispatch_next!(0); }, CallIndirect(idx) => dispatch_flow!(executor.exec_call_indirect::(*idx, instr_ptr + 1)?), @@ -63,10 +63,10 @@ macro_rules! instruction_handlers { Throw(tag) => dispatch_flow!(executor.exec_throw(*tag, instr_ptr)?), ThrowRef => dispatch_flow!(executor.exec_throw_ref(instr_ptr)?), Jump(ip) => dispatch_next!(*ip as usize), - JumpIfZero32(ip) => if i32::stack_pop(&mut executor.store.value_stack) == 0 { dispatch_next!(*ip as usize) }, - JumpIfNonZero32(ip) => if i32::stack_pop(&mut executor.store.value_stack) != 0 { dispatch_next!(*ip as usize) }, - JumpIfZero64(ip) => if i64::stack_pop(&mut executor.store.value_stack) == 0 { dispatch_next!(*ip as usize) }, - JumpIfNonZero64(ip) => if i64::stack_pop(&mut executor.store.value_stack) != 0 { dispatch_next!(*ip as usize) }, + JumpIfZero32(ip) => if i32::stack_pop(&mut executor.value_stack) == 0 { dispatch_next!(*ip as usize) }, + JumpIfNonZero32(ip) => if i32::stack_pop(&mut executor.value_stack) != 0 { dispatch_next!(*ip as usize) }, + JumpIfZero64(ip) => if i64::stack_pop(&mut executor.value_stack) == 0 { dispatch_next!(*ip as usize) }, + JumpIfNonZero64(ip) => if i64::stack_pop(&mut executor.value_stack) != 0 { dispatch_next!(*ip as usize) }, JumpIfRefNull(ip) => { let ip = *ip; if executor.exec_jump_if_ref::() { dispatch_next!(ip as usize) } }, JumpIfRefNonNull(ip) => { let ip = *ip; if executor.exec_jump_if_ref::() { dispatch_next!(ip as usize) } }, BrOnCast(idx) => if let Some(ip) = executor.exec_br_on_cast::(*idx) { dispatch_next!(ip) }, @@ -91,33 +91,33 @@ macro_rules! instruction_handlers { JumpCmpLocalConst64(packed) => if let Some(ip) = executor.exec_jump_cmp_local_const64(*packed) { dispatch_next!(ip) }, JumpCmpLocalLocal32(packed) => if let Some(ip) = executor.exec_jump_cmp_local_local32(*packed) { dispatch_next!(ip) }, JumpCmpLocalLocal64(packed) => if let Some(ip) = executor.exec_jump_cmp_local_local64(*packed) { dispatch_next!(ip) }, - DropKeep32 { base, keep } => executor.store.value_stack.stack_32.truncate_keep((executor.cf.stack_base().s32 + u32::from(*base)) as usize, *keep as usize), - DropKeep64 { base, keep } => executor.store.value_stack.stack_64.truncate_keep((executor.cf.stack_base().s64 + u32::from(*base)) as usize, *keep as usize), - DropKeep128 { base, keep } => executor.store.value_stack.stack_128.truncate_keep((executor.cf.stack_base().s128 + u32::from(*base)) as usize, *keep as usize), + DropKeep32 { base, keep } => executor.value_stack.stack_32.truncate_keep((executor.cf.stack_base().s32 + u32::from(*base)) as usize, *keep as usize), + DropKeep64 { base, keep } => executor.value_stack.stack_64.truncate_keep((executor.cf.stack_base().s64 + u32::from(*base)) as usize, *keep as usize), + DropKeep128 { base, keep } => executor.value_stack.stack_128.truncate_keep((executor.cf.stack_base().s128 + u32::from(*base)) as usize, *keep as usize), BranchTable(idx) => dispatch_next!(executor.exec_branch_table(*idx)), Return => dispatch_flow!(executor.exec_return()), ReturnVoid => dispatch_flow!(executor.exec_return_void()), Return32 => dispatch_flow!(executor.exec_return_32()), Return64 => dispatch_flow!(executor.exec_return_64()), Return128 => dispatch_flow!(executor.exec_return_128()), - LocalGet32(local_index) => Value32::local_push(&mut executor.store.value_stack, &executor.cf, *local_index), - LocalGet64(local_index) => Value64::local_push(&mut executor.store.value_stack, &executor.cf, *local_index), - LocalGet128(local_index) => Value128::local_push(&mut executor.store.value_stack, &executor.cf, *local_index), + LocalGet32(local_index) => Value32::local_push(&mut executor.value_stack, &executor.cf, *local_index), + LocalGet64(local_index) => Value64::local_push(&mut executor.value_stack, &executor.cf, *local_index), + LocalGet128(local_index) => Value128::local_push(&mut executor.value_stack, &executor.cf, *local_index), LocalSet32(local_index) => executor.exec_local_set_pop::(*local_index), LocalSet64(local_index) => executor.exec_local_set_pop::(*local_index), LocalSet128(local_index) => executor.exec_local_set_pop::(*local_index), - LocalCopy32(from, to) => Value32::local_copy(&mut executor.store.value_stack, &executor.cf, *from, *to), - LocalCopy64(from, to) => Value64::local_copy(&mut executor.store.value_stack, &executor.cf, *from, *to), - LocalCopy128(from, to) => Value128::local_copy(&mut executor.store.value_stack, &executor.cf, *from, *to), - AddConst32(c) => { i32::stack_update(&mut executor.store.value_stack, |value| value.wrapping_add(*c)); }, - AndConst32(c) => { i32::stack_update(&mut executor.store.value_stack, |value| value & *c); }, - XorConst32(c) => { i32::stack_update(&mut executor.store.value_stack, |value| value ^ *c); }, - ShrUConst32(c) => { u32::stack_update(&mut executor.store.value_stack, |value| value.wrapping_shr(*c as u32)); }, - AddConst64(idx) => { let rhs = idx.resolve(&executor.func.data).value(); i64::stack_update(&mut executor.store.value_stack, |value| value.wrapping_add(rhs)); }, - BinOpStackConst32(op, rhs) => { u32::stack_update(&mut executor.store.value_stack, |lhs| op.exec(lhs, *rhs as u32)); }, - BinOpStackConst64(packed) => { let rhs = packed.index.resolve(&executor.func.data).value() as u64; u64::stack_update(&mut executor.store.value_stack, |lhs| packed.op.exec(lhs, rhs)); }, - IncLocal32(arg) => { i32::local_update(&mut executor.store.value_stack, &executor.cf, arg.local, |value| value.wrapping_add(arg.value)); }, - IncLocal64(packed) => { let rhs = packed.index.resolve(&executor.func.data).value(); i64::local_update(&mut executor.store.value_stack, &executor.cf, packed.op, |value| value.wrapping_add(rhs)); }, + LocalCopy32(from, to) => Value32::local_copy(&mut executor.value_stack, &executor.cf, *from, *to), + LocalCopy64(from, to) => Value64::local_copy(&mut executor.value_stack, &executor.cf, *from, *to), + LocalCopy128(from, to) => Value128::local_copy(&mut executor.value_stack, &executor.cf, *from, *to), + AddConst32(c) => { i32::stack_update(&mut executor.value_stack, |value| value.wrapping_add(*c)); }, + AndConst32(c) => { i32::stack_update(&mut executor.value_stack, |value| value & *c); }, + XorConst32(c) => { i32::stack_update(&mut executor.value_stack, |value| value ^ *c); }, + ShrUConst32(c) => { u32::stack_update(&mut executor.value_stack, |value| value.wrapping_shr(*c as u32)); }, + AddConst64(idx) => { let rhs = idx.resolve(&executor.func.data).value(); i64::stack_update(&mut executor.value_stack, |value| value.wrapping_add(rhs)); }, + BinOpStackConst32(op, rhs) => { u32::stack_update(&mut executor.value_stack, |lhs| op.exec(lhs, *rhs as u32)); }, + BinOpStackConst64(packed) => { let rhs = packed.index.resolve(&executor.func.data).value() as u64; u64::stack_update(&mut executor.value_stack, |lhs| packed.op.exec(lhs, rhs)); }, + IncLocal32(arg) => { i32::local_update(&mut executor.value_stack, &executor.cf, arg.local, |value| value.wrapping_add(arg.value)); }, + IncLocal64(packed) => { let rhs = packed.index.resolve(&executor.func.data).value(); i64::local_update(&mut executor.value_stack, &executor.cf, packed.op, |value| value.wrapping_add(rhs)); }, I32Add3 => exec_op!(executor; ternary i32 => i32, |a, b, c| a.wrapping_add(b).wrapping_add(c)), I64Add3 => exec_op!(executor; ternary i64 => i64, |a, b, c| a.wrapping_add(b).wrapping_add(c)), MulAccLocal32(acc) => executor.exec_mul_acc_local::(*acc, i32::wrapping_mul, i32::wrapping_add), @@ -160,9 +160,9 @@ macro_rules! instruction_handlers { BinOpGlobalConst32(packed) => { let v = packed.index.resolve(&executor.func.data); executor.exec_binop_global_const::(v.a(), v.b(), packed.op)?; }, BinOpGlobalConst64(packed) => { let v = packed.index.resolve(&executor.func.data); executor.exec_binop_global_const::(v.a(), v.b(), packed.op)?; }, BinOpGlobalConst128(packed) => { let v = packed.index.resolve(&executor.func.data); let rhs = Value128(v.b().resolve(&executor.func.data).value()); executor.exec_binop_global_const::(v.a(), rhs, packed.op)?; }, - SetLocalConst32(arg) => i32::local_set(&mut executor.store.value_stack, &executor.cf, arg.local, arg.value), - SetLocalConst64(packed) => { let v = packed.index.resolve(&executor.func.data).value(); i64::local_set(&mut executor.store.value_stack, &executor.cf, packed.op, v); }, - SetLocalConst128(packed) => Value128::local_set(&mut executor.store.value_stack, &executor.cf, packed.op, Value128(packed.index.resolve(&executor.func.data).value())), + SetLocalConst32(arg) => i32::local_set(&mut executor.value_stack, &executor.cf, arg.local, arg.value), + SetLocalConst64(packed) => { let v = packed.index.resolve(&executor.func.data).value(); i64::local_set(&mut executor.value_stack, &executor.cf, packed.op, v); }, + SetLocalConst128(packed) => Value128::local_set(&mut executor.value_stack, &executor.cf, packed.op, Value128(packed.index.resolve(&executor.func.data).value())), IncMemoryLocal32(arg) => executor.exec_inc_memory_local::(arg.memory_arg_idx, arg.local1, #[inline(always)] |v| v.wrapping_add(1))?, IncMemoryLocal64(arg) => executor.exec_inc_memory_local::(arg.memory_arg_idx, arg.local1, #[inline(always)] |v| v.wrapping_add(1))?, StoreLocalLocal32(arg) => executor.exec_store_local_local::(arg.memory_arg_idx, arg.local1, arg.local2)?, @@ -202,10 +202,10 @@ macro_rules! instruction_handlers { GlobalTee32(global_index) => executor.exec_global_tee::(*global_index), GlobalTee64(global_index) => executor.exec_global_tee::(*global_index), GlobalTee128(global_index) => executor.exec_global_tee::(*global_index), - Const32(val) => i32::stack_push(&mut executor.store.value_stack, *val), - Const64Imm(val) => i64::stack_push(&mut executor.store.value_stack, i64::from(*val)), - Const64(idx) => i64::stack_push(&mut executor.store.value_stack, idx.resolve(&executor.func.data).value()), - Const128Imm(val) => Value128::stack_push(&mut executor.store.value_stack, Value128(u128::from(*val).to_le_bytes())), + Const32(val) => i32::stack_push(&mut executor.value_stack, *val), + Const64Imm(val) => i64::stack_push(&mut executor.value_stack, i64::from(*val)), + Const64(idx) => i64::stack_push(&mut executor.value_stack, idx.resolve(&executor.func.data).value()), + Const128Imm(val) => Value128::stack_push(&mut executor.value_stack, Value128(u128::from(*val).to_le_bytes())), I64Eqz => exec_op!(executor; unary i64 => i32, |v| i32::from(v == 0)), I32Eqz => exec_op!(executor; unary i32 => i32, |v| i32::from(v == 0)), I32Eq => exec_op!(executor; binary i32 => i32, |a, b| i32::from(a == b)), @@ -290,8 +290,8 @@ macro_rules! instruction_handlers { I64Popcnt => exec_op!(executor; unary i64 => i64, |v| i64::from(v.count_ones())), // Reference types - RefFunc(func_idx) => ValueRef::stack_push(&mut executor.store.value_stack, ValueRef::from_category_addr(executor.module.resolve_func_addr(*func_idx))), - RefNull(_) => ValueRef::stack_push(&mut executor.store.value_stack, ValueRef::NULL), + RefFunc(func_idx) => ValueRef::stack_push(&mut executor.value_stack, ValueRef::from_category_addr(executor.module.resolve_func_addr(*func_idx))), + RefNull(_) => ValueRef::stack_push(&mut executor.value_stack, ValueRef::NULL), RefIsNull => executor.exec_ref_is_null()?, RefAsNonNull => executor.exec_ref_as_non_null()?, RefI31 => exec_op!(executor; unary i32 => ValueRef, |v| ValueRef::from_i31(v)), @@ -460,7 +460,7 @@ macro_rules! instruction_handlers { V128Store64Lane(arg) => executor.exec_mem_store_lane::(*arg)?, V128Load32Zero(idx) => executor.exec_mem_load::(idx.resolve(&executor.func.data), |v| Value128::from_i32x4([v, 0, 0, 0]))?, V128Load64Zero(idx) => executor.exec_mem_load::(idx.resolve(&executor.func.data), |v| Value128::from_i64x2([v, 0]))?, - Const128(arg) => Value128::stack_push(&mut executor.store.value_stack, Value128(arg.resolve(&executor.func.data).value())), + Const128(arg) => Value128::stack_push(&mut executor.value_stack, Value128(arg.resolve(&executor.func.data).value())), I8x16ExtractLaneS(lane) => executor.exec_simd_extract_lane::(*lane, |v, lane| v.extract_lane_i8(lane) as i32)?, I8x16ExtractLaneU(lane) => executor.exec_simd_extract_lane::(*lane, |v, lane| v.extract_lane_u8(lane) as i32)?, I16x8ExtractLaneS(lane) => executor.exec_simd_extract_lane::(*lane, |v, lane| v.extract_lane_i16(lane) as i32)?, diff --git a/crates/tinywasm/src/interpreter/executor/mod.rs b/crates/tinywasm/src/interpreter/executor/mod.rs index 932a2b3..33bec02 100644 --- a/crates/tinywasm/src/interpreter/executor/mod.rs +++ b/crates/tinywasm/src/interpreter/executor/mod.rs @@ -78,6 +78,11 @@ impl ExecFlow { /// instance ends the run with [`Executor::left`] set; [`InterpreterRuntime`](super::InterpreterRuntime) /// resumes that frame with an executor for its instance. pub(crate) struct Executor<'store, 'module> { + /// The store's value stack, which the executor holds while it runs and gives back for host + /// calls and when it is dropped. Reached through the executor, which each handler receives + /// as a `&mut`, stack accesses need one load fewer, and the compiler knows that no store to a + /// slot or to memory changes a stack's height. + value_stack: ValueStack, cf: CallFrame, func: &'module WasmFunction, module: &'module ModuleInstance, @@ -111,7 +116,9 @@ impl<'store, 'module> Executor<'store, 'module> { call_stack_base: u32, ) -> Self { let func = module.local_func(cf.func_addr).expect("call frame from another module instance"); + let value_stack = core::mem::take(&mut store.value_stack); Self { + value_stack, module, cf, func, @@ -140,6 +147,27 @@ impl<'store, 'module> Executor<'store, 'module> { ExecFlow::Complete } + /// Runs `f` with the value stack back in the store, as host functions expect it. + #[inline(always)] + fn with_store_value_stack(&mut self, f: impl FnOnce(&mut Store) -> R) -> R { + /// Takes the stack back when `f` returns or unwinds: a host function may catch a panic + /// from a nested call, and the executor must then still hold the stack. + struct Lent<'a, 'store> { + store: &'a mut &'store mut Store, + value_stack: &'a mut ValueStack, + } + impl Drop for Lent<'_, '_> { + #[inline(always)] + fn drop(&mut self) { + *self.value_stack = core::mem::take(&mut self.store.value_stack); + } + } + + self.store.value_stack = core::mem::take(&mut self.value_stack); + let lent = Lent { store: &mut self.store, value_stack: &mut self.value_stack }; + f(lent.store) + } + /// Resolves a module-local memory index to its store address, caching the common memory-0 case. #[inline(always)] fn mem_addr(&self, idx: MemAddr) -> MemAddr { @@ -160,9 +188,9 @@ impl<'store, 'module> Executor<'store, 'module> { #[inline(always)] fn exec_jump_if_ref(&mut self) -> bool { - let is_null = ValueRef::stack_peek(&self.store.value_stack).is_null(); + let is_null = ValueRef::stack_peek(&self.value_stack).is_null(); if is_null { - ValueRef::stack_pop(&mut self.store.value_stack); + ValueRef::stack_pop(&mut self.value_stack); } is_null == ON_NULL } @@ -179,41 +207,41 @@ impl<'store, 'module> Executor<'store, 'module> { &self, local: LocalAddr, ) -> bool { - (T::local_get(&self.store.value_stack, &self.cf, local) == T::default()) == ON_ZERO + (T::local_get(&self.value_stack, &self.cf, local) == T::default()) == ON_ZERO } #[inline(always)] fn exec_jump_cmp_stack_const32(&mut self, packed: PackedOp64) -> Option { let operand = packed.index.resolve(&self.func.data); - packed.op.cmp(i32::stack_pop(&mut self.store.value_stack), operand.b()).then_some(operand.target() as usize) + packed.op.cmp(i32::stack_pop(&mut self.value_stack), operand.b()).then_some(operand.target() as usize) } #[inline(always)] fn exec_jump_cmp_stack_const64(&mut self, packed: PackedOp128) -> Option { let operand = packed.index.resolve(&self.func.data); - packed.op.cmp(i64::stack_pop(&mut self.store.value_stack), operand.b()).then_some(operand.target() as usize) + packed.op.cmp(i64::stack_pop(&mut self.value_stack), operand.b()).then_some(operand.target() as usize) } #[inline(always)] fn exec_jump_cmp_stack_local32(&mut self, packed: PackedOp64) -> Option { let operand = packed.index.resolve(&self.func.data); - let lhs = i32::stack_pop(&mut self.store.value_stack); - let rhs = i32::local_get(&self.store.value_stack, &self.cf, operand.b()); + let lhs = i32::stack_pop(&mut self.value_stack); + let rhs = i32::local_get(&self.value_stack, &self.cf, operand.b()); packed.op.cmp(lhs, rhs).then_some(operand.target() as usize) } #[inline(always)] fn exec_jump_cmp_stack_local64(&mut self, packed: PackedOp64) -> Option { let operand = packed.index.resolve(&self.func.data); - let lhs = i64::stack_pop(&mut self.store.value_stack); - let rhs = i64::local_get(&self.store.value_stack, &self.cf, operand.b()); + let lhs = i64::stack_pop(&mut self.value_stack); + let rhs = i64::local_get(&self.value_stack, &self.cf, operand.b()); packed.op.cmp(lhs, rhs).then_some(operand.target() as usize) } #[inline(always)] fn exec_binop_local_const_jump(&mut self, packed: PackedOp128) -> Option { let operand = packed.index.resolve(&self.func.data); - let value = i32::local_update(&mut self.store.value_stack, &self.cf, operand.local(), |value| { + let value = i32::local_update(&mut self.value_stack, &self.cf, operand.local(), |value| { packed.op.exec(value as u32, operand.value() as u32) as i32 }); ((value == 0) == operand.on_zero()).then_some(operand.target() as usize) @@ -225,10 +253,10 @@ impl<'store, 'module> Executor<'store, 'module> { packed: PackedOp128<(BinOp, CmpOp), LocalUpdateCmpOperand>, ) -> Option { let operand = packed.index.resolve(&self.func.data); - let lhs = i32::local_update(&mut self.store.value_stack, &self.cf, operand.local(), |value| { + let lhs = i32::local_update(&mut self.value_stack, &self.cf, operand.local(), |value| { packed.op.0.exec(value as u32, operand.value() as u32) as i32 }); - let rhs = i32::local_get(&self.store.value_stack, &self.cf, operand.right()); + let rhs = i32::local_get(&self.value_stack, &self.cf, operand.right()); packed.op.1.cmp(lhs, rhs).then_some(operand.target() as usize) } @@ -238,10 +266,9 @@ impl<'store, 'module> Executor<'store, 'module> { packed: PackedOp128, ) -> Result, Trap> { let operand = packed.index.resolve(&self.func.data); - let value = i32::stack_update(&mut self.store.value_stack, |lhs| { - packed.op.exec(lhs as u32, operand.value() as u32) as i32 - }); - i32::local_set(&mut self.store.value_stack, &self.cf, operand.local(), value); + let value = + i32::stack_update(&mut self.value_stack, |lhs| packed.op.exec(lhs as u32, operand.value() as u32) as i32); + i32::local_set(&mut self.value_stack, &self.cf, operand.local(), value); Ok(((value == 0) == operand.on_zero()).then_some(operand.target() as usize)) } @@ -258,7 +285,7 @@ impl<'store, 'module> Executor<'store, 'module> { #[inline(always)] fn exec_inc_local_jump(&mut self, index: Operand128Idx) -> Option { let operand = index.resolve(&self.func.data); - let value = i32::local_update(&mut self.store.value_stack, &self.cf, operand.local(), |value| { + let value = i32::local_update(&mut self.value_stack, &self.cf, operand.local(), |value| { value.wrapping_add(operand.value()) }); ((value == 0) == operand.on_zero()).then_some(operand.target() as usize) @@ -270,8 +297,8 @@ impl<'store, 'module> Executor<'store, 'module> { index: Operand128Idx, ) -> Result, Trap> { let operand = index.resolve(&self.func.data); - let value = i32::stack_update(&mut self.store.value_stack, |value| value.wrapping_add(operand.value())); - i32::local_set(&mut self.store.value_stack, &self.cf, operand.local(), value); + let value = i32::stack_update(&mut self.value_stack, |value| value.wrapping_add(operand.value())); + i32::local_set(&mut self.value_stack, &self.cf, operand.local(), value); Ok(((value == 0) == operand.on_zero()).then_some(operand.target() as usize)) } @@ -287,84 +314,84 @@ impl<'store, 'module> Executor<'store, 'module> { #[inline(always)] fn exec_inc_local_jump_cmp_local(&mut self, packed: PackedOp128) -> Option { let operand = packed.index.resolve(&self.func.data); - let lhs = i32::local_update(&mut self.store.value_stack, &self.cf, operand.local(), |value| { + let lhs = i32::local_update(&mut self.value_stack, &self.cf, operand.local(), |value| { value.wrapping_add(operand.value()) }); - let rhs = i32::local_get(&self.store.value_stack, &self.cf, operand.right()); + let rhs = i32::local_get(&self.value_stack, &self.cf, operand.right()); packed.op.cmp(lhs, rhs).then_some(operand.target() as usize) } #[inline(always)] fn exec_jump_cmp_local_const32(&self, packed: PackedOp128) -> Option { let operand = packed.index.resolve(&self.func.data); - let lhs = i32::local_get(&self.store.value_stack, &self.cf, operand.c()); + let lhs = i32::local_get(&self.value_stack, &self.cf, operand.c()); packed.op.cmp(lhs, operand.b()).then_some(operand.target() as usize) } #[inline(always)] fn exec_jump_cmp_local_const64(&self, packed: PackedOp128) -> Option { let operand = packed.index.resolve(&self.func.data); - let lhs = i64::local_get(&self.store.value_stack, &self.cf, operand.c()); + let lhs = i64::local_get(&self.value_stack, &self.cf, operand.c()); packed.op.cmp(lhs, i64::from(operand.b())).then_some(operand.target() as usize) } #[inline(always)] fn exec_jump_cmp_local_local32(&self, packed: PackedOp64) -> Option { let operand = packed.index.resolve(&self.func.data); - let lhs = i32::local_get(&self.store.value_stack, &self.cf, operand.b()); - let rhs = i32::local_get(&self.store.value_stack, &self.cf, operand.c()); + let lhs = i32::local_get(&self.value_stack, &self.cf, operand.b()); + let rhs = i32::local_get(&self.value_stack, &self.cf, operand.c()); packed.op.cmp(lhs, rhs).then_some(operand.target() as usize) } #[inline(always)] fn exec_jump_cmp_local_local64(&self, packed: PackedOp64) -> Option { let operand = packed.index.resolve(&self.func.data); - let lhs = i64::local_get(&self.store.value_stack, &self.cf, operand.b()); - let rhs = i64::local_get(&self.store.value_stack, &self.cf, operand.c()); + let lhs = i64::local_get(&self.value_stack, &self.cf, operand.b()); + let rhs = i64::local_get(&self.value_stack, &self.cf, operand.c()); packed.op.cmp(lhs, rhs).then_some(operand.target() as usize) } fn exec_i64_add128(&mut self) -> Result<(), Trap> { - let b_hi = i64::stack_pop(&mut self.store.value_stack); - let b_lo = i64::stack_pop(&mut self.store.value_stack); - let a_hi = i64::stack_pop(&mut self.store.value_stack); - let a_lo = i64::stack_pop(&mut self.store.value_stack); + let b_hi = i64::stack_pop(&mut self.value_stack); + let b_lo = i64::stack_pop(&mut self.value_stack); + let a_hi = i64::stack_pop(&mut self.value_stack); + let a_lo = i64::stack_pop(&mut self.value_stack); let lo = a_lo.wrapping_add(b_lo); let carry = u64::from((lo as u64) < (a_lo as u64)); let hi = a_hi.wrapping_add(b_hi).wrapping_add(carry as i64); - i64::stack_push(&mut self.store.value_stack, lo); - i64::stack_push(&mut self.store.value_stack, hi); + i64::stack_push(&mut self.value_stack, lo); + i64::stack_push(&mut self.value_stack, hi); Ok(()) } fn exec_i64_sub128(&mut self) -> Result<(), Trap> { - let b_hi = i64::stack_pop(&mut self.store.value_stack); - let b_lo = i64::stack_pop(&mut self.store.value_stack); - let a_hi = i64::stack_pop(&mut self.store.value_stack); - let a_lo = i64::stack_pop(&mut self.store.value_stack); + let b_hi = i64::stack_pop(&mut self.value_stack); + let b_lo = i64::stack_pop(&mut self.value_stack); + let a_hi = i64::stack_pop(&mut self.value_stack); + let a_lo = i64::stack_pop(&mut self.value_stack); let lo = a_lo.wrapping_sub(b_lo); let borrow = u64::from((a_lo as u64) < (b_lo as u64)); let hi = a_hi.wrapping_sub(b_hi).wrapping_sub(borrow as i64); - i64::stack_push(&mut self.store.value_stack, lo); - i64::stack_push(&mut self.store.value_stack, hi); + i64::stack_push(&mut self.value_stack, lo); + i64::stack_push(&mut self.value_stack, hi); Ok(()) } fn exec_i64_mul_wide_s(&mut self) -> Result<(), Trap> { - let rhs = i64::stack_pop(&mut self.store.value_stack); - let lhs = i64::stack_pop(&mut self.store.value_stack); + let rhs = i64::stack_pop(&mut self.value_stack); + let lhs = i64::stack_pop(&mut self.value_stack); let product = (lhs as i128).wrapping_mul(rhs as i128); - i64::stack_push(&mut self.store.value_stack, product as i64); - i64::stack_push(&mut self.store.value_stack, (product >> 64) as i64); + i64::stack_push(&mut self.value_stack, product as i64); + i64::stack_push(&mut self.value_stack, (product >> 64) as i64); Ok(()) } fn exec_i64_mul_wide_u(&mut self) -> Result<(), Trap> { - let rhs = i64::stack_pop(&mut self.store.value_stack); - let lhs = i64::stack_pop(&mut self.store.value_stack); + let rhs = i64::stack_pop(&mut self.value_stack); + let lhs = i64::stack_pop(&mut self.value_stack); let product = (lhs as u64 as u128).wrapping_mul(rhs as u64 as u128); - i64::stack_push(&mut self.store.value_stack, product as u64 as i64); - i64::stack_push(&mut self.store.value_stack, (product >> 64) as u64 as i64); + i64::stack_push(&mut self.value_stack, product as u64 as i64); + i64::stack_push(&mut self.value_stack, (product >> 64) as u64 as i64); Ok(()) } @@ -374,8 +401,8 @@ impl<'store, 'module> Executor<'store, 'module> { lane: u8, operation: impl FnOnce(Value128, u8) -> TO, ) -> Result<(), Trap> { - let vector = Value128::stack_pop(&mut self.store.value_stack); - TO::stack_push(&mut self.store.value_stack, operation(vector, lane)); + let vector = Value128::stack_pop(&mut self.value_stack); + TO::stack_push(&mut self.value_stack, operation(vector, lane)); Ok(()) } @@ -385,50 +412,50 @@ impl<'store, 'module> Executor<'store, 'module> { lane: u8, operation: impl FnOnce(VALUE, Value128, u8) -> Value128, ) -> Result<(), Trap> { - let vector = Value128::stack_pop(&mut self.store.value_stack); - let value = VALUE::stack_pop(&mut self.store.value_stack); - Value128::stack_push(&mut self.store.value_stack, operation(value, vector, lane)); + let vector = Value128::stack_pop(&mut self.value_stack); + let value = VALUE::stack_pop(&mut self.value_stack); + Value128::stack_push(&mut self.value_stack, operation(value, vector, lane)); Ok(()) } fn exec_simd_shuffle(&mut self, lanes: Value128) -> Result<(), Trap> { - let rhs = Value128::stack_pop(&mut self.store.value_stack); - let lhs = Value128::stack_pop(&mut self.store.value_stack); - Value128::stack_push(&mut self.store.value_stack, Value128::i8x16_shuffle(lhs, rhs, lanes)); + let rhs = Value128::stack_pop(&mut self.value_stack); + let lhs = Value128::stack_pop(&mut self.value_stack); + Value128::stack_push(&mut self.value_stack, Value128::i8x16_shuffle(lhs, rhs, lanes)); Ok(()) } #[inline(always)] fn exec_local_set_pop(&mut self, local: LocalAddr) { - let value = T::stack_pop(&mut self.store.value_stack); - T::local_set(&mut self.store.value_stack, &self.cf, local, value); + let value = T::stack_pop(&mut self.value_stack); + T::local_set(&mut self.value_stack, &self.cf, local, value); } #[inline(always)] fn exec_local_tee(&mut self, local: LocalAddr) { - let value = T::stack_peek(&self.store.value_stack); - T::local_set(&mut self.store.value_stack, &self.cf, local, value); + let value = T::stack_peek(&self.value_stack); + T::local_set(&mut self.value_stack, &self.cf, local, value); } #[inline(always)] fn exec_global_get(&mut self, global: GlobalAddr) -> Result<(), Trap> { let addr = self.module.resolve_global_addr(global); let value = T::global_get(&self.store.state.globals, addr); - T::stack_push(&mut self.store.value_stack, value); + T::stack_push(&mut self.value_stack, value); Ok(()) } #[inline(always)] fn exec_global_set(&mut self, global: GlobalAddr) { let addr = self.module.resolve_global_addr(global); - let value = T::stack_pop(&mut self.store.value_stack); + let value = T::stack_pop(&mut self.value_stack); T::global_set(&mut self.store.state.globals, addr, value); } #[inline(always)] fn exec_global_tee(&mut self, global: GlobalAddr) { let addr = self.module.resolve_global_addr(global); - let value = T::stack_peek(&self.store.value_stack); + let value = T::stack_peek(&self.value_stack); T::global_set(&mut self.store.state.globals, addr, value); } @@ -439,10 +466,10 @@ impl<'store, 'module> Executor<'store, 'module> { val: T, ) -> Result<(), Trap> { if let Some(dst) = dst { - T::local_set(&mut self.store.value_stack, &self.cf, dst, val); + T::local_set(&mut self.value_stack, &self.cf, dst, val); } if PUSH { - T::stack_push(&mut self.store.value_stack, val); + T::stack_push(&mut self.value_stack, val); } Ok(()) } @@ -455,8 +482,8 @@ impl<'store, 'module> Executor<'store, 'module> { destination: Option, op: impl BinOpExt, ) -> Result<(), Trap> { - let lhs = T::local_get(&self.store.value_stack, &self.cf, lhs); - let rhs = T::local_get(&self.store.value_stack, &self.cf, rhs); + let lhs = T::local_get(&self.value_stack, &self.cf, lhs); + let rhs = T::local_get(&self.value_stack, &self.cf, rhs); self.exec_binop_result::(destination, op.exec(lhs, rhs)) } @@ -478,7 +505,7 @@ impl<'store, 'module> Executor<'store, 'module> { destination: Option, op: impl BinOpExt, ) -> Result<(), Trap> { - let lhs = T::local_get(&self.store.value_stack, &self.cf, lhs); + let lhs = T::local_get(&self.value_stack, &self.cf, lhs); self.exec_binop_result::(destination, op.exec(lhs, rhs)) } @@ -490,7 +517,7 @@ impl<'store, 'module> Executor<'store, 'module> { op: impl BinOpExt, ) -> Result<(), Trap> { let lhs = T::global_get(&self.store.state.globals, self.module.resolve_global_addr(lhs)); - T::stack_push(&mut self.store.value_stack, op.exec(lhs, rhs)); + T::stack_push(&mut self.value_stack, op.exec(lhs, rhs)); Ok(()) } @@ -500,9 +527,9 @@ impl<'store, 'module> Executor<'store, 'module> { T: InternalValue, CmpOp: CmpOpExt, { - let lhs = T::local_get(&self.store.value_stack, &self.cf, lhs); - let rhs = T::local_get(&self.store.value_stack, &self.cf, rhs); - i32::stack_push(&mut self.store.value_stack, i32::from(op.cmp(lhs, rhs))); + let lhs = T::local_get(&self.value_stack, &self.cf, lhs); + let rhs = T::local_get(&self.value_stack, &self.cf, rhs); + i32::stack_push(&mut self.value_stack, i32::from(op.cmp(lhs, rhs))); Ok(()) } @@ -514,7 +541,7 @@ impl<'store, 'module> Executor<'store, 'module> { ) -> Result<(), Trap> { let addr = self.module.resolve_global_addr(global); let rhs = T::global_get(&self.store.state.globals, addr); - T::stack_update(&mut self.store.value_stack, |lhs| op.exec(lhs, rhs)); + T::stack_update(&mut self.value_stack, |lhs| op.exec(lhs, rhs)); Ok(()) } @@ -525,8 +552,8 @@ impl<'store, 'module> Executor<'store, 'module> { destination: Option, op: impl BinOpExt, ) -> Result<(), Trap> { - let rhs = T::local_get(&self.store.value_stack, &self.cf, local); - let lhs = T::stack_pop(&mut self.store.value_stack); + let rhs = T::local_get(&self.value_stack, &self.cf, local); + let lhs = T::stack_pop(&mut self.value_stack); self.exec_binop_result::(destination, op.exec(lhs, rhs)) } @@ -537,8 +564,8 @@ impl<'store, 'module> Executor<'store, 'module> { destination: LocalAddr, op: impl BinOpExt, ) -> Result<(), Trap> { - let value = T::stack_update(&mut self.store.value_stack, |lhs| op.exec(lhs, rhs)); - T::local_set(&mut self.store.value_stack, &self.cf, destination, value); + let value = T::stack_update(&mut self.value_stack, |lhs| op.exec(lhs, rhs)); + T::local_set(&mut self.value_stack, &self.cf, destination, value); Ok(()) } @@ -549,15 +576,15 @@ impl<'store, 'module> Executor<'store, 'module> { multiply: fn(T, T) -> T, add: fn(T, T) -> T, ) { - let rhs = T::stack_pop(&mut self.store.value_stack); - let lhs = T::stack_pop(&mut self.store.value_stack); + let rhs = T::stack_pop(&mut self.value_stack); + let lhs = T::stack_pop(&mut self.value_stack); let product = multiply(lhs, rhs); - T::local_update(&mut self.store.value_stack, &self.cf, accumulator, |value| add(product, value)); + T::local_update(&mut self.value_stack, &self.cf, accumulator, |value| add(product, value)); } fn exec_branch_table(&mut self, index: Operand128Idx) -> usize { let v = index.resolve(&self.func.data); - let idx = ::stack_pop(&mut self.store.value_stack); + let idx = ::stack_pop(&mut self.value_stack); let target_ip = if idx >= 0 && (idx as u32) < v.size() { self.func.data.branch_table_targets.get((v.start() + idx as u32) as usize).copied().unwrap_or(v.target()) } else { @@ -574,7 +601,7 @@ impl<'store, 'module> Executor<'store, 'module> { self.store.state.gc.check_allocation(payload_len, true)?; let mut payload = Vec::new(); cold_err!(payload.try_reserve_exact(payload_len)).map_err(|_| Trap::OutOfMemory)?; - let value_stack = &mut self.store.value_stack; + let value_stack = &mut self.value_stack; for index in (0..payload_len).rev() { let ty = self.store.state.get_canonical_func_type(type_addr).params()[index]; payload.push(match ty { @@ -585,7 +612,7 @@ impl<'store, 'module> Executor<'store, 'module> { }); } payload.reverse(); - let roots = (&self.store.value_stack.stack_32).into_iter().copied().map(ValueRef::from_raw); + let roots = (&self.value_stack.stack_32).into_iter().copied().map(ValueRef::from_raw); self.store.state.alloc_exception(tag_addr, payload, roots) } @@ -595,7 +622,7 @@ impl<'store, 'module> Executor<'store, 'module> { } fn exec_throw_ref(&mut self, instr_ptr: usize) -> ExecResult { - let exception = ValueRef::stack_pop(&mut self.store.value_stack); + let exception = ValueRef::stack_pop(&mut self.value_stack); if exception.is_null() { return Err(Trap::NullReference.into()); } @@ -654,16 +681,15 @@ impl<'store, 'module> Executor<'store, 'module> { s64: stack_base.s64 + base.c64 as u32, s128: stack_base.s128 + base.c128 as u32, }; - self.store.value_stack.truncate_to_base(target); + self.value_stack.truncate_to_base(target); if include_payload { - let Store { state, value_stack, .. } = self.store; - let object = state.gc.get(exception).ok_or(Trap::InvalidReference)?; + let object = self.store.state.gc.get(exception).ok_or(Trap::InvalidReference)?; for value in object.values.iter().copied() { - value_stack.push_reserved(value); + self.value_stack.push_reserved(value); } } if with_ref { - ValueRef::stack_push(&mut self.store.value_stack, exception); + ValueRef::stack_push(&mut self.value_stack, exception); } if foreign.is_some() { self.cf.instr_ptr = landing_pad as usize; @@ -677,7 +703,7 @@ impl<'store, 'module> Executor<'store, 'module> { })); } - self.store.value_stack.truncate_to_base(self.cf.locals_base); + self.value_stack.truncate_to_base(self.cf.locals_base); let Some(caller) = self.store.call_stack.pop_frame(self.call_stack_base) else { return Ok(None); }; @@ -706,8 +732,9 @@ impl<'store, 'module> Executor<'store, 'module> { type_addr: TypeAddr, return_instr_ptr: usize, ) -> ExecResult { + let module_id = self.module.id(); if let Some(host_func) = host_func.typed_callback() { - cold_err!(host_func.call_stack(self.store, self.module.id(), type_addr)) + cold_err!(self.with_store_value_stack(|store| host_func.call_stack(store, module_id, type_addr))) .map_err(|error| Trap::HostFunction(Box::new(error)))?; if TAIL { return Ok(self.exec_return()); @@ -717,11 +744,10 @@ impl<'store, 'module> Executor<'store, 'module> { let (param_count, result_count, base) = { let ty = self.store.state.get_canonical_func_type(type_addr); - (ty.params().len(), ty.results().len(), self.store.value_stack.base_before(ty.params().iter().collect())) + (ty.params().len(), ty.results().len(), self.value_stack.base_before(ty.params().iter().collect())) }; - let module_id = self.module.id(); - self.store - .with_scratch_values(param_count + result_count, |store, values| { + self.with_store_value_stack(|store| { + store.with_scratch_values(param_count + result_count, |store, values| { let host_values = store.stack_value_iter(type_addr, crate::store::FuncValueTypes::Params, base)?; for (slot, value) in values[..param_count].iter_mut().zip(host_values) { *slot = value; @@ -733,10 +759,11 @@ impl<'store, 'module> Executor<'store, 'module> { .map_err(|error| Error::Trap(Trap::HostFunction(Box::new(error))))?; store.push_wasm_values(results) }) - .map_err(|error| match error { - Error::Trap(trap) => trap, - other => Trap::HostFunction(Box::new(other)), - })?; + }) + .map_err(|error| match error { + Error::Trap(trap) => trap, + other => Trap::HostFunction(Box::new(other)), + })?; if TAIL { Ok(self.exec_return()) } else { Ok(ExecFlow::Next(return_instr_ptr)) } } @@ -775,8 +802,7 @@ impl<'store, 'module> Executor<'store, 'module> { fn exec_call_self(&mut self, return_instr_ptr: usize) -> ExecResult<()> { self.charge_call_fuel(FUEL_COST_CALL_TOTAL); - let locals_base = - self.store.value_stack.enter_locals(&self.func.params, &self.func.locals, &self.func.max_stack)?; + let locals_base = self.value_stack.enter_locals(&self.func.params, &self.func.locals, &self.func.max_stack)?; let new = CallFrame::new(self.cf.func_addr, locals_base, self.func.locals); self.store.call_stack.push(core::mem::replace(&mut self.cf, new), return_instr_ptr)?; Ok(()) @@ -785,9 +811,8 @@ impl<'store, 'module> Executor<'store, 'module> { fn exec_return_call_self(&mut self) -> ExecResult<()> { self.charge_call_fuel(FUEL_COST_CALL_TOTAL); - self.store.value_stack.truncate_keep_counts(self.cf.locals_base, self.func.params); - let locals_base = - self.store.value_stack.enter_locals(&self.func.params, &self.func.locals, &self.func.max_stack)?; + self.value_stack.truncate_keep_counts(self.cf.locals_base, self.func.params); + let locals_base = self.value_stack.enter_locals(&self.func.params, &self.func.locals, &self.func.max_stack)?; self.cf = CallFrame::new(self.cf.func_addr, locals_base, self.func.locals); Ok(()) } @@ -892,9 +917,9 @@ impl<'store, 'module> Executor<'store, 'module> { return_instr_ptr: usize, ) -> ExecResult<()> { if TAIL { - self.store.value_stack.truncate_keep_counts(self.cf.locals_base, func.params); + self.value_stack.truncate_keep_counts(self.cf.locals_base, func.params); } - let locals_base = self.store.value_stack.enter_locals(&func.params, &func.locals, &func.max_stack)?; + let locals_base = self.value_stack.enter_locals(&func.params, &func.locals, &func.max_stack)?; if !TAIL { self.store.call_stack.push(self.cf, return_instr_ptr)?; } @@ -904,7 +929,7 @@ impl<'store, 'module> Executor<'store, 'module> { fn exec_call_ref(&mut self, type_addr: u32, return_instr_ptr: usize) -> ExecResult { self.charge_call_fuel(FUEL_COST_CALL_TOTAL); - let func_ref = ValueRef::stack_pop(&mut self.store.value_stack); + let func_ref = ValueRef::stack_pop(&mut self.value_stack); let Some(func_addr) = func_ref.addr() else { return cold!(Err(Trap::NullFunctionReference.into())); }; @@ -913,7 +938,7 @@ impl<'store, 'module> Executor<'store, 'module> { } fn exec_return(&mut self) -> ExecFlow { - self.store.value_stack.truncate_keep_counts(self.cf.locals_base, self.func.results); + self.value_stack.truncate_keep_counts(self.cf.locals_base, self.func.results); self.finish_return() } @@ -939,28 +964,28 @@ impl<'store, 'module> Executor<'store, 'module> { } fn exec_return_void(&mut self) -> ExecFlow { - self.store.value_stack.truncate_to_base(self.cf.locals_base); + self.value_stack.truncate_to_base(self.cf.locals_base); self.finish_return() } fn exec_return_32(&mut self) -> ExecFlow { - self.store.value_stack.stack_32.truncate_to_one_tail(self.cf.locals_base.s32 as usize); - self.store.value_stack.stack_64.truncate_to(self.cf.locals_base.s64 as usize); - self.store.value_stack.stack_128.truncate_to(self.cf.locals_base.s128 as usize); + self.value_stack.stack_32.truncate_to_one_tail(self.cf.locals_base.s32 as usize); + self.value_stack.stack_64.truncate_to(self.cf.locals_base.s64 as usize); + self.value_stack.stack_128.truncate_to(self.cf.locals_base.s128 as usize); self.finish_return() } fn exec_return_64(&mut self) -> ExecFlow { - self.store.value_stack.stack_32.truncate_to(self.cf.locals_base.s32 as usize); - self.store.value_stack.stack_64.truncate_to_one_tail(self.cf.locals_base.s64 as usize); - self.store.value_stack.stack_128.truncate_to(self.cf.locals_base.s128 as usize); + self.value_stack.stack_32.truncate_to(self.cf.locals_base.s32 as usize); + self.value_stack.stack_64.truncate_to_one_tail(self.cf.locals_base.s64 as usize); + self.value_stack.stack_128.truncate_to(self.cf.locals_base.s128 as usize); self.finish_return() } fn exec_return_128(&mut self) -> ExecFlow { - self.store.value_stack.stack_32.truncate_to(self.cf.locals_base.s32 as usize); - self.store.value_stack.stack_64.truncate_to(self.cf.locals_base.s64 as usize); - self.store.value_stack.stack_128.truncate_to_one_tail(self.cf.locals_base.s128 as usize); + self.value_stack.stack_32.truncate_to(self.cf.locals_base.s32 as usize); + self.value_stack.stack_64.truncate_to(self.cf.locals_base.s64 as usize); + self.value_stack.stack_128.truncate_to_one_tail(self.cf.locals_base.s128 as usize); self.finish_return() } @@ -972,18 +997,18 @@ impl<'store, 'module> Executor<'store, 'module> { value_local: u8, ) -> Result<(), Trap> { let memarg = index.resolve(&self.func.data); - let value = T::local_get(&self.store.value_stack, &self.cf, u16::from(value_local)); + let value = T::local_get(&self.value_stack, &self.cf, u16::from(value_local)); let mem_addr = self.mem_addr(MemAddr::from(memarg.memory())); crate::store::with_memory!(self.store.state, mem_addr, |mem, kind| { let base = if kind.arch() == MemoryArch::I64 { - let base = u64::local_get(&self.store.value_stack, &self.cf, u16::from(addr_local)); + let base = u64::local_get(&self.value_stack, &self.cf, u16::from(addr_local)); cold_err!(usize::try_from(base).map_err(|_| Trap::MemoryOutOfBounds { offset: usize::MAX, len: N, max: mem.len(), }))? } else { - u32::local_get(&self.store.value_stack, &self.cf, u16::from(addr_local)) as usize + u32::local_get(&self.value_stack, &self.cf, u16::from(addr_local)) as usize }; let addr = cold_err!(mem.effective_addr::(base, u64::from(memarg.offset())))?; value.store_at(&mut *mem, addr) @@ -1001,14 +1026,14 @@ impl<'store, 'module> Executor<'store, 'module> { let mem_addr = self.mem_addr(MemAddr::from(memarg.memory())); crate::store::with_memory!(self.store.state, mem_addr, |mem, kind| { let base = if kind.arch() == MemoryArch::I64 { - let base = i64::local_get(&self.store.value_stack, &self.cf, u16::from(addr_local)) as u64; + let base = i64::local_get(&self.value_stack, &self.cf, u16::from(addr_local)) as u64; cold_err!(usize::try_from(base).map_err(|_| Trap::MemoryOutOfBounds { offset: usize::MAX, len: N, max: mem.len(), }))? } else { - u32::local_get(&self.store.value_stack, &self.cf, u16::from(addr_local)) as usize + u32::local_get(&self.value_stack, &self.cf, u16::from(addr_local)) as usize }; let addr = cold_err!(mem.effective_addr::(base, u64::from(memarg.offset())))?; @@ -1025,13 +1050,13 @@ impl<'store, 'module> Executor<'store, 'module> { &mut self, m: CompactMemoryArg, ) -> Result<(), Trap> { - let rhs = T::stack_pop(&mut self.store.value_stack); - let lhs = T::stack_pop(&mut self.store.value_stack); - let acc = T::stack_pop(&mut self.store.value_stack); + let rhs = T::stack_pop(&mut self.value_stack); + let lhs = T::stack_pop(&mut self.value_stack); + let acc = T::stack_pop(&mut self.value_stack); let fma = acc + lhs * rhs; let mem_addr = self.mem_addr(m.mem_addr()); crate::store::with_memory!(self.store.state, mem_addr, |mem, kind| { - let base = self.store.value_stack.pop_memory_operand(kind.arch())?; + let base = self.value_stack.pop_memory_operand(kind.arch())?; let addr = cold_err!(mem.effective_addr::(base, m.offset()))?; cold_err!(fma.store_at(&mut *mem, addr)) }) @@ -1056,37 +1081,37 @@ impl<'store, 'module> Executor<'store, 'module> { let mem_addr = self.mem_addr(MemAddr::from(memarg.memory())); crate::store::with_memory!(self.store.state, mem_addr, |mem, kind| { let base = if kind.arch() == MemoryArch::I64 { - let base = i64::local_get(&self.store.value_stack, &self.cf, u16::from(addr_local)) as u64; + let base = i64::local_get(&self.value_stack, &self.cf, u16::from(addr_local)) as u64; cold_err!(usize::try_from(base)).map_err(|_| Trap::MemoryOutOfBounds { offset: usize::MAX, len: N, max: mem.len(), })? } else { - u32::local_get(&self.store.value_stack, &self.cf, u16::from(addr_local)) as usize + u32::local_get(&self.value_stack, &self.cf, u16::from(addr_local)) as usize }; let addr = cold_err!(mem.effective_addr::(base, u64::from(memarg.offset())))?; let value = cast(cold_err!(LOAD::load_at(&*mem, addr))?); if SET_LOCAL { - TARGET::local_set(&mut self.store.value_stack, &self.cf, u16::from(dst_local), value); + TARGET::local_set(&mut self.value_stack, &self.cf, u16::from(dst_local), value); } if !SET_LOCAL || TEE { - TARGET::stack_push(&mut self.store.value_stack, value); + TARGET::stack_push(&mut self.value_stack, value); } Ok(()) }) } fn exec_ref_is_null(&mut self) -> Result<(), Trap> { - let is_null = i32::from(ValueRef::stack_pop(&mut self.store.value_stack).is_null()); - i32::stack_push(&mut self.store.value_stack, is_null); + let is_null = i32::from(ValueRef::stack_pop(&mut self.value_stack).is_null()); + i32::stack_push(&mut self.value_stack, is_null); Ok(()) } fn exec_ref_as_non_null(&mut self) -> Result<(), Trap> { - if ValueRef::stack_peek(&self.store.value_stack).is_null() { + if ValueRef::stack_peek(&self.value_stack).is_null() { return cold!(Err(Trap::NullReference)); } Ok(()) @@ -1098,14 +1123,14 @@ impl<'store, 'module> Executor<'store, 'module> { } fn exec_ref_matches(&self, ty: RefType) -> bool { - let value = ValueRef::stack_peek(&self.store.value_stack); + let value = ValueRef::stack_peek(&self.value_stack); self.store.state.value_ref_matches(value, self.canonical_ref_type(ty)) } fn exec_ref_test(&mut self, ty: RefType) -> Result<(), Trap> { - let value = ValueRef::stack_pop(&mut self.store.value_stack); + let value = ValueRef::stack_pop(&mut self.value_stack); let matches = self.store.state.value_ref_matches(value, self.canonical_ref_type(ty)); - i32::stack_push(&mut self.store.value_stack, i32::from(matches)); + i32::stack_push(&mut self.value_stack, i32::from(matches)); Ok(()) } @@ -1117,7 +1142,7 @@ impl<'store, 'module> Executor<'store, 'module> { } fn exec_i31_get(&mut self, signed: bool) -> Result<(), Trap> { - let value = ValueRef::stack_pop(&mut self.store.value_stack); + let value = ValueRef::stack_pop(&mut self.value_stack); if value.is_null() { return cold!(Err(Trap::NullI31Reference)); } @@ -1126,14 +1151,14 @@ impl<'store, 'module> Executor<'store, 'module> { } else { value.i31_u().expect("validated i31.get operand") as i32 }; - i32::stack_push(&mut self.store.value_stack, value); + i32::stack_push(&mut self.value_stack, value); Ok(()) } fn push_gc_object(&mut self, type_addr: TypeAddr, values: Vec) -> Result<(), Trap> { - let roots = (&self.store.value_stack.stack_32).into_iter().copied().map(ValueRef::from_raw); + let roots = (&self.value_stack.stack_32).into_iter().copied().map(ValueRef::from_raw); let reference = self.store.state.alloc_gc_object(type_addr, values, roots)?; - ValueRef::stack_push(&mut self.store.value_stack, reference); + ValueRef::stack_push(&mut self.value_stack, reference); Ok(()) } @@ -1158,7 +1183,7 @@ impl<'store, 'module> Executor<'store, 'module> { } else { for index in (0..field_count).rev() { let storage = self.store.state.get_type(type_addr).as_struct().unwrap().fields[index].storage; - values.push(pop_value(&mut self.store.value_stack, storage)); + values.push(pop_value(&mut self.value_stack, storage)); } values.reverse(); } @@ -1169,7 +1194,7 @@ impl<'store, 'module> Executor<'store, 'module> { let operand = index.resolve(&self.func.data); let type_index = operand.a(); let field_index = operand.b(); - let reference = ValueRef::stack_pop(&mut self.store.value_stack); + let reference = ValueRef::stack_pop(&mut self.value_stack); let type_addr = self.module.resolve_type_addr(type_index); let storage = self.store.state.get_type(type_addr).as_struct().expect("validated struct.get type").fields [field_index as usize] @@ -1177,7 +1202,7 @@ impl<'store, 'module> Executor<'store, 'module> { let object = self.store.state.gc_object(reference, type_addr)?; let object = self.store.state.gc.get_handle(object).ok_or(Trap::Other("invalid GC reference"))?; let value = *object.values.get(field_index as usize).expect("validated struct field index"); - push_value(&mut self.store.value_stack, value, storage, signed); + push_value(&mut self.value_stack, value, storage, signed); Ok(()) } @@ -1189,8 +1214,8 @@ impl<'store, 'module> Executor<'store, 'module> { let storage = self.store.state.get_type(type_addr).as_struct().expect("validated struct.set type").fields [field_index as usize] .storage; - let value = pop_value(&mut self.store.value_stack, storage); - let reference = ValueRef::stack_pop(&mut self.store.value_stack); + let value = pop_value(&mut self.value_stack, storage); + let reference = ValueRef::stack_pop(&mut self.value_stack); let object = self.store.state.gc_object(reference, type_addr)?; self.store.state.gc.set(object, field_index as usize, value).expect("live struct field"); Ok(()) @@ -1199,9 +1224,9 @@ impl<'store, 'module> Executor<'store, 'module> { fn exec_array_new(&mut self, type_index: TypeAddr, default: bool) -> Result<(), Trap> { let type_addr = self.module.resolve_type_addr(type_index); let storage = self.store.state.get_type(type_addr).as_array().expect("validated array.new type").field.storage; - let len = u32::stack_pop(&mut self.store.value_stack) as usize; + let len = u32::stack_pop(&mut self.value_stack) as usize; self.store.state.gc.check_allocation(len, self.store.state.gc_type_has_references(type_addr))?; - let value = if default { default_value(storage) } else { pop_value(&mut self.store.value_stack, storage) }; + let value = if default { default_value(storage) } else { pop_value(&mut self.value_stack, storage) }; let mut values = Vec::new(); cold_err!(values.try_reserve_exact(len)).map_err(|_| Trap::OutOfMemory)?; values.resize(len, value); @@ -1219,36 +1244,36 @@ impl<'store, 'module> Executor<'store, 'module> { let mut values = Vec::new(); cold_err!(values.try_reserve_exact(len)).map_err(|_| Trap::OutOfMemory)?; for _ in 0..len { - values.push(pop_value(&mut self.store.value_stack, arr_type.field.storage)); + values.push(pop_value(&mut self.value_stack, arr_type.field.storage)); } values.reverse(); self.push_gc_object(type_addr, values) } fn exec_array_get(&mut self, type_index: TypeAddr, signed: Option) -> Result<(), Trap> { - let index = u32::stack_pop(&mut self.store.value_stack) as usize; - let reference = ValueRef::stack_pop(&mut self.store.value_stack); + let index = u32::stack_pop(&mut self.value_stack) as usize; + let reference = ValueRef::stack_pop(&mut self.value_stack); let type_addr = self.module.resolve_type_addr(type_index); let storage = self.store.state.get_type(type_addr).as_array().expect("validated array.get type").field.storage; let object = self.store.state.gc_object(reference, type_addr)?; let object = self.store.state.gc.get_handle(object).ok_or(Trap::Other("invalid GC reference"))?; let value = *object.values.get(index).ok_or(Trap::ArrayOutOfBounds)?; - push_value(&mut self.store.value_stack, value, storage, signed); + push_value(&mut self.value_stack, value, storage, signed); Ok(()) } fn exec_array_set(&mut self, type_index: TypeAddr) -> Result<(), Trap> { let type_addr = self.module.resolve_type_addr(type_index); let storage = self.store.state.get_type(type_addr).as_array().expect("validated array.set type").field.storage; - let value = pop_value(&mut self.store.value_stack, storage); - let index = u32::stack_pop(&mut self.store.value_stack) as usize; - let reference = ValueRef::stack_pop(&mut self.store.value_stack); + let value = pop_value(&mut self.value_stack, storage); + let index = u32::stack_pop(&mut self.value_stack) as usize; + let reference = ValueRef::stack_pop(&mut self.value_stack); let object = self.store.state.gc_object(reference, type_addr)?; self.store.state.gc.set(object, index, value).ok_or(Trap::ArrayOutOfBounds) } fn exec_array_len(&mut self) -> Result<(), Trap> { - let reference = ValueRef::stack_pop(&mut self.store.value_stack); + let reference = ValueRef::stack_pop(&mut self.value_stack); if reference.is_null() { return Err(Trap::NullArrayReference); } @@ -1259,17 +1284,17 @@ impl<'store, 'module> Executor<'store, 'module> { if self.store.state.get_type(type_addr).as_array().is_none() { return Err(Trap::Other("GC reference is not an array")); } - i32::stack_push(&mut self.store.value_stack, object.values.len() as i32); + i32::stack_push(&mut self.value_stack, object.values.len() as i32); Ok(()) } fn exec_array_fill(&mut self, type_index: TypeAddr) -> Result<(), Trap> { let type_addr = self.module.resolve_type_addr(type_index); let storage = self.store.state.get_type(type_addr).as_array().expect("validated array.fill type").field.storage; - let len = u32::stack_pop(&mut self.store.value_stack) as usize; - let value = pop_value(&mut self.store.value_stack, storage); - let index = u32::stack_pop(&mut self.store.value_stack) as usize; - let reference = ValueRef::stack_pop(&mut self.store.value_stack); + let len = u32::stack_pop(&mut self.value_stack) as usize; + let value = pop_value(&mut self.value_stack, storage); + let index = u32::stack_pop(&mut self.value_stack) as usize; + let reference = ValueRef::stack_pop(&mut self.value_stack); let object = self.store.state.gc_object(reference, type_addr)?; let object_ref = self.store.state.gc.get_handle(object).ok_or(Trap::Other("invalid GC reference"))?; let end = index.checked_add(len).filter(|end| *end <= object_ref.values.len()).ok_or(Trap::ArrayOutOfBounds)?; @@ -1281,11 +1306,11 @@ impl<'store, 'module> Executor<'store, 'module> { let operand = index.resolve(&self.func.data); let dst_type = operand.a(); let src_type = operand.b(); - let len = u32::stack_pop(&mut self.store.value_stack) as usize; - let src_index = u32::stack_pop(&mut self.store.value_stack) as usize; - let src = ValueRef::stack_pop(&mut self.store.value_stack); - let dst_index = u32::stack_pop(&mut self.store.value_stack) as usize; - let dst = ValueRef::stack_pop(&mut self.store.value_stack); + let len = u32::stack_pop(&mut self.value_stack) as usize; + let src_index = u32::stack_pop(&mut self.value_stack) as usize; + let src = ValueRef::stack_pop(&mut self.value_stack); + let dst_index = u32::stack_pop(&mut self.value_stack) as usize; + let dst = ValueRef::stack_pop(&mut self.value_stack); let dst_type = self.module.resolve_type_addr(dst_type); let src_type = self.module.resolve_type_addr(src_type); let dst_handle = self.store.state.gc_object(dst, dst_type)?; @@ -1311,8 +1336,8 @@ impl<'store, 'module> Executor<'store, 'module> { let operand = index.resolve(&self.func.data); let type_index = operand.a(); let data_index = operand.b(); - let len = u32::stack_pop(&mut self.store.value_stack) as usize; - let src = u32::stack_pop(&mut self.store.value_stack) as usize; + let len = u32::stack_pop(&mut self.value_stack) as usize; + let src = u32::stack_pop(&mut self.value_stack) as usize; let type_addr = self.module.resolve_type_addr(type_index); let storage = self.store.state.get_type(type_addr).as_array().expect("validated array type").field.storage; let data_addr = self.module.resolve_data_addr(data_index); @@ -1327,8 +1352,8 @@ impl<'store, 'module> Executor<'store, 'module> { let operand = index.resolve(&self.func.data); let type_index = operand.a(); let elem_index = operand.b(); - let len = u32::stack_pop(&mut self.store.value_stack) as usize; - let src = u32::stack_pop(&mut self.store.value_stack) as usize; + let len = u32::stack_pop(&mut self.value_stack) as usize; + let src = u32::stack_pop(&mut self.value_stack) as usize; let type_addr = self.module.resolve_type_addr(type_index); let elem_addr = self.module.resolve_elem_addr(elem_index); let items = self.store.state.elements[elem_addr as usize].items_range(src, len)?; @@ -1343,10 +1368,10 @@ impl<'store, 'module> Executor<'store, 'module> { let operand = index.resolve(&self.func.data); let type_index = operand.a(); let data_index = operand.b(); - let len = u32::stack_pop(&mut self.store.value_stack) as usize; - let src = u32::stack_pop(&mut self.store.value_stack) as usize; - let dst = u32::stack_pop(&mut self.store.value_stack) as usize; - let reference = ValueRef::stack_pop(&mut self.store.value_stack); + let len = u32::stack_pop(&mut self.value_stack) as usize; + let src = u32::stack_pop(&mut self.value_stack) as usize; + let dst = u32::stack_pop(&mut self.value_stack) as usize; + let reference = ValueRef::stack_pop(&mut self.value_stack); let type_addr = self.module.resolve_type_addr(type_index); let storage = self.store.state.get_type(type_addr).as_array().expect("validated array type").field.storage; let object = self.store.state.gc_object(reference, type_addr)?; @@ -1363,10 +1388,10 @@ impl<'store, 'module> Executor<'store, 'module> { let operand = index.resolve(&self.func.data); let type_index = operand.a(); let elem_index = operand.b(); - let len = u32::stack_pop(&mut self.store.value_stack) as usize; - let src = u32::stack_pop(&mut self.store.value_stack) as usize; - let dst = u32::stack_pop(&mut self.store.value_stack) as usize; - let reference = ValueRef::stack_pop(&mut self.store.value_stack); + let len = u32::stack_pop(&mut self.value_stack) as usize; + let src = u32::stack_pop(&mut self.value_stack) as usize; + let dst = u32::stack_pop(&mut self.value_stack) as usize; + let reference = ValueRef::stack_pop(&mut self.value_stack); let type_addr = self.module.resolve_type_addr(type_index); let object = self.store.state.gc_object(reference, type_addr)?; let object_len = self.store.state.gc.get_handle(object).expect("validated array").values.len(); @@ -1382,8 +1407,8 @@ impl<'store, 'module> Executor<'store, 'module> { fn exec_memory_size(&mut self, addr: u32) -> Result<(), Trap> { match self.store.state.memory_size(self.mem_addr(addr)) { - (MemoryArch::I64, pages) => i64::stack_push(&mut self.store.value_stack, pages as i64), - (MemoryArch::I32, pages) => i32::stack_push(&mut self.store.value_stack, pages as i32), + (MemoryArch::I64, pages) => i64::stack_push(&mut self.value_stack, pages as i64), + (MemoryArch::I32, pages) => i32::stack_push(&mut self.value_stack, pages as i32), } Ok(()) } @@ -1393,14 +1418,14 @@ impl<'store, 'module> Executor<'store, 'module> { let limiter = self.store.engine.config().resource_limiter.as_deref(); let is_64bit = self.store.state.memory_type(mem_addr).arch() == MemoryArch::I64; let pages_delta = match is_64bit { - true => i64::stack_pop(&mut self.store.value_stack), - false => i64::from(i32::stack_pop(&mut self.store.value_stack)), + true => i64::stack_pop(&mut self.value_stack), + false => i64::from(i32::stack_pop(&mut self.value_stack)), }; let size = self.store.state.grow_mem(mem_addr, pages_delta, limiter)?.unwrap_or(-1); match is_64bit { - true => i64::stack_push(&mut self.store.value_stack, size), - false => i32::stack_push(&mut self.store.value_stack, size as i32), + true => i64::stack_push(&mut self.value_stack, size), + false => i32::stack_push(&mut self.value_stack, size as i32), }; Ok(()) @@ -1416,9 +1441,9 @@ impl<'store, 'module> Executor<'store, 'module> { let src_arch = self.store.state.memory_type(src_mem_addr).arch(); let len_arch = if dst_arch == MemoryArch::I32 || src_arch == MemoryArch::I32 { MemoryArch::I32 } else { MemoryArch::I64 }; - let size = self.store.value_stack.pop_memory_operand(len_arch)?; - let src = self.store.value_stack.pop_memory_operand(src_arch)?; - let dst = self.store.value_stack.pop_memory_operand(dst_arch)?; + let size = self.value_stack.pop_memory_operand(len_arch)?; + let src = self.value_stack.pop_memory_operand(src_arch)?; + let dst = self.value_stack.pop_memory_operand(dst_arch)?; self.store.state.copy_memories(dst_mem_addr, dst, src_mem_addr, src, size) } @@ -1426,9 +1451,9 @@ impl<'store, 'module> Executor<'store, 'module> { fn exec_memory_fill(&mut self, addr: u32) -> Result<(), Trap> { let mem_addr = self.mem_addr(addr); let arch = self.store.state.memory_type(mem_addr).arch(); - let size = self.store.value_stack.pop_memory_operand(arch)?; - let val = i32::stack_pop(&mut self.store.value_stack); - let dst = self.store.value_stack.pop_memory_operand(arch)?; + let size = self.value_stack.pop_memory_operand(arch)?; + let val = i32::stack_pop(&mut self.value_stack); + let dst = self.value_stack.pop_memory_operand(arch)?; self.exec_memory_fill_impl(mem_addr, dst, val as u8, size) } @@ -1436,7 +1461,7 @@ impl<'store, 'module> Executor<'store, 'module> { let operand = index.resolve(&self.func.data); let mem_addr = self.mem_addr(operand.memory()); let arch = self.store.state.memory_type(mem_addr).arch(); - let dst = self.store.value_stack.pop_memory_operand(arch)?; + let dst = self.value_stack.pop_memory_operand(arch)?; self.exec_memory_fill_impl(mem_addr, dst, operand.byte(), operand.value() as u32 as usize) } @@ -1454,11 +1479,11 @@ impl<'store, 'module> Executor<'store, 'module> { let operand = index.resolve(&self.func.data); let data_index = operand.a(); let mem_index = operand.b(); - let size = u32::stack_pop(&mut self.store.value_stack) as usize; - let offset = u32::stack_pop(&mut self.store.value_stack) as usize; + let size = u32::stack_pop(&mut self.value_stack) as usize; + let offset = u32::stack_pop(&mut self.value_stack) as usize; let mem_addr = self.mem_addr(mem_index); let arch = self.store.state.memory_type(mem_addr).arch(); - let dst = self.store.value_stack.pop_memory_operand(arch)?; + let dst = self.value_stack.pop_memory_operand(arch)?; let data = self.store.state.data[self.module.resolve_data_addr(data_index) as usize].data.clone(); // Dropped segments behave like empty segments, including valid zero-length copies. @@ -1509,11 +1534,11 @@ impl<'store, 'module> Executor<'store, 'module> { let m = arg.memory_arg_idx.resolve(&self.func.data); let mem_addr = self.mem_addr(m.memory()); crate::store::with_memory!(self.store.state, mem_addr, |mem, kind| { - let base = self.store.value_stack.pop_memory_operand(kind.arch())?; + let base = self.value_stack.pop_memory_operand(kind.arch())?; let addr = cold_err!(mem.effective_addr::(base, m.offset()))?; let val = cold_err!(LOAD::load_at(&*mem, addr))?; let offset = arg.lane as usize * LOAD_SIZE; - Value128::stack_update(&mut self.store.value_stack, |value| { + Value128::stack_update(&mut self.value_stack, |value| { let mut bytes = value.to_mem_bytes(); bytes[offset..offset + LOAD_SIZE].copy_from_slice(&val.to_mem_bytes()); Value128(bytes) @@ -1530,10 +1555,10 @@ impl<'store, 'module> Executor<'store, 'module> { ) -> Result<(), Trap> { let mem_addr = self.mem_addr(m.memory()); crate::store::with_memory!(self.store.state, mem_addr, |mem, kind| { - let base = self.store.value_stack.pop_memory_operand(kind.arch())?; + let base = self.value_stack.pop_memory_operand(kind.arch())?; let addr = cold_err!(mem.effective_addr::(base, m.offset()))?; let value = cold_err!(LOAD::load_at(&*mem, addr))?; - TARGET::stack_push(&mut self.store.value_stack, cast(value)); + TARGET::stack_push(&mut self.value_stack, cast(value)); Ok(()) }) } @@ -1548,16 +1573,16 @@ impl<'store, 'module> Executor<'store, 'module> { } fn exec_atomic_wait(&mut self, arg: Operand128Idx, op: AtomicWaitOp) -> Result<(), Trap> { - let timeout = if op == AtomicWaitOp::Notify { 0 } else { i64::stack_pop(&mut self.store.value_stack) }; + let timeout = if op == AtomicWaitOp::Notify { 0 } else { i64::stack_pop(&mut self.value_stack) }; let value = match op { - AtomicWaitOp::Wait64 => u64::stack_pop(&mut self.store.value_stack), - _ => u32::stack_pop(&mut self.store.value_stack) as u64, + AtomicWaitOp::Wait64 => u64::stack_pop(&mut self.value_stack), + _ => u32::stack_pop(&mut self.value_stack) as u64, }; let memory = arg.resolve(&self.func.data); let mem_addr = self.mem_addr(memory.memory()); let width = if op == AtomicWaitOp::Wait64 { 8 } else { 4 }; let addr = crate::store::with_memory!(@lock_inline self.store.state, mem_addr, |mem, kind| { - let base = self.store.value_stack.pop_memory_operand(kind.arch())?; + let base = self.value_stack.pop_memory_operand(kind.arch())?; let addr = cold_err!(mem.effective_addr::<1>(base, memory.offset()))?; if addr % width != 0 { return cold!(Err(Trap::UnalignedAtomic)); @@ -1586,11 +1611,11 @@ impl<'store, 'module> Executor<'store, 'module> { AtomicWaitOp::Wait32 => shared.wait::<4>(addr, value, timeout)?, AtomicWaitOp::Wait64 => shared.wait::<8>(addr, value, timeout)?, }; - u32::stack_push(&mut self.store.value_stack, result); + u32::stack_push(&mut self.value_stack, result); return Ok(()); } - u32::stack_push(&mut self.store.value_stack, 0); + u32::stack_push(&mut self.value_stack, 0); Ok(()) } @@ -1598,20 +1623,20 @@ impl<'store, 'module> Executor<'store, 'module> { let op = arg.op(); let value = match op { AtomicOp::Load => 0, - _ if arg.is_64() => u64::stack_pop(&mut self.store.value_stack), - _ => u32::stack_pop(&mut self.store.value_stack) as u64, + _ if arg.is_64() => u64::stack_pop(&mut self.value_stack), + _ => u32::stack_pop(&mut self.value_stack) as u64, }; let expected = match op { - AtomicOp::Cmpxchg if arg.is_64() => u64::stack_pop(&mut self.store.value_stack), - AtomicOp::Cmpxchg => u32::stack_pop(&mut self.store.value_stack) as u64, + AtomicOp::Cmpxchg if arg.is_64() => u64::stack_pop(&mut self.value_stack), + AtomicOp::Cmpxchg => u32::stack_pop(&mut self.value_stack) as u64, _ => 0, }; let memory = arg.memory.resolve(&self.func.data); let mem_addr = self.mem_addr(memory.memory()); crate::store::with_memory!(@lock_inline self.store.state, mem_addr, |mem, kind| { - let base = self.store.value_stack.pop_memory_operand(kind.arch())?; + let base = self.value_stack.pop_memory_operand(kind.arch())?; let addr = cold_err!(mem.effective_addr::(base, memory.offset()))?; if addr % N != 0 { return cold!(Err(Trap::UnalignedAtomic)); @@ -1644,8 +1669,8 @@ impl<'store, 'module> Executor<'store, 'module> { } match arg.is_64() { - true => u64::stack_push(&mut self.store.value_stack, old), - false => u32::stack_push(&mut self.store.value_stack, old as u32), + true => u64::stack_push(&mut self.value_stack, old), + false => u32::stack_push(&mut self.value_stack, old as u32), } Ok(()) }) @@ -1653,7 +1678,7 @@ impl<'store, 'module> Executor<'store, 'module> { #[inline(always)] fn exec_mem_store_lane + Copy, const N: usize>(&mut self, arg: MemoryLaneArg) -> Result<(), Trap> { - let bytes = Value128::stack_pop(&mut self.store.value_stack).to_mem_bytes(); + let bytes = Value128::stack_pop(&mut self.value_stack).to_mem_bytes(); let lane_offset = arg.lane as usize * N; let mut val_bytes = [0u8; N]; val_bytes.copy_from_slice(&bytes[lane_offset..lane_offset + N]); @@ -1661,7 +1686,7 @@ impl<'store, 'module> Executor<'store, 'module> { let m = arg.memory_arg_idx.resolve(&self.func.data); let mem_addr = self.mem_addr(m.memory()); crate::store::with_memory!(self.store.state, mem_addr, |mem, kind| { - let base = self.store.value_stack.pop_memory_operand(kind.arch())?; + let base = self.value_stack.pop_memory_operand(kind.arch())?; let addr = cold_err!(mem.effective_addr::(base, m.offset()))?; cold_err!(val.store_at(&mut *mem, addr)) }) @@ -1673,7 +1698,7 @@ impl<'store, 'module> Executor<'store, 'module> { memory: Operand128, cast: impl Fn(T) -> U, ) -> Result<(), Trap> { - let val = cast(::stack_pop(&mut self.store.value_stack)); + let val = cast(::stack_pop(&mut self.value_stack)); self.exec_mem_store_value(self.mem_addr(memory.memory()), memory.offset(), val) } @@ -1682,9 +1707,9 @@ impl<'store, 'module> Executor<'store, 'module> { &mut self, memory: Operand128, ) -> Result<(), Trap> { - let condition = Value32::stack_pop(&mut self.store.value_stack); - let false_value = T::stack_pop(&mut self.store.value_stack); - let true_value = T::stack_pop(&mut self.store.value_stack); + let condition = Value32::stack_pop(&mut self.value_stack); + let false_value = T::stack_pop(&mut self.value_stack); + let true_value = T::stack_pop(&mut self.value_stack); let value = if condition == 0 { false_value } else { true_value }; self.exec_mem_store_value(self.mem_addr(memory.memory()), memory.offset(), value) } @@ -1697,7 +1722,7 @@ impl<'store, 'module> Executor<'store, 'module> { val: U, ) -> Result<(), Trap> { crate::store::with_memory!(self.store.state, memory_addr, |mem, kind| { - let base = self.store.value_stack.pop_memory_operand(kind.arch())?; + let base = self.value_stack.pop_memory_operand(kind.arch())?; let addr = cold_err!(mem.effective_addr::(base, offset))?; cold_err!(val.store_at(&mut *mem, addr)) }) @@ -1707,12 +1732,12 @@ impl<'store, 'module> Executor<'store, 'module> { let table_addr = self.module.resolve_table_addr(table_index); let idx = self.pop_table_operand(self.store.state.get_table(table_addr).kind.arch())?; let value = *self.store.state.get_table(table_addr).get(idx)?; - ValueRef::stack_push(&mut self.store.value_stack, value); + ValueRef::stack_push(&mut self.value_stack, value); Ok(()) } fn exec_table_set(&mut self, table_index: u32) -> Result<(), Trap> { - let val = ValueRef::stack_pop(&mut self.store.value_stack); + let val = ValueRef::stack_pop(&mut self.value_stack); let table_addr = self.module.resolve_table_addr(table_index); let idx = self.pop_table_operand(self.store.state.get_table(table_addr).kind.arch())?; let table = self.store.state.get_table_mut(table_addr); @@ -1722,8 +1747,8 @@ impl<'store, 'module> Executor<'store, 'module> { fn exec_table_size(&mut self, table_index: u32) -> Result<(), Trap> { let table = self.store.state.get_table(self.module.resolve_table_addr(table_index)); match table.kind.arch() { - MemoryArch::I32 => i32::stack_push(&mut self.store.value_stack, table.size() as i32), - MemoryArch::I64 => i64::stack_push(&mut self.store.value_stack, table.size() as i64), + MemoryArch::I32 => i32::stack_push(&mut self.value_stack, table.size() as i32), + MemoryArch::I64 => i64::stack_push(&mut self.value_stack, table.size() as i64), } Ok(()) } @@ -1746,16 +1771,16 @@ impl<'store, 'module> Executor<'store, 'module> { let table_addr = self.module.resolve_table_addr(table_index); let arch = self.store.state.get_table(table_addr).kind.arch(); let n = self.pop_table_operand(arch)?; - let val = ValueRef::stack_pop(&mut self.store.value_stack); + let val = ValueRef::stack_pop(&mut self.value_stack); let limiter = self.store.engine.config().resource_limiter.as_deref(); let table = self.store.state.get_table_mut(table_addr); let sz = table.size(); let grew = table.grow(n, val, limiter)?; match (arch, grew) { - (MemoryArch::I32, true) => i32::stack_push(&mut self.store.value_stack, sz as i32), - (MemoryArch::I32, false) => i32::stack_push(&mut self.store.value_stack, -1), - (MemoryArch::I64, true) => i64::stack_push(&mut self.store.value_stack, sz as i64), - (MemoryArch::I64, false) => i64::stack_push(&mut self.store.value_stack, -1), + (MemoryArch::I32, true) => i32::stack_push(&mut self.value_stack, sz as i32), + (MemoryArch::I32, false) => i32::stack_push(&mut self.value_stack, -1), + (MemoryArch::I64, true) => i64::stack_push(&mut self.value_stack, sz as i64), + (MemoryArch::I64, false) => i64::stack_push(&mut self.value_stack, -1), } Ok(()) } @@ -1764,15 +1789,15 @@ impl<'store, 'module> Executor<'store, 'module> { let table_addr = self.module.resolve_table_addr(table_index); let arch = self.store.state.get_table(table_addr).kind.arch(); let n = self.pop_table_operand(arch)?; - let val = ValueRef::stack_pop(&mut self.store.value_stack); + let val = ValueRef::stack_pop(&mut self.value_stack); let i = self.pop_table_operand(arch)?; self.store.state.get_table_mut(table_addr).fill(i, n, val) } fn pop_table_operand(&mut self, arch: MemoryArch) -> Result { let value = match arch { - MemoryArch::I32 => i32::stack_pop(&mut self.store.value_stack) as u32 as u64, - MemoryArch::I64 => i64::stack_pop(&mut self.store.value_stack) as u64, + MemoryArch::I32 => i32::stack_pop(&mut self.value_stack) as u32 as u64, + MemoryArch::I64 => i64::stack_pop(&mut self.value_stack) as u64, }; cold_err!(usize::try_from(value).map_err(|_| Trap::TableOutOfBounds { offset: usize::MAX, @@ -1781,3 +1806,9 @@ impl<'store, 'module> Executor<'store, 'module> { })) } } + +impl Drop for Executor<'_, '_> { + fn drop(&mut self) { + self.store.value_stack = core::mem::take(&mut self.value_stack); + } +} diff --git a/crates/tinywasm/src/interpreter/num_helpers.rs b/crates/tinywasm/src/interpreter/num_helpers.rs index 53632bd..aa805a9 100644 --- a/crates/tinywasm/src/interpreter/num_helpers.rs +++ b/crates/tinywasm/src/interpreter/num_helpers.rs @@ -25,7 +25,7 @@ macro_rules! checked_conv_float { ($from:tt, $to:tt, $self:expr) => {{ checked_conv_float!($from, $to, $to, $self) }}; // Conversion with an intermediate unsigned type and error checking (three types) ($from:tt, $intermediate:tt, $to:tt, $self:expr) => {{ - let v = <$from>::stack_pop(&mut $self.store.value_stack); + let v = <$from>::stack_pop(&mut $self.value_stack); let (min, max) = float_min_max!($from, $intermediate); if v.is_nan() { core::hint::cold_path(); @@ -35,7 +35,7 @@ macro_rules! checked_conv_float { core::hint::cold_path(); return Err(crate::Trap::IntegerOverflow.into()); } - <$to>::stack_push(&mut $self.store.value_stack, (v as $intermediate as $to).into()); + <$to>::stack_push(&mut $self.value_stack, (v as $intermediate as $to).into()); }}; } diff --git a/crates/tinywasm/src/interpreter/stack/value_stack.rs b/crates/tinywasm/src/interpreter/stack/value_stack.rs index 457194e..20db970 100644 --- a/crates/tinywasm/src/interpreter/stack/value_stack.rs +++ b/crates/tinywasm/src/interpreter/stack/value_stack.rs @@ -6,6 +6,7 @@ use crate::engine::{Config, StackConfig}; use crate::interpreter::*; use crate::{Result, Trap}; +#[derive(Default)] #[cfg_attr(feature = "debug", derive(Debug))] /// Physical value lanes used by the interpreter. /// @@ -35,6 +36,13 @@ pub(crate) struct Stack { /// The most slots [`Stack::enter_locals`] writes ahead of the pushes that reach them. const WRITTEN_RESERVATION: usize = 64; +/// An empty stack without an allocation: what a store holds while an executor runs. +impl Default for Stack { + fn default() -> Self { + Self { data: Vec::new(), len: 0, max_size: 0, dynamic: false } + } +} + impl Stack { pub(crate) fn new(config: StackConfig) -> Self { Self { From bbbd500031c6ba516f8ccb66f7fe2060bae5d5e1 Mon Sep 17 00:00:00 2001 From: Matt Hargett Date: Tue, 29 Sep 2026 19:14:08 -0700 Subject: [PATCH 08/10] perf: pass the 32-bit value stack's height through the tail-call handlers Each tail-call handler receives the height of the 32-bit value stack, which most instructions push to or pop from, as its last argument. It writes the height to the stack before its body and reads it back before dispatching, and with the stack in the executor the compiler can keep it in a register in between, as it does in most handlers. A handler no longer loads the height its predecessor stored, a store-to-load round trip through memory on every dispatch. The Unbudgeted handlers now take six integer arguments and the Bounded ones five, which stay in registers on arm64 (eight) and x86-64 System V (six). Windows x64 passes four, so there the height goes on the stack along with the Instruction, which already does without this change. --- .../interpreter/executor/dispatch_become.rs | 49 ++++++++++++------- .../src/interpreter/stack/value_stack.rs | 10 ++++ 2 files changed, 42 insertions(+), 17 deletions(-) diff --git a/crates/tinywasm/src/interpreter/executor/dispatch_become.rs b/crates/tinywasm/src/interpreter/executor/dispatch_become.rs index c1c62fd..3cdcda9 100644 --- a/crates/tinywasm/src/interpreter/executor/dispatch_become.rs +++ b/crates/tinywasm/src/interpreter/executor/dispatch_become.rs @@ -3,11 +3,18 @@ use super::*; struct Unbudgeted; struct Bounded; +// The last argument of both handler types is the height of the 32-bit value stack, which most +// instructions push to or pop from. Each handler writes it back to the stack before its body runs +// and reads it again before dispatching, so no handler loads the height its predecessor stored. +// Most keep it in a register in between. A few load it back before dispatching: the 8- and 16-lane +// SIMD ops, and the fused `LoadLocal*` handlers, whose hot path joins a cold call. +// // Between two handlers only what the calling convention passes in registers stays out of memory. // On arm64_32 (watchOS) the Rust ABI passes an aggregate larger than a pointer, the 8-byte // `Instruction`, by reference, so every dispatch would store it and the next handler load it back. // The C convention passes it in a register; `C-unwind` still lets a host function's panic unwind. -// Elsewhere the handlers keep the Rust ABI. +// Elsewhere the handlers keep the Rust ABI, which passes eight integer arguments in registers on +// arm64, six on x86-64 System V (as many as the Unbudgeted handlers take) and four on Windows x64. macro_rules! handler_fn { ($(#[$meta:meta])* fn $($rest:tt)*) => { #[cfg(all(target_arch = "aarch64", target_pointer_width = "32"))] @@ -23,9 +30,9 @@ macro_rules! handler_types { // Both sides are Rust, so the C convention's view of these types need not be FFI-safe. #[allow(improper_ctypes_definitions)] type UnbudgetedHandler = - for<'store, 'module> $(extern $abi)? fn(&mut Executor<'store, 'module>, &[Instruction], usize, Instruction) -> ExecResult<()>; + for<'store, 'module> $(extern $abi)? fn(&mut Executor<'store, 'module>, &[Instruction], usize, Instruction, usize) -> ExecResult<()>; #[allow(improper_ctypes_definitions)] - type BoundedHandler = for<'store, 'module> $(extern $abi)? fn(&mut Executor<'store, 'module>, usize, Instruction, u32) -> ExecResult<()>; + type BoundedHandler = for<'store, 'module> $(extern $abi)? fn(&mut Executor<'store, 'module>, usize, Instruction, u32, usize) -> ExecResult<()>; }; } #[cfg(all(target_arch = "aarch64", target_pointer_width = "32"))] @@ -53,15 +60,17 @@ macro_rules! define_unbudgeted_tail_dispatch { instructions: &[Instruction], $instr_ptr: usize, instruction: Instruction, + height32: usize, ) -> ExecResult<()> { macro_rules! $dispatch_next { ($next_instr_ptr:expr) => {{ let next_instr_ptr = $next_instr_ptr; + let height32 = $executor.value_stack.stack_32.len(); let Some(&next) = instructions.get(next_instr_ptr) else { - become Self::invalid_instr_ptr($executor, instructions, next_instr_ptr, instruction); + become Self::invalid_instr_ptr($executor, instructions, next_instr_ptr, instruction, height32); }; let handler = Self::handler_for(next.opcode()); - become handler($executor, instructions, next_instr_ptr, next); + become handler($executor, instructions, next_instr_ptr, next, height32); }}; } macro_rules! $dispatch_flow { @@ -83,11 +92,12 @@ macro_rules! define_unbudgeted_tail_dispatch { } use tinywasm_types::Instruction::*; $(let $variant($($arg),*) = &instruction else { - become Self::handler_mismatch($executor, instructions, $instr_ptr, instruction); + become Self::handler_mismatch($executor, instructions, $instr_ptr, instruction, height32); };)? $(let $variant { $($field),* } = &instruction else { - become Self::handler_mismatch($executor, instructions, $instr_ptr, instruction); + become Self::handler_mismatch($executor, instructions, $instr_ptr, instruction, height32); };)? + $executor.value_stack.stack_32.set_len(height32); $body; $dispatch_next!($instr_ptr + 1) } @@ -115,10 +125,12 @@ macro_rules! define_bounded_tail_dispatch { $instr_ptr: usize, instruction: Instruction, instructions_until_checkpoint: u32, + height32: usize, ) -> ExecResult<()> { macro_rules! $dispatch_next { ($next_instr_ptr:expr) => {{ let next_instr_ptr = $next_instr_ptr; + let height32 = $executor.value_stack.stack_32.len(); if instructions_until_checkpoint == 0 { return cold!({ $executor.cf.instr_ptr = next_instr_ptr; @@ -127,10 +139,10 @@ macro_rules! define_bounded_tail_dispatch { } let Some(&next) = $executor.func.instructions.get(next_instr_ptr) else { - become Self::invalid_instr_ptr($executor, next_instr_ptr, instruction, instructions_until_checkpoint); + become Self::invalid_instr_ptr($executor, next_instr_ptr, instruction, instructions_until_checkpoint, height32); }; let handler = Self::handler_for(next.opcode()); - become handler($executor, next_instr_ptr, next, instructions_until_checkpoint - 1); + become handler($executor, next_instr_ptr, next, instructions_until_checkpoint - 1, height32); }}; } macro_rules! $dispatch_flow { @@ -150,11 +162,12 @@ macro_rules! define_bounded_tail_dispatch { } use tinywasm_types::Instruction::*; $(let $variant($($arg),*) = &instruction else { - become Self::handler_mismatch($executor, $instr_ptr, instruction, instructions_until_checkpoint); + become Self::handler_mismatch($executor, $instr_ptr, instruction, instructions_until_checkpoint, height32); };)? $(let $variant { $($field),* } = &instruction else { - become Self::handler_mismatch($executor, $instr_ptr, instruction, instructions_until_checkpoint); + become Self::handler_mismatch($executor, $instr_ptr, instruction, instructions_until_checkpoint, height32); };)? + $executor.value_stack.stack_32.set_len(height32); $body; $dispatch_next!($instr_ptr + 1) } @@ -171,7 +184,7 @@ impl Unbudgeted { handler_fn! { #[cold] #[inline(never)] - fn handler_mismatch(_: &mut Executor<'_, '_>, _: &[Instruction], _: usize, _: Instruction) -> ExecResult<()> { + fn handler_mismatch(_: &mut Executor<'_, '_>, _: &[Instruction], _: usize, _: Instruction, _: usize) -> ExecResult<()> { unreachable!("instruction handler mismatch") } } @@ -179,7 +192,7 @@ impl Unbudgeted { handler_fn! { #[cold] #[inline(never)] - fn invalid_instr_ptr(_: &mut Executor<'_, '_>, _: &[Instruction], instr_ptr: usize, _: Instruction) -> ExecResult<()> { + fn invalid_instr_ptr(_: &mut Executor<'_, '_>, _: &[Instruction], instr_ptr: usize, _: Instruction, _: usize) -> ExecResult<()> { unreachable!("instruction pointer {instr_ptr} out of range, this is a bug") } } @@ -193,7 +206,7 @@ impl Bounded { handler_fn! { #[cold] #[inline(never)] - fn handler_mismatch(_: &mut Executor<'_, '_>, _: usize, _: Instruction, _: u32) -> ExecResult<()> { + fn handler_mismatch(_: &mut Executor<'_, '_>, _: usize, _: Instruction, _: u32, _: usize) -> ExecResult<()> { unreachable!("instruction handler mismatch") } } @@ -201,7 +214,7 @@ impl Bounded { handler_fn! { #[cold] #[inline(never)] - fn invalid_instr_ptr(_: &mut Executor<'_, '_>, instr_ptr: usize, _: Instruction, _: u32) -> ExecResult<()> { + fn invalid_instr_ptr(_: &mut Executor<'_, '_>, instr_ptr: usize, _: Instruction, _: u32, _: usize) -> ExecResult<()> { unreachable!("instruction pointer {instr_ptr} out of range, this is a bug") } } @@ -212,7 +225,8 @@ impl Bounded { let instr_ptr = executor.cf.instr_ptr; let instruction = executor.func.instructions[instr_ptr]; let handler = Self::handler_for(instruction.opcode()); - handler(executor, instr_ptr, instruction, chunk_left - 1) + let height32 = executor.value_stack.stack_32.len(); + handler(executor, instr_ptr, instruction, chunk_left - 1, height32) } } @@ -226,7 +240,8 @@ impl Executor<'_, '_> { let instr_ptr = self.cf.instr_ptr; let instruction = instructions[instr_ptr]; let handler = Unbudgeted::handler_for(instruction.opcode()); - handler(&mut self, instructions, instr_ptr, instruction)?; + let height32 = self.value_stack.stack_32.len(); + handler(&mut self, instructions, instr_ptr, instruction, height32)?; if self.completed || self.left { return Ok(self.left()); } diff --git a/crates/tinywasm/src/interpreter/stack/value_stack.rs b/crates/tinywasm/src/interpreter/stack/value_stack.rs index 20db970..d897126 100644 --- a/crates/tinywasm/src/interpreter/stack/value_stack.rs +++ b/crates/tinywasm/src/interpreter/stack/value_stack.rs @@ -62,6 +62,16 @@ impl Stack { self.len } + /// Sets the height to one read from [`Self::len`] earlier, with nothing pushed or popped since. + /// The tail-call handlers pass the height between them and write it back on entry, so the + /// compiler can use the value in a register instead of reloading it from memory. + #[cfg(feature = "nightly-tail-calls")] + #[inline(always)] + pub(crate) fn set_len(&mut self, len: usize) { + debug_assert!(len <= self.data.len()); + self.len = len; + } + /// Pushes a value inside a function body. `enter_locals` reserved the function's whole operand /// stack, so the stack is never full here, and the handlers make no calls. #[inline(always)] From 54ff1131e12593011d65b95e5295838d55241cf9 Mon Sep 17 00:00:00 2001 From: Matt Hargett Date: Thu, 15 Oct 2026 00:09:47 -0700 Subject: [PATCH 09/10] perf: pass the tail-call handlers' arguments in six registers on Windows x64 Windows' calling convention passes four integer arguments in registers, so every dispatch stored the Unbudgeted handlers' last two and the next handler loaded them back. On Windows x64 the handlers now use the System V convention, which passes six, so the 32-bit height stays in a register. The instruction still goes through the stack: under the System V rules this enum is passed in memory. A CI job runs the tests with tail calls on Windows. --- .github/workflows/test.yaml | 5 +++++ .../src/interpreter/executor/dispatch_become.rs | 17 +++++++++++++---- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 8a94aa2..9ea1d1a 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -74,6 +74,11 @@ jobs: rust: stable name: "Windows x86_64" target: x86_64-pc-windows-msvc + - os: windows-2025 + rust: nightly + name: "Windows x86_64 (nightly tail calls)" + target: x86_64-pc-windows-msvc + args: "--features tinywasm/nightly-tail-calls" - os: ubuntu-26.04-arm rust: stable name: "Linux arm64" diff --git a/crates/tinywasm/src/interpreter/executor/dispatch_become.rs b/crates/tinywasm/src/interpreter/executor/dispatch_become.rs index 3cdcda9..62a131c 100644 --- a/crates/tinywasm/src/interpreter/executor/dispatch_become.rs +++ b/crates/tinywasm/src/interpreter/executor/dispatch_become.rs @@ -13,14 +13,21 @@ struct Bounded; // On arm64_32 (watchOS) the Rust ABI passes an aggregate larger than a pointer, the 8-byte // `Instruction`, by reference, so every dispatch would store it and the next handler load it back. // The C convention passes it in a register; `C-unwind` still lets a host function's panic unwind. -// Elsewhere the handlers keep the Rust ABI, which passes eight integer arguments in registers on -// arm64, six on x86-64 System V (as many as the Unbudgeted handlers take) and four on Windows x64. +// On Windows x64 the handlers use the System V convention (`sysv64-unwind`), which passes six +// integer arguments in registers where Windows' own passes four, so a dispatch no longer stores the +// height for the next handler to load back. The `Instruction` still goes through the stack there: +// the System V rules pass this enum in memory. Elsewhere the handlers keep the Rust ABI, which +// passes eight integer arguments in registers on arm64 and six on x86-64 System V, with the +// `Instruction` in one of them. macro_rules! handler_fn { ($(#[$meta:meta])* fn $($rest:tt)*) => { #[cfg(all(target_arch = "aarch64", target_pointer_width = "32"))] #[allow(improper_ctypes_definitions)] $(#[$meta])* extern "C-unwind" fn $($rest)* - #[cfg(not(all(target_arch = "aarch64", target_pointer_width = "32")))] + #[cfg(all(target_arch = "x86_64", windows))] + #[allow(improper_ctypes_definitions)] + $(#[$meta])* extern "sysv64-unwind" fn $($rest)* + #[cfg(not(any(all(target_arch = "aarch64", target_pointer_width = "32"), all(target_arch = "x86_64", windows))))] $(#[$meta])* fn $($rest)* }; } @@ -37,7 +44,9 @@ macro_rules! handler_types { } #[cfg(all(target_arch = "aarch64", target_pointer_width = "32"))] handler_types!("C-unwind"); -#[cfg(not(all(target_arch = "aarch64", target_pointer_width = "32")))] +#[cfg(all(target_arch = "x86_64", windows))] +handler_types!("sysv64-unwind"); +#[cfg(not(any(all(target_arch = "aarch64", target_pointer_width = "32"), all(target_arch = "x86_64", windows))))] handler_types!(); macro_rules! define_unbudgeted_tail_dispatch { From 81c5b50d83188b9556ff050444f004d665a90c32 Mon Sep 17 00:00:00 2001 From: Matt Hargett Date: Fri, 16 Oct 2026 21:11:16 -0700 Subject: [PATCH 10/10] perf: keep the bounded handlers in Windows x64's four argument registers Windows x64 passes four integer arguments in registers. The bounded handlers (fuel and time budgets) take four besides the 32-bit height, so passing the height put a fifth argument on the stack under either convention: the height under the Rust ABI, the instruction under System V, which passes this enum in memory. On Windows x64 the bounded handlers keep the Rust ABI and leave the height out: they read it from the stack before dispatching, as before the height was passed. The unbudgeted handlers keep the System V convention and the height. --- .../interpreter/executor/dispatch_become.rs | 72 +++++++++++++++---- 1 file changed, 57 insertions(+), 15 deletions(-) diff --git a/crates/tinywasm/src/interpreter/executor/dispatch_become.rs b/crates/tinywasm/src/interpreter/executor/dispatch_become.rs index 62a131c..1f4f3f2 100644 --- a/crates/tinywasm/src/interpreter/executor/dispatch_become.rs +++ b/crates/tinywasm/src/interpreter/executor/dispatch_become.rs @@ -13,10 +13,12 @@ struct Bounded; // On arm64_32 (watchOS) the Rust ABI passes an aggregate larger than a pointer, the 8-byte // `Instruction`, by reference, so every dispatch would store it and the next handler load it back. // The C convention passes it in a register; `C-unwind` still lets a host function's panic unwind. -// On Windows x64 the handlers use the System V convention (`sysv64-unwind`), which passes six -// integer arguments in registers where Windows' own passes four, so a dispatch no longer stores the -// height for the next handler to load back. The `Instruction` still goes through the stack there: -// the System V rules pass this enum in memory. Elsewhere the handlers keep the Rust ABI, which +// On Windows x64 the unbudgeted handlers use the System V convention (`sysv64-unwind`), which +// passes six integer arguments in registers where Windows' own passes four, so a dispatch no longer +// stores the height for the next handler to load back. The `Instruction` still goes through the +// stack there: the System V rules pass this enum in memory. The bounded handlers keep the Rust ABI +// on Windows x64 and do not pass the height, so their four arguments, the `Instruction` among them, +// stay in Windows' four argument registers. Elsewhere the handlers keep the Rust ABI, which // passes eight integer arguments in registers on arm64 and six on x86-64 System V, with the // `Instruction` in one of them. macro_rules! handler_fn { @@ -32,17 +34,57 @@ macro_rules! handler_fn { }; } +macro_rules! bounded_handler_fn { + ($(#[$meta:meta])* fn $($rest:tt)*) => { + #[cfg(all(target_arch = "aarch64", target_pointer_width = "32"))] + #[allow(improper_ctypes_definitions)] + $(#[$meta])* extern "C-unwind" fn $($rest)* + #[cfg(not(all(target_arch = "aarch64", target_pointer_width = "32")))] + $(#[$meta])* fn $($rest)* + }; +} + +/// The height a bounded handler passes on: nothing on Windows x64. +#[cfg(not(all(target_arch = "x86_64", windows)))] +type BoundedHeight = usize; +#[cfg(all(target_arch = "x86_64", windows))] +type BoundedHeight = (); + +#[inline(always)] +fn bounded_height(executor: &Executor<'_, '_>) -> BoundedHeight { + #[cfg(not(all(target_arch = "x86_64", windows)))] + return executor.value_stack.stack_32.len(); + #[cfg(all(target_arch = "x86_64", windows))] + { + let _ = executor; + } +} + +/// Writes a passed height back to the stack. +#[inline(always)] +fn enter_bounded_height(executor: &mut Executor<'_, '_>, height32: BoundedHeight) { + #[cfg(not(all(target_arch = "x86_64", windows)))] + executor.value_stack.stack_32.set_len(height32); + #[cfg(all(target_arch = "x86_64", windows))] + { + let _ = (executor, height32); + } +} + macro_rules! handler_types { ($($abi:literal)?) => { // Both sides are Rust, so the C convention's view of these types need not be FFI-safe. #[allow(improper_ctypes_definitions)] type UnbudgetedHandler = for<'store, 'module> $(extern $abi)? fn(&mut Executor<'store, 'module>, &[Instruction], usize, Instruction, usize) -> ExecResult<()>; - #[allow(improper_ctypes_definitions)] - type BoundedHandler = for<'store, 'module> $(extern $abi)? fn(&mut Executor<'store, 'module>, usize, Instruction, u32, usize) -> ExecResult<()>; }; } #[cfg(all(target_arch = "aarch64", target_pointer_width = "32"))] +#[allow(improper_ctypes_definitions)] +type BoundedHandler = for<'store, 'module> extern "C-unwind" fn(&mut Executor<'store, 'module>, usize, Instruction, u32, BoundedHeight) -> ExecResult<()>; +#[cfg(not(all(target_arch = "aarch64", target_pointer_width = "32")))] +type BoundedHandler = for<'store, 'module> fn(&mut Executor<'store, 'module>, usize, Instruction, u32, BoundedHeight) -> ExecResult<()>; +#[cfg(all(target_arch = "aarch64", target_pointer_width = "32"))] handler_types!("C-unwind"); #[cfg(all(target_arch = "x86_64", windows))] handler_types!("sysv64-unwind"); @@ -127,19 +169,19 @@ macro_rules! define_bounded_tail_dispatch { HANDLERS[opcode as usize] } - $(handler_fn! { + $(bounded_handler_fn! { #[allow(non_snake_case, unreachable_code, unused_imports, unused_macros, unused_variables)] fn $variant( $executor: &mut Executor<'_, '_>, $instr_ptr: usize, instruction: Instruction, instructions_until_checkpoint: u32, - height32: usize, + height32: BoundedHeight, ) -> ExecResult<()> { macro_rules! $dispatch_next { ($next_instr_ptr:expr) => {{ let next_instr_ptr = $next_instr_ptr; - let height32 = $executor.value_stack.stack_32.len(); + let height32 = bounded_height($executor); if instructions_until_checkpoint == 0 { return cold!({ $executor.cf.instr_ptr = next_instr_ptr; @@ -176,7 +218,7 @@ macro_rules! define_bounded_tail_dispatch { $(let $variant { $($field),* } = &instruction else { become Self::handler_mismatch($executor, $instr_ptr, instruction, instructions_until_checkpoint, height32); };)? - $executor.value_stack.stack_32.set_len(height32); + enter_bounded_height($executor, height32); $body; $dispatch_next!($instr_ptr + 1) } @@ -212,18 +254,18 @@ impl Bounded { // Tail-called like `Unbudgeted`'s. - handler_fn! { + bounded_handler_fn! { #[cold] #[inline(never)] - fn handler_mismatch(_: &mut Executor<'_, '_>, _: usize, _: Instruction, _: u32, _: usize) -> ExecResult<()> { + fn handler_mismatch(_: &mut Executor<'_, '_>, _: usize, _: Instruction, _: u32, _: BoundedHeight) -> ExecResult<()> { unreachable!("instruction handler mismatch") } } - handler_fn! { + bounded_handler_fn! { #[cold] #[inline(never)] - fn invalid_instr_ptr(_: &mut Executor<'_, '_>, instr_ptr: usize, _: Instruction, _: u32, _: usize) -> ExecResult<()> { + fn invalid_instr_ptr(_: &mut Executor<'_, '_>, instr_ptr: usize, _: Instruction, _: u32, _: BoundedHeight) -> ExecResult<()> { unreachable!("instruction pointer {instr_ptr} out of range, this is a bug") } } @@ -234,7 +276,7 @@ impl Bounded { let instr_ptr = executor.cf.instr_ptr; let instruction = executor.func.instructions[instr_ptr]; let handler = Self::handler_for(instruction.opcode()); - let height32 = executor.value_stack.stack_32.len(); + let height32 = bounded_height(executor); handler(executor, instr_ptr, instruction, chunk_left - 1, height32) } }