diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 6be4ae55..9ea1d1a5 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -46,6 +46,11 @@ jobs: name: "Linux x86 (nightly tail calls)" target: x86_64-unknown-linux-gnu args: "--features tinywasm/nightly-tail-calls" + - os: ubuntu-26.04 + rust: nightly + name: "Linux x86 (nightly tail calls, release with debug assertions)" + target: x86_64-unknown-linux-gnu + args: "--release --config profile.release.debug-assertions=true --features tinywasm/nightly-tail-calls" - os: ubuntu-26.04 rust: stable name: "Linux x86 (stable, no default features)" @@ -69,6 +74,11 @@ jobs: rust: stable name: "Windows x86_64" target: x86_64-pc-windows-msvc + - os: windows-2025 + rust: nightly + name: "Windows x86_64 (nightly tail calls)" + target: x86_64-pc-windows-msvc + args: "--features tinywasm/nightly-tail-calls" - os: ubuntu-26.04-arm rust: stable name: "Linux arm64" diff --git a/crates/tinywasm/src/instance.rs b/crates/tinywasm/src/instance.rs index f92765dd..b0b0b248 100644 --- a/crates/tinywasm/src/instance.rs +++ b/crates/tinywasm/src/instance.rs @@ -60,6 +60,12 @@ struct ModuleInstanceInner { id: ModuleInstanceId, type_addrs: Box<[TypeAddr]>, func_addrs: Box<[FuncAddr]>, + /// The module, whose functions are the instance's own (non-imported) ones in index order. The + /// store allocates them contiguously from `local_func_base`, so an executor can borrow any of + /// them for a whole run. Holding the module costs one reference count per instance. + module: Module, + local_func_base: FuncAddr, + imported_funcs: u32, table_addrs: Box<[TableAddr]>, mem_addrs: Box<[MemAddr]>, #[cfg(feature = "std")] @@ -84,6 +90,20 @@ impl ModuleInstance { self.0.func_addrs[addr as usize] } + /// The store address and body of the module's own function `idx`, or `None` for an import. + #[inline(always)] + pub(crate) fn local_func_by_index(&self, idx: FuncAddr) -> Option<(FuncAddr, &WasmFunction)> { + let local = idx.wrapping_sub(self.0.imported_funcs); + let func = self.0.module.funcs.get(local as usize)?; + Some((self.0.local_func_base + local, func)) + } + + /// The body of the function at store address `addr`, if this instance owns it. + #[inline(always)] + pub(crate) fn local_func(&self, addr: FuncAddr) -> Option<&WasmFunction> { + self.0.module.funcs.get(addr.wrapping_sub(self.0.local_func_base) as usize).map(|func| &**func) + } + /// resolve a table address to the global store address #[inline] pub(crate) fn resolve_table_addr(&self, addr: TableAddr) -> TableAddr { @@ -211,7 +231,9 @@ impl ModuleInstance { let id = store.next_module_instance_id(); let mut addrs = crate::imports::ResolvedImports::new(store, module, &type_addrs, imports)?; let imported_funcs = addrs.funcs.len(); - addrs.funcs.extend(store.init_funcs(&module.funcs, id, &module.func_type_idxs[imported_funcs..], &type_addrs)); + let local_funcs = store.init_funcs(&module.funcs, id, &module.func_type_idxs[imported_funcs..], &type_addrs); + let local_func_base = local_funcs.start; + addrs.funcs.extend(local_funcs); addrs.tags.extend(store.init_tags(&module.tags, &type_addrs)); let limiter = store.engine.config().resource_limiter.clone(); if !module.skip_local_memory_allocation { @@ -238,6 +260,9 @@ impl ModuleInstance { id, type_addrs, func_addrs: addrs.funcs.into_boxed_slice(), + module: module.clone(), + local_func_base, + imported_funcs: imported_funcs as u32, table_addrs: addrs.tables.into_boxed_slice(), mem_addrs: addrs.memories.into_boxed_slice(), #[cfg(feature = "std")] diff --git a/crates/tinywasm/src/interpreter/executor/dispatch.rs b/crates/tinywasm/src/interpreter/executor/dispatch.rs index ac520475..08477f73 100644 --- a/crates/tinywasm/src/interpreter/executor/dispatch.rs +++ b/crates/tinywasm/src/interpreter/executor/dispatch.rs @@ -4,7 +4,11 @@ macro_rules! define_stable_dispatch { ($executor:ident, $instr_ptr:ident, $dispatch_next:ident, $dispatch_flow:ident; $($variant:ident $(($($arg:pat),*))? $({ $($field:ident),* })? => $body:expr),* $(,)?) => { #[inline(always)] - fn exec_step($executor: &mut Self, $instr_ptr: usize) -> ExecResult { + fn exec_step( + $executor: &mut Self, + instructions: &[tinywasm_types::Instruction], + $instr_ptr: usize, + ) -> ExecResult { macro_rules! $dispatch_next { ($next_instr_ptr:expr) => {{ return Ok(ExecFlow::Next($next_instr_ptr)); @@ -16,7 +20,7 @@ macro_rules! define_stable_dispatch { }}; } use tinywasm_types::Instruction::*; - match &$executor.func.instructions[$instr_ptr] { + match &instructions[$instr_ptr] { $($variant $(($($arg),*))? $({ $($field),* })? => $body,)* } Ok(ExecFlow::Next($instr_ptr + 1)) @@ -24,67 +28,104 @@ macro_rules! define_stable_dispatch { }; } -impl Executor<'_> { +impl<'module> Executor<'_, 'module> { instruction_handlers!(define_stable_dispatch); + /// The executing function's instructions. They are borrowed from the module instance, not from + /// the executor, so the dispatch loop keeps them in a local across steps. #[inline(always)] - pub(crate) fn run_to_completion(mut self) -> Result<()> { + fn instructions(&self) -> &'module [tinywasm_types::Instruction] { + let func: &'module WasmFunction = self.func; + &func.instructions + } + + /// One step from `instr_ptr`. A step that switches functions reloads `instructions`. + #[inline(always)] + fn step( + &mut self, + instructions: &mut &'module [tinywasm_types::Instruction], + instr_ptr: usize, + ) -> ExecResult> { + Ok(match Self::exec_step(self, instructions, instr_ptr)? { + ExecFlow::Next(next_instr_ptr) => Some(next_instr_ptr), + ExecFlow::Switch(next_instr_ptr) => { + *instructions = self.instructions(); + Some(next_instr_ptr) + } + ExecFlow::Complete => None, + }) + } + + /// Runs until the call completes (`None`) or continues in another module instance's frame. + #[inline(always)] + pub(crate) fn run_to_completion(mut self) -> Result> { let mut instr_ptr = self.cf.instr_ptr; + let mut instructions = self.instructions(); loop { - match Self::exec_step(&mut self, instr_ptr)?.next_instr_ptr() { + match self.step(&mut instructions, instr_ptr)? { Some(next_instr_ptr) => instr_ptr = next_instr_ptr, - None => return cold!(Ok(())), + None => return cold!(Ok(self.left())), } } } + /// Runs `chunk_left` instructions, then checkpoint by checkpoint until `time_budget` has + /// elapsed since `start`. #[cfg(feature = "std")] #[inline(always)] - pub(crate) fn run_with_time_budget(mut self, time_budget: core::time::Duration) -> Result { - use crate::std::time::Instant; - - if time_budget.is_zero() { - return Ok(ExecState::Suspended(self.cf)); - } - let start = Instant::now(); + pub(crate) fn run_with_time_budget( + mut self, + start: crate::std::time::Instant, + time_budget: core::time::Duration, + mut chunk_left: u32, + ) -> Result { let mut instr_ptr = self.cf.instr_ptr; + let mut instructions = self.instructions(); loop { - for _ in 0..CHECKPOINT_INTERVAL { - match Self::exec_step(&mut self, instr_ptr)?.next_instr_ptr() { + while chunk_left != 0 { + chunk_left -= 1; + match self.step(&mut instructions, instr_ptr)? { Some(next_instr_ptr) => instr_ptr = next_instr_ptr, - None => return Ok(ExecState::Completed), + None => return Ok(self.run_end(chunk_left)), } } + chunk_left = CHECKPOINT_INTERVAL; if start.elapsed() >= time_budget { self.cf.instr_ptr = instr_ptr; - return Ok(ExecState::Suspended(self.cf)); + return Ok(RunEnd::State(ExecState::Suspended(self.cf))); } } } + /// Runs `chunk_left` instructions, then checkpoint by checkpoint until the store's fuel is out. #[inline(always)] - pub(crate) fn run_with_fuel(mut self, fuel: u32) -> Result { + pub(crate) fn run_with_fuel(mut self, mut chunk_left: u32) -> Result { self.fuel_metered = true; - self.store.execution_fuel = fuel; - if self.store.execution_fuel == 0 { - return Ok(ExecState::Suspended(self.cf)); - } - let mut instr_ptr = self.cf.instr_ptr; + let mut instructions = self.instructions(); loop { - for _ in 0..CHECKPOINT_INTERVAL { - match Self::exec_step(&mut self, instr_ptr)?.next_instr_ptr() { + while chunk_left != 0 { + chunk_left -= 1; + match self.step(&mut instructions, instr_ptr)? { Some(next_instr_ptr) => instr_ptr = next_instr_ptr, - None => return Ok(ExecState::Completed), + None => return Ok(self.run_end(chunk_left)), } } + chunk_left = CHECKPOINT_INTERVAL; self.store.execution_fuel = self.store.execution_fuel.saturating_sub(CHECKPOINT_INTERVAL); if self.store.execution_fuel == 0 { self.cf.instr_ptr = instr_ptr; - return Ok(ExecState::Suspended(self.cf)); + return Ok(RunEnd::State(ExecState::Suspended(self.cf))); } } } + + fn run_end(&self, chunk_left: u32) -> RunEnd { + match self.left() { + Some(frame) => RunEnd::Left(frame, chunk_left), + None => RunEnd::State(ExecState::Completed), + } + } } diff --git a/crates/tinywasm/src/interpreter/executor/dispatch_become.rs b/crates/tinywasm/src/interpreter/executor/dispatch_become.rs index f790d63e..1f4f3f2a 100644 --- a/crates/tinywasm/src/interpreter/executor/dispatch_become.rs +++ b/crates/tinywasm/src/interpreter/executor/dispatch_become.rs @@ -3,8 +3,93 @@ use super::*; struct Unbudgeted; struct Bounded; -type UnbudgetedHandler = for<'store> fn(&mut Executor<'store>, &[Instruction], usize, Instruction) -> ExecResult<()>; -type BoundedHandler = for<'store> fn(&mut Executor<'store>, usize, Instruction, u32) -> ExecResult<()>; +// The last argument of both handler types is the height of the 32-bit value stack, which most +// instructions push to or pop from. Each handler writes it back to the stack before its body runs +// and reads it again before dispatching, so no handler loads the height its predecessor stored. +// Most keep it in a register in between. A few load it back before dispatching: the 8- and 16-lane +// SIMD ops, and the fused `LoadLocal*` handlers, whose hot path joins a cold call. +// +// Between two handlers only what the calling convention passes in registers stays out of memory. +// On arm64_32 (watchOS) the Rust ABI passes an aggregate larger than a pointer, the 8-byte +// `Instruction`, by reference, so every dispatch would store it and the next handler load it back. +// The C convention passes it in a register; `C-unwind` still lets a host function's panic unwind. +// On Windows x64 the unbudgeted handlers use the System V convention (`sysv64-unwind`), which +// passes six integer arguments in registers where Windows' own passes four, so a dispatch no longer +// stores the height for the next handler to load back. The `Instruction` still goes through the +// stack there: the System V rules pass this enum in memory. The bounded handlers keep the Rust ABI +// on Windows x64 and do not pass the height, so their four arguments, the `Instruction` among them, +// stay in Windows' four argument registers. Elsewhere the handlers keep the Rust ABI, which +// passes eight integer arguments in registers on arm64 and six on x86-64 System V, with the +// `Instruction` in one of them. +macro_rules! handler_fn { + ($(#[$meta:meta])* fn $($rest:tt)*) => { + #[cfg(all(target_arch = "aarch64", target_pointer_width = "32"))] + #[allow(improper_ctypes_definitions)] + $(#[$meta])* extern "C-unwind" fn $($rest)* + #[cfg(all(target_arch = "x86_64", windows))] + #[allow(improper_ctypes_definitions)] + $(#[$meta])* extern "sysv64-unwind" fn $($rest)* + #[cfg(not(any(all(target_arch = "aarch64", target_pointer_width = "32"), all(target_arch = "x86_64", windows))))] + $(#[$meta])* fn $($rest)* + }; +} + +macro_rules! bounded_handler_fn { + ($(#[$meta:meta])* fn $($rest:tt)*) => { + #[cfg(all(target_arch = "aarch64", target_pointer_width = "32"))] + #[allow(improper_ctypes_definitions)] + $(#[$meta])* extern "C-unwind" fn $($rest)* + #[cfg(not(all(target_arch = "aarch64", target_pointer_width = "32")))] + $(#[$meta])* fn $($rest)* + }; +} + +/// The height a bounded handler passes on: nothing on Windows x64. +#[cfg(not(all(target_arch = "x86_64", windows)))] +type BoundedHeight = usize; +#[cfg(all(target_arch = "x86_64", windows))] +type BoundedHeight = (); + +#[inline(always)] +fn bounded_height(executor: &Executor<'_, '_>) -> BoundedHeight { + #[cfg(not(all(target_arch = "x86_64", windows)))] + return executor.value_stack.stack_32.len(); + #[cfg(all(target_arch = "x86_64", windows))] + { + let _ = executor; + } +} + +/// Writes a passed height back to the stack. +#[inline(always)] +fn enter_bounded_height(executor: &mut Executor<'_, '_>, height32: BoundedHeight) { + #[cfg(not(all(target_arch = "x86_64", windows)))] + executor.value_stack.stack_32.set_len(height32); + #[cfg(all(target_arch = "x86_64", windows))] + { + let _ = (executor, height32); + } +} + +macro_rules! handler_types { + ($($abi:literal)?) => { + // Both sides are Rust, so the C convention's view of these types need not be FFI-safe. + #[allow(improper_ctypes_definitions)] + type UnbudgetedHandler = + for<'store, 'module> $(extern $abi)? fn(&mut Executor<'store, 'module>, &[Instruction], usize, Instruction, usize) -> ExecResult<()>; + }; +} +#[cfg(all(target_arch = "aarch64", target_pointer_width = "32"))] +#[allow(improper_ctypes_definitions)] +type BoundedHandler = for<'store, 'module> extern "C-unwind" fn(&mut Executor<'store, 'module>, usize, Instruction, u32, BoundedHeight) -> ExecResult<()>; +#[cfg(not(all(target_arch = "aarch64", target_pointer_width = "32")))] +type BoundedHandler = for<'store, 'module> fn(&mut Executor<'store, 'module>, usize, Instruction, u32, BoundedHeight) -> ExecResult<()>; +#[cfg(all(target_arch = "aarch64", target_pointer_width = "32"))] +handler_types!("C-unwind"); +#[cfg(all(target_arch = "x86_64", windows))] +handler_types!("sysv64-unwind"); +#[cfg(not(any(all(target_arch = "aarch64", target_pointer_width = "32"), all(target_arch = "x86_64", windows))))] +handler_types!(); macro_rules! define_unbudgeted_tail_dispatch { ($executor:ident, $instr_ptr:ident, $dispatch_next:ident, $dispatch_flow:ident; @@ -19,22 +104,24 @@ macro_rules! define_unbudgeted_tail_dispatch { HANDLERS[opcode as usize] } - $( + $(handler_fn! { #[allow(non_snake_case, unreachable_code, unused_imports, unused_macros, unused_variables)] fn $variant( - $executor: &mut Executor<'_>, + $executor: &mut Executor<'_, '_>, instructions: &[Instruction], $instr_ptr: usize, instruction: Instruction, + height32: usize, ) -> ExecResult<()> { macro_rules! $dispatch_next { ($next_instr_ptr:expr) => {{ let next_instr_ptr = $next_instr_ptr; + let height32 = $executor.value_stack.stack_32.len(); let Some(&next) = instructions.get(next_instr_ptr) else { - become Self::invalid_instr_ptr($executor, instructions, next_instr_ptr, instruction); + become Self::invalid_instr_ptr($executor, instructions, next_instr_ptr, instruction, height32); }; let handler = Self::handler_for(next.opcode()); - become handler($executor, instructions, next_instr_ptr, next); + become handler($executor, instructions, next_instr_ptr, next, height32); }}; } macro_rules! $dispatch_flow { @@ -45,21 +132,27 @@ macro_rules! define_unbudgeted_tail_dispatch { $executor.cf.instr_ptr = next_instr_ptr; return Ok(()); }, - ExecFlow::Complete => return cold!({ $executor.completed = true; Ok(()) }), + ExecFlow::Complete => return cold!({ + if !$executor.left { + $executor.completed = true; + } + Ok(()) + }), } }}; } use tinywasm_types::Instruction::*; $(let $variant($($arg),*) = &instruction else { - become Self::handler_mismatch($executor, instructions, $instr_ptr, instruction); + become Self::handler_mismatch($executor, instructions, $instr_ptr, instruction, height32); };)? $(let $variant { $($field),* } = &instruction else { - become Self::handler_mismatch($executor, instructions, $instr_ptr, instruction); + become Self::handler_mismatch($executor, instructions, $instr_ptr, instruction, height32); };)? + $executor.value_stack.stack_32.set_len(height32); $body; $dispatch_next!($instr_ptr + 1) } - )* + })* }; } @@ -76,17 +169,19 @@ macro_rules! define_bounded_tail_dispatch { HANDLERS[opcode as usize] } - $( + $(bounded_handler_fn! { #[allow(non_snake_case, unreachable_code, unused_imports, unused_macros, unused_variables)] fn $variant( - $executor: &mut Executor<'_>, + $executor: &mut Executor<'_, '_>, $instr_ptr: usize, instruction: Instruction, instructions_until_checkpoint: u32, + height32: BoundedHeight, ) -> ExecResult<()> { macro_rules! $dispatch_next { ($next_instr_ptr:expr) => {{ let next_instr_ptr = $next_instr_ptr; + let height32 = bounded_height($executor); if instructions_until_checkpoint == 0 { return cold!({ $executor.cf.instr_ptr = next_instr_ptr; @@ -95,10 +190,10 @@ macro_rules! define_bounded_tail_dispatch { } let Some(&next) = $executor.func.instructions.get(next_instr_ptr) else { - become Self::invalid_instr_ptr($executor, next_instr_ptr, instruction, instructions_until_checkpoint); + become Self::invalid_instr_ptr($executor, next_instr_ptr, instruction, instructions_until_checkpoint, height32); }; let handler = Self::handler_for(next.opcode()); - become handler($executor, next_instr_ptr, next, instructions_until_checkpoint - 1); + become handler($executor, next_instr_ptr, next, instructions_until_checkpoint - 1, height32); }}; } macro_rules! $dispatch_flow { @@ -106,7 +201,11 @@ macro_rules! define_bounded_tail_dispatch { match $flow.next_instr_ptr() { Some(next_instr_ptr) => $dispatch_next!(next_instr_ptr), None => return cold!({ - $executor.completed = true; + if $executor.left { + $executor.chunk_left = instructions_until_checkpoint; + } else { + $executor.completed = true; + } Ok(()) }), } @@ -114,15 +213,16 @@ macro_rules! define_bounded_tail_dispatch { } use tinywasm_types::Instruction::*; $(let $variant($($arg),*) = &instruction else { - become Self::handler_mismatch($executor, $instr_ptr, instruction, instructions_until_checkpoint); + become Self::handler_mismatch($executor, $instr_ptr, instruction, instructions_until_checkpoint, height32); };)? $(let $variant { $($field),* } = &instruction else { - become Self::handler_mismatch($executor, $instr_ptr, instruction, instructions_until_checkpoint); + become Self::handler_mismatch($executor, $instr_ptr, instruction, instructions_until_checkpoint, height32); };)? + enter_bounded_height($executor, height32); $body; $dispatch_next!($instr_ptr + 1) } - )* + })* }; } @@ -132,16 +232,20 @@ impl Unbudgeted { // The handlers tail-call these cold paths instead of calling them: a call would make every // handler save a stack frame. - #[cold] - #[inline(never)] - fn handler_mismatch(_: &mut Executor<'_>, _: &[Instruction], _: usize, _: Instruction) -> ExecResult<()> { - unreachable!("instruction handler mismatch") + handler_fn! { + #[cold] + #[inline(never)] + fn handler_mismatch(_: &mut Executor<'_, '_>, _: &[Instruction], _: usize, _: Instruction, _: usize) -> ExecResult<()> { + unreachable!("instruction handler mismatch") + } } - #[cold] - #[inline(never)] - fn invalid_instr_ptr(_: &mut Executor<'_>, _: &[Instruction], instr_ptr: usize, _: Instruction) -> ExecResult<()> { - unreachable!("instruction pointer {instr_ptr} out of range, this is a bug") + handler_fn! { + #[cold] + #[inline(never)] + fn invalid_instr_ptr(_: &mut Executor<'_, '_>, _: &[Instruction], instr_ptr: usize, _: Instruction, _: usize) -> ExecResult<()> { + unreachable!("instruction pointer {instr_ptr} out of range, this is a bug") + } } } @@ -150,81 +254,100 @@ impl Bounded { // Tail-called like `Unbudgeted`'s. - #[cold] - #[inline(never)] - fn handler_mismatch(_: &mut Executor<'_>, _: usize, _: Instruction, _: u32) -> ExecResult<()> { - unreachable!("instruction handler mismatch") + bounded_handler_fn! { + #[cold] + #[inline(never)] + fn handler_mismatch(_: &mut Executor<'_, '_>, _: usize, _: Instruction, _: u32, _: BoundedHeight) -> ExecResult<()> { + unreachable!("instruction handler mismatch") + } } - #[cold] - #[inline(never)] - fn invalid_instr_ptr(_: &mut Executor<'_>, instr_ptr: usize, _: Instruction, _: u32) -> ExecResult<()> { - unreachable!("instruction pointer {instr_ptr} out of range, this is a bug") + bounded_handler_fn! { + #[cold] + #[inline(never)] + fn invalid_instr_ptr(_: &mut Executor<'_, '_>, instr_ptr: usize, _: Instruction, _: u32, _: BoundedHeight) -> ExecResult<()> { + unreachable!("instruction pointer {instr_ptr} out of range, this is a bug") + } } + /// Runs up to `chunk_left` (at least 1) instructions from `executor.cf`. #[inline(always)] - fn run(executor: &mut Executor<'_>) -> ExecResult<()> { + fn run(executor: &mut Executor<'_, '_>, chunk_left: u32) -> ExecResult<()> { let instr_ptr = executor.cf.instr_ptr; let instruction = executor.func.instructions[instr_ptr]; let handler = Self::handler_for(instruction.opcode()); - handler(executor, instr_ptr, instruction, CHECKPOINT_INTERVAL - 1) + let height32 = bounded_height(executor); + handler(executor, instr_ptr, instruction, chunk_left - 1, height32) } } -impl<'store> Executor<'store> { +impl Executor<'_, '_> { + /// Runs until the call completes (`None`) or continues in another module instance's frame. #[inline(always)] - pub(crate) fn run_to_completion(mut self) -> Result<()> { + pub(crate) fn run_to_completion(mut self) -> Result> { loop { - let func = self.func.clone(); + let func = self.func; let instructions = &func.instructions; let instr_ptr = self.cf.instr_ptr; let instruction = instructions[instr_ptr]; let handler = Unbudgeted::handler_for(instruction.opcode()); - handler(&mut self, instructions, instr_ptr, instruction)?; - if self.completed { - return Ok(()); + let height32 = self.value_stack.stack_32.len(); + handler(&mut self, instructions, instr_ptr, instruction, height32)?; + if self.completed || self.left { + return Ok(self.left()); } } } + /// Runs `chunk_left` instructions, then checkpoint by checkpoint until `time_budget` has + /// elapsed since `start`. #[cfg(feature = "std")] #[inline(always)] - pub(crate) fn run_with_time_budget(mut self, time_budget: core::time::Duration) -> Result { - use crate::std::time::Instant; - - if time_budget.is_zero() { - return Ok(ExecState::Suspended(self.cf)); - } - let start = Instant::now(); - + pub(crate) fn run_with_time_budget( + mut self, + start: crate::std::time::Instant, + time_budget: core::time::Duration, + mut chunk_left: u32, + ) -> Result { loop { - Bounded::run(&mut self)?; - if self.completed { - return cold!(Ok(ExecState::Completed)); + if chunk_left != 0 { + Bounded::run(&mut self, chunk_left)?; + if let Some(end) = self.run_end() { + return Ok(end); + } } + chunk_left = CHECKPOINT_INTERVAL; if start.elapsed() >= time_budget { - return cold!(Ok(ExecState::Suspended(self.cf))); + return cold!(Ok(RunEnd::State(ExecState::Suspended(self.cf)))); } } } + /// Runs `chunk_left` instructions, then checkpoint by checkpoint until the store's fuel is out. #[inline(always)] - pub(crate) fn run_with_fuel(mut self, fuel: u32) -> Result { + pub(crate) fn run_with_fuel(mut self, mut chunk_left: u32) -> Result { self.fuel_metered = true; - self.store.execution_fuel = fuel; - if self.store.execution_fuel == 0 { - return Ok(ExecState::Suspended(self.cf)); - } - loop { - Bounded::run(&mut self)?; - if self.completed { - return cold!(Ok(ExecState::Completed)); + if chunk_left != 0 { + Bounded::run(&mut self, chunk_left)?; + if let Some(end) = self.run_end() { + return Ok(end); + } } + chunk_left = CHECKPOINT_INTERVAL; self.store.execution_fuel = self.store.execution_fuel.saturating_sub(CHECKPOINT_INTERVAL); if self.store.execution_fuel == 0 { - return cold!(Ok(ExecState::Suspended(self.cf))); + return cold!(Ok(RunEnd::State(ExecState::Suspended(self.cf)))); } } } + + /// How a bounded chain that stopped ended, unless it stopped at a checkpoint. + #[inline(always)] + fn run_end(&self) -> Option { + if self.completed { + return cold!(Some(RunEnd::State(ExecState::Completed))); + } + self.left().map(|frame| RunEnd::Left(frame, self.chunk_left)) + } } diff --git a/crates/tinywasm/src/interpreter/executor/instructions.rs b/crates/tinywasm/src/interpreter/executor/instructions.rs index 22892ebf..15932fa3 100644 --- a/crates/tinywasm/src/interpreter/executor/instructions.rs +++ b/crates/tinywasm/src/interpreter/executor/instructions.rs @@ -6,14 +6,14 @@ macro_rules! exec_op { <$ty>::stack_push(value_stack, $expr?); Ok(()) } - exec_binary_fallible(&mut $executor.store.value_stack)?; + exec_binary_fallible(&mut $executor.value_stack)?; }}; ($executor:ident; unary $from:ty => $to:ty, |$v:ident| $expr:expr) => {{ fn exec_unary(value_stack: &mut ValueStack) { let $v = <$from>::stack_pop(value_stack); <$to>::stack_push(value_stack, $expr); } - exec_unary(&mut $executor.store.value_stack); + exec_unary(&mut $executor.value_stack); }}; ($executor:ident; binary $from:ty => $to:ty, |$lhs:ident, $rhs:ident| $expr:expr) => {{ exec_op!($executor; binary $from, $from => $to, |$lhs, $rhs| $expr) @@ -24,7 +24,7 @@ macro_rules! exec_op { let $lhs = <$lhs_ty>::stack_pop(value_stack); <$res>::stack_push(value_stack, $expr); } - exec_binary(&mut $executor.store.value_stack); + exec_binary(&mut $executor.value_stack); }}; ($executor:ident; ternary $from:ty => $to:ty, |$a:ident, $b:ident, $c:ident| $expr:expr) => {{ fn exec_ternary(value_stack: &mut ValueStack) { @@ -33,7 +33,7 @@ macro_rules! exec_op { let $a = <$from>::stack_pop(value_stack); <$to>::stack_push(value_stack, $expr); } - exec_ternary(&mut $executor.store.value_stack); + exec_ternary(&mut $executor.value_stack); }}; } @@ -43,15 +43,15 @@ macro_rules! instruction_handlers { ($emit:ident) => { $emit! { executor, instr_ptr, dispatch_next, dispatch_flow; Unreachable => { return cold!(Err(Trap::Unreachable.into())); }, - Drop32 => { _ = Value32::stack_pop(&mut executor.store.value_stack)}, - Drop64 => { _ = Value64::stack_pop(&mut executor.store.value_stack)}, - Drop128 => { _ = Value128::stack_pop(&mut executor.store.value_stack)}, - Select32 => Value32::stack_select(&mut executor.store.value_stack), - Select64 => Value64::stack_select(&mut executor.store.value_stack), - Select128 => Value128::stack_select(&mut executor.store.value_stack), + Drop32 => { _ = Value32::stack_pop(&mut executor.value_stack)}, + Drop64 => { _ = Value64::stack_pop(&mut executor.value_stack)}, + Drop128 => { _ = Value128::stack_pop(&mut executor.value_stack)}, + Select32 => Value32::stack_select(&mut executor.value_stack), + Select64 => Value64::stack_select(&mut executor.value_stack), + Select128 => Value128::stack_select(&mut executor.value_stack), SelectStore32(idx) => executor.exec_select_store::(idx.resolve(&executor.func.data))?, SelectStore64(idx) => executor.exec_select_store::(idx.resolve(&executor.func.data))?, - SelectMulti(counts) => executor.store.value_stack.select_multi(*counts), + SelectMulti(counts) => executor.value_stack.select_multi(*counts), Call(v) => dispatch_flow!(executor.exec_call_direct(*v, instr_ptr + 1)?), CallSelf => { executor.exec_call_self(instr_ptr + 1)?; dispatch_next!(0); }, CallIndirect(idx) => dispatch_flow!(executor.exec_call_indirect::(*idx, instr_ptr + 1)?), @@ -63,10 +63,10 @@ macro_rules! instruction_handlers { Throw(tag) => dispatch_flow!(executor.exec_throw(*tag, instr_ptr)?), ThrowRef => dispatch_flow!(executor.exec_throw_ref(instr_ptr)?), Jump(ip) => dispatch_next!(*ip as usize), - JumpIfZero32(ip) => if i32::stack_pop(&mut executor.store.value_stack) == 0 { dispatch_next!(*ip as usize) }, - JumpIfNonZero32(ip) => if i32::stack_pop(&mut executor.store.value_stack) != 0 { dispatch_next!(*ip as usize) }, - JumpIfZero64(ip) => if i64::stack_pop(&mut executor.store.value_stack) == 0 { dispatch_next!(*ip as usize) }, - JumpIfNonZero64(ip) => if i64::stack_pop(&mut executor.store.value_stack) != 0 { dispatch_next!(*ip as usize) }, + JumpIfZero32(ip) => if i32::stack_pop(&mut executor.value_stack) == 0 { dispatch_next!(*ip as usize) }, + JumpIfNonZero32(ip) => if i32::stack_pop(&mut executor.value_stack) != 0 { dispatch_next!(*ip as usize) }, + JumpIfZero64(ip) => if i64::stack_pop(&mut executor.value_stack) == 0 { dispatch_next!(*ip as usize) }, + JumpIfNonZero64(ip) => if i64::stack_pop(&mut executor.value_stack) != 0 { dispatch_next!(*ip as usize) }, JumpIfRefNull(ip) => { let ip = *ip; if executor.exec_jump_if_ref::() { dispatch_next!(ip as usize) } }, JumpIfRefNonNull(ip) => { let ip = *ip; if executor.exec_jump_if_ref::() { dispatch_next!(ip as usize) } }, BrOnCast(idx) => if let Some(ip) = executor.exec_br_on_cast::(*idx) { dispatch_next!(ip) }, @@ -91,33 +91,33 @@ macro_rules! instruction_handlers { JumpCmpLocalConst64(packed) => if let Some(ip) = executor.exec_jump_cmp_local_const64(*packed) { dispatch_next!(ip) }, JumpCmpLocalLocal32(packed) => if let Some(ip) = executor.exec_jump_cmp_local_local32(*packed) { dispatch_next!(ip) }, JumpCmpLocalLocal64(packed) => if let Some(ip) = executor.exec_jump_cmp_local_local64(*packed) { dispatch_next!(ip) }, - DropKeep32 { base, keep } => executor.store.value_stack.stack_32.truncate_keep((executor.cf.stack_base().s32 + u32::from(*base)) as usize, *keep as usize), - DropKeep64 { base, keep } => executor.store.value_stack.stack_64.truncate_keep((executor.cf.stack_base().s64 + u32::from(*base)) as usize, *keep as usize), - DropKeep128 { base, keep } => executor.store.value_stack.stack_128.truncate_keep((executor.cf.stack_base().s128 + u32::from(*base)) as usize, *keep as usize), + DropKeep32 { base, keep } => executor.value_stack.stack_32.truncate_keep((executor.cf.stack_base().s32 + u32::from(*base)) as usize, *keep as usize), + DropKeep64 { base, keep } => executor.value_stack.stack_64.truncate_keep((executor.cf.stack_base().s64 + u32::from(*base)) as usize, *keep as usize), + DropKeep128 { base, keep } => executor.value_stack.stack_128.truncate_keep((executor.cf.stack_base().s128 + u32::from(*base)) as usize, *keep as usize), BranchTable(idx) => dispatch_next!(executor.exec_branch_table(*idx)), Return => dispatch_flow!(executor.exec_return()), ReturnVoid => dispatch_flow!(executor.exec_return_void()), Return32 => dispatch_flow!(executor.exec_return_32()), Return64 => dispatch_flow!(executor.exec_return_64()), Return128 => dispatch_flow!(executor.exec_return_128()), - LocalGet32(local_index) => Value32::local_push(&mut executor.store.value_stack, &executor.cf, *local_index), - LocalGet64(local_index) => Value64::local_push(&mut executor.store.value_stack, &executor.cf, *local_index), - LocalGet128(local_index) => Value128::local_push(&mut executor.store.value_stack, &executor.cf, *local_index), + LocalGet32(local_index) => Value32::local_push(&mut executor.value_stack, &executor.cf, *local_index), + LocalGet64(local_index) => Value64::local_push(&mut executor.value_stack, &executor.cf, *local_index), + LocalGet128(local_index) => Value128::local_push(&mut executor.value_stack, &executor.cf, *local_index), LocalSet32(local_index) => executor.exec_local_set_pop::(*local_index), LocalSet64(local_index) => executor.exec_local_set_pop::(*local_index), LocalSet128(local_index) => executor.exec_local_set_pop::(*local_index), - LocalCopy32(from, to) => Value32::local_copy(&mut executor.store.value_stack, &executor.cf, *from, *to), - LocalCopy64(from, to) => Value64::local_copy(&mut executor.store.value_stack, &executor.cf, *from, *to), - LocalCopy128(from, to) => Value128::local_copy(&mut executor.store.value_stack, &executor.cf, *from, *to), - AddConst32(c) => { i32::stack_update(&mut executor.store.value_stack, |value| value.wrapping_add(*c)); }, - AndConst32(c) => { i32::stack_update(&mut executor.store.value_stack, |value| value & *c); }, - XorConst32(c) => { i32::stack_update(&mut executor.store.value_stack, |value| value ^ *c); }, - ShrUConst32(c) => { u32::stack_update(&mut executor.store.value_stack, |value| value.wrapping_shr(*c as u32)); }, - AddConst64(idx) => { let rhs = idx.resolve(&executor.func.data).value(); i64::stack_update(&mut executor.store.value_stack, |value| value.wrapping_add(rhs)); }, - BinOpStackConst32(op, rhs) => { u32::stack_update(&mut executor.store.value_stack, |lhs| op.exec(lhs, *rhs as u32)); }, - BinOpStackConst64(packed) => { let rhs = packed.index.resolve(&executor.func.data).value() as u64; u64::stack_update(&mut executor.store.value_stack, |lhs| packed.op.exec(lhs, rhs)); }, - IncLocal32(arg) => { i32::local_update(&mut executor.store.value_stack, &executor.cf, arg.local, |value| value.wrapping_add(arg.value)); }, - IncLocal64(packed) => { let rhs = packed.index.resolve(&executor.func.data).value(); i64::local_update(&mut executor.store.value_stack, &executor.cf, packed.op, |value| value.wrapping_add(rhs)); }, + LocalCopy32(from, to) => Value32::local_copy(&mut executor.value_stack, &executor.cf, *from, *to), + LocalCopy64(from, to) => Value64::local_copy(&mut executor.value_stack, &executor.cf, *from, *to), + LocalCopy128(from, to) => Value128::local_copy(&mut executor.value_stack, &executor.cf, *from, *to), + AddConst32(c) => { i32::stack_update(&mut executor.value_stack, |value| value.wrapping_add(*c)); }, + AndConst32(c) => { i32::stack_update(&mut executor.value_stack, |value| value & *c); }, + XorConst32(c) => { i32::stack_update(&mut executor.value_stack, |value| value ^ *c); }, + ShrUConst32(c) => { u32::stack_update(&mut executor.value_stack, |value| value.wrapping_shr(*c as u32)); }, + AddConst64(idx) => { let rhs = idx.resolve(&executor.func.data).value(); i64::stack_update(&mut executor.value_stack, |value| value.wrapping_add(rhs)); }, + BinOpStackConst32(op, rhs) => { u32::stack_update(&mut executor.value_stack, |lhs| op.exec(lhs, *rhs as u32)); }, + BinOpStackConst64(packed) => { let rhs = packed.index.resolve(&executor.func.data).value() as u64; u64::stack_update(&mut executor.value_stack, |lhs| packed.op.exec(lhs, rhs)); }, + IncLocal32(arg) => { i32::local_update(&mut executor.value_stack, &executor.cf, arg.local, |value| value.wrapping_add(arg.value)); }, + IncLocal64(packed) => { let rhs = packed.index.resolve(&executor.func.data).value(); i64::local_update(&mut executor.value_stack, &executor.cf, packed.op, |value| value.wrapping_add(rhs)); }, I32Add3 => exec_op!(executor; ternary i32 => i32, |a, b, c| a.wrapping_add(b).wrapping_add(c)), I64Add3 => exec_op!(executor; ternary i64 => i64, |a, b, c| a.wrapping_add(b).wrapping_add(c)), MulAccLocal32(acc) => executor.exec_mul_acc_local::(*acc, i32::wrapping_mul, i32::wrapping_add), @@ -160,9 +160,9 @@ macro_rules! instruction_handlers { BinOpGlobalConst32(packed) => { let v = packed.index.resolve(&executor.func.data); executor.exec_binop_global_const::(v.a(), v.b(), packed.op)?; }, BinOpGlobalConst64(packed) => { let v = packed.index.resolve(&executor.func.data); executor.exec_binop_global_const::(v.a(), v.b(), packed.op)?; }, BinOpGlobalConst128(packed) => { let v = packed.index.resolve(&executor.func.data); let rhs = Value128(v.b().resolve(&executor.func.data).value()); executor.exec_binop_global_const::(v.a(), rhs, packed.op)?; }, - SetLocalConst32(arg) => i32::local_set(&mut executor.store.value_stack, &executor.cf, arg.local, arg.value), - SetLocalConst64(packed) => { let v = packed.index.resolve(&executor.func.data).value(); i64::local_set(&mut executor.store.value_stack, &executor.cf, packed.op, v); }, - SetLocalConst128(packed) => Value128::local_set(&mut executor.store.value_stack, &executor.cf, packed.op, Value128(packed.index.resolve(&executor.func.data).value())), + SetLocalConst32(arg) => i32::local_set(&mut executor.value_stack, &executor.cf, arg.local, arg.value), + SetLocalConst64(packed) => { let v = packed.index.resolve(&executor.func.data).value(); i64::local_set(&mut executor.value_stack, &executor.cf, packed.op, v); }, + SetLocalConst128(packed) => Value128::local_set(&mut executor.value_stack, &executor.cf, packed.op, Value128(packed.index.resolve(&executor.func.data).value())), IncMemoryLocal32(arg) => executor.exec_inc_memory_local::(arg.memory_arg_idx, arg.local1, #[inline(always)] |v| v.wrapping_add(1))?, IncMemoryLocal64(arg) => executor.exec_inc_memory_local::(arg.memory_arg_idx, arg.local1, #[inline(always)] |v| v.wrapping_add(1))?, StoreLocalLocal32(arg) => executor.exec_store_local_local::(arg.memory_arg_idx, arg.local1, arg.local2)?, @@ -202,10 +202,10 @@ macro_rules! instruction_handlers { GlobalTee32(global_index) => executor.exec_global_tee::(*global_index), GlobalTee64(global_index) => executor.exec_global_tee::(*global_index), GlobalTee128(global_index) => executor.exec_global_tee::(*global_index), - Const32(val) => i32::stack_push(&mut executor.store.value_stack, *val), - Const64Imm(val) => i64::stack_push(&mut executor.store.value_stack, i64::from(*val)), - Const64(idx) => i64::stack_push(&mut executor.store.value_stack, idx.resolve(&executor.func.data).value()), - Const128Imm(val) => Value128::stack_push(&mut executor.store.value_stack, Value128(u128::from(*val).to_le_bytes())), + Const32(val) => i32::stack_push(&mut executor.value_stack, *val), + Const64Imm(val) => i64::stack_push(&mut executor.value_stack, i64::from(*val)), + Const64(idx) => i64::stack_push(&mut executor.value_stack, idx.resolve(&executor.func.data).value()), + Const128Imm(val) => Value128::stack_push(&mut executor.value_stack, Value128(u128::from(*val).to_le_bytes())), I64Eqz => exec_op!(executor; unary i64 => i32, |v| i32::from(v == 0)), I32Eqz => exec_op!(executor; unary i32 => i32, |v| i32::from(v == 0)), I32Eq => exec_op!(executor; binary i32 => i32, |a, b| i32::from(a == b)), @@ -290,8 +290,8 @@ macro_rules! instruction_handlers { I64Popcnt => exec_op!(executor; unary i64 => i64, |v| i64::from(v.count_ones())), // Reference types - RefFunc(func_idx) => ValueRef::stack_push(&mut executor.store.value_stack, ValueRef::from_category_addr(executor.module.resolve_func_addr(*func_idx))), - RefNull(_) => ValueRef::stack_push(&mut executor.store.value_stack, ValueRef::NULL), + RefFunc(func_idx) => ValueRef::stack_push(&mut executor.value_stack, ValueRef::from_category_addr(executor.module.resolve_func_addr(*func_idx))), + RefNull(_) => ValueRef::stack_push(&mut executor.value_stack, ValueRef::NULL), RefIsNull => executor.exec_ref_is_null()?, RefAsNonNull => executor.exec_ref_as_non_null()?, RefI31 => exec_op!(executor; unary i32 => ValueRef, |v| ValueRef::from_i31(v)), @@ -460,7 +460,7 @@ macro_rules! instruction_handlers { V128Store64Lane(arg) => executor.exec_mem_store_lane::(*arg)?, V128Load32Zero(idx) => executor.exec_mem_load::(idx.resolve(&executor.func.data), |v| Value128::from_i32x4([v, 0, 0, 0]))?, V128Load64Zero(idx) => executor.exec_mem_load::(idx.resolve(&executor.func.data), |v| Value128::from_i64x2([v, 0]))?, - Const128(arg) => Value128::stack_push(&mut executor.store.value_stack, Value128(arg.resolve(&executor.func.data).value())), + Const128(arg) => Value128::stack_push(&mut executor.value_stack, Value128(arg.resolve(&executor.func.data).value())), I8x16ExtractLaneS(lane) => executor.exec_simd_extract_lane::(*lane, |v, lane| v.extract_lane_i8(lane) as i32)?, I8x16ExtractLaneU(lane) => executor.exec_simd_extract_lane::(*lane, |v, lane| v.extract_lane_u8(lane) as i32)?, I16x8ExtractLaneS(lane) => executor.exec_simd_extract_lane::(*lane, |v, lane| v.extract_lane_i16(lane) as i32)?, diff --git a/crates/tinywasm/src/interpreter/executor/mod.rs b/crates/tinywasm/src/interpreter/executor/mod.rs index decc92be..33bec020 100644 --- a/crates/tinywasm/src/interpreter/executor/mod.rs +++ b/crates/tinywasm/src/interpreter/executor/mod.rs @@ -26,7 +26,7 @@ mod dispatch; #[cfg(feature = "nightly-tail-calls")] mod dispatch_become; -const CHECKPOINT_INTERVAL: u32 = 128; +pub(crate) const CHECKPOINT_INTERVAL: u32 = 128; const FUEL_COST_CALL_TOTAL: u32 = 5; struct ExecError(Box); @@ -62,6 +62,7 @@ enum ExecFlow { } impl ExecFlow { + #[cfg(feature = "nightly-tail-calls")] #[inline(always)] fn next_instr_ptr(self) -> Option { match self { @@ -71,49 +72,108 @@ impl ExecFlow { } } -pub(crate) struct Executor<'store> { +/// Runs frames of one module instance. The executing function is borrowed from the instance, which +/// the caller keeps alive for the whole run, so calls and returns within the instance switch a +/// reference instead of cloning a `Shared` handle. Execution that continues in a frame of another +/// instance ends the run with [`Executor::left`] set; [`InterpreterRuntime`](super::InterpreterRuntime) +/// resumes that frame with an executor for its instance. +pub(crate) struct Executor<'store, 'module> { + /// The store's value stack, which the executor holds while it runs and gives back for host + /// calls and when it is dropped. Reached through the executor, which each handler receives + /// as a `&mut`, stack accesses need one load fewer, and the compiler knows that no store to a + /// slot or to memory changes a stack's height. + value_stack: ValueStack, cf: CallFrame, - func: Shared, - module: ModuleInstance, + func: &'module WasmFunction, + module: &'module ModuleInstance, store: &'store mut Store, call_stack_base: u32, mem0: MemAddr, fuel_metered: bool, + /// Execution continues in `cf`, which belongs to another module instance. + left: bool, + /// Budgeted runs: instructions left before the next checkpoint when the run left. + #[cfg(feature = "nightly-tail-calls")] + chunk_left: u32, #[cfg(feature = "nightly-tail-calls")] completed: bool, } -impl<'store> Executor<'store> { - pub(crate) fn new(store: &'store mut Store, cf: CallFrame, call_stack_base: u32) -> Self { - let wasm_func = store.state.funcs.wasm(cf.func_addr); - let module = store.get_module_instance(wasm_func.owner).expect("invalid module instance").clone(); - let mem0 = module.mem0_addr(); +/// How a budgeted run of one executor ended. +pub(crate) enum RunEnd { + State(ExecState), + /// Execution continues in this frame of another module instance, with this many instructions + /// left before the next checkpoint. + Left(CallFrame, u32), +} + +impl<'store, 'module> Executor<'store, 'module> { + /// `module` must own the function of `cf`. + pub(crate) fn new( + store: &'store mut Store, + module: &'module ModuleInstance, + cf: CallFrame, + call_stack_base: u32, + ) -> Self { + let func = module.local_func(cf.func_addr).expect("call frame from another module instance"); + let value_stack = core::mem::take(&mut store.value_stack); Self { + value_stack, module, cf, - func: wasm_func.func.clone(), + func, store, call_stack_base, - mem0, + mem0: module.mem0_addr(), fuel_metered: false, + left: false, + #[cfg(feature = "nightly-tail-calls")] + chunk_left: 0, #[cfg(feature = "nightly-tail-calls")] completed: false, } } + /// The frame to resume in another module instance, if execution left this one. + #[inline(always)] + pub(crate) fn left(&self) -> Option { + self.left.then_some(self.cf) + } + + /// Ends the run because `self.cf` belongs to another module instance. + #[cold] + fn leave(&mut self) -> ExecFlow { + self.left = true; + ExecFlow::Complete + } + + /// Runs `f` with the value stack back in the store, as host functions expect it. + #[inline(always)] + fn with_store_value_stack(&mut self, f: impl FnOnce(&mut Store) -> R) -> R { + /// Takes the stack back when `f` returns or unwinds: a host function may catch a panic + /// from a nested call, and the executor must then still hold the stack. + struct Lent<'a, 'store> { + store: &'a mut &'store mut Store, + value_stack: &'a mut ValueStack, + } + impl Drop for Lent<'_, '_> { + #[inline(always)] + fn drop(&mut self) { + *self.value_stack = core::mem::take(&mut self.store.value_stack); + } + } + + self.store.value_stack = core::mem::take(&mut self.value_stack); + let lent = Lent { store: &mut self.store, value_stack: &mut self.value_stack }; + f(lent.store) + } + /// Resolves a module-local memory index to its store address, caching the common memory-0 case. #[inline(always)] fn mem_addr(&self, idx: MemAddr) -> MemAddr { if idx == 0 { self.mem0 } else { self.module.resolve_mem_addr(idx) } } - /// Switches the executor to another module, keeping the cached memory-0 address in sync. - #[inline] - fn set_module(&mut self, owner: ModuleInstanceId) { - self.module = self.store.get_module_instance(owner).expect("invalid module instance").clone(); - self.mem0 = self.module.mem0_addr(); - } - #[inline(always)] fn charge_call_fuel(&mut self, total_fuel_cost: u32) { if self.fuel_metered { @@ -128,9 +188,9 @@ impl<'store> Executor<'store> { #[inline(always)] fn exec_jump_if_ref(&mut self) -> bool { - let is_null = ValueRef::stack_peek(&self.store.value_stack).is_null(); + let is_null = ValueRef::stack_peek(&self.value_stack).is_null(); if is_null { - ValueRef::stack_pop(&mut self.store.value_stack); + ValueRef::stack_pop(&mut self.value_stack); } is_null == ON_NULL } @@ -147,41 +207,41 @@ impl<'store> Executor<'store> { &self, local: LocalAddr, ) -> bool { - (T::local_get(&self.store.value_stack, &self.cf, local) == T::default()) == ON_ZERO + (T::local_get(&self.value_stack, &self.cf, local) == T::default()) == ON_ZERO } #[inline(always)] fn exec_jump_cmp_stack_const32(&mut self, packed: PackedOp64) -> Option { let operand = packed.index.resolve(&self.func.data); - packed.op.cmp(i32::stack_pop(&mut self.store.value_stack), operand.b()).then_some(operand.target() as usize) + packed.op.cmp(i32::stack_pop(&mut self.value_stack), operand.b()).then_some(operand.target() as usize) } #[inline(always)] fn exec_jump_cmp_stack_const64(&mut self, packed: PackedOp128) -> Option { let operand = packed.index.resolve(&self.func.data); - packed.op.cmp(i64::stack_pop(&mut self.store.value_stack), operand.b()).then_some(operand.target() as usize) + packed.op.cmp(i64::stack_pop(&mut self.value_stack), operand.b()).then_some(operand.target() as usize) } #[inline(always)] fn exec_jump_cmp_stack_local32(&mut self, packed: PackedOp64) -> Option { let operand = packed.index.resolve(&self.func.data); - let lhs = i32::stack_pop(&mut self.store.value_stack); - let rhs = i32::local_get(&self.store.value_stack, &self.cf, operand.b()); + let lhs = i32::stack_pop(&mut self.value_stack); + let rhs = i32::local_get(&self.value_stack, &self.cf, operand.b()); packed.op.cmp(lhs, rhs).then_some(operand.target() as usize) } #[inline(always)] fn exec_jump_cmp_stack_local64(&mut self, packed: PackedOp64) -> Option { let operand = packed.index.resolve(&self.func.data); - let lhs = i64::stack_pop(&mut self.store.value_stack); - let rhs = i64::local_get(&self.store.value_stack, &self.cf, operand.b()); + let lhs = i64::stack_pop(&mut self.value_stack); + let rhs = i64::local_get(&self.value_stack, &self.cf, operand.b()); packed.op.cmp(lhs, rhs).then_some(operand.target() as usize) } #[inline(always)] fn exec_binop_local_const_jump(&mut self, packed: PackedOp128) -> Option { let operand = packed.index.resolve(&self.func.data); - let value = i32::local_update(&mut self.store.value_stack, &self.cf, operand.local(), |value| { + let value = i32::local_update(&mut self.value_stack, &self.cf, operand.local(), |value| { packed.op.exec(value as u32, operand.value() as u32) as i32 }); ((value == 0) == operand.on_zero()).then_some(operand.target() as usize) @@ -193,10 +253,10 @@ impl<'store> Executor<'store> { packed: PackedOp128<(BinOp, CmpOp), LocalUpdateCmpOperand>, ) -> Option { let operand = packed.index.resolve(&self.func.data); - let lhs = i32::local_update(&mut self.store.value_stack, &self.cf, operand.local(), |value| { + let lhs = i32::local_update(&mut self.value_stack, &self.cf, operand.local(), |value| { packed.op.0.exec(value as u32, operand.value() as u32) as i32 }); - let rhs = i32::local_get(&self.store.value_stack, &self.cf, operand.right()); + let rhs = i32::local_get(&self.value_stack, &self.cf, operand.right()); packed.op.1.cmp(lhs, rhs).then_some(operand.target() as usize) } @@ -206,10 +266,9 @@ impl<'store> Executor<'store> { packed: PackedOp128, ) -> Result, Trap> { let operand = packed.index.resolve(&self.func.data); - let value = i32::stack_update(&mut self.store.value_stack, |lhs| { - packed.op.exec(lhs as u32, operand.value() as u32) as i32 - }); - i32::local_set(&mut self.store.value_stack, &self.cf, operand.local(), value); + let value = + i32::stack_update(&mut self.value_stack, |lhs| packed.op.exec(lhs as u32, operand.value() as u32) as i32); + i32::local_set(&mut self.value_stack, &self.cf, operand.local(), value); Ok(((value == 0) == operand.on_zero()).then_some(operand.target() as usize)) } @@ -226,7 +285,7 @@ impl<'store> Executor<'store> { #[inline(always)] fn exec_inc_local_jump(&mut self, index: Operand128Idx) -> Option { let operand = index.resolve(&self.func.data); - let value = i32::local_update(&mut self.store.value_stack, &self.cf, operand.local(), |value| { + let value = i32::local_update(&mut self.value_stack, &self.cf, operand.local(), |value| { value.wrapping_add(operand.value()) }); ((value == 0) == operand.on_zero()).then_some(operand.target() as usize) @@ -238,8 +297,8 @@ impl<'store> Executor<'store> { index: Operand128Idx, ) -> Result, Trap> { let operand = index.resolve(&self.func.data); - let value = i32::stack_update(&mut self.store.value_stack, |value| value.wrapping_add(operand.value())); - i32::local_set(&mut self.store.value_stack, &self.cf, operand.local(), value); + let value = i32::stack_update(&mut self.value_stack, |value| value.wrapping_add(operand.value())); + i32::local_set(&mut self.value_stack, &self.cf, operand.local(), value); Ok(((value == 0) == operand.on_zero()).then_some(operand.target() as usize)) } @@ -255,84 +314,84 @@ impl<'store> Executor<'store> { #[inline(always)] fn exec_inc_local_jump_cmp_local(&mut self, packed: PackedOp128) -> Option { let operand = packed.index.resolve(&self.func.data); - let lhs = i32::local_update(&mut self.store.value_stack, &self.cf, operand.local(), |value| { + let lhs = i32::local_update(&mut self.value_stack, &self.cf, operand.local(), |value| { value.wrapping_add(operand.value()) }); - let rhs = i32::local_get(&self.store.value_stack, &self.cf, operand.right()); + let rhs = i32::local_get(&self.value_stack, &self.cf, operand.right()); packed.op.cmp(lhs, rhs).then_some(operand.target() as usize) } #[inline(always)] fn exec_jump_cmp_local_const32(&self, packed: PackedOp128) -> Option { let operand = packed.index.resolve(&self.func.data); - let lhs = i32::local_get(&self.store.value_stack, &self.cf, operand.c()); + let lhs = i32::local_get(&self.value_stack, &self.cf, operand.c()); packed.op.cmp(lhs, operand.b()).then_some(operand.target() as usize) } #[inline(always)] fn exec_jump_cmp_local_const64(&self, packed: PackedOp128) -> Option { let operand = packed.index.resolve(&self.func.data); - let lhs = i64::local_get(&self.store.value_stack, &self.cf, operand.c()); + let lhs = i64::local_get(&self.value_stack, &self.cf, operand.c()); packed.op.cmp(lhs, i64::from(operand.b())).then_some(operand.target() as usize) } #[inline(always)] fn exec_jump_cmp_local_local32(&self, packed: PackedOp64) -> Option { let operand = packed.index.resolve(&self.func.data); - let lhs = i32::local_get(&self.store.value_stack, &self.cf, operand.b()); - let rhs = i32::local_get(&self.store.value_stack, &self.cf, operand.c()); + let lhs = i32::local_get(&self.value_stack, &self.cf, operand.b()); + let rhs = i32::local_get(&self.value_stack, &self.cf, operand.c()); packed.op.cmp(lhs, rhs).then_some(operand.target() as usize) } #[inline(always)] fn exec_jump_cmp_local_local64(&self, packed: PackedOp64) -> Option { let operand = packed.index.resolve(&self.func.data); - let lhs = i64::local_get(&self.store.value_stack, &self.cf, operand.b()); - let rhs = i64::local_get(&self.store.value_stack, &self.cf, operand.c()); + let lhs = i64::local_get(&self.value_stack, &self.cf, operand.b()); + let rhs = i64::local_get(&self.value_stack, &self.cf, operand.c()); packed.op.cmp(lhs, rhs).then_some(operand.target() as usize) } fn exec_i64_add128(&mut self) -> Result<(), Trap> { - let b_hi = i64::stack_pop(&mut self.store.value_stack); - let b_lo = i64::stack_pop(&mut self.store.value_stack); - let a_hi = i64::stack_pop(&mut self.store.value_stack); - let a_lo = i64::stack_pop(&mut self.store.value_stack); + let b_hi = i64::stack_pop(&mut self.value_stack); + let b_lo = i64::stack_pop(&mut self.value_stack); + let a_hi = i64::stack_pop(&mut self.value_stack); + let a_lo = i64::stack_pop(&mut self.value_stack); let lo = a_lo.wrapping_add(b_lo); let carry = u64::from((lo as u64) < (a_lo as u64)); let hi = a_hi.wrapping_add(b_hi).wrapping_add(carry as i64); - i64::stack_push(&mut self.store.value_stack, lo); - i64::stack_push(&mut self.store.value_stack, hi); + i64::stack_push(&mut self.value_stack, lo); + i64::stack_push(&mut self.value_stack, hi); Ok(()) } fn exec_i64_sub128(&mut self) -> Result<(), Trap> { - let b_hi = i64::stack_pop(&mut self.store.value_stack); - let b_lo = i64::stack_pop(&mut self.store.value_stack); - let a_hi = i64::stack_pop(&mut self.store.value_stack); - let a_lo = i64::stack_pop(&mut self.store.value_stack); + let b_hi = i64::stack_pop(&mut self.value_stack); + let b_lo = i64::stack_pop(&mut self.value_stack); + let a_hi = i64::stack_pop(&mut self.value_stack); + let a_lo = i64::stack_pop(&mut self.value_stack); let lo = a_lo.wrapping_sub(b_lo); let borrow = u64::from((a_lo as u64) < (b_lo as u64)); let hi = a_hi.wrapping_sub(b_hi).wrapping_sub(borrow as i64); - i64::stack_push(&mut self.store.value_stack, lo); - i64::stack_push(&mut self.store.value_stack, hi); + i64::stack_push(&mut self.value_stack, lo); + i64::stack_push(&mut self.value_stack, hi); Ok(()) } fn exec_i64_mul_wide_s(&mut self) -> Result<(), Trap> { - let rhs = i64::stack_pop(&mut self.store.value_stack); - let lhs = i64::stack_pop(&mut self.store.value_stack); + let rhs = i64::stack_pop(&mut self.value_stack); + let lhs = i64::stack_pop(&mut self.value_stack); let product = (lhs as i128).wrapping_mul(rhs as i128); - i64::stack_push(&mut self.store.value_stack, product as i64); - i64::stack_push(&mut self.store.value_stack, (product >> 64) as i64); + i64::stack_push(&mut self.value_stack, product as i64); + i64::stack_push(&mut self.value_stack, (product >> 64) as i64); Ok(()) } fn exec_i64_mul_wide_u(&mut self) -> Result<(), Trap> { - let rhs = i64::stack_pop(&mut self.store.value_stack); - let lhs = i64::stack_pop(&mut self.store.value_stack); + let rhs = i64::stack_pop(&mut self.value_stack); + let lhs = i64::stack_pop(&mut self.value_stack); let product = (lhs as u64 as u128).wrapping_mul(rhs as u64 as u128); - i64::stack_push(&mut self.store.value_stack, product as u64 as i64); - i64::stack_push(&mut self.store.value_stack, (product >> 64) as u64 as i64); + i64::stack_push(&mut self.value_stack, product as u64 as i64); + i64::stack_push(&mut self.value_stack, (product >> 64) as u64 as i64); Ok(()) } @@ -342,8 +401,8 @@ impl<'store> Executor<'store> { lane: u8, operation: impl FnOnce(Value128, u8) -> TO, ) -> Result<(), Trap> { - let vector = Value128::stack_pop(&mut self.store.value_stack); - TO::stack_push(&mut self.store.value_stack, operation(vector, lane)); + let vector = Value128::stack_pop(&mut self.value_stack); + TO::stack_push(&mut self.value_stack, operation(vector, lane)); Ok(()) } @@ -353,50 +412,50 @@ impl<'store> Executor<'store> { lane: u8, operation: impl FnOnce(VALUE, Value128, u8) -> Value128, ) -> Result<(), Trap> { - let vector = Value128::stack_pop(&mut self.store.value_stack); - let value = VALUE::stack_pop(&mut self.store.value_stack); - Value128::stack_push(&mut self.store.value_stack, operation(value, vector, lane)); + let vector = Value128::stack_pop(&mut self.value_stack); + let value = VALUE::stack_pop(&mut self.value_stack); + Value128::stack_push(&mut self.value_stack, operation(value, vector, lane)); Ok(()) } fn exec_simd_shuffle(&mut self, lanes: Value128) -> Result<(), Trap> { - let rhs = Value128::stack_pop(&mut self.store.value_stack); - let lhs = Value128::stack_pop(&mut self.store.value_stack); - Value128::stack_push(&mut self.store.value_stack, Value128::i8x16_shuffle(lhs, rhs, lanes)); + let rhs = Value128::stack_pop(&mut self.value_stack); + let lhs = Value128::stack_pop(&mut self.value_stack); + Value128::stack_push(&mut self.value_stack, Value128::i8x16_shuffle(lhs, rhs, lanes)); Ok(()) } #[inline(always)] fn exec_local_set_pop(&mut self, local: LocalAddr) { - let value = T::stack_pop(&mut self.store.value_stack); - T::local_set(&mut self.store.value_stack, &self.cf, local, value); + let value = T::stack_pop(&mut self.value_stack); + T::local_set(&mut self.value_stack, &self.cf, local, value); } #[inline(always)] fn exec_local_tee(&mut self, local: LocalAddr) { - let value = T::stack_peek(&self.store.value_stack); - T::local_set(&mut self.store.value_stack, &self.cf, local, value); + let value = T::stack_peek(&self.value_stack); + T::local_set(&mut self.value_stack, &self.cf, local, value); } #[inline(always)] fn exec_global_get(&mut self, global: GlobalAddr) -> Result<(), Trap> { let addr = self.module.resolve_global_addr(global); let value = T::global_get(&self.store.state.globals, addr); - T::stack_push(&mut self.store.value_stack, value); + T::stack_push(&mut self.value_stack, value); Ok(()) } #[inline(always)] fn exec_global_set(&mut self, global: GlobalAddr) { let addr = self.module.resolve_global_addr(global); - let value = T::stack_pop(&mut self.store.value_stack); + let value = T::stack_pop(&mut self.value_stack); T::global_set(&mut self.store.state.globals, addr, value); } #[inline(always)] fn exec_global_tee(&mut self, global: GlobalAddr) { let addr = self.module.resolve_global_addr(global); - let value = T::stack_peek(&self.store.value_stack); + let value = T::stack_peek(&self.value_stack); T::global_set(&mut self.store.state.globals, addr, value); } @@ -407,10 +466,10 @@ impl<'store> Executor<'store> { val: T, ) -> Result<(), Trap> { if let Some(dst) = dst { - T::local_set(&mut self.store.value_stack, &self.cf, dst, val); + T::local_set(&mut self.value_stack, &self.cf, dst, val); } if PUSH { - T::stack_push(&mut self.store.value_stack, val); + T::stack_push(&mut self.value_stack, val); } Ok(()) } @@ -423,8 +482,8 @@ impl<'store> Executor<'store> { destination: Option, op: impl BinOpExt, ) -> Result<(), Trap> { - let lhs = T::local_get(&self.store.value_stack, &self.cf, lhs); - let rhs = T::local_get(&self.store.value_stack, &self.cf, rhs); + let lhs = T::local_get(&self.value_stack, &self.cf, lhs); + let rhs = T::local_get(&self.value_stack, &self.cf, rhs); self.exec_binop_result::(destination, op.exec(lhs, rhs)) } @@ -446,7 +505,7 @@ impl<'store> Executor<'store> { destination: Option, op: impl BinOpExt, ) -> Result<(), Trap> { - let lhs = T::local_get(&self.store.value_stack, &self.cf, lhs); + let lhs = T::local_get(&self.value_stack, &self.cf, lhs); self.exec_binop_result::(destination, op.exec(lhs, rhs)) } @@ -458,7 +517,7 @@ impl<'store> Executor<'store> { op: impl BinOpExt, ) -> Result<(), Trap> { let lhs = T::global_get(&self.store.state.globals, self.module.resolve_global_addr(lhs)); - T::stack_push(&mut self.store.value_stack, op.exec(lhs, rhs)); + T::stack_push(&mut self.value_stack, op.exec(lhs, rhs)); Ok(()) } @@ -468,9 +527,9 @@ impl<'store> Executor<'store> { T: InternalValue, CmpOp: CmpOpExt, { - let lhs = T::local_get(&self.store.value_stack, &self.cf, lhs); - let rhs = T::local_get(&self.store.value_stack, &self.cf, rhs); - i32::stack_push(&mut self.store.value_stack, i32::from(op.cmp(lhs, rhs))); + let lhs = T::local_get(&self.value_stack, &self.cf, lhs); + let rhs = T::local_get(&self.value_stack, &self.cf, rhs); + i32::stack_push(&mut self.value_stack, i32::from(op.cmp(lhs, rhs))); Ok(()) } @@ -482,7 +541,7 @@ impl<'store> Executor<'store> { ) -> Result<(), Trap> { let addr = self.module.resolve_global_addr(global); let rhs = T::global_get(&self.store.state.globals, addr); - T::stack_update(&mut self.store.value_stack, |lhs| op.exec(lhs, rhs)); + T::stack_update(&mut self.value_stack, |lhs| op.exec(lhs, rhs)); Ok(()) } @@ -493,8 +552,8 @@ impl<'store> Executor<'store> { destination: Option, op: impl BinOpExt, ) -> Result<(), Trap> { - let rhs = T::local_get(&self.store.value_stack, &self.cf, local); - let lhs = T::stack_pop(&mut self.store.value_stack); + let rhs = T::local_get(&self.value_stack, &self.cf, local); + let lhs = T::stack_pop(&mut self.value_stack); self.exec_binop_result::(destination, op.exec(lhs, rhs)) } @@ -505,8 +564,8 @@ impl<'store> Executor<'store> { destination: LocalAddr, op: impl BinOpExt, ) -> Result<(), Trap> { - let value = T::stack_update(&mut self.store.value_stack, |lhs| op.exec(lhs, rhs)); - T::local_set(&mut self.store.value_stack, &self.cf, destination, value); + let value = T::stack_update(&mut self.value_stack, |lhs| op.exec(lhs, rhs)); + T::local_set(&mut self.value_stack, &self.cf, destination, value); Ok(()) } @@ -517,15 +576,15 @@ impl<'store> Executor<'store> { multiply: fn(T, T) -> T, add: fn(T, T) -> T, ) { - let rhs = T::stack_pop(&mut self.store.value_stack); - let lhs = T::stack_pop(&mut self.store.value_stack); + let rhs = T::stack_pop(&mut self.value_stack); + let lhs = T::stack_pop(&mut self.value_stack); let product = multiply(lhs, rhs); - T::local_update(&mut self.store.value_stack, &self.cf, accumulator, |value| add(product, value)); + T::local_update(&mut self.value_stack, &self.cf, accumulator, |value| add(product, value)); } fn exec_branch_table(&mut self, index: Operand128Idx) -> usize { let v = index.resolve(&self.func.data); - let idx = ::stack_pop(&mut self.store.value_stack); + let idx = ::stack_pop(&mut self.value_stack); let target_ip = if idx >= 0 && (idx as u32) < v.size() { self.func.data.branch_table_targets.get((v.start() + idx as u32) as usize).copied().unwrap_or(v.target()) } else { @@ -542,7 +601,7 @@ impl<'store> Executor<'store> { self.store.state.gc.check_allocation(payload_len, true)?; let mut payload = Vec::new(); cold_err!(payload.try_reserve_exact(payload_len)).map_err(|_| Trap::OutOfMemory)?; - let value_stack = &mut self.store.value_stack; + let value_stack = &mut self.value_stack; for index in (0..payload_len).rev() { let ty = self.store.state.get_canonical_func_type(type_addr).params()[index]; payload.push(match ty { @@ -553,7 +612,7 @@ impl<'store> Executor<'store> { }); } payload.reverse(); - let roots = (&self.store.value_stack.stack_32).into_iter().copied().map(ValueRef::from_raw); + let roots = (&self.value_stack.stack_32).into_iter().copied().map(ValueRef::from_raw); self.store.state.alloc_exception(tag_addr, payload, roots) } @@ -563,7 +622,7 @@ impl<'store> Executor<'store> { } fn exec_throw_ref(&mut self, instr_ptr: usize) -> ExecResult { - let exception = ValueRef::stack_pop(&mut self.store.value_stack); + let exception = ValueRef::stack_pop(&mut self.value_stack); if exception.is_null() { return Err(Trap::NullReference.into()); } @@ -577,8 +636,13 @@ impl<'store> Executor<'store> { } } - fn matching_catch(&self, protected_ip: usize, tag_addr: TagAddr) -> Option { - let handlers = &self.func.data.exception_handlers; + fn matching_catch( + func: &WasmFunction, + module: &ModuleInstance, + protected_ip: usize, + tag_addr: TagAddr, + ) -> Option { + let handlers = &func.data.exception_handlers; let end = handlers.partition_point(|handler| handler.start_ip as usize <= protected_ip); handlers[..end] .iter() @@ -586,35 +650,27 @@ impl<'store> Executor<'store> { .filter(|handler| protected_ip < handler.end_ip as usize) .flat_map(|handler| handler.catches.iter().copied()) .find(|catch| match catch { - ExceptionCatch::Tag { tag, .. } => self.module.resolve_tag_addr(*tag) == tag_addr, + ExceptionCatch::Tag { tag, .. } => module.resolve_tag_addr(*tag) == tag_addr, ExceptionCatch::All { .. } => true, }) } - #[inline(always)] - fn switch_to_frame(&mut self, frame: CallFrame) { - let previous = core::mem::replace(&mut self.cf, frame); - if previous.func_addr == self.cf.func_addr { - return; - } - - let wasm_func = self.store.state.funcs.wasm(self.cf.func_addr); - if !Shared::ptr_eq(&self.func, &wasm_func.func) { - self.func = wasm_func.func.clone(); - } - if wasm_func.owner != self.module.id() { - self.set_module(wasm_func.owner); - } - } - + /// Unwinds to the innermost handler that catches `exception`. Returns the flow to its landing + /// pad, or `None` if no frame of this run catches it. fn dispatch_exception(&mut self, exception: ValueRef, mut protected_ip: usize) -> Result, Trap> { let object = self.store.state.gc.get(exception).ok_or(Trap::InvalidReference)?; let crate::store::GcObjectKind::Exception(tag_addr) = object.kind else { return Err(Trap::InvalidReference); }; - let mut switched = false; + let entry_func_addr = self.cf.func_addr; + // The function and instance of the frame being searched while it belongs to another instance. + let mut foreign: Option<(Shared, ModuleInstance)> = None; loop { - if let Some(catch) = self.matching_catch(protected_ip, tag_addr) { + let catch = match &foreign { + None => Self::matching_catch(self.func, self.module, protected_ip, tag_addr), + Some((func, module)) => Self::matching_catch(func, module, protected_ip, tag_addr), + }; + if let Some(catch) = catch { let (landing_pad, base, with_ref, include_payload) = match catch { ExceptionCatch::Tag { landing_pad, base, with_ref, .. } => (landing_pad, base, with_ref, true), ExceptionCatch::All { landing_pad, base, with_ref } => (landing_pad, base, with_ref, false), @@ -625,30 +681,47 @@ impl<'store> Executor<'store> { s64: stack_base.s64 + base.c64 as u32, s128: stack_base.s128 + base.c128 as u32, }; - self.store.value_stack.truncate_to_base(target); + self.value_stack.truncate_to_base(target); if include_payload { - let Store { state, value_stack, .. } = self.store; - let object = state.gc.get(exception).ok_or(Trap::InvalidReference)?; + let object = self.store.state.gc.get(exception).ok_or(Trap::InvalidReference)?; for value in object.values.iter().copied() { - value_stack.push_reserved(value); + self.value_stack.push_reserved(value); } } if with_ref { - ValueRef::stack_push(&mut self.store.value_stack, exception); + ValueRef::stack_push(&mut self.value_stack, exception); + } + if foreign.is_some() { + self.cf.instr_ptr = landing_pad as usize; + return Ok(Some(self.leave())); } - return Ok(Some(if switched { - ExecFlow::Switch(landing_pad as usize) + let landing_pad = landing_pad as usize; + return Ok(Some(if self.cf.func_addr == entry_func_addr { + ExecFlow::Next(landing_pad) } else { - ExecFlow::Next(landing_pad as usize) + ExecFlow::Switch(landing_pad) })); } - self.store.value_stack.truncate_to_base(self.cf.locals_base); + self.value_stack.truncate_to_base(self.cf.locals_base); let Some(caller) = self.store.call_stack.pop_frame(self.call_stack_base) else { return Ok(None); }; - switched = true; - self.switch_to_frame(caller); + if caller.func_addr != self.cf.func_addr { + let module = self.module; + match module.local_func(caller.func_addr) { + Some(func) => { + self.func = func; + foreign = None; + } + None => { + let wasm_func = self.store.state.funcs.wasm(caller.func_addr); + let owner = self.store.get_module_instance(wasm_func.owner).expect("invalid module instance"); + foreign = Some((wasm_func.func.clone(), owner.clone())); + } + } + } + self.cf = caller; protected_ip = self.cf.instr_ptr.checked_sub(1).expect("invalid caller IP"); } } @@ -659,8 +732,9 @@ impl<'store> Executor<'store> { type_addr: TypeAddr, return_instr_ptr: usize, ) -> ExecResult { + let module_id = self.module.id(); if let Some(host_func) = host_func.typed_callback() { - cold_err!(host_func.call_stack(self.store, self.module.id(), type_addr)) + cold_err!(self.with_store_value_stack(|store| host_func.call_stack(store, module_id, type_addr))) .map_err(|error| Trap::HostFunction(Box::new(error)))?; if TAIL { return Ok(self.exec_return()); @@ -670,11 +744,10 @@ impl<'store> Executor<'store> { let (param_count, result_count, base) = { let ty = self.store.state.get_canonical_func_type(type_addr); - (ty.params().len(), ty.results().len(), self.store.value_stack.base_before(ty.params().iter().collect())) + (ty.params().len(), ty.results().len(), self.value_stack.base_before(ty.params().iter().collect())) }; - let module_id = self.module.id(); - self.store - .with_scratch_values(param_count + result_count, |store, values| { + self.with_store_value_stack(|store| { + store.with_scratch_values(param_count + result_count, |store, values| { let host_values = store.stack_value_iter(type_addr, crate::store::FuncValueTypes::Params, base)?; for (slot, value) in values[..param_count].iter_mut().zip(host_values) { *slot = value; @@ -686,43 +759,50 @@ impl<'store> Executor<'store> { .map_err(|error| Error::Trap(Trap::HostFunction(Box::new(error))))?; store.push_wasm_values(results) }) - .map_err(|error| match error { - Error::Trap(trap) => trap, - other => Trap::HostFunction(Box::new(other)), - })?; + }) + .map_err(|error| match error { + Error::Trap(trap) => trap, + other => Trap::HostFunction(Box::new(other)), + })?; if TAIL { Ok(self.exec_return()) } else { Ok(ExecFlow::Next(return_instr_ptr)) } } fn exec_call_direct(&mut self, v: u32, return_instr_ptr: usize) -> ExecResult { self.charge_call_fuel(FUEL_COST_CALL_TOTAL); - let addr = self.module.resolve_func_addr(v); - if self.store.state.funcs.is_host(addr) { - let host_func = self.store.state.funcs.host(addr); - let type_addr = host_func.type_addr; - let host_func = host_func.func.clone(); - self.exec_call_host::(host_func, type_addr, return_instr_ptr) - } else { - self.exec_call_wasm::(addr, return_instr_ptr) + let module = self.module; + match module.local_func_by_index(v) { + Some((addr, func)) => self.exec_call_local::(addr, func, return_instr_ptr), + None => self.exec_call_import::(v, return_instr_ptr), } } fn exec_return_call_direct(&mut self, v: u32) -> ExecResult { self.charge_call_fuel(FUEL_COST_CALL_TOTAL); + let module = self.module; + match module.local_func_by_index(v) { + Some((addr, func)) => self.exec_call_local::(addr, func, 0), + None => self.exec_call_import::(v, 0), + } + } + + /// Calls an imported function: a host function or another instance's Wasm function. + #[inline(always)] + fn exec_call_import(&mut self, v: u32, return_instr_ptr: usize) -> ExecResult { let addr = self.module.resolve_func_addr(v); if self.store.state.funcs.is_host(addr) { let host_func = self.store.state.funcs.host(addr); let type_addr = host_func.type_addr; let host_func = host_func.func.clone(); - self.exec_call_host::(host_func, type_addr, 0) + self.exec_call_host::(host_func, type_addr, return_instr_ptr) } else { - self.exec_call_wasm::(addr, 0) + // An instance cannot import its own functions. + self.exec_call_foreign::(addr, return_instr_ptr) } } fn exec_call_self(&mut self, return_instr_ptr: usize) -> ExecResult<()> { self.charge_call_fuel(FUEL_COST_CALL_TOTAL); - let locals_base = - self.store.value_stack.enter_locals(&self.func.params, &self.func.locals, &self.func.max_stack)?; + let locals_base = self.value_stack.enter_locals(&self.func.params, &self.func.locals, &self.func.max_stack)?; let new = CallFrame::new(self.cf.func_addr, locals_base, self.func.locals); self.store.call_stack.push(core::mem::replace(&mut self.cf, new), return_instr_ptr)?; Ok(()) @@ -731,9 +811,8 @@ impl<'store> Executor<'store> { fn exec_return_call_self(&mut self) -> ExecResult<()> { self.charge_call_fuel(FUEL_COST_CALL_TOTAL); - self.store.value_stack.truncate_keep_counts(self.cf.locals_base, self.func.params); - let locals_base = - self.store.value_stack.enter_locals(&self.func.params, &self.func.locals, &self.func.max_stack)?; + self.value_stack.truncate_keep_counts(self.cf.locals_base, self.func.params); + let locals_base = self.value_stack.enter_locals(&self.func.params, &self.func.locals, &self.func.max_stack)?; self.cf = CallFrame::new(self.cf.func_addr, locals_base, self.func.locals); Ok(()) } @@ -787,40 +866,70 @@ impl<'store> Executor<'store> { self.exec_call_wasm::(func_addr, return_instr_ptr) } + /// Calls the Wasm function at store address `func_addr`. #[inline(always)] fn exec_call_wasm( &mut self, func_addr: FuncAddr, return_instr_ptr: usize, ) -> ExecResult { - let wasm_func = self.store.state.funcs.wasm(func_addr); - let (params, locals, max_stack, owner, next_func) = { - let next_func = (!Shared::ptr_eq(&self.func, &wasm_func.func)).then(|| wasm_func.func.clone()); - (wasm_func.func.params, wasm_func.func.locals, wasm_func.func.max_stack, wasm_func.owner, next_func) - }; - if TAIL { - self.store.value_stack.truncate_keep_counts(self.cf.locals_base, params); + let module = self.module; + match module.local_func(func_addr) { + Some(func) => self.exec_call_local::(func_addr, func, return_instr_ptr), + None => self.exec_call_foreign::(func_addr, return_instr_ptr), } - let locals_base = self.store.value_stack.enter_locals(¶ms, &locals, &max_stack)?; - if TAIL { - self.cf = CallFrame::new(func_addr, locals_base, locals); - } else { - self.store.call_stack.push(self.cf, return_instr_ptr)?; - self.cf = CallFrame::new(func_addr, locals_base, locals); + } + + /// Enters `func`, which this run's module instance owns. + #[inline(always)] + fn exec_call_local( + &mut self, + func_addr: FuncAddr, + func: &'module WasmFunction, + return_instr_ptr: usize, + ) -> ExecResult { + self.enter_frame::(func_addr, func, return_instr_ptr)?; + if core::ptr::eq(self.func, func) { + return Ok(ExecFlow::Next(0)); } - let switched = next_func.is_some(); - if let Some(next_func) = next_func { - self.func = next_func; + self.func = func; + Ok(ExecFlow::Switch(0)) + } + + /// Enters a function of another module instance, which ends this run. + #[inline(never)] + fn exec_call_foreign( + &mut self, + func_addr: FuncAddr, + return_instr_ptr: usize, + ) -> ExecResult { + let func = self.store.state.funcs.wasm(func_addr).func.clone(); + self.enter_frame::(func_addr, &func, return_instr_ptr)?; + Ok(self.leave()) + } + + /// Sets up the frame of a call to `func`, replacing the current one for a tail call. + #[inline(always)] + fn enter_frame( + &mut self, + func_addr: FuncAddr, + func: &WasmFunction, + return_instr_ptr: usize, + ) -> ExecResult<()> { + if TAIL { + self.value_stack.truncate_keep_counts(self.cf.locals_base, func.params); } - if owner != self.module.id() { - self.set_module(owner); + let locals_base = self.value_stack.enter_locals(&func.params, &func.locals, &func.max_stack)?; + if !TAIL { + self.store.call_stack.push(self.cf, return_instr_ptr)?; } - Ok(if switched { ExecFlow::Switch(0) } else { ExecFlow::Next(0) }) + self.cf = CallFrame::new(func_addr, locals_base, func.locals); + Ok(()) } fn exec_call_ref(&mut self, type_addr: u32, return_instr_ptr: usize) -> ExecResult { self.charge_call_fuel(FUEL_COST_CALL_TOTAL); - let func_ref = ValueRef::stack_pop(&mut self.store.value_stack); + let func_ref = ValueRef::stack_pop(&mut self.value_stack); let Some(func_addr) = func_ref.addr() else { return cold!(Err(Trap::NullFunctionReference.into())); }; @@ -829,7 +938,7 @@ impl<'store> Executor<'store> { } fn exec_return(&mut self) -> ExecFlow { - self.store.value_stack.truncate_keep_counts(self.cf.locals_base, self.func.results); + self.value_stack.truncate_keep_counts(self.cf.locals_base, self.func.results); self.finish_return() } @@ -839,38 +948,44 @@ impl<'store> Executor<'store> { return ExecFlow::Complete; }; let instr_ptr = caller.instr_ptr; - if caller.func_addr == self.cf.func_addr { - self.cf = caller; - ExecFlow::Next(instr_ptr) - } else { - self.switch_to_frame(caller); - ExecFlow::Switch(instr_ptr) + let switched = caller.func_addr != self.cf.func_addr; + self.cf = caller; + if !switched { + return ExecFlow::Next(instr_ptr); + } + let module = self.module; + match module.local_func(caller.func_addr) { + Some(func) => { + self.func = func; + ExecFlow::Switch(instr_ptr) + } + None => self.leave(), } } fn exec_return_void(&mut self) -> ExecFlow { - self.store.value_stack.truncate_to_base(self.cf.locals_base); + self.value_stack.truncate_to_base(self.cf.locals_base); self.finish_return() } fn exec_return_32(&mut self) -> ExecFlow { - self.store.value_stack.stack_32.truncate_to_one_tail(self.cf.locals_base.s32 as usize); - self.store.value_stack.stack_64.truncate_to(self.cf.locals_base.s64 as usize); - self.store.value_stack.stack_128.truncate_to(self.cf.locals_base.s128 as usize); + self.value_stack.stack_32.truncate_to_one_tail(self.cf.locals_base.s32 as usize); + self.value_stack.stack_64.truncate_to(self.cf.locals_base.s64 as usize); + self.value_stack.stack_128.truncate_to(self.cf.locals_base.s128 as usize); self.finish_return() } fn exec_return_64(&mut self) -> ExecFlow { - self.store.value_stack.stack_32.truncate_to(self.cf.locals_base.s32 as usize); - self.store.value_stack.stack_64.truncate_to_one_tail(self.cf.locals_base.s64 as usize); - self.store.value_stack.stack_128.truncate_to(self.cf.locals_base.s128 as usize); + self.value_stack.stack_32.truncate_to(self.cf.locals_base.s32 as usize); + self.value_stack.stack_64.truncate_to_one_tail(self.cf.locals_base.s64 as usize); + self.value_stack.stack_128.truncate_to(self.cf.locals_base.s128 as usize); self.finish_return() } fn exec_return_128(&mut self) -> ExecFlow { - self.store.value_stack.stack_32.truncate_to(self.cf.locals_base.s32 as usize); - self.store.value_stack.stack_64.truncate_to(self.cf.locals_base.s64 as usize); - self.store.value_stack.stack_128.truncate_to_one_tail(self.cf.locals_base.s128 as usize); + self.value_stack.stack_32.truncate_to(self.cf.locals_base.s32 as usize); + self.value_stack.stack_64.truncate_to(self.cf.locals_base.s64 as usize); + self.value_stack.stack_128.truncate_to_one_tail(self.cf.locals_base.s128 as usize); self.finish_return() } @@ -882,18 +997,18 @@ impl<'store> Executor<'store> { value_local: u8, ) -> Result<(), Trap> { let memarg = index.resolve(&self.func.data); - let value = T::local_get(&self.store.value_stack, &self.cf, u16::from(value_local)); + let value = T::local_get(&self.value_stack, &self.cf, u16::from(value_local)); let mem_addr = self.mem_addr(MemAddr::from(memarg.memory())); crate::store::with_memory!(self.store.state, mem_addr, |mem, kind| { let base = if kind.arch() == MemoryArch::I64 { - let base = u64::local_get(&self.store.value_stack, &self.cf, u16::from(addr_local)); + let base = u64::local_get(&self.value_stack, &self.cf, u16::from(addr_local)); cold_err!(usize::try_from(base).map_err(|_| Trap::MemoryOutOfBounds { offset: usize::MAX, len: N, max: mem.len(), }))? } else { - u32::local_get(&self.store.value_stack, &self.cf, u16::from(addr_local)) as usize + u32::local_get(&self.value_stack, &self.cf, u16::from(addr_local)) as usize }; let addr = cold_err!(mem.effective_addr::(base, u64::from(memarg.offset())))?; value.store_at(&mut *mem, addr) @@ -911,14 +1026,14 @@ impl<'store> Executor<'store> { let mem_addr = self.mem_addr(MemAddr::from(memarg.memory())); crate::store::with_memory!(self.store.state, mem_addr, |mem, kind| { let base = if kind.arch() == MemoryArch::I64 { - let base = i64::local_get(&self.store.value_stack, &self.cf, u16::from(addr_local)) as u64; + let base = i64::local_get(&self.value_stack, &self.cf, u16::from(addr_local)) as u64; cold_err!(usize::try_from(base).map_err(|_| Trap::MemoryOutOfBounds { offset: usize::MAX, len: N, max: mem.len(), }))? } else { - u32::local_get(&self.store.value_stack, &self.cf, u16::from(addr_local)) as usize + u32::local_get(&self.value_stack, &self.cf, u16::from(addr_local)) as usize }; let addr = cold_err!(mem.effective_addr::(base, u64::from(memarg.offset())))?; @@ -935,13 +1050,13 @@ impl<'store> Executor<'store> { &mut self, m: CompactMemoryArg, ) -> Result<(), Trap> { - let rhs = T::stack_pop(&mut self.store.value_stack); - let lhs = T::stack_pop(&mut self.store.value_stack); - let acc = T::stack_pop(&mut self.store.value_stack); + let rhs = T::stack_pop(&mut self.value_stack); + let lhs = T::stack_pop(&mut self.value_stack); + let acc = T::stack_pop(&mut self.value_stack); let fma = acc + lhs * rhs; let mem_addr = self.mem_addr(m.mem_addr()); crate::store::with_memory!(self.store.state, mem_addr, |mem, kind| { - let base = self.store.value_stack.pop_memory_operand(kind.arch())?; + let base = self.value_stack.pop_memory_operand(kind.arch())?; let addr = cold_err!(mem.effective_addr::(base, m.offset()))?; cold_err!(fma.store_at(&mut *mem, addr)) }) @@ -966,37 +1081,37 @@ impl<'store> Executor<'store> { let mem_addr = self.mem_addr(MemAddr::from(memarg.memory())); crate::store::with_memory!(self.store.state, mem_addr, |mem, kind| { let base = if kind.arch() == MemoryArch::I64 { - let base = i64::local_get(&self.store.value_stack, &self.cf, u16::from(addr_local)) as u64; + let base = i64::local_get(&self.value_stack, &self.cf, u16::from(addr_local)) as u64; cold_err!(usize::try_from(base)).map_err(|_| Trap::MemoryOutOfBounds { offset: usize::MAX, len: N, max: mem.len(), })? } else { - u32::local_get(&self.store.value_stack, &self.cf, u16::from(addr_local)) as usize + u32::local_get(&self.value_stack, &self.cf, u16::from(addr_local)) as usize }; let addr = cold_err!(mem.effective_addr::(base, u64::from(memarg.offset())))?; let value = cast(cold_err!(LOAD::load_at(&*mem, addr))?); if SET_LOCAL { - TARGET::local_set(&mut self.store.value_stack, &self.cf, u16::from(dst_local), value); + TARGET::local_set(&mut self.value_stack, &self.cf, u16::from(dst_local), value); } if !SET_LOCAL || TEE { - TARGET::stack_push(&mut self.store.value_stack, value); + TARGET::stack_push(&mut self.value_stack, value); } Ok(()) }) } fn exec_ref_is_null(&mut self) -> Result<(), Trap> { - let is_null = i32::from(ValueRef::stack_pop(&mut self.store.value_stack).is_null()); - i32::stack_push(&mut self.store.value_stack, is_null); + let is_null = i32::from(ValueRef::stack_pop(&mut self.value_stack).is_null()); + i32::stack_push(&mut self.value_stack, is_null); Ok(()) } fn exec_ref_as_non_null(&mut self) -> Result<(), Trap> { - if ValueRef::stack_peek(&self.store.value_stack).is_null() { + if ValueRef::stack_peek(&self.value_stack).is_null() { return cold!(Err(Trap::NullReference)); } Ok(()) @@ -1008,14 +1123,14 @@ impl<'store> Executor<'store> { } fn exec_ref_matches(&self, ty: RefType) -> bool { - let value = ValueRef::stack_peek(&self.store.value_stack); + let value = ValueRef::stack_peek(&self.value_stack); self.store.state.value_ref_matches(value, self.canonical_ref_type(ty)) } fn exec_ref_test(&mut self, ty: RefType) -> Result<(), Trap> { - let value = ValueRef::stack_pop(&mut self.store.value_stack); + let value = ValueRef::stack_pop(&mut self.value_stack); let matches = self.store.state.value_ref_matches(value, self.canonical_ref_type(ty)); - i32::stack_push(&mut self.store.value_stack, i32::from(matches)); + i32::stack_push(&mut self.value_stack, i32::from(matches)); Ok(()) } @@ -1027,7 +1142,7 @@ impl<'store> Executor<'store> { } fn exec_i31_get(&mut self, signed: bool) -> Result<(), Trap> { - let value = ValueRef::stack_pop(&mut self.store.value_stack); + let value = ValueRef::stack_pop(&mut self.value_stack); if value.is_null() { return cold!(Err(Trap::NullI31Reference)); } @@ -1036,14 +1151,14 @@ impl<'store> Executor<'store> { } else { value.i31_u().expect("validated i31.get operand") as i32 }; - i32::stack_push(&mut self.store.value_stack, value); + i32::stack_push(&mut self.value_stack, value); Ok(()) } fn push_gc_object(&mut self, type_addr: TypeAddr, values: Vec) -> Result<(), Trap> { - let roots = (&self.store.value_stack.stack_32).into_iter().copied().map(ValueRef::from_raw); + let roots = (&self.value_stack.stack_32).into_iter().copied().map(ValueRef::from_raw); let reference = self.store.state.alloc_gc_object(type_addr, values, roots)?; - ValueRef::stack_push(&mut self.store.value_stack, reference); + ValueRef::stack_push(&mut self.value_stack, reference); Ok(()) } @@ -1068,7 +1183,7 @@ impl<'store> Executor<'store> { } else { for index in (0..field_count).rev() { let storage = self.store.state.get_type(type_addr).as_struct().unwrap().fields[index].storage; - values.push(pop_value(&mut self.store.value_stack, storage)); + values.push(pop_value(&mut self.value_stack, storage)); } values.reverse(); } @@ -1079,7 +1194,7 @@ impl<'store> Executor<'store> { let operand = index.resolve(&self.func.data); let type_index = operand.a(); let field_index = operand.b(); - let reference = ValueRef::stack_pop(&mut self.store.value_stack); + let reference = ValueRef::stack_pop(&mut self.value_stack); let type_addr = self.module.resolve_type_addr(type_index); let storage = self.store.state.get_type(type_addr).as_struct().expect("validated struct.get type").fields [field_index as usize] @@ -1087,7 +1202,7 @@ impl<'store> Executor<'store> { let object = self.store.state.gc_object(reference, type_addr)?; let object = self.store.state.gc.get_handle(object).ok_or(Trap::Other("invalid GC reference"))?; let value = *object.values.get(field_index as usize).expect("validated struct field index"); - push_value(&mut self.store.value_stack, value, storage, signed); + push_value(&mut self.value_stack, value, storage, signed); Ok(()) } @@ -1099,8 +1214,8 @@ impl<'store> Executor<'store> { let storage = self.store.state.get_type(type_addr).as_struct().expect("validated struct.set type").fields [field_index as usize] .storage; - let value = pop_value(&mut self.store.value_stack, storage); - let reference = ValueRef::stack_pop(&mut self.store.value_stack); + let value = pop_value(&mut self.value_stack, storage); + let reference = ValueRef::stack_pop(&mut self.value_stack); let object = self.store.state.gc_object(reference, type_addr)?; self.store.state.gc.set(object, field_index as usize, value).expect("live struct field"); Ok(()) @@ -1109,9 +1224,9 @@ impl<'store> Executor<'store> { fn exec_array_new(&mut self, type_index: TypeAddr, default: bool) -> Result<(), Trap> { let type_addr = self.module.resolve_type_addr(type_index); let storage = self.store.state.get_type(type_addr).as_array().expect("validated array.new type").field.storage; - let len = u32::stack_pop(&mut self.store.value_stack) as usize; + let len = u32::stack_pop(&mut self.value_stack) as usize; self.store.state.gc.check_allocation(len, self.store.state.gc_type_has_references(type_addr))?; - let value = if default { default_value(storage) } else { pop_value(&mut self.store.value_stack, storage) }; + let value = if default { default_value(storage) } else { pop_value(&mut self.value_stack, storage) }; let mut values = Vec::new(); cold_err!(values.try_reserve_exact(len)).map_err(|_| Trap::OutOfMemory)?; values.resize(len, value); @@ -1129,36 +1244,36 @@ impl<'store> Executor<'store> { let mut values = Vec::new(); cold_err!(values.try_reserve_exact(len)).map_err(|_| Trap::OutOfMemory)?; for _ in 0..len { - values.push(pop_value(&mut self.store.value_stack, arr_type.field.storage)); + values.push(pop_value(&mut self.value_stack, arr_type.field.storage)); } values.reverse(); self.push_gc_object(type_addr, values) } fn exec_array_get(&mut self, type_index: TypeAddr, signed: Option) -> Result<(), Trap> { - let index = u32::stack_pop(&mut self.store.value_stack) as usize; - let reference = ValueRef::stack_pop(&mut self.store.value_stack); + let index = u32::stack_pop(&mut self.value_stack) as usize; + let reference = ValueRef::stack_pop(&mut self.value_stack); let type_addr = self.module.resolve_type_addr(type_index); let storage = self.store.state.get_type(type_addr).as_array().expect("validated array.get type").field.storage; let object = self.store.state.gc_object(reference, type_addr)?; let object = self.store.state.gc.get_handle(object).ok_or(Trap::Other("invalid GC reference"))?; let value = *object.values.get(index).ok_or(Trap::ArrayOutOfBounds)?; - push_value(&mut self.store.value_stack, value, storage, signed); + push_value(&mut self.value_stack, value, storage, signed); Ok(()) } fn exec_array_set(&mut self, type_index: TypeAddr) -> Result<(), Trap> { let type_addr = self.module.resolve_type_addr(type_index); let storage = self.store.state.get_type(type_addr).as_array().expect("validated array.set type").field.storage; - let value = pop_value(&mut self.store.value_stack, storage); - let index = u32::stack_pop(&mut self.store.value_stack) as usize; - let reference = ValueRef::stack_pop(&mut self.store.value_stack); + let value = pop_value(&mut self.value_stack, storage); + let index = u32::stack_pop(&mut self.value_stack) as usize; + let reference = ValueRef::stack_pop(&mut self.value_stack); let object = self.store.state.gc_object(reference, type_addr)?; self.store.state.gc.set(object, index, value).ok_or(Trap::ArrayOutOfBounds) } fn exec_array_len(&mut self) -> Result<(), Trap> { - let reference = ValueRef::stack_pop(&mut self.store.value_stack); + let reference = ValueRef::stack_pop(&mut self.value_stack); if reference.is_null() { return Err(Trap::NullArrayReference); } @@ -1169,17 +1284,17 @@ impl<'store> Executor<'store> { if self.store.state.get_type(type_addr).as_array().is_none() { return Err(Trap::Other("GC reference is not an array")); } - i32::stack_push(&mut self.store.value_stack, object.values.len() as i32); + i32::stack_push(&mut self.value_stack, object.values.len() as i32); Ok(()) } fn exec_array_fill(&mut self, type_index: TypeAddr) -> Result<(), Trap> { let type_addr = self.module.resolve_type_addr(type_index); let storage = self.store.state.get_type(type_addr).as_array().expect("validated array.fill type").field.storage; - let len = u32::stack_pop(&mut self.store.value_stack) as usize; - let value = pop_value(&mut self.store.value_stack, storage); - let index = u32::stack_pop(&mut self.store.value_stack) as usize; - let reference = ValueRef::stack_pop(&mut self.store.value_stack); + let len = u32::stack_pop(&mut self.value_stack) as usize; + let value = pop_value(&mut self.value_stack, storage); + let index = u32::stack_pop(&mut self.value_stack) as usize; + let reference = ValueRef::stack_pop(&mut self.value_stack); let object = self.store.state.gc_object(reference, type_addr)?; let object_ref = self.store.state.gc.get_handle(object).ok_or(Trap::Other("invalid GC reference"))?; let end = index.checked_add(len).filter(|end| *end <= object_ref.values.len()).ok_or(Trap::ArrayOutOfBounds)?; @@ -1191,11 +1306,11 @@ impl<'store> Executor<'store> { let operand = index.resolve(&self.func.data); let dst_type = operand.a(); let src_type = operand.b(); - let len = u32::stack_pop(&mut self.store.value_stack) as usize; - let src_index = u32::stack_pop(&mut self.store.value_stack) as usize; - let src = ValueRef::stack_pop(&mut self.store.value_stack); - let dst_index = u32::stack_pop(&mut self.store.value_stack) as usize; - let dst = ValueRef::stack_pop(&mut self.store.value_stack); + let len = u32::stack_pop(&mut self.value_stack) as usize; + let src_index = u32::stack_pop(&mut self.value_stack) as usize; + let src = ValueRef::stack_pop(&mut self.value_stack); + let dst_index = u32::stack_pop(&mut self.value_stack) as usize; + let dst = ValueRef::stack_pop(&mut self.value_stack); let dst_type = self.module.resolve_type_addr(dst_type); let src_type = self.module.resolve_type_addr(src_type); let dst_handle = self.store.state.gc_object(dst, dst_type)?; @@ -1221,8 +1336,8 @@ impl<'store> Executor<'store> { let operand = index.resolve(&self.func.data); let type_index = operand.a(); let data_index = operand.b(); - let len = u32::stack_pop(&mut self.store.value_stack) as usize; - let src = u32::stack_pop(&mut self.store.value_stack) as usize; + let len = u32::stack_pop(&mut self.value_stack) as usize; + let src = u32::stack_pop(&mut self.value_stack) as usize; let type_addr = self.module.resolve_type_addr(type_index); let storage = self.store.state.get_type(type_addr).as_array().expect("validated array type").field.storage; let data_addr = self.module.resolve_data_addr(data_index); @@ -1237,8 +1352,8 @@ impl<'store> Executor<'store> { let operand = index.resolve(&self.func.data); let type_index = operand.a(); let elem_index = operand.b(); - let len = u32::stack_pop(&mut self.store.value_stack) as usize; - let src = u32::stack_pop(&mut self.store.value_stack) as usize; + let len = u32::stack_pop(&mut self.value_stack) as usize; + let src = u32::stack_pop(&mut self.value_stack) as usize; let type_addr = self.module.resolve_type_addr(type_index); let elem_addr = self.module.resolve_elem_addr(elem_index); let items = self.store.state.elements[elem_addr as usize].items_range(src, len)?; @@ -1253,10 +1368,10 @@ impl<'store> Executor<'store> { let operand = index.resolve(&self.func.data); let type_index = operand.a(); let data_index = operand.b(); - let len = u32::stack_pop(&mut self.store.value_stack) as usize; - let src = u32::stack_pop(&mut self.store.value_stack) as usize; - let dst = u32::stack_pop(&mut self.store.value_stack) as usize; - let reference = ValueRef::stack_pop(&mut self.store.value_stack); + let len = u32::stack_pop(&mut self.value_stack) as usize; + let src = u32::stack_pop(&mut self.value_stack) as usize; + let dst = u32::stack_pop(&mut self.value_stack) as usize; + let reference = ValueRef::stack_pop(&mut self.value_stack); let type_addr = self.module.resolve_type_addr(type_index); let storage = self.store.state.get_type(type_addr).as_array().expect("validated array type").field.storage; let object = self.store.state.gc_object(reference, type_addr)?; @@ -1273,10 +1388,10 @@ impl<'store> Executor<'store> { let operand = index.resolve(&self.func.data); let type_index = operand.a(); let elem_index = operand.b(); - let len = u32::stack_pop(&mut self.store.value_stack) as usize; - let src = u32::stack_pop(&mut self.store.value_stack) as usize; - let dst = u32::stack_pop(&mut self.store.value_stack) as usize; - let reference = ValueRef::stack_pop(&mut self.store.value_stack); + let len = u32::stack_pop(&mut self.value_stack) as usize; + let src = u32::stack_pop(&mut self.value_stack) as usize; + let dst = u32::stack_pop(&mut self.value_stack) as usize; + let reference = ValueRef::stack_pop(&mut self.value_stack); let type_addr = self.module.resolve_type_addr(type_index); let object = self.store.state.gc_object(reference, type_addr)?; let object_len = self.store.state.gc.get_handle(object).expect("validated array").values.len(); @@ -1292,8 +1407,8 @@ impl<'store> Executor<'store> { fn exec_memory_size(&mut self, addr: u32) -> Result<(), Trap> { match self.store.state.memory_size(self.mem_addr(addr)) { - (MemoryArch::I64, pages) => i64::stack_push(&mut self.store.value_stack, pages as i64), - (MemoryArch::I32, pages) => i32::stack_push(&mut self.store.value_stack, pages as i32), + (MemoryArch::I64, pages) => i64::stack_push(&mut self.value_stack, pages as i64), + (MemoryArch::I32, pages) => i32::stack_push(&mut self.value_stack, pages as i32), } Ok(()) } @@ -1303,14 +1418,14 @@ impl<'store> Executor<'store> { let limiter = self.store.engine.config().resource_limiter.as_deref(); let is_64bit = self.store.state.memory_type(mem_addr).arch() == MemoryArch::I64; let pages_delta = match is_64bit { - true => i64::stack_pop(&mut self.store.value_stack), - false => i64::from(i32::stack_pop(&mut self.store.value_stack)), + true => i64::stack_pop(&mut self.value_stack), + false => i64::from(i32::stack_pop(&mut self.value_stack)), }; let size = self.store.state.grow_mem(mem_addr, pages_delta, limiter)?.unwrap_or(-1); match is_64bit { - true => i64::stack_push(&mut self.store.value_stack, size), - false => i32::stack_push(&mut self.store.value_stack, size as i32), + true => i64::stack_push(&mut self.value_stack, size), + false => i32::stack_push(&mut self.value_stack, size as i32), }; Ok(()) @@ -1326,9 +1441,9 @@ impl<'store> Executor<'store> { let src_arch = self.store.state.memory_type(src_mem_addr).arch(); let len_arch = if dst_arch == MemoryArch::I32 || src_arch == MemoryArch::I32 { MemoryArch::I32 } else { MemoryArch::I64 }; - let size = self.store.value_stack.pop_memory_operand(len_arch)?; - let src = self.store.value_stack.pop_memory_operand(src_arch)?; - let dst = self.store.value_stack.pop_memory_operand(dst_arch)?; + let size = self.value_stack.pop_memory_operand(len_arch)?; + let src = self.value_stack.pop_memory_operand(src_arch)?; + let dst = self.value_stack.pop_memory_operand(dst_arch)?; self.store.state.copy_memories(dst_mem_addr, dst, src_mem_addr, src, size) } @@ -1336,9 +1451,9 @@ impl<'store> Executor<'store> { fn exec_memory_fill(&mut self, addr: u32) -> Result<(), Trap> { let mem_addr = self.mem_addr(addr); let arch = self.store.state.memory_type(mem_addr).arch(); - let size = self.store.value_stack.pop_memory_operand(arch)?; - let val = i32::stack_pop(&mut self.store.value_stack); - let dst = self.store.value_stack.pop_memory_operand(arch)?; + let size = self.value_stack.pop_memory_operand(arch)?; + let val = i32::stack_pop(&mut self.value_stack); + let dst = self.value_stack.pop_memory_operand(arch)?; self.exec_memory_fill_impl(mem_addr, dst, val as u8, size) } @@ -1346,7 +1461,7 @@ impl<'store> Executor<'store> { let operand = index.resolve(&self.func.data); let mem_addr = self.mem_addr(operand.memory()); let arch = self.store.state.memory_type(mem_addr).arch(); - let dst = self.store.value_stack.pop_memory_operand(arch)?; + let dst = self.value_stack.pop_memory_operand(arch)?; self.exec_memory_fill_impl(mem_addr, dst, operand.byte(), operand.value() as u32 as usize) } @@ -1364,11 +1479,11 @@ impl<'store> Executor<'store> { let operand = index.resolve(&self.func.data); let data_index = operand.a(); let mem_index = operand.b(); - let size = u32::stack_pop(&mut self.store.value_stack) as usize; - let offset = u32::stack_pop(&mut self.store.value_stack) as usize; + let size = u32::stack_pop(&mut self.value_stack) as usize; + let offset = u32::stack_pop(&mut self.value_stack) as usize; let mem_addr = self.mem_addr(mem_index); let arch = self.store.state.memory_type(mem_addr).arch(); - let dst = self.store.value_stack.pop_memory_operand(arch)?; + let dst = self.value_stack.pop_memory_operand(arch)?; let data = self.store.state.data[self.module.resolve_data_addr(data_index) as usize].data.clone(); // Dropped segments behave like empty segments, including valid zero-length copies. @@ -1419,11 +1534,11 @@ impl<'store> Executor<'store> { let m = arg.memory_arg_idx.resolve(&self.func.data); let mem_addr = self.mem_addr(m.memory()); crate::store::with_memory!(self.store.state, mem_addr, |mem, kind| { - let base = self.store.value_stack.pop_memory_operand(kind.arch())?; + let base = self.value_stack.pop_memory_operand(kind.arch())?; let addr = cold_err!(mem.effective_addr::(base, m.offset()))?; let val = cold_err!(LOAD::load_at(&*mem, addr))?; let offset = arg.lane as usize * LOAD_SIZE; - Value128::stack_update(&mut self.store.value_stack, |value| { + Value128::stack_update(&mut self.value_stack, |value| { let mut bytes = value.to_mem_bytes(); bytes[offset..offset + LOAD_SIZE].copy_from_slice(&val.to_mem_bytes()); Value128(bytes) @@ -1440,10 +1555,10 @@ impl<'store> Executor<'store> { ) -> Result<(), Trap> { let mem_addr = self.mem_addr(m.memory()); crate::store::with_memory!(self.store.state, mem_addr, |mem, kind| { - let base = self.store.value_stack.pop_memory_operand(kind.arch())?; + let base = self.value_stack.pop_memory_operand(kind.arch())?; let addr = cold_err!(mem.effective_addr::(base, m.offset()))?; let value = cold_err!(LOAD::load_at(&*mem, addr))?; - TARGET::stack_push(&mut self.store.value_stack, cast(value)); + TARGET::stack_push(&mut self.value_stack, cast(value)); Ok(()) }) } @@ -1458,16 +1573,16 @@ impl<'store> Executor<'store> { } fn exec_atomic_wait(&mut self, arg: Operand128Idx, op: AtomicWaitOp) -> Result<(), Trap> { - let timeout = if op == AtomicWaitOp::Notify { 0 } else { i64::stack_pop(&mut self.store.value_stack) }; + let timeout = if op == AtomicWaitOp::Notify { 0 } else { i64::stack_pop(&mut self.value_stack) }; let value = match op { - AtomicWaitOp::Wait64 => u64::stack_pop(&mut self.store.value_stack), - _ => u32::stack_pop(&mut self.store.value_stack) as u64, + AtomicWaitOp::Wait64 => u64::stack_pop(&mut self.value_stack), + _ => u32::stack_pop(&mut self.value_stack) as u64, }; let memory = arg.resolve(&self.func.data); let mem_addr = self.mem_addr(memory.memory()); let width = if op == AtomicWaitOp::Wait64 { 8 } else { 4 }; let addr = crate::store::with_memory!(@lock_inline self.store.state, mem_addr, |mem, kind| { - let base = self.store.value_stack.pop_memory_operand(kind.arch())?; + let base = self.value_stack.pop_memory_operand(kind.arch())?; let addr = cold_err!(mem.effective_addr::<1>(base, memory.offset()))?; if addr % width != 0 { return cold!(Err(Trap::UnalignedAtomic)); @@ -1496,11 +1611,11 @@ impl<'store> Executor<'store> { AtomicWaitOp::Wait32 => shared.wait::<4>(addr, value, timeout)?, AtomicWaitOp::Wait64 => shared.wait::<8>(addr, value, timeout)?, }; - u32::stack_push(&mut self.store.value_stack, result); + u32::stack_push(&mut self.value_stack, result); return Ok(()); } - u32::stack_push(&mut self.store.value_stack, 0); + u32::stack_push(&mut self.value_stack, 0); Ok(()) } @@ -1508,20 +1623,20 @@ impl<'store> Executor<'store> { let op = arg.op(); let value = match op { AtomicOp::Load => 0, - _ if arg.is_64() => u64::stack_pop(&mut self.store.value_stack), - _ => u32::stack_pop(&mut self.store.value_stack) as u64, + _ if arg.is_64() => u64::stack_pop(&mut self.value_stack), + _ => u32::stack_pop(&mut self.value_stack) as u64, }; let expected = match op { - AtomicOp::Cmpxchg if arg.is_64() => u64::stack_pop(&mut self.store.value_stack), - AtomicOp::Cmpxchg => u32::stack_pop(&mut self.store.value_stack) as u64, + AtomicOp::Cmpxchg if arg.is_64() => u64::stack_pop(&mut self.value_stack), + AtomicOp::Cmpxchg => u32::stack_pop(&mut self.value_stack) as u64, _ => 0, }; let memory = arg.memory.resolve(&self.func.data); let mem_addr = self.mem_addr(memory.memory()); crate::store::with_memory!(@lock_inline self.store.state, mem_addr, |mem, kind| { - let base = self.store.value_stack.pop_memory_operand(kind.arch())?; + let base = self.value_stack.pop_memory_operand(kind.arch())?; let addr = cold_err!(mem.effective_addr::(base, memory.offset()))?; if addr % N != 0 { return cold!(Err(Trap::UnalignedAtomic)); @@ -1554,8 +1669,8 @@ impl<'store> Executor<'store> { } match arg.is_64() { - true => u64::stack_push(&mut self.store.value_stack, old), - false => u32::stack_push(&mut self.store.value_stack, old as u32), + true => u64::stack_push(&mut self.value_stack, old), + false => u32::stack_push(&mut self.value_stack, old as u32), } Ok(()) }) @@ -1563,7 +1678,7 @@ impl<'store> Executor<'store> { #[inline(always)] fn exec_mem_store_lane + Copy, const N: usize>(&mut self, arg: MemoryLaneArg) -> Result<(), Trap> { - let bytes = Value128::stack_pop(&mut self.store.value_stack).to_mem_bytes(); + let bytes = Value128::stack_pop(&mut self.value_stack).to_mem_bytes(); let lane_offset = arg.lane as usize * N; let mut val_bytes = [0u8; N]; val_bytes.copy_from_slice(&bytes[lane_offset..lane_offset + N]); @@ -1571,7 +1686,7 @@ impl<'store> Executor<'store> { let m = arg.memory_arg_idx.resolve(&self.func.data); let mem_addr = self.mem_addr(m.memory()); crate::store::with_memory!(self.store.state, mem_addr, |mem, kind| { - let base = self.store.value_stack.pop_memory_operand(kind.arch())?; + let base = self.value_stack.pop_memory_operand(kind.arch())?; let addr = cold_err!(mem.effective_addr::(base, m.offset()))?; cold_err!(val.store_at(&mut *mem, addr)) }) @@ -1583,7 +1698,7 @@ impl<'store> Executor<'store> { memory: Operand128, cast: impl Fn(T) -> U, ) -> Result<(), Trap> { - let val = cast(::stack_pop(&mut self.store.value_stack)); + let val = cast(::stack_pop(&mut self.value_stack)); self.exec_mem_store_value(self.mem_addr(memory.memory()), memory.offset(), val) } @@ -1592,9 +1707,9 @@ impl<'store> Executor<'store> { &mut self, memory: Operand128, ) -> Result<(), Trap> { - let condition = Value32::stack_pop(&mut self.store.value_stack); - let false_value = T::stack_pop(&mut self.store.value_stack); - let true_value = T::stack_pop(&mut self.store.value_stack); + let condition = Value32::stack_pop(&mut self.value_stack); + let false_value = T::stack_pop(&mut self.value_stack); + let true_value = T::stack_pop(&mut self.value_stack); let value = if condition == 0 { false_value } else { true_value }; self.exec_mem_store_value(self.mem_addr(memory.memory()), memory.offset(), value) } @@ -1607,7 +1722,7 @@ impl<'store> Executor<'store> { val: U, ) -> Result<(), Trap> { crate::store::with_memory!(self.store.state, memory_addr, |mem, kind| { - let base = self.store.value_stack.pop_memory_operand(kind.arch())?; + let base = self.value_stack.pop_memory_operand(kind.arch())?; let addr = cold_err!(mem.effective_addr::(base, offset))?; cold_err!(val.store_at(&mut *mem, addr)) }) @@ -1617,12 +1732,12 @@ impl<'store> Executor<'store> { let table_addr = self.module.resolve_table_addr(table_index); let idx = self.pop_table_operand(self.store.state.get_table(table_addr).kind.arch())?; let value = *self.store.state.get_table(table_addr).get(idx)?; - ValueRef::stack_push(&mut self.store.value_stack, value); + ValueRef::stack_push(&mut self.value_stack, value); Ok(()) } fn exec_table_set(&mut self, table_index: u32) -> Result<(), Trap> { - let val = ValueRef::stack_pop(&mut self.store.value_stack); + let val = ValueRef::stack_pop(&mut self.value_stack); let table_addr = self.module.resolve_table_addr(table_index); let idx = self.pop_table_operand(self.store.state.get_table(table_addr).kind.arch())?; let table = self.store.state.get_table_mut(table_addr); @@ -1632,8 +1747,8 @@ impl<'store> Executor<'store> { fn exec_table_size(&mut self, table_index: u32) -> Result<(), Trap> { let table = self.store.state.get_table(self.module.resolve_table_addr(table_index)); match table.kind.arch() { - MemoryArch::I32 => i32::stack_push(&mut self.store.value_stack, table.size() as i32), - MemoryArch::I64 => i64::stack_push(&mut self.store.value_stack, table.size() as i64), + MemoryArch::I32 => i32::stack_push(&mut self.value_stack, table.size() as i32), + MemoryArch::I64 => i64::stack_push(&mut self.value_stack, table.size() as i64), } Ok(()) } @@ -1656,16 +1771,16 @@ impl<'store> Executor<'store> { let table_addr = self.module.resolve_table_addr(table_index); let arch = self.store.state.get_table(table_addr).kind.arch(); let n = self.pop_table_operand(arch)?; - let val = ValueRef::stack_pop(&mut self.store.value_stack); + let val = ValueRef::stack_pop(&mut self.value_stack); let limiter = self.store.engine.config().resource_limiter.as_deref(); let table = self.store.state.get_table_mut(table_addr); let sz = table.size(); let grew = table.grow(n, val, limiter)?; match (arch, grew) { - (MemoryArch::I32, true) => i32::stack_push(&mut self.store.value_stack, sz as i32), - (MemoryArch::I32, false) => i32::stack_push(&mut self.store.value_stack, -1), - (MemoryArch::I64, true) => i64::stack_push(&mut self.store.value_stack, sz as i64), - (MemoryArch::I64, false) => i64::stack_push(&mut self.store.value_stack, -1), + (MemoryArch::I32, true) => i32::stack_push(&mut self.value_stack, sz as i32), + (MemoryArch::I32, false) => i32::stack_push(&mut self.value_stack, -1), + (MemoryArch::I64, true) => i64::stack_push(&mut self.value_stack, sz as i64), + (MemoryArch::I64, false) => i64::stack_push(&mut self.value_stack, -1), } Ok(()) } @@ -1674,15 +1789,15 @@ impl<'store> Executor<'store> { let table_addr = self.module.resolve_table_addr(table_index); let arch = self.store.state.get_table(table_addr).kind.arch(); let n = self.pop_table_operand(arch)?; - let val = ValueRef::stack_pop(&mut self.store.value_stack); + let val = ValueRef::stack_pop(&mut self.value_stack); let i = self.pop_table_operand(arch)?; self.store.state.get_table_mut(table_addr).fill(i, n, val) } fn pop_table_operand(&mut self, arch: MemoryArch) -> Result { let value = match arch { - MemoryArch::I32 => i32::stack_pop(&mut self.store.value_stack) as u32 as u64, - MemoryArch::I64 => i64::stack_pop(&mut self.store.value_stack) as u64, + MemoryArch::I32 => i32::stack_pop(&mut self.value_stack) as u32 as u64, + MemoryArch::I64 => i64::stack_pop(&mut self.value_stack) as u64, }; cold_err!(usize::try_from(value).map_err(|_| Trap::TableOutOfBounds { offset: usize::MAX, @@ -1691,3 +1806,9 @@ impl<'store> Executor<'store> { })) } } + +impl Drop for Executor<'_, '_> { + fn drop(&mut self) { + self.store.value_stack = core::mem::take(&mut self.value_stack); + } +} diff --git a/crates/tinywasm/src/interpreter/mod.rs b/crates/tinywasm/src/interpreter/mod.rs index 0f7ae20d..3400a94e 100644 --- a/crates/tinywasm/src/interpreter/mod.rs +++ b/crates/tinywasm/src/interpreter/mod.rs @@ -26,20 +26,54 @@ pub(crate) enum ExecState { pub(crate) struct InterpreterRuntime; impl InterpreterRuntime { - pub(crate) fn exec(store: &mut Store, cf: CallFrame, call_stack_base: u32) -> Result<()> { - executor::Executor::new(store, cf, call_stack_base).run_to_completion() + pub(crate) fn exec(store: &mut Store, mut cf: CallFrame, call_stack_base: u32) -> Result<()> { + loop { + let module = Self::frame_module(store, &cf); + match executor::Executor::new(store, &module, cf, call_stack_base).run_to_completion()? { + None => return Ok(()), + Some(frame) => cf = frame, + } + } } - pub(crate) fn exec_with_fuel(store: &mut Store, cf: CallFrame, fuel: u32) -> Result { - executor::Executor::new(store, cf, 0).run_with_fuel(fuel) + pub(crate) fn exec_with_fuel(store: &mut Store, mut cf: CallFrame, fuel: u32) -> Result { + store.execution_fuel = fuel; + if fuel == 0 { + return Ok(ExecState::Suspended(cf)); + } + let mut chunk_left = executor::CHECKPOINT_INTERVAL; + loop { + let module = Self::frame_module(store, &cf); + match executor::Executor::new(store, &module, cf, 0).run_with_fuel(chunk_left)? { + executor::RunEnd::State(state) => return Ok(state), + executor::RunEnd::Left(frame, left) => (cf, chunk_left) = (frame, left), + } + } } #[cfg(feature = "std")] pub(crate) fn exec_with_time_budget( store: &mut Store, - cf: CallFrame, + mut cf: CallFrame, time_budget: core::time::Duration, ) -> Result { - executor::Executor::new(store, cf, 0).run_with_time_budget(time_budget) + if time_budget.is_zero() { + return Ok(ExecState::Suspended(cf)); + } + let start = crate::std::time::Instant::now(); + let mut chunk_left = executor::CHECKPOINT_INTERVAL; + loop { + let module = Self::frame_module(store, &cf); + match executor::Executor::new(store, &module, cf, 0).run_with_time_budget(start, time_budget, chunk_left)? { + executor::RunEnd::State(state) => return Ok(state), + executor::RunEnd::Left(frame, left) => (cf, chunk_left) = (frame, left), + } + } + } + + /// The module instance that owns the function of `cf`. + fn frame_module(store: &Store, cf: &CallFrame) -> crate::ModuleInstance { + let owner = store.state.funcs.wasm(cf.func_addr).owner; + store.get_module_instance(owner).expect("invalid module instance").clone() } } diff --git a/crates/tinywasm/src/interpreter/num_helpers.rs b/crates/tinywasm/src/interpreter/num_helpers.rs index 53632bd0..aa805a9b 100644 --- a/crates/tinywasm/src/interpreter/num_helpers.rs +++ b/crates/tinywasm/src/interpreter/num_helpers.rs @@ -25,7 +25,7 @@ macro_rules! checked_conv_float { ($from:tt, $to:tt, $self:expr) => {{ checked_conv_float!($from, $to, $to, $self) }}; // Conversion with an intermediate unsigned type and error checking (three types) ($from:tt, $intermediate:tt, $to:tt, $self:expr) => {{ - let v = <$from>::stack_pop(&mut $self.store.value_stack); + let v = <$from>::stack_pop(&mut $self.value_stack); let (min, max) = float_min_max!($from, $intermediate); if v.is_nan() { core::hint::cold_path(); @@ -35,7 +35,7 @@ macro_rules! checked_conv_float { core::hint::cold_path(); return Err(crate::Trap::IntegerOverflow.into()); } - <$to>::stack_push(&mut $self.store.value_stack, (v as $intermediate as $to).into()); + <$to>::stack_push(&mut $self.value_stack, (v as $intermediate as $to).into()); }}; } diff --git a/crates/tinywasm/src/interpreter/simd/instructions.rs b/crates/tinywasm/src/interpreter/simd/instructions.rs index 1ad4c25a..a3bc0789 100644 --- a/crates/tinywasm/src/interpreter/simd/instructions.rs +++ b/crates/tinywasm/src/interpreter/simd/instructions.rs @@ -30,7 +30,7 @@ impl Value128 { pub(crate) fn v128_any_true(self) -> bool { simd_impl! { wasm => { wasm::v128_any_true(self.to_wasm_v128()) } - generic => { self.0.iter().any(|&b| b != 0) } + generic => { u128::from_ne_bytes(self.0) != 0 } } } @@ -498,7 +498,11 @@ impl Value128 { "f64x2.extract_lane" => extract_lane_f64(self, lane: u8) -> f64 => f64::from_bits(self.extract_lane_i64(lane) as u64); // Truth checks, popcount, and averaging - "i8x16.all_true" => i8x16_all_true(self) -> bool => self.0.iter().all(|&b| b != 0); + "i8x16.all_true" => i8x16_all_true(self) -> bool => { + // No byte is zero: the classic zero-byte test on the whole vector. + let v = u128::from_ne_bytes(self.0); + v.wrapping_sub(u128::from_ne_bytes([1; 16])) & !v & u128::from_ne_bytes([0x80; 16]) == 0 + }; "i16x8.all_true" => i16x8_all_true(self) -> bool => self.as_i16x8().iter().all(|&x| x != 0); "i32x4.all_true" => i32x4_all_true(self) -> bool => self.as_i32x4().iter().all(|&x| x != 0); "i64x2.all_true" => i64x2_all_true(self) -> bool => self.as_i64x2().iter().all(|&x| x != 0); diff --git a/crates/tinywasm/src/interpreter/stack/value_stack.rs b/crates/tinywasm/src/interpreter/stack/value_stack.rs index 1c40e31c..d8971260 100644 --- a/crates/tinywasm/src/interpreter/stack/value_stack.rs +++ b/crates/tinywasm/src/interpreter/stack/value_stack.rs @@ -6,6 +6,7 @@ use crate::engine::{Config, StackConfig}; use crate::interpreter::*; use crate::{Result, Trap}; +#[derive(Default)] #[cfg_attr(feature = "debug", derive(Debug))] /// Physical value lanes used by the interpreter. /// @@ -18,31 +19,76 @@ pub(crate) struct ValueStack { } #[cfg_attr(feature = "debug", derive(Debug))] +/// One value lane: a stack of `len` values over `data`. +/// +/// `data` holds every slot the stack has reached and only grows. The slots above `len` hold stale +/// values that are written before they are read again. A large operand-stack reservation stays +/// capacity, so its slots are written, and their pages touched, only when a push first reaches them. +/// Keeping the height outside the `Vec` lets the tail-call handlers carry it in a register and +/// write it back without `unsafe`. pub(crate) struct Stack { data: Vec, + len: usize, max_size: usize, dynamic: bool, } +/// The most slots [`Stack::enter_locals`] writes ahead of the pushes that reach them. +const WRITTEN_RESERVATION: usize = 64; + +/// An empty stack without an allocation: what a store holds while an executor runs. +impl Default for Stack { + fn default() -> Self { + Self { data: Vec::new(), len: 0, max_size: 0, dynamic: false } + } +} + impl Stack { pub(crate) fn new(config: StackConfig) -> Self { - Self { data: Vec::with_capacity(config.initial_size), max_size: config.max_size, dynamic: config.dynamic } + Self { + data: Vec::with_capacity(config.initial_size), + len: 0, + max_size: config.max_size, + dynamic: config.dynamic, + } } pub(crate) fn clear(&mut self) { - self.data.clear(); + self.len = 0; } #[inline(always)] pub(crate) fn len(&self) -> usize { - self.data.len() + self.len + } + + /// Sets the height to one read from [`Self::len`] earlier, with nothing pushed or popped since. + /// The tail-call handlers pass the height between them and write it back on entry, so the + /// compiler can use the value in a register instead of reloading it from memory. + #[cfg(feature = "nightly-tail-calls")] + #[inline(always)] + pub(crate) fn set_len(&mut self, len: usize) { + debug_assert!(len <= self.data.len()); + self.len = len; } /// Pushes a value inside a function body. `enter_locals` reserved the function's whole operand - /// stack, so the stack is never full here. After this check `Vec::push` cannot reach its own - /// growth path, so the instruction handlers make no calls. + /// stack, so the stack is never full here, and the handlers make no calls. #[inline(always)] pub(crate) fn push(&mut self, value: T) { + let len = self.len; + match self.data.get_mut(len) { + Some(slot) => *slot = value, + None => self.push_first(value), + } + self.len = len + 1; + } + + /// Adds the first slot at a height the stack has not reached before, within the reserved + /// capacity: it never reallocates. On the tail-call build this makes no call. + #[inline(always)] + fn push_first(&mut self, value: T) { + core::hint::cold_path(); if self.data.len() == self.data.capacity() { crate::invariant_violated("value stack push beyond the function's reservation"); } @@ -53,10 +99,12 @@ impl Stack { /// covers, so a dynamic stack grows here if needed. #[inline(always)] pub(crate) fn push_or_grow(&mut self, value: T) -> Result<(), Trap> { - if self.data.len() == self.data.capacity() { - return self.push_grow(value); + let len = self.len; + match self.data.get_mut(len) { + Some(slot) => *slot = value, + None => return self.push_grow(value), } - self.data.push(value); + self.len = len + 1; Ok(()) } @@ -69,33 +117,38 @@ impl Stack { #[cold] #[inline(never)] fn push_grow(&mut self, value: T) -> Result<(), Trap> { - // Check the limit only at capacity to avoid an extra hot-path check. Vec growth may - // intentionally overshoot max_size. - if !self.dynamic || self.data.len() >= self.max_size { + // Only the reserved capacity is free to use. Past it, check the limit, which Vec growth + // may intentionally overshoot. + if self.data.len() == self.data.capacity() && (!self.dynamic || self.data.len() >= self.max_size) { return Err(Trap::ValueStackOverflow); } self.data.push(value); + self.len = self.data.len(); Ok(()) } + /// Pops the top value. On an empty stack the index wraps around, so the bounds check in + /// [`Self::get`] also catches an underflow. #[inline(always)] pub(crate) fn pop(&mut self) -> T { - match self.data.pop() { - Some(value) => value, - None => crate::invariant_violated("value stack underflow"), - } + let index = self.len.wrapping_sub(1); + let value = *self.get(index); + self.len = index; + value } #[inline(always)] pub(crate) fn last(&self) -> &T { - match self.data.last() { - Some(value) => value, - None => crate::invariant_violated("value stack underflow"), - } + self.get(self.len.wrapping_sub(1)) } + /// The slot at `index`, which validation keeps below the height. The check is against the + /// slots in use at any height, which is what memory safety needs. Builds with debug assertions + /// also check the height, and CI runs the tests optimized with them, with the release profile's + /// wrapping arithmetic. #[inline(always)] pub(crate) fn get(&self, index: usize) -> &T { + debug_assert!(index < self.len); match self.data.get(index) { Some(value) => value, None => crate::invariant_violated("value stack index out of range"), @@ -104,6 +157,7 @@ impl Stack { #[inline(always)] pub(crate) fn set(&mut self, index: usize, value: T) { + debug_assert!(index < self.len); match self.data.get_mut(index) { Some(slot) => *slot = value, None => crate::invariant_violated("value stack index out of range"), @@ -118,29 +172,36 @@ impl Stack { #[inline(always)] pub(crate) fn truncate_keep(&mut self, n: usize, end_keep: usize) { - let len = self.data.len(); + let len = self.len; debug_assert!(n <= len); if n >= len { return; } let keep = len.wrapping_sub(n).min(end_keep); - self.data.copy_within(len.wrapping_sub(keep)..len, n); - self.data.truncate(n.wrapping_add(keep)); + if keep != 0 { + // Copying to the start of the values above `n` leaves copy_within no check to fail. + match self.data.get_mut(n..len) { + Some(above) => above.copy_within(above.len().wrapping_sub(keep).., 0), + None => crate::invariant_violated("value stack index out of range"), + } + } + self.len = n.wrapping_add(keep); } #[inline(always)] pub(crate) fn truncate_to(&mut self, n: usize) { - debug_assert!(n <= self.data.len()); - self.data.truncate(n); + debug_assert!(n <= self.len); + self.len = n; } + /// Moves the top value down to index `n` and drops everything above it (a function's result + /// replacing its frame). #[inline(always)] pub(crate) fn truncate_to_one_tail(&mut self, n: usize) { - debug_assert!(n < self.data.len()); - let last = self.pop(); - self.data.truncate(n); - self.data.push(last); + debug_assert!(n < self.len); + self.copy(self.len.wrapping_sub(1), n); + self.len = n + 1; } /// Enters a function: turns its parameters into the first locals, zeroes the rest, and reserves @@ -154,36 +215,80 @@ impl Stack { max_stack: usize, ) -> Result { debug_assert!(param_count <= local_count); - debug_assert!(param_count <= self.data.len()); + debug_assert!(param_count <= self.len); - let len = self.data.len(); + let len = self.len; let start = len - param_count; let end = start + local_count; let reserve = end + max_stack; - if reserve > self.data.capacity() { + // A reservation larger than `WRITTEN_RESERVATION` stays capacity, so once the locals have + // slots and the capacity covers it, entries skip `reserve_slots`. + let len_slots = self.data.len(); + if reserve > len_slots + && (end > len_slots || reserve > self.data.capacity() || reserve - len_slots <= WRITTEN_RESERVATION) + { core::hint::cold_path(); - if reserve > self.max_size || !self.dynamic { - return Err(Trap::ValueStackOverflow); - } - let cap = self.data.capacity(); - let target = reserve.max(cap.max(1).saturating_mul(2)).min(self.max_size); - if self.data.try_reserve(target - len).is_err() { - return Err(Trap::ValueStackOverflow); - } + self.reserve_slots(end, reserve)?; } - self.data.resize(end, T::default()); + // Most functions have no or few locals in a lane. Store the first and last directly and + // fill only what lies between: `fill` becomes a memset call even for one value. + if end > len { + match self.data.get_mut(len..end) { + Some([a]) => *a = T::default(), + Some([a, middle @ .., b]) => { + if !middle.is_empty() { + middle.fill(T::default()); + } + (*a, *b) = (T::default(), T::default()); + } + _ => crate::invariant_violated("value stack index out of range"), + } + } + self.len = end; Ok(start as u32) } + /// Makes slots for a function's locals and reserves its operand stack when they reach past the + /// slots the stack has had. A reservation of up to [`WRITTEN_RESERVATION`] slots is written out + /// with the locals, so later entries at this height skip this. A larger one stays capacity and + /// its slots are written as they are pushed, so a function whose deep branch rarely runs does + /// not keep pages for it. + #[cold] + #[inline(never)] + fn reserve_slots(&mut self, end: usize, reserve: usize) -> Result<(), Trap> { + if reserve > self.data.capacity() { + self.grow_to(reserve)?; + } + let slots = if reserve - self.data.len() <= WRITTEN_RESERVATION { reserve } else { end }; + if slots > self.data.len() { + self.data.resize(slots, T::default()); + } + Ok(()) + } + + /// Makes room for `reserve` slots, growing the allocation if a dynamic stack allows it. + #[cold] + #[inline(never)] + fn grow_to(&mut self, reserve: usize) -> Result<(), Trap> { + if reserve > self.max_size || !self.dynamic { + return Err(Trap::ValueStackOverflow); + } + let target = reserve.max(self.data.capacity().max(1).saturating_mul(2)).min(self.max_size); + if self.data.try_reserve(target - self.data.len()).is_err() { + return Err(Trap::ValueStackOverflow); + } + Ok(()) + } + #[inline(always)] pub(crate) fn select_many(&mut self, count: usize, condition: bool) { if count == 0 { return; } - let len = self.data.len(); + let len = self.len; let needed = count.wrapping_mul(2); if len < needed { @@ -193,10 +298,13 @@ impl Stack { if !condition { let dst = len.wrapping_sub(needed); let src = len.wrapping_sub(count); - self.data.copy_within(src..len, dst); + match self.data.get_mut(..len) { + Some(live) => live.copy_within(src..len, dst), + None => crate::invariant_violated("value stack index out of range"), + } } - self.data.truncate(len.wrapping_sub(count)); + self.len = len.wrapping_sub(count); } } @@ -205,7 +313,7 @@ impl<'a, T: Copy + Default> IntoIterator for &'a Stack { type IntoIter = core::slice::Iter<'a, T>; fn into_iter(self) -> Self::IntoIter { - self.data.iter() + self.data[..self.len].iter() } } impl ValueStack { @@ -276,10 +384,17 @@ impl ValueStack { ) -> Result { let locals_base32 = self.stack_32.enter_locals(params.c32 as usize, locals.c32 as usize, max_stack.c32 as usize)?; - let locals_base64 = - self.stack_64.enter_locals(params.c64 as usize, locals.c64 as usize, max_stack.c64 as usize)?; - let locals_base128 = - self.stack_128.enter_locals(params.c128 as usize, locals.c128 as usize, max_stack.c128 as usize)?; + // Most functions use only the 32-bit lane; an unused lane needs no reservation. + let locals_base64 = if locals.c64 | max_stack.c64 == 0 { + self.stack_64.len() as u32 + } else { + self.stack_64.enter_locals(params.c64 as usize, locals.c64 as usize, max_stack.c64 as usize)? + }; + let locals_base128 = if locals.c128 | max_stack.c128 == 0 { + self.stack_128.len() as u32 + } else { + self.stack_128.enter_locals(params.c128 as usize, locals.c128 as usize, max_stack.c128 as usize)? + }; Ok(StackBase { s32: locals_base32, s64: locals_base64, s128: locals_base128 }) } @@ -317,3 +432,60 @@ impl ValueStack { } } } + +#[cfg(test)] +mod tests { + use super::*; + + /// A stack of height 2 whose third slot still holds a popped value. + #[cfg(debug_assertions)] + fn popped() -> Stack { + let mut stack = Stack::new(StackConfig::fixed(8)); + for value in [1, 2, 3] { + stack.push_or_grow(value).unwrap(); + } + assert_eq!(stack.pop(), 3); + stack + } + + #[test] + #[cfg(debug_assertions)] + #[should_panic] + fn get_above_the_height() { + popped().get(2); + } + + #[test] + #[cfg(debug_assertions)] + #[should_panic] + fn set_above_the_height() { + popped().set(2, 0); + } + + /// A large operand-stack reservation is not written: only the locals and the values pushed + /// become slots. A small one is written with the locals. + #[test] + fn large_reservations_are_written_as_they_are_pushed() { + let mut stack = Stack::::new(StackConfig::fixed(1024)); + assert!(matches!(stack.enter_locals(0, 2, 500), Ok(0))); + assert_eq!(stack.data.len(), 2); + for value in 0..500 { + stack.push(value); + } + assert_eq!((stack.len(), stack.data.len()), (502, 502)); + + // Below the slots the stack already has, entering a function writes only its locals. + stack.truncate_to(2); + stack.push(7); + assert!(matches!(stack.enter_locals(0, 2, 10), Ok(3))); + assert_eq!((stack.len(), stack.data.len()), (5, 502)); + + stack.truncate_to(0); + for value in 0..500 { + stack.push_or_grow(value).unwrap(); + } + assert!(matches!(stack.enter_locals(0, 1, 8), Ok(500))); + assert_eq!(stack.data.len(), 509); + assert!(matches!(stack.enter_locals(0, 2, 1000), Err(Trap::ValueStackOverflow))); + } +} diff --git a/crates/tinywasm/src/store/mod.rs b/crates/tinywasm/src/store/mod.rs index 7bbab959..4e348fc3 100644 --- a/crates/tinywasm/src/store/mod.rs +++ b/crates/tinywasm/src/store/mod.rs @@ -568,14 +568,14 @@ impl Store { result } - /// Add functions to the store, returning their addresses in the store + /// Add functions to the store, returning their (contiguous) addresses in the store pub(crate) fn init_funcs( &mut self, funcs: &[Shared], owner: ModuleInstanceId, module_type_idxs: &[TypeAddr], type_addrs: &[TypeAddr], - ) -> impl ExactSizeIterator { + ) -> core::ops::Range { debug_assert_eq!(funcs.len(), module_type_idxs.len()); let funcs = funcs.iter().cloned().zip(module_type_idxs).map(|(func, &type_idx)| (type_addrs[type_idx as usize], func)); diff --git a/crates/tinywasm/tests/cross_instance_calls.rs b/crates/tinywasm/tests/cross_instance_calls.rs new file mode 100644 index 00000000..1a3054f0 --- /dev/null +++ b/crates/tinywasm/tests/cross_instance_calls.rs @@ -0,0 +1,173 @@ +//! Calls, returns and exceptions that move between module instances. The interpreter runs one +//! instance's frames at a time and hands over to the other instance at each crossing. + +use tinywasm::{ExecProgress, Imports, ModuleInstance, Store}; + +type TestResult = Result<(), Box>; + +const MODULE_A: &str = r#" + (module + (type $i2i (func (param i32) (result i32))) + (tag $e (export "e") (param i32)) + (table $t (export "t") 4 funcref) + (func $add1 (export "add1") (param i32) (result i32) + (i32.add (local.get 0) (i32.const 1))) + (func (export "throw") (param i32) + (throw $e (local.get 0))) + ;; Calls table slot 0, which the other instance fills. + (func (export "apply") (param i32) (result i32) + (i32.add (call_indirect $t (type $i2i) (local.get 0) (i32.const 0)) (i32.const 100))) + ;; Same, catching `$e` from the callee. + (func (export "apply_catching") (param i32) (result i32) + (block $caught (result i32) + (try_table (result i32) (catch $e $caught) + (call_indirect $t (type $i2i) (local.get 0) (i32.const 0))) + (return)) + (i32.add (i32.const 1000))) + (elem (table $t) (i32.const 1) func $add1)) +"#; + +const MODULE_B: &str = r#" + (module + (type $i2i (func (param i32) (result i32))) + (import "a" "e" (tag $e (param i32))) + (import "a" "t" (table $t 4 funcref)) + (import "a" "add1" (func $add1 (param i32) (result i32))) + (import "a" "throw" (func $throw (param i32))) + (import "a" "apply" (func $apply (param i32) (result i32))) + (import "a" "apply_catching" (func $apply_catching (param i32) (result i32))) + (func $double (param i32) (result i32) + (i32.mul (local.get 0) (i32.const 2))) + (func $throws (param i32) (result i32) + (throw $e (local.get 0))) + (elem declare func $double $throws) + + (func (export "direct") (param i32) (result i32) + (call $add1 (call $add1 (local.get 0)))) + (func (export "tail") (param i32) (result i32) + (return_call $add1 (local.get 0))) + (func (export "indirect") (param i32) (result i32) + (call_indirect $t (type $i2i) (local.get 0) (i32.const 1))) + ;; B -> A -> B -> A -> B + (func (export "callback") (param i32) (result i32) + (table.set $t (i32.const 0) (ref.func $double)) + (call $apply (local.get 0))) + (func (export "catch_from_import") (param i32) (result i32) + (block $caught (result i32) + (try_table (catch $e $caught) + (call $throw (local.get 0))) + (i32.const -1))) + ;; Thrown in B, unwinds through A, caught in B. + (func (export "catch_through_import") (param i32) (result i32) + (table.set $t (i32.const 0) (ref.func $throws)) + (block $caught (result i32) + (try_table (result i32) (catch $e $caught) + (call $apply (local.get 0))) + (return)) + (i32.add (i32.const 2000))) + ;; Thrown in B, caught in A. + (func (export "caught_in_import") (param i32) (result i32) + (table.set $t (i32.const 0) (ref.func $throws)) + (call $apply_catching (local.get 0))) + ;; Four crossings per iteration. + (func (export "crossings") (param $n i32) (result i32) (local $sum i32) + (table.set $t (i32.const 0) (ref.func $double)) + (loop $next + (local.set $sum (i32.add (local.get $sum) (call $apply (local.get $n)))) + (local.set $n (i32.sub (local.get $n) (i32.const 1))) + (br_if $next (local.get $n))) + (local.get $sum))) +"#; + +fn instantiate(store: &mut Store) -> Result> { + let a = tinywasm::parse_bytes(&wat::parse_str(MODULE_A)?)?; + let b = tinywasm::parse_bytes(&wat::parse_str(MODULE_B)?)?; + let a = ModuleInstance::instantiate(store, &a, None)?; + let mut imports = Imports::new(); + imports.link_module("a", a)?; + Ok(ModuleInstance::instantiate(store, &b, Some(&imports))?) +} + +fn call( + store: &mut Store, + instance: &ModuleInstance, + name: &str, + arg: i32, +) -> Result> { + Ok(instance.func::(store, name)?.call(store, arg)?) +} + +#[test] +fn calls_and_returns_cross_instances() -> TestResult { + let mut store = Store::default(); + let b = instantiate(&mut store)?; + assert_eq!(call(&mut store, &b, "direct", 5)?, 7); + assert_eq!(call(&mut store, &b, "tail", 5)?, 6); + assert_eq!(call(&mut store, &b, "indirect", 5)?, 6); + assert_eq!(call(&mut store, &b, "callback", 5)?, 110); + Ok(()) +} + +#[test] +fn exceptions_cross_instances() -> TestResult { + let mut store = Store::default(); + let b = instantiate(&mut store)?; + assert_eq!(call(&mut store, &b, "catch_from_import", 7)?, 7); + assert_eq!(call(&mut store, &b, "catch_through_import", 9)?, 2009); + assert_eq!(call(&mut store, &b, "caught_in_import", 9)?, 1009); + // The instances keep working after each unwind. + assert_eq!(call(&mut store, &b, "callback", 5)?, 110); + Ok(()) +} + +#[test] +fn budgeted_runs_cross_instances() -> TestResult { + const N: i32 = 1000; + const EXPECTED: i32 = N * (N + 1) + 100 * N; + + let mut store = Store::default(); + let b = instantiate(&mut store)?; + assert_eq!(call(&mut store, &b, "crossings", N)?, EXPECTED); + + for fuel in [1, 3, 64, 1000] { + let mut store = Store::default(); + let b = instantiate(&mut store)?; + let func = b.func::(&store, "crossings")?; + let mut exec = func.call_resumable(&mut store, N)?; + let mut suspensions = 0; + let result = loop { + match exec.resume_with_fuel(fuel)? { + ExecProgress::Completed(value) => break value, + ExecProgress::Suspended => suspensions += 1, + } + }; + assert_eq!(result, EXPECTED, "fuel {fuel}"); + assert!(suspensions > 0, "fuel {fuel}"); + } + + #[cfg(feature = "std")] + { + let mut store = Store::default(); + let b = instantiate(&mut store)?; + let func = b.func::(&store, "crossings")?; + let mut exec = func.call_resumable(&mut store, N)?; + let result = loop { + match exec.resume_with_time_budget(std::time::Duration::from_micros(10))? { + ExecProgress::Completed(value) => break value, + ExecProgress::Suspended => {} + } + }; + assert_eq!(result, EXPECTED); + + // An effectively unlimited budget completes in one resume. + let mut store = Store::default(); + let b = instantiate(&mut store)?; + let func = b.func::(&store, "crossings")?; + let mut exec = func.call_resumable(&mut store, N)?; + match exec.resume_with_time_budget(std::time::Duration::MAX)? { + ExecProgress::Completed(value) => assert_eq!(value, EXPECTED), + ExecProgress::Suspended => panic!("suspended with an unlimited time budget"), + } + } + Ok(()) +}