perf: keep the value stack's height in a register, stacked on cheaper calls

.github/workflows/test.yaml

@@ -46,6 +46,11 @@ jobs:
             name: "Linux x86 (nightly tail calls)"
             target: x86_64-unknown-linux-gnu
             args: "--features tinywasm/nightly-tail-calls"
+          - os: ubuntu-26.04
+            rust: nightly
+            name: "Linux x86 (nightly tail calls, release with debug assertions)"
+            target: x86_64-unknown-linux-gnu
+            args: "--release --config profile.release.debug-assertions=true --features tinywasm/nightly-tail-calls"
           - os: ubuntu-26.04
             rust: stable
             name: "Linux x86 (stable, no default features)"
@@ -69,6 +74,11 @@ jobs:
             rust: stable
             name: "Windows x86_64"
             target: x86_64-pc-windows-msvc
+          - os: windows-2025
+            rust: nightly
+            name: "Windows x86_64 (nightly tail calls)"
+            target: x86_64-pc-windows-msvc
+            args: "--features tinywasm/nightly-tail-calls"
           - os: ubuntu-26.04-arm
             rust: stable
             name: "Linux arm64"

crates/tinywasm/src/instance.rs

@@ -60,6 +60,12 @@ struct ModuleInstanceInner {
     id: ModuleInstanceId,
     type_addrs: Box<[TypeAddr]>,
     func_addrs: Box<[FuncAddr]>,
+    /// The module, whose functions are the instance's own (non-imported) ones in index order. The
+    /// store allocates them contiguously from `local_func_base`, so an executor can borrow any of
+    /// them for a whole run. Holding the module costs one reference count per instance.
+    module: Module,
+    local_func_base: FuncAddr,
+    imported_funcs: u32,
     table_addrs: Box<[TableAddr]>,
     mem_addrs: Box<[MemAddr]>,
     #[cfg(feature = "std")]
@@ -84,6 +90,20 @@ impl ModuleInstance {
         self.0.func_addrs[addr as usize]
     }
 
+    /// The store address and body of the module's own function `idx`, or `None` for an import.
+    #[inline(always)]
+    pub(crate) fn local_func_by_index(&self, idx: FuncAddr) -> Option<(FuncAddr, &WasmFunction)> {
+        let local = idx.wrapping_sub(self.0.imported_funcs);
+        let func = self.0.module.funcs.get(local as usize)?;
+        Some((self.0.local_func_base + local, func))
+    }
+
+    /// The body of the function at store address `addr`, if this instance owns it.
+    #[inline(always)]
+    pub(crate) fn local_func(&self, addr: FuncAddr) -> Option<&WasmFunction> {
+        self.0.module.funcs.get(addr.wrapping_sub(self.0.local_func_base) as usize).map(|func| &**func)
+    }
+
     /// resolve a table address to the global store address
     #[inline]
     pub(crate) fn resolve_table_addr(&self, addr: TableAddr) -> TableAddr {
@@ -211,7 +231,9 @@ impl ModuleInstance {
         let id = store.next_module_instance_id();
         let mut addrs = crate::imports::ResolvedImports::new(store, module, &type_addrs, imports)?;
         let imported_funcs = addrs.funcs.len();
-        addrs.funcs.extend(store.init_funcs(&module.funcs, id, &module.func_type_idxs[imported_funcs..], &type_addrs));
+        let local_funcs = store.init_funcs(&module.funcs, id, &module.func_type_idxs[imported_funcs..], &type_addrs);
+        let local_func_base = local_funcs.start;
+        addrs.funcs.extend(local_funcs);
         addrs.tags.extend(store.init_tags(&module.tags, &type_addrs));
         let limiter = store.engine.config().resource_limiter.clone();
         if !module.skip_local_memory_allocation {
@@ -238,6 +260,9 @@ impl ModuleInstance {
             id,
             type_addrs,
             func_addrs: addrs.funcs.into_boxed_slice(),
+            module: module.clone(),
+            local_func_base,
+            imported_funcs: imported_funcs as u32,
             table_addrs: addrs.tables.into_boxed_slice(),
             mem_addrs: addrs.memories.into_boxed_slice(),
             #[cfg(feature = "std")]

crates/tinywasm/src/interpreter/executor/dispatch.rs

@@ -4,7 +4,11 @@ macro_rules! define_stable_dispatch {
     ($executor:ident, $instr_ptr:ident, $dispatch_next:ident, $dispatch_flow:ident;
      $($variant:ident $(($($arg:pat),*))? $({ $($field:ident),* })? => $body:expr),* $(,)?) => {
         #[inline(always)]
-        fn exec_step($executor: &mut Self, $instr_ptr: usize) -> ExecResult<ExecFlow> {
+        fn exec_step(
+            $executor: &mut Self,
+            instructions: &[tinywasm_types::Instruction],
+            $instr_ptr: usize,
+        ) -> ExecResult<ExecFlow> {
             macro_rules! $dispatch_next {
                 ($next_instr_ptr:expr) => {{
                     return Ok(ExecFlow::Next($next_instr_ptr));
@@ -16,75 +20,112 @@ macro_rules! define_stable_dispatch {
                 }};
             }
             use tinywasm_types::Instruction::*;
-            match &$executor.func.instructions[$instr_ptr] {
+            match &instructions[$instr_ptr] {
                 $($variant $(($($arg),*))? $({ $($field),* })? => $body,)*
             }
             Ok(ExecFlow::Next($instr_ptr + 1))
         }
     };
 }
 
-impl Executor<'_> {
+impl<'module> Executor<'_, 'module> {
     instruction_handlers!(define_stable_dispatch);
 
+    /// The executing function's instructions. They are borrowed from the module instance, not from
+    /// the executor, so the dispatch loop keeps them in a local across steps.
     #[inline(always)]
-    pub(crate) fn run_to_completion(mut self) -> Result<()> {
+    fn instructions(&self) -> &'module [tinywasm_types::Instruction] {
+        let func: &'module WasmFunction = self.func;
+        &func.instructions
+    }
+
+    /// One step from `instr_ptr`. A step that switches functions reloads `instructions`.
+    #[inline(always)]
+    fn step(
+        &mut self,
+        instructions: &mut &'module [tinywasm_types::Instruction],
+        instr_ptr: usize,
+    ) -> ExecResult<Option<usize>> {
+        Ok(match Self::exec_step(self, instructions, instr_ptr)? {
+            ExecFlow::Next(next_instr_ptr) => Some(next_instr_ptr),
+            ExecFlow::Switch(next_instr_ptr) => {
+                *instructions = self.instructions();
+                Some(next_instr_ptr)
+            }
+            ExecFlow::Complete => None,
+        })
+    }
+
+    /// Runs until the call completes (`None`) or continues in another module instance's frame.
+    #[inline(always)]
+    pub(crate) fn run_to_completion(mut self) -> Result<Option<CallFrame>> {
         let mut instr_ptr = self.cf.instr_ptr;
+        let mut instructions = self.instructions();
         loop {
-            match Self::exec_step(&mut self, instr_ptr)?.next_instr_ptr() {
+            match self.step(&mut instructions, instr_ptr)? {
                 Some(next_instr_ptr) => instr_ptr = next_instr_ptr,
-                None => return cold!(Ok(())),
+                None => return cold!(Ok(self.left())),
             }
         }
     }
 
+    /// Runs `chunk_left` instructions, then checkpoint by checkpoint until `time_budget` has
+    /// elapsed since `start`.
     #[cfg(feature = "std")]
     #[inline(always)]
-    pub(crate) fn run_with_time_budget(mut self, time_budget: core::time::Duration) -> Result<ExecState> {
-        use crate::std::time::Instant;
-
-        if time_budget.is_zero() {
-            return Ok(ExecState::Suspended(self.cf));
-        }
-        let start = Instant::now();
+    pub(crate) fn run_with_time_budget(
+        mut self,
+        start: crate::std::time::Instant,
+        time_budget: core::time::Duration,
+        mut chunk_left: u32,
+    ) -> Result<RunEnd> {
         let mut instr_ptr = self.cf.instr_ptr;
+        let mut instructions = self.instructions();
         loop {
-            for _ in 0..CHECKPOINT_INTERVAL {
-                match Self::exec_step(&mut self, instr_ptr)?.next_instr_ptr() {
+            while chunk_left != 0 {
+                chunk_left -= 1;
+                match self.step(&mut instructions, instr_ptr)? {
                     Some(next_instr_ptr) => instr_ptr = next_instr_ptr,
-                    None => return Ok(ExecState::Completed),
+                    None => return Ok(self.run_end(chunk_left)),
                 }
             }
+            chunk_left = CHECKPOINT_INTERVAL;
 
             if start.elapsed() >= time_budget {
                 self.cf.instr_ptr = instr_ptr;
-                return Ok(ExecState::Suspended(self.cf));
+                return Ok(RunEnd::State(ExecState::Suspended(self.cf)));
             }
         }
     }
 
+    /// Runs `chunk_left` instructions, then checkpoint by checkpoint until the store's fuel is out.
     #[inline(always)]
-    pub(crate) fn run_with_fuel(mut self, fuel: u32) -> Result<ExecState> {
+    pub(crate) fn run_with_fuel(mut self, mut chunk_left: u32) -> Result<RunEnd> {
         self.fuel_metered = true;
-        self.store.execution_fuel = fuel;
-        if self.store.execution_fuel == 0 {
-            return Ok(ExecState::Suspended(self.cf));
-        }
-
         let mut instr_ptr = self.cf.instr_ptr;
+        let mut instructions = self.instructions();
         loop {
-            for _ in 0..CHECKPOINT_INTERVAL {
-                match Self::exec_step(&mut self, instr_ptr)?.next_instr_ptr() {
+            while chunk_left != 0 {
+                chunk_left -= 1;
+                match self.step(&mut instructions, instr_ptr)? {
                     Some(next_instr_ptr) => instr_ptr = next_instr_ptr,
-                    None => return Ok(ExecState::Completed),
+                    None => return Ok(self.run_end(chunk_left)),
                 }
             }
+            chunk_left = CHECKPOINT_INTERVAL;
 
             self.store.execution_fuel = self.store.execution_fuel.saturating_sub(CHECKPOINT_INTERVAL);
             if self.store.execution_fuel == 0 {
                 self.cf.instr_ptr = instr_ptr;
-                return Ok(ExecState::Suspended(self.cf));
+                return Ok(RunEnd::State(ExecState::Suspended(self.cf)));
             }
         }
     }
+
+    fn run_end(&self, chunk_left: u32) -> RunEnd {
+        match self.left() {
+            Some(frame) => RunEnd::Left(frame, chunk_left),
+            None => RunEnd::State(ExecState::Completed),
+        }
+    }
 }