From 4790e6a18df48cf5d3ed5ee035210ce32f3488ce Mon Sep 17 00:00:00 2001 From: Matt Hargett Date: Tue, 29 Sep 2026 19:14:08 -0700 Subject: [PATCH 1/4] perf: pass the tail-call handlers' Instruction in a register on arm64_32 On arm64_32 (watchOS) the Rust ABI passes the 8-byte Instruction by reference, so every dispatch stores it and the next handler loads it back. The C convention passes it in a register, and `C-unwind` still lets a host function's panic unwind. Other targets keep the Rust ABI. --- .../interpreter/executor/dispatch_become.rs | 79 +++++++++++++------ 1 file changed, 57 insertions(+), 22 deletions(-) diff --git a/crates/tinywasm/src/interpreter/executor/dispatch_become.rs b/crates/tinywasm/src/interpreter/executor/dispatch_become.rs index f790d63..52e7d18 100644 --- a/crates/tinywasm/src/interpreter/executor/dispatch_become.rs +++ b/crates/tinywasm/src/interpreter/executor/dispatch_become.rs @@ -3,8 +3,35 @@ use super::*; struct Unbudgeted; struct Bounded; -type UnbudgetedHandler = for<'store> fn(&mut Executor<'store>, &[Instruction], usize, Instruction) -> ExecResult<()>; -type BoundedHandler = for<'store> fn(&mut Executor<'store>, usize, Instruction, u32) -> ExecResult<()>; +// Between two handlers only what the calling convention passes in registers stays out of memory. +// On arm64_32 (watchOS) the Rust ABI passes an aggregate larger than a pointer, the 8-byte +// `Instruction`, by reference, so every dispatch would store it and the next handler load it back. +// The C convention passes it in a register; `C-unwind` still lets a host function's panic unwind. +// Elsewhere the handlers keep the Rust ABI. +macro_rules! handler_fn { + ($(#[$meta:meta])* fn $($rest:tt)*) => { + #[cfg(all(target_arch = "aarch64", target_pointer_width = "32"))] + #[allow(improper_ctypes_definitions)] + $(#[$meta])* extern "C-unwind" fn $($rest)* + #[cfg(not(all(target_arch = "aarch64", target_pointer_width = "32")))] + $(#[$meta])* fn $($rest)* + }; +} + +macro_rules! handler_types { + ($($abi:literal)?) => { + // Both sides are Rust, so the C convention's view of these types need not be FFI-safe. + #[allow(improper_ctypes_definitions)] + type UnbudgetedHandler = + for<'store> $(extern $abi)? fn(&mut Executor<'store>, &[Instruction], usize, Instruction) -> ExecResult<()>; + #[allow(improper_ctypes_definitions)] + type BoundedHandler = for<'store> $(extern $abi)? fn(&mut Executor<'store>, usize, Instruction, u32) -> ExecResult<()>; + }; +} +#[cfg(all(target_arch = "aarch64", target_pointer_width = "32"))] +handler_types!("C-unwind"); +#[cfg(not(all(target_arch = "aarch64", target_pointer_width = "32")))] +handler_types!(); macro_rules! define_unbudgeted_tail_dispatch { ($executor:ident, $instr_ptr:ident, $dispatch_next:ident, $dispatch_flow:ident; @@ -19,7 +46,7 @@ macro_rules! define_unbudgeted_tail_dispatch { HANDLERS[opcode as usize] } - $( + $(handler_fn! { #[allow(non_snake_case, unreachable_code, unused_imports, unused_macros, unused_variables)] fn $variant( $executor: &mut Executor<'_>, @@ -59,7 +86,7 @@ macro_rules! define_unbudgeted_tail_dispatch { $body; $dispatch_next!($instr_ptr + 1) } - )* + })* }; } @@ -76,7 +103,7 @@ macro_rules! define_bounded_tail_dispatch { HANDLERS[opcode as usize] } - $( + $(handler_fn! { #[allow(non_snake_case, unreachable_code, unused_imports, unused_macros, unused_variables)] fn $variant( $executor: &mut Executor<'_>, @@ -122,7 +149,7 @@ macro_rules! define_bounded_tail_dispatch { $body; $dispatch_next!($instr_ptr + 1) } - )* + })* }; } @@ -132,16 +159,20 @@ impl Unbudgeted { // The handlers tail-call these cold paths instead of calling them: a call would make every // handler save a stack frame. - #[cold] - #[inline(never)] - fn handler_mismatch(_: &mut Executor<'_>, _: &[Instruction], _: usize, _: Instruction) -> ExecResult<()> { - unreachable!("instruction handler mismatch") + handler_fn! { + #[cold] + #[inline(never)] + fn handler_mismatch(_: &mut Executor<'_>, _: &[Instruction], _: usize, _: Instruction) -> ExecResult<()> { + unreachable!("instruction handler mismatch") + } } - #[cold] - #[inline(never)] - fn invalid_instr_ptr(_: &mut Executor<'_>, _: &[Instruction], instr_ptr: usize, _: Instruction) -> ExecResult<()> { - unreachable!("instruction pointer {instr_ptr} out of range, this is a bug") + handler_fn! { + #[cold] + #[inline(never)] + fn invalid_instr_ptr(_: &mut Executor<'_>, _: &[Instruction], instr_ptr: usize, _: Instruction) -> ExecResult<()> { + unreachable!("instruction pointer {instr_ptr} out of range, this is a bug") + } } } @@ -150,16 +181,20 @@ impl Bounded { // Tail-called like `Unbudgeted`'s. - #[cold] - #[inline(never)] - fn handler_mismatch(_: &mut Executor<'_>, _: usize, _: Instruction, _: u32) -> ExecResult<()> { - unreachable!("instruction handler mismatch") + handler_fn! { + #[cold] + #[inline(never)] + fn handler_mismatch(_: &mut Executor<'_>, _: usize, _: Instruction, _: u32) -> ExecResult<()> { + unreachable!("instruction handler mismatch") + } } - #[cold] - #[inline(never)] - fn invalid_instr_ptr(_: &mut Executor<'_>, instr_ptr: usize, _: Instruction, _: u32) -> ExecResult<()> { - unreachable!("instruction pointer {instr_ptr} out of range, this is a bug") + handler_fn! { + #[cold] + #[inline(never)] + fn invalid_instr_ptr(_: &mut Executor<'_>, instr_ptr: usize, _: Instruction, _: u32) -> ExecResult<()> { + unreachable!("instruction pointer {instr_ptr} out of range, this is a bug") + } } #[inline(always)] From 15982f1a7cf3e23b46625b4b12cd41125e2c0081 Mon Sep 17 00:00:00 2001 From: Matt Hargett Date: Tue, 29 Sep 2026 13:23:15 -0700 Subject: [PATCH 2/4] perf: keep each value stack's height outside its Vec The height of a value-stack lane is its own field, over a Vec that holds every slot the lane has reached. Slots above the height hold stale values that are written before they are read again. The tail-call handlers can then pass the height between them in a register and write it back without unsafe code. Entering a function writes its locals, and its operand-stack reservation if that is at most 64 slots, so later entries at that height compare once. A larger reservation stays capacity, and a push writes each slot the first time it reaches it, so a function whose deep branch rarely runs keeps no pages for it. With 64 stores of a module whose untaken branch declares a 30000-deep i64 stack, each store keeps next's footprint (35 KB on macOS, 16-19 KB on glibc) instead of 273 KB and 250 KB. v128.any_true and i8x16.all_true test the vector with integer arithmetic: LLVM scalarized their iterator forms once pushes gained that path. Only builds with debug assertions check indices against the height. CI also runs the tests optimized with them, keeping the release profile's wrapping arithmetic. Entering a function zeroes one or two locals with direct stores, and only the ones in between with memset. --- .github/workflows/test.yaml | 5 + .../src/interpreter/simd/instructions.rs | 8 +- .../src/interpreter/stack/value_stack.rs | 227 ++++++++++++++---- 3 files changed, 195 insertions(+), 45 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 6be4ae5..8a94aa2 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -46,6 +46,11 @@ jobs: name: "Linux x86 (nightly tail calls)" target: x86_64-unknown-linux-gnu args: "--features tinywasm/nightly-tail-calls" + - os: ubuntu-26.04 + rust: nightly + name: "Linux x86 (nightly tail calls, release with debug assertions)" + target: x86_64-unknown-linux-gnu + args: "--release --config profile.release.debug-assertions=true --features tinywasm/nightly-tail-calls" - os: ubuntu-26.04 rust: stable name: "Linux x86 (stable, no default features)" diff --git a/crates/tinywasm/src/interpreter/simd/instructions.rs b/crates/tinywasm/src/interpreter/simd/instructions.rs index 1ad4c25..a3bc078 100644 --- a/crates/tinywasm/src/interpreter/simd/instructions.rs +++ b/crates/tinywasm/src/interpreter/simd/instructions.rs @@ -30,7 +30,7 @@ impl Value128 { pub(crate) fn v128_any_true(self) -> bool { simd_impl! { wasm => { wasm::v128_any_true(self.to_wasm_v128()) } - generic => { self.0.iter().any(|&b| b != 0) } + generic => { u128::from_ne_bytes(self.0) != 0 } } } @@ -498,7 +498,11 @@ impl Value128 { "f64x2.extract_lane" => extract_lane_f64(self, lane: u8) -> f64 => f64::from_bits(self.extract_lane_i64(lane) as u64); // Truth checks, popcount, and averaging - "i8x16.all_true" => i8x16_all_true(self) -> bool => self.0.iter().all(|&b| b != 0); + "i8x16.all_true" => i8x16_all_true(self) -> bool => { + // No byte is zero: the classic zero-byte test on the whole vector. + let v = u128::from_ne_bytes(self.0); + v.wrapping_sub(u128::from_ne_bytes([1; 16])) & !v & u128::from_ne_bytes([0x80; 16]) == 0 + }; "i16x8.all_true" => i16x8_all_true(self) -> bool => self.as_i16x8().iter().all(|&x| x != 0); "i32x4.all_true" => i32x4_all_true(self) -> bool => self.as_i32x4().iter().all(|&x| x != 0); "i64x2.all_true" => i64x2_all_true(self) -> bool => self.as_i64x2().iter().all(|&x| x != 0); diff --git a/crates/tinywasm/src/interpreter/stack/value_stack.rs b/crates/tinywasm/src/interpreter/stack/value_stack.rs index 1c40e31..386453b 100644 --- a/crates/tinywasm/src/interpreter/stack/value_stack.rs +++ b/crates/tinywasm/src/interpreter/stack/value_stack.rs @@ -18,31 +18,59 @@ pub(crate) struct ValueStack { } #[cfg_attr(feature = "debug", derive(Debug))] +/// One value lane: a stack of `len` values over `data`. +/// +/// `data` holds every slot the stack has reached and only grows. The slots above `len` hold stale +/// values that are written before they are read again. A large operand-stack reservation stays +/// capacity, so its slots are written, and their pages touched, only when a push first reaches them. +/// Keeping the height outside the `Vec` lets the tail-call handlers carry it in a register and +/// write it back without `unsafe`. pub(crate) struct Stack { data: Vec, + len: usize, max_size: usize, dynamic: bool, } +/// The most slots [`Stack::enter_locals`] writes ahead of the pushes that reach them. +const WRITTEN_RESERVATION: usize = 64; + impl Stack { pub(crate) fn new(config: StackConfig) -> Self { - Self { data: Vec::with_capacity(config.initial_size), max_size: config.max_size, dynamic: config.dynamic } + Self { + data: Vec::with_capacity(config.initial_size), + len: 0, + max_size: config.max_size, + dynamic: config.dynamic, + } } pub(crate) fn clear(&mut self) { - self.data.clear(); + self.len = 0; } #[inline(always)] pub(crate) fn len(&self) -> usize { - self.data.len() + self.len } /// Pushes a value inside a function body. `enter_locals` reserved the function's whole operand - /// stack, so the stack is never full here. After this check `Vec::push` cannot reach its own - /// growth path, so the instruction handlers make no calls. + /// stack, so the stack is never full here, and the handlers make no calls. #[inline(always)] pub(crate) fn push(&mut self, value: T) { + let len = self.len; + match self.data.get_mut(len) { + Some(slot) => *slot = value, + None => self.push_first(value), + } + self.len = len + 1; + } + + /// Adds the first slot at a height the stack has not reached before, within the reserved + /// capacity: it never reallocates. On the tail-call build this makes no call. + #[inline(always)] + fn push_first(&mut self, value: T) { + core::hint::cold_path(); if self.data.len() == self.data.capacity() { crate::invariant_violated("value stack push beyond the function's reservation"); } @@ -53,10 +81,12 @@ impl Stack { /// covers, so a dynamic stack grows here if needed. #[inline(always)] pub(crate) fn push_or_grow(&mut self, value: T) -> Result<(), Trap> { - if self.data.len() == self.data.capacity() { - return self.push_grow(value); + let len = self.len; + match self.data.get_mut(len) { + Some(slot) => *slot = value, + None => return self.push_grow(value), } - self.data.push(value); + self.len = len + 1; Ok(()) } @@ -69,33 +99,38 @@ impl Stack { #[cold] #[inline(never)] fn push_grow(&mut self, value: T) -> Result<(), Trap> { - // Check the limit only at capacity to avoid an extra hot-path check. Vec growth may - // intentionally overshoot max_size. - if !self.dynamic || self.data.len() >= self.max_size { + // Only the reserved capacity is free to use. Past it, check the limit, which Vec growth + // may intentionally overshoot. + if self.data.len() == self.data.capacity() && (!self.dynamic || self.data.len() >= self.max_size) { return Err(Trap::ValueStackOverflow); } self.data.push(value); + self.len = self.data.len(); Ok(()) } + /// Pops the top value. On an empty stack the index wraps around, so the bounds check in + /// [`Self::get`] also catches an underflow. #[inline(always)] pub(crate) fn pop(&mut self) -> T { - match self.data.pop() { - Some(value) => value, - None => crate::invariant_violated("value stack underflow"), - } + let index = self.len.wrapping_sub(1); + let value = *self.get(index); + self.len = index; + value } #[inline(always)] pub(crate) fn last(&self) -> &T { - match self.data.last() { - Some(value) => value, - None => crate::invariant_violated("value stack underflow"), - } + self.get(self.len.wrapping_sub(1)) } + /// The slot at `index`, which validation keeps below the height. The check is against the + /// slots in use at any height, which is what memory safety needs. Builds with debug assertions + /// also check the height, and CI runs the tests optimized with them, with the release profile's + /// wrapping arithmetic. #[inline(always)] pub(crate) fn get(&self, index: usize) -> &T { + debug_assert!(index < self.len); match self.data.get(index) { Some(value) => value, None => crate::invariant_violated("value stack index out of range"), @@ -104,6 +139,7 @@ impl Stack { #[inline(always)] pub(crate) fn set(&mut self, index: usize, value: T) { + debug_assert!(index < self.len); match self.data.get_mut(index) { Some(slot) => *slot = value, None => crate::invariant_violated("value stack index out of range"), @@ -118,29 +154,35 @@ impl Stack { #[inline(always)] pub(crate) fn truncate_keep(&mut self, n: usize, end_keep: usize) { - let len = self.data.len(); + let len = self.len; debug_assert!(n <= len); if n >= len { return; } let keep = len.wrapping_sub(n).min(end_keep); - self.data.copy_within(len.wrapping_sub(keep)..len, n); - self.data.truncate(n.wrapping_add(keep)); + if keep != 0 { + // Copying to the start of the values above `n` leaves copy_within no check to fail. + match self.data.get_mut(n..len) { + Some(above) => above.copy_within(above.len().wrapping_sub(keep).., 0), + None => crate::invariant_violated("value stack index out of range"), + } + } + self.len = n.wrapping_add(keep); } #[inline(always)] pub(crate) fn truncate_to(&mut self, n: usize) { - debug_assert!(n <= self.data.len()); - self.data.truncate(n); + debug_assert!(n <= self.len); + self.len = n; } #[inline(always)] pub(crate) fn truncate_to_one_tail(&mut self, n: usize) { - debug_assert!(n < self.data.len()); + debug_assert!(n < self.len); let last = self.pop(); - self.data.truncate(n); - self.data.push(last); + self.len = n; + self.push(last); } /// Enters a function: turns its parameters into the first locals, zeroes the rest, and reserves @@ -154,36 +196,75 @@ impl Stack { max_stack: usize, ) -> Result { debug_assert!(param_count <= local_count); - debug_assert!(param_count <= self.data.len()); + debug_assert!(param_count <= self.len); - let len = self.data.len(); + let len = self.len; let start = len - param_count; let end = start + local_count; let reserve = end + max_stack; - if reserve > self.data.capacity() { + if reserve > self.data.len() { core::hint::cold_path(); - if reserve > self.max_size || !self.dynamic { - return Err(Trap::ValueStackOverflow); - } - let cap = self.data.capacity(); - let target = reserve.max(cap.max(1).saturating_mul(2)).min(self.max_size); - if self.data.try_reserve(target - len).is_err() { - return Err(Trap::ValueStackOverflow); - } + self.reserve_slots(end, reserve)?; } - self.data.resize(end, T::default()); + // Most functions have no or few locals in a lane. Store the first and last directly and + // fill only what lies between: `fill` becomes a memset call even for one value. + if end > len { + match self.data.get_mut(len..end) { + Some([a]) => *a = T::default(), + Some([a, middle @ .., b]) => { + if !middle.is_empty() { + middle.fill(T::default()); + } + (*a, *b) = (T::default(), T::default()); + } + _ => crate::invariant_violated("value stack index out of range"), + } + } + self.len = end; Ok(start as u32) } + /// Makes slots for a function's locals and reserves its operand stack when they reach past the + /// slots the stack has had. A reservation of up to [`WRITTEN_RESERVATION`] slots is written out + /// with the locals, so later entries at this height skip this. A larger one stays capacity and + /// its slots are written as they are pushed, so a function whose deep branch rarely runs does + /// not keep pages for it; its entries come back here. + #[cold] + #[inline(never)] + fn reserve_slots(&mut self, end: usize, reserve: usize) -> Result<(), Trap> { + if reserve > self.data.capacity() { + self.grow_to(reserve)?; + } + let slots = if reserve - self.data.len() <= WRITTEN_RESERVATION { reserve } else { end }; + if slots > self.data.len() { + self.data.resize(slots, T::default()); + } + Ok(()) + } + + /// Makes room for `reserve` slots, growing the allocation if a dynamic stack allows it. + #[cold] + #[inline(never)] + fn grow_to(&mut self, reserve: usize) -> Result<(), Trap> { + if reserve > self.max_size || !self.dynamic { + return Err(Trap::ValueStackOverflow); + } + let target = reserve.max(self.data.capacity().max(1).saturating_mul(2)).min(self.max_size); + if self.data.try_reserve(target - self.data.len()).is_err() { + return Err(Trap::ValueStackOverflow); + } + Ok(()) + } + #[inline(always)] pub(crate) fn select_many(&mut self, count: usize, condition: bool) { if count == 0 { return; } - let len = self.data.len(); + let len = self.len; let needed = count.wrapping_mul(2); if len < needed { @@ -193,10 +274,13 @@ impl Stack { if !condition { let dst = len.wrapping_sub(needed); let src = len.wrapping_sub(count); - self.data.copy_within(src..len, dst); + match self.data.get_mut(..len) { + Some(live) => live.copy_within(src..len, dst), + None => crate::invariant_violated("value stack index out of range"), + } } - self.data.truncate(len.wrapping_sub(count)); + self.len = len.wrapping_sub(count); } } @@ -205,7 +289,7 @@ impl<'a, T: Copy + Default> IntoIterator for &'a Stack { type IntoIter = core::slice::Iter<'a, T>; fn into_iter(self) -> Self::IntoIter { - self.data.iter() + self.data[..self.len].iter() } } impl ValueStack { @@ -317,3 +401,60 @@ impl ValueStack { } } } + +#[cfg(test)] +mod tests { + use super::*; + + /// A stack of height 2 whose third slot still holds a popped value. + #[cfg(debug_assertions)] + fn popped() -> Stack { + let mut stack = Stack::new(StackConfig::fixed(8)); + for value in [1, 2, 3] { + stack.push_or_grow(value).unwrap(); + } + assert_eq!(stack.pop(), 3); + stack + } + + #[test] + #[cfg(debug_assertions)] + #[should_panic] + fn get_above_the_height() { + popped().get(2); + } + + #[test] + #[cfg(debug_assertions)] + #[should_panic] + fn set_above_the_height() { + popped().set(2, 0); + } + + /// A large operand-stack reservation is not written: only the locals and the values pushed + /// become slots. A small one is written with the locals. + #[test] + fn large_reservations_are_written_as_they_are_pushed() { + let mut stack = Stack::::new(StackConfig::fixed(1024)); + assert!(matches!(stack.enter_locals(0, 2, 500), Ok(0))); + assert_eq!(stack.data.len(), 2); + for value in 0..500 { + stack.push(value); + } + assert_eq!((stack.len(), stack.data.len()), (502, 502)); + + // Below the slots the stack already has, entering a function writes only its locals. + stack.truncate_to(2); + stack.push(7); + assert!(matches!(stack.enter_locals(0, 2, 10), Ok(3))); + assert_eq!((stack.len(), stack.data.len()), (5, 502)); + + stack.truncate_to(0); + for value in 0..500 { + stack.push_or_grow(value).unwrap(); + } + assert!(matches!(stack.enter_locals(0, 1, 8), Ok(500))); + assert_eq!(stack.data.len(), 509); + assert!(matches!(stack.enter_locals(0, 2, 1000), Err(Trap::ValueStackOverflow))); + } +} From acd22defbe0d69f06db265a8603acc4e0380d51e Mon Sep 17 00:00:00 2001 From: Matt Hargett Date: Tue, 29 Sep 2026 13:33:05 -0700 Subject: [PATCH 3/4] perf: hold the value stack in the executor while it runs The executor takes the store's value stack when it is created and gives it back when it is dropped, and around host calls, which read their arguments from the store and may call back into wasm. Every handler receives the executor as a `&mut`, so a stack access reached through it needs one load fewer (the store pointer), and the compiler knows that no store to a slot or to memory changes a stack's height, so it can keep the heights in registers within a handler. --- .../src/interpreter/executor/instructions.rs | 86 +-- .../tinywasm/src/interpreter/executor/mod.rs | 489 ++++++++++-------- .../tinywasm/src/interpreter/num_helpers.rs | 4 +- .../src/interpreter/stack/value_stack.rs | 8 + 4 files changed, 313 insertions(+), 274 deletions(-) diff --git a/crates/tinywasm/src/interpreter/executor/instructions.rs b/crates/tinywasm/src/interpreter/executor/instructions.rs index 22892eb..15932fa 100644 --- a/crates/tinywasm/src/interpreter/executor/instructions.rs +++ b/crates/tinywasm/src/interpreter/executor/instructions.rs @@ -6,14 +6,14 @@ macro_rules! exec_op { <$ty>::stack_push(value_stack, $expr?); Ok(()) } - exec_binary_fallible(&mut $executor.store.value_stack)?; + exec_binary_fallible(&mut $executor.value_stack)?; }}; ($executor:ident; unary $from:ty => $to:ty, |$v:ident| $expr:expr) => {{ fn exec_unary(value_stack: &mut ValueStack) { let $v = <$from>::stack_pop(value_stack); <$to>::stack_push(value_stack, $expr); } - exec_unary(&mut $executor.store.value_stack); + exec_unary(&mut $executor.value_stack); }}; ($executor:ident; binary $from:ty => $to:ty, |$lhs:ident, $rhs:ident| $expr:expr) => {{ exec_op!($executor; binary $from, $from => $to, |$lhs, $rhs| $expr) @@ -24,7 +24,7 @@ macro_rules! exec_op { let $lhs = <$lhs_ty>::stack_pop(value_stack); <$res>::stack_push(value_stack, $expr); } - exec_binary(&mut $executor.store.value_stack); + exec_binary(&mut $executor.value_stack); }}; ($executor:ident; ternary $from:ty => $to:ty, |$a:ident, $b:ident, $c:ident| $expr:expr) => {{ fn exec_ternary(value_stack: &mut ValueStack) { @@ -33,7 +33,7 @@ macro_rules! exec_op { let $a = <$from>::stack_pop(value_stack); <$to>::stack_push(value_stack, $expr); } - exec_ternary(&mut $executor.store.value_stack); + exec_ternary(&mut $executor.value_stack); }}; } @@ -43,15 +43,15 @@ macro_rules! instruction_handlers { ($emit:ident) => { $emit! { executor, instr_ptr, dispatch_next, dispatch_flow; Unreachable => { return cold!(Err(Trap::Unreachable.into())); }, - Drop32 => { _ = Value32::stack_pop(&mut executor.store.value_stack)}, - Drop64 => { _ = Value64::stack_pop(&mut executor.store.value_stack)}, - Drop128 => { _ = Value128::stack_pop(&mut executor.store.value_stack)}, - Select32 => Value32::stack_select(&mut executor.store.value_stack), - Select64 => Value64::stack_select(&mut executor.store.value_stack), - Select128 => Value128::stack_select(&mut executor.store.value_stack), + Drop32 => { _ = Value32::stack_pop(&mut executor.value_stack)}, + Drop64 => { _ = Value64::stack_pop(&mut executor.value_stack)}, + Drop128 => { _ = Value128::stack_pop(&mut executor.value_stack)}, + Select32 => Value32::stack_select(&mut executor.value_stack), + Select64 => Value64::stack_select(&mut executor.value_stack), + Select128 => Value128::stack_select(&mut executor.value_stack), SelectStore32(idx) => executor.exec_select_store::(idx.resolve(&executor.func.data))?, SelectStore64(idx) => executor.exec_select_store::(idx.resolve(&executor.func.data))?, - SelectMulti(counts) => executor.store.value_stack.select_multi(*counts), + SelectMulti(counts) => executor.value_stack.select_multi(*counts), Call(v) => dispatch_flow!(executor.exec_call_direct(*v, instr_ptr + 1)?), CallSelf => { executor.exec_call_self(instr_ptr + 1)?; dispatch_next!(0); }, CallIndirect(idx) => dispatch_flow!(executor.exec_call_indirect::(*idx, instr_ptr + 1)?), @@ -63,10 +63,10 @@ macro_rules! instruction_handlers { Throw(tag) => dispatch_flow!(executor.exec_throw(*tag, instr_ptr)?), ThrowRef => dispatch_flow!(executor.exec_throw_ref(instr_ptr)?), Jump(ip) => dispatch_next!(*ip as usize), - JumpIfZero32(ip) => if i32::stack_pop(&mut executor.store.value_stack) == 0 { dispatch_next!(*ip as usize) }, - JumpIfNonZero32(ip) => if i32::stack_pop(&mut executor.store.value_stack) != 0 { dispatch_next!(*ip as usize) }, - JumpIfZero64(ip) => if i64::stack_pop(&mut executor.store.value_stack) == 0 { dispatch_next!(*ip as usize) }, - JumpIfNonZero64(ip) => if i64::stack_pop(&mut executor.store.value_stack) != 0 { dispatch_next!(*ip as usize) }, + JumpIfZero32(ip) => if i32::stack_pop(&mut executor.value_stack) == 0 { dispatch_next!(*ip as usize) }, + JumpIfNonZero32(ip) => if i32::stack_pop(&mut executor.value_stack) != 0 { dispatch_next!(*ip as usize) }, + JumpIfZero64(ip) => if i64::stack_pop(&mut executor.value_stack) == 0 { dispatch_next!(*ip as usize) }, + JumpIfNonZero64(ip) => if i64::stack_pop(&mut executor.value_stack) != 0 { dispatch_next!(*ip as usize) }, JumpIfRefNull(ip) => { let ip = *ip; if executor.exec_jump_if_ref::() { dispatch_next!(ip as usize) } }, JumpIfRefNonNull(ip) => { let ip = *ip; if executor.exec_jump_if_ref::() { dispatch_next!(ip as usize) } }, BrOnCast(idx) => if let Some(ip) = executor.exec_br_on_cast::(*idx) { dispatch_next!(ip) }, @@ -91,33 +91,33 @@ macro_rules! instruction_handlers { JumpCmpLocalConst64(packed) => if let Some(ip) = executor.exec_jump_cmp_local_const64(*packed) { dispatch_next!(ip) }, JumpCmpLocalLocal32(packed) => if let Some(ip) = executor.exec_jump_cmp_local_local32(*packed) { dispatch_next!(ip) }, JumpCmpLocalLocal64(packed) => if let Some(ip) = executor.exec_jump_cmp_local_local64(*packed) { dispatch_next!(ip) }, - DropKeep32 { base, keep } => executor.store.value_stack.stack_32.truncate_keep((executor.cf.stack_base().s32 + u32::from(*base)) as usize, *keep as usize), - DropKeep64 { base, keep } => executor.store.value_stack.stack_64.truncate_keep((executor.cf.stack_base().s64 + u32::from(*base)) as usize, *keep as usize), - DropKeep128 { base, keep } => executor.store.value_stack.stack_128.truncate_keep((executor.cf.stack_base().s128 + u32::from(*base)) as usize, *keep as usize), + DropKeep32 { base, keep } => executor.value_stack.stack_32.truncate_keep((executor.cf.stack_base().s32 + u32::from(*base)) as usize, *keep as usize), + DropKeep64 { base, keep } => executor.value_stack.stack_64.truncate_keep((executor.cf.stack_base().s64 + u32::from(*base)) as usize, *keep as usize), + DropKeep128 { base, keep } => executor.value_stack.stack_128.truncate_keep((executor.cf.stack_base().s128 + u32::from(*base)) as usize, *keep as usize), BranchTable(idx) => dispatch_next!(executor.exec_branch_table(*idx)), Return => dispatch_flow!(executor.exec_return()), ReturnVoid => dispatch_flow!(executor.exec_return_void()), Return32 => dispatch_flow!(executor.exec_return_32()), Return64 => dispatch_flow!(executor.exec_return_64()), Return128 => dispatch_flow!(executor.exec_return_128()), - LocalGet32(local_index) => Value32::local_push(&mut executor.store.value_stack, &executor.cf, *local_index), - LocalGet64(local_index) => Value64::local_push(&mut executor.store.value_stack, &executor.cf, *local_index), - LocalGet128(local_index) => Value128::local_push(&mut executor.store.value_stack, &executor.cf, *local_index), + LocalGet32(local_index) => Value32::local_push(&mut executor.value_stack, &executor.cf, *local_index), + LocalGet64(local_index) => Value64::local_push(&mut executor.value_stack, &executor.cf, *local_index), + LocalGet128(local_index) => Value128::local_push(&mut executor.value_stack, &executor.cf, *local_index), LocalSet32(local_index) => executor.exec_local_set_pop::(*local_index), LocalSet64(local_index) => executor.exec_local_set_pop::(*local_index), LocalSet128(local_index) => executor.exec_local_set_pop::(*local_index), - LocalCopy32(from, to) => Value32::local_copy(&mut executor.store.value_stack, &executor.cf, *from, *to), - LocalCopy64(from, to) => Value64::local_copy(&mut executor.store.value_stack, &executor.cf, *from, *to), - LocalCopy128(from, to) => Value128::local_copy(&mut executor.store.value_stack, &executor.cf, *from, *to), - AddConst32(c) => { i32::stack_update(&mut executor.store.value_stack, |value| value.wrapping_add(*c)); }, - AndConst32(c) => { i32::stack_update(&mut executor.store.value_stack, |value| value & *c); }, - XorConst32(c) => { i32::stack_update(&mut executor.store.value_stack, |value| value ^ *c); }, - ShrUConst32(c) => { u32::stack_update(&mut executor.store.value_stack, |value| value.wrapping_shr(*c as u32)); }, - AddConst64(idx) => { let rhs = idx.resolve(&executor.func.data).value(); i64::stack_update(&mut executor.store.value_stack, |value| value.wrapping_add(rhs)); }, - BinOpStackConst32(op, rhs) => { u32::stack_update(&mut executor.store.value_stack, |lhs| op.exec(lhs, *rhs as u32)); }, - BinOpStackConst64(packed) => { let rhs = packed.index.resolve(&executor.func.data).value() as u64; u64::stack_update(&mut executor.store.value_stack, |lhs| packed.op.exec(lhs, rhs)); }, - IncLocal32(arg) => { i32::local_update(&mut executor.store.value_stack, &executor.cf, arg.local, |value| value.wrapping_add(arg.value)); }, - IncLocal64(packed) => { let rhs = packed.index.resolve(&executor.func.data).value(); i64::local_update(&mut executor.store.value_stack, &executor.cf, packed.op, |value| value.wrapping_add(rhs)); }, + LocalCopy32(from, to) => Value32::local_copy(&mut executor.value_stack, &executor.cf, *from, *to), + LocalCopy64(from, to) => Value64::local_copy(&mut executor.value_stack, &executor.cf, *from, *to), + LocalCopy128(from, to) => Value128::local_copy(&mut executor.value_stack, &executor.cf, *from, *to), + AddConst32(c) => { i32::stack_update(&mut executor.value_stack, |value| value.wrapping_add(*c)); }, + AndConst32(c) => { i32::stack_update(&mut executor.value_stack, |value| value & *c); }, + XorConst32(c) => { i32::stack_update(&mut executor.value_stack, |value| value ^ *c); }, + ShrUConst32(c) => { u32::stack_update(&mut executor.value_stack, |value| value.wrapping_shr(*c as u32)); }, + AddConst64(idx) => { let rhs = idx.resolve(&executor.func.data).value(); i64::stack_update(&mut executor.value_stack, |value| value.wrapping_add(rhs)); }, + BinOpStackConst32(op, rhs) => { u32::stack_update(&mut executor.value_stack, |lhs| op.exec(lhs, *rhs as u32)); }, + BinOpStackConst64(packed) => { let rhs = packed.index.resolve(&executor.func.data).value() as u64; u64::stack_update(&mut executor.value_stack, |lhs| packed.op.exec(lhs, rhs)); }, + IncLocal32(arg) => { i32::local_update(&mut executor.value_stack, &executor.cf, arg.local, |value| value.wrapping_add(arg.value)); }, + IncLocal64(packed) => { let rhs = packed.index.resolve(&executor.func.data).value(); i64::local_update(&mut executor.value_stack, &executor.cf, packed.op, |value| value.wrapping_add(rhs)); }, I32Add3 => exec_op!(executor; ternary i32 => i32, |a, b, c| a.wrapping_add(b).wrapping_add(c)), I64Add3 => exec_op!(executor; ternary i64 => i64, |a, b, c| a.wrapping_add(b).wrapping_add(c)), MulAccLocal32(acc) => executor.exec_mul_acc_local::(*acc, i32::wrapping_mul, i32::wrapping_add), @@ -160,9 +160,9 @@ macro_rules! instruction_handlers { BinOpGlobalConst32(packed) => { let v = packed.index.resolve(&executor.func.data); executor.exec_binop_global_const::(v.a(), v.b(), packed.op)?; }, BinOpGlobalConst64(packed) => { let v = packed.index.resolve(&executor.func.data); executor.exec_binop_global_const::(v.a(), v.b(), packed.op)?; }, BinOpGlobalConst128(packed) => { let v = packed.index.resolve(&executor.func.data); let rhs = Value128(v.b().resolve(&executor.func.data).value()); executor.exec_binop_global_const::(v.a(), rhs, packed.op)?; }, - SetLocalConst32(arg) => i32::local_set(&mut executor.store.value_stack, &executor.cf, arg.local, arg.value), - SetLocalConst64(packed) => { let v = packed.index.resolve(&executor.func.data).value(); i64::local_set(&mut executor.store.value_stack, &executor.cf, packed.op, v); }, - SetLocalConst128(packed) => Value128::local_set(&mut executor.store.value_stack, &executor.cf, packed.op, Value128(packed.index.resolve(&executor.func.data).value())), + SetLocalConst32(arg) => i32::local_set(&mut executor.value_stack, &executor.cf, arg.local, arg.value), + SetLocalConst64(packed) => { let v = packed.index.resolve(&executor.func.data).value(); i64::local_set(&mut executor.value_stack, &executor.cf, packed.op, v); }, + SetLocalConst128(packed) => Value128::local_set(&mut executor.value_stack, &executor.cf, packed.op, Value128(packed.index.resolve(&executor.func.data).value())), IncMemoryLocal32(arg) => executor.exec_inc_memory_local::(arg.memory_arg_idx, arg.local1, #[inline(always)] |v| v.wrapping_add(1))?, IncMemoryLocal64(arg) => executor.exec_inc_memory_local::(arg.memory_arg_idx, arg.local1, #[inline(always)] |v| v.wrapping_add(1))?, StoreLocalLocal32(arg) => executor.exec_store_local_local::(arg.memory_arg_idx, arg.local1, arg.local2)?, @@ -202,10 +202,10 @@ macro_rules! instruction_handlers { GlobalTee32(global_index) => executor.exec_global_tee::(*global_index), GlobalTee64(global_index) => executor.exec_global_tee::(*global_index), GlobalTee128(global_index) => executor.exec_global_tee::(*global_index), - Const32(val) => i32::stack_push(&mut executor.store.value_stack, *val), - Const64Imm(val) => i64::stack_push(&mut executor.store.value_stack, i64::from(*val)), - Const64(idx) => i64::stack_push(&mut executor.store.value_stack, idx.resolve(&executor.func.data).value()), - Const128Imm(val) => Value128::stack_push(&mut executor.store.value_stack, Value128(u128::from(*val).to_le_bytes())), + Const32(val) => i32::stack_push(&mut executor.value_stack, *val), + Const64Imm(val) => i64::stack_push(&mut executor.value_stack, i64::from(*val)), + Const64(idx) => i64::stack_push(&mut executor.value_stack, idx.resolve(&executor.func.data).value()), + Const128Imm(val) => Value128::stack_push(&mut executor.value_stack, Value128(u128::from(*val).to_le_bytes())), I64Eqz => exec_op!(executor; unary i64 => i32, |v| i32::from(v == 0)), I32Eqz => exec_op!(executor; unary i32 => i32, |v| i32::from(v == 0)), I32Eq => exec_op!(executor; binary i32 => i32, |a, b| i32::from(a == b)), @@ -290,8 +290,8 @@ macro_rules! instruction_handlers { I64Popcnt => exec_op!(executor; unary i64 => i64, |v| i64::from(v.count_ones())), // Reference types - RefFunc(func_idx) => ValueRef::stack_push(&mut executor.store.value_stack, ValueRef::from_category_addr(executor.module.resolve_func_addr(*func_idx))), - RefNull(_) => ValueRef::stack_push(&mut executor.store.value_stack, ValueRef::NULL), + RefFunc(func_idx) => ValueRef::stack_push(&mut executor.value_stack, ValueRef::from_category_addr(executor.module.resolve_func_addr(*func_idx))), + RefNull(_) => ValueRef::stack_push(&mut executor.value_stack, ValueRef::NULL), RefIsNull => executor.exec_ref_is_null()?, RefAsNonNull => executor.exec_ref_as_non_null()?, RefI31 => exec_op!(executor; unary i32 => ValueRef, |v| ValueRef::from_i31(v)), @@ -460,7 +460,7 @@ macro_rules! instruction_handlers { V128Store64Lane(arg) => executor.exec_mem_store_lane::(*arg)?, V128Load32Zero(idx) => executor.exec_mem_load::(idx.resolve(&executor.func.data), |v| Value128::from_i32x4([v, 0, 0, 0]))?, V128Load64Zero(idx) => executor.exec_mem_load::(idx.resolve(&executor.func.data), |v| Value128::from_i64x2([v, 0]))?, - Const128(arg) => Value128::stack_push(&mut executor.store.value_stack, Value128(arg.resolve(&executor.func.data).value())), + Const128(arg) => Value128::stack_push(&mut executor.value_stack, Value128(arg.resolve(&executor.func.data).value())), I8x16ExtractLaneS(lane) => executor.exec_simd_extract_lane::(*lane, |v, lane| v.extract_lane_i8(lane) as i32)?, I8x16ExtractLaneU(lane) => executor.exec_simd_extract_lane::(*lane, |v, lane| v.extract_lane_u8(lane) as i32)?, I16x8ExtractLaneS(lane) => executor.exec_simd_extract_lane::(*lane, |v, lane| v.extract_lane_i16(lane) as i32)?, diff --git a/crates/tinywasm/src/interpreter/executor/mod.rs b/crates/tinywasm/src/interpreter/executor/mod.rs index decc92b..dd78116 100644 --- a/crates/tinywasm/src/interpreter/executor/mod.rs +++ b/crates/tinywasm/src/interpreter/executor/mod.rs @@ -72,6 +72,11 @@ impl ExecFlow { } pub(crate) struct Executor<'store> { + /// The store's value stack, which the executor holds while it runs and gives back for host + /// calls and when it is dropped. Reached through the executor, which each handler receives + /// as a `&mut`, stack accesses need one load fewer, and the compiler knows that no store to a + /// slot or to memory changes a stack's height. + value_stack: ValueStack, cf: CallFrame, func: Shared, module: ModuleInstance, @@ -88,7 +93,9 @@ impl<'store> Executor<'store> { let wasm_func = store.state.funcs.wasm(cf.func_addr); let module = store.get_module_instance(wasm_func.owner).expect("invalid module instance").clone(); let mem0 = module.mem0_addr(); + let value_stack = core::mem::take(&mut store.value_stack); Self { + value_stack, module, cf, func: wasm_func.func.clone(), @@ -101,6 +108,27 @@ impl<'store> Executor<'store> { } } + /// Runs `f` with the value stack back in the store, as host functions expect it. + #[inline(always)] + fn with_store_value_stack(&mut self, f: impl FnOnce(&mut Store) -> R) -> R { + /// Takes the stack back when `f` returns or unwinds: a host function may catch a panic + /// from a nested call, and the executor must then still hold the stack. + struct Lent<'a, 'store> { + store: &'a mut &'store mut Store, + value_stack: &'a mut ValueStack, + } + impl Drop for Lent<'_, '_> { + #[inline(always)] + fn drop(&mut self) { + *self.value_stack = core::mem::take(&mut self.store.value_stack); + } + } + + self.store.value_stack = core::mem::take(&mut self.value_stack); + let lent = Lent { store: &mut self.store, value_stack: &mut self.value_stack }; + f(lent.store) + } + /// Resolves a module-local memory index to its store address, caching the common memory-0 case. #[inline(always)] fn mem_addr(&self, idx: MemAddr) -> MemAddr { @@ -128,9 +156,9 @@ impl<'store> Executor<'store> { #[inline(always)] fn exec_jump_if_ref(&mut self) -> bool { - let is_null = ValueRef::stack_peek(&self.store.value_stack).is_null(); + let is_null = ValueRef::stack_peek(&self.value_stack).is_null(); if is_null { - ValueRef::stack_pop(&mut self.store.value_stack); + ValueRef::stack_pop(&mut self.value_stack); } is_null == ON_NULL } @@ -147,41 +175,41 @@ impl<'store> Executor<'store> { &self, local: LocalAddr, ) -> bool { - (T::local_get(&self.store.value_stack, &self.cf, local) == T::default()) == ON_ZERO + (T::local_get(&self.value_stack, &self.cf, local) == T::default()) == ON_ZERO } #[inline(always)] fn exec_jump_cmp_stack_const32(&mut self, packed: PackedOp64) -> Option { let operand = packed.index.resolve(&self.func.data); - packed.op.cmp(i32::stack_pop(&mut self.store.value_stack), operand.b()).then_some(operand.target() as usize) + packed.op.cmp(i32::stack_pop(&mut self.value_stack), operand.b()).then_some(operand.target() as usize) } #[inline(always)] fn exec_jump_cmp_stack_const64(&mut self, packed: PackedOp128) -> Option { let operand = packed.index.resolve(&self.func.data); - packed.op.cmp(i64::stack_pop(&mut self.store.value_stack), operand.b()).then_some(operand.target() as usize) + packed.op.cmp(i64::stack_pop(&mut self.value_stack), operand.b()).then_some(operand.target() as usize) } #[inline(always)] fn exec_jump_cmp_stack_local32(&mut self, packed: PackedOp64) -> Option { let operand = packed.index.resolve(&self.func.data); - let lhs = i32::stack_pop(&mut self.store.value_stack); - let rhs = i32::local_get(&self.store.value_stack, &self.cf, operand.b()); + let lhs = i32::stack_pop(&mut self.value_stack); + let rhs = i32::local_get(&self.value_stack, &self.cf, operand.b()); packed.op.cmp(lhs, rhs).then_some(operand.target() as usize) } #[inline(always)] fn exec_jump_cmp_stack_local64(&mut self, packed: PackedOp64) -> Option { let operand = packed.index.resolve(&self.func.data); - let lhs = i64::stack_pop(&mut self.store.value_stack); - let rhs = i64::local_get(&self.store.value_stack, &self.cf, operand.b()); + let lhs = i64::stack_pop(&mut self.value_stack); + let rhs = i64::local_get(&self.value_stack, &self.cf, operand.b()); packed.op.cmp(lhs, rhs).then_some(operand.target() as usize) } #[inline(always)] fn exec_binop_local_const_jump(&mut self, packed: PackedOp128) -> Option { let operand = packed.index.resolve(&self.func.data); - let value = i32::local_update(&mut self.store.value_stack, &self.cf, operand.local(), |value| { + let value = i32::local_update(&mut self.value_stack, &self.cf, operand.local(), |value| { packed.op.exec(value as u32, operand.value() as u32) as i32 }); ((value == 0) == operand.on_zero()).then_some(operand.target() as usize) @@ -193,10 +221,10 @@ impl<'store> Executor<'store> { packed: PackedOp128<(BinOp, CmpOp), LocalUpdateCmpOperand>, ) -> Option { let operand = packed.index.resolve(&self.func.data); - let lhs = i32::local_update(&mut self.store.value_stack, &self.cf, operand.local(), |value| { + let lhs = i32::local_update(&mut self.value_stack, &self.cf, operand.local(), |value| { packed.op.0.exec(value as u32, operand.value() as u32) as i32 }); - let rhs = i32::local_get(&self.store.value_stack, &self.cf, operand.right()); + let rhs = i32::local_get(&self.value_stack, &self.cf, operand.right()); packed.op.1.cmp(lhs, rhs).then_some(operand.target() as usize) } @@ -206,10 +234,9 @@ impl<'store> Executor<'store> { packed: PackedOp128, ) -> Result, Trap> { let operand = packed.index.resolve(&self.func.data); - let value = i32::stack_update(&mut self.store.value_stack, |lhs| { - packed.op.exec(lhs as u32, operand.value() as u32) as i32 - }); - i32::local_set(&mut self.store.value_stack, &self.cf, operand.local(), value); + let value = + i32::stack_update(&mut self.value_stack, |lhs| packed.op.exec(lhs as u32, operand.value() as u32) as i32); + i32::local_set(&mut self.value_stack, &self.cf, operand.local(), value); Ok(((value == 0) == operand.on_zero()).then_some(operand.target() as usize)) } @@ -226,7 +253,7 @@ impl<'store> Executor<'store> { #[inline(always)] fn exec_inc_local_jump(&mut self, index: Operand128Idx) -> Option { let operand = index.resolve(&self.func.data); - let value = i32::local_update(&mut self.store.value_stack, &self.cf, operand.local(), |value| { + let value = i32::local_update(&mut self.value_stack, &self.cf, operand.local(), |value| { value.wrapping_add(operand.value()) }); ((value == 0) == operand.on_zero()).then_some(operand.target() as usize) @@ -238,8 +265,8 @@ impl<'store> Executor<'store> { index: Operand128Idx, ) -> Result, Trap> { let operand = index.resolve(&self.func.data); - let value = i32::stack_update(&mut self.store.value_stack, |value| value.wrapping_add(operand.value())); - i32::local_set(&mut self.store.value_stack, &self.cf, operand.local(), value); + let value = i32::stack_update(&mut self.value_stack, |value| value.wrapping_add(operand.value())); + i32::local_set(&mut self.value_stack, &self.cf, operand.local(), value); Ok(((value == 0) == operand.on_zero()).then_some(operand.target() as usize)) } @@ -255,84 +282,84 @@ impl<'store> Executor<'store> { #[inline(always)] fn exec_inc_local_jump_cmp_local(&mut self, packed: PackedOp128) -> Option { let operand = packed.index.resolve(&self.func.data); - let lhs = i32::local_update(&mut self.store.value_stack, &self.cf, operand.local(), |value| { + let lhs = i32::local_update(&mut self.value_stack, &self.cf, operand.local(), |value| { value.wrapping_add(operand.value()) }); - let rhs = i32::local_get(&self.store.value_stack, &self.cf, operand.right()); + let rhs = i32::local_get(&self.value_stack, &self.cf, operand.right()); packed.op.cmp(lhs, rhs).then_some(operand.target() as usize) } #[inline(always)] fn exec_jump_cmp_local_const32(&self, packed: PackedOp128) -> Option { let operand = packed.index.resolve(&self.func.data); - let lhs = i32::local_get(&self.store.value_stack, &self.cf, operand.c()); + let lhs = i32::local_get(&self.value_stack, &self.cf, operand.c()); packed.op.cmp(lhs, operand.b()).then_some(operand.target() as usize) } #[inline(always)] fn exec_jump_cmp_local_const64(&self, packed: PackedOp128) -> Option { let operand = packed.index.resolve(&self.func.data); - let lhs = i64::local_get(&self.store.value_stack, &self.cf, operand.c()); + let lhs = i64::local_get(&self.value_stack, &self.cf, operand.c()); packed.op.cmp(lhs, i64::from(operand.b())).then_some(operand.target() as usize) } #[inline(always)] fn exec_jump_cmp_local_local32(&self, packed: PackedOp64) -> Option { let operand = packed.index.resolve(&self.func.data); - let lhs = i32::local_get(&self.store.value_stack, &self.cf, operand.b()); - let rhs = i32::local_get(&self.store.value_stack, &self.cf, operand.c()); + let lhs = i32::local_get(&self.value_stack, &self.cf, operand.b()); + let rhs = i32::local_get(&self.value_stack, &self.cf, operand.c()); packed.op.cmp(lhs, rhs).then_some(operand.target() as usize) } #[inline(always)] fn exec_jump_cmp_local_local64(&self, packed: PackedOp64) -> Option { let operand = packed.index.resolve(&self.func.data); - let lhs = i64::local_get(&self.store.value_stack, &self.cf, operand.b()); - let rhs = i64::local_get(&self.store.value_stack, &self.cf, operand.c()); + let lhs = i64::local_get(&self.value_stack, &self.cf, operand.b()); + let rhs = i64::local_get(&self.value_stack, &self.cf, operand.c()); packed.op.cmp(lhs, rhs).then_some(operand.target() as usize) } fn exec_i64_add128(&mut self) -> Result<(), Trap> { - let b_hi = i64::stack_pop(&mut self.store.value_stack); - let b_lo = i64::stack_pop(&mut self.store.value_stack); - let a_hi = i64::stack_pop(&mut self.store.value_stack); - let a_lo = i64::stack_pop(&mut self.store.value_stack); + let b_hi = i64::stack_pop(&mut self.value_stack); + let b_lo = i64::stack_pop(&mut self.value_stack); + let a_hi = i64::stack_pop(&mut self.value_stack); + let a_lo = i64::stack_pop(&mut self.value_stack); let lo = a_lo.wrapping_add(b_lo); let carry = u64::from((lo as u64) < (a_lo as u64)); let hi = a_hi.wrapping_add(b_hi).wrapping_add(carry as i64); - i64::stack_push(&mut self.store.value_stack, lo); - i64::stack_push(&mut self.store.value_stack, hi); + i64::stack_push(&mut self.value_stack, lo); + i64::stack_push(&mut self.value_stack, hi); Ok(()) } fn exec_i64_sub128(&mut self) -> Result<(), Trap> { - let b_hi = i64::stack_pop(&mut self.store.value_stack); - let b_lo = i64::stack_pop(&mut self.store.value_stack); - let a_hi = i64::stack_pop(&mut self.store.value_stack); - let a_lo = i64::stack_pop(&mut self.store.value_stack); + let b_hi = i64::stack_pop(&mut self.value_stack); + let b_lo = i64::stack_pop(&mut self.value_stack); + let a_hi = i64::stack_pop(&mut self.value_stack); + let a_lo = i64::stack_pop(&mut self.value_stack); let lo = a_lo.wrapping_sub(b_lo); let borrow = u64::from((a_lo as u64) < (b_lo as u64)); let hi = a_hi.wrapping_sub(b_hi).wrapping_sub(borrow as i64); - i64::stack_push(&mut self.store.value_stack, lo); - i64::stack_push(&mut self.store.value_stack, hi); + i64::stack_push(&mut self.value_stack, lo); + i64::stack_push(&mut self.value_stack, hi); Ok(()) } fn exec_i64_mul_wide_s(&mut self) -> Result<(), Trap> { - let rhs = i64::stack_pop(&mut self.store.value_stack); - let lhs = i64::stack_pop(&mut self.store.value_stack); + let rhs = i64::stack_pop(&mut self.value_stack); + let lhs = i64::stack_pop(&mut self.value_stack); let product = (lhs as i128).wrapping_mul(rhs as i128); - i64::stack_push(&mut self.store.value_stack, product as i64); - i64::stack_push(&mut self.store.value_stack, (product >> 64) as i64); + i64::stack_push(&mut self.value_stack, product as i64); + i64::stack_push(&mut self.value_stack, (product >> 64) as i64); Ok(()) } fn exec_i64_mul_wide_u(&mut self) -> Result<(), Trap> { - let rhs = i64::stack_pop(&mut self.store.value_stack); - let lhs = i64::stack_pop(&mut self.store.value_stack); + let rhs = i64::stack_pop(&mut self.value_stack); + let lhs = i64::stack_pop(&mut self.value_stack); let product = (lhs as u64 as u128).wrapping_mul(rhs as u64 as u128); - i64::stack_push(&mut self.store.value_stack, product as u64 as i64); - i64::stack_push(&mut self.store.value_stack, (product >> 64) as u64 as i64); + i64::stack_push(&mut self.value_stack, product as u64 as i64); + i64::stack_push(&mut self.value_stack, (product >> 64) as u64 as i64); Ok(()) } @@ -342,8 +369,8 @@ impl<'store> Executor<'store> { lane: u8, operation: impl FnOnce(Value128, u8) -> TO, ) -> Result<(), Trap> { - let vector = Value128::stack_pop(&mut self.store.value_stack); - TO::stack_push(&mut self.store.value_stack, operation(vector, lane)); + let vector = Value128::stack_pop(&mut self.value_stack); + TO::stack_push(&mut self.value_stack, operation(vector, lane)); Ok(()) } @@ -353,50 +380,50 @@ impl<'store> Executor<'store> { lane: u8, operation: impl FnOnce(VALUE, Value128, u8) -> Value128, ) -> Result<(), Trap> { - let vector = Value128::stack_pop(&mut self.store.value_stack); - let value = VALUE::stack_pop(&mut self.store.value_stack); - Value128::stack_push(&mut self.store.value_stack, operation(value, vector, lane)); + let vector = Value128::stack_pop(&mut self.value_stack); + let value = VALUE::stack_pop(&mut self.value_stack); + Value128::stack_push(&mut self.value_stack, operation(value, vector, lane)); Ok(()) } fn exec_simd_shuffle(&mut self, lanes: Value128) -> Result<(), Trap> { - let rhs = Value128::stack_pop(&mut self.store.value_stack); - let lhs = Value128::stack_pop(&mut self.store.value_stack); - Value128::stack_push(&mut self.store.value_stack, Value128::i8x16_shuffle(lhs, rhs, lanes)); + let rhs = Value128::stack_pop(&mut self.value_stack); + let lhs = Value128::stack_pop(&mut self.value_stack); + Value128::stack_push(&mut self.value_stack, Value128::i8x16_shuffle(lhs, rhs, lanes)); Ok(()) } #[inline(always)] fn exec_local_set_pop(&mut self, local: LocalAddr) { - let value = T::stack_pop(&mut self.store.value_stack); - T::local_set(&mut self.store.value_stack, &self.cf, local, value); + let value = T::stack_pop(&mut self.value_stack); + T::local_set(&mut self.value_stack, &self.cf, local, value); } #[inline(always)] fn exec_local_tee(&mut self, local: LocalAddr) { - let value = T::stack_peek(&self.store.value_stack); - T::local_set(&mut self.store.value_stack, &self.cf, local, value); + let value = T::stack_peek(&self.value_stack); + T::local_set(&mut self.value_stack, &self.cf, local, value); } #[inline(always)] fn exec_global_get(&mut self, global: GlobalAddr) -> Result<(), Trap> { let addr = self.module.resolve_global_addr(global); let value = T::global_get(&self.store.state.globals, addr); - T::stack_push(&mut self.store.value_stack, value); + T::stack_push(&mut self.value_stack, value); Ok(()) } #[inline(always)] fn exec_global_set(&mut self, global: GlobalAddr) { let addr = self.module.resolve_global_addr(global); - let value = T::stack_pop(&mut self.store.value_stack); + let value = T::stack_pop(&mut self.value_stack); T::global_set(&mut self.store.state.globals, addr, value); } #[inline(always)] fn exec_global_tee(&mut self, global: GlobalAddr) { let addr = self.module.resolve_global_addr(global); - let value = T::stack_peek(&self.store.value_stack); + let value = T::stack_peek(&self.value_stack); T::global_set(&mut self.store.state.globals, addr, value); } @@ -407,10 +434,10 @@ impl<'store> Executor<'store> { val: T, ) -> Result<(), Trap> { if let Some(dst) = dst { - T::local_set(&mut self.store.value_stack, &self.cf, dst, val); + T::local_set(&mut self.value_stack, &self.cf, dst, val); } if PUSH { - T::stack_push(&mut self.store.value_stack, val); + T::stack_push(&mut self.value_stack, val); } Ok(()) } @@ -423,8 +450,8 @@ impl<'store> Executor<'store> { destination: Option, op: impl BinOpExt, ) -> Result<(), Trap> { - let lhs = T::local_get(&self.store.value_stack, &self.cf, lhs); - let rhs = T::local_get(&self.store.value_stack, &self.cf, rhs); + let lhs = T::local_get(&self.value_stack, &self.cf, lhs); + let rhs = T::local_get(&self.value_stack, &self.cf, rhs); self.exec_binop_result::(destination, op.exec(lhs, rhs)) } @@ -446,7 +473,7 @@ impl<'store> Executor<'store> { destination: Option, op: impl BinOpExt, ) -> Result<(), Trap> { - let lhs = T::local_get(&self.store.value_stack, &self.cf, lhs); + let lhs = T::local_get(&self.value_stack, &self.cf, lhs); self.exec_binop_result::(destination, op.exec(lhs, rhs)) } @@ -458,7 +485,7 @@ impl<'store> Executor<'store> { op: impl BinOpExt, ) -> Result<(), Trap> { let lhs = T::global_get(&self.store.state.globals, self.module.resolve_global_addr(lhs)); - T::stack_push(&mut self.store.value_stack, op.exec(lhs, rhs)); + T::stack_push(&mut self.value_stack, op.exec(lhs, rhs)); Ok(()) } @@ -468,9 +495,9 @@ impl<'store> Executor<'store> { T: InternalValue, CmpOp: CmpOpExt, { - let lhs = T::local_get(&self.store.value_stack, &self.cf, lhs); - let rhs = T::local_get(&self.store.value_stack, &self.cf, rhs); - i32::stack_push(&mut self.store.value_stack, i32::from(op.cmp(lhs, rhs))); + let lhs = T::local_get(&self.value_stack, &self.cf, lhs); + let rhs = T::local_get(&self.value_stack, &self.cf, rhs); + i32::stack_push(&mut self.value_stack, i32::from(op.cmp(lhs, rhs))); Ok(()) } @@ -482,7 +509,7 @@ impl<'store> Executor<'store> { ) -> Result<(), Trap> { let addr = self.module.resolve_global_addr(global); let rhs = T::global_get(&self.store.state.globals, addr); - T::stack_update(&mut self.store.value_stack, |lhs| op.exec(lhs, rhs)); + T::stack_update(&mut self.value_stack, |lhs| op.exec(lhs, rhs)); Ok(()) } @@ -493,8 +520,8 @@ impl<'store> Executor<'store> { destination: Option, op: impl BinOpExt, ) -> Result<(), Trap> { - let rhs = T::local_get(&self.store.value_stack, &self.cf, local); - let lhs = T::stack_pop(&mut self.store.value_stack); + let rhs = T::local_get(&self.value_stack, &self.cf, local); + let lhs = T::stack_pop(&mut self.value_stack); self.exec_binop_result::(destination, op.exec(lhs, rhs)) } @@ -505,8 +532,8 @@ impl<'store> Executor<'store> { destination: LocalAddr, op: impl BinOpExt, ) -> Result<(), Trap> { - let value = T::stack_update(&mut self.store.value_stack, |lhs| op.exec(lhs, rhs)); - T::local_set(&mut self.store.value_stack, &self.cf, destination, value); + let value = T::stack_update(&mut self.value_stack, |lhs| op.exec(lhs, rhs)); + T::local_set(&mut self.value_stack, &self.cf, destination, value); Ok(()) } @@ -517,15 +544,15 @@ impl<'store> Executor<'store> { multiply: fn(T, T) -> T, add: fn(T, T) -> T, ) { - let rhs = T::stack_pop(&mut self.store.value_stack); - let lhs = T::stack_pop(&mut self.store.value_stack); + let rhs = T::stack_pop(&mut self.value_stack); + let lhs = T::stack_pop(&mut self.value_stack); let product = multiply(lhs, rhs); - T::local_update(&mut self.store.value_stack, &self.cf, accumulator, |value| add(product, value)); + T::local_update(&mut self.value_stack, &self.cf, accumulator, |value| add(product, value)); } fn exec_branch_table(&mut self, index: Operand128Idx) -> usize { let v = index.resolve(&self.func.data); - let idx = ::stack_pop(&mut self.store.value_stack); + let idx = ::stack_pop(&mut self.value_stack); let target_ip = if idx >= 0 && (idx as u32) < v.size() { self.func.data.branch_table_targets.get((v.start() + idx as u32) as usize).copied().unwrap_or(v.target()) } else { @@ -542,7 +569,7 @@ impl<'store> Executor<'store> { self.store.state.gc.check_allocation(payload_len, true)?; let mut payload = Vec::new(); cold_err!(payload.try_reserve_exact(payload_len)).map_err(|_| Trap::OutOfMemory)?; - let value_stack = &mut self.store.value_stack; + let value_stack = &mut self.value_stack; for index in (0..payload_len).rev() { let ty = self.store.state.get_canonical_func_type(type_addr).params()[index]; payload.push(match ty { @@ -553,7 +580,7 @@ impl<'store> Executor<'store> { }); } payload.reverse(); - let roots = (&self.store.value_stack.stack_32).into_iter().copied().map(ValueRef::from_raw); + let roots = (&self.value_stack.stack_32).into_iter().copied().map(ValueRef::from_raw); self.store.state.alloc_exception(tag_addr, payload, roots) } @@ -563,7 +590,7 @@ impl<'store> Executor<'store> { } fn exec_throw_ref(&mut self, instr_ptr: usize) -> ExecResult { - let exception = ValueRef::stack_pop(&mut self.store.value_stack); + let exception = ValueRef::stack_pop(&mut self.value_stack); if exception.is_null() { return Err(Trap::NullReference.into()); } @@ -625,16 +652,15 @@ impl<'store> Executor<'store> { s64: stack_base.s64 + base.c64 as u32, s128: stack_base.s128 + base.c128 as u32, }; - self.store.value_stack.truncate_to_base(target); + self.value_stack.truncate_to_base(target); if include_payload { - let Store { state, value_stack, .. } = self.store; - let object = state.gc.get(exception).ok_or(Trap::InvalidReference)?; + let object = self.store.state.gc.get(exception).ok_or(Trap::InvalidReference)?; for value in object.values.iter().copied() { - value_stack.push_reserved(value); + self.value_stack.push_reserved(value); } } if with_ref { - ValueRef::stack_push(&mut self.store.value_stack, exception); + ValueRef::stack_push(&mut self.value_stack, exception); } return Ok(Some(if switched { ExecFlow::Switch(landing_pad as usize) @@ -643,7 +669,7 @@ impl<'store> Executor<'store> { })); } - self.store.value_stack.truncate_to_base(self.cf.locals_base); + self.value_stack.truncate_to_base(self.cf.locals_base); let Some(caller) = self.store.call_stack.pop_frame(self.call_stack_base) else { return Ok(None); }; @@ -659,8 +685,9 @@ impl<'store> Executor<'store> { type_addr: TypeAddr, return_instr_ptr: usize, ) -> ExecResult { + let module_id = self.module.id(); if let Some(host_func) = host_func.typed_callback() { - cold_err!(host_func.call_stack(self.store, self.module.id(), type_addr)) + cold_err!(self.with_store_value_stack(|store| host_func.call_stack(store, module_id, type_addr))) .map_err(|error| Trap::HostFunction(Box::new(error)))?; if TAIL { return Ok(self.exec_return()); @@ -670,11 +697,10 @@ impl<'store> Executor<'store> { let (param_count, result_count, base) = { let ty = self.store.state.get_canonical_func_type(type_addr); - (ty.params().len(), ty.results().len(), self.store.value_stack.base_before(ty.params().iter().collect())) + (ty.params().len(), ty.results().len(), self.value_stack.base_before(ty.params().iter().collect())) }; - let module_id = self.module.id(); - self.store - .with_scratch_values(param_count + result_count, |store, values| { + self.with_store_value_stack(|store| { + store.with_scratch_values(param_count + result_count, |store, values| { let host_values = store.stack_value_iter(type_addr, crate::store::FuncValueTypes::Params, base)?; for (slot, value) in values[..param_count].iter_mut().zip(host_values) { *slot = value; @@ -686,10 +712,11 @@ impl<'store> Executor<'store> { .map_err(|error| Error::Trap(Trap::HostFunction(Box::new(error))))?; store.push_wasm_values(results) }) - .map_err(|error| match error { - Error::Trap(trap) => trap, - other => Trap::HostFunction(Box::new(other)), - })?; + }) + .map_err(|error| match error { + Error::Trap(trap) => trap, + other => Trap::HostFunction(Box::new(other)), + })?; if TAIL { Ok(self.exec_return()) } else { Ok(ExecFlow::Next(return_instr_ptr)) } } @@ -721,8 +748,7 @@ impl<'store> Executor<'store> { fn exec_call_self(&mut self, return_instr_ptr: usize) -> ExecResult<()> { self.charge_call_fuel(FUEL_COST_CALL_TOTAL); - let locals_base = - self.store.value_stack.enter_locals(&self.func.params, &self.func.locals, &self.func.max_stack)?; + let locals_base = self.value_stack.enter_locals(&self.func.params, &self.func.locals, &self.func.max_stack)?; let new = CallFrame::new(self.cf.func_addr, locals_base, self.func.locals); self.store.call_stack.push(core::mem::replace(&mut self.cf, new), return_instr_ptr)?; Ok(()) @@ -731,9 +757,8 @@ impl<'store> Executor<'store> { fn exec_return_call_self(&mut self) -> ExecResult<()> { self.charge_call_fuel(FUEL_COST_CALL_TOTAL); - self.store.value_stack.truncate_keep_counts(self.cf.locals_base, self.func.params); - let locals_base = - self.store.value_stack.enter_locals(&self.func.params, &self.func.locals, &self.func.max_stack)?; + self.value_stack.truncate_keep_counts(self.cf.locals_base, self.func.params); + let locals_base = self.value_stack.enter_locals(&self.func.params, &self.func.locals, &self.func.max_stack)?; self.cf = CallFrame::new(self.cf.func_addr, locals_base, self.func.locals); Ok(()) } @@ -799,9 +824,9 @@ impl<'store> Executor<'store> { (wasm_func.func.params, wasm_func.func.locals, wasm_func.func.max_stack, wasm_func.owner, next_func) }; if TAIL { - self.store.value_stack.truncate_keep_counts(self.cf.locals_base, params); + self.value_stack.truncate_keep_counts(self.cf.locals_base, params); } - let locals_base = self.store.value_stack.enter_locals(¶ms, &locals, &max_stack)?; + let locals_base = self.value_stack.enter_locals(¶ms, &locals, &max_stack)?; if TAIL { self.cf = CallFrame::new(func_addr, locals_base, locals); } else { @@ -820,7 +845,7 @@ impl<'store> Executor<'store> { fn exec_call_ref(&mut self, type_addr: u32, return_instr_ptr: usize) -> ExecResult { self.charge_call_fuel(FUEL_COST_CALL_TOTAL); - let func_ref = ValueRef::stack_pop(&mut self.store.value_stack); + let func_ref = ValueRef::stack_pop(&mut self.value_stack); let Some(func_addr) = func_ref.addr() else { return cold!(Err(Trap::NullFunctionReference.into())); }; @@ -829,7 +854,7 @@ impl<'store> Executor<'store> { } fn exec_return(&mut self) -> ExecFlow { - self.store.value_stack.truncate_keep_counts(self.cf.locals_base, self.func.results); + self.value_stack.truncate_keep_counts(self.cf.locals_base, self.func.results); self.finish_return() } @@ -849,28 +874,28 @@ impl<'store> Executor<'store> { } fn exec_return_void(&mut self) -> ExecFlow { - self.store.value_stack.truncate_to_base(self.cf.locals_base); + self.value_stack.truncate_to_base(self.cf.locals_base); self.finish_return() } fn exec_return_32(&mut self) -> ExecFlow { - self.store.value_stack.stack_32.truncate_to_one_tail(self.cf.locals_base.s32 as usize); - self.store.value_stack.stack_64.truncate_to(self.cf.locals_base.s64 as usize); - self.store.value_stack.stack_128.truncate_to(self.cf.locals_base.s128 as usize); + self.value_stack.stack_32.truncate_to_one_tail(self.cf.locals_base.s32 as usize); + self.value_stack.stack_64.truncate_to(self.cf.locals_base.s64 as usize); + self.value_stack.stack_128.truncate_to(self.cf.locals_base.s128 as usize); self.finish_return() } fn exec_return_64(&mut self) -> ExecFlow { - self.store.value_stack.stack_32.truncate_to(self.cf.locals_base.s32 as usize); - self.store.value_stack.stack_64.truncate_to_one_tail(self.cf.locals_base.s64 as usize); - self.store.value_stack.stack_128.truncate_to(self.cf.locals_base.s128 as usize); + self.value_stack.stack_32.truncate_to(self.cf.locals_base.s32 as usize); + self.value_stack.stack_64.truncate_to_one_tail(self.cf.locals_base.s64 as usize); + self.value_stack.stack_128.truncate_to(self.cf.locals_base.s128 as usize); self.finish_return() } fn exec_return_128(&mut self) -> ExecFlow { - self.store.value_stack.stack_32.truncate_to(self.cf.locals_base.s32 as usize); - self.store.value_stack.stack_64.truncate_to(self.cf.locals_base.s64 as usize); - self.store.value_stack.stack_128.truncate_to_one_tail(self.cf.locals_base.s128 as usize); + self.value_stack.stack_32.truncate_to(self.cf.locals_base.s32 as usize); + self.value_stack.stack_64.truncate_to(self.cf.locals_base.s64 as usize); + self.value_stack.stack_128.truncate_to_one_tail(self.cf.locals_base.s128 as usize); self.finish_return() } @@ -882,18 +907,18 @@ impl<'store> Executor<'store> { value_local: u8, ) -> Result<(), Trap> { let memarg = index.resolve(&self.func.data); - let value = T::local_get(&self.store.value_stack, &self.cf, u16::from(value_local)); + let value = T::local_get(&self.value_stack, &self.cf, u16::from(value_local)); let mem_addr = self.mem_addr(MemAddr::from(memarg.memory())); crate::store::with_memory!(self.store.state, mem_addr, |mem, kind| { let base = if kind.arch() == MemoryArch::I64 { - let base = u64::local_get(&self.store.value_stack, &self.cf, u16::from(addr_local)); + let base = u64::local_get(&self.value_stack, &self.cf, u16::from(addr_local)); cold_err!(usize::try_from(base).map_err(|_| Trap::MemoryOutOfBounds { offset: usize::MAX, len: N, max: mem.len(), }))? } else { - u32::local_get(&self.store.value_stack, &self.cf, u16::from(addr_local)) as usize + u32::local_get(&self.value_stack, &self.cf, u16::from(addr_local)) as usize }; let addr = cold_err!(mem.effective_addr::(base, u64::from(memarg.offset())))?; value.store_at(&mut *mem, addr) @@ -911,14 +936,14 @@ impl<'store> Executor<'store> { let mem_addr = self.mem_addr(MemAddr::from(memarg.memory())); crate::store::with_memory!(self.store.state, mem_addr, |mem, kind| { let base = if kind.arch() == MemoryArch::I64 { - let base = i64::local_get(&self.store.value_stack, &self.cf, u16::from(addr_local)) as u64; + let base = i64::local_get(&self.value_stack, &self.cf, u16::from(addr_local)) as u64; cold_err!(usize::try_from(base).map_err(|_| Trap::MemoryOutOfBounds { offset: usize::MAX, len: N, max: mem.len(), }))? } else { - u32::local_get(&self.store.value_stack, &self.cf, u16::from(addr_local)) as usize + u32::local_get(&self.value_stack, &self.cf, u16::from(addr_local)) as usize }; let addr = cold_err!(mem.effective_addr::(base, u64::from(memarg.offset())))?; @@ -935,13 +960,13 @@ impl<'store> Executor<'store> { &mut self, m: CompactMemoryArg, ) -> Result<(), Trap> { - let rhs = T::stack_pop(&mut self.store.value_stack); - let lhs = T::stack_pop(&mut self.store.value_stack); - let acc = T::stack_pop(&mut self.store.value_stack); + let rhs = T::stack_pop(&mut self.value_stack); + let lhs = T::stack_pop(&mut self.value_stack); + let acc = T::stack_pop(&mut self.value_stack); let fma = acc + lhs * rhs; let mem_addr = self.mem_addr(m.mem_addr()); crate::store::with_memory!(self.store.state, mem_addr, |mem, kind| { - let base = self.store.value_stack.pop_memory_operand(kind.arch())?; + let base = self.value_stack.pop_memory_operand(kind.arch())?; let addr = cold_err!(mem.effective_addr::(base, m.offset()))?; cold_err!(fma.store_at(&mut *mem, addr)) }) @@ -966,37 +991,37 @@ impl<'store> Executor<'store> { let mem_addr = self.mem_addr(MemAddr::from(memarg.memory())); crate::store::with_memory!(self.store.state, mem_addr, |mem, kind| { let base = if kind.arch() == MemoryArch::I64 { - let base = i64::local_get(&self.store.value_stack, &self.cf, u16::from(addr_local)) as u64; + let base = i64::local_get(&self.value_stack, &self.cf, u16::from(addr_local)) as u64; cold_err!(usize::try_from(base)).map_err(|_| Trap::MemoryOutOfBounds { offset: usize::MAX, len: N, max: mem.len(), })? } else { - u32::local_get(&self.store.value_stack, &self.cf, u16::from(addr_local)) as usize + u32::local_get(&self.value_stack, &self.cf, u16::from(addr_local)) as usize }; let addr = cold_err!(mem.effective_addr::(base, u64::from(memarg.offset())))?; let value = cast(cold_err!(LOAD::load_at(&*mem, addr))?); if SET_LOCAL { - TARGET::local_set(&mut self.store.value_stack, &self.cf, u16::from(dst_local), value); + TARGET::local_set(&mut self.value_stack, &self.cf, u16::from(dst_local), value); } if !SET_LOCAL || TEE { - TARGET::stack_push(&mut self.store.value_stack, value); + TARGET::stack_push(&mut self.value_stack, value); } Ok(()) }) } fn exec_ref_is_null(&mut self) -> Result<(), Trap> { - let is_null = i32::from(ValueRef::stack_pop(&mut self.store.value_stack).is_null()); - i32::stack_push(&mut self.store.value_stack, is_null); + let is_null = i32::from(ValueRef::stack_pop(&mut self.value_stack).is_null()); + i32::stack_push(&mut self.value_stack, is_null); Ok(()) } fn exec_ref_as_non_null(&mut self) -> Result<(), Trap> { - if ValueRef::stack_peek(&self.store.value_stack).is_null() { + if ValueRef::stack_peek(&self.value_stack).is_null() { return cold!(Err(Trap::NullReference)); } Ok(()) @@ -1008,14 +1033,14 @@ impl<'store> Executor<'store> { } fn exec_ref_matches(&self, ty: RefType) -> bool { - let value = ValueRef::stack_peek(&self.store.value_stack); + let value = ValueRef::stack_peek(&self.value_stack); self.store.state.value_ref_matches(value, self.canonical_ref_type(ty)) } fn exec_ref_test(&mut self, ty: RefType) -> Result<(), Trap> { - let value = ValueRef::stack_pop(&mut self.store.value_stack); + let value = ValueRef::stack_pop(&mut self.value_stack); let matches = self.store.state.value_ref_matches(value, self.canonical_ref_type(ty)); - i32::stack_push(&mut self.store.value_stack, i32::from(matches)); + i32::stack_push(&mut self.value_stack, i32::from(matches)); Ok(()) } @@ -1027,7 +1052,7 @@ impl<'store> Executor<'store> { } fn exec_i31_get(&mut self, signed: bool) -> Result<(), Trap> { - let value = ValueRef::stack_pop(&mut self.store.value_stack); + let value = ValueRef::stack_pop(&mut self.value_stack); if value.is_null() { return cold!(Err(Trap::NullI31Reference)); } @@ -1036,14 +1061,14 @@ impl<'store> Executor<'store> { } else { value.i31_u().expect("validated i31.get operand") as i32 }; - i32::stack_push(&mut self.store.value_stack, value); + i32::stack_push(&mut self.value_stack, value); Ok(()) } fn push_gc_object(&mut self, type_addr: TypeAddr, values: Vec) -> Result<(), Trap> { - let roots = (&self.store.value_stack.stack_32).into_iter().copied().map(ValueRef::from_raw); + let roots = (&self.value_stack.stack_32).into_iter().copied().map(ValueRef::from_raw); let reference = self.store.state.alloc_gc_object(type_addr, values, roots)?; - ValueRef::stack_push(&mut self.store.value_stack, reference); + ValueRef::stack_push(&mut self.value_stack, reference); Ok(()) } @@ -1068,7 +1093,7 @@ impl<'store> Executor<'store> { } else { for index in (0..field_count).rev() { let storage = self.store.state.get_type(type_addr).as_struct().unwrap().fields[index].storage; - values.push(pop_value(&mut self.store.value_stack, storage)); + values.push(pop_value(&mut self.value_stack, storage)); } values.reverse(); } @@ -1079,7 +1104,7 @@ impl<'store> Executor<'store> { let operand = index.resolve(&self.func.data); let type_index = operand.a(); let field_index = operand.b(); - let reference = ValueRef::stack_pop(&mut self.store.value_stack); + let reference = ValueRef::stack_pop(&mut self.value_stack); let type_addr = self.module.resolve_type_addr(type_index); let storage = self.store.state.get_type(type_addr).as_struct().expect("validated struct.get type").fields [field_index as usize] @@ -1087,7 +1112,7 @@ impl<'store> Executor<'store> { let object = self.store.state.gc_object(reference, type_addr)?; let object = self.store.state.gc.get_handle(object).ok_or(Trap::Other("invalid GC reference"))?; let value = *object.values.get(field_index as usize).expect("validated struct field index"); - push_value(&mut self.store.value_stack, value, storage, signed); + push_value(&mut self.value_stack, value, storage, signed); Ok(()) } @@ -1099,8 +1124,8 @@ impl<'store> Executor<'store> { let storage = self.store.state.get_type(type_addr).as_struct().expect("validated struct.set type").fields [field_index as usize] .storage; - let value = pop_value(&mut self.store.value_stack, storage); - let reference = ValueRef::stack_pop(&mut self.store.value_stack); + let value = pop_value(&mut self.value_stack, storage); + let reference = ValueRef::stack_pop(&mut self.value_stack); let object = self.store.state.gc_object(reference, type_addr)?; self.store.state.gc.set(object, field_index as usize, value).expect("live struct field"); Ok(()) @@ -1109,9 +1134,9 @@ impl<'store> Executor<'store> { fn exec_array_new(&mut self, type_index: TypeAddr, default: bool) -> Result<(), Trap> { let type_addr = self.module.resolve_type_addr(type_index); let storage = self.store.state.get_type(type_addr).as_array().expect("validated array.new type").field.storage; - let len = u32::stack_pop(&mut self.store.value_stack) as usize; + let len = u32::stack_pop(&mut self.value_stack) as usize; self.store.state.gc.check_allocation(len, self.store.state.gc_type_has_references(type_addr))?; - let value = if default { default_value(storage) } else { pop_value(&mut self.store.value_stack, storage) }; + let value = if default { default_value(storage) } else { pop_value(&mut self.value_stack, storage) }; let mut values = Vec::new(); cold_err!(values.try_reserve_exact(len)).map_err(|_| Trap::OutOfMemory)?; values.resize(len, value); @@ -1129,36 +1154,36 @@ impl<'store> Executor<'store> { let mut values = Vec::new(); cold_err!(values.try_reserve_exact(len)).map_err(|_| Trap::OutOfMemory)?; for _ in 0..len { - values.push(pop_value(&mut self.store.value_stack, arr_type.field.storage)); + values.push(pop_value(&mut self.value_stack, arr_type.field.storage)); } values.reverse(); self.push_gc_object(type_addr, values) } fn exec_array_get(&mut self, type_index: TypeAddr, signed: Option) -> Result<(), Trap> { - let index = u32::stack_pop(&mut self.store.value_stack) as usize; - let reference = ValueRef::stack_pop(&mut self.store.value_stack); + let index = u32::stack_pop(&mut self.value_stack) as usize; + let reference = ValueRef::stack_pop(&mut self.value_stack); let type_addr = self.module.resolve_type_addr(type_index); let storage = self.store.state.get_type(type_addr).as_array().expect("validated array.get type").field.storage; let object = self.store.state.gc_object(reference, type_addr)?; let object = self.store.state.gc.get_handle(object).ok_or(Trap::Other("invalid GC reference"))?; let value = *object.values.get(index).ok_or(Trap::ArrayOutOfBounds)?; - push_value(&mut self.store.value_stack, value, storage, signed); + push_value(&mut self.value_stack, value, storage, signed); Ok(()) } fn exec_array_set(&mut self, type_index: TypeAddr) -> Result<(), Trap> { let type_addr = self.module.resolve_type_addr(type_index); let storage = self.store.state.get_type(type_addr).as_array().expect("validated array.set type").field.storage; - let value = pop_value(&mut self.store.value_stack, storage); - let index = u32::stack_pop(&mut self.store.value_stack) as usize; - let reference = ValueRef::stack_pop(&mut self.store.value_stack); + let value = pop_value(&mut self.value_stack, storage); + let index = u32::stack_pop(&mut self.value_stack) as usize; + let reference = ValueRef::stack_pop(&mut self.value_stack); let object = self.store.state.gc_object(reference, type_addr)?; self.store.state.gc.set(object, index, value).ok_or(Trap::ArrayOutOfBounds) } fn exec_array_len(&mut self) -> Result<(), Trap> { - let reference = ValueRef::stack_pop(&mut self.store.value_stack); + let reference = ValueRef::stack_pop(&mut self.value_stack); if reference.is_null() { return Err(Trap::NullArrayReference); } @@ -1169,17 +1194,17 @@ impl<'store> Executor<'store> { if self.store.state.get_type(type_addr).as_array().is_none() { return Err(Trap::Other("GC reference is not an array")); } - i32::stack_push(&mut self.store.value_stack, object.values.len() as i32); + i32::stack_push(&mut self.value_stack, object.values.len() as i32); Ok(()) } fn exec_array_fill(&mut self, type_index: TypeAddr) -> Result<(), Trap> { let type_addr = self.module.resolve_type_addr(type_index); let storage = self.store.state.get_type(type_addr).as_array().expect("validated array.fill type").field.storage; - let len = u32::stack_pop(&mut self.store.value_stack) as usize; - let value = pop_value(&mut self.store.value_stack, storage); - let index = u32::stack_pop(&mut self.store.value_stack) as usize; - let reference = ValueRef::stack_pop(&mut self.store.value_stack); + let len = u32::stack_pop(&mut self.value_stack) as usize; + let value = pop_value(&mut self.value_stack, storage); + let index = u32::stack_pop(&mut self.value_stack) as usize; + let reference = ValueRef::stack_pop(&mut self.value_stack); let object = self.store.state.gc_object(reference, type_addr)?; let object_ref = self.store.state.gc.get_handle(object).ok_or(Trap::Other("invalid GC reference"))?; let end = index.checked_add(len).filter(|end| *end <= object_ref.values.len()).ok_or(Trap::ArrayOutOfBounds)?; @@ -1191,11 +1216,11 @@ impl<'store> Executor<'store> { let operand = index.resolve(&self.func.data); let dst_type = operand.a(); let src_type = operand.b(); - let len = u32::stack_pop(&mut self.store.value_stack) as usize; - let src_index = u32::stack_pop(&mut self.store.value_stack) as usize; - let src = ValueRef::stack_pop(&mut self.store.value_stack); - let dst_index = u32::stack_pop(&mut self.store.value_stack) as usize; - let dst = ValueRef::stack_pop(&mut self.store.value_stack); + let len = u32::stack_pop(&mut self.value_stack) as usize; + let src_index = u32::stack_pop(&mut self.value_stack) as usize; + let src = ValueRef::stack_pop(&mut self.value_stack); + let dst_index = u32::stack_pop(&mut self.value_stack) as usize; + let dst = ValueRef::stack_pop(&mut self.value_stack); let dst_type = self.module.resolve_type_addr(dst_type); let src_type = self.module.resolve_type_addr(src_type); let dst_handle = self.store.state.gc_object(dst, dst_type)?; @@ -1221,8 +1246,8 @@ impl<'store> Executor<'store> { let operand = index.resolve(&self.func.data); let type_index = operand.a(); let data_index = operand.b(); - let len = u32::stack_pop(&mut self.store.value_stack) as usize; - let src = u32::stack_pop(&mut self.store.value_stack) as usize; + let len = u32::stack_pop(&mut self.value_stack) as usize; + let src = u32::stack_pop(&mut self.value_stack) as usize; let type_addr = self.module.resolve_type_addr(type_index); let storage = self.store.state.get_type(type_addr).as_array().expect("validated array type").field.storage; let data_addr = self.module.resolve_data_addr(data_index); @@ -1237,8 +1262,8 @@ impl<'store> Executor<'store> { let operand = index.resolve(&self.func.data); let type_index = operand.a(); let elem_index = operand.b(); - let len = u32::stack_pop(&mut self.store.value_stack) as usize; - let src = u32::stack_pop(&mut self.store.value_stack) as usize; + let len = u32::stack_pop(&mut self.value_stack) as usize; + let src = u32::stack_pop(&mut self.value_stack) as usize; let type_addr = self.module.resolve_type_addr(type_index); let elem_addr = self.module.resolve_elem_addr(elem_index); let items = self.store.state.elements[elem_addr as usize].items_range(src, len)?; @@ -1253,10 +1278,10 @@ impl<'store> Executor<'store> { let operand = index.resolve(&self.func.data); let type_index = operand.a(); let data_index = operand.b(); - let len = u32::stack_pop(&mut self.store.value_stack) as usize; - let src = u32::stack_pop(&mut self.store.value_stack) as usize; - let dst = u32::stack_pop(&mut self.store.value_stack) as usize; - let reference = ValueRef::stack_pop(&mut self.store.value_stack); + let len = u32::stack_pop(&mut self.value_stack) as usize; + let src = u32::stack_pop(&mut self.value_stack) as usize; + let dst = u32::stack_pop(&mut self.value_stack) as usize; + let reference = ValueRef::stack_pop(&mut self.value_stack); let type_addr = self.module.resolve_type_addr(type_index); let storage = self.store.state.get_type(type_addr).as_array().expect("validated array type").field.storage; let object = self.store.state.gc_object(reference, type_addr)?; @@ -1273,10 +1298,10 @@ impl<'store> Executor<'store> { let operand = index.resolve(&self.func.data); let type_index = operand.a(); let elem_index = operand.b(); - let len = u32::stack_pop(&mut self.store.value_stack) as usize; - let src = u32::stack_pop(&mut self.store.value_stack) as usize; - let dst = u32::stack_pop(&mut self.store.value_stack) as usize; - let reference = ValueRef::stack_pop(&mut self.store.value_stack); + let len = u32::stack_pop(&mut self.value_stack) as usize; + let src = u32::stack_pop(&mut self.value_stack) as usize; + let dst = u32::stack_pop(&mut self.value_stack) as usize; + let reference = ValueRef::stack_pop(&mut self.value_stack); let type_addr = self.module.resolve_type_addr(type_index); let object = self.store.state.gc_object(reference, type_addr)?; let object_len = self.store.state.gc.get_handle(object).expect("validated array").values.len(); @@ -1292,8 +1317,8 @@ impl<'store> Executor<'store> { fn exec_memory_size(&mut self, addr: u32) -> Result<(), Trap> { match self.store.state.memory_size(self.mem_addr(addr)) { - (MemoryArch::I64, pages) => i64::stack_push(&mut self.store.value_stack, pages as i64), - (MemoryArch::I32, pages) => i32::stack_push(&mut self.store.value_stack, pages as i32), + (MemoryArch::I64, pages) => i64::stack_push(&mut self.value_stack, pages as i64), + (MemoryArch::I32, pages) => i32::stack_push(&mut self.value_stack, pages as i32), } Ok(()) } @@ -1303,14 +1328,14 @@ impl<'store> Executor<'store> { let limiter = self.store.engine.config().resource_limiter.as_deref(); let is_64bit = self.store.state.memory_type(mem_addr).arch() == MemoryArch::I64; let pages_delta = match is_64bit { - true => i64::stack_pop(&mut self.store.value_stack), - false => i64::from(i32::stack_pop(&mut self.store.value_stack)), + true => i64::stack_pop(&mut self.value_stack), + false => i64::from(i32::stack_pop(&mut self.value_stack)), }; let size = self.store.state.grow_mem(mem_addr, pages_delta, limiter)?.unwrap_or(-1); match is_64bit { - true => i64::stack_push(&mut self.store.value_stack, size), - false => i32::stack_push(&mut self.store.value_stack, size as i32), + true => i64::stack_push(&mut self.value_stack, size), + false => i32::stack_push(&mut self.value_stack, size as i32), }; Ok(()) @@ -1326,9 +1351,9 @@ impl<'store> Executor<'store> { let src_arch = self.store.state.memory_type(src_mem_addr).arch(); let len_arch = if dst_arch == MemoryArch::I32 || src_arch == MemoryArch::I32 { MemoryArch::I32 } else { MemoryArch::I64 }; - let size = self.store.value_stack.pop_memory_operand(len_arch)?; - let src = self.store.value_stack.pop_memory_operand(src_arch)?; - let dst = self.store.value_stack.pop_memory_operand(dst_arch)?; + let size = self.value_stack.pop_memory_operand(len_arch)?; + let src = self.value_stack.pop_memory_operand(src_arch)?; + let dst = self.value_stack.pop_memory_operand(dst_arch)?; self.store.state.copy_memories(dst_mem_addr, dst, src_mem_addr, src, size) } @@ -1336,9 +1361,9 @@ impl<'store> Executor<'store> { fn exec_memory_fill(&mut self, addr: u32) -> Result<(), Trap> { let mem_addr = self.mem_addr(addr); let arch = self.store.state.memory_type(mem_addr).arch(); - let size = self.store.value_stack.pop_memory_operand(arch)?; - let val = i32::stack_pop(&mut self.store.value_stack); - let dst = self.store.value_stack.pop_memory_operand(arch)?; + let size = self.value_stack.pop_memory_operand(arch)?; + let val = i32::stack_pop(&mut self.value_stack); + let dst = self.value_stack.pop_memory_operand(arch)?; self.exec_memory_fill_impl(mem_addr, dst, val as u8, size) } @@ -1346,7 +1371,7 @@ impl<'store> Executor<'store> { let operand = index.resolve(&self.func.data); let mem_addr = self.mem_addr(operand.memory()); let arch = self.store.state.memory_type(mem_addr).arch(); - let dst = self.store.value_stack.pop_memory_operand(arch)?; + let dst = self.value_stack.pop_memory_operand(arch)?; self.exec_memory_fill_impl(mem_addr, dst, operand.byte(), operand.value() as u32 as usize) } @@ -1364,11 +1389,11 @@ impl<'store> Executor<'store> { let operand = index.resolve(&self.func.data); let data_index = operand.a(); let mem_index = operand.b(); - let size = u32::stack_pop(&mut self.store.value_stack) as usize; - let offset = u32::stack_pop(&mut self.store.value_stack) as usize; + let size = u32::stack_pop(&mut self.value_stack) as usize; + let offset = u32::stack_pop(&mut self.value_stack) as usize; let mem_addr = self.mem_addr(mem_index); let arch = self.store.state.memory_type(mem_addr).arch(); - let dst = self.store.value_stack.pop_memory_operand(arch)?; + let dst = self.value_stack.pop_memory_operand(arch)?; let data = self.store.state.data[self.module.resolve_data_addr(data_index) as usize].data.clone(); // Dropped segments behave like empty segments, including valid zero-length copies. @@ -1419,11 +1444,11 @@ impl<'store> Executor<'store> { let m = arg.memory_arg_idx.resolve(&self.func.data); let mem_addr = self.mem_addr(m.memory()); crate::store::with_memory!(self.store.state, mem_addr, |mem, kind| { - let base = self.store.value_stack.pop_memory_operand(kind.arch())?; + let base = self.value_stack.pop_memory_operand(kind.arch())?; let addr = cold_err!(mem.effective_addr::(base, m.offset()))?; let val = cold_err!(LOAD::load_at(&*mem, addr))?; let offset = arg.lane as usize * LOAD_SIZE; - Value128::stack_update(&mut self.store.value_stack, |value| { + Value128::stack_update(&mut self.value_stack, |value| { let mut bytes = value.to_mem_bytes(); bytes[offset..offset + LOAD_SIZE].copy_from_slice(&val.to_mem_bytes()); Value128(bytes) @@ -1440,10 +1465,10 @@ impl<'store> Executor<'store> { ) -> Result<(), Trap> { let mem_addr = self.mem_addr(m.memory()); crate::store::with_memory!(self.store.state, mem_addr, |mem, kind| { - let base = self.store.value_stack.pop_memory_operand(kind.arch())?; + let base = self.value_stack.pop_memory_operand(kind.arch())?; let addr = cold_err!(mem.effective_addr::(base, m.offset()))?; let value = cold_err!(LOAD::load_at(&*mem, addr))?; - TARGET::stack_push(&mut self.store.value_stack, cast(value)); + TARGET::stack_push(&mut self.value_stack, cast(value)); Ok(()) }) } @@ -1458,16 +1483,16 @@ impl<'store> Executor<'store> { } fn exec_atomic_wait(&mut self, arg: Operand128Idx, op: AtomicWaitOp) -> Result<(), Trap> { - let timeout = if op == AtomicWaitOp::Notify { 0 } else { i64::stack_pop(&mut self.store.value_stack) }; + let timeout = if op == AtomicWaitOp::Notify { 0 } else { i64::stack_pop(&mut self.value_stack) }; let value = match op { - AtomicWaitOp::Wait64 => u64::stack_pop(&mut self.store.value_stack), - _ => u32::stack_pop(&mut self.store.value_stack) as u64, + AtomicWaitOp::Wait64 => u64::stack_pop(&mut self.value_stack), + _ => u32::stack_pop(&mut self.value_stack) as u64, }; let memory = arg.resolve(&self.func.data); let mem_addr = self.mem_addr(memory.memory()); let width = if op == AtomicWaitOp::Wait64 { 8 } else { 4 }; let addr = crate::store::with_memory!(@lock_inline self.store.state, mem_addr, |mem, kind| { - let base = self.store.value_stack.pop_memory_operand(kind.arch())?; + let base = self.value_stack.pop_memory_operand(kind.arch())?; let addr = cold_err!(mem.effective_addr::<1>(base, memory.offset()))?; if addr % width != 0 { return cold!(Err(Trap::UnalignedAtomic)); @@ -1496,11 +1521,11 @@ impl<'store> Executor<'store> { AtomicWaitOp::Wait32 => shared.wait::<4>(addr, value, timeout)?, AtomicWaitOp::Wait64 => shared.wait::<8>(addr, value, timeout)?, }; - u32::stack_push(&mut self.store.value_stack, result); + u32::stack_push(&mut self.value_stack, result); return Ok(()); } - u32::stack_push(&mut self.store.value_stack, 0); + u32::stack_push(&mut self.value_stack, 0); Ok(()) } @@ -1508,20 +1533,20 @@ impl<'store> Executor<'store> { let op = arg.op(); let value = match op { AtomicOp::Load => 0, - _ if arg.is_64() => u64::stack_pop(&mut self.store.value_stack), - _ => u32::stack_pop(&mut self.store.value_stack) as u64, + _ if arg.is_64() => u64::stack_pop(&mut self.value_stack), + _ => u32::stack_pop(&mut self.value_stack) as u64, }; let expected = match op { - AtomicOp::Cmpxchg if arg.is_64() => u64::stack_pop(&mut self.store.value_stack), - AtomicOp::Cmpxchg => u32::stack_pop(&mut self.store.value_stack) as u64, + AtomicOp::Cmpxchg if arg.is_64() => u64::stack_pop(&mut self.value_stack), + AtomicOp::Cmpxchg => u32::stack_pop(&mut self.value_stack) as u64, _ => 0, }; let memory = arg.memory.resolve(&self.func.data); let mem_addr = self.mem_addr(memory.memory()); crate::store::with_memory!(@lock_inline self.store.state, mem_addr, |mem, kind| { - let base = self.store.value_stack.pop_memory_operand(kind.arch())?; + let base = self.value_stack.pop_memory_operand(kind.arch())?; let addr = cold_err!(mem.effective_addr::(base, memory.offset()))?; if addr % N != 0 { return cold!(Err(Trap::UnalignedAtomic)); @@ -1554,8 +1579,8 @@ impl<'store> Executor<'store> { } match arg.is_64() { - true => u64::stack_push(&mut self.store.value_stack, old), - false => u32::stack_push(&mut self.store.value_stack, old as u32), + true => u64::stack_push(&mut self.value_stack, old), + false => u32::stack_push(&mut self.value_stack, old as u32), } Ok(()) }) @@ -1563,7 +1588,7 @@ impl<'store> Executor<'store> { #[inline(always)] fn exec_mem_store_lane + Copy, const N: usize>(&mut self, arg: MemoryLaneArg) -> Result<(), Trap> { - let bytes = Value128::stack_pop(&mut self.store.value_stack).to_mem_bytes(); + let bytes = Value128::stack_pop(&mut self.value_stack).to_mem_bytes(); let lane_offset = arg.lane as usize * N; let mut val_bytes = [0u8; N]; val_bytes.copy_from_slice(&bytes[lane_offset..lane_offset + N]); @@ -1571,7 +1596,7 @@ impl<'store> Executor<'store> { let m = arg.memory_arg_idx.resolve(&self.func.data); let mem_addr = self.mem_addr(m.memory()); crate::store::with_memory!(self.store.state, mem_addr, |mem, kind| { - let base = self.store.value_stack.pop_memory_operand(kind.arch())?; + let base = self.value_stack.pop_memory_operand(kind.arch())?; let addr = cold_err!(mem.effective_addr::(base, m.offset()))?; cold_err!(val.store_at(&mut *mem, addr)) }) @@ -1583,7 +1608,7 @@ impl<'store> Executor<'store> { memory: Operand128, cast: impl Fn(T) -> U, ) -> Result<(), Trap> { - let val = cast(::stack_pop(&mut self.store.value_stack)); + let val = cast(::stack_pop(&mut self.value_stack)); self.exec_mem_store_value(self.mem_addr(memory.memory()), memory.offset(), val) } @@ -1592,9 +1617,9 @@ impl<'store> Executor<'store> { &mut self, memory: Operand128, ) -> Result<(), Trap> { - let condition = Value32::stack_pop(&mut self.store.value_stack); - let false_value = T::stack_pop(&mut self.store.value_stack); - let true_value = T::stack_pop(&mut self.store.value_stack); + let condition = Value32::stack_pop(&mut self.value_stack); + let false_value = T::stack_pop(&mut self.value_stack); + let true_value = T::stack_pop(&mut self.value_stack); let value = if condition == 0 { false_value } else { true_value }; self.exec_mem_store_value(self.mem_addr(memory.memory()), memory.offset(), value) } @@ -1607,7 +1632,7 @@ impl<'store> Executor<'store> { val: U, ) -> Result<(), Trap> { crate::store::with_memory!(self.store.state, memory_addr, |mem, kind| { - let base = self.store.value_stack.pop_memory_operand(kind.arch())?; + let base = self.value_stack.pop_memory_operand(kind.arch())?; let addr = cold_err!(mem.effective_addr::(base, offset))?; cold_err!(val.store_at(&mut *mem, addr)) }) @@ -1617,12 +1642,12 @@ impl<'store> Executor<'store> { let table_addr = self.module.resolve_table_addr(table_index); let idx = self.pop_table_operand(self.store.state.get_table(table_addr).kind.arch())?; let value = *self.store.state.get_table(table_addr).get(idx)?; - ValueRef::stack_push(&mut self.store.value_stack, value); + ValueRef::stack_push(&mut self.value_stack, value); Ok(()) } fn exec_table_set(&mut self, table_index: u32) -> Result<(), Trap> { - let val = ValueRef::stack_pop(&mut self.store.value_stack); + let val = ValueRef::stack_pop(&mut self.value_stack); let table_addr = self.module.resolve_table_addr(table_index); let idx = self.pop_table_operand(self.store.state.get_table(table_addr).kind.arch())?; let table = self.store.state.get_table_mut(table_addr); @@ -1632,8 +1657,8 @@ impl<'store> Executor<'store> { fn exec_table_size(&mut self, table_index: u32) -> Result<(), Trap> { let table = self.store.state.get_table(self.module.resolve_table_addr(table_index)); match table.kind.arch() { - MemoryArch::I32 => i32::stack_push(&mut self.store.value_stack, table.size() as i32), - MemoryArch::I64 => i64::stack_push(&mut self.store.value_stack, table.size() as i64), + MemoryArch::I32 => i32::stack_push(&mut self.value_stack, table.size() as i32), + MemoryArch::I64 => i64::stack_push(&mut self.value_stack, table.size() as i64), } Ok(()) } @@ -1656,16 +1681,16 @@ impl<'store> Executor<'store> { let table_addr = self.module.resolve_table_addr(table_index); let arch = self.store.state.get_table(table_addr).kind.arch(); let n = self.pop_table_operand(arch)?; - let val = ValueRef::stack_pop(&mut self.store.value_stack); + let val = ValueRef::stack_pop(&mut self.value_stack); let limiter = self.store.engine.config().resource_limiter.as_deref(); let table = self.store.state.get_table_mut(table_addr); let sz = table.size(); let grew = table.grow(n, val, limiter)?; match (arch, grew) { - (MemoryArch::I32, true) => i32::stack_push(&mut self.store.value_stack, sz as i32), - (MemoryArch::I32, false) => i32::stack_push(&mut self.store.value_stack, -1), - (MemoryArch::I64, true) => i64::stack_push(&mut self.store.value_stack, sz as i64), - (MemoryArch::I64, false) => i64::stack_push(&mut self.store.value_stack, -1), + (MemoryArch::I32, true) => i32::stack_push(&mut self.value_stack, sz as i32), + (MemoryArch::I32, false) => i32::stack_push(&mut self.value_stack, -1), + (MemoryArch::I64, true) => i64::stack_push(&mut self.value_stack, sz as i64), + (MemoryArch::I64, false) => i64::stack_push(&mut self.value_stack, -1), } Ok(()) } @@ -1674,15 +1699,15 @@ impl<'store> Executor<'store> { let table_addr = self.module.resolve_table_addr(table_index); let arch = self.store.state.get_table(table_addr).kind.arch(); let n = self.pop_table_operand(arch)?; - let val = ValueRef::stack_pop(&mut self.store.value_stack); + let val = ValueRef::stack_pop(&mut self.value_stack); let i = self.pop_table_operand(arch)?; self.store.state.get_table_mut(table_addr).fill(i, n, val) } fn pop_table_operand(&mut self, arch: MemoryArch) -> Result { let value = match arch { - MemoryArch::I32 => i32::stack_pop(&mut self.store.value_stack) as u32 as u64, - MemoryArch::I64 => i64::stack_pop(&mut self.store.value_stack) as u64, + MemoryArch::I32 => i32::stack_pop(&mut self.value_stack) as u32 as u64, + MemoryArch::I64 => i64::stack_pop(&mut self.value_stack) as u64, }; cold_err!(usize::try_from(value).map_err(|_| Trap::TableOutOfBounds { offset: usize::MAX, @@ -1691,3 +1716,9 @@ impl<'store> Executor<'store> { })) } } + +impl Drop for Executor<'_> { + fn drop(&mut self) { + self.store.value_stack = core::mem::take(&mut self.value_stack); + } +} diff --git a/crates/tinywasm/src/interpreter/num_helpers.rs b/crates/tinywasm/src/interpreter/num_helpers.rs index 53632bd..aa805a9 100644 --- a/crates/tinywasm/src/interpreter/num_helpers.rs +++ b/crates/tinywasm/src/interpreter/num_helpers.rs @@ -25,7 +25,7 @@ macro_rules! checked_conv_float { ($from:tt, $to:tt, $self:expr) => {{ checked_conv_float!($from, $to, $to, $self) }}; // Conversion with an intermediate unsigned type and error checking (three types) ($from:tt, $intermediate:tt, $to:tt, $self:expr) => {{ - let v = <$from>::stack_pop(&mut $self.store.value_stack); + let v = <$from>::stack_pop(&mut $self.value_stack); let (min, max) = float_min_max!($from, $intermediate); if v.is_nan() { core::hint::cold_path(); @@ -35,7 +35,7 @@ macro_rules! checked_conv_float { core::hint::cold_path(); return Err(crate::Trap::IntegerOverflow.into()); } - <$to>::stack_push(&mut $self.store.value_stack, (v as $intermediate as $to).into()); + <$to>::stack_push(&mut $self.value_stack, (v as $intermediate as $to).into()); }}; } diff --git a/crates/tinywasm/src/interpreter/stack/value_stack.rs b/crates/tinywasm/src/interpreter/stack/value_stack.rs index 386453b..6d33ddd 100644 --- a/crates/tinywasm/src/interpreter/stack/value_stack.rs +++ b/crates/tinywasm/src/interpreter/stack/value_stack.rs @@ -6,6 +6,7 @@ use crate::engine::{Config, StackConfig}; use crate::interpreter::*; use crate::{Result, Trap}; +#[derive(Default)] #[cfg_attr(feature = "debug", derive(Debug))] /// Physical value lanes used by the interpreter. /// @@ -35,6 +36,13 @@ pub(crate) struct Stack { /// The most slots [`Stack::enter_locals`] writes ahead of the pushes that reach them. const WRITTEN_RESERVATION: usize = 64; +/// An empty stack without an allocation: what a store holds while an executor runs. +impl Default for Stack { + fn default() -> Self { + Self { data: Vec::new(), len: 0, max_size: 0, dynamic: false } + } +} + impl Stack { pub(crate) fn new(config: StackConfig) -> Self { Self { From fc1f8174766592b799c2802e0c7375a2037c0a4e Mon Sep 17 00:00:00 2001 From: Matt Hargett Date: Tue, 29 Sep 2026 19:14:08 -0700 Subject: [PATCH 4/4] perf: pass the 32-bit value stack's height through the tail-call handlers Each tail-call handler receives the height of the 32-bit value stack, which most instructions push to or pop from, as its last argument. It writes the height to the stack before its body and reads it back before dispatching, and with the stack in the executor the compiler can keep it in a register in between, as it does in most handlers. A handler no longer loads the height its predecessor stored, a store-to-load round trip through memory on every dispatch. The Unbudgeted handlers now take six integer arguments and the Bounded ones five, which stay in registers on arm64 (eight) and x86-64 System V (six). Windows x64 passes four, so there the height goes on the stack along with the Instruction, which already does without this change. --- .../interpreter/executor/dispatch_become.rs | 49 ++++++++++++------- .../src/interpreter/stack/value_stack.rs | 10 ++++ 2 files changed, 42 insertions(+), 17 deletions(-) diff --git a/crates/tinywasm/src/interpreter/executor/dispatch_become.rs b/crates/tinywasm/src/interpreter/executor/dispatch_become.rs index 52e7d18..f62cbdc 100644 --- a/crates/tinywasm/src/interpreter/executor/dispatch_become.rs +++ b/crates/tinywasm/src/interpreter/executor/dispatch_become.rs @@ -3,11 +3,18 @@ use super::*; struct Unbudgeted; struct Bounded; +// The last argument of both handler types is the height of the 32-bit value stack, which most +// instructions push to or pop from. Each handler writes it back to the stack before its body runs +// and reads it again before dispatching, so no handler loads the height its predecessor stored. +// Most keep it in a register in between. A few load it back before dispatching: the 8- and 16-lane +// SIMD ops, and the fused `LoadLocal*` handlers, whose hot path joins a cold call. +// // Between two handlers only what the calling convention passes in registers stays out of memory. // On arm64_32 (watchOS) the Rust ABI passes an aggregate larger than a pointer, the 8-byte // `Instruction`, by reference, so every dispatch would store it and the next handler load it back. // The C convention passes it in a register; `C-unwind` still lets a host function's panic unwind. -// Elsewhere the handlers keep the Rust ABI. +// Elsewhere the handlers keep the Rust ABI, which passes eight integer arguments in registers on +// arm64, six on x86-64 System V (as many as the Unbudgeted handlers take) and four on Windows x64. macro_rules! handler_fn { ($(#[$meta:meta])* fn $($rest:tt)*) => { #[cfg(all(target_arch = "aarch64", target_pointer_width = "32"))] @@ -23,9 +30,9 @@ macro_rules! handler_types { // Both sides are Rust, so the C convention's view of these types need not be FFI-safe. #[allow(improper_ctypes_definitions)] type UnbudgetedHandler = - for<'store> $(extern $abi)? fn(&mut Executor<'store>, &[Instruction], usize, Instruction) -> ExecResult<()>; + for<'store> $(extern $abi)? fn(&mut Executor<'store>, &[Instruction], usize, Instruction, usize) -> ExecResult<()>; #[allow(improper_ctypes_definitions)] - type BoundedHandler = for<'store> $(extern $abi)? fn(&mut Executor<'store>, usize, Instruction, u32) -> ExecResult<()>; + type BoundedHandler = for<'store> $(extern $abi)? fn(&mut Executor<'store>, usize, Instruction, u32, usize) -> ExecResult<()>; }; } #[cfg(all(target_arch = "aarch64", target_pointer_width = "32"))] @@ -53,15 +60,17 @@ macro_rules! define_unbudgeted_tail_dispatch { instructions: &[Instruction], $instr_ptr: usize, instruction: Instruction, + height32: usize, ) -> ExecResult<()> { macro_rules! $dispatch_next { ($next_instr_ptr:expr) => {{ let next_instr_ptr = $next_instr_ptr; + let height32 = $executor.value_stack.stack_32.len(); let Some(&next) = instructions.get(next_instr_ptr) else { - become Self::invalid_instr_ptr($executor, instructions, next_instr_ptr, instruction); + become Self::invalid_instr_ptr($executor, instructions, next_instr_ptr, instruction, height32); }; let handler = Self::handler_for(next.opcode()); - become handler($executor, instructions, next_instr_ptr, next); + become handler($executor, instructions, next_instr_ptr, next, height32); }}; } macro_rules! $dispatch_flow { @@ -78,11 +87,12 @@ macro_rules! define_unbudgeted_tail_dispatch { } use tinywasm_types::Instruction::*; $(let $variant($($arg),*) = &instruction else { - become Self::handler_mismatch($executor, instructions, $instr_ptr, instruction); + become Self::handler_mismatch($executor, instructions, $instr_ptr, instruction, height32); };)? $(let $variant { $($field),* } = &instruction else { - become Self::handler_mismatch($executor, instructions, $instr_ptr, instruction); + become Self::handler_mismatch($executor, instructions, $instr_ptr, instruction, height32); };)? + $executor.value_stack.stack_32.set_len(height32); $body; $dispatch_next!($instr_ptr + 1) } @@ -110,10 +120,12 @@ macro_rules! define_bounded_tail_dispatch { $instr_ptr: usize, instruction: Instruction, instructions_until_checkpoint: u32, + height32: usize, ) -> ExecResult<()> { macro_rules! $dispatch_next { ($next_instr_ptr:expr) => {{ let next_instr_ptr = $next_instr_ptr; + let height32 = $executor.value_stack.stack_32.len(); if instructions_until_checkpoint == 0 { return cold!({ $executor.cf.instr_ptr = next_instr_ptr; @@ -122,10 +134,10 @@ macro_rules! define_bounded_tail_dispatch { } let Some(&next) = $executor.func.instructions.get(next_instr_ptr) else { - become Self::invalid_instr_ptr($executor, next_instr_ptr, instruction, instructions_until_checkpoint); + become Self::invalid_instr_ptr($executor, next_instr_ptr, instruction, instructions_until_checkpoint, height32); }; let handler = Self::handler_for(next.opcode()); - become handler($executor, next_instr_ptr, next, instructions_until_checkpoint - 1); + become handler($executor, next_instr_ptr, next, instructions_until_checkpoint - 1, height32); }}; } macro_rules! $dispatch_flow { @@ -141,11 +153,12 @@ macro_rules! define_bounded_tail_dispatch { } use tinywasm_types::Instruction::*; $(let $variant($($arg),*) = &instruction else { - become Self::handler_mismatch($executor, $instr_ptr, instruction, instructions_until_checkpoint); + become Self::handler_mismatch($executor, $instr_ptr, instruction, instructions_until_checkpoint, height32); };)? $(let $variant { $($field),* } = &instruction else { - become Self::handler_mismatch($executor, $instr_ptr, instruction, instructions_until_checkpoint); + become Self::handler_mismatch($executor, $instr_ptr, instruction, instructions_until_checkpoint, height32); };)? + $executor.value_stack.stack_32.set_len(height32); $body; $dispatch_next!($instr_ptr + 1) } @@ -162,7 +175,7 @@ impl Unbudgeted { handler_fn! { #[cold] #[inline(never)] - fn handler_mismatch(_: &mut Executor<'_>, _: &[Instruction], _: usize, _: Instruction) -> ExecResult<()> { + fn handler_mismatch(_: &mut Executor<'_>, _: &[Instruction], _: usize, _: Instruction, _: usize) -> ExecResult<()> { unreachable!("instruction handler mismatch") } } @@ -170,7 +183,7 @@ impl Unbudgeted { handler_fn! { #[cold] #[inline(never)] - fn invalid_instr_ptr(_: &mut Executor<'_>, _: &[Instruction], instr_ptr: usize, _: Instruction) -> ExecResult<()> { + fn invalid_instr_ptr(_: &mut Executor<'_>, _: &[Instruction], instr_ptr: usize, _: Instruction, _: usize) -> ExecResult<()> { unreachable!("instruction pointer {instr_ptr} out of range, this is a bug") } } @@ -184,7 +197,7 @@ impl Bounded { handler_fn! { #[cold] #[inline(never)] - fn handler_mismatch(_: &mut Executor<'_>, _: usize, _: Instruction, _: u32) -> ExecResult<()> { + fn handler_mismatch(_: &mut Executor<'_>, _: usize, _: Instruction, _: u32, _: usize) -> ExecResult<()> { unreachable!("instruction handler mismatch") } } @@ -192,7 +205,7 @@ impl Bounded { handler_fn! { #[cold] #[inline(never)] - fn invalid_instr_ptr(_: &mut Executor<'_>, instr_ptr: usize, _: Instruction, _: u32) -> ExecResult<()> { + fn invalid_instr_ptr(_: &mut Executor<'_>, instr_ptr: usize, _: Instruction, _: u32, _: usize) -> ExecResult<()> { unreachable!("instruction pointer {instr_ptr} out of range, this is a bug") } } @@ -202,7 +215,8 @@ impl Bounded { let instr_ptr = executor.cf.instr_ptr; let instruction = executor.func.instructions[instr_ptr]; let handler = Self::handler_for(instruction.opcode()); - handler(executor, instr_ptr, instruction, CHECKPOINT_INTERVAL - 1) + let height32 = executor.value_stack.stack_32.len(); + handler(executor, instr_ptr, instruction, CHECKPOINT_INTERVAL - 1, height32) } } @@ -215,7 +229,8 @@ impl<'store> Executor<'store> { let instr_ptr = self.cf.instr_ptr; let instruction = instructions[instr_ptr]; let handler = Unbudgeted::handler_for(instruction.opcode()); - handler(&mut self, instructions, instr_ptr, instruction)?; + let height32 = self.value_stack.stack_32.len(); + handler(&mut self, instructions, instr_ptr, instruction, height32)?; if self.completed { return Ok(()); } diff --git a/crates/tinywasm/src/interpreter/stack/value_stack.rs b/crates/tinywasm/src/interpreter/stack/value_stack.rs index 6d33ddd..82e2e5f 100644 --- a/crates/tinywasm/src/interpreter/stack/value_stack.rs +++ b/crates/tinywasm/src/interpreter/stack/value_stack.rs @@ -62,6 +62,16 @@ impl Stack { self.len } + /// Sets the height to one read from [`Self::len`] earlier, with nothing pushed or popped since. + /// The tail-call handlers pass the height between them and write it back on entry, so the + /// compiler can use the value in a register instead of reloading it from memory. + #[cfg(feature = "nightly-tail-calls")] + #[inline(always)] + pub(crate) fn set_len(&mut self, len: usize) { + debug_assert!(len <= self.data.len()); + self.len = len; + } + /// Pushes a value inside a function body. `enter_locals` reserved the function's whole operand /// stack, so the stack is never full here, and the handlers make no calls. #[inline(always)]