diff --git a/crates/tinywasm/src/interpreter/executor/dispatch_become.rs b/crates/tinywasm/src/interpreter/executor/dispatch_become.rs index d5c86442..f790d63e 100644 --- a/crates/tinywasm/src/interpreter/executor/dispatch_become.rs +++ b/crates/tinywasm/src/interpreter/executor/dispatch_become.rs @@ -6,12 +6,6 @@ struct Bounded; type UnbudgetedHandler = for<'store> fn(&mut Executor<'store>, &[Instruction], usize, Instruction) -> ExecResult<()>; type BoundedHandler = for<'store> fn(&mut Executor<'store>, usize, Instruction, u32) -> ExecResult<()>; -#[cold] -#[inline(never)] -fn instruction_handler_mismatch() -> ! { - unreachable!("instruction handler mismatch") -} - macro_rules! define_unbudgeted_tail_dispatch { ($executor:ident, $instr_ptr:ident, $dispatch_next:ident, $dispatch_flow:ident; $($variant:ident $(($($arg:pat),*))? $({ $($field:ident),* })? => $body:expr),* $(,)?) => { @@ -36,9 +30,11 @@ macro_rules! define_unbudgeted_tail_dispatch { macro_rules! $dispatch_next { ($next_instr_ptr:expr) => {{ let next_instr_ptr = $next_instr_ptr; - let instruction = instructions[next_instr_ptr]; - let handler = Self::handler_for(instruction.opcode()); - become handler($executor, instructions, next_instr_ptr, instruction); + let Some(&next) = instructions.get(next_instr_ptr) else { + become Self::invalid_instr_ptr($executor, instructions, next_instr_ptr, instruction); + }; + let handler = Self::handler_for(next.opcode()); + become handler($executor, instructions, next_instr_ptr, next); }}; } macro_rules! $dispatch_flow { @@ -55,10 +51,10 @@ macro_rules! define_unbudgeted_tail_dispatch { } use tinywasm_types::Instruction::*; $(let $variant($($arg),*) = &instruction else { - cold!(instruction_handler_mismatch()) + become Self::handler_mismatch($executor, instructions, $instr_ptr, instruction); };)? $(let $variant { $($field),* } = &instruction else { - cold!(instruction_handler_mismatch()) + become Self::handler_mismatch($executor, instructions, $instr_ptr, instruction); };)? $body; $dispatch_next!($instr_ptr + 1) @@ -98,9 +94,11 @@ macro_rules! define_bounded_tail_dispatch { }); } - let instruction = $executor.func.instructions[next_instr_ptr]; - let handler = Self::handler_for(instruction.opcode()); - become handler($executor, next_instr_ptr, instruction, instructions_until_checkpoint - 1); + let Some(&next) = $executor.func.instructions.get(next_instr_ptr) else { + become Self::invalid_instr_ptr($executor, next_instr_ptr, instruction, instructions_until_checkpoint); + }; + let handler = Self::handler_for(next.opcode()); + become handler($executor, next_instr_ptr, next, instructions_until_checkpoint - 1); }}; } macro_rules! $dispatch_flow { @@ -116,10 +114,10 @@ macro_rules! define_bounded_tail_dispatch { } use tinywasm_types::Instruction::*; $(let $variant($($arg),*) = &instruction else { - cold!(instruction_handler_mismatch()) + become Self::handler_mismatch($executor, $instr_ptr, instruction, instructions_until_checkpoint); };)? $(let $variant { $($field),* } = &instruction else { - cold!(instruction_handler_mismatch()) + become Self::handler_mismatch($executor, $instr_ptr, instruction, instructions_until_checkpoint); };)? $body; $dispatch_next!($instr_ptr + 1) @@ -130,11 +128,40 @@ macro_rules! define_bounded_tail_dispatch { impl Unbudgeted { instruction_handlers!(define_unbudgeted_tail_dispatch); + + // The handlers tail-call these cold paths instead of calling them: a call would make every + // handler save a stack frame. + + #[cold] + #[inline(never)] + fn handler_mismatch(_: &mut Executor<'_>, _: &[Instruction], _: usize, _: Instruction) -> ExecResult<()> { + unreachable!("instruction handler mismatch") + } + + #[cold] + #[inline(never)] + fn invalid_instr_ptr(_: &mut Executor<'_>, _: &[Instruction], instr_ptr: usize, _: Instruction) -> ExecResult<()> { + unreachable!("instruction pointer {instr_ptr} out of range, this is a bug") + } } impl Bounded { instruction_handlers!(define_bounded_tail_dispatch); + // Tail-called like `Unbudgeted`'s. + + #[cold] + #[inline(never)] + fn handler_mismatch(_: &mut Executor<'_>, _: usize, _: Instruction, _: u32) -> ExecResult<()> { + unreachable!("instruction handler mismatch") + } + + #[cold] + #[inline(never)] + fn invalid_instr_ptr(_: &mut Executor<'_>, instr_ptr: usize, _: Instruction, _: u32) -> ExecResult<()> { + unreachable!("instruction pointer {instr_ptr} out of range, this is a bug") + } + #[inline(always)] fn run(executor: &mut Executor<'_>) -> ExecResult<()> { let instr_ptr = executor.cf.instr_ptr; diff --git a/crates/tinywasm/src/interpreter/executor/instructions.rs b/crates/tinywasm/src/interpreter/executor/instructions.rs index 57eef102..22892ebf 100644 --- a/crates/tinywasm/src/interpreter/executor/instructions.rs +++ b/crates/tinywasm/src/interpreter/executor/instructions.rs @@ -3,36 +3,37 @@ macro_rules! exec_op { fn exec_binary_fallible(value_stack: &mut ValueStack) -> Result<(), Trap> { let $rhs = <$ty>::stack_pop(value_stack); let $lhs = <$ty>::stack_pop(value_stack); - <$ty>::stack_push(value_stack, $expr?) + <$ty>::stack_push(value_stack, $expr?); + Ok(()) } exec_binary_fallible(&mut $executor.store.value_stack)?; }}; ($executor:ident; unary $from:ty => $to:ty, |$v:ident| $expr:expr) => {{ - fn exec_unary(value_stack: &mut ValueStack) -> Result<(), Trap> { + fn exec_unary(value_stack: &mut ValueStack) { let $v = <$from>::stack_pop(value_stack); - <$to>::stack_push(value_stack, $expr) + <$to>::stack_push(value_stack, $expr); } - exec_unary(&mut $executor.store.value_stack)?; + exec_unary(&mut $executor.store.value_stack); }}; ($executor:ident; binary $from:ty => $to:ty, |$lhs:ident, $rhs:ident| $expr:expr) => {{ exec_op!($executor; binary $from, $from => $to, |$lhs, $rhs| $expr) }}; ($executor:ident; binary $lhs_ty:ty, $rhs_ty:ty => $res:ty, |$lhs:ident, $rhs:ident| $expr:expr) => {{ - fn exec_binary(value_stack: &mut ValueStack) -> Result<(), Trap> { + fn exec_binary(value_stack: &mut ValueStack) { let $rhs = <$rhs_ty>::stack_pop(value_stack); let $lhs = <$lhs_ty>::stack_pop(value_stack); - <$res>::stack_push(value_stack, $expr) + <$res>::stack_push(value_stack, $expr); } - exec_binary(&mut $executor.store.value_stack)?; + exec_binary(&mut $executor.store.value_stack); }}; ($executor:ident; ternary $from:ty => $to:ty, |$a:ident, $b:ident, $c:ident| $expr:expr) => {{ - fn exec_ternary(value_stack: &mut ValueStack) -> Result<(), Trap> { + fn exec_ternary(value_stack: &mut ValueStack) { let $c = <$from>::stack_pop(value_stack); let $b = <$from>::stack_pop(value_stack); let $a = <$from>::stack_pop(value_stack); - <$to>::stack_push(value_stack, $expr) + <$to>::stack_push(value_stack, $expr); } - exec_ternary(&mut $executor.store.value_stack)?; + exec_ternary(&mut $executor.store.value_stack); }}; } @@ -99,9 +100,9 @@ macro_rules! instruction_handlers { Return32 => dispatch_flow!(executor.exec_return_32()), Return64 => dispatch_flow!(executor.exec_return_64()), Return128 => dispatch_flow!(executor.exec_return_128()), - LocalGet32(local_index) => Value32::local_push(&mut executor.store.value_stack, &executor.cf, *local_index)?, - LocalGet64(local_index) => Value64::local_push(&mut executor.store.value_stack, &executor.cf, *local_index)?, - LocalGet128(local_index) => Value128::local_push(&mut executor.store.value_stack, &executor.cf, *local_index)?, + LocalGet32(local_index) => Value32::local_push(&mut executor.store.value_stack, &executor.cf, *local_index), + LocalGet64(local_index) => Value64::local_push(&mut executor.store.value_stack, &executor.cf, *local_index), + LocalGet128(local_index) => Value128::local_push(&mut executor.store.value_stack, &executor.cf, *local_index), LocalSet32(local_index) => executor.exec_local_set_pop::(*local_index), LocalSet64(local_index) => executor.exec_local_set_pop::(*local_index), LocalSet128(local_index) => executor.exec_local_set_pop::(*local_index), @@ -201,10 +202,10 @@ macro_rules! instruction_handlers { GlobalTee32(global_index) => executor.exec_global_tee::(*global_index), GlobalTee64(global_index) => executor.exec_global_tee::(*global_index), GlobalTee128(global_index) => executor.exec_global_tee::(*global_index), - Const32(val) => i32::stack_push(&mut executor.store.value_stack, *val)?, - Const64Imm(val) => i64::stack_push(&mut executor.store.value_stack, i64::from(*val))?, - Const64(idx) => i64::stack_push(&mut executor.store.value_stack, idx.resolve(&executor.func.data).value())?, - Const128Imm(val) => Value128::stack_push(&mut executor.store.value_stack, Value128(u128::from(*val).to_le_bytes()))?, + Const32(val) => i32::stack_push(&mut executor.store.value_stack, *val), + Const64Imm(val) => i64::stack_push(&mut executor.store.value_stack, i64::from(*val)), + Const64(idx) => i64::stack_push(&mut executor.store.value_stack, idx.resolve(&executor.func.data).value()), + Const128Imm(val) => Value128::stack_push(&mut executor.store.value_stack, Value128(u128::from(*val).to_le_bytes())), I64Eqz => exec_op!(executor; unary i64 => i32, |v| i32::from(v == 0)), I32Eqz => exec_op!(executor; unary i32 => i32, |v| i32::from(v == 0)), I32Eq => exec_op!(executor; binary i32 => i32, |a, b| i32::from(a == b)), @@ -289,8 +290,8 @@ macro_rules! instruction_handlers { I64Popcnt => exec_op!(executor; unary i64 => i64, |v| i64::from(v.count_ones())), // Reference types - RefFunc(func_idx) => ValueRef::stack_push(&mut executor.store.value_stack, ValueRef::from_category_addr(executor.module.resolve_func_addr(*func_idx)))?, - RefNull(_) => ValueRef::stack_push(&mut executor.store.value_stack, ValueRef::NULL)?, + RefFunc(func_idx) => ValueRef::stack_push(&mut executor.store.value_stack, ValueRef::from_category_addr(executor.module.resolve_func_addr(*func_idx))), + RefNull(_) => ValueRef::stack_push(&mut executor.store.value_stack, ValueRef::NULL), RefIsNull => executor.exec_ref_is_null()?, RefAsNonNull => executor.exec_ref_as_non_null()?, RefI31 => exec_op!(executor; unary i32 => ValueRef, |v| ValueRef::from_i31(v)), @@ -459,7 +460,7 @@ macro_rules! instruction_handlers { V128Store64Lane(arg) => executor.exec_mem_store_lane::(*arg)?, V128Load32Zero(idx) => executor.exec_mem_load::(idx.resolve(&executor.func.data), |v| Value128::from_i32x4([v, 0, 0, 0]))?, V128Load64Zero(idx) => executor.exec_mem_load::(idx.resolve(&executor.func.data), |v| Value128::from_i64x2([v, 0]))?, - Const128(arg) => Value128::stack_push(&mut executor.store.value_stack, Value128(arg.resolve(&executor.func.data).value()))?, + Const128(arg) => Value128::stack_push(&mut executor.store.value_stack, Value128(arg.resolve(&executor.func.data).value())), I8x16ExtractLaneS(lane) => executor.exec_simd_extract_lane::(*lane, |v, lane| v.extract_lane_i8(lane) as i32)?, I8x16ExtractLaneU(lane) => executor.exec_simd_extract_lane::(*lane, |v, lane| v.extract_lane_u8(lane) as i32)?, I16x8ExtractLaneS(lane) => executor.exec_simd_extract_lane::(*lane, |v, lane| v.extract_lane_i16(lane) as i32)?, diff --git a/crates/tinywasm/src/interpreter/executor/mod.rs b/crates/tinywasm/src/interpreter/executor/mod.rs index 149ebc3a..decc92be 100644 --- a/crates/tinywasm/src/interpreter/executor/mod.rs +++ b/crates/tinywasm/src/interpreter/executor/mod.rs @@ -300,8 +300,9 @@ impl<'store> Executor<'store> { let lo = a_lo.wrapping_add(b_lo); let carry = u64::from((lo as u64) < (a_lo as u64)); let hi = a_hi.wrapping_add(b_hi).wrapping_add(carry as i64); - i64::stack_push(&mut self.store.value_stack, lo)?; - i64::stack_push(&mut self.store.value_stack, hi) + i64::stack_push(&mut self.store.value_stack, lo); + i64::stack_push(&mut self.store.value_stack, hi); + Ok(()) } fn exec_i64_sub128(&mut self) -> Result<(), Trap> { @@ -312,24 +313,27 @@ impl<'store> Executor<'store> { let lo = a_lo.wrapping_sub(b_lo); let borrow = u64::from((a_lo as u64) < (b_lo as u64)); let hi = a_hi.wrapping_sub(b_hi).wrapping_sub(borrow as i64); - i64::stack_push(&mut self.store.value_stack, lo)?; - i64::stack_push(&mut self.store.value_stack, hi) + i64::stack_push(&mut self.store.value_stack, lo); + i64::stack_push(&mut self.store.value_stack, hi); + Ok(()) } fn exec_i64_mul_wide_s(&mut self) -> Result<(), Trap> { let rhs = i64::stack_pop(&mut self.store.value_stack); let lhs = i64::stack_pop(&mut self.store.value_stack); let product = (lhs as i128).wrapping_mul(rhs as i128); - i64::stack_push(&mut self.store.value_stack, product as i64)?; - i64::stack_push(&mut self.store.value_stack, (product >> 64) as i64) + i64::stack_push(&mut self.store.value_stack, product as i64); + i64::stack_push(&mut self.store.value_stack, (product >> 64) as i64); + Ok(()) } fn exec_i64_mul_wide_u(&mut self) -> Result<(), Trap> { let rhs = i64::stack_pop(&mut self.store.value_stack); let lhs = i64::stack_pop(&mut self.store.value_stack); let product = (lhs as u64 as u128).wrapping_mul(rhs as u64 as u128); - i64::stack_push(&mut self.store.value_stack, product as u64 as i64)?; - i64::stack_push(&mut self.store.value_stack, (product >> 64) as u64 as i64) + i64::stack_push(&mut self.store.value_stack, product as u64 as i64); + i64::stack_push(&mut self.store.value_stack, (product >> 64) as u64 as i64); + Ok(()) } #[inline(always)] @@ -339,7 +343,8 @@ impl<'store> Executor<'store> { operation: impl FnOnce(Value128, u8) -> TO, ) -> Result<(), Trap> { let vector = Value128::stack_pop(&mut self.store.value_stack); - TO::stack_push(&mut self.store.value_stack, operation(vector, lane)) + TO::stack_push(&mut self.store.value_stack, operation(vector, lane)); + Ok(()) } #[inline(always)] @@ -350,13 +355,15 @@ impl<'store> Executor<'store> { ) -> Result<(), Trap> { let vector = Value128::stack_pop(&mut self.store.value_stack); let value = VALUE::stack_pop(&mut self.store.value_stack); - Value128::stack_push(&mut self.store.value_stack, operation(value, vector, lane)) + Value128::stack_push(&mut self.store.value_stack, operation(value, vector, lane)); + Ok(()) } fn exec_simd_shuffle(&mut self, lanes: Value128) -> Result<(), Trap> { let rhs = Value128::stack_pop(&mut self.store.value_stack); let lhs = Value128::stack_pop(&mut self.store.value_stack); - Value128::stack_push(&mut self.store.value_stack, Value128::i8x16_shuffle(lhs, rhs, lanes)) + Value128::stack_push(&mut self.store.value_stack, Value128::i8x16_shuffle(lhs, rhs, lanes)); + Ok(()) } #[inline(always)] @@ -375,7 +382,8 @@ impl<'store> Executor<'store> { fn exec_global_get(&mut self, global: GlobalAddr) -> Result<(), Trap> { let addr = self.module.resolve_global_addr(global); let value = T::global_get(&self.store.state.globals, addr); - T::stack_push(&mut self.store.value_stack, value) + T::stack_push(&mut self.store.value_stack, value); + Ok(()) } #[inline(always)] @@ -401,7 +409,10 @@ impl<'store> Executor<'store> { if let Some(dst) = dst { T::local_set(&mut self.store.value_stack, &self.cf, dst, val); } - if PUSH { T::stack_push(&mut self.store.value_stack, val) } else { Ok(()) } + if PUSH { + T::stack_push(&mut self.store.value_stack, val); + } + Ok(()) } #[inline(always)] @@ -447,7 +458,8 @@ impl<'store> Executor<'store> { op: impl BinOpExt, ) -> Result<(), Trap> { let lhs = T::global_get(&self.store.state.globals, self.module.resolve_global_addr(lhs)); - T::stack_push(&mut self.store.value_stack, op.exec(lhs, rhs)) + T::stack_push(&mut self.store.value_stack, op.exec(lhs, rhs)); + Ok(()) } #[inline(always)] @@ -458,7 +470,8 @@ impl<'store> Executor<'store> { { let lhs = T::local_get(&self.store.value_stack, &self.cf, lhs); let rhs = T::local_get(&self.store.value_stack, &self.cf, rhs); - i32::stack_push(&mut self.store.value_stack, i32::from(op.cmp(lhs, rhs))) + i32::stack_push(&mut self.store.value_stack, i32::from(op.cmp(lhs, rhs))); + Ok(()) } #[inline(always)] @@ -617,11 +630,11 @@ impl<'store> Executor<'store> { let Store { state, value_stack, .. } = self.store; let object = state.gc.get(exception).ok_or(Trap::InvalidReference)?; for value in object.values.iter().copied() { - value_stack.push_reserved(value)?; + value_stack.push_reserved(value); } } if with_ref { - ValueRef::stack_push(&mut self.store.value_stack, exception)?; + ValueRef::stack_push(&mut self.store.value_stack, exception); } return Ok(Some(if switched { ExecFlow::Switch(landing_pad as usize) @@ -861,6 +874,7 @@ impl<'store> Executor<'store> { self.finish_return() } + #[inline(always)] fn exec_store_local_local, const N: usize>( &mut self, index: Operand64Idx, @@ -886,6 +900,7 @@ impl<'store> Executor<'store> { }) } + #[inline(always)] fn exec_inc_memory_local, const N: usize>( &mut self, index: Operand64Idx, @@ -912,6 +927,7 @@ impl<'store> Executor<'store> { }) } + #[inline(always)] fn exec_fma_store< T: InternalValue + MemValue + core::ops::Add + core::ops::Mul, const N: usize, @@ -931,6 +947,7 @@ impl<'store> Executor<'store> { }) } + #[inline(always)] fn exec_load_local< LOAD: MemValue, const N: usize, @@ -966,7 +983,7 @@ impl<'store> Executor<'store> { TARGET::local_set(&mut self.store.value_stack, &self.cf, u16::from(dst_local), value); } if !SET_LOCAL || TEE { - TARGET::stack_push(&mut self.store.value_stack, value)?; + TARGET::stack_push(&mut self.store.value_stack, value); } Ok(()) }) @@ -974,7 +991,8 @@ impl<'store> Executor<'store> { fn exec_ref_is_null(&mut self) -> Result<(), Trap> { let is_null = i32::from(ValueRef::stack_pop(&mut self.store.value_stack).is_null()); - i32::stack_push(&mut self.store.value_stack, is_null) + i32::stack_push(&mut self.store.value_stack, is_null); + Ok(()) } fn exec_ref_as_non_null(&mut self) -> Result<(), Trap> { @@ -997,7 +1015,8 @@ impl<'store> Executor<'store> { fn exec_ref_test(&mut self, ty: RefType) -> Result<(), Trap> { let value = ValueRef::stack_pop(&mut self.store.value_stack); let matches = self.store.state.value_ref_matches(value, self.canonical_ref_type(ty)); - i32::stack_push(&mut self.store.value_stack, i32::from(matches)) + i32::stack_push(&mut self.store.value_stack, i32::from(matches)); + Ok(()) } fn exec_ref_cast(&self, ty: RefType) -> Result<(), Trap> { @@ -1017,13 +1036,15 @@ impl<'store> Executor<'store> { } else { value.i31_u().expect("validated i31.get operand") as i32 }; - i32::stack_push(&mut self.store.value_stack, value) + i32::stack_push(&mut self.store.value_stack, value); + Ok(()) } fn push_gc_object(&mut self, type_addr: TypeAddr, values: Vec) -> Result<(), Trap> { let roots = (&self.store.value_stack.stack_32).into_iter().copied().map(ValueRef::from_raw); let reference = self.store.state.alloc_gc_object(type_addr, values, roots)?; - ValueRef::stack_push(&mut self.store.value_stack, reference) + ValueRef::stack_push(&mut self.store.value_stack, reference); + Ok(()) } fn exec_struct_new(&mut self, type_index: TypeAddr, default: bool) -> Result<(), Trap> { @@ -1066,7 +1087,8 @@ impl<'store> Executor<'store> { let object = self.store.state.gc_object(reference, type_addr)?; let object = self.store.state.gc.get_handle(object).ok_or(Trap::Other("invalid GC reference"))?; let value = *object.values.get(field_index as usize).expect("validated struct field index"); - push_value(&mut self.store.value_stack, value, storage, signed) + push_value(&mut self.store.value_stack, value, storage, signed); + Ok(()) } fn exec_struct_set(&mut self, index: Operand64Idx<(u32, u32)>) -> Result<(), Trap> { @@ -1121,7 +1143,8 @@ impl<'store> Executor<'store> { let object = self.store.state.gc_object(reference, type_addr)?; let object = self.store.state.gc.get_handle(object).ok_or(Trap::Other("invalid GC reference"))?; let value = *object.values.get(index).ok_or(Trap::ArrayOutOfBounds)?; - push_value(&mut self.store.value_stack, value, storage, signed) + push_value(&mut self.store.value_stack, value, storage, signed); + Ok(()) } fn exec_array_set(&mut self, type_index: TypeAddr) -> Result<(), Trap> { @@ -1146,7 +1169,8 @@ impl<'store> Executor<'store> { if self.store.state.get_type(type_addr).as_array().is_none() { return Err(Trap::Other("GC reference is not an array")); } - i32::stack_push(&mut self.store.value_stack, object.values.len() as i32) + i32::stack_push(&mut self.store.value_stack, object.values.len() as i32); + Ok(()) } fn exec_array_fill(&mut self, type_index: TypeAddr) -> Result<(), Trap> { @@ -1271,6 +1295,7 @@ impl<'store> Executor<'store> { (MemoryArch::I64, pages) => i64::stack_push(&mut self.store.value_stack, pages as i64), (MemoryArch::I32, pages) => i32::stack_push(&mut self.store.value_stack, pages as i32), } + Ok(()) } fn exec_memory_grow(&mut self, addr: u32) -> Result<(), Trap> { @@ -1284,8 +1309,8 @@ impl<'store> Executor<'store> { let size = self.store.state.grow_mem(mem_addr, pages_delta, limiter)?.unwrap_or(-1); match is_64bit { - true => i64::stack_push(&mut self.store.value_stack, size)?, - false => i32::stack_push(&mut self.store.value_stack, size as i32)?, + true => i64::stack_push(&mut self.store.value_stack, size), + false => i32::stack_push(&mut self.store.value_stack, size as i32), }; Ok(()) @@ -1386,6 +1411,7 @@ impl<'store> Executor<'store> { } } + #[inline(always)] fn exec_mem_load_lane, const LOAD_SIZE: usize>( &mut self, arg: MemoryLaneArg, @@ -1417,7 +1443,8 @@ impl<'store> Executor<'store> { let base = self.store.value_stack.pop_memory_operand(kind.arch())?; let addr = cold_err!(mem.effective_addr::(base, m.offset()))?; let value = cold_err!(LOAD::load_at(&*mem, addr))?; - TARGET::stack_push(&mut self.store.value_stack, cast(value)) + TARGET::stack_push(&mut self.store.value_stack, cast(value)); + Ok(()) }) } @@ -1469,10 +1496,12 @@ impl<'store> Executor<'store> { AtomicWaitOp::Wait32 => shared.wait::<4>(addr, value, timeout)?, AtomicWaitOp::Wait64 => shared.wait::<8>(addr, value, timeout)?, }; - return u32::stack_push(&mut self.store.value_stack, result); + u32::stack_push(&mut self.store.value_stack, result); + return Ok(()); } - u32::stack_push(&mut self.store.value_stack, 0) + u32::stack_push(&mut self.store.value_stack, 0); + Ok(()) } fn exec_atomic_width(&mut self, arg: AtomicArg) -> Result<(), Trap> { @@ -1528,6 +1557,7 @@ impl<'store> Executor<'store> { true => u64::stack_push(&mut self.store.value_stack, old), false => u32::stack_push(&mut self.store.value_stack, old as u32), } + Ok(()) }) } @@ -1587,7 +1617,8 @@ impl<'store> Executor<'store> { let table_addr = self.module.resolve_table_addr(table_index); let idx = self.pop_table_operand(self.store.state.get_table(table_addr).kind.arch())?; let value = *self.store.state.get_table(table_addr).get(idx)?; - ValueRef::stack_push(&mut self.store.value_stack, value) + ValueRef::stack_push(&mut self.store.value_stack, value); + Ok(()) } fn exec_table_set(&mut self, table_index: u32) -> Result<(), Trap> { @@ -1604,6 +1635,7 @@ impl<'store> Executor<'store> { MemoryArch::I32 => i32::stack_push(&mut self.store.value_stack, table.size() as i32), MemoryArch::I64 => i64::stack_push(&mut self.store.value_stack, table.size() as i64), } + Ok(()) } fn exec_table_init(&mut self, index: Operand64Idx<(u32, u32)>) -> Result<(), Trap> { @@ -1635,6 +1667,7 @@ impl<'store> Executor<'store> { (MemoryArch::I64, true) => i64::stack_push(&mut self.store.value_stack, sz as i64), (MemoryArch::I64, false) => i64::stack_push(&mut self.store.value_stack, -1), } + Ok(()) } fn exec_table_fill(&mut self, table_index: u32) -> Result<(), Trap> { diff --git a/crates/tinywasm/src/interpreter/num_helpers.rs b/crates/tinywasm/src/interpreter/num_helpers.rs index 6e9ec191..53632bd0 100644 --- a/crates/tinywasm/src/interpreter/num_helpers.rs +++ b/crates/tinywasm/src/interpreter/num_helpers.rs @@ -35,7 +35,7 @@ macro_rules! checked_conv_float { core::hint::cold_path(); return Err(crate::Trap::IntegerOverflow.into()); } - <$to>::stack_push(&mut $self.store.value_stack, (v as $intermediate as $to).into())?; + <$to>::stack_push(&mut $self.store.value_stack, (v as $intermediate as $to).into()); }}; } diff --git a/crates/tinywasm/src/interpreter/stack/value_stack.rs b/crates/tinywasm/src/interpreter/stack/value_stack.rs index b3ac91a0..1c40e31c 100644 --- a/crates/tinywasm/src/interpreter/stack/value_stack.rs +++ b/crates/tinywasm/src/interpreter/stack/value_stack.rs @@ -24,12 +24,6 @@ pub(crate) struct Stack { dynamic: bool, } -#[cold] -#[inline(never)] -fn stack_underflow() -> ! { - unreachable!("ValueStack underflow, this is a bug") -} - impl Stack { pub(crate) fn new(config: StackConfig) -> Self { Self { data: Vec::with_capacity(config.initial_size), max_size: config.max_size, dynamic: config.dynamic } @@ -45,15 +39,14 @@ impl Stack { } /// Pushes a value inside a function body. `enter_locals` reserved the function's whole operand - /// stack, so a full stack here is the limit and there is nothing to grow. After this check - /// `Vec::push` cannot reach its own growth path, so the instruction handlers make no calls. + /// stack, so the stack is never full here. After this check `Vec::push` cannot reach its own + /// growth path, so the instruction handlers make no calls. #[inline(always)] - pub(crate) fn push(&mut self, value: T) -> Result<(), Trap> { + pub(crate) fn push(&mut self, value: T) { if self.data.len() == self.data.capacity() { - return cold!(Err(Trap::ValueStackOverflow)); + crate::invariant_violated("value stack push beyond the function's reservation"); } self.data.push(value); - Ok(()) } /// Pushes a value outside a function body (host arguments and results), which no reservation @@ -68,9 +61,9 @@ impl Stack { } #[inline(always)] - pub(crate) fn push_copy(&mut self, index: usize) -> Result<(), Trap> { - let value = self.data[index]; - self.push(value) + pub(crate) fn push_copy(&mut self, index: usize) { + let value = *self.get(index); + self.push(value); } #[cold] @@ -89,7 +82,7 @@ impl Stack { pub(crate) fn pop(&mut self) -> T { match self.data.pop() { Some(value) => value, - None => cold!(stack_underflow()), + None => crate::invariant_violated("value stack underflow"), } } @@ -97,23 +90,30 @@ impl Stack { pub(crate) fn last(&self) -> &T { match self.data.last() { Some(value) => value, - None => cold!(stack_underflow()), + None => crate::invariant_violated("value stack underflow"), } } #[inline(always)] pub(crate) fn get(&self, index: usize) -> &T { - &self.data[index] + match self.data.get(index) { + Some(value) => value, + None => crate::invariant_violated("value stack index out of range"), + } } #[inline(always)] pub(crate) fn set(&mut self, index: usize, value: T) { - self.data[index] = value; + match self.data.get_mut(index) { + Some(slot) => *slot = value, + None => crate::invariant_violated("value stack index out of range"), + } } #[inline(always)] pub(crate) fn copy(&mut self, from: usize, to: usize) { - self.data[to] = self.data[from]; + let value = *self.get(from); + self.set(to, value); } #[inline(always)] @@ -187,7 +187,7 @@ impl Stack { let needed = count.wrapping_mul(2); if len < needed { - cold!(stack_underflow()); + crate::invariant_violated("value stack underflow"); } if !condition { @@ -298,7 +298,7 @@ impl ValueStack { } /// Pushes a dynamically typed value inside a function body using its entry reservation. - pub(crate) fn push_reserved(&mut self, value: RuntimeValue) -> Result<(), Trap> { + pub(crate) fn push_reserved(&mut self, value: RuntimeValue) { match value { RuntimeValue::Value32(value) => self.stack_32.push(value), RuntimeValue::Value64(value) => self.stack_64.push(value), diff --git a/crates/tinywasm/src/interpreter/values.rs b/crates/tinywasm/src/interpreter/values.rs index ec69fe97..4125ce43 100644 --- a/crates/tinywasm/src/interpreter/values.rs +++ b/crates/tinywasm/src/interpreter/values.rs @@ -107,13 +107,13 @@ impl RuntimeValue { /// Typed access to values in their physical [`ValueStack`] and [`Globals`] lanes. pub(crate) trait InternalValue: Copy + Default { - fn stack_push(stack: &mut ValueStack, value: Self) -> Result<(), crate::Trap>; + fn stack_push(stack: &mut ValueStack, value: Self); fn stack_pop(stack: &mut ValueStack) -> Self; fn stack_peek(stack: &ValueStack) -> Self; fn stack_update(stack: &mut ValueStack, f: impl FnOnce(Self) -> Self) -> Self; fn stack_select(stack: &mut ValueStack); fn local_get(stack: &ValueStack, frame: &CallFrame, index: LocalAddr) -> Self; - fn local_push(stack: &mut ValueStack, frame: &CallFrame, index: LocalAddr) -> Result<(), crate::Trap>; + fn local_push(stack: &mut ValueStack, frame: &CallFrame, index: LocalAddr); fn local_set(stack: &mut ValueStack, frame: &CallFrame, index: LocalAddr, value: Self); fn local_update(stack: &mut ValueStack, frame: &CallFrame, index: LocalAddr, f: impl FnOnce(Self) -> Self) -> Self; fn local_copy(stack: &mut ValueStack, frame: &CallFrame, from: LocalAddr, to: LocalAddr); @@ -140,8 +140,8 @@ macro_rules! impl_internalvalue { } #[inline(always)] - fn local_push(stack: &mut ValueStack, frame: &CallFrame, index: LocalAddr) -> Result<(), crate::Trap> { - stack.$stack.push_copy(frame.locals_base.$stack_base as usize + index as usize) + fn local_push(stack: &mut ValueStack, frame: &CallFrame, index: LocalAddr) { + stack.$stack.push_copy(frame.locals_base.$stack_base as usize + index as usize); } #[inline(always)] @@ -195,10 +195,9 @@ macro_rules! impl_internalvalue { } #[inline(always)] - fn stack_push(stack: &mut ValueStack, value: Self) -> Result<(), crate::Trap> { + fn stack_push(stack: &mut ValueStack, value: Self) { let $to_stack_v = value; - cold_err!(stack.$stack.push($to_stack))?; - Ok(()) + stack.$stack.push($to_stack); } #[inline(always)] diff --git a/crates/tinywasm/src/lib.rs b/crates/tinywasm/src/lib.rs index 47da470d..a2789f61 100644 --- a/crates/tinywasm/src/lib.rs +++ b/crates/tinywasm/src/lib.rs @@ -1,6 +1,7 @@ #![no_std] #![cfg_attr(feature = "nightly-tail-calls", allow(incomplete_features))] #![cfg_attr(feature = "nightly-tail-calls", feature(explicit_tail_calls))] +#![cfg_attr(all(feature = "nightly-tail-calls", not(debug_assertions)), feature(abort_immediate))] #![doc(test( no_crate_inject, attr(deny(warnings, rust_2018_idioms), allow(dead_code, unused_assignments, unused_variables)) @@ -94,6 +95,7 @@ #[macro_use] mod macros; +pub(crate) use macros::invariant_violated; mod std; extern crate alloc; diff --git a/crates/tinywasm/src/macros.rs b/crates/tinywasm/src/macros.rs index 351ae886..c5f1bce3 100644 --- a/crates/tinywasm/src/macros.rs +++ b/crates/tinywasm/src/macros.rs @@ -1,3 +1,25 @@ +/// Stops on an interpreter invariant that validated code cannot break, such as a value-stack or +/// global access out of range. +/// +/// With the nightly tail-call dispatch, release builds trap in place: a call here, even a cold one, +/// makes every instruction handler that touches the value stack save a stack frame. Debug builds +/// and the loop dispatch panic. +#[cfg(all(feature = "nightly-tail-calls", not(debug_assertions)))] +#[inline(always)] +pub(crate) fn invariant_violated(_what: &'static str) -> ! { + core::process::abort_immediate() +} + +/// Stops on an interpreter invariant that validated code cannot break, such as a value-stack or +/// global access out of range. +#[cfg(not(all(feature = "nightly-tail-calls", not(debug_assertions))))] +#[cold] +#[inline(never)] +#[track_caller] +pub(crate) fn invariant_violated(what: &'static str) -> ! { + unreachable!("{what}, this is a bug") +} + macro_rules! cold { ($value:expr) => {{ core::hint::cold_path(); diff --git a/crates/tinywasm/src/store/gc/mod.rs b/crates/tinywasm/src/store/gc/mod.rs index 598907d9..f55d7b48 100644 --- a/crates/tinywasm/src/store/gc/mod.rs +++ b/crates/tinywasm/src/store/gc/mod.rs @@ -39,12 +39,7 @@ pub(crate) fn pop_value(stack: &mut ValueStack, storage: StorageType) -> Runtime } /// Extends a packed value and pushes it onto the operand stack. -pub(crate) fn push_value( - stack: &mut ValueStack, - value: RuntimeValue, - storage: StorageType, - signed: Option, -) -> Result<(), Trap> { +pub(crate) fn push_value(stack: &mut ValueStack, value: RuntimeValue, storage: StorageType, signed: Option) { let value = match (value, storage, signed) { (RuntimeValue::Value32(value), StorageType::I8, Some(true)) => RuntimeValue::Value32(value as i8 as i32 as u32), (RuntimeValue::Value32(value), StorageType::I16, Some(true)) => { diff --git a/crates/tinywasm/src/store/global.rs b/crates/tinywasm/src/store/global.rs index 75aa163e..8c696f1e 100644 --- a/crates/tinywasm/src/store/global.rs +++ b/crates/tinywasm/src/store/global.rs @@ -25,12 +25,18 @@ impl GlobalLane { #[inline] fn get(&self, index: usize) -> T { - self.values[index] + match self.values.get(index) { + Some(value) => *value, + None => crate::invariant_violated("global address out of range"), + } } #[inline] fn set(&mut self, index: usize, value: T) { - self.values[index] = value; + match self.values.get_mut(index) { + Some(slot) => *slot = value, + None => crate::invariant_violated("global address out of range"), + } } fn ty(&self, index: usize) -> GlobalType {