T0b1/wasmtime
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
Files
b8a68ff86d14aa70cc51960bf1b8676233110f5e
wasmtime/crates/environ/src/fact/trampoline.rs
Alex Crichton b8a68ff86d Tweak adapter cost of lists (#4853) 
I noticed an oss-fuzz-based timeout that was reported for the
`component_api` fuzzer where the adapter module generated takes 1.5
seconds to compile the singular function in release mode (no fuzzing
enabled). The test case in question was a deeply recursive
list-of-list-of-etc and only one function was generated instead of
multiple. I updated the cost of strings/lists to cost more in the
approximate cost calculation which now forces the one giant function to
get split up and the large function is now split up into multiple
smaller function that take milliseconds to compile.
2022-09-02 18:11:48 +00:00

3162 lines
120 KiB
Rust
Raw Blame History

//! Low-level compilation of an fused adapter function.
//!
//! This module is tasked with the top-level `compile` function which creates a
//! single WebAssembly function which will perform the steps of the fused
//! adapter for an `AdapterData` provided. This is the "meat" of compilation
//! where the validation of the canonical ABI or similar all happens to
//! translate arguments from one module to another.
//!
//! ## Traps and their ordering
//!
//! Currently this compiler is pretty "loose" about the ordering of precisely
//! what trap happens where. The main reason for this is that to core wasm all
//! traps are the same and for fused adapters if a trap happens no intermediate
//! side effects are visible (as designed by the canonical ABI itself). For this
//! it's important to note that some of the precise choices of control flow here
//! can be somewhat arbitrary, an intentional decision.
use crate::component::{
CanonicalAbiInfo, ComponentTypesBuilder, FlatType, InterfaceType, StringEncoding,
TypeEnumIndex, TypeFlagsIndex, TypeListIndex, TypeOptionIndex, TypeRecordIndex,
TypeResultIndex, TypeTupleIndex, TypeUnionIndex, TypeVariantIndex, VariantInfo, FLAG_MAY_ENTER,
FLAG_MAY_LEAVE, MAX_FLAT_PARAMS, MAX_FLAT_RESULTS,
};
use crate::fact::signature::Signature;
use crate::fact::transcode::{FixedEncoding as FE, Transcode, Transcoder};
use crate::fact::traps::Trap;
use crate::fact::{
AdapterData, Body, Context, Function, FunctionId, Helper, HelperLocation, HelperType, Module,
Options,
};
use crate::{FuncIndex, GlobalIndex};
use std::collections::HashMap;
use std::mem;
use std::ops::Range;
use wasm_encoder::{BlockType, Encode, Instruction, Instruction::*, MemArg, ValType};
use wasmtime_component_util::{DiscriminantSize, FlagsSize};
const MAX_STRING_BYTE_LENGTH: u32 = 1 << 31;
const UTF16_TAG: u32 = 1 << 31;
/// This value is arbitrarily chosen and should be fine to change at any time,
/// it just seemed like a halfway reasonable starting point.
const INITIAL_FUEL: usize = 1_000;
struct Compiler<'a, 'b> {
types: &'a ComponentTypesBuilder,
module: &'b mut Module<'a>,
result: FunctionId,
/// The encoded WebAssembly function body so far, not including locals.
code: Vec<u8>,
/// Total number of locals generated so far.
nlocals: u32,
/// Locals partitioned by type which are not currently in use.
free_locals: HashMap<ValType, Vec<u32>>,
/// Metadata about all `unreachable` trap instructions in this function and
/// what the trap represents. The offset within `self.code` is recorded as
/// well.
traps: Vec<(usize, Trap)>,
/// A heuristic which is intended to limit the size of a generated function
/// to a certain maximum to avoid generating arbitrarily large functions.
///
/// This fuel counter is decremented each time `translate` is called and
/// when fuel is entirely consumed further translations, if necessary, will
/// be done through calls to other functions in the module. This is intended
/// to be a heuristic to split up the main function into theoretically
/// reusable portions.
fuel: usize,
}
pub(super) fn compile(module: &mut Module<'_>, adapter: &AdapterData) {
let lower_sig = module.types.signature(&adapter.lower, Context::Lower);
let lift_sig = module.types.signature(&adapter.lift, Context::Lift);
let ty = module
.core_types
.function(&lower_sig.params, &lower_sig.results);
let result = module
.funcs
.push(Function::new(Some(adapter.name.clone()), ty));
Compiler {
types: module.types,
module,
code: Vec::new(),
nlocals: lower_sig.params.len() as u32,
free_locals: HashMap::new(),
traps: Vec::new(),
result,
fuel: INITIAL_FUEL,
}
.compile_adapter(adapter, &lower_sig, &lift_sig)
}
/// Compiles a helper function as specified by the `Helper` configuration.
///
/// This function is invoked when the translation process runs out of fuel for
/// some prior function which enqueues a helper to get translated later. This
/// translation function will perform one type translation as specified by
/// `Helper` which can either be in the stack or memory for each side.
pub(super) fn compile_helper(module: &mut Module<'_>, result: FunctionId, helper: Helper) {
let mut nlocals = 0;
let src_flat;
let src = match helper.src.loc {
// If the source is on the stack then it's specified in the parameters
// to the function, so this creates the flattened representation and
// then lists those as the locals with appropriate types for the source
// values.
HelperLocation::Stack => {
src_flat = module
.types
.flatten_types(&helper.src.opts, usize::MAX, [helper.src.ty])
.unwrap()
.iter()
.enumerate()
.map(|(i, ty)| (i as u32, *ty))
.collect::<Vec<_>>();
nlocals += src_flat.len() as u32;
Source::Stack(Stack {
locals: &src_flat,
opts: &helper.src.opts,
})
}
// If the source is in memory then that's just propagated here as the
// first local is the pointer to the source.
HelperLocation::Memory => {
nlocals += 1;
Source::Memory(Memory {
opts: &helper.src.opts,
addr: TempLocal::new(0, helper.src.opts.ptr()),
offset: 0,
})
}
};
let dst_flat;
let dst = match helper.dst.loc {
// This is the same as the stack-based source although `Destination` is
// configured slightly differently.
HelperLocation::Stack => {
dst_flat = module
.types
.flatten_types(&helper.dst.opts, usize::MAX, [helper.dst.ty])
.unwrap();
Destination::Stack(&dst_flat, &helper.dst.opts)
}
// This is the same as a memroy-based source but note that the address
// of the destination is passed as the final parameter to the function.
HelperLocation::Memory => {
nlocals += 1;
Destination::Memory(Memory {
opts: &helper.dst.opts,
addr: TempLocal::new(nlocals - 1, helper.dst.opts.ptr()),
offset: 0,
})
}
};
let mut compiler = Compiler {
types: module.types,
module,
code: Vec::new(),
nlocals,
free_locals: HashMap::new(),
traps: Vec::new(),
result,
fuel: INITIAL_FUEL,
};
compiler.translate(&helper.src.ty, &src, &helper.dst.ty, &dst);
compiler.finish();
}
/// Possible ways that a interface value is represented in the core wasm
/// canonical ABI.
enum Source<'a> {
/// This value is stored on the "stack" in wasm locals.
///
/// This could mean that it's inline from the parameters to the function or
/// that after a function call the results were stored in locals and the
/// locals are the inline results.
Stack(Stack<'a>),
/// This value is stored in linear memory described by the `Memory`
/// structure.
Memory(Memory<'a>),
}
/// Same as `Source` but for where values are translated into.
enum Destination<'a> {
/// This value is destined for the WebAssembly stack which means that
/// results are simply pushed as we go along.
///
/// The types listed are the types that are expected to be on the stack at
/// the end of translation.
Stack(&'a [ValType], &'a Options),
/// This value is to be placed in linear memory described by `Memory`.
Memory(Memory<'a>),
}
struct Stack<'a> {
/// The locals that comprise a particular value.
///
/// The length of this list represents the flattened list of types that make
/// up the component value. Each list has the index of the local being
/// accessed as well as the type of the local itself.
locals: &'a [(u32, ValType)],
/// The lifting/lowering options for where this stack of values comes from
opts: &'a Options,
}
/// Representation of where a value is going to be stored in linear memory.
struct Memory<'a> {
/// The lifting/lowering options with memory configuration
opts: &'a Options,
/// The index of the local that contains the base address of where the
/// storage is happening.
addr: TempLocal,
/// A "static" offset that will be baked into wasm instructions for where
/// memory loads/stores happen.
offset: u32,
}
impl Compiler<'_, '_> {
fn compile_adapter(
mut self,
adapter: &AdapterData,
lower_sig: &Signature,
lift_sig: &Signature,
) {
// Check the instance flags required for this trampoline.
//
// This inserts the initial check required by `canon_lower` that the
// caller instance can be left and additionally checks the
// flags on the callee if necessary whether it can be entered.
self.trap_if_not_flag(adapter.lower.flags, FLAG_MAY_LEAVE, Trap::CannotLeave);
if adapter.called_as_export {
self.trap_if_not_flag(adapter.lift.flags, FLAG_MAY_ENTER, Trap::CannotEnter);
self.set_flag(adapter.lift.flags, FLAG_MAY_ENTER, false);
} else if self.module.debug {
self.assert_not_flag(
adapter.lift.flags,
FLAG_MAY_ENTER,
"may_enter should be unset",
);
}
// Perform the translation of arguments. Note that `FLAG_MAY_LEAVE` is
// cleared around this invocation for the callee as per the
// `canon_lift` definition in the spec. Additionally note that the
// precise ordering of traps here is not required since internal state
// is not visible to either instance and a trap will "lock down" both
// instances to no longer be visible. This means that we're free to
// reorder lifts/lowers and flags and such as is necessary and
// convenient here.
//
// TODO: if translation doesn't actually call any functions in either
// instance then there's no need to set/clear the flag here and that can
// be optimized away.
self.set_flag(adapter.lift.flags, FLAG_MAY_LEAVE, false);
let param_locals = lower_sig
.params
.iter()
.enumerate()
.map(|(i, ty)| (i as u32, *ty))
.collect::<Vec<_>>();
self.translate_params(adapter, &param_locals);
self.set_flag(adapter.lift.flags, FLAG_MAY_LEAVE, true);
// With all the arguments on the stack the actual target function is
// now invoked. The core wasm results of the function are then placed
// into locals for result translation afterwards.
self.instruction(Call(adapter.callee.as_u32()));
let mut result_locals = Vec::with_capacity(lift_sig.results.len());
let mut temps = Vec::new();
for ty in lift_sig.results.iter().rev() {
let local = self.local_set_new_tmp(*ty);
result_locals.push((local.idx, *ty));
temps.push(local);
}
result_locals.reverse();
// Like above during the translation of results the caller cannot be
// left (as we might invoke things like `realloc`). Again the precise
// order of everything doesn't matter since intermediate states cannot
// be witnessed, hence the setting of flags here to encapsulate both
// liftings and lowerings.
//
// TODO: like above the management of the `MAY_LEAVE` flag can probably
// be elided here for "simple" results.
self.set_flag(adapter.lower.flags, FLAG_MAY_LEAVE, false);
self.translate_results(adapter, &param_locals, &result_locals);
self.set_flag(adapter.lower.flags, FLAG_MAY_LEAVE, true);
// And finally post-return state is handled here once all results/etc
// are all translated.
if let Some(func) = adapter.lift.post_return {
for (result, _) in result_locals.iter() {
self.instruction(LocalGet(*result));
}
self.instruction(Call(func.as_u32()));
}
if adapter.called_as_export {
self.set_flag(adapter.lift.flags, FLAG_MAY_ENTER, true);
}
for tmp in temps {
self.free_temp_local(tmp);
}
self.finish()
}
fn translate_params(&mut self, adapter: &AdapterData, param_locals: &[(u32, ValType)]) {
let src_tys = &self.types[adapter.lower.ty].params;
let src_tys = src_tys.iter().map(|(_, ty)| *ty).collect::<Vec<_>>();
let dst_tys = &self.types[adapter.lift.ty].params;
let dst_tys = dst_tys.iter().map(|(_, ty)| *ty).collect::<Vec<_>>();
let lift_opts = &adapter.lift.options;
let lower_opts = &adapter.lower.options;
// TODO: handle subtyping
assert_eq!(src_tys.len(), dst_tys.len());
let src_flat =
self.types
.flatten_types(lower_opts, MAX_FLAT_PARAMS, src_tys.iter().copied());
let dst_flat =
self.types
.flatten_types(lift_opts, MAX_FLAT_PARAMS, dst_tys.iter().copied());
let src = if let Some(flat) = &src_flat {
Source::Stack(Stack {
locals: &param_locals[..flat.len()],
opts: lower_opts,
})
} else {
// If there are too many parameters then that means the parameters
// are actually a tuple stored in linear memory addressed by the
// first parameter local.
let (addr, ty) = param_locals[0];
assert_eq!(ty, lower_opts.ptr());
let align = src_tys
.iter()
.map(|t| self.types.align(lower_opts, t))
.max()
.unwrap_or(1);
Source::Memory(self.memory_operand(lower_opts, TempLocal::new(addr, ty), align))
};
let dst = if let Some(flat) = &dst_flat {
Destination::Stack(flat, lift_opts)
} else {
// If there are too many parameters then space is allocated in the
// destination module for the parameters via its `realloc` function.
let abi = CanonicalAbiInfo::record(dst_tys.iter().map(|t| self.types.canonical_abi(t)));
let (size, align) = if lift_opts.memory64 {
(abi.size64, abi.align64)
} else {
(abi.size32, abi.align32)
};
let size = MallocSize::Const(size);
Destination::Memory(self.malloc(lift_opts, size, align))
};
let srcs = src
.record_field_srcs(self.types, src_tys.iter().copied())
.zip(src_tys.iter());
let dsts = dst
.record_field_dsts(self.types, dst_tys.iter().copied())
.zip(dst_tys.iter());
for ((src, src_ty), (dst, dst_ty)) in srcs.zip(dsts) {
self.translate(&src_ty, &src, &dst_ty, &dst);
}
// If the destination was linear memory instead of the stack then the
// actual parameter that we're passing is the address of the values
// stored, so ensure that's happening in the wasm body here.
if let Destination::Memory(mem) = dst {
self.instruction(LocalGet(mem.addr.idx));
self.free_temp_local(mem.addr);
}
}
fn translate_results(
&mut self,
adapter: &AdapterData,
param_locals: &[(u32, ValType)],
result_locals: &[(u32, ValType)],
) {
let src_tys = &self.types[adapter.lift.ty].results;
let src_tys = src_tys.iter().map(|(_, ty)| *ty).collect::<Vec<_>>();
let dst_tys = &self.types[adapter.lower.ty].results;
let dst_tys = dst_tys.iter().map(|(_, ty)| *ty).collect::<Vec<_>>();
let lift_opts = &adapter.lift.options;
let lower_opts = &adapter.lower.options;
let src_flat =
self.types
.flatten_types(lift_opts, MAX_FLAT_RESULTS, src_tys.iter().copied());
let dst_flat =
self.types
.flatten_types(lower_opts, MAX_FLAT_RESULTS, dst_tys.iter().copied());
let src = if src_flat.is_some() {
Source::Stack(Stack {
locals: result_locals,
opts: lift_opts,
})
} else {
// The original results to read from in this case come from the
// return value of the function itself. The imported function will
// return a linear memory address at which the values can be read
// from.
let align = src_tys
.iter()
.map(|t| self.types.align(lift_opts, t))
.max()
.unwrap_or(1);
assert_eq!(result_locals.len(), 1);
let (addr, ty) = result_locals[0];
assert_eq!(ty, lift_opts.ptr());
Source::Memory(self.memory_operand(lift_opts, TempLocal::new(addr, ty), align))
};
let dst = if let Some(flat) = &dst_flat {
Destination::Stack(flat, lower_opts)
} else {
// This is slightly different than `translate_params` where the
// return pointer was provided by the caller of this function
// meaning the last parameter local is a pointer into linear memory.
let align = dst_tys
.iter()
.map(|t| self.types.align(lower_opts, t))
.max()
.unwrap_or(1);
let (addr, ty) = *param_locals.last().expect("no retptr");
assert_eq!(ty, lower_opts.ptr());
Destination::Memory(self.memory_operand(lower_opts, TempLocal::new(addr, ty), align))
};
let srcs = src
.record_field_srcs(self.types, src_tys.iter().copied())
.zip(src_tys.iter());
let dsts = dst
.record_field_dsts(self.types, dst_tys.iter().copied())
.zip(dst_tys.iter());
for ((src, src_ty), (dst, dst_ty)) in srcs.zip(dsts) {
self.translate(&src_ty, &src, &dst_ty, &dst);
}
}
fn translate(
&mut self,
src_ty: &InterfaceType,
src: &Source<'_>,
dst_ty: &InterfaceType,
dst: &Destination,
) {
if let Source::Memory(mem) = src {
self.assert_aligned(src_ty, mem);
}
if let Destination::Memory(mem) = dst {
self.assert_aligned(dst_ty, mem);
}
// Calculate a cost heuristic for what the translation of this specific
// layer of the type is going to incur. The purpose of this cost is that
// we'll deduct it from `self.fuel` and if no fuel is remaining then
// translation is outlined into a separate function rather than being
// translated into this function.
//
// The general goal is to avoid creating an exponentially sized function
// for a linearly sized input (the type section). By outlining helper
// functions there will ideally be a constant set of helper functions
// per type (to accomodate in-memory or on-stack transfers as well as
// src/dst options) which means that each function is at most a certain
// size and we have a linear number of functions which should guarantee
// an overall linear size of the output.
//
// To implement this the current heuristic is that each layer of
// translating a type has a cost associated with it and this cost is
// accounted for in `self.fuel`. Some conversions are considered free as
// they generate basically as much code as the `call` to the translation
// function while other are considered proportionally expensive to the
// size of the type. The hope is that some upper layers are of a type's
// translation are all inlined into one function but bottom layers end
// up getting outlined to separate functions. Theoretically, again this
// is built on hopes and dreams, the outlining can be shared amongst
// tightly-intertwined type hierarchies which will reduce the size of
// the output module due to the helpers being used.
//
// This heuristic of how to split functions has changed a few times in
// the past and this isn't necessarily guaranteed to be the final
// iteration.
let cost = match src_ty {
// These types are all quite simple to load/store and equate to
// basically the same cost of the `call` instruction to call an
// out-of-line translation function, so give them 0 cost.
InterfaceType::Bool
| InterfaceType::U8
| InterfaceType::S8
| InterfaceType::U16
| InterfaceType::S16
| InterfaceType::U32
| InterfaceType::S32
| InterfaceType::U64
| InterfaceType::S64
| InterfaceType::Float32
| InterfaceType::Float64 => 0,
// This has a small amount of validation associated with it, so
// give it a cost of 1.
InterfaceType::Char => 1,
// This has a fair bit of code behind it depending on the
// strings/encodings in play, so arbitrarily assign it this cost.
InterfaceType::String => 40,
// Iteration of a loop is along the lines of the cost of a string
// so give it the same cost
InterfaceType::List(_) => 40,
InterfaceType::Flags(i) => {
let count = self.module.types[*i].names.len();
match FlagsSize::from_count(count) {
FlagsSize::Size0 => 0,
FlagsSize::Size1 | FlagsSize::Size2 => 1,
FlagsSize::Size4Plus(n) => n.into(),
}
}
InterfaceType::Record(i) => self.types[*i].fields.len(),
InterfaceType::Tuple(i) => self.types[*i].types.len(),
InterfaceType::Variant(i) => self.types[*i].cases.len(),
InterfaceType::Union(i) => self.types[*i].types.len(),
InterfaceType::Enum(i) => self.types[*i].names.len(),
// 2 cases to consider for each of these variants.
InterfaceType::Option(_) | InterfaceType::Result(_) => 2,
};
match self.fuel.checked_sub(cost) {
// This function has enough fuel to perform the layer of translation
// necessary for this type, so the fuel is updated in-place and
// translation continues. Note that the recursion here is bounded by
// the static recursion limit for all interface types as imposed
// during the translation phase.
Some(n) => {
self.fuel = n;
match src_ty {
InterfaceType::Bool => self.translate_bool(src, dst_ty, dst),
InterfaceType::U8 => self.translate_u8(src, dst_ty, dst),
InterfaceType::S8 => self.translate_s8(src, dst_ty, dst),
InterfaceType::U16 => self.translate_u16(src, dst_ty, dst),
InterfaceType::S16 => self.translate_s16(src, dst_ty, dst),
InterfaceType::U32 => self.translate_u32(src, dst_ty, dst),
InterfaceType::S32 => self.translate_s32(src, dst_ty, dst),
InterfaceType::U64 => self.translate_u64(src, dst_ty, dst),
InterfaceType::S64 => self.translate_s64(src, dst_ty, dst),
InterfaceType::Float32 => self.translate_f32(src, dst_ty, dst),
InterfaceType::Float64 => self.translate_f64(src, dst_ty, dst),
InterfaceType::Char => self.translate_char(src, dst_ty, dst),
InterfaceType::String => self.translate_string(src, dst_ty, dst),
InterfaceType::List(t) => self.translate_list(*t, src, dst_ty, dst),
InterfaceType::Record(t) => self.translate_record(*t, src, dst_ty, dst),
InterfaceType::Flags(f) => self.translate_flags(*f, src, dst_ty, dst),
InterfaceType::Tuple(t) => self.translate_tuple(*t, src, dst_ty, dst),
InterfaceType::Variant(v) => self.translate_variant(*v, src, dst_ty, dst),
InterfaceType::Union(u) => self.translate_union(*u, src, dst_ty, dst),
InterfaceType::Enum(t) => self.translate_enum(*t, src, dst_ty, dst),
InterfaceType::Option(t) => self.translate_option(*t, src, dst_ty, dst),
InterfaceType::Result(t) => self.translate_result(*t, src, dst_ty, dst),
}
}
// This function does not have enough fuel left to perform this
// layer of translation so the translation is deferred to a helper
// function. The actual translation here is then done by marshalling
// the src/dst into the function we're calling and then processing
// the results.
None => {
let src_loc = match src {
// If the source is on the stack then `stack_get` is used to
// convert everything to the appropriate flat representation
// for the source type.
Source::Stack(stack) => {
for (i, ty) in stack
.opts
.flat_types(src_ty, self.types)
.unwrap()
.iter()
.enumerate()
{
let stack = stack.slice(i..i + 1);
self.stack_get(&stack, (*ty).into());
}
HelperLocation::Stack
}
// If the source is in memory then the pointer is passed
// through, but note that the offset must be factored in
// here since the translation function will start from
// offset 0.
Source::Memory(mem) => {
self.push_mem_addr(mem);
HelperLocation::Memory
}
};
let dst_loc = match dst {
Destination::Stack(..) => HelperLocation::Stack,
Destination::Memory(mem) => {
self.push_mem_addr(mem);
HelperLocation::Memory
}
};
// Generate a `FunctionId` corresponding to the `Helper`
// configuration that is necessary here. This will ideally be a
// "cache hit" and use a preexisting helper which represents
// outlining what would otherwise be duplicate code within a
// function to one function.
let helper = self.module.translate_helper(Helper {
src: HelperType {
ty: *src_ty,
opts: *src.opts(),
loc: src_loc,
},
dst: HelperType {
ty: *dst_ty,
opts: *dst.opts(),
loc: dst_loc,
},
});
// Emit a `call` instruction which will get "relocated" to a
// function index once translation has completely finished.
self.flush_code();
self.module.funcs[self.result].body.push(Body::Call(helper));
// If the destination of the translation was on the stack then
// the types on the stack need to be optionally converted to
// different types (e.g. if the result here is part of a variant
// somewhere else).
//
// This translation happens inline here by popping the results
// into new locals and then using those locals to do a
// `stack_set`.
if let Destination::Stack(tys, opts) = dst {
let flat = self
.types
.flatten_types(opts, usize::MAX, [*dst_ty])
.unwrap();
assert_eq!(flat.len(), tys.len());
let locals = flat
.iter()
.rev()
.map(|ty| self.local_set_new_tmp(*ty))
.collect::<Vec<_>>();
for (ty, local) in tys.iter().zip(locals.into_iter().rev()) {
self.instruction(LocalGet(local.idx));
self.stack_set(std::slice::from_ref(ty), local.ty);
self.free_temp_local(local);
}
}
}
}
}
fn push_mem_addr(&mut self, mem: &Memory<'_>) {
self.instruction(LocalGet(mem.addr.idx));
if mem.offset != 0 {
self.ptr_uconst(mem.opts, mem.offset);
self.ptr_add(mem.opts);
}
}
fn translate_bool(&mut self, src: &Source<'_>, dst_ty: &InterfaceType, dst: &Destination) {
// TODO: subtyping
assert!(matches!(dst_ty, InterfaceType::Bool));
self.push_dst_addr(dst);
// Booleans are canonicalized to 0 or 1 as they pass through the
// component boundary, so use a `select` instruction to do so.
self.instruction(I32Const(1));
self.instruction(I32Const(0));
match src {
Source::Memory(mem) => self.i32_load8u(mem),
Source::Stack(stack) => self.stack_get(stack, ValType::I32),
}
self.instruction(Select);
match dst {
Destination::Memory(mem) => self.i32_store8(mem),
Destination::Stack(stack, _) => self.stack_set(stack, ValType::I32),
}
}
fn translate_u8(&mut self, src: &Source<'_>, dst_ty: &InterfaceType, dst: &Destination) {
// TODO: subtyping
assert!(matches!(dst_ty, InterfaceType::U8));
self.convert_u8_mask(src, dst, 0xff);
}
fn convert_u8_mask(&mut self, src: &Source<'_>, dst: &Destination<'_>, mask: u8) {
self.push_dst_addr(dst);
let mut needs_mask = true;
match src {
Source::Memory(mem) => {
self.i32_load8u(mem);
needs_mask = mask != 0xff;
}
Source::Stack(stack) => {
self.stack_get(stack, ValType::I32);
}
}
if needs_mask {
self.instruction(I32Const(i32::from(mask)));
self.instruction(I32And);
}
match dst {
Destination::Memory(mem) => self.i32_store8(mem),
Destination::Stack(stack, _) => self.stack_set(stack, ValType::I32),
}
}
fn translate_s8(&mut self, src: &Source<'_>, dst_ty: &InterfaceType, dst: &Destination) {
// TODO: subtyping
assert!(matches!(dst_ty, InterfaceType::S8));
self.push_dst_addr(dst);
match src {
Source::Memory(mem) => self.i32_load8s(mem),
Source::Stack(stack) => {
self.stack_get(stack, ValType::I32);
self.instruction(I32Extend8S);
}
}
match dst {
Destination::Memory(mem) => self.i32_store8(mem),
Destination::Stack(stack, _) => self.stack_set(stack, ValType::I32),
}
}
fn translate_u16(&mut self, src: &Source<'_>, dst_ty: &InterfaceType, dst: &Destination) {
// TODO: subtyping
assert!(matches!(dst_ty, InterfaceType::U16));
self.convert_u16_mask(src, dst, 0xffff);
}
fn convert_u16_mask(&mut self, src: &Source<'_>, dst: &Destination<'_>, mask: u16) {
self.push_dst_addr(dst);
let mut needs_mask = true;
match src {
Source::Memory(mem) => {
self.i32_load16u(mem);
needs_mask = mask != 0xffff;
}
Source::Stack(stack) => {
self.stack_get(stack, ValType::I32);
}
}
if needs_mask {
self.instruction(I32Const(i32::from(mask)));
self.instruction(I32And);
}
match dst {
Destination::Memory(mem) => self.i32_store16(mem),
Destination::Stack(stack, _) => self.stack_set(stack, ValType::I32),
}
}
fn translate_s16(&mut self, src: &Source<'_>, dst_ty: &InterfaceType, dst: &Destination) {
// TODO: subtyping
assert!(matches!(dst_ty, InterfaceType::S16));
self.push_dst_addr(dst);
match src {
Source::Memory(mem) => self.i32_load16s(mem),
Source::Stack(stack) => {
self.stack_get(stack, ValType::I32);
self.instruction(I32Extend16S);
}
}
match dst {
Destination::Memory(mem) => self.i32_store16(mem),
Destination::Stack(stack, _) => self.stack_set(stack, ValType::I32),
}
}
fn translate_u32(&mut self, src: &Source<'_>, dst_ty: &InterfaceType, dst: &Destination) {
// TODO: subtyping
assert!(matches!(dst_ty, InterfaceType::U32));
self.convert_u32_mask(src, dst, 0xffffffff)
}
fn convert_u32_mask(&mut self, src: &Source<'_>, dst: &Destination<'_>, mask: u32) {
self.push_dst_addr(dst);
match src {
Source::Memory(mem) => self.i32_load(mem),
Source::Stack(stack) => self.stack_get(stack, ValType::I32),
}
if mask != 0xffffffff {
self.instruction(I32Const(mask as i32));
self.instruction(I32And);
}
match dst {
Destination::Memory(mem) => self.i32_store(mem),
Destination::Stack(stack, _) => self.stack_set(stack, ValType::I32),
}
}
fn translate_s32(&mut self, src: &Source<'_>, dst_ty: &InterfaceType, dst: &Destination) {
// TODO: subtyping
assert!(matches!(dst_ty, InterfaceType::S32));
self.push_dst_addr(dst);
match src {
Source::Memory(mem) => self.i32_load(mem),
Source::Stack(stack) => self.stack_get(stack, ValType::I32),
}
match dst {
Destination::Memory(mem) => self.i32_store(mem),
Destination::Stack(stack, _) => self.stack_set(stack, ValType::I32),
}
}
fn translate_u64(&mut self, src: &Source<'_>, dst_ty: &InterfaceType, dst: &Destination) {
// TODO: subtyping
assert!(matches!(dst_ty, InterfaceType::U64));
self.push_dst_addr(dst);
match src {
Source::Memory(mem) => self.i64_load(mem),
Source::Stack(stack) => self.stack_get(stack, ValType::I64),
}
match dst {
Destination::Memory(mem) => self.i64_store(mem),
Destination::Stack(stack, _) => self.stack_set(stack, ValType::I64),
}
}
fn translate_s64(&mut self, src: &Source<'_>, dst_ty: &InterfaceType, dst: &Destination) {
// TODO: subtyping
assert!(matches!(dst_ty, InterfaceType::S64));
self.push_dst_addr(dst);
match src {
Source::Memory(mem) => self.i64_load(mem),
Source::Stack(stack) => self.stack_get(stack, ValType::I64),
}
match dst {
Destination::Memory(mem) => self.i64_store(mem),
Destination::Stack(stack, _) => self.stack_set(stack, ValType::I64),
}
}
fn translate_f32(&mut self, src: &Source<'_>, dst_ty: &InterfaceType, dst: &Destination) {
// TODO: subtyping
assert!(matches!(dst_ty, InterfaceType::Float32));
self.push_dst_addr(dst);
match src {
Source::Memory(mem) => self.f32_load(mem),
Source::Stack(stack) => self.stack_get(stack, ValType::F32),
}
match dst {
Destination::Memory(mem) => self.f32_store(mem),
Destination::Stack(stack, _) => self.stack_set(stack, ValType::F32),
}
}
fn translate_f64(&mut self, src: &Source<'_>, dst_ty: &InterfaceType, dst: &Destination) {
// TODO: subtyping
assert!(matches!(dst_ty, InterfaceType::Float64));
self.push_dst_addr(dst);
match src {
Source::Memory(mem) => self.f64_load(mem),
Source::Stack(stack) => self.stack_get(stack, ValType::F64),
}
match dst {
Destination::Memory(mem) => self.f64_store(mem),
Destination::Stack(stack, _) => self.stack_set(stack, ValType::F64),
}
}
fn translate_char(&mut self, src: &Source<'_>, dst_ty: &InterfaceType, dst: &Destination) {
assert!(matches!(dst_ty, InterfaceType::Char));
match src {
Source::Memory(mem) => self.i32_load(mem),
Source::Stack(stack) => self.stack_get(stack, ValType::I32),
}
let local = self.local_set_new_tmp(ValType::I32);
// This sequence is copied from the output of LLVM for:
//
// pub extern "C" fn foo(x: u32) -> char {
// char::try_from(x)
// .unwrap_or_else(|_| std::arch::wasm32::unreachable())
// }
//
// Apparently this does what's required by the canonical ABI:
//
// def i32_to_char(opts, i):
// trap_if(i >= 0x110000)
// trap_if(0xD800 <= i <= 0xDFFF)
// return chr(i)
//
// ... but I don't know how it works other than "well I trust LLVM"
self.instruction(Block(BlockType::Empty));
self.instruction(Block(BlockType::Empty));
self.instruction(LocalGet(local.idx));
self.instruction(I32Const(0xd800));
self.instruction(I32Xor);
self.instruction(I32Const(-0x110000));
self.instruction(I32Add);
self.instruction(I32Const(-0x10f800));
self.instruction(I32LtU);
self.instruction(BrIf(0));
self.instruction(LocalGet(local.idx));
self.instruction(I32Const(0x110000));
self.instruction(I32Ne);
self.instruction(BrIf(1));
self.instruction(End);
self.trap(Trap::InvalidChar);
self.instruction(End);
self.push_dst_addr(dst);
self.instruction(LocalGet(local.idx));
match dst {
Destination::Memory(mem) => {
self.i32_store(mem);
}
Destination::Stack(stack, _) => self.stack_set(stack, ValType::I32),
}
self.free_temp_local(local);
}
fn translate_string(&mut self, src: &Source<'_>, dst_ty: &InterfaceType, dst: &Destination) {
assert!(matches!(dst_ty, InterfaceType::String));
let src_opts = src.opts();
let dst_opts = dst.opts();
// Load the pointer/length of this string into temporary locals. These
// will be referenced a good deal so this just makes it easier to deal
// with them consistently below rather than trying to reload from memory
// for example.
match src {
Source::Stack(s) => {
assert_eq!(s.locals.len(), 2);
self.stack_get(&s.slice(0..1), src_opts.ptr());
self.stack_get(&s.slice(1..2), src_opts.ptr());
}
Source::Memory(mem) => {
self.ptr_load(mem);
self.ptr_load(&mem.bump(src_opts.ptr_size().into()));
}
}
let src_len = self.local_set_new_tmp(src_opts.ptr());
let src_ptr = self.local_set_new_tmp(src_opts.ptr());
let src_str = WasmString {
ptr: src_ptr,
len: src_len,
opts: src_opts,
};
let dst_str = match src_opts.string_encoding {
StringEncoding::Utf8 => match dst_opts.string_encoding {
StringEncoding::Utf8 => self.string_copy(&src_str, FE::Utf8, dst_opts, FE::Utf8),
StringEncoding::Utf16 => self.string_utf8_to_utf16(&src_str, dst_opts),
StringEncoding::CompactUtf16 => {
self.string_to_compact(&src_str, FE::Utf8, dst_opts)
}
},
StringEncoding::Utf16 => {
self.verify_aligned(src_opts, src_str.ptr.idx, 2);
match dst_opts.string_encoding {
StringEncoding::Utf8 => {
self.string_deflate_to_utf8(&src_str, FE::Utf16, dst_opts)
}
StringEncoding::Utf16 => {
self.string_copy(&src_str, FE::Utf16, dst_opts, FE::Utf16)
}
StringEncoding::CompactUtf16 => {
self.string_to_compact(&src_str, FE::Utf16, dst_opts)
}
}
}
StringEncoding::CompactUtf16 => {
self.verify_aligned(src_opts, src_str.ptr.idx, 2);
// Test the tag big to see if this is a utf16 or a latin1 string
// at runtime...
self.instruction(LocalGet(src_str.len.idx));
self.ptr_uconst(src_opts, UTF16_TAG);
self.ptr_and(src_opts);
self.ptr_if(src_opts, BlockType::Empty);
// In the utf16 block unset the upper bit from the length local
// so further calculations have the right value. Afterwards the
// string transcode proceeds assuming utf16.
self.instruction(LocalGet(src_str.len.idx));
self.ptr_uconst(src_opts, UTF16_TAG);
self.ptr_xor(src_opts);
self.instruction(LocalSet(src_str.len.idx));
let s1 = match dst_opts.string_encoding {
StringEncoding::Utf8 => {
self.string_deflate_to_utf8(&src_str, FE::Utf16, dst_opts)
}
StringEncoding::Utf16 => {
self.string_copy(&src_str, FE::Utf16, dst_opts, FE::Utf16)
}
StringEncoding::CompactUtf16 => {
self.string_compact_utf16_to_compact(&src_str, dst_opts)
}
};
self.instruction(Else);
// In the latin1 block the `src_len` local is already the number
// of code units, so the string transcoding is all that needs to
// happen.
let s2 = match dst_opts.string_encoding {
StringEncoding::Utf16 => {
self.string_copy(&src_str, FE::Latin1, dst_opts, FE::Utf16)
}
StringEncoding::Utf8 => {
self.string_deflate_to_utf8(&src_str, FE::Latin1, dst_opts)
}
StringEncoding::CompactUtf16 => {
self.string_copy(&src_str, FE::Latin1, dst_opts, FE::Latin1)
}
};
// Set our `s2` generated locals to the `s2` generated locals
// as the resulting pointer of this transcode.
self.instruction(LocalGet(s2.ptr.idx));
self.instruction(LocalSet(s1.ptr.idx));
self.instruction(LocalGet(s2.len.idx));
self.instruction(LocalSet(s1.len.idx));
self.instruction(End);
self.free_temp_local(s2.ptr);
self.free_temp_local(s2.len);
s1
}
};
// Store the ptr/length in the desired destination
match dst {
Destination::Stack(s, _) => {
self.instruction(LocalGet(dst_str.ptr.idx));
self.stack_set(&s[..1], dst_opts.ptr());
self.instruction(LocalGet(dst_str.len.idx));
self.stack_set(&s[1..], dst_opts.ptr());
}
Destination::Memory(mem) => {
self.instruction(LocalGet(mem.addr.idx));
self.instruction(LocalGet(dst_str.ptr.idx));
self.ptr_store(mem);
self.instruction(LocalGet(mem.addr.idx));
self.instruction(LocalGet(dst_str.len.idx));
self.ptr_store(&mem.bump(dst_opts.ptr_size().into()));
}
}
self.free_temp_local(src_str.ptr);
self.free_temp_local(src_str.len);
self.free_temp_local(dst_str.ptr);
self.free_temp_local(dst_str.len);
}
// Corresponding function for `store_string_copy` in the spec.
//
// This performs a transcoding of the string with a one-pass copy from
// the `src` encoding to the `dst` encoding. This is only possible for
// fixed encodings where the first allocation is guaranteed to be an
// appropriate fit so it's not suitable for all encodings.
//
// Imported host transcoding functions here take the src/dst pointers as
// well as the number of code units in the source (which always matches
// the number of code units in the destination). There is no return
// value from the transcode function since the encoding should always
// work on the first pass.
fn string_copy<'a>(
&mut self,
src: &WasmString<'_>,
src_enc: FE,
dst_opts: &'a Options,
dst_enc: FE,
) -> WasmString<'a> {
assert!(dst_enc.width() >= src_enc.width());
self.validate_string_length(src, dst_enc);
// Calculate the source byte length given the size of each code
// unit. Note that this shouldn't overflow given
// `validate_string_length` above.
let mut src_byte_len_tmp = None;
let src_byte_len = if src_enc.width() == 1 {
src.len.idx
} else {
assert_eq!(src_enc.width(), 2);
self.instruction(LocalGet(src.len.idx));
self.ptr_uconst(src.opts, 1);
self.ptr_shl(src.opts);
let tmp = self.local_set_new_tmp(src.opts.ptr());
let ret = tmp.idx;
src_byte_len_tmp = Some(tmp);
ret
};
// Convert the source code units length to the destination byte
// length type.
self.convert_src_len_to_dst(src.len.idx, src.opts.ptr(), dst_opts.ptr());
let dst_len = self.local_tee_new_tmp(dst_opts.ptr());
if dst_enc.width() > 1 {
assert_eq!(dst_enc.width(), 2);
self.ptr_uconst(dst_opts, 1);
self.ptr_shl(dst_opts);
}
let dst_byte_len = self.local_set_new_tmp(dst_opts.ptr());
// Allocate space in the destination using the calculated byte
// length.
let dst = {
let dst_mem = self.malloc(
dst_opts,
MallocSize::Local(dst_byte_len.idx),
dst_enc.width().into(),
);
WasmString {
ptr: dst_mem.addr,
len: dst_len,
opts: dst_opts,
}
};
// Validate that `src_len + src_ptr` and
// `dst_mem.addr_local + dst_byte_len` are both in-bounds. This
// is done by loading the last byte of the string and if that
// doesn't trap then it's known valid.
self.validate_string_inbounds(src, src_byte_len);
self.validate_string_inbounds(&dst, dst_byte_len.idx);
// If the validations pass then the host `transcode` intrinsic
// is invoked. This will either raise a trap or otherwise succeed
// in which case we're done.
let op = if src_enc == dst_enc {
Transcode::Copy(src_enc)
} else {
assert_eq!(src_enc, FE::Latin1);
assert_eq!(dst_enc, FE::Utf16);
Transcode::Latin1ToUtf16
};
let transcode = self.transcoder(src, &dst, op);
self.instruction(LocalGet(src.ptr.idx));
self.instruction(LocalGet(src.len.idx));
self.instruction(LocalGet(dst.ptr.idx));
self.instruction(Call(transcode.as_u32()));
self.free_temp_local(dst_byte_len);
if let Some(tmp) = src_byte_len_tmp {
self.free_temp_local(tmp);
}
dst
}
// Corresponding function for `store_string_to_utf8` in the spec.
//
// This translation works by possibly performing a number of
// reallocations. First a buffer of size input-code-units is used to try
// to get the transcoding correct on the first try. If that fails the
// maximum worst-case size is used and then that is resized down if it's
// too large.
//
// The host transcoding function imported here will receive src ptr/len
// and dst ptr/len and return how many code units were consumed on both
// sides. The amount of code units consumed in the source dictates which
// branches are taken in this conversion.
fn string_deflate_to_utf8<'a>(
&mut self,
src: &WasmString<'_>,
src_enc: FE,
dst_opts: &'a Options,
) -> WasmString<'a> {
self.validate_string_length(src, src_enc);
// Optimistically assume that the code unit length of the source is
// all that's needed in the destination. Perform that allocaiton
// here and proceed to transcoding below.
self.convert_src_len_to_dst(src.len.idx, src.opts.ptr(), dst_opts.ptr());
let dst_len = self.local_tee_new_tmp(dst_opts.ptr());
let dst_byte_len = self.local_set_new_tmp(dst_opts.ptr());
let dst = {
let dst_mem = self.malloc(dst_opts, MallocSize::Local(dst_byte_len.idx), 1);
WasmString {
ptr: dst_mem.addr,
len: dst_len,
opts: dst_opts,
}
};
// Ensure buffers are all in-bounds
let mut src_byte_len_tmp = None;
let src_byte_len = match src_enc {
FE::Latin1 => src.len.idx,
FE::Utf16 => {
self.instruction(LocalGet(src.len.idx));
self.ptr_uconst(src.opts, 1);
self.ptr_shl(src.opts);
let tmp = self.local_set_new_tmp(src.opts.ptr());
let ret = tmp.idx;
src_byte_len_tmp = Some(tmp);
ret
}
FE::Utf8 => unreachable!(),
};
self.validate_string_inbounds(src, src_byte_len);
self.validate_string_inbounds(&dst, dst_byte_len.idx);
// Perform the initial transcode
let op = match src_enc {
FE::Latin1 => Transcode::Latin1ToUtf8,
FE::Utf16 => Transcode::Utf16ToUtf8,
FE::Utf8 => unreachable!(),
};
let transcode = self.transcoder(src, &dst, op);
self.instruction(LocalGet(src.ptr.idx));
self.instruction(LocalGet(src.len.idx));
self.instruction(LocalGet(dst.ptr.idx));
self.instruction(LocalGet(dst_byte_len.idx));
self.instruction(Call(transcode.as_u32()));
self.instruction(LocalSet(dst.len.idx));
let src_len_tmp = self.local_set_new_tmp(src.opts.ptr());
// Test if the source was entirely transcoded by comparing
// `src_len_tmp`, the number of code units transcoded from the
// source, with `src_len`, the original number of code units.
self.instruction(LocalGet(src_len_tmp.idx));
self.instruction(LocalGet(src.len.idx));
self.ptr_ne(src.opts);
self.instruction(If(BlockType::Empty));
// Here a worst-case reallocation is performed to grow `dst_mem`.
// In-line a check is also performed that the worst-case byte size
// fits within the maximum size of strings.
self.instruction(LocalGet(dst.ptr.idx)); // old_ptr
self.instruction(LocalGet(dst_byte_len.idx)); // old_size
self.ptr_uconst(dst.opts, 1); // align
let factor = match src_enc {
FE::Latin1 => 2,
FE::Utf16 => 3,
_ => unreachable!(),
};
self.validate_string_length_u8(src, factor);
self.convert_src_len_to_dst(src.len.idx, src.opts.ptr(), dst_opts.ptr());
self.ptr_uconst(dst_opts, factor.into());
self.ptr_mul(dst_opts);
self.instruction(LocalTee(dst_byte_len.idx));
self.instruction(Call(dst_opts.realloc.unwrap().as_u32()));
self.instruction(LocalSet(dst.ptr.idx));
// Verify that the destination is still in-bounds
self.validate_string_inbounds(&dst, dst_byte_len.idx);
// Perform another round of transcoding that should be guaranteed
// to succeed. Note that all the parameters here are offset by the
// results of the first transcoding to only perform the remaining
// transcode on the final units.
self.instruction(LocalGet(src.ptr.idx));
self.instruction(LocalGet(src_len_tmp.idx));
if let FE::Utf16 = src_enc {
self.ptr_uconst(src.opts, 1);
self.ptr_shl(src.opts);
}
self.ptr_add(src.opts);
self.instruction(LocalGet(src.len.idx));
self.instruction(LocalGet(src_len_tmp.idx));
self.ptr_sub(src.opts);
self.instruction(LocalGet(dst.ptr.idx));
self.instruction(LocalGet(dst.len.idx));
self.ptr_add(dst.opts);
self.instruction(LocalGet(dst_byte_len.idx));
self.instruction(LocalGet(dst.len.idx));
self.ptr_sub(dst.opts);
self.instruction(Call(transcode.as_u32()));
// Add the second result, the amount of destination units encoded,
// to `dst_len` so it's an accurate reflection of the final size of
// the destination buffer.
self.instruction(LocalGet(dst.len.idx));
self.ptr_add(dst.opts);
self.instruction(LocalSet(dst.len.idx));
// In debug mode verify the first result consumed the entire string,
// otherwise simply discard it.
if self.module.debug {
self.instruction(LocalGet(src.len.idx));
self.instruction(LocalGet(src_len_tmp.idx));
self.ptr_sub(src.opts);
self.ptr_ne(src.opts);
self.instruction(If(BlockType::Empty));
self.trap(Trap::AssertFailed("should have finished encoding"));
self.instruction(End);
} else {
self.instruction(Drop);
}
// Perform a downsizing if the worst-case size was too large
self.instruction(LocalGet(dst.len.idx));
self.instruction(LocalGet(dst_byte_len.idx));
self.ptr_ne(dst.opts);
self.instruction(If(BlockType::Empty));
self.instruction(LocalGet(dst.ptr.idx)); // old_ptr
self.instruction(LocalGet(dst_byte_len.idx)); // old_size
self.ptr_uconst(dst.opts, 1); // align
self.instruction(LocalGet(dst.len.idx)); // new_size
self.instruction(Call(dst.opts.realloc.unwrap().as_u32()));
self.instruction(LocalSet(dst.ptr.idx));
self.instruction(End);
// If the first transcode was enough then assert that the returned
// amount of destination items written equals the byte size.
if self.module.debug {
self.instruction(Else);
self.instruction(LocalGet(dst.len.idx));
self.instruction(LocalGet(dst_byte_len.idx));
self.ptr_ne(dst_opts);
self.instruction(If(BlockType::Empty));
self.trap(Trap::AssertFailed("should have finished encoding"));
self.instruction(End);
}
self.instruction(End); // end of "first transcode not enough"
self.free_temp_local(src_len_tmp);
self.free_temp_local(dst_byte_len);
if let Some(tmp) = src_byte_len_tmp {
self.free_temp_local(tmp);
}
dst
}
// Corresponds to the `store_utf8_to_utf16` function in the spec.
//
// When converting utf-8 to utf-16 a pessimistic allocation is
// done which is twice the byte length of the utf-8 string.
// The host then transcodes and returns how many code units were
// actually used during the transcoding and if it's beneath the
// pessimistic maximum then the buffer is reallocated down to
// a smaller amount.
//
// The host-imported transcoding function takes the src/dst pointer as
// well as the code unit size of both the source and destination. The
// destination should always be big enough to hold the result of the
// transcode and so the result of the host function is how many code
// units were written to the destination.
fn string_utf8_to_utf16<'a>(
&mut self,
src: &WasmString<'_>,
dst_opts: &'a Options,
) -> WasmString<'a> {
self.validate_string_length(src, FE::Utf16);
self.convert_src_len_to_dst(src.len.idx, src.opts.ptr(), dst_opts.ptr());
let dst_len = self.local_tee_new_tmp(dst_opts.ptr());
self.ptr_uconst(dst_opts, 1);
self.ptr_shl(dst_opts);
let dst_byte_len = self.local_set_new_tmp(dst_opts.ptr());
let dst = {
let dst_mem = self.malloc(dst_opts, MallocSize::Local(dst_byte_len.idx), 2);
WasmString {
ptr: dst_mem.addr,
len: dst_len,
opts: dst_opts,
}
};
self.validate_string_inbounds(src, src.len.idx);
self.validate_string_inbounds(&dst, dst_byte_len.idx);
let transcode = self.transcoder(src, &dst, Transcode::Utf8ToUtf16);
self.instruction(LocalGet(src.ptr.idx));
self.instruction(LocalGet(src.len.idx));
self.instruction(LocalGet(dst.ptr.idx));
self.instruction(Call(transcode.as_u32()));
self.instruction(LocalSet(dst.len.idx));
// If the number of code units returned by transcode is not
// equal to the original number of code units then
// the buffer must be shrunk.
//
// Note that the byte length of the final allocation we
// want is twice the code unit length returned by the
// transcoding function.
self.convert_src_len_to_dst(src.len.idx, src.opts.ptr(), dst.opts.ptr());
self.instruction(LocalGet(dst.len.idx));
self.ptr_ne(dst_opts);
self.instruction(If(BlockType::Empty));
self.instruction(LocalGet(dst.ptr.idx));
self.instruction(LocalGet(dst_byte_len.idx));
self.ptr_uconst(dst.opts, 2);
self.instruction(LocalGet(dst.len.idx));
self.ptr_uconst(dst.opts, 1);
self.ptr_shl(dst.opts);
self.instruction(Call(dst.opts.realloc.unwrap().as_u32()));
self.instruction(LocalSet(dst.ptr.idx));
self.instruction(End); // end of shrink-to-fit
self.free_temp_local(dst_byte_len);
dst
}
// Corresponds to `store_probably_utf16_to_latin1_or_utf16` in the spec.
//
// This will try to transcode the input utf16 string to utf16 in the
// destination. If utf16 isn't needed though and latin1 could be used
// then that's used instead and a reallocation to downsize occurs
// afterwards.
//
// The host transcode function here will take the src/dst pointers as
// well as src length. The destination byte length is twice the src code
// unit length. The return value is the tagged length of the returned
// string. If the upper bit is set then utf16 was used and the
// conversion is done. If the upper bit is not set then latin1 was used
// and a downsizing needs to happen.
fn string_compact_utf16_to_compact<'a>(
&mut self,
src: &WasmString<'_>,
dst_opts: &'a Options,
) -> WasmString<'a> {
self.validate_string_length(src, FE::Utf16);
self.convert_src_len_to_dst(src.len.idx, src.opts.ptr(), dst_opts.ptr());
let dst_len = self.local_tee_new_tmp(dst_opts.ptr());
self.ptr_uconst(dst_opts, 1);
self.ptr_shl(dst_opts);
let dst_byte_len = self.local_set_new_tmp(dst_opts.ptr());
let dst = {
let dst_mem = self.malloc(dst_opts, MallocSize::Local(dst_byte_len.idx), 2);
WasmString {
ptr: dst_mem.addr,
len: dst_len,
opts: dst_opts,
}
};
self.convert_src_len_to_dst(dst_byte_len.idx, dst.opts.ptr(), src.opts.ptr());
let src_byte_len = self.local_set_new_tmp(src.opts.ptr());
self.validate_string_inbounds(src, src_byte_len.idx);
self.validate_string_inbounds(&dst, dst_byte_len.idx);
let transcode = self.transcoder(src, &dst, Transcode::Utf16ToCompactProbablyUtf16);
self.instruction(LocalGet(src.ptr.idx));
self.instruction(LocalGet(src.len.idx));
self.instruction(LocalGet(dst.ptr.idx));
self.instruction(Call(transcode.as_u32()));
self.instruction(LocalSet(dst.len.idx));
// Assert that the untagged code unit length is the same as the
// source code unit length.
if self.module.debug {
self.instruction(LocalGet(dst.len.idx));
self.ptr_uconst(dst.opts, !UTF16_TAG);
self.ptr_and(dst.opts);
self.convert_src_len_to_dst(src.len.idx, src.opts.ptr(), dst.opts.ptr());
self.ptr_ne(dst.opts);
self.instruction(If(BlockType::Empty));
self.trap(Trap::AssertFailed("expected equal code units"));
self.instruction(End);
}
// If the UTF16_TAG is set then utf16 was used and the destination
// should be appropriately sized. Bail out of the "is this string
// empty" block and fall through otherwise to resizing.
self.instruction(LocalGet(dst.len.idx));
self.ptr_uconst(dst.opts, UTF16_TAG);
self.ptr_and(dst.opts);
self.ptr_br_if(dst.opts, 0);
// Here `realloc` is used to downsize the string
self.instruction(LocalGet(dst.ptr.idx)); // old_ptr
self.instruction(LocalGet(dst_byte_len.idx)); // old_size
self.ptr_uconst(dst.opts, 2); // align
self.instruction(LocalGet(dst.len.idx)); // new_size
self.instruction(Call(dst.opts.realloc.unwrap().as_u32()));
self.instruction(LocalSet(dst.ptr.idx));
self.free_temp_local(dst_byte_len);
self.free_temp_local(src_byte_len);
dst
}
// Corresponds to `store_string_to_latin1_or_utf16` in the spec.
//
// This will attempt a first pass of transcoding to latin1 and on
// failure a larger buffer is allocated for utf16 and then utf16 is
// encoded in-place into the buffer. After either latin1 or utf16 the
// buffer is then resized to fit the final string allocation.
fn string_to_compact<'a>(
&mut self,
src: &WasmString<'_>,
src_enc: FE,
dst_opts: &'a Options,
) -> WasmString<'a> {
self.validate_string_length(src, src_enc);
self.convert_src_len_to_dst(src.len.idx, src.opts.ptr(), dst_opts.ptr());
let dst_len = self.local_tee_new_tmp(dst_opts.ptr());
let dst_byte_len = self.local_set_new_tmp(dst_opts.ptr());
let dst = {
let dst_mem = self.malloc(dst_opts, MallocSize::Local(dst_byte_len.idx), 2);
WasmString {
ptr: dst_mem.addr,
len: dst_len,
opts: dst_opts,
}
};
self.validate_string_inbounds(src, src.len.idx);
self.validate_string_inbounds(&dst, dst_byte_len.idx);
// Perform the initial latin1 transcode. This returns the number of
// source code units consumed and the number of destination code
// units (bytes) written.
let (latin1, utf16) = match src_enc {
FE::Utf8 => (Transcode::Utf8ToLatin1, Transcode::Utf8ToCompactUtf16),
FE::Utf16 => (Transcode::Utf16ToLatin1, Transcode::Utf16ToCompactUtf16),
FE::Latin1 => unreachable!(),
};
let transcode_latin1 = self.transcoder(src, &dst, latin1);
let transcode_utf16 = self.transcoder(src, &dst, utf16);
self.instruction(LocalGet(src.ptr.idx));
self.instruction(LocalGet(src.len.idx));
self.instruction(LocalGet(dst.ptr.idx));
self.instruction(Call(transcode_latin1.as_u32()));
self.instruction(LocalSet(dst.len.idx));
let src_len_tmp = self.local_set_new_tmp(src.opts.ptr());
// If the source was entirely consumed then the transcode completed
// and all that's necessary is to optionally shrink the buffer.
self.instruction(LocalGet(src_len_tmp.idx));
self.instruction(LocalGet(src.len.idx));
self.ptr_eq(src.opts);
self.instruction(If(BlockType::Empty)); // if latin1-or-utf16 block
// Test if the original byte length of the allocation is the same as
// the number of written bytes, and if not then shrink the buffer
// with a call to `realloc`.
self.instruction(LocalGet(dst_byte_len.idx));
self.instruction(LocalGet(dst.len.idx));
self.ptr_ne(dst.opts);
self.instruction(If(BlockType::Empty));
self.instruction(LocalGet(dst.ptr.idx)); // old_ptr
self.instruction(LocalGet(dst_byte_len.idx)); // old_size
self.ptr_uconst(dst.opts, 2); // align
self.instruction(LocalGet(dst.len.idx)); // new_size
self.instruction(Call(dst.opts.realloc.unwrap().as_u32()));
self.instruction(LocalSet(dst.ptr.idx));
self.instruction(End);
// In this block the latin1 encoding failed. The host transcode
// returned how many units were consumed from the source and how
// many bytes were written to the destination. Here the buffer is
// inflated and sized and the second utf16 intrinsic is invoked to
// perform the final inflation.
self.instruction(Else); // else latin1-or-utf16 block
// For utf8 validate that the inflated size is still within bounds.
if src_enc.width() == 1 {
self.validate_string_length_u8(src, 2);
}
// Reallocate the buffer with twice the source code units in byte
// size.
self.instruction(LocalGet(dst.ptr.idx)); // old_ptr
self.instruction(LocalGet(dst_byte_len.idx)); // old_size
self.ptr_uconst(dst.opts, 2); // align
self.convert_src_len_to_dst(src.len.idx, src.opts.ptr(), dst.opts.ptr());
self.ptr_uconst(dst.opts, 1);
self.ptr_shl(dst.opts);
self.instruction(LocalTee(dst_byte_len.idx));
self.instruction(Call(dst.opts.realloc.unwrap().as_u32()));
self.instruction(LocalSet(dst.ptr.idx));
// Call the host utf16 transcoding function. This will inflate the
// prior latin1 bytes and then encode the rest of the source string
// as utf16 into the remaining space in the destination buffer.
self.instruction(LocalGet(src.ptr.idx));
self.instruction(LocalGet(src_len_tmp.idx));
if let FE::Utf16 = src_enc {
self.ptr_uconst(src.opts, 1);
self.ptr_shl(src.opts);
}
self.ptr_add(src.opts);
self.instruction(LocalGet(src.len.idx));
self.instruction(LocalGet(src_len_tmp.idx));
self.ptr_sub(src.opts);
self.instruction(LocalGet(dst.ptr.idx));
self.convert_src_len_to_dst(src.len.idx, src.opts.ptr(), dst.opts.ptr());
self.instruction(LocalGet(dst.len.idx));
self.instruction(Call(transcode_utf16.as_u32()));
self.instruction(LocalSet(dst.len.idx));
// If the returned number of code units written to the destination
// is not equal to the size of the allocation then the allocation is
// resized down to the appropriate size.
//
// Note that the byte size desired is `2*dst_len` and the current
// byte buffer size is `2*src_len` so the `2` factor isn't checked
// here, just the lengths.
self.instruction(LocalGet(dst.len.idx));
self.convert_src_len_to_dst(src.len.idx, src.opts.ptr(), dst.opts.ptr());
self.ptr_ne(dst.opts);
self.instruction(If(BlockType::Empty));
self.instruction(LocalGet(dst.ptr.idx)); // old_ptr
self.instruction(LocalGet(dst_byte_len.idx)); // old_size
self.ptr_uconst(dst.opts, 2); // align
self.instruction(LocalGet(dst.len.idx));
self.ptr_uconst(dst.opts, 1);
self.ptr_shl(dst.opts);
self.instruction(Call(dst.opts.realloc.unwrap().as_u32()));
self.instruction(LocalSet(dst.ptr.idx));
self.instruction(End);
// Tag the returned pointer as utf16
self.instruction(LocalGet(dst.len.idx));
self.ptr_uconst(dst.opts, UTF16_TAG);
self.ptr_or(dst.opts);
self.instruction(LocalSet(dst.len.idx));
self.instruction(End); // end latin1-or-utf16 block
self.free_temp_local(src_len_tmp);
self.free_temp_local(dst_byte_len);
dst
}
fn validate_string_length(&mut self, src: &WasmString<'_>, dst: FE) {
self.validate_string_length_u8(src, dst.width())
}
fn validate_string_length_u8(&mut self, s: &WasmString<'_>, dst: u8) {
// Check to see if the source byte length is out of bounds in
// which case a trap is generated.
self.instruction(LocalGet(s.len.idx));
let max = MAX_STRING_BYTE_LENGTH / u32::from(dst);
self.ptr_uconst(s.opts, max);
self.ptr_ge_u(s.opts);
self.instruction(If(BlockType::Empty));
self.trap(Trap::StringLengthTooBig);
self.instruction(End);
}
fn transcoder(
&mut self,
src: &WasmString<'_>,
dst: &WasmString<'_>,
op: Transcode,
) -> FuncIndex {
self.module.import_transcoder(Transcoder {
from_memory: src.opts.memory.unwrap(),
from_memory64: src.opts.memory64,
to_memory: dst.opts.memory.unwrap(),
to_memory64: dst.opts.memory64,
op,
})
}
fn validate_string_inbounds(&mut self, s: &WasmString<'_>, byte_len: u32) {
self.validate_memory_inbounds(s.opts, s.ptr.idx, byte_len, Trap::StringLengthOverflow)
}
fn validate_memory_inbounds(
&mut self,
opts: &Options,
ptr_local: u32,
byte_len_local: u32,
trap: Trap,
) {
let extend_to_64 = |me: &mut Self| {
if !opts.memory64 {
me.instruction(I64ExtendI32U);
}
};
self.instruction(Block(BlockType::Empty));
self.instruction(Block(BlockType::Empty));
// Calculate the full byte size of memory with `memory.size`. Note that
// arithmetic here is done always in 64-bits to accomodate 4G memories.
// Additionally it's assumed that 64-bit memories never fill up
// entirely.
self.instruction(MemorySize(opts.memory.unwrap().as_u32()));
extend_to_64(self);
self.instruction(I64Const(16));
self.instruction(I64Shl);
// Calculate the end address of the string. This is done by adding the
// base pointer to the byte length. For 32-bit memories there's no need
// to check for overflow since everything is extended to 64-bit, but for
// 64-bit memories overflow is checked.
self.instruction(LocalGet(ptr_local));
extend_to_64(self);
self.instruction(LocalGet(byte_len_local));
extend_to_64(self);
self.instruction(I64Add);
if opts.memory64 {
let tmp = self.local_tee_new_tmp(ValType::I64);
self.instruction(LocalGet(ptr_local));
self.ptr_lt_u(opts);
self.instruction(BrIf(0));
self.instruction(LocalGet(tmp.idx));
self.free_temp_local(tmp);
}
// If the byte size of memory is greater than the final address of the
// string then the string is invalid. Note that if it's precisely equal
// then that's ok.
self.instruction(I64GeU);
self.instruction(BrIf(1));
self.instruction(End);
self.trap(trap);
self.instruction(End);
}
fn translate_list(
&mut self,
src_ty: TypeListIndex,
src: &Source<'_>,
dst_ty: &InterfaceType,
dst: &Destination,
) {
let src_element_ty = &self.types[src_ty].element;
let dst_element_ty = match dst_ty {
InterfaceType::List(r) => &self.types[*r].element,
_ => panic!("expected a list"),
};
let src_opts = src.opts();
let dst_opts = dst.opts();
let (src_size, src_align) = self.types.size_align(src_opts, src_element_ty);
let (dst_size, dst_align) = self.types.size_align(dst_opts, dst_element_ty);
// Load the pointer/length of this list into temporary locals. These
// will be referenced a good deal so this just makes it easier to deal
// with them consistently below rather than trying to reload from memory
// for example.
match src {
Source::Stack(s) => {
assert_eq!(s.locals.len(), 2);
self.stack_get(&s.slice(0..1), src_opts.ptr());
self.stack_get(&s.slice(1..2), src_opts.ptr());
}
Source::Memory(mem) => {
self.ptr_load(mem);
self.ptr_load(&mem.bump(src_opts.ptr_size().into()));
}
}
let src_len = self.local_set_new_tmp(src_opts.ptr());
let src_ptr = self.local_set_new_tmp(src_opts.ptr());
// Create a `Memory` operand which will internally assert that the
// `src_ptr` value is properly aligned.
let src_mem = self.memory_operand(src_opts, src_ptr, src_align);
// Calculate the source/destination byte lengths into unique locals.
let src_byte_len = self.calculate_list_byte_len(src_opts, src_len.idx, src_size);
let dst_byte_len = if src_size == dst_size {
self.convert_src_len_to_dst(src_byte_len.idx, src_opts.ptr(), dst_opts.ptr());
self.local_set_new_tmp(dst_opts.ptr())
} else if src_opts.ptr() == dst_opts.ptr() {
self.calculate_list_byte_len(dst_opts, src_len.idx, dst_size)
} else {
self.convert_src_len_to_dst(src_byte_len.idx, src_opts.ptr(), dst_opts.ptr());
let tmp = self.local_set_new_tmp(dst_opts.ptr());
let ret = self.calculate_list_byte_len(dst_opts, tmp.idx, dst_size);
self.free_temp_local(tmp);
ret
};
// Here `realloc` is invoked (in a `malloc`-like fashion) to allocate
// space for the list in the destination memory. This will also
// internally insert checks that the returned pointer is aligned
// correctly for the destination.
let dst_mem = self.malloc(dst_opts, MallocSize::Local(dst_byte_len.idx), dst_align);
// With all the pointers and byte lengths verity that both the source
// and the destination buffers are in-bounds.
self.validate_memory_inbounds(
src_opts,
src_mem.addr.idx,
src_byte_len.idx,
Trap::ListByteLengthOverflow,
);
self.validate_memory_inbounds(
dst_opts,
dst_mem.addr.idx,
dst_byte_len.idx,
Trap::ListByteLengthOverflow,
);
self.free_temp_local(src_byte_len);
self.free_temp_local(dst_byte_len);
// This is the main body of the loop to actually translate list types.
// Note that if both element sizes are 0 then this won't actually do
// anything so the loop is removed entirely.
if src_size > 0 || dst_size > 0 {
// This block encompasses the entire loop and is use to exit before even
// entering the loop if the list size is zero.
self.instruction(Block(BlockType::Empty));
// Set the `remaining` local and only continue if it's > 0
self.instruction(LocalGet(src_len.idx));
let remaining = self.local_tee_new_tmp(src_opts.ptr());
self.ptr_eqz(src_opts);
self.instruction(BrIf(0));
// Initialize the two destination pointers to their initial values
self.instruction(LocalGet(src_mem.addr.idx));
let cur_src_ptr = self.local_set_new_tmp(src_opts.ptr());
self.instruction(LocalGet(dst_mem.addr.idx));
let cur_dst_ptr = self.local_set_new_tmp(dst_opts.ptr());
self.instruction(Loop(BlockType::Empty));
// Translate the next element in the list
let element_src = Source::Memory(Memory {
opts: src_opts,
offset: 0,
addr: TempLocal::new(cur_src_ptr.idx, cur_src_ptr.ty),
});
let element_dst = Destination::Memory(Memory {
opts: dst_opts,
offset: 0,
addr: TempLocal::new(cur_dst_ptr.idx, cur_dst_ptr.ty),
});
self.translate(src_element_ty, &element_src, dst_element_ty, &element_dst);
// Update the two loop pointers
if src_size > 0 {
self.instruction(LocalGet(cur_src_ptr.idx));
self.ptr_uconst(src_opts, src_size);
self.ptr_add(src_opts);
self.instruction(LocalSet(cur_src_ptr.idx));
}
if dst_size > 0 {
self.instruction(LocalGet(cur_dst_ptr.idx));
self.ptr_uconst(dst_opts, dst_size);
self.ptr_add(dst_opts);
self.instruction(LocalSet(cur_dst_ptr.idx));
}
// Update the remaining count, falling through to break out if it's zero
// now.
self.instruction(LocalGet(remaining.idx));
self.ptr_iconst(src_opts, -1);
self.ptr_add(src_opts);
self.instruction(LocalTee(remaining.idx));
self.ptr_br_if(src_opts, 0);
self.instruction(End); // end of loop
self.instruction(End); // end of block
self.free_temp_local(cur_dst_ptr);
self.free_temp_local(cur_src_ptr);
self.free_temp_local(remaining);
}
// Store the ptr/length in the desired destination
match dst {
Destination::Stack(s, _) => {
self.instruction(LocalGet(dst_mem.addr.idx));
self.stack_set(&s[..1], dst_opts.ptr());
self.convert_src_len_to_dst(src_len.idx, src_opts.ptr(), dst_opts.ptr());
self.stack_set(&s[1..], dst_opts.ptr());
}
Destination::Memory(mem) => {
self.instruction(LocalGet(mem.addr.idx));
self.instruction(LocalGet(dst_mem.addr.idx));
self.ptr_store(mem);
self.instruction(LocalGet(mem.addr.idx));
self.convert_src_len_to_dst(src_len.idx, src_opts.ptr(), dst_opts.ptr());
self.ptr_store(&mem.bump(dst_opts.ptr_size().into()));
}
}
self.free_temp_local(src_len);
self.free_temp_local(src_mem.addr);
self.free_temp_local(dst_mem.addr);
}
fn calculate_list_byte_len(
&mut self,
opts: &Options,
len_local: u32,
elt_size: u32,
) -> TempLocal {
// Zero-size types are easy to handle here because the byte size of the
// destination is always zero.
if elt_size == 0 {
self.ptr_uconst(opts, 0);
return self.local_set_new_tmp(opts.ptr());
}
// For one-byte elements in the destination the check here can be a bit
// more optimal than the general case below. In these situations if the
// source pointer type is 32-bit then we're guaranteed to not overflow,
// so the source length is simply casted to the destination's type.
//
// If the source is 64-bit then all that needs to be checked is to
// ensure that it does not have the upper 32-bits set.
if elt_size == 1 {
if let ValType::I64 = opts.ptr() {
self.instruction(LocalGet(len_local));
self.instruction(I64Const(32));
self.instruction(I64ShrU);
self.instruction(I32WrapI64);
self.instruction(If(BlockType::Empty));
self.trap(Trap::ListByteLengthOverflow);
self.instruction(End);
}
self.instruction(LocalGet(len_local));
return self.local_set_new_tmp(opts.ptr());
}
// The main check implemented by this function is to verify that
// `src_len_local` does not exceed the 32-bit range. Byte sizes for
// lists must always fit in 32-bits to get transferred to 32-bit
// memories.
self.instruction(Block(BlockType::Empty));
self.instruction(Block(BlockType::Empty));
self.instruction(LocalGet(len_local));
match opts.ptr() {
// The source's list length is guaranteed to be less than 32-bits
// so simply extend it up to a 64-bit type for the multiplication
// below.
ValType::I32 => self.instruction(I64ExtendI32U),
// If the source is a 64-bit memory then if the item length doesn't
// fit in 32-bits the byte length definitely won't, so generate a
// branch to our overflow trap here if any of the upper 32-bits are set.
ValType::I64 => {
self.instruction(I64Const(32));
self.instruction(I64ShrU);
self.instruction(I32WrapI64);
self.instruction(BrIf(0));
self.instruction(LocalGet(len_local));
}
_ => unreachable!(),
}
// Next perform a 64-bit multiplication with the element byte size that
// is itself guaranteed to fit in 32-bits. The result is then checked
// to see if we overflowed the 32-bit space. The two input operands to
// the multiplication are guaranteed to be 32-bits at most which means
// that this multiplication shouldn't overflow.
//
// The result of the multiplication is saved into a local as well to
// get the result afterwards.
self.instruction(I64Const(elt_size.into()));
self.instruction(I64Mul);
let tmp = self.local_tee_new_tmp(ValType::I64);
// Branch to success if the upper 32-bits are zero, otherwise
// fall-through to the trap.
self.instruction(I64Const(32));
self.instruction(I64ShrU);
self.instruction(I64Eqz);
self.instruction(BrIf(1));
self.instruction(End);
self.trap(Trap::ListByteLengthOverflow);
self.instruction(End);
// If a fresh local was used to store the result of the multiplication
// then convert it down to 32-bits which should be guaranteed to not
// lose information at this point.
if opts.ptr() == ValType::I64 {
tmp
} else {
self.instruction(LocalGet(tmp.idx));
self.instruction(I32WrapI64);
self.free_temp_local(tmp);
self.local_set_new_tmp(ValType::I32)
}
}
fn convert_src_len_to_dst(
&mut self,
src_len_local: u32,
src_ptr_ty: ValType,
dst_ptr_ty: ValType,
) {
self.instruction(LocalGet(src_len_local));
match (src_ptr_ty, dst_ptr_ty) {
(ValType::I32, ValType::I64) => self.instruction(I64ExtendI32U),
(ValType::I64, ValType::I32) => self.instruction(I32WrapI64),
(src, dst) => assert_eq!(src, dst),
}
}
fn translate_record(
&mut self,
src_ty: TypeRecordIndex,
src: &Source<'_>,
dst_ty: &InterfaceType,
dst: &Destination,
) {
let src_ty = &self.types[src_ty];
let dst_ty = match dst_ty {
InterfaceType::Record(r) => &self.types[*r],
_ => panic!("expected a record"),
};
// TODO: subtyping
assert_eq!(src_ty.fields.len(), dst_ty.fields.len());
// First a map is made of the source fields to where they're coming
// from (e.g. which offset or which locals). This map is keyed by the
// fields' names
let mut src_fields = HashMap::new();
for (i, src) in src
.record_field_srcs(self.types, src_ty.fields.iter().map(|f| f.ty))
.enumerate()
{
let field = &src_ty.fields[i];
src_fields.insert(&field.name, (src, &field.ty));
}
// .. and next translation is performed in the order of the destination
// fields in case the destination is the stack to ensure that the stack
// has the fields all in the right order.
//
// Note that the lookup in `src_fields` is an infallible lookup which
// will panic if the field isn't found.
//
// TODO: should that lookup be fallible with subtyping?
for (i, dst) in dst
.record_field_dsts(self.types, dst_ty.fields.iter().map(|f| f.ty))
.enumerate()
{
let field = &dst_ty.fields[i];
let (src, src_ty) = &src_fields[&field.name];
self.translate(src_ty, src, &field.ty, &dst);
}
}
fn translate_flags(
&mut self,
src_ty: TypeFlagsIndex,
src: &Source<'_>,
dst_ty: &InterfaceType,
dst: &Destination,
) {
let src_ty = &self.types[src_ty];
let dst_ty = match dst_ty {
InterfaceType::Flags(r) => &self.types[*r],
_ => panic!("expected a record"),
};
// TODO: subtyping
//
// Notably this implementation does not support reordering flags from
// the source to the destination nor having more flags in the
// destination. Currently this is a copy from source to destination
// in-bulk. Otherwise reordering indices would have to have some sort of
// fancy bit twiddling tricks or something like that.
assert_eq!(src_ty.names, dst_ty.names);
let cnt = src_ty.names.len();
match FlagsSize::from_count(cnt) {
FlagsSize::Size0 => {}
FlagsSize::Size1 => {
let mask = if cnt == 8 { 0xff } else { (1 << cnt) - 1 };
self.convert_u8_mask(src, dst, mask);
}
FlagsSize::Size2 => {
let mask = if cnt == 16 { 0xffff } else { (1 << cnt) - 1 };
self.convert_u16_mask(src, dst, mask);
}
FlagsSize::Size4Plus(n) => {
let srcs = src.record_field_srcs(self.types, (0..n).map(|_| InterfaceType::U32));
let dsts = dst.record_field_dsts(self.types, (0..n).map(|_| InterfaceType::U32));
let n = usize::from(n);
for (i, (src, dst)) in srcs.zip(dsts).enumerate() {
let mask = if i == n - 1 && (cnt % 32 != 0) {
(1 << (cnt % 32)) - 1
} else {
0xffffffff
};
self.convert_u32_mask(&src, &dst, mask);
}
}
}
}
fn translate_tuple(
&mut self,
src_ty: TypeTupleIndex,
src: &Source<'_>,
dst_ty: &InterfaceType,
dst: &Destination,
) {
let src_ty = &self.types[src_ty];
let dst_ty = match dst_ty {
InterfaceType::Tuple(t) => &self.types[*t],
_ => panic!("expected a tuple"),
};
// TODO: subtyping
assert_eq!(src_ty.types.len(), dst_ty.types.len());
let srcs = src
.record_field_srcs(self.types, src_ty.types.iter().copied())
.zip(src_ty.types.iter());
let dsts = dst
.record_field_dsts(self.types, dst_ty.types.iter().copied())
.zip(dst_ty.types.iter());
for ((src, src_ty), (dst, dst_ty)) in srcs.zip(dsts) {
self.translate(src_ty, &src, dst_ty, &dst);
}
}
fn translate_variant(
&mut self,
src_ty: TypeVariantIndex,
src: &Source<'_>,
dst_ty: &InterfaceType,
dst: &Destination,
) {
let src_ty = &self.types[src_ty];
let dst_ty = match dst_ty {
InterfaceType::Variant(t) => &self.types[*t],
_ => panic!("expected a variant"),
};
let src_info = variant_info(self.types, src_ty.cases.iter().map(|c| c.ty.as_ref()));
let dst_info = variant_info(self.types, dst_ty.cases.iter().map(|c| c.ty.as_ref()));
let iter = src_ty.cases.iter().enumerate().map(|(src_i, src_case)| {
let dst_i = dst_ty
.cases
.iter()
.position(|c| c.name == src_case.name)
.unwrap();
let dst_case = &dst_ty.cases[dst_i];
let src_i = u32::try_from(src_i).unwrap();
let dst_i = u32::try_from(dst_i).unwrap();
VariantCase {
src_i,
src_ty: src_case.ty.as_ref(),
dst_i,
dst_ty: dst_case.ty.as_ref(),
}
});
self.convert_variant(src, &src_info, dst, &dst_info, iter);
}
fn translate_union(
&mut self,
src_ty: TypeUnionIndex,
src: &Source<'_>,
dst_ty: &InterfaceType,
dst: &Destination,
) {
let src_ty = &self.types[src_ty];
let dst_ty = match dst_ty {
InterfaceType::Union(t) => &self.types[*t],
_ => panic!("expected an option"),
};
assert_eq!(src_ty.types.len(), dst_ty.types.len());
let src_info = variant_info(self.types, src_ty.types.iter().map(Some));
let dst_info = variant_info(self.types, dst_ty.types.iter().map(Some));
self.convert_variant(
src,
&src_info,
dst,
&dst_info,
src_ty
.types
.iter()
.zip(dst_ty.types.iter())
.enumerate()
.map(|(i, (src_ty, dst_ty))| {
let i = u32::try_from(i).unwrap();
VariantCase {
src_i: i,
dst_i: i,
src_ty: Some(src_ty),
dst_ty: Some(dst_ty),
}
}),
);
}
fn translate_enum(
&mut self,
src_ty: TypeEnumIndex,
src: &Source<'_>,
dst_ty: &InterfaceType,
dst: &Destination,
) {
let src_ty = &self.types[src_ty];
let dst_ty = match dst_ty {
InterfaceType::Enum(t) => &self.types[*t],
_ => panic!("expected an option"),
};
let src_info = variant_info(self.types, src_ty.names.iter().map(|_| None));
let dst_info = variant_info(self.types, dst_ty.names.iter().map(|_| None));
self.convert_variant(
src,
&src_info,
dst,
&dst_info,
src_ty.names.iter().enumerate().map(|(src_i, src_name)| {
let dst_i = dst_ty.names.iter().position(|n| n == src_name).unwrap();
let src_i = u32::try_from(src_i).unwrap();
let dst_i = u32::try_from(dst_i).unwrap();
VariantCase {
src_i,
dst_i,
src_ty: None,
dst_ty: None,
}
}),
);
}
fn translate_option(
&mut self,
src_ty: TypeOptionIndex,
src: &Source<'_>,
dst_ty: &InterfaceType,
dst: &Destination,
) {
let src_ty = &self.types[src_ty].ty;
let dst_ty = match dst_ty {
InterfaceType::Option(t) => &self.types[*t].ty,
_ => panic!("expected an option"),
};
let src_ty = Some(src_ty);
let dst_ty = Some(dst_ty);
let src_info = variant_info(self.types, [None, src_ty]);
let dst_info = variant_info(self.types, [None, dst_ty]);
self.convert_variant(
src,
&src_info,
dst,
&dst_info,
[
VariantCase {
src_i: 0,
dst_i: 0,
src_ty: None,
dst_ty: None,
},
VariantCase {
src_i: 1,
dst_i: 1,
src_ty,
dst_ty,
},
]
.into_iter(),
);
}
fn translate_result(
&mut self,
src_ty: TypeResultIndex,
src: &Source<'_>,
dst_ty: &InterfaceType,
dst: &Destination,
) {
let src_ty = &self.types[src_ty];
let dst_ty = match dst_ty {
InterfaceType::Result(t) => &self.types[*t],
_ => panic!("expected a result"),
};
let src_info = variant_info(self.types, [src_ty.ok.as_ref(), src_ty.err.as_ref()]);
let dst_info = variant_info(self.types, [dst_ty.ok.as_ref(), dst_ty.err.as_ref()]);
self.convert_variant(
src,
&src_info,
dst,
&dst_info,
[
VariantCase {
src_i: 0,
dst_i: 0,
src_ty: src_ty.ok.as_ref(),
dst_ty: dst_ty.ok.as_ref(),
},
VariantCase {
src_i: 1,
dst_i: 1,
src_ty: src_ty.err.as_ref(),
dst_ty: dst_ty.err.as_ref(),
},
]
.into_iter(),
);
}
fn convert_variant<'a>(
&mut self,
src: &Source<'_>,
src_info: &VariantInfo,
dst: &Destination,
dst_info: &VariantInfo,
src_cases: impl ExactSizeIterator<Item = VariantCase<'a>>,
) {
// The outermost block is special since it has the result type of the
// translation here. That will depend on the `dst`.
let outer_block_ty = match dst {
Destination::Stack(dst_flat, _) => match dst_flat.len() {
0 => BlockType::Empty,
1 => BlockType::Result(dst_flat[0]),
_ => {
let ty = self.module.core_types.function(&[], &dst_flat);
BlockType::FunctionType(ty)
}
},
Destination::Memory(_) => BlockType::Empty,
};
self.instruction(Block(outer_block_ty));
// After the outermost block generate a new block for each of the
// remaining cases.
let src_cases_len = src_cases.len();
for _ in 0..src_cases_len - 1 {
self.instruction(Block(BlockType::Empty));
}
// Generate a block for an invalid variant discriminant
self.instruction(Block(BlockType::Empty));
// And generate one final block that we'll be jumping out of with the
// `br_table`
self.instruction(Block(BlockType::Empty));
// Load the discriminant
match src {
Source::Stack(s) => self.stack_get(&s.slice(0..1), ValType::I32),
Source::Memory(mem) => match src_info.size {
DiscriminantSize::Size1 => self.i32_load8u(mem),
DiscriminantSize::Size2 => self.i32_load16u(mem),
DiscriminantSize::Size4 => self.i32_load(mem),
},
}
// Generate the `br_table` for the discriminant. Each case has an
// offset of 1 to skip the trapping block.
let mut targets = Vec::new();
for i in 0..src_cases_len {
targets.push((i + 1) as u32);
}
self.instruction(BrTable(targets[..].into(), 0));
self.instruction(End); // end the `br_table` block
self.trap(Trap::InvalidDiscriminant);
self.instruction(End); // end the "invalid discriminant" block
// Translate each case individually within its own block. Note that the
// iteration order here places the first case in the innermost block
// and the last case in the outermost block. This matches the order
// of the jump targets in the `br_table` instruction.
let src_cases_len = u32::try_from(src_cases_len).unwrap();
for case in src_cases {
let VariantCase {
src_i,
src_ty,
dst_i,
dst_ty,
} = case;
// Translate the discriminant here, noting that `dst_i` may be
// different than `src_i`.
self.push_dst_addr(dst);
self.instruction(I32Const(dst_i as i32));
match dst {
Destination::Stack(stack, _) => self.stack_set(&stack[..1], ValType::I32),
Destination::Memory(mem) => match dst_info.size {
DiscriminantSize::Size1 => self.i32_store8(mem),
DiscriminantSize::Size2 => self.i32_store16(mem),
DiscriminantSize::Size4 => self.i32_store(mem),
},
}
let src_payload = src.payload_src(self.types, src_info, src_ty);
let dst_payload = dst.payload_dst(self.types, dst_info, dst_ty);
// Translate the payload of this case using the various types from
// the dst/src.
match (src_ty, dst_ty) {
(Some(src_ty), Some(dst_ty)) => {
self.translate(src_ty, &src_payload, dst_ty, &dst_payload);
}
(None, None) => {}
_ => unimplemented!(),
}
// If the results of this translation were placed on the stack then
// the stack values may need to be padded with more zeros due to
// this particular case being possibly smaller than the entire
// variant. That's handled here by pushing remaining zeros after
// accounting for the discriminant pushed as well as the results of
// this individual payload.
if let Destination::Stack(payload_results, _) = dst_payload {
if let Destination::Stack(dst_results, _) = dst {
let remaining = &dst_results[1..][payload_results.len()..];
for ty in remaining {
match ty {
ValType::I32 => self.instruction(I32Const(0)),
ValType::I64 => self.instruction(I64Const(0)),
ValType::F32 => self.instruction(F32Const(0.0)),
ValType::F64 => self.instruction(F64Const(0.0)),
_ => unreachable!(),
}
}
}
}
// Branch to the outermost block. Note that this isn't needed for
// the outermost case since it simply falls through.
if src_i != src_cases_len - 1 {
self.instruction(Br(src_cases_len - src_i - 1));
}
self.instruction(End); // end this case's block
}
}
fn trap_if_not_flag(&mut self, flags_global: GlobalIndex, flag_to_test: i32, trap: Trap) {
self.instruction(GlobalGet(flags_global.as_u32()));
self.instruction(I32Const(flag_to_test));
self.instruction(I32And);
self.instruction(I32Eqz);
self.instruction(If(BlockType::Empty));
self.trap(trap);
self.instruction(End);
}
fn assert_not_flag(&mut self, flags_global: GlobalIndex, flag_to_test: i32, msg: &'static str) {
self.instruction(GlobalGet(flags_global.as_u32()));
self.instruction(I32Const(flag_to_test));
self.instruction(I32And);
self.instruction(If(BlockType::Empty));
self.trap(Trap::AssertFailed(msg));
self.instruction(End);
}
fn set_flag(&mut self, flags_global: GlobalIndex, flag_to_set: i32, value: bool) {
self.instruction(GlobalGet(flags_global.as_u32()));
if value {
self.instruction(I32Const(flag_to_set));
self.instruction(I32Or);
} else {
self.instruction(I32Const(!flag_to_set));
self.instruction(I32And);
}
self.instruction(GlobalSet(flags_global.as_u32()));
}
fn verify_aligned(&mut self, opts: &Options, addr_local: u32, align: u32) {
// If the alignment is 1 then everything is trivially aligned and the
// check can be omitted.
if align == 1 {
return;
}
self.instruction(LocalGet(addr_local));
assert!(align.is_power_of_two());
self.ptr_uconst(opts, align - 1);
self.ptr_and(opts);
self.ptr_if(opts, BlockType::Empty);
self.trap(Trap::UnalignedPointer);
self.instruction(End);
}
fn assert_aligned(&mut self, ty: &InterfaceType, mem: &Memory) {
if !self.module.debug {
return;
}
let align = self.types.align(mem.opts, ty);
if align == 1 {
return;
}
assert!(align.is_power_of_two());
self.instruction(LocalGet(mem.addr.idx));
self.ptr_uconst(mem.opts, mem.offset);
self.ptr_add(mem.opts);
self.ptr_uconst(mem.opts, align - 1);
self.ptr_and(mem.opts);
self.ptr_if(mem.opts, BlockType::Empty);
self.trap(Trap::AssertFailed("pointer not aligned"));
self.instruction(End);
}
fn malloc<'a>(&mut self, opts: &'a Options, size: MallocSize, align: u32) -> Memory<'a> {
let realloc = opts.realloc.unwrap();
self.ptr_uconst(opts, 0);
self.ptr_uconst(opts, 0);
self.ptr_uconst(opts, align);
match size {
MallocSize::Const(size) => self.ptr_uconst(opts, size),
MallocSize::Local(idx) => self.instruction(LocalGet(idx)),
}
self.instruction(Call(realloc.as_u32()));
let addr = self.local_set_new_tmp(opts.ptr());
self.memory_operand(opts, addr, align)
}
fn memory_operand<'a>(&mut self, opts: &'a Options, addr: TempLocal, align: u32) -> Memory<'a> {
let ret = Memory {
addr,
offset: 0,
opts,
};
self.verify_aligned(opts, ret.addr.idx, align);
ret
}
/// Generates a new local in this function of the `ty` specified,
/// initializing it with the top value on the current wasm stack.
///
/// The returned `TempLocal` must be freed after it is finished with
/// `free_temp_local`.
fn local_tee_new_tmp(&mut self, ty: ValType) -> TempLocal {
self.gen_temp_local(ty, LocalTee)
}
/// Same as `local_tee_new_tmp` but initializes the local with `LocalSet`
/// instead of `LocalTee`.
fn local_set_new_tmp(&mut self, ty: ValType) -> TempLocal {
self.gen_temp_local(ty, LocalSet)
}
fn gen_temp_local(&mut self, ty: ValType, insn: fn(u32) -> Instruction<'static>) -> TempLocal {
// First check to see if any locals are available in this function which
// were previously generated but are no longer in use.
if let Some(idx) = self.free_locals.get_mut(&ty).and_then(|v| v.pop()) {
self.instruction(insn(idx));
return TempLocal {
ty,
idx,
needs_free: true,
};
}
// Failing that generate a fresh new local.
let locals = &mut self.module.funcs[self.result].locals;
match locals.last_mut() {
Some((cnt, prev_ty)) if ty == *prev_ty => *cnt += 1,
_ => locals.push((1, ty)),
}
self.nlocals += 1;
let idx = self.nlocals - 1;
self.instruction(insn(idx));
TempLocal {
ty,
idx,
needs_free: true,
}
}
/// Used to release a `TempLocal` from a particular lexical scope to allow
/// its possible reuse in later scopes.
fn free_temp_local(&mut self, mut local: TempLocal) {
assert!(local.needs_free);
self.free_locals
.entry(local.ty)
.or_insert(Vec::new())
.push(local.idx);
local.needs_free = false;
}
fn instruction(&mut self, instr: Instruction) {
instr.encode(&mut self.code);
}
fn trap(&mut self, trap: Trap) {
self.traps.push((self.code.len(), trap));
self.instruction(Unreachable);
}
/// Flushes out the current `code` instructions (and `traps` if there are
/// any) into the destination function.
///
/// This is a noop if no instructions have been encoded yet.
fn flush_code(&mut self) {
if self.code.is_empty() {
return;
}
self.module.funcs[self.result].body.push(Body::Raw(
mem::take(&mut self.code),
mem::take(&mut self.traps),
));
}
fn finish(mut self) {
// Append the final `end` instruction which all functions require, and
// then empty out the temporary buffer in `Compiler`.
self.instruction(End);
self.flush_code();
// Flag the function as "done" which helps with an assert later on in
// emission that everything was eventually finished.
self.module.funcs[self.result].filled_in = true;
}
/// Fetches the value contained with the local specified by `stack` and
/// converts it to `dst_ty`.
///
/// This is only intended for use in primitive operations where `stack` is
/// guaranteed to have only one local. The type of the local on the stack is
/// then converted to `dst_ty` appropriately. Note that the types may be
/// different due to the "flattening" of variant types.
fn stack_get(&mut self, stack: &Stack<'_>, dst_ty: ValType) {
assert_eq!(stack.locals.len(), 1);
let (idx, src_ty) = stack.locals[0];
self.instruction(LocalGet(idx));
match (src_ty, dst_ty) {
(ValType::I32, ValType::I32)
| (ValType::I64, ValType::I64)
| (ValType::F32, ValType::F32)
| (ValType::F64, ValType::F64) => {}
(ValType::I32, ValType::F32) => self.instruction(F32ReinterpretI32),
(ValType::I64, ValType::I32) => {
self.assert_i64_upper_bits_not_set(idx);
self.instruction(I32WrapI64);
}
(ValType::I64, ValType::F64) => self.instruction(F64ReinterpretI64),
(ValType::I64, ValType::F32) => {
self.assert_i64_upper_bits_not_set(idx);
self.instruction(I32WrapI64);
self.instruction(F32ReinterpretI32);
}
// should not be possible given the `join` function for variants
(ValType::I32, ValType::I64)
| (ValType::I32, ValType::F64)
| (ValType::F32, ValType::I32)
| (ValType::F32, ValType::I64)
| (ValType::F32, ValType::F64)
| (ValType::F64, ValType::I32)
| (ValType::F64, ValType::I64)
| (ValType::F64, ValType::F32)
// not used in the component model
| (ValType::ExternRef, _)
| (_, ValType::ExternRef)
| (ValType::FuncRef, _)
| (_, ValType::FuncRef)
| (ValType::V128, _)
| (_, ValType::V128) => {
panic!("cannot get {dst_ty:?} from {src_ty:?} local");
}
}
}
fn assert_i64_upper_bits_not_set(&mut self, local: u32) {
if !self.module.debug {
return;
}
self.instruction(LocalGet(local));
self.instruction(I64Const(32));
self.instruction(I64ShrU);
self.instruction(I32WrapI64);
self.instruction(If(BlockType::Empty));
self.trap(Trap::AssertFailed("upper bits are unexpectedly set"));
self.instruction(End);
}
/// Converts the top value on the WebAssembly stack which has type
/// `src_ty` to `dst_tys[0]`.
///
/// This is only intended for conversion of primitives where the `dst_tys`
/// list is known to be of length 1.
fn stack_set(&mut self, dst_tys: &[ValType], src_ty: ValType) {
assert_eq!(dst_tys.len(), 1);
let dst_ty = dst_tys[0];
match (src_ty, dst_ty) {
(ValType::I32, ValType::I32)
| (ValType::I64, ValType::I64)
| (ValType::F32, ValType::F32)
| (ValType::F64, ValType::F64) => {}
(ValType::F32, ValType::I32) => self.instruction(I32ReinterpretF32),
(ValType::I32, ValType::I64) => self.instruction(I64ExtendI32U),
(ValType::F64, ValType::I64) => self.instruction(I64ReinterpretF64),
(ValType::F32, ValType::I64) => {
self.instruction(I32ReinterpretF32);
self.instruction(I64ExtendI32U);
}
// should not be possible given the `join` function for variants
(ValType::I64, ValType::I32)
| (ValType::F64, ValType::I32)
| (ValType::I32, ValType::F32)
| (ValType::I64, ValType::F32)
| (ValType::F64, ValType::F32)
| (ValType::I32, ValType::F64)
| (ValType::I64, ValType::F64)
| (ValType::F32, ValType::F64)
// not used in the component model
| (ValType::ExternRef, _)
| (_, ValType::ExternRef)
| (ValType::FuncRef, _)
| (_, ValType::FuncRef)
| (ValType::V128, _)
| (_, ValType::V128) => {
panic!("cannot get {dst_ty:?} from {src_ty:?} local");
}
}
}
fn i32_load8u(&mut self, mem: &Memory) {
self.instruction(LocalGet(mem.addr.idx));
self.instruction(I32Load8_U(mem.memarg(0)));
}
fn i32_load8s(&mut self, mem: &Memory) {
self.instruction(LocalGet(mem.addr.idx));
self.instruction(I32Load8_S(mem.memarg(0)));
}
fn i32_load16u(&mut self, mem: &Memory) {
self.instruction(LocalGet(mem.addr.idx));
self.instruction(I32Load16_U(mem.memarg(1)));
}
fn i32_load16s(&mut self, mem: &Memory) {
self.instruction(LocalGet(mem.addr.idx));
self.instruction(I32Load16_S(mem.memarg(1)));
}
fn i32_load(&mut self, mem: &Memory) {
self.instruction(LocalGet(mem.addr.idx));
self.instruction(I32Load(mem.memarg(2)));
}
fn i64_load(&mut self, mem: &Memory) {
self.instruction(LocalGet(mem.addr.idx));
self.instruction(I64Load(mem.memarg(3)));
}
fn ptr_load(&mut self, mem: &Memory) {
if mem.opts.memory64 {
self.i64_load(mem);
} else {
self.i32_load(mem);
}
}
fn ptr_add(&mut self, opts: &Options) {
if opts.memory64 {
self.instruction(I64Add);
} else {
self.instruction(I32Add);
}
}
fn ptr_sub(&mut self, opts: &Options) {
if opts.memory64 {
self.instruction(I64Sub);
} else {
self.instruction(I32Sub);
}
}
fn ptr_mul(&mut self, opts: &Options) {
if opts.memory64 {
self.instruction(I64Mul);
} else {
self.instruction(I32Mul);
}
}
fn ptr_ge_u(&mut self, opts: &Options) {
if opts.memory64 {
self.instruction(I64GeU);
} else {
self.instruction(I32GeU);
}
}
fn ptr_lt_u(&mut self, opts: &Options) {
if opts.memory64 {
self.instruction(I64LtU);
} else {
self.instruction(I32LtU);
}
}
fn ptr_shl(&mut self, opts: &Options) {
if opts.memory64 {
self.instruction(I64Shl);
} else {
self.instruction(I32Shl);
}
}
fn ptr_eqz(&mut self, opts: &Options) {
if opts.memory64 {
self.instruction(I64Eqz);
} else {
self.instruction(I32Eqz);
}
}
fn ptr_uconst(&mut self, opts: &Options, val: u32) {
if opts.memory64 {
self.instruction(I64Const(val.into()));
} else {
self.instruction(I32Const(val as i32));
}
}
fn ptr_iconst(&mut self, opts: &Options, val: i32) {
if opts.memory64 {
self.instruction(I64Const(val.into()));
} else {
self.instruction(I32Const(val));
}
}
fn ptr_eq(&mut self, opts: &Options) {
if opts.memory64 {
self.instruction(I64Eq);
} else {
self.instruction(I32Eq);
}
}
fn ptr_ne(&mut self, opts: &Options) {
if opts.memory64 {
self.instruction(I64Ne);
} else {
self.instruction(I32Ne);
}
}
fn ptr_and(&mut self, opts: &Options) {
if opts.memory64 {
self.instruction(I64And);
} else {
self.instruction(I32And);
}
}
fn ptr_or(&mut self, opts: &Options) {
if opts.memory64 {
self.instruction(I64Or);
} else {
self.instruction(I32Or);
}
}
fn ptr_xor(&mut self, opts: &Options) {
if opts.memory64 {
self.instruction(I64Xor);
} else {
self.instruction(I32Xor);
}
}
fn ptr_if(&mut self, opts: &Options, ty: BlockType) {
if opts.memory64 {
self.instruction(I64Const(0));
self.instruction(I64Ne);
}
self.instruction(If(ty));
}
fn ptr_br_if(&mut self, opts: &Options, depth: u32) {
if opts.memory64 {
self.instruction(I64Const(0));
self.instruction(I64Ne);
}
self.instruction(BrIf(depth));
}
fn f32_load(&mut self, mem: &Memory) {
self.instruction(LocalGet(mem.addr.idx));
self.instruction(F32Load(mem.memarg(2)));
}
fn f64_load(&mut self, mem: &Memory) {
self.instruction(LocalGet(mem.addr.idx));
self.instruction(F64Load(mem.memarg(3)));
}
fn push_dst_addr(&mut self, dst: &Destination) {
if let Destination::Memory(mem) = dst {
self.instruction(LocalGet(mem.addr.idx));
}
}
fn i32_store8(&mut self, mem: &Memory) {
self.instruction(I32Store8(mem.memarg(0)));
}
fn i32_store16(&mut self, mem: &Memory) {
self.instruction(I32Store16(mem.memarg(1)));
}
fn i32_store(&mut self, mem: &Memory) {
self.instruction(I32Store(mem.memarg(2)));
}
fn i64_store(&mut self, mem: &Memory) {
self.instruction(I64Store(mem.memarg(3)));
}
fn ptr_store(&mut self, mem: &Memory) {
if mem.opts.memory64 {
self.i64_store(mem);
} else {
self.i32_store(mem);
}
}
fn f32_store(&mut self, mem: &Memory) {
self.instruction(F32Store(mem.memarg(2)));
}
fn f64_store(&mut self, mem: &Memory) {
self.instruction(F64Store(mem.memarg(3)));
}
}
impl<'a> Source<'a> {
/// Given this `Source` returns an iterator over the `Source` for each of
/// the component `fields` specified.
///
/// This will automatically slice stack-based locals to the appropriate
/// width for each component type and additionally calculate the appropriate
/// offset for each memory-based type.
fn record_field_srcs<'b>(
&'b self,
types: &'b ComponentTypesBuilder,
fields: impl IntoIterator<Item = InterfaceType> + 'b,
) -> impl Iterator<Item = Source<'a>> + 'b
where
'a: 'b,
{
let mut offset = 0;
fields.into_iter().map(move |ty| match self {
Source::Memory(mem) => {
let mem = next_field_offset(&mut offset, types, &ty, mem);
Source::Memory(mem)
}
Source::Stack(stack) => {
let cnt = types.flat_types(&ty).unwrap().len() as u32;
offset += cnt;
Source::Stack(stack.slice((offset - cnt) as usize..offset as usize))
}
})
}
/// Returns the corresponding discriminant source and payload source f
fn payload_src(
&self,
types: &ComponentTypesBuilder,
info: &VariantInfo,
case: Option<&InterfaceType>,
) -> Source<'a> {
match self {
Source::Stack(s) => {
let flat_len = match case {
Some(case) => types.flat_types(case).unwrap().len(),
None => 0,
};
Source::Stack(s.slice(1..s.locals.len()).slice(0..flat_len))
}
Source::Memory(mem) => {
let mem = if mem.opts.memory64 {
mem.bump(info.payload_offset64)
} else {
mem.bump(info.payload_offset32)
};
Source::Memory(mem)
}
}
}
fn opts(&self) -> &'a Options {
match self {
Source::Stack(s) => s.opts,
Source::Memory(mem) => mem.opts,
}
}
}
impl<'a> Destination<'a> {
/// Same as `Source::record_field_srcs` but for destinations.
fn record_field_dsts<'b>(
&'b self,
types: &'b ComponentTypesBuilder,
fields: impl IntoIterator<Item = InterfaceType> + 'b,
) -> impl Iterator<Item = Destination> + 'b
where
'a: 'b,
{
let mut offset = 0;
fields.into_iter().map(move |ty| match self {
Destination::Memory(mem) => {
let mem = next_field_offset(&mut offset, types, &ty, mem);
Destination::Memory(mem)
}
Destination::Stack(s, opts) => {
let cnt = types.flat_types(&ty).unwrap().len() as u32;
offset += cnt;
Destination::Stack(&s[(offset - cnt) as usize..offset as usize], opts)
}
})
}
/// Returns the corresponding discriminant source and payload source f
fn payload_dst(
&self,
types: &ComponentTypesBuilder,
info: &VariantInfo,
case: Option<&InterfaceType>,
) -> Destination {
match self {
Destination::Stack(s, opts) => {
let flat_len = match case {
Some(case) => types.flat_types(case).unwrap().len(),
None => 0,
};
Destination::Stack(&s[1..][..flat_len], opts)
}
Destination::Memory(mem) => {
let mem = if mem.opts.memory64 {
mem.bump(info.payload_offset64)
} else {
mem.bump(info.payload_offset32)
};
Destination::Memory(mem)
}
}
}
fn opts(&self) -> &'a Options {
match self {
Destination::Stack(_, opts) => opts,
Destination::Memory(mem) => mem.opts,
}
}
}
fn next_field_offset<'a>(
offset: &mut u32,
types: &ComponentTypesBuilder,
field: &InterfaceType,
mem: &Memory<'a>,
) -> Memory<'a> {
let abi = types.canonical_abi(field);
let offset = if mem.opts.memory64 {
abi.next_field64(offset)
} else {
abi.next_field32(offset)
};
mem.bump(offset)
}
impl<'a> Memory<'a> {
fn memarg(&self, align: u32) -> MemArg {
MemArg {
offset: u64::from(self.offset),
align,
memory_index: self.opts.memory.unwrap().as_u32(),
}
}
fn bump(&self, offset: u32) -> Memory<'a> {
Memory {
opts: self.opts,
addr: TempLocal::new(self.addr.idx, self.addr.ty),
offset: self.offset + offset,
}
}
}
impl<'a> Stack<'a> {
fn slice(&self, range: Range<usize>) -> Stack<'a> {
Stack {
locals: &self.locals[range],
opts: self.opts,
}
}
}
struct VariantCase<'a> {
src_i: u32,
src_ty: Option<&'a InterfaceType>,
dst_i: u32,
dst_ty: Option<&'a InterfaceType>,
}
fn variant_info<'a, I>(types: &ComponentTypesBuilder, cases: I) -> VariantInfo
where
I: IntoIterator<Item = Option<&'a InterfaceType>>,
I::IntoIter: ExactSizeIterator,
{
VariantInfo::new(
cases
.into_iter()
.map(|ty| ty.map(|ty| types.canonical_abi(ty))),
)
.0
}
enum MallocSize {
Const(u32),
Local(u32),
}
struct WasmString<'a> {
ptr: TempLocal,
len: TempLocal,
opts: &'a Options,
}
struct TempLocal {
idx: u32,
ty: ValType,
needs_free: bool,
}
impl TempLocal {
fn new(idx: u32, ty: ValType) -> TempLocal {
TempLocal {
idx,
ty,
needs_free: false,
}
}
}
impl std::ops::Drop for TempLocal {
fn drop(&mut self) {
if self.needs_free {
panic!("temporary local not free'd");
}
}
}
impl From<FlatType> for ValType {
fn from(ty: FlatType) -> ValType {
match ty {
FlatType::I32 => ValType::I32,
FlatType::I64 => ValType::I64,
FlatType::F32 => ValType::F32,
FlatType::F64 => ValType::F64,
}
}
}
Reference in New Issue View Git Blame Copy Permalink