Support for SpiderMonkey's "Wasm ABI 2020".

As part of a Wasm JIT update, SpiderMonkey is changing its internal
WebAssembly function ABI. The new ABI's frame format includes "caller
TLS" and "callee TLS" slots. The details of where these come from are
not important; from Cranelift's point of view, the only relevant
requirement is that we have two on-stack args that are always present
(offsetting other on-stack args), and that we define special argument
purposes so that we can supply values for these slots.

Note that this adds a *new* ABI (a variant of the Baldrdash ABI) because
we do not want to tightly couple the landing of this PR to the landing
of the changes in SpiderMonkey; it's better if both the old and new
behavior remain available in Cranelift, so SpiderMonkey can continue to
vendor Cranelift even if it does not land (or backs out) the ABI change.

Furthermore, note that this needs to be a Cranelift-level change (i.e.
cannot be done purely from the translator environment implementation)
because the special TLS arguments must always go on the stack, which
would not otherwise happen with the usual argument-placement logic; and
there is no primitive to push a value directly in CLIF code (the notion
of a stack frame is a lower-level concept).

cranelift/codegen/meta/src/shared/settings.rs

@@ -194,6 +194,7 @@ pub(crate) fn define() -> SettingGroup {
             "windows_fastcall",
             "baldrdash_system_v",
             "baldrdash_windows",
+            "baldrdash_2020",
             "probestack",
         ],
     );

cranelift/codegen/src/ir/extfunc.rs

@@ -335,6 +335,20 @@ pub enum ArgumentPurpose {
     /// This is a pointer to a stack limit. It is used to check the current stack pointer
     /// against. Can only appear once in a signature.
     StackLimit,
+
+    /// A callee TLS value.
+    ///
+    /// In the Baldrdash-2020 calling convention, the stack upon entry to the callee contains the
+    /// TLS-register values for the caller and the callee. This argument is used to provide the
+    /// value for the callee.
+    CalleeTLS,
+
+    /// A caller TLS value.
+    ///
+    /// In the Baldrdash-2020 calling convention, the stack upon entry to the callee contains the
+    /// TLS-register values for the caller and the callee. This argument is used to provide the
+    /// value for the caller.
+    CallerTLS,
 }
 
 impl fmt::Display for ArgumentPurpose {
@@ -349,6 +363,8 @@ impl fmt::Display for ArgumentPurpose {
             Self::VMContext => "vmctx",
             Self::SignatureId => "sigid",
             Self::StackLimit => "stack_limit",
+            Self::CalleeTLS => "callee_tls",
+            Self::CallerTLS => "caller_tls",
         })
     }
 }
@@ -470,6 +486,7 @@ mod tests {
             CallConv::WindowsFastcall,
             CallConv::BaldrdashSystemV,
             CallConv::BaldrdashWindows,
+            CallConv::Baldrdash2020,
         ] {
             assert_eq!(Ok(cc), cc.to_string().parse())
         }

cranelift/codegen/src/isa/aarch64/abi.rs

@@ -31,6 +31,11 @@ static BALDRDASH_SIG_REG: u8 = 10;
 /// This is SpiderMonkey's `WasmTlsReg`.
 static BALDRDASH_TLS_REG: u8 = 23;
 
+/// Offset in stack-arg area to callee-TLS slot in Baldrdash-2020 calling convention.
+static BALDRDASH_CALLEE_TLS_OFFSET: i64 = 0;
+/// Offset in stack-arg area to caller-TLS slot in Baldrdash-2020 calling convention.
+static BALDRDASH_CALLER_TLS_OFFSET: i64 = 8;
+
 // These two lists represent the registers the JIT may *not* use at any point in generated code.
 //
 // So these are callee-preserved from the JIT's point of view, and every register not in this list
@@ -75,6 +80,7 @@ fn try_fill_baldrdash_reg(call_conv: isa::CallConv, param: &ir::AbiParam) -> Opt
                     xreg(BALDRDASH_TLS_REG).to_real_reg(),
                     ir::types::I64,
                     param.extension,
+                    param.purpose,
                 ))
             }
             &ir::ArgumentPurpose::SignatureId => {
@@ -83,6 +89,27 @@ fn try_fill_baldrdash_reg(call_conv: isa::CallConv, param: &ir::AbiParam) -> Opt
                     xreg(BALDRDASH_SIG_REG).to_real_reg(),
                     ir::types::I64,
                     param.extension,
+                    param.purpose,
+                ))
+            }
+            &ir::ArgumentPurpose::CalleeTLS => {
+                // This is SpiderMonkey's callee TLS slot in the extended frame of Wasm's ABI-2020.
+                assert!(call_conv == isa::CallConv::Baldrdash2020);
+                Some(ABIArg::Stack(
+                    BALDRDASH_CALLEE_TLS_OFFSET,
+                    ir::types::I64,
+                    ir::ArgumentExtension::None,
+                    param.purpose,
+                ))
+            }
+            &ir::ArgumentPurpose::CallerTLS => {
+                // This is SpiderMonkey's caller TLS slot in the extended frame of Wasm's ABI-2020.
+                assert!(call_conv == isa::CallConv::Baldrdash2020);
+                Some(ABIArg::Stack(
+                    BALDRDASH_CALLER_TLS_OFFSET,
+                    ir::types::I64,
+                    ir::ArgumentExtension::None,
+                    param.purpose,
                 ))
             }
             _ => None,
@@ -120,6 +147,7 @@ impl ABIMachineSpec for AArch64MachineDeps {
         add_ret_area_ptr: bool,
     ) -> CodegenResult<(Vec<ABIArg>, i64, Option<usize>)> {
         let is_baldrdash = call_conv.extends_baldrdash();
+        let has_baldrdash_tls = call_conv == isa::CallConv::Baldrdash2020;
 
         // See AArch64 ABI (https://c9x.me/compile/bib/abi-arm64.pdf), sections 5.4.
         let mut next_xreg = 0;
@@ -127,6 +155,12 @@ impl ABIMachineSpec for AArch64MachineDeps {
         let mut next_stack: u64 = 0;
         let mut ret = vec![];
 
+        if args_or_rets == ArgsOrRets::Args && has_baldrdash_tls {
+            // Baldrdash ABI-2020 always has two stack-arg slots reserved, for the callee and
+            // caller TLS-register values, respectively.
+            next_stack = 16;
+        }
+
         // Note on return values: on the regular non-baldrdash ABI, we may return values in 8
         // registers for V128 and I64 registers independently of the number of register values
         // returned in the other class. That is, we can return values in up to 8 integer and 8
@@ -155,7 +189,9 @@ impl ABIMachineSpec for AArch64MachineDeps {
                 &ir::ArgumentPurpose::VMContext
                 | &ir::ArgumentPurpose::Normal
                 | &ir::ArgumentPurpose::StackLimit
-                | &ir::ArgumentPurpose::SignatureId => {}
+                | &ir::ArgumentPurpose::SignatureId
+                | &ir::ArgumentPurpose::CallerTLS
+                | &ir::ArgumentPurpose::CalleeTLS => {}
                 _ => panic!(
                     "Unsupported argument purpose {:?} in signature: {:?}",
                     param.purpose, params
@@ -188,6 +224,7 @@ impl ABIMachineSpec for AArch64MachineDeps {
                     reg.to_real_reg(),
                     param.value_type,
                     param.extension,
+                    param.purpose,
                 ));
                 *next_reg += 1;
                 remaining_reg_vals -= 1;
@@ -203,6 +240,7 @@ impl ABIMachineSpec for AArch64MachineDeps {
                     next_stack as i64,
                     param.value_type,
                     param.extension,
+                    param.purpose,
                 ));
                 next_stack += size;
             }
@@ -219,12 +257,14 @@ impl ABIMachineSpec for AArch64MachineDeps {
                     xreg(next_xreg).to_real_reg(),
                     I64,
                     ir::ArgumentExtension::None,
+                    ir::ArgumentPurpose::Normal,
                 ));
             } else {
                 ret.push(ABIArg::Stack(
                     next_stack as i64,
                     I64,
                     ir::ArgumentExtension::None,
+                    ir::ArgumentPurpose::Normal,
                 ));
                 next_stack += 8;
             }
@@ -453,6 +493,7 @@ impl ABIMachineSpec for AArch64MachineDeps {
     // nominal SP offset; abi_impl generic code will do that.
     fn gen_clobber_save(
         call_conv: isa::CallConv,
+        _: &settings::Flags,
         clobbers: &Set<Writable<RealReg>>,
     ) -> (u64, SmallVec<[Inst; 16]>) {
         let mut insts = SmallVec::new();
@@ -503,6 +544,7 @@ impl ABIMachineSpec for AArch64MachineDeps {
 
     fn gen_clobber_restore(
         call_conv: isa::CallConv,
+        flags: &settings::Flags,
         clobbers: &Set<Writable<RealReg>>,
     ) -> SmallVec<[Inst; 16]> {
         let mut insts = SmallVec::new();
@@ -549,6 +591,18 @@ impl ABIMachineSpec for AArch64MachineDeps {
             });
         }
 
+        // If this is Baldrdash-2020, restore the callee (i.e., our) TLS
+        // register. We may have allocated it for something else and clobbered
+        // it, but the ABI expects us to leave the TLS register unchanged.
+        if call_conv == isa::CallConv::Baldrdash2020 {
+            let off = BALDRDASH_CALLEE_TLS_OFFSET + Self::fp_to_arg_offset(call_conv, flags);
+            insts.push(Inst::gen_load(
+                writable_xreg(BALDRDASH_TLS_REG),
+                AMode::UnsignedOffset(fp_reg(), UImm12Scaled::maybe_from_i64(off, I64).unwrap()),
+                I64,
+            ));
+        }
+
         insts
     }
 

cranelift/codegen/src/isa/call_conv.rs

@@ -22,6 +22,9 @@ pub enum CallConv {
     BaldrdashSystemV,
     /// SpiderMonkey WebAssembly convention on Windows
     BaldrdashWindows,
+    /// SpiderMonkey WebAssembly convention for "ABI-2020", with extra TLS
+    /// register slots in the frame.
+    Baldrdash2020,
     /// Specialized convention for the probestack function
     Probestack,
 }
@@ -48,6 +51,7 @@ impl CallConv {
             LibcallCallConv::WindowsFastcall => Self::WindowsFastcall,
             LibcallCallConv::BaldrdashSystemV => Self::BaldrdashSystemV,
             LibcallCallConv::BaldrdashWindows => Self::BaldrdashWindows,
+            LibcallCallConv::Baldrdash2020 => Self::Baldrdash2020,
             LibcallCallConv::Probestack => Self::Probestack,
         }
     }
@@ -63,7 +67,7 @@ impl CallConv {
     /// Is the calling convention extending the Baldrdash ABI?
     pub fn extends_baldrdash(self) -> bool {
         match self {
-            Self::BaldrdashSystemV | Self::BaldrdashWindows => true,
+            Self::BaldrdashSystemV | Self::BaldrdashWindows | Self::Baldrdash2020 => true,
             _ => false,
         }
     }
@@ -78,6 +82,7 @@ impl fmt::Display for CallConv {
             Self::WindowsFastcall => "windows_fastcall",
             Self::BaldrdashSystemV => "baldrdash_system_v",
             Self::BaldrdashWindows => "baldrdash_windows",
+            Self::Baldrdash2020 => "baldrdash_2020",
             Self::Probestack => "probestack",
         })
     }
@@ -93,6 +98,7 @@ impl str::FromStr for CallConv {
             "windows_fastcall" => Ok(Self::WindowsFastcall),
             "baldrdash_system_v" => Ok(Self::BaldrdashSystemV),
             "baldrdash_windows" => Ok(Self::BaldrdashWindows),
+            "baldrdash_2020" => Ok(Self::Baldrdash2020),
             "probestack" => Ok(Self::Probestack),
             _ => Err(()),
         }

cranelift/codegen/src/isa/x64/abi.rs

@@ -20,6 +20,11 @@ use std::convert::TryFrom;
 /// with 32-bit arithmetic: for now, 128 MB.
 static STACK_ARG_RET_SIZE_LIMIT: u64 = 128 * 1024 * 1024;
 
+/// Offset in stack-arg area to callee-TLS slot in Baldrdash-2020 calling convention.
+static BALDRDASH_CALLEE_TLS_OFFSET: i64 = 0;
+/// Offset in stack-arg area to caller-TLS slot in Baldrdash-2020 calling convention.
+static BALDRDASH_CALLER_TLS_OFFSET: i64 = 8;
+
 /// Try to fill a Baldrdash register, returning it if it was found.
 fn try_fill_baldrdash_reg(call_conv: CallConv, param: &ir::AbiParam) -> Option<ABIArg> {
     if call_conv.extends_baldrdash() {
@@ -30,6 +35,7 @@ fn try_fill_baldrdash_reg(call_conv: CallConv, param: &ir::AbiParam) -> Option<A
                     regs::r14().to_real_reg(),
                     types::I64,
                     param.extension,
+                    param.purpose,
                 ))
             }
             &ir::ArgumentPurpose::SignatureId => {
@@ -38,6 +44,27 @@ fn try_fill_baldrdash_reg(call_conv: CallConv, param: &ir::AbiParam) -> Option<A
                     regs::r10().to_real_reg(),
                     types::I64,
                     param.extension,
+                    param.purpose,
+                ))
+            }
+            &ir::ArgumentPurpose::CalleeTLS => {
+                // This is SpiderMonkey's callee TLS slot in the extended frame of Wasm's ABI-2020.
+                assert!(call_conv == isa::CallConv::Baldrdash2020);
+                Some(ABIArg::Stack(
+                    BALDRDASH_CALLEE_TLS_OFFSET,
+                    ir::types::I64,
+                    ir::ArgumentExtension::None,
+                    param.purpose,
+                ))
+            }
+            &ir::ArgumentPurpose::CallerTLS => {
+                // This is SpiderMonkey's caller TLS slot in the extended frame of Wasm's ABI-2020.
+                assert!(call_conv == isa::CallConv::Baldrdash2020);
+                Some(ABIArg::Stack(
+                    BALDRDASH_CALLER_TLS_OFFSET,
+                    ir::types::I64,
+                    ir::ArgumentExtension::None,
+                    param.purpose,
                 ))
             }
             _ => None,
@@ -70,12 +97,19 @@ impl ABIMachineSpec for X64ABIMachineSpec {
         add_ret_area_ptr: bool,
     ) -> CodegenResult<(Vec<ABIArg>, i64, Option<usize>)> {
         let is_baldrdash = call_conv.extends_baldrdash();
+        let has_baldrdash_tls = call_conv == isa::CallConv::Baldrdash2020;
 
         let mut next_gpr = 0;
         let mut next_vreg = 0;
         let mut next_stack: u64 = 0;
         let mut ret = vec![];
 
+        if args_or_rets == ArgsOrRets::Args && has_baldrdash_tls {
+            // Baldrdash ABI-2020 always has two stack-arg slots reserved, for the callee and
+            // caller TLS-register values, respectively.
+            next_stack = 16;
+        }
+
         for i in 0..params.len() {
             // Process returns backward, according to the SpiderMonkey ABI (which we
             // adopt internally if `is_baldrdash` is set).
@@ -90,7 +124,9 @@ impl ABIMachineSpec for X64ABIMachineSpec {
                 &ir::ArgumentPurpose::VMContext
                 | &ir::ArgumentPurpose::Normal
                 | &ir::ArgumentPurpose::StackLimit
-                | &ir::ArgumentPurpose::SignatureId => {}
+                | &ir::ArgumentPurpose::SignatureId
+                | &ir::ArgumentPurpose::CalleeTLS
+                | &ir::ArgumentPurpose::CallerTLS => {}
                 _ => panic!(
                     "Unsupported argument purpose {:?} in signature: {:?}",
                     param.purpose, params
@@ -130,6 +166,7 @@ impl ABIMachineSpec for X64ABIMachineSpec {
                     reg.to_real_reg(),
                     param.value_type,
                     param.extension,
+                    param.purpose,
                 ));
                 *next_reg += 1;
             } else {
@@ -144,6 +181,7 @@ impl ABIMachineSpec for X64ABIMachineSpec {
                     next_stack as i64,
                     param.value_type,
                     param.extension,
+                    param.purpose,
                 ));
                 next_stack += size;
             }
@@ -160,12 +198,14 @@ impl ABIMachineSpec for X64ABIMachineSpec {
                     reg.to_real_reg(),
                     types::I64,
                     ir::ArgumentExtension::None,
+                    ir::ArgumentPurpose::Normal,
                 ));
             } else {
                 ret.push(ABIArg::Stack(
                     next_stack as i64,
                     types::I64,
                     ir::ArgumentExtension::None,
+                    ir::ArgumentPurpose::Normal,
                 ));
                 next_stack += 8;
             }
@@ -351,6 +391,7 @@ impl ABIMachineSpec for X64ABIMachineSpec {
 
     fn gen_clobber_save(
         call_conv: isa::CallConv,
+        _: &settings::Flags,
         clobbers: &Set<Writable<RealReg>>,
     ) -> (u64, SmallVec<[Self::I; 16]>) {
         let mut insts = SmallVec::new();
@@ -396,6 +437,7 @@ impl ABIMachineSpec for X64ABIMachineSpec {
 
     fn gen_clobber_restore(
         call_conv: isa::CallConv,
+        flags: &settings::Flags,
         clobbers: &Set<Writable<RealReg>>,
     ) -> SmallVec<[Self::I; 16]> {
         let mut insts = SmallVec::new();
@@ -430,6 +472,18 @@ impl ABIMachineSpec for X64ABIMachineSpec {
             ));
         }
 
+        // If this is Baldrdash-2020, restore the callee (i.e., our) TLS
+        // register. We may have allocated it for something else and clobbered
+        // it, but the ABI expects us to leave the TLS register unchanged.
+        if call_conv == isa::CallConv::Baldrdash2020 {
+            let off = BALDRDASH_CALLEE_TLS_OFFSET + Self::fp_to_arg_offset(call_conv, flags);
+            insts.push(Inst::mov64_m_r(
+                Amode::imm_reg(off as u32, regs::rbp()),
+                Writable::from_reg(regs::r14()),
+                None,
+            ));
+        }
+
         insts
     }
 
@@ -598,7 +652,11 @@ fn in_vec_reg(ty: types::Type) -> bool {
 
 fn get_intreg_for_arg_systemv(call_conv: &CallConv, idx: usize) -> Option<Reg> {
     match call_conv {
-        CallConv::Fast | CallConv::Cold | CallConv::SystemV | CallConv::BaldrdashSystemV => {}
+        CallConv::Fast
+        | CallConv::Cold
+        | CallConv::SystemV
+        | CallConv::BaldrdashSystemV
+        | CallConv::Baldrdash2020 => {}
         _ => panic!("int args only supported for SysV calling convention"),
     };
     match idx {
@@ -614,7 +672,11 @@ fn get_intreg_for_arg_systemv(call_conv: &CallConv, idx: usize) -> Option<Reg> {
 
 fn get_fltreg_for_arg_systemv(call_conv: &CallConv, idx: usize) -> Option<Reg> {
     match call_conv {
-        CallConv::Fast | CallConv::Cold | CallConv::SystemV | CallConv::BaldrdashSystemV => {}
+        CallConv::Fast
+        | CallConv::Cold
+        | CallConv::SystemV
+        | CallConv::BaldrdashSystemV
+        | CallConv::Baldrdash2020 => {}
         _ => panic!("float args only supported for SysV calling convention"),
     };
     match idx {
@@ -641,7 +703,7 @@ fn get_intreg_for_retval_systemv(
             1 => Some(regs::rdx()),
             _ => None,
         },
-        CallConv::BaldrdashSystemV => {
+        CallConv::BaldrdashSystemV | CallConv::Baldrdash2020 => {
             if intreg_idx == 0 && retval_idx == 0 {
                 Some(regs::rax())
             } else {
@@ -663,7 +725,7 @@ fn get_fltreg_for_retval_systemv(
             1 => Some(regs::xmm1()),
             _ => None,
         },
-        CallConv::BaldrdashSystemV => {
+        CallConv::BaldrdashSystemV | CallConv::Baldrdash2020 => {
             if fltreg_idx == 0 && retval_idx == 0 {
                 Some(regs::xmm0())
             } else {
@@ -705,7 +767,7 @@ fn is_callee_save_baldrdash(r: RealReg) -> bool {
 
 fn get_callee_saves(call_conv: &CallConv, regs: &Set<Writable<RealReg>>) -> Vec<Writable<RealReg>> {
     let mut regs: Vec<Writable<RealReg>> = match call_conv {
-        CallConv::BaldrdashSystemV => regs
+        CallConv::BaldrdashSystemV | CallConv::Baldrdash2020 => regs
             .iter()
             .cloned()
             .filter(|r| is_callee_save_baldrdash(r.to_reg()))

cranelift/codegen/src/isa/x86/abi.rs

@@ -506,6 +506,7 @@ pub fn prologue_epilogue(func: &mut ir::Function, isa: &dyn TargetIsa) -> Codege
             baldrdash_prologue_epilogue(func, isa)
         }
         CallConv::Probestack => unimplemented!("probestack calling convention"),
+        CallConv::Baldrdash2020 => unimplemented!("Baldrdash ABI 2020"),
     }
 }
 

cranelift/codegen/src/legalizer/boundary.rs

@@ -158,6 +158,7 @@ fn legalize_entry_params(func: &mut Function, entry: Block) {
                     has_stack_limit = true;
                 }
                 ArgumentPurpose::Link => panic!("Unexpected link arg {}", abi_type),
+                ArgumentPurpose::CallerTLS | ArgumentPurpose::CalleeTLS => {}
             }
             abi_arg += 1;
         } else {
@@ -217,6 +218,7 @@ fn legalize_entry_params(func: &mut Function, entry: Block) {
                 debug_assert!(!has_stack_limit, "Multiple stack_limit parameters found");
                 has_stack_limit = true;
             }
+            ArgumentPurpose::CallerTLS | ArgumentPurpose::CalleeTLS => {}
         }
 
         // Just create entry block values to match here. We will use them in `handle_return_abi()`

cranelift/codegen/src/machinst/abi.rs

@@ -44,6 +44,11 @@ pub trait ABICallee {
     /// register.
     fn gen_copy_arg_to_reg(&self, idx: usize, into_reg: Writable<Reg>) -> Self::I;
 
+    /// Is the given argument needed in the body (as opposed to, e.g., serving
+    /// only as a special ABI-specific placeholder)? This controls whether
+    /// lowering will copy it to a virtual reg use by CLIF instructions.
+    fn arg_is_needed_in_body(&self, idx: usize) -> bool;
+
     /// Generate any setup instruction needed to save values to the
     /// return-value area. This is usually used when were are multiple return
     /// values or an otherwise large return value that must be passed on the

cranelift/codegen/src/machinst/abi_impl.rs

@@ -127,9 +127,24 @@ use std::mem;
 #[derive(Clone, Copy, Debug)]
 pub enum ABIArg {
     /// In a real register.
-    Reg(RealReg, ir::Type, ir::ArgumentExtension),
+    Reg(
+        RealReg,
+        ir::Type,
+        ir::ArgumentExtension,
+        ir::ArgumentPurpose,
+    ),
     /// Arguments only: on stack, at given offset from SP at entry.
-    Stack(i64, ir::Type, ir::ArgumentExtension),
+    Stack(i64, ir::Type, ir::ArgumentExtension, ir::ArgumentPurpose),
+}
+
+impl ABIArg {
+    /// Get the purpose of this arg.
+    fn get_purpose(self) -> ir::ArgumentPurpose {
+        match self {
+            ABIArg::Reg(_, _, _, purpose) => purpose,
+            ABIArg::Stack(_, _, _, purpose) => purpose,
+        }
+    }
 }
 
 /// Are we computing information about arguments or return values? Much of the
@@ -308,6 +323,7 @@ pub trait ABIMachineSpec {
     /// nominal SP offset; caller will do that.
     fn gen_clobber_save(
         call_conv: isa::CallConv,
+        flags: &settings::Flags,
         clobbers: &Set<Writable<RealReg>>,
     ) -> (u64, SmallVec<[Self::I; 16]>);
 
@@ -317,6 +333,7 @@ pub trait ABIMachineSpec {
     /// clobber-save sequence finished.
     fn gen_clobber_restore(
         call_conv: isa::CallConv,
+        flags: &settings::Flags,
         clobbers: &Set<Writable<RealReg>>,
     ) -> SmallVec<[Self::I; 16]>;
 
@@ -700,8 +717,8 @@ impl<M: ABIMachineSpec> ABICallee for ABICalleeImpl<M> {
         match &self.sig.args[idx] {
             // Extension mode doesn't matter (we're copying out, not in; we
             // ignore high bits by convention).
-            &ABIArg::Reg(r, ty, _) => M::gen_move(into_reg, r.to_reg(), ty),
+            &ABIArg::Reg(r, ty, ..) => M::gen_move(into_reg, r.to_reg(), ty),
-            &ABIArg::Stack(off, ty, _) => M::gen_load_stack(
+            &ABIArg::Stack(off, ty, ..) => M::gen_load_stack(
                 StackAMode::FPOffset(M::fp_to_arg_offset(self.call_conv, &self.flags) + off, ty),
                 into_reg,
                 ty,
@@ -709,11 +726,21 @@ impl<M: ABIMachineSpec> ABICallee for ABICalleeImpl<M> {
         }
     }
 
+    fn arg_is_needed_in_body(&self, idx: usize) -> bool {
+        match self.sig.args[idx].get_purpose() {
+            // Special Baldrdash-specific pseudo-args that are present only to
+            // fill stack slots.  Won't ever be used as ordinary values in the
+            // body.
+            ir::ArgumentPurpose::CalleeTLS | ir::ArgumentPurpose::CallerTLS => false,
+            _ => true,
+        }
+    }
+
     fn gen_copy_reg_to_retval(&self, idx: usize, from_reg: Writable<Reg>) -> Vec<Self::I> {
         let mut ret = Vec::new();
         let word_bits = M::word_bits() as u8;
         match &self.sig.rets[idx] {
-            &ABIArg::Reg(r, ty, ext) => {
+            &ABIArg::Reg(r, ty, ext, ..) => {
                 let from_bits = ty_bits(ty) as u8;
                 let dest_reg = Writable::from_reg(r.to_reg());
                 match (ext, from_bits) {
@@ -732,7 +759,7 @@ impl<M: ABIMachineSpec> ABICallee for ABICalleeImpl<M> {
                     _ => ret.push(M::gen_move(dest_reg, from_reg.to_reg(), ty)),
                 };
             }
-            &ABIArg::Stack(off, mut ty, ext) => {
+            &ABIArg::Stack(off, mut ty, ext, ..) => {
                 let from_bits = ty_bits(ty) as u8;
                 // A machine ABI implementation should ensure that stack frames
                 // have "reasonable" size. All current ABIs for machinst
@@ -937,7 +964,8 @@ impl<M: ABIMachineSpec> ABICallee for ABICalleeImpl<M> {
         }
 
         // Save clobbered registers.
-        let (clobber_size, clobber_insts) = M::gen_clobber_save(self.call_conv, &self.clobbered);
+        let (clobber_size, clobber_insts) =
+            M::gen_clobber_save(self.call_conv, &self.flags, &self.clobbered);
         insts.extend(clobber_insts);
 
         if clobber_size > 0 {
@@ -952,7 +980,11 @@ impl<M: ABIMachineSpec> ABICallee for ABICalleeImpl<M> {
         let mut insts = vec![];
 
         // Restore clobbered registers.
-        insts.extend(M::gen_clobber_restore(self.call_conv, &self.clobbered));
+        insts.extend(M::gen_clobber_restore(
+            self.call_conv,
+            &self.flags,
+            &self.clobbered,
+        ));
 
         // N.B.: we do *not* emit a nominal SP adjustment here, because (i) there will be no
         // references to nominal SP offsets before the return below, and (ii) the instruction
@@ -1135,7 +1167,7 @@ impl<M: ABIMachineSpec> ABICaller for ABICallerImpl<M> {
         let word_rc = M::word_reg_class();
         let word_bits = M::word_bits() as usize;
         match &self.sig.args[idx] {
-            &ABIArg::Reg(reg, ty, ext)
+            &ABIArg::Reg(reg, ty, ext, _)
                 if ext != ir::ArgumentExtension::None && ty_bits(ty) < word_bits =>
             {
                 assert_eq!(word_rc, reg.get_class());
@@ -1152,10 +1184,10 @@ impl<M: ABIMachineSpec> ABICaller for ABICallerImpl<M> {
                     word_bits as u8,
                 ));
             }
-            &ABIArg::Reg(reg, ty, _) => {
+            &ABIArg::Reg(reg, ty, _, _) => {
                 ctx.emit(M::gen_move(Writable::from_reg(reg.to_reg()), from_reg, ty));
             }
-            &ABIArg::Stack(off, mut ty, ext) => {
+            &ABIArg::Stack(off, mut ty, ext, _) => {
                 if ext != ir::ArgumentExtension::None && ty_bits(ty) < word_bits {
                     assert_eq!(word_rc, from_reg.get_class());
                     let signed = match ext {
@@ -1194,8 +1226,8 @@ impl<M: ABIMachineSpec> ABICaller for ABICallerImpl<M> {
         match &self.sig.rets[idx] {
             // Extension mode doesn't matter because we're copying out, not in,
             // and we ignore high bits in our own registers by convention.
-            &ABIArg::Reg(reg, ty, _) => ctx.emit(M::gen_move(into_reg, reg.to_reg(), ty)),
+            &ABIArg::Reg(reg, ty, _, _) => ctx.emit(M::gen_move(into_reg, reg.to_reg(), ty)),
-            &ABIArg::Stack(off, ty, _) => {
+            &ABIArg::Stack(off, ty, _, _) => {
                 let ret_area_base = self.sig.stack_arg_space;
                 ctx.emit(M::gen_load_stack(
                     StackAMode::SPOffset(off + ret_area_base, ty),

cranelift/codegen/src/machinst/lower.rs

@@ -420,6 +420,9 @@ impl<'func, I: VCodeInst> Lower<'func, I> {
                 self.f.dfg.block_params(entry_bb)
             );
             for (i, param) in self.f.dfg.block_params(entry_bb).iter().enumerate() {
+                if !self.vcode.abi().arg_is_needed_in_body(i) {
+                    continue;
+                }
                 let reg = Writable::from_reg(self.value_regs[*param]);
                 let insn = self.vcode.abi().gen_copy_arg_to_reg(i, reg);
                 self.emit(insn);