aarch64: Migrate uextend/sextend to ISLE

This commit migrates the sign/zero extension instructions from
`lower_inst.rs` to ISLE. There's actually a fair amount going on in this
migration since a few other pieces needed touching up along the way as
well:

* First is the actual migration of `uextend` and `sextend`. These
  instructions are relatively simple but end up having a number of special
  cases. I've attempted to replicate all the cases here but
  double-checks would be good.

* This commit actually fixes a few issues where if the result of a vector
  extraction is sign/zero-extended into i128 that actually results in
  panics in the current backend.

* This commit adds exhaustive testing for
  extension-of-a-vector-extraction is a noop wrt extraction.

* A bugfix around ISLE glue was required to get this commit working,
  notably the case where the `RegMapper` implementation was trying to
  map an input to an output (meaning ISLE was passing through an input
  unmodified to the output) wasn't working. This requires a `mov`
  instruction to be generated and this commit updates the glue to do
  this. At the same time this commit updates the ISLE glue to share more
  infrastructure between x64 and aarch64 so both backends get this fix
  instead of just aarch64.

Overall I think that the translation to ISLE was a net benefit for these
instructions. It's relatively obvious what all the cases are now unlike
before where it took a few reads of the code and some boolean switches
to figure out which path was taken for each flavor of input. I think
there's still possible improvements here where, for example, the
`put_in_reg_{s,z}ext64` helper doesn't use this logic so technically
those helpers could also pattern match the "well atomic loads and vector
extractions automatically do this for us" but that's a possible future
improvement for later (and shouldn't be too too hard with some ISLE
refactoring).

cranelift/codegen/src/machinst/isle.rs

@@ -1,5 +1,7 @@
-use crate::ir::Value;
+use crate::ir::{Inst, Value};
+use crate::machinst::{get_output_reg, InsnOutput, LowerCtx, MachInst, RegRenamer};
 use regalloc::{Reg, Writable};
+use smallvec::SmallVec;
 
 pub type Unit = ();
 pub type ValueSlice<'a> = &'a [Value];
@@ -8,6 +10,8 @@ pub type ValueArray3 = [Value; 3];
 pub type WritableReg = Writable<Reg>;
 pub type ValueRegs = crate::machinst::ValueRegs<Reg>;
 
+/// Helper macro to define methods in `prelude.isle` within `impl Context for
+/// ...` for each backend. These methods are shared amongst all backends.
 #[macro_export]
 #[doc(hidden)]
 macro_rules! isle_prelude_methods {
@@ -228,6 +232,102 @@ macro_rules! isle_prelude_methods {
     };
 }
 
+/// This structure is used to implement the ISLE-generated `Context` trait and
+/// internally has a temporary reference to a machinst `LowerCtx`.
+pub(crate) struct IsleContext<'a, C: LowerCtx, F, const N: usize>
+where
+    [C::I; N]: smallvec::Array,
+{
+    pub lower_ctx: &'a mut C,
+    pub isa_flags: &'a F,
+    pub emitted_insts: SmallVec<[C::I; N]>,
+}
+
+/// Shared lowering code amongst all backends for doing ISLE-based lowering.
+///
+/// The `isle_lower` argument here is an ISLE-generated function for `lower` and
+/// then this function otherwise handles register mapping and such around the
+/// lowering.
+pub(crate) fn lower_common<C, F, const N: usize>(
+    lower_ctx: &mut C,
+    isa_flags: &F,
+    outputs: &[InsnOutput],
+    inst: Inst,
+    isle_lower: fn(&mut IsleContext<'_, C, F, N>, Inst) -> Option<ValueRegs>,
+    map_regs: fn(&mut C::I, &RegRenamer),
+) -> Result<(), ()>
+where
+    C: LowerCtx,
+    [C::I; N]: smallvec::Array<Item = C::I>,
+{
+    // TODO: reuse the ISLE context across lowerings so we can reuse its
+    // internal heap allocations.
+    let mut isle_ctx = IsleContext {
+        lower_ctx,
+        isa_flags,
+        emitted_insts: SmallVec::new(),
+    };
+
+    let temp_regs = isle_lower(&mut isle_ctx, inst).ok_or(())?;
+    let mut temp_regs = temp_regs.regs().iter();
+
+    #[cfg(debug_assertions)]
+    {
+        let all_dsts_len = outputs
+            .iter()
+            .map(|out| get_output_reg(isle_ctx.lower_ctx, *out).len())
+            .sum();
+        debug_assert_eq!(
+            temp_regs.len(),
+            all_dsts_len,
+            "the number of temporary registers and destination registers do \
+         not match ({} != {}); ensure the correct registers are being \
+         returned.",
+            temp_regs.len(),
+            all_dsts_len,
+        );
+    }
+
+    // The ISLE generated code emits its own registers to define the
+    // instruction's lowered values in. We rename those registers to the
+    // registers they were assigned when their value was used as an operand in
+    // earlier lowerings.
+    let mut renamer = RegRenamer::default();
+    for output in outputs {
+        let dsts = get_output_reg(isle_ctx.lower_ctx, *output);
+        let ty = isle_ctx.lower_ctx.output_ty(output.insn, output.output);
+        let (_, tys) = <C::I>::rc_for_type(ty).unwrap();
+        for ((temp, dst), ty) in temp_regs.by_ref().zip(dsts.regs()).zip(tys) {
+            renamer.add_rename(*temp, dst.to_reg(), *ty);
+        }
+    }
+    for inst in isle_ctx.emitted_insts.iter_mut() {
+        map_regs(inst, &renamer);
+    }
+
+    // If any renamed register wasn't actually defined in the ISLE-generated
+    // instructions then what we're actually doing is "renaming" an input to a
+    // new name which requires manually inserting a `mov` instruction. Note that
+    // this typically doesn't happen and is only here for cases where the input
+    // is sometimes passed through unmodified to the output, such as
+    // zero-extending a 64-bit input to a 128-bit output which doesn't actually
+    // change the input and simply produces another zero'd register.
+    for (old, new, ty) in renamer.unmapped_defs() {
+        isle_ctx
+            .lower_ctx
+            .emit(<C::I>::gen_move(Writable::from_reg(new), old, ty));
+    }
+
+    // Once everything is remapped we forward all emitted instructions to the
+    // `lower_ctx`. Note that this happens after the synthetic mov's above in
+    // case any of these instruction use those movs.
+    for inst in isle_ctx.emitted_insts {
+        lower_ctx.emit(inst);
+    }
+
+    Ok(())
+}
+
 #[inline(never)]
 #[cold]
 pub fn out_of_line_panic(msg: &str) -> ! {

cranelift/codegen/src/machinst/regmapping.rs

@@ -1,5 +1,7 @@
+use crate::ir::Type;
 use regalloc::{Reg, RegUsageMapper, Writable};
 use smallvec::SmallVec;
+use std::cell::Cell;
 
 // Define our own register-mapping trait so we can do arbitrary register
 // renaming that are more free form than what `regalloc` constrains us to with
@@ -48,36 +50,59 @@ where
     }
 }
 
-#[derive(Default)]
+#[derive(Debug, Default)]
 pub struct RegRenamer {
-    // Map of `(old, new)` register names. Use a `SmallVec` because we typically
-    // only have one or two renamings.
-    renames: SmallVec<[(Reg, Reg); 2]>,
+    // Map of `(old, new, used, ty)` register names. Use a `SmallVec` because
+    // we typically only have one or two renamings.
+    //
+    // The `used` flag indicates whether the mapping has been used for
+    // `get_def`, later used afterwards during `unmapped_defs` to know what
+    // moves need to be generated.
+    renames: SmallVec<[(Reg, Reg, Cell<bool>, Type); 2]>,
 }
 
 impl RegRenamer {
-    pub fn add_rename(&mut self, old: Reg, new: Reg) {
-        self.renames.push((old, new));
+    /// Adds a new mapping which means that `old` reg should now be called
+    /// `new`. The type of `old` is `ty` as specified.
+    pub fn add_rename(&mut self, old: Reg, new: Reg, ty: Type) {
+        self.renames.push((old, new, Cell::new(false), ty));
     }
 
-    fn get_rename(&self, reg: Reg) -> Option<Reg> {
-        self.renames
-            .iter()
-            .find(|(old, _)| reg == *old)
-            .map(|(_, new)| *new)
+    fn get_rename(&self, reg: Reg, set_used_def: bool) -> Option<Reg> {
+        let (_, new, used_def, _) = self.renames.iter().find(|(old, _, _, _)| reg == *old)?;
+        used_def.set(used_def.get() || set_used_def);
+        Some(*new)
+    }
+
+    /// Returns the list of register mappings, with their type, which were not
+    /// actually mapped.
+    ///
+    /// This list is used because it means that the `old` name for the register
+    /// was never actually defined, so to correctly rename this register the
+    /// caller needs to move `old` into `new`.
+    ///
+    /// This yields tuples of `(old, new, ty)`.
+    pub fn unmapped_defs(&self) -> impl Iterator<Item = (Reg, Reg, Type)> + '_ {
+        self.renames.iter().filter_map(|(old, new, used_def, ty)| {
+            if used_def.get() {
+                None
+            } else {
+                Some((*old, *new, *ty))
+            }
+        })
     }
 }
 
 impl RegMapper for RegRenamer {
     fn get_use(&self, reg: Reg) -> Option<Reg> {
-        self.get_rename(reg)
+        self.get_rename(reg, false)
     }
 
     fn get_def(&self, reg: Reg) -> Option<Reg> {
-        self.get_rename(reg)
+        self.get_rename(reg, true)
     }
 
     fn get_mod(&self, reg: Reg) -> Option<Reg> {
-        self.get_rename(reg)
+        self.get_rename(reg, false)
     }
 }