machinst x64: implement support for dynamic heaps and explicit bound checks;

cranelift/codegen/shared/src/condcodes.rs

@@ -122,6 +122,27 @@ impl IntCC {
         }
     }
 
+    /// Determines whether this condcode interprets inputs as signed or
+    /// unsigned.  See the documentation for the `icmp` instruction in
+    /// cranelift-codegen/meta/src/shared/instructions.rs for further insights
+    /// into this.
+    pub fn is_signed(&self) -> bool {
+        match self {
+            IntCC::Equal
+            | IntCC::UnsignedGreaterThanOrEqual
+            | IntCC::UnsignedGreaterThan
+            | IntCC::UnsignedLessThanOrEqual
+            | IntCC::UnsignedLessThan
+            | IntCC::NotEqual => false,
+            IntCC::SignedGreaterThanOrEqual
+            | IntCC::SignedGreaterThan
+            | IntCC::SignedLessThanOrEqual
+            | IntCC::SignedLessThan
+            | IntCC::Overflow
+            | IntCC::NotOverflow => true,
+        }
+    }
+
     /// Get the corresponding string condition code for the IntCC object.
     pub fn to_static_str(self) -> &'static str {
         use self::IntCC::*;

cranelift/codegen/src/isa/aarch64/lower.rs

@@ -798,27 +798,6 @@ pub(crate) fn lower_vector_compare<C: LowerCtx<I = Inst>>(
     Ok(())
 }
 
-/// Determines whether this condcode interprets inputs as signed or
-/// unsigned.  See the documentation for the `icmp` instruction in
-/// cranelift-codegen/meta/src/shared/instructions.rs for further insights
-/// into this.
-pub fn condcode_is_signed(cc: IntCC) -> bool {
-    match cc {
-        IntCC::Equal => false,
-        IntCC::NotEqual => false,
-        IntCC::SignedGreaterThanOrEqual => true,
-        IntCC::SignedGreaterThan => true,
-        IntCC::SignedLessThanOrEqual => true,
-        IntCC::SignedLessThan => true,
-        IntCC::UnsignedGreaterThanOrEqual => false,
-        IntCC::UnsignedGreaterThan => false,
-        IntCC::UnsignedLessThanOrEqual => false,
-        IntCC::UnsignedLessThan => false,
-        IntCC::Overflow => true,
-        IntCC::NotOverflow => true,
-    }
-}
-
 //=============================================================================
 // Helpers for instruction lowering.
 

cranelift/codegen/src/isa/aarch64/lower_inst.rs

@@ -1115,7 +1115,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
             {
                 let condcode = inst_condcode(ctx.data(icmp_insn)).unwrap();
                 let cond = lower_condcode(condcode);
-                let is_signed = condcode_is_signed(condcode);
+                let is_signed = condcode.is_signed();
                 lower_icmp_or_ifcmp_to_flags(ctx, icmp_insn, is_signed);
                 cond
             } else if let Some(fcmp_insn) =
@@ -1161,7 +1161,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
         Opcode::Selectif | Opcode::SelectifSpectreGuard => {
             let condcode = inst_condcode(ctx.data(insn)).unwrap();
             let cond = lower_condcode(condcode);
-            let is_signed = condcode_is_signed(condcode);
+            let is_signed = condcode.is_signed();
             // Verification ensures that the input is always a
             // single-def ifcmp.
             let ifcmp_insn = maybe_input_insn(ctx, inputs[0], Opcode::Ifcmp).unwrap();
@@ -1232,7 +1232,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
         Opcode::Trueif => {
             let condcode = inst_condcode(ctx.data(insn)).unwrap();
             let cond = lower_condcode(condcode);
-            let is_signed = condcode_is_signed(condcode);
+            let is_signed = condcode.is_signed();
             // Verification ensures that the input is always a
             // single-def ifcmp.
             let ifcmp_insn = maybe_input_insn(ctx, inputs[0], Opcode::Ifcmp).unwrap();
@@ -1433,7 +1433,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
         Opcode::Icmp => {
             let condcode = inst_condcode(ctx.data(insn)).unwrap();
             let cond = lower_condcode(condcode);
-            let is_signed = condcode_is_signed(condcode);
+            let is_signed = condcode.is_signed();
             let rd = get_output_reg(ctx, outputs[0]);
             let ty = ctx.input_ty(insn, 0);
             let bits = ty_bits(ty);
@@ -1509,7 +1509,7 @@ pub(crate) fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
             } else if op == Opcode::Trapif {
                 let condcode = inst_condcode(ctx.data(insn)).unwrap();
                 let cond = lower_condcode(condcode);
-                let is_signed = condcode_is_signed(condcode);
+                let is_signed = condcode.is_signed();
 
                 // Verification ensures that the input is always a single-def ifcmp.
                 let ifcmp_insn = maybe_input_insn(ctx, inputs[0], Opcode::Ifcmp).unwrap();
@@ -2360,7 +2360,7 @@ pub(crate) fn lower_branch<C: LowerCtx<I = Inst>>(
                 {
                     let condcode = inst_condcode(ctx.data(icmp_insn)).unwrap();
                     let cond = lower_condcode(condcode);
-                    let is_signed = condcode_is_signed(condcode);
+                    let is_signed = condcode.is_signed();
                     let negated = op0 == Opcode::Brz;
                     let cond = if negated { cond.invert() } else { cond };
 
@@ -2410,7 +2410,7 @@ pub(crate) fn lower_branch<C: LowerCtx<I = Inst>>(
                 let cond = lower_condcode(condcode);
                 let kind = CondBrKind::Cond(cond);
 
-                let is_signed = condcode_is_signed(condcode);
+                let is_signed = condcode.is_signed();
                 let ty = ctx.input_ty(branches[0], 0);
                 let bits = ty_bits(ty);
                 let narrow_mode = match (bits <= 32, is_signed) {
@@ -2451,7 +2451,7 @@ pub(crate) fn lower_branch<C: LowerCtx<I = Inst>>(
                 let cond = lower_condcode(condcode);
                 let kind = CondBrKind::Cond(cond);
 
-                let is_signed = condcode_is_signed(condcode);
+                let is_signed = condcode.is_signed();
                 let flag_input = InsnInput {
                     insn: branches[0],
                     input: 0,

cranelift/codegen/src/isa/x64/inst/args.rs

@@ -726,9 +726,9 @@ impl CC {
             | FloatCC::LessThan
             | FloatCC::LessThanOrEqual
             | FloatCC::UnorderedOrGreaterThan
-            | FloatCC::UnorderedOrGreaterThanOrEqual => unimplemented!(
+            | FloatCC::UnorderedOrGreaterThanOrEqual => {
-                "No single condition code to guarantee ordered. Treat as special case."
+                panic!("No single condition code to guarantee ordered. Treat as special case.")
-            ),
+            }
         }
     }
 

cranelift/codegen/src/isa/x64/lower.rs

@@ -135,6 +135,17 @@ struct InsnOutput {
     output: usize,
 }
 
+fn matches_input<C: LowerCtx<I = Inst>>(c: &mut C, input: InsnInput, op: Opcode) -> Option<IRInst> {
+    let inputs = c.get_input(input.insn, input.input);
+    if let Some((src_inst, _)) = inputs.inst {
+        let data = c.data(src_inst);
+        if data.opcode() == op {
+            return Some(src_inst);
+        }
+    }
+    None
+}
+
 fn input_to_reg(ctx: Ctx, spec: InsnInput) -> Reg {
     let inputs = ctx.get_input(spec.insn, spec.input);
     ctx.use_input_reg(inputs);
@@ -328,7 +339,13 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
             }
         }
 
-        Opcode::Iadd | Opcode::Isub | Opcode::Imul | Opcode::Band | Opcode::Bor | Opcode::Bxor => {
+        Opcode::Iadd
+        | Opcode::IaddIfcout
+        | Opcode::Isub
+        | Opcode::Imul
+        | Opcode::Band
+        | Opcode::Bor
+        | Opcode::Bxor => {
             let lhs = input_to_reg(ctx, inputs[0]);
             let rhs = input_to_reg_mem_imm(ctx, inputs[1]);
             let dst = output_to_reg(ctx, outputs[0]);
@@ -338,7 +355,7 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
 
             let is_64 = int_ty_is_64(ty.unwrap());
             let alu_op = match op {
-                Opcode::Iadd => AluRmiROpcode::Add,
+                Opcode::Iadd | Opcode::IaddIfcout => AluRmiROpcode::Add,
                 Opcode::Isub => AluRmiROpcode::Sub,
                 Opcode::Imul => AluRmiROpcode::Mul,
                 Opcode::Band => AluRmiROpcode::And,
@@ -965,6 +982,64 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
             ctx.emit(Inst::Ud2 { trap_info })
         }
 
+        Opcode::Trapif | Opcode::Trapff => {
+            let srcloc = ctx.srcloc(insn);
+            let trap_code = inst_trapcode(ctx.data(insn)).unwrap();
+
+            let cc = if matches_input(ctx, inputs[0], Opcode::IaddIfcout).is_some() {
+                let condcode = inst_condcode(ctx.data(insn));
+                // The flags must not have been clobbered by any other instruction between the
+                // iadd_ifcout and this instruction, as verified by the CLIF validator; so we can
+                // simply use the flags here.
+                CC::from_intcc(condcode)
+            } else if op == Opcode::Trapif {
+                let condcode = inst_condcode(ctx.data(insn));
+                let cc = CC::from_intcc(condcode);
+
+                // Verification ensures that the input is always a single-def ifcmp.
+                let ifcmp_insn = matches_input(ctx, inputs[0], Opcode::Ifcmp).unwrap();
+                emit_cmp(ctx, ifcmp_insn);
+                cc
+            } else {
+                let condcode = inst_fp_condcode(ctx.data(insn)).unwrap();
+                let cc = CC::from_floatcc(condcode);
+
+                // Verification ensures that the input is always a single-def ffcmp.
+                let ffcmp_insn = matches_input(ctx, inputs[0], Opcode::Ffcmp).unwrap();
+                {
+                    // The only valid CC constructed with `from_floatcc` can be put in the flag
+                    // register with a direct float comparison; do this here.
+                    let input_ty = ctx.input_ty(ffcmp_insn, 0);
+                    let op = match input_ty {
+                        F32 => SseOpcode::Ucomiss,
+                        F64 => SseOpcode::Ucomisd,
+                        _ => panic!("Bad input type to Fcmp"),
+                    };
+                    let inputs = &[
+                        InsnInput {
+                            insn: ffcmp_insn,
+                            input: 0,
+                        },
+                        InsnInput {
+                            insn: ffcmp_insn,
+                            input: 1,
+                        },
+                    ];
+                    let lhs = input_to_reg(ctx, inputs[0]);
+                    let rhs = input_to_reg_mem(ctx, inputs[1]);
+                    ctx.emit(Inst::xmm_cmp_rm_r(op, rhs, lhs));
+                }
+
+                cc
+            };
+
+            ctx.emit(Inst::TrapIf {
+                trap_code,
+                srcloc,
+                cc,
+            });
+        }
+
         Opcode::F64const => {
             // TODO use xorpd for 0
             let value = ctx.get_constant(insn).unwrap();
@@ -1724,7 +1799,6 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
         | Opcode::IaddCin
         | Opcode::IaddIfcin
         | Opcode::IaddCout
-        | Opcode::IaddIfcout
         | Opcode::IaddCarry
         | Opcode::IaddIfcarry
         | Opcode::IsubBin