diff --git a/cranelift/codegen/shared/src/condcodes.rs b/cranelift/codegen/shared/src/condcodes.rs index 03ae865ce4..2695b5b471 100644 --- a/cranelift/codegen/shared/src/condcodes.rs +++ b/cranelift/codegen/shared/src/condcodes.rs @@ -122,6 +122,27 @@ impl IntCC { } } + /// Determines whether this condcode interprets inputs as signed or + /// unsigned. See the documentation for the `icmp` instruction in + /// cranelift-codegen/meta/src/shared/instructions.rs for further insights + /// into this. + pub fn is_signed(&self) -> bool { + match self { + IntCC::Equal + | IntCC::UnsignedGreaterThanOrEqual + | IntCC::UnsignedGreaterThan + | IntCC::UnsignedLessThanOrEqual + | IntCC::UnsignedLessThan + | IntCC::NotEqual => false, + IntCC::SignedGreaterThanOrEqual + | IntCC::SignedGreaterThan + | IntCC::SignedLessThanOrEqual + | IntCC::SignedLessThan + | IntCC::Overflow + | IntCC::NotOverflow => true, + } + } + /// Get the corresponding string condition code for the IntCC object. pub fn to_static_str(self) -> &'static str { use self::IntCC::*; diff --git a/cranelift/codegen/src/isa/aarch64/lower.rs b/cranelift/codegen/src/isa/aarch64/lower.rs index d8011d0789..c98cb128b7 100644 --- a/cranelift/codegen/src/isa/aarch64/lower.rs +++ b/cranelift/codegen/src/isa/aarch64/lower.rs @@ -798,27 +798,6 @@ pub(crate) fn lower_vector_compare>( Ok(()) } -/// Determines whether this condcode interprets inputs as signed or -/// unsigned. See the documentation for the `icmp` instruction in -/// cranelift-codegen/meta/src/shared/instructions.rs for further insights -/// into this. -pub fn condcode_is_signed(cc: IntCC) -> bool { - match cc { - IntCC::Equal => false, - IntCC::NotEqual => false, - IntCC::SignedGreaterThanOrEqual => true, - IntCC::SignedGreaterThan => true, - IntCC::SignedLessThanOrEqual => true, - IntCC::SignedLessThan => true, - IntCC::UnsignedGreaterThanOrEqual => false, - IntCC::UnsignedGreaterThan => false, - IntCC::UnsignedLessThanOrEqual => false, - IntCC::UnsignedLessThan => false, - IntCC::Overflow => true, - IntCC::NotOverflow => true, - } -} - //============================================================================= // Helpers for instruction lowering. diff --git a/cranelift/codegen/src/isa/aarch64/lower_inst.rs b/cranelift/codegen/src/isa/aarch64/lower_inst.rs index 5db776765d..5b89ce1436 100644 --- a/cranelift/codegen/src/isa/aarch64/lower_inst.rs +++ b/cranelift/codegen/src/isa/aarch64/lower_inst.rs @@ -1115,7 +1115,7 @@ pub(crate) fn lower_insn_to_regs>( { let condcode = inst_condcode(ctx.data(icmp_insn)).unwrap(); let cond = lower_condcode(condcode); - let is_signed = condcode_is_signed(condcode); + let is_signed = condcode.is_signed(); lower_icmp_or_ifcmp_to_flags(ctx, icmp_insn, is_signed); cond } else if let Some(fcmp_insn) = @@ -1161,7 +1161,7 @@ pub(crate) fn lower_insn_to_regs>( Opcode::Selectif | Opcode::SelectifSpectreGuard => { let condcode = inst_condcode(ctx.data(insn)).unwrap(); let cond = lower_condcode(condcode); - let is_signed = condcode_is_signed(condcode); + let is_signed = condcode.is_signed(); // Verification ensures that the input is always a // single-def ifcmp. let ifcmp_insn = maybe_input_insn(ctx, inputs[0], Opcode::Ifcmp).unwrap(); @@ -1232,7 +1232,7 @@ pub(crate) fn lower_insn_to_regs>( Opcode::Trueif => { let condcode = inst_condcode(ctx.data(insn)).unwrap(); let cond = lower_condcode(condcode); - let is_signed = condcode_is_signed(condcode); + let is_signed = condcode.is_signed(); // Verification ensures that the input is always a // single-def ifcmp. let ifcmp_insn = maybe_input_insn(ctx, inputs[0], Opcode::Ifcmp).unwrap(); @@ -1433,7 +1433,7 @@ pub(crate) fn lower_insn_to_regs>( Opcode::Icmp => { let condcode = inst_condcode(ctx.data(insn)).unwrap(); let cond = lower_condcode(condcode); - let is_signed = condcode_is_signed(condcode); + let is_signed = condcode.is_signed(); let rd = get_output_reg(ctx, outputs[0]); let ty = ctx.input_ty(insn, 0); let bits = ty_bits(ty); @@ -1509,7 +1509,7 @@ pub(crate) fn lower_insn_to_regs>( } else if op == Opcode::Trapif { let condcode = inst_condcode(ctx.data(insn)).unwrap(); let cond = lower_condcode(condcode); - let is_signed = condcode_is_signed(condcode); + let is_signed = condcode.is_signed(); // Verification ensures that the input is always a single-def ifcmp. let ifcmp_insn = maybe_input_insn(ctx, inputs[0], Opcode::Ifcmp).unwrap(); @@ -2360,7 +2360,7 @@ pub(crate) fn lower_branch>( { let condcode = inst_condcode(ctx.data(icmp_insn)).unwrap(); let cond = lower_condcode(condcode); - let is_signed = condcode_is_signed(condcode); + let is_signed = condcode.is_signed(); let negated = op0 == Opcode::Brz; let cond = if negated { cond.invert() } else { cond }; @@ -2410,7 +2410,7 @@ pub(crate) fn lower_branch>( let cond = lower_condcode(condcode); let kind = CondBrKind::Cond(cond); - let is_signed = condcode_is_signed(condcode); + let is_signed = condcode.is_signed(); let ty = ctx.input_ty(branches[0], 0); let bits = ty_bits(ty); let narrow_mode = match (bits <= 32, is_signed) { @@ -2451,7 +2451,7 @@ pub(crate) fn lower_branch>( let cond = lower_condcode(condcode); let kind = CondBrKind::Cond(cond); - let is_signed = condcode_is_signed(condcode); + let is_signed = condcode.is_signed(); let flag_input = InsnInput { insn: branches[0], input: 0, diff --git a/cranelift/codegen/src/isa/x64/inst/args.rs b/cranelift/codegen/src/isa/x64/inst/args.rs index 162ed8c2d1..f469be43c7 100644 --- a/cranelift/codegen/src/isa/x64/inst/args.rs +++ b/cranelift/codegen/src/isa/x64/inst/args.rs @@ -726,9 +726,9 @@ impl CC { | FloatCC::LessThan | FloatCC::LessThanOrEqual | FloatCC::UnorderedOrGreaterThan - | FloatCC::UnorderedOrGreaterThanOrEqual => unimplemented!( - "No single condition code to guarantee ordered. Treat as special case." - ), + | FloatCC::UnorderedOrGreaterThanOrEqual => { + panic!("No single condition code to guarantee ordered. Treat as special case.") + } } } diff --git a/cranelift/codegen/src/isa/x64/lower.rs b/cranelift/codegen/src/isa/x64/lower.rs index feb72703ce..0ffb4cef93 100644 --- a/cranelift/codegen/src/isa/x64/lower.rs +++ b/cranelift/codegen/src/isa/x64/lower.rs @@ -135,6 +135,17 @@ struct InsnOutput { output: usize, } +fn matches_input>(c: &mut C, input: InsnInput, op: Opcode) -> Option { + let inputs = c.get_input(input.insn, input.input); + if let Some((src_inst, _)) = inputs.inst { + let data = c.data(src_inst); + if data.opcode() == op { + return Some(src_inst); + } + } + None +} + fn input_to_reg(ctx: Ctx, spec: InsnInput) -> Reg { let inputs = ctx.get_input(spec.insn, spec.input); ctx.use_input_reg(inputs); @@ -328,7 +339,13 @@ fn lower_insn_to_regs>( } } - Opcode::Iadd | Opcode::Isub | Opcode::Imul | Opcode::Band | Opcode::Bor | Opcode::Bxor => { + Opcode::Iadd + | Opcode::IaddIfcout + | Opcode::Isub + | Opcode::Imul + | Opcode::Band + | Opcode::Bor + | Opcode::Bxor => { let lhs = input_to_reg(ctx, inputs[0]); let rhs = input_to_reg_mem_imm(ctx, inputs[1]); let dst = output_to_reg(ctx, outputs[0]); @@ -338,7 +355,7 @@ fn lower_insn_to_regs>( let is_64 = int_ty_is_64(ty.unwrap()); let alu_op = match op { - Opcode::Iadd => AluRmiROpcode::Add, + Opcode::Iadd | Opcode::IaddIfcout => AluRmiROpcode::Add, Opcode::Isub => AluRmiROpcode::Sub, Opcode::Imul => AluRmiROpcode::Mul, Opcode::Band => AluRmiROpcode::And, @@ -965,6 +982,64 @@ fn lower_insn_to_regs>( ctx.emit(Inst::Ud2 { trap_info }) } + Opcode::Trapif | Opcode::Trapff => { + let srcloc = ctx.srcloc(insn); + let trap_code = inst_trapcode(ctx.data(insn)).unwrap(); + + let cc = if matches_input(ctx, inputs[0], Opcode::IaddIfcout).is_some() { + let condcode = inst_condcode(ctx.data(insn)); + // The flags must not have been clobbered by any other instruction between the + // iadd_ifcout and this instruction, as verified by the CLIF validator; so we can + // simply use the flags here. + CC::from_intcc(condcode) + } else if op == Opcode::Trapif { + let condcode = inst_condcode(ctx.data(insn)); + let cc = CC::from_intcc(condcode); + + // Verification ensures that the input is always a single-def ifcmp. + let ifcmp_insn = matches_input(ctx, inputs[0], Opcode::Ifcmp).unwrap(); + emit_cmp(ctx, ifcmp_insn); + cc + } else { + let condcode = inst_fp_condcode(ctx.data(insn)).unwrap(); + let cc = CC::from_floatcc(condcode); + + // Verification ensures that the input is always a single-def ffcmp. + let ffcmp_insn = matches_input(ctx, inputs[0], Opcode::Ffcmp).unwrap(); + { + // The only valid CC constructed with `from_floatcc` can be put in the flag + // register with a direct float comparison; do this here. + let input_ty = ctx.input_ty(ffcmp_insn, 0); + let op = match input_ty { + F32 => SseOpcode::Ucomiss, + F64 => SseOpcode::Ucomisd, + _ => panic!("Bad input type to Fcmp"), + }; + let inputs = &[ + InsnInput { + insn: ffcmp_insn, + input: 0, + }, + InsnInput { + insn: ffcmp_insn, + input: 1, + }, + ]; + let lhs = input_to_reg(ctx, inputs[0]); + let rhs = input_to_reg_mem(ctx, inputs[1]); + ctx.emit(Inst::xmm_cmp_rm_r(op, rhs, lhs)); + } + + cc + }; + + ctx.emit(Inst::TrapIf { + trap_code, + srcloc, + cc, + }); + } + Opcode::F64const => { // TODO use xorpd for 0 let value = ctx.get_constant(insn).unwrap(); @@ -1724,7 +1799,6 @@ fn lower_insn_to_regs>( | Opcode::IaddCin | Opcode::IaddIfcin | Opcode::IaddCout - | Opcode::IaddIfcout | Opcode::IaddCarry | Opcode::IaddIfcarry | Opcode::IsubBin