diff --git a/cranelift/codegen/src/isa/x64/inst.isle b/cranelift/codegen/src/isa/x64/inst.isle index d52f0107cf..88a3f1e6f9 100644 --- a/cranelift/codegen/src/isa/x64/inst.isle +++ b/cranelift/codegen/src/isa/x64/inst.isle @@ -515,6 +515,16 @@ (type BoxVecMachLabel extern (enum)) +(type MachLabelSlice extern (enum)) + +;; Extract a the target from a MachLabelSlice with exactly one target. +(decl single_target (MachLabel) MachLabelSlice) +(extern extractor single_target single_target) + +;; Extract a the targets from a MachLabelSlice with exactly two targets. +(decl two_targets (MachLabel MachLabel) MachLabelSlice) +(extern extractor two_targets two_targets) + ;; Get the `OperandSize` for a given `Type`, rounding smaller types up to 32 bits. (decl operand_size_of_type_32_64 (Type) OperandSize) (extern constructor operand_size_of_type_32_64 operand_size_of_type_32_64) @@ -1057,9 +1067,13 @@ NLE P NP)) + (decl intcc_to_cc (IntCC) CC) (extern constructor intcc_to_cc intcc_to_cc) +(decl cc_invert (CC) CC) +(extern constructor cc_invert cc_invert) + (type AvxOpcode extern (enum Vfmadd213ps Vfmadd213pd)) @@ -3039,6 +3053,23 @@ (rule (trap_if_fcmp (FcmpCondResult.OrCondition producer cc1 cc2) tc) (with_flags_side_effect producer (trap_if_or cc1 cc2 tc))) +;;;; Jumps ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; Unconditional jump. +(decl jmp_known (MachLabel) SideEffectNoResult) +(rule (jmp_known target) + (SideEffectNoResult.Inst (MInst.JmpKnown target))) + +;; Conditional jump based on the condition code. +(decl jmp_cond (CC MachLabel MachLabel) ConsumesFlags) +(rule (jmp_cond cc taken not_taken) + (ConsumesFlags.ConsumesFlagsSideEffect (MInst.JmpCond cc taken not_taken))) + +;; Conditional jump based on the result of an icmp. +(decl jmp_cond_icmp (IcmpCondResult MachLabel MachLabel) SideEffectNoResult) +(rule (jmp_cond_icmp (IcmpCondResult.Condition producer cc) taken not_taken) + (with_flags_side_effect producer (jmp_cond cc taken not_taken))) + ;;;; Comparisons ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (type IcmpCondResult (enum (Condition (producer ProducesFlags) (cc CC)))) @@ -3046,6 +3077,10 @@ (decl icmp_cond_result (ProducesFlags CC) IcmpCondResult) (rule (icmp_cond_result producer cc) (IcmpCondResult.Condition producer cc)) +(decl invert_icmp_cond_result (IcmpCondResult) IcmpCondResult) +(rule (invert_icmp_cond_result (IcmpCondResult.Condition producer cc)) + (icmp_cond_result producer (cc_invert cc))) + ;; Lower an Icmp result into a boolean value in a register. (decl lower_icmp_bool (IcmpCondResult) ValueRegs) (rule (lower_icmp_bool (IcmpCondResult.Condition producer cc)) diff --git a/cranelift/codegen/src/isa/x64/lower.isle b/cranelift/codegen/src/isa/x64/lower.isle index 1004207c47..376746656d 100644 --- a/cranelift/codegen/src/isa/x64/lower.isle +++ b/cranelift/codegen/src/isa/x64/lower.isle @@ -4,6 +4,11 @@ ;; register(s) within which the lowered instruction's result values live. (decl lower (Inst) InstOutput) +;; A variant of the main lowering constructor term, used for branches. +;; The only difference is that it gets an extra argument holding a vector +;; of branch targets to be used. +(decl lower_branch (Inst MachLabelSlice) InstOutput) + ;;;; Rules for `iconst` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; `i64` and smaller. @@ -2856,3 +2861,30 @@ (x64_load $I64 (Amode.ImmReg 8 (preg_rbp) (mem_flags_trusted)) (ExtKind.None))) + +;; Rules for `jump` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(rule (lower_branch (jump _ _) (single_target target)) + (side_effect (jmp_known target))) + +;; Rules for `brif` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(rule (lower_branch (brif cc (ifcmp a b) _ _) (two_targets taken not_taken)) + (side_effect (jmp_cond_icmp (emit_cmp cc a b) taken not_taken))) + +;; Rules for `brz` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(rule (lower_branch (brz (icmp cc a b) _ _) (two_targets taken not_taken)) + (let ((cmp IcmpCondResult (invert_icmp_cond_result (emit_cmp cc a b)))) + (side_effect (jmp_cond_icmp cmp taken not_taken)))) + +;; Rules for `brnz` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(rule (lower_branch (brnz (icmp cc a b) _ _) (two_targets taken not_taken)) + (side_effect (jmp_cond_icmp (emit_cmp cc a b) taken not_taken))) + + +;; Rules for `bricmp` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(rule (lower_branch (br_icmp cc a b _ _) (two_targets taken not_taken)) + (side_effect (jmp_cond_icmp (emit_cmp cc a b) taken not_taken))) diff --git a/cranelift/codegen/src/isa/x64/lower.rs b/cranelift/codegen/src/isa/x64/lower.rs index 4758058f12..dc7335fbc3 100644 --- a/cranelift/codegen/src/isa/x64/lower.rs +++ b/cranelift/codegen/src/isa/x64/lower.rs @@ -2833,6 +2833,24 @@ impl LowerBackend for X64Backend { // verifier pass. assert!(branches.len() <= 2); + if let Ok(()) = isle::lower_branch( + ctx, + &self.triple, + &self.flags, + &self.x64_flags, + branches[0], + targets, + ) { + return Ok(()); + } + + let implemented_in_isle = |ctx: &mut C| { + unreachable!( + "branch implemented in ISLE: inst = `{}`", + ctx.dfg().display_inst(branches[0]) + ) + }; + if branches.len() == 2 { // Must be a conditional branch followed by an unconditional branch. let op0 = ctx.data(branches[0]).opcode(); @@ -2858,18 +2876,8 @@ impl LowerBackend for X64Backend { let src_ty = ctx.input_ty(branches[0], 0); - if let Some(icmp) = matches_input(ctx, flag_input, Opcode::Icmp) { - let cond_code = ctx.data(icmp).cond_code().unwrap(); - let cond_code = emit_cmp(ctx, icmp, cond_code); - - let cond_code = if op0 == Opcode::Brz { - cond_code.inverse() - } else { - cond_code - }; - - let cc = CC::from_intcc(cond_code); - ctx.emit(Inst::jmp_cond(cc, taken, not_taken)); + if let Some(_icmp) = matches_input(ctx, flag_input, Opcode::Icmp) { + implemented_in_isle(ctx) } else if let Some(fcmp) = matches_input(ctx, flag_input, Opcode::Fcmp) { let cond_code = ctx.data(fcmp).fp_cond_code().unwrap(); let cond_code = if op0 == Opcode::Brz { @@ -2960,49 +2968,7 @@ impl LowerBackend for X64Backend { } } - Opcode::BrIcmp => { - let src_ty = ctx.input_ty(branches[0], 0); - if is_int_or_ref_ty(src_ty) || is_bool_ty(src_ty) { - let lhs = put_input_in_reg( - ctx, - InsnInput { - insn: branches[0], - input: 0, - }, - ); - let rhs = input_to_reg_mem_imm( - ctx, - InsnInput { - insn: branches[0], - input: 1, - }, - ); - let cc = CC::from_intcc(ctx.data(branches[0]).cond_code().unwrap()); - // Cranelift's icmp semantics want to compare lhs - rhs, while Intel gives - // us dst - src at the machine instruction level, so invert operands. - ctx.emit(Inst::cmp_rmi_r(OperandSize::from_ty(src_ty), rhs, lhs)); - ctx.emit(Inst::jmp_cond(cc, taken, not_taken)); - } else { - unimplemented!("bricmp with non-int type {:?}", src_ty); - } - } - - Opcode::Brif => { - let flag_input = InsnInput { - insn: branches[0], - input: 0, - }; - - if let Some(ifcmp) = matches_input(ctx, flag_input, Opcode::Ifcmp) { - let cond_code = ctx.data(branches[0]).cond_code().unwrap(); - let cond_code = emit_cmp(ctx, ifcmp, cond_code); - let cc = CC::from_intcc(cond_code); - ctx.emit(Inst::jmp_cond(cc, taken, not_taken)); - } else { - // Should be disallowed by flags checks in verifier. - unimplemented!("Brif with non-ifcmp input"); - } - } + Opcode::BrIcmp | Opcode::Brif => implemented_in_isle(ctx), Opcode::Brff => { let flag_input = InsnInput { insn: branches[0], @@ -3039,9 +3005,7 @@ impl LowerBackend for X64Backend { // Must be an unconditional branch or trap. let op = ctx.data(branches[0]).opcode(); match op { - Opcode::Jump => { - ctx.emit(Inst::jmp_known(targets[0])); - } + Opcode::Jump => implemented_in_isle(ctx), Opcode::BrTable => { let jt_size = targets.len() - 1; diff --git a/cranelift/codegen/src/isa/x64/lower/isle.rs b/cranelift/codegen/src/isa/x64/lower/isle.rs index 046ff28972..7eae08707e 100644 --- a/cranelift/codegen/src/isa/x64/lower/isle.rs +++ b/cranelift/codegen/src/isa/x64/lower/isle.rs @@ -41,6 +41,7 @@ use target_lexicon::Triple; type BoxCallInfo = Box; type BoxVecMachLabel = Box>; +type MachLabelSlice = [MachLabel]; pub struct SinkableLoad { inst: Inst, @@ -71,6 +72,28 @@ where ) } +pub(crate) fn lower_branch( + lower_ctx: &mut C, + triple: &Triple, + flags: &Flags, + isa_flags: &IsaFlags, + branch: Inst, + targets: &[MachLabel], +) -> Result<(), ()> +where + C: LowerCtx, +{ + lower_common( + lower_ctx, + triple, + flags, + isa_flags, + &[], + branch, + |cx, insn| generated_code::constructor_lower_branch(cx, insn, targets), + ) +} + impl Context for IsleContext<'_, C, Flags, IsaFlags, 6> where C: LowerCtx, @@ -562,6 +585,11 @@ where CC::from_intcc(*intcc) } + #[inline] + fn cc_invert(&mut self, cc: &CC) -> CC { + cc.invert() + } + #[inline] fn sum_extend_fits_in_32_bits( &mut self, @@ -675,6 +703,24 @@ where output_reg.to_reg() } + + #[inline] + fn single_target(&mut self, targets: &MachLabelSlice) -> Option { + if targets.len() == 1 { + Some(targets[0]) + } else { + None + } + } + + #[inline] + fn two_targets(&mut self, targets: &MachLabelSlice) -> Option<(MachLabel, MachLabel)> { + if targets.len() == 2 { + Some((targets[0], targets[1])) + } else { + None + } + } } impl IsleContext<'_, C, Flags, IsaFlags, 6>