T0b1/wasmtime
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

cranelift: Remove redundant branch and select instructions (#5097)

Browse Source 
As discussed in the 2022/10/19 meeting, this PR removes many of the branch and select instructions that used iflags, in favor if using brz/brnz and select in their place. Additionally, it reworks selectif_spectre_guard to take an i8 input instead of an iflags input.

For reference, the removed instructions are: br_icmp, brif, brff, trueif, trueff, and selectif.
This commit is contained in:
Trevor Elliott
2022-10-24 16:14:35 -07:00
committed by GitHub
parent 30589170b4
commit ec12415b1f
40 changed files with 579 additions and 2527 deletions
Download Patch File Download Diff File Expand all files Collapse all files

39
cranelift/codegen/meta/src/shared/formats.rs
View File

@@ -9,23 +9,17 @@ pub(crate) struct Formats {
pub(crate) binary_imm8: Rc<InstructionFormat>,
pub(crate) binary_imm64: Rc<InstructionFormat>,
pub(crate) branch: Rc<InstructionFormat>,
pub(crate) branch_float: Rc<InstructionFormat>,
pub(crate) branch_icmp: Rc<InstructionFormat>,
pub(crate) branch_int: Rc<InstructionFormat>,
pub(crate) branch_table: Rc<InstructionFormat>,
pub(crate) call: Rc<InstructionFormat>,
pub(crate) call_indirect: Rc<InstructionFormat>,
pub(crate) cond_trap: Rc<InstructionFormat>,
pub(crate) float_compare: Rc<InstructionFormat>,
pub(crate) float_cond: Rc<InstructionFormat>,
pub(crate) float_cond_trap: Rc<InstructionFormat>,
pub(crate) func_addr: Rc<InstructionFormat>,
pub(crate) heap_addr: Rc<InstructionFormat>,
pub(crate) int_compare: Rc<InstructionFormat>,
pub(crate) int_compare_imm: Rc<InstructionFormat>,
pub(crate) int_cond: Rc<InstructionFormat>,
pub(crate) int_cond_trap: Rc<InstructionFormat>,
pub(crate) int_select: Rc<InstructionFormat>,
pub(crate) jump: Rc<InstructionFormat>,
pub(crate) load: Rc<InstructionFormat>,
pub(crate) load_no_offset: Rc<InstructionFormat>,
@@ -113,23 +107,12 @@ impl Formats {
.imm(&imm.imm64)
.build(),
int_cond: Builder::new("IntCond").imm(&imm.intcc).value().build(),
float_compare: Builder::new("FloatCompare")
.imm(&imm.floatcc)
.value()
.value()
.build(),
float_cond: Builder::new("FloatCond").imm(&imm.floatcc).value().build(),
int_select: Builder::new("IntSelect")
.imm(&imm.intcc)
.value()
.value()
.value()
.build(),
jump: Builder::new("Jump").imm(&entities.block).varargs().build(),
branch: Builder::new("Branch")
@@ -138,28 +121,6 @@ impl Formats {
.varargs()
.build(),
branch_int: Builder::new("BranchInt")
.imm(&imm.intcc)
.value()
.imm(&entities.block)
.varargs()
.build(),
branch_float: Builder::new("BranchFloat")
.imm(&imm.floatcc)
.value()
.imm(&entities.block)
.varargs()
.build(),
branch_icmp: Builder::new("BranchIcmp")
.imm(&imm.intcc)
.value()
.value()
.imm(&entities.block)
.varargs()
.build(),
branch_table: Builder::new("BranchTable")
.value()
.imm(&entities.block)

133
cranelift/codegen/meta/src/shared/instructions.rs
View File

@@ -75,82 +75,9 @@ fn define_control_flow(
);
}
let iB = &TypeVar::new(
"iB",
"A scalar integer type",
TypeSetBuilder::new().ints(Interval::All).build(),
);
let iflags: &TypeVar = &ValueType::Special(types::Flag::IFlags.into()).into();
let fflags: &TypeVar = &ValueType::Special(types::Flag::FFlags.into()).into();
{
let Cond = &Operand::new("Cond", &imm.intcc);
let x = &Operand::new("x", iB);
let y = &Operand::new("y", iB);
ig.push(
Inst::new(
"br_icmp",
r#"
Compare scalar integers and branch.
Compare ``x`` and ``y`` in the same way as the `icmp` instruction
and take the branch if the condition is true:
```text
br_icmp ugt v1, v2, block4(v5, v6)
```
is semantically equivalent to:
```text
v10 = icmp ugt, v1, v2
brnz v10, block4(v5, v6)
```
Some RISC architectures like MIPS and RISC-V provide instructions that
implement all or some of the condition codes. The instruction can also
be used to represent *macro-op fusion* on architectures like Intel's.
"#,
&formats.branch_icmp,
)
.operands_in(vec![Cond, x, y, block, args])
.is_branch(true),
);
let f = &Operand::new("f", iflags);
ig.push(
Inst::new(
"brif",
r#"
Branch when condition is true in integer CPU flags.
"#,
&formats.branch_int,
)
.operands_in(vec![Cond, f, block, args])
.is_branch(true),
);
}
{
let Cond = &Operand::new("Cond", &imm.floatcc);
let f = &Operand::new("f", fflags);
ig.push(
Inst::new(
"brff",
r#"
Branch when condition is true in floating point CPU flags.
"#,
&formats.branch_float,
)
.operands_in(vec![Cond, f, block, args])
.is_branch(true),
);
}
{
let _i32 = &TypeVar::new(
"i32",
@@ -1498,28 +1425,13 @@ pub(crate) fn define(
.operands_out(vec![a]),
);
let cc = &Operand::new("cc", &imm.intcc).with_doc("Controlling condition code");
let flags = &Operand::new("flags", iflags).with_doc("The machine's flag register");
ig.push(
Inst::new(
"selectif",
r#"
Conditional select, dependent on integer condition codes.
"#,
&formats.int_select,
)
.operands_in(vec![cc, flags, x, y])
.operands_out(vec![a]),
);
ig.push(
Inst::new(
"selectif_spectre_guard",
"select_spectre_guard",
r#"
Conditional select intended for Spectre guards.
This operation is semantically equivalent to a selectif instruction.
This operation is semantically equivalent to a select instruction.
However, it is guaranteed to not be removed or otherwise altered by any
optimization pass, and is guaranteed to result in a conditional-move
instruction, not a branch-based lowering. As such, it is suitable
@@ -1534,9 +1446,9 @@ pub(crate) fn define(
speculative path, this ensures that no Spectre vulnerability will
exist.
"#,
&formats.int_select,
&formats.ternary,
)
.operands_in(vec![cc, flags, x, y])
.operands_in(vec![c, x, y])
.operands_out(vec![a])
.other_side_effects(true),
);
@@ -3194,43 +3106,6 @@ pub(crate) fn define(
.operands_out(vec![a]),
);
let Cond = &Operand::new("Cond", &imm.intcc);
let f = &Operand::new("f", iflags);
let a = &Operand::new("a", i8);
ig.push(
Inst::new(
"trueif",
r#"
Test integer CPU flags for a specific condition.
Check the CPU flags in ``f`` against the ``Cond`` condition code and
return true when the condition code is satisfied.
"#,
&formats.int_cond,
)
.operands_in(vec![Cond, f])
.operands_out(vec![a]),
);
let Cond = &Operand::new("Cond", &imm.floatcc);
let f = &Operand::new("f", fflags);
ig.push(
Inst::new(
"trueff",
r#"
Test floating point CPU flags for a specific condition.
Check the CPU flags in ``f`` against the ``Cond`` condition code and
return true when the condition code is satisfied.
"#,
&formats.float_cond,
)
.operands_in(vec![Cond, f])
.operands_out(vec![a]),
);
let x = &Operand::new("x", Mem);
let a = &Operand::new("a", MemTo).with_doc("Bits of `x` reinterpreted");

45
cranelift/codegen/src/ir/instructions.rs
View File

@@ -185,26 +185,11 @@ impl InstructionData {
ref args,
..
} => BranchInfo::SingleDest(destination, args.as_slice(pool)),
Self::BranchInt {
destination,
ref args,
..
}
| Self::BranchFloat {
destination,
ref args,
..
}
| Self::Branch {
Self::Branch {
destination,
ref args,
..
} => BranchInfo::SingleDest(destination, &args.as_slice(pool)[1..]),
Self::BranchIcmp {
destination,
ref args,
..
} => BranchInfo::SingleDest(destination, &args.as_slice(pool)[2..]),
Self::BranchTable {
table, destination, ..
} => BranchInfo::Table(table, Some(destination)),
@@ -221,11 +206,7 @@ impl InstructionData {
/// Multi-destination branches like `br_table` return `None`.
pub fn branch_destination(&self) -> Option<Block> {
match *self {
Self::Jump { destination, .. }
| Self::Branch { destination, .. }
| Self::BranchInt { destination, .. }
| Self::BranchFloat { destination, .. }
| Self::BranchIcmp { destination, .. } => Some(destination),
Self::Jump { destination, .. } | Self::Branch { destination, .. } => Some(destination),
Self::BranchTable { .. } => None,
_ => {
debug_assert!(!self.opcode().is_branch());
@@ -247,18 +228,6 @@ impl InstructionData {
| Self::Branch {
ref mut destination,
..
}
| Self::BranchInt {
ref mut destination,
..
}
| Self::BranchFloat {
ref mut destination,
..
}
| Self::BranchIcmp {
ref mut destination,
..
} => Some(destination),
Self::BranchTable { .. } => None,
_ => {
@@ -284,12 +253,8 @@ impl InstructionData {
/// condition. Otherwise, return `None`.
pub fn cond_code(&self) -> Option<IntCC> {
match self {
&InstructionData::IntCond { cond, .. }
| &InstructionData::BranchIcmp { cond, .. }
| &InstructionData::IntCompare { cond, .. }
&InstructionData::IntCompare { cond, .. }
| &InstructionData::IntCondTrap { cond, .. }
| &InstructionData::BranchInt { cond, .. }
| &InstructionData::IntSelect { cond, .. }
| &InstructionData::IntCompareImm { cond, .. } => Some(cond),
_ => None,
}
@@ -299,9 +264,7 @@ impl InstructionData {
/// condition. Otherwise, return `None`.
pub fn fp_cond_code(&self) -> Option<FloatCC> {
match self {
&InstructionData::BranchFloat { cond, .. }
| &InstructionData::FloatCompare { cond, .. }
| &InstructionData::FloatCond { cond, .. }
&InstructionData::FloatCompare { cond, .. }
| &InstructionData::FloatCondTrap { cond, .. } => Some(cond),
_ => None,
}

86
cranelift/codegen/src/isa/aarch64/lower.isle
View File

@@ -1706,21 +1706,6 @@
(rule (lower (resumable_trap trap_code))
(side_effect (udf trap_code)))
;;;; Rules for `trueif` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Verification ensures the input is always a single-def ifcmp.
(rule (lower (has_type out_ty
(trueif cc insn @ (ifcmp x @ (value_type in_ty) y))))
(lower_icmp_into_reg cc x y in_ty out_ty))
;;;; Rules for `trueff` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Verification ensures the input is always a single-def ffcmp.
(rule (lower (trueff cc insn @ (ffcmp x @ (value_type in_ty) y)))
(with_flags_reg
(fpu_cmp (scalar_size in_ty) x y)
(materialize_bool_result (fp_cond_code cc))))
;;;; Rules for `select` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(rule (lower (has_type ty
@@ -1755,22 +1740,11 @@
(cmp (OperandSize.Size64) rcond (zero_reg))
(Cond.Ne) ty rn rm)))
;;;; Rules for `selectif` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Verification ensures that the input is always a single-def ifcmp.
(rule (lower (has_type ty
(selectif cc flags @ (ifcmp x @ (value_type in_ty) y)
if_true if_false)))
(let ((cond Cond (cond_code cc)))
(lower_select
(lower_icmp_into_flags cc x y in_ty)
cond ty if_true if_false)))
;;;; Rules for `selectif_spectre_guard` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;;;; Rules for `select_spectre_guard` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(rule (lower (has_type ty
(selectif_spectre_guard cc
flags @ (ifcmp x @ (value_type in_ty) y) if_true if_false)))
(select_spectre_guard
(icmp cc x @ (value_type in_ty) y) if_true if_false)))
(let ((cond Cond (cond_code cc))
(dst ValueRegs (lower_select
(lower_icmp_into_flags cc x y in_ty)
@@ -2422,7 +2396,7 @@
(rule (lower (fvpromote_low val))
(vec_rr_long (VecRRLongOp.Fcvtl32) val $false))
;;; Rules for `brz`/`brnz`/`brif`/`brff`/`bricmp` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;;; Rules for `brz`/`brnz` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; `brz` following `icmp`
(rule (lower_branch (brz (icmp cc x @ (value_type ty) y) _ _) targets)
@@ -2505,58 +2479,6 @@
(with_flags_side_effect flags
(cond_br taken not_taken (cond_br_not_zero rt))))))
;; `br_icmp`
(rule (lower_branch (br_icmp cc x @ (value_type ty) y _ _) targets)
(let ((cond Cond (cond_code cc))
(taken BranchTarget (branch_target targets 0))
(not_taken BranchTarget (branch_target targets 1)))
(side_effect
(with_flags_side_effect (lower_icmp_into_flags cc x y ty)
(cond_br taken not_taken
(cond_br_cond cond))))))
;; `brif`
(rule (lower_branch (brif cc (ifcmp x @ (value_type ty) y) _ _) targets)
(let ((cond Cond (cond_code cc))
(taken BranchTarget (branch_target targets 0))
(not_taken BranchTarget (branch_target targets 1)))
(side_effect
(with_flags_side_effect (lower_icmp_into_flags cc x y ty)
(cond_br taken not_taken
(cond_br_cond cond))))))
;; If the `ifcmp` result is actually placed in a register, we need to move it
;; back into the flags.
(rule -1 (lower_branch (brif cc f _ _) targets)
(let ((cond Cond (cond_code cc))
(rn Reg (put_in_reg f))
(taken BranchTarget (branch_target targets 0))
(not_taken BranchTarget (branch_target targets 1)))
(side_effect
(with_flags_side_effect (mov_to_nzcv rn)
(cond_br taken not_taken
(cond_br_cond cond))))))
;; `brff`
(rule (lower_branch (brff cc (ffcmp x @ (value_type ty) y) _ _) targets)
(let ((cond Cond (fp_cond_code cc))
(taken BranchTarget (branch_target targets 0))
(not_taken BranchTarget (branch_target targets 1)))
(side_effect
(with_flags_side_effect (fpu_cmp (scalar_size ty) x y)
(cond_br taken not_taken
(cond_br_cond cond))))))
;; If the `ffcmp` result is actually placed in a register, we need to move it
;; back into the flags.
(rule -1 (lower_branch (brff cc f _ _) targets)
(let ((cond Cond (fp_cond_code cc))
(rn Reg (put_in_reg f))
(taken BranchTarget (branch_target targets 0))
(not_taken BranchTarget (branch_target targets 1)))
(side_effect
(with_flags_side_effect (mov_to_nzcv rn)
(cond_br taken not_taken
(cond_br_cond cond))))))
;;; Rules for `jump` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(rule (lower_branch (jump _ _) targets)

14
cranelift/codegen/src/isa/aarch64/lower_inst.rs
View File

@@ -151,14 +151,10 @@ pub(crate) fn lower_insn_to_regs(
Opcode::Select => implemented_in_isle(ctx),
Opcode::Selectif | Opcode::SelectifSpectreGuard => implemented_in_isle(ctx),
Opcode::SelectSpectreGuard => implemented_in_isle(ctx),
Opcode::Bitselect | Opcode::Vselect => implemented_in_isle(ctx),
Opcode::Trueif => implemented_in_isle(ctx),
Opcode::Trueff => implemented_in_isle(ctx),
Opcode::IsNull | Opcode::IsInvalid => implemented_in_isle(ctx),
Opcode::Copy => implemented_in_isle(ctx),
@@ -205,13 +201,7 @@ pub(crate) fn lower_insn_to_regs(
Opcode::GetPinnedReg | Opcode::SetPinnedReg => implemented_in_isle(ctx),
Opcode::Jump
| Opcode::Brz
| Opcode::Brnz
| Opcode::BrIcmp
| Opcode::Brif
| Opcode::Brff
| Opcode::BrTable => {
Opcode::Jump | Opcode::Brz | Opcode::Brnz | Opcode::BrTable => {
panic!("Branch opcode reached non-branch lowering logic!");
}

23
cranelift/codegen/src/isa/riscv64/inst.isle
View File

@@ -1915,24 +1915,29 @@
(rule
(lower_branch (brz v @ (value_type ty) _ _) targets)
(lower_brz_or_nz (IntCC.Equal) (normalize_value ty v) targets ty))
(rule 1
(lower_branch (brz (icmp cc a @ (value_type ty) b) _ _) targets)
(lower_br_icmp (intcc_inverse cc) a b targets ty))
(rule 1
(lower_branch (brz (fcmp cc a @ (value_type ty) b) _ _) targets)
(lower_br_fcmp (floatcc_inverse cc) a b targets ty))
;;;;
(rule
(lower_branch (brnz v @ (value_type ty) _ _) targets)
(lower_brz_or_nz (IntCC.NotEqual) (normalize_value ty v) targets ty))
;;;
(rule
(lower_branch (br_icmp cc a @ (value_type ty) b _ _) targets)
(rule 1
(lower_branch (brnz (icmp cc a @ (value_type ty) b) _ _) targets)
(lower_br_icmp cc a b targets ty))
(rule
(lower_branch (brif cc (ifcmp a @ (value_type ty) b) _ _) targets)
(lower_br_icmp cc a b targets ty))
(rule
(lower_branch (brff cc (ffcmp a @ (value_type ty) b) _ _) targets)
(rule 1
(lower_branch (brnz (fcmp cc a @ (value_type ty) b) _ _) targets)
(lower_br_fcmp cc a b targets ty))
;;;
(decl lower_br_table (Reg VecMachLabel) InstOutput)
(extern constructor lower_br_table lower_br_table)

29
cranelift/codegen/src/isa/riscv64/lower.isle
View File

@@ -605,6 +605,11 @@
(gen_select ty (normalize_value cty c) x y)
)
(rule 1
(lower (has_type ty (select (icmp cc a b) x y)))
(gen_select_reg cc a b x y)
)
;;;;; Rules for `bitselect`;;;;;;;;;
(rule
@@ -832,35 +837,15 @@
(gen_float_round (FloatRoundOP.Nearest) x ty))
;;;;; Rules for `selectif`;;;;;;;;;
;;;;; Rules for `select_spectre_guard`;;;;;;;;;
(rule
(lower (has_type r_ty (selectif cc (ifcmp ca @ (value_type cty) cb) a b)))
(let
((dst VecWritableReg (alloc_vec_writable r_ty))
(r Reg (lower_icmp cc ca cb cty))
(_ Unit (emit (MInst.SelectIf $false (vec_writable_clone dst) r a b))))
(vec_writable_to_regs dst)))
;;;;; Rules for `selectif_spectre_guard`;;;;;;;;;
(rule
(lower (has_type r_ty (selectif_spectre_guard cc (ifcmp ca @ (value_type cty) cb) a b)))
(lower (has_type r_ty (select_spectre_guard (icmp cc ca @ (value_type cty) cb) a b)))
(let
((dst VecWritableReg (alloc_vec_writable r_ty))
(r Reg (lower_icmp cc ca cb cty))
(_ Unit (emit (MInst.SelectIf $true (vec_writable_clone dst) r a b))))
(vec_writable_to_regs dst)))
;;;;; Rules for `trueif`;;;;;;;;;
(rule
(lower (has_type ty (trueif cc (ifcmp ca @ (value_type cty) cb))))
(lower_icmp cc ca cb cty))
;;;;; Rules for `trueff`;;;;;;;;;
(rule
(lower (has_type ty (trueff cc (ffcmp ca @ (value_type cty) cb))))
(gen_fcmp cc ca cb cty))
;;;;; Rules for `trapif`;;;;;;;;;
(rule

17
cranelift/codegen/src/isa/s390x/lower.isle
View File

@@ -3726,15 +3726,15 @@
(put_in_reg val_true) (put_in_reg val_false)))
;;;; Rules for `selectif_spectre_guard` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;;;; Rules for `select_spectre_guard` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; We do not support the `iflags` mechanism on our platform. However, common
;; code will unconditionally emit certain patterns using `iflags` which we
;; need to handle somehow. Note that only those specific patterns are
;; recognized by the code below, other uses will fail to lower.
(rule (lower (has_type ty (selectif_spectre_guard int_cc
(ifcmp x y) val_true val_false)))
(rule (lower (has_type ty (select_spectre_guard
(icmp int_cc x y) val_true val_false)))
(select_bool_reg ty (icmp_val $false int_cc x y)
(put_in_reg val_true) (put_in_reg val_false)))
@@ -3801,17 +3801,6 @@
(vec_element targets 1))))
;;;; Rules for `brif` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Similarly to `selectif_spectre_guard`, we only recognize specific patterns
;; generated by common code here. Others will fail to lower.
(rule (lower_branch (brif int_cc (ifcmp x y) _ _) targets)
(side_effect (cond_br_bool (icmp_val $false int_cc x y)
(vec_element targets 0)
(vec_element targets 1))))
;;;; Rules for `trap` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(rule (lower (trap trap_code))

17
cranelift/codegen/src/isa/s390x/lower.rs
View File

@@ -171,7 +171,7 @@ impl LowerBackend for S390xBackend {
| Opcode::IsNull
| Opcode::IsInvalid
| Opcode::Select
| Opcode::SelectifSpectreGuard
| Opcode::SelectSpectreGuard
| Opcode::Trap
| Opcode::ResumableTrap
| Opcode::Trapz
@@ -223,21 +223,10 @@ impl LowerBackend for S390xBackend {
Opcode::GlobalValue => {
panic!("global_value should have been removed by legalization!");
}
Opcode::Ifcmp
| Opcode::Ffcmp
| Opcode::Trapff
| Opcode::Trueif
| Opcode::Trueff
| Opcode::Selectif => {
Opcode::Ifcmp | Opcode::Ffcmp | Opcode::Trapff => {
panic!("Flags opcode should not be encountered.");
}
Opcode::Jump
| Opcode::Brz
| Opcode::Brnz
| Opcode::BrIcmp
| Opcode::Brif
| Opcode::Brff
| Opcode::BrTable => {
Opcode::Jump | Opcode::Brz | Opcode::Brnz | Opcode::BrTable => {
panic!("Branch opcode reached non-branch lowering logic!");
}
Opcode::IaddImm

6
cranelift/codegen/src/isa/x64/inst.isle
View File

@@ -1089,12 +1089,6 @@
(decl cc_invert (CC) CC)
(extern constructor cc_invert cc_invert)
(decl intcc_reverse (IntCC) IntCC)
(extern constructor intcc_reverse intcc_reverse)
(decl floatcc_inverse (FloatCC) FloatCC)
(extern constructor floatcc_inverse floatcc_inverse)
;; Fails if the argument is not either CC.NZ or CC.Z.
(decl cc_nz_or_z (CC) CC)
(extern extractor cc_nz_or_z cc_nz_or_z)

22
cranelift/codegen/src/isa/x64/lower.isle
View File

@@ -2760,16 +2760,6 @@
(rule (lower_branch (jump _ _) (single_target target))
(side_effect (jmp_known target)))
;; Rules for `brif` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(rule (lower_branch (brif cc (ifcmp a b) _ _) (two_targets taken not_taken))
(side_effect (jmp_cond_icmp (emit_cmp cc a b) taken not_taken)))
;; Rules for `brff` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(rule (lower_branch (brff cc (ffcmp a b) _ _) (two_targets taken not_taken))
(side_effect (jmp_cond_fcmp (emit_fcmp cc a b) taken not_taken)))
;; Rules for `brz` and `brnz` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(rule 2 (lower_branch (brz (icmp cc a b) _ _) (two_targets taken not_taken))
@@ -2828,22 +2818,14 @@
(src Gpr val))
(x64_test size src src)))
;; Rules for `bricmp` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(rule (lower_branch (br_icmp cc a b _ _) (two_targets taken not_taken))
(side_effect (jmp_cond_icmp (emit_cmp cc a b) taken not_taken)))
;; Rules for `br_table` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(rule (lower_branch (br_table idx @ (value_type ty) _ _) (jump_table_targets default_target jt_targets))
(side_effect (jmp_table_seq ty idx default_target jt_targets)))
;; Rules for `selectif` and `selectif_spectre_guard` ;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Rules for `select_spectre_guard` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(rule (lower (selectif cc (ifcmp a b) x y))
(select_icmp (emit_cmp cc a b) x y))
(rule (lower (selectif_spectre_guard cc (ifcmp a b) x y))
(rule (lower (select_spectre_guard (icmp cc a b) x y))
(select_icmp (emit_cmp cc a b) x y))
;; Rules for `fcvt_from_sint` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

13
cranelift/codegen/src/isa/x64/lower.rs
View File

@@ -419,8 +419,7 @@ fn lower_insn_to_regs(
| Opcode::GetStackPointer
| Opcode::GetReturnAddress
| Opcode::Select
| Opcode::Selectif
| Opcode::SelectifSpectreGuard
| Opcode::SelectSpectreGuard
| Opcode::FcvtFromSint
| Opcode::FcvtLowFromSint
| Opcode::FcvtFromUint
@@ -499,8 +498,6 @@ fn lower_insn_to_regs(
Opcode::Bmask => unimplemented!("Bmask not implemented"),
Opcode::Trueif | Opcode::Trueff => unimplemented!("trueif / trueff not implemented"),
Opcode::Vsplit | Opcode::Vconcat => {
unimplemented!("Vector split/concat ops not implemented.");
}
@@ -570,13 +567,7 @@ fn lower_insn_to_regs(
panic!("trapz / trapnz / resumable_trapnz should have been removed by legalization!");
}
Opcode::Jump
| Opcode::Brz
| Opcode::Brnz
| Opcode::BrIcmp
| Opcode::Brif
| Opcode::Brff
| Opcode::BrTable => {
Opcode::Jump | Opcode::Brz | Opcode::Brnz | Opcode::BrTable => {
panic!("Branch opcode reached non-branch lowering logic!");
}
}

10
cranelift/codegen/src/isa/x64/lower/isle.rs
View File

@@ -593,16 +593,6 @@ impl Context for IsleContext<'_, '_, MInst, Flags, IsaFlags, 6> {
}
}
#[inline]
fn intcc_reverse(&mut self, cc: &IntCC) -> IntCC {
cc.reverse()
}
#[inline]
fn floatcc_inverse(&mut self, cc: &FloatCC) -> FloatCC {
cc.inverse()
}
#[inline]
fn sum_extend_fits_in_32_bits(
&mut self,

4
cranelift/codegen/src/legalizer/heap.rs
View File

@@ -250,11 +250,11 @@ fn compute_addr(
if let Some((cc, a, b)) = spectre_oob_comparison {
let final_addr = pos.ins().iadd(base, offset);
let zero = pos.ins().iconst(addr_ty, 0);
let flags = pos.ins().ifcmp(a, b);
let cmp = pos.ins().icmp(cc, a, b);
pos.func
.dfg
.replace(inst)
.selectif_spectre_guard(addr_ty, cc, flags, zero, final_addr);
.select_spectre_guard(cmp, zero, final_addr);
} else {
pos.func.dfg.replace(inst).iadd(base, offset);
}

21
cranelift/codegen/src/legalizer/mod.rs
View File

@@ -54,27 +54,6 @@ pub fn simple_legalize(func: &mut ir::Function, cfg: &mut ControlFlowGraph, isa:
while let Some(inst) = pos.next_inst() {
match pos.func.dfg[inst] {
// control flow
InstructionData::BranchIcmp {
opcode: ir::Opcode::BrIcmp,
cond,
destination,
ref args,
} => {
let a = args.get(0, &pos.func.dfg.value_lists).unwrap();
let b = args.get(1, &pos.func.dfg.value_lists).unwrap();
let block_args = args.as_slice(&pos.func.dfg.value_lists)[2..].to_vec();
let old_block = pos.func.layout.pp_block(inst);
pos.func.dfg.clear_results(inst);
let icmp_res = pos.func.dfg.replace(inst).icmp(cond, a, b);
let mut pos = FuncCursor::new(pos.func).after_inst(inst);
pos.use_srcloc(inst);
pos.ins().brnz(icmp_res, destination, &block_args);
cfg.recompute_block(pos.func, destination);
cfg.recompute_block(pos.func, old_block);
}
InstructionData::CondTrap {
opcode:
opcode @ (ir::Opcode::Trapnz | ir::Opcode::Trapz | ir::Opcode::ResumableTrapnz),

12
cranelift/codegen/src/legalizer/table.rs
View File

@@ -99,15 +99,11 @@ fn compute_addr(
};
let element_addr = if let Some((index, bound)) = spectre_oob_cmp {
let flags = pos.ins().ifcmp(index, bound);
let cond = pos
.ins()
.icmp(IntCC::UnsignedGreaterThanOrEqual, index, bound);
// If out-of-bounds, choose the table base on the misspeculation path.
pos.ins().selectif_spectre_guard(
addr_ty,
IntCC::UnsignedGreaterThanOrEqual,
flags,
base,
element_addr,
)
pos.ins().select_spectre_guard(cond, base, element_addr)
} else {
element_addr
};

24
cranelift/codegen/src/machinst/isle.rs
View File

@@ -8,8 +8,8 @@ use target_lexicon::Triple;
pub use super::MachLabel;
pub use crate::ir::{
condcodes, dynamic_to_fixed, ArgumentExtension, Constant, DynamicStackSlot, ExternalName,
FuncRef, GlobalValue, Immediate, SigRef, StackSlot,
condcodes, condcodes::CondCode, dynamic_to_fixed, ArgumentExtension, Constant,
DynamicStackSlot, ExternalName, FuncRef, GlobalValue, Immediate, SigRef, StackSlot,
};
pub use crate::isa::unwind::UnwindInst;
pub use crate::machinst::{
@@ -570,6 +570,26 @@ macro_rules! isle_lower_prelude_methods {
fn gen_move(&mut self, ty: Type, dst: WritableReg, src: Reg) -> MInst {
MInst::gen_move(dst, src, ty)
}
#[inline]
fn intcc_reverse(&mut self, cc: &IntCC) -> IntCC {
cc.reverse()
}
#[inline]
fn intcc_inverse(&mut self, cc: &IntCC) -> IntCC {
cc.inverse()
}
#[inline]
fn floatcc_reverse(&mut self, cc: &FloatCC) -> FloatCC {
cc.reverse()
}
#[inline]
fn floatcc_inverse(&mut self, cc: &FloatCC) -> FloatCC {
cc.inverse()
}
};
}

17
cranelift/codegen/src/prelude_lower.isle
View File

@@ -320,6 +320,23 @@
;;;; Helpers for Working with Flags ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Reverse an IntCC flag.
(decl intcc_reverse (IntCC) IntCC)
(extern constructor intcc_reverse intcc_reverse)
;; Invert an IntCC flag.
(decl intcc_inverse (IntCC) IntCC)
(extern constructor intcc_inverse intcc_inverse)
;; Reverse an FloatCC flag.
(decl floatcc_reverse (FloatCC) FloatCC)
(extern constructor floatcc_reverse floatcc_reverse)
;; Invert an FloatCC flag.
(decl floatcc_inverse (FloatCC) FloatCC)
(extern constructor floatcc_inverse floatcc_inverse)
;; Newtype wrapper around `MInst` for instructions that are used for their
;; effect on flags.
;;

35
cranelift/codegen/src/simple_preopt.rs
View File

@@ -469,7 +469,6 @@ fn do_divrem_transformation(divrem_info: &DivRemByConstInfo, pos: &mut FuncCurso
enum BranchOrderKind {
BrzToBrnz(Value),
BrnzToBrz(Value),
InvertIcmpCond(IntCC, Value, Value),
}
/// Reorder branches to encourage fallthroughs.
@@ -538,27 +537,6 @@ fn branch_order(pos: &mut FuncCursor, cfg: &mut ControlFlowGraph, block: Block,
kind,
)
}
InstructionData::BranchIcmp {
opcode: Opcode::BrIcmp,
cond,
destination: cond_dest,
args: ref prev_args,
} => {
let (x_arg, y_arg) = {
let args = pos.func.dfg.inst_args(prev_inst);
(args[0], args[1])
};
(
inst,
args.clone(),
destination,
prev_inst,
prev_args.clone(),
*cond_dest,
BranchOrderKind::InvertIcmpCond(*cond, x_arg, y_arg),
)
}
_ => return,
}
}
@@ -590,19 +568,6 @@ fn branch_order(pos: &mut FuncCursor, cfg: &mut ControlFlowGraph, block: Block,
.replace(cond_inst)
.brnz(cond_arg, term_dest, &term_args);
}
BranchOrderKind::InvertIcmpCond(cond, x_arg, y_arg) => {
pos.func
.dfg
.replace(term_inst)
.jump(cond_dest, &cond_args[2..]);
pos.func.dfg.replace(cond_inst).br_icmp(
cond.inverse(),
x_arg,
y_arg,
term_dest,
&term_args,
);
}
}
cfg.recompute_block(pos.func, block);

18
cranelift/codegen/src/verifier/mod.rs
View File

@@ -639,21 +639,6 @@ impl<'a> Verifier<'a> {
destination,
ref args,
..
}
| BranchInt {
destination,
ref args,
..
}
| BranchFloat {
destination,
ref args,
..
}
| BranchIcmp {
destination,
ref args,
..
} => {
self.verify_block(inst, destination, errors)?;
self.verify_value_list(inst, args, errors)?;
@@ -776,10 +761,7 @@ impl<'a> Verifier<'a> {
| Shuffle { .. }
| IntCompare { .. }
| IntCompareImm { .. }
| IntCond { .. }
| FloatCompare { .. }
| FloatCond { .. }
| IntSelect { .. }
| Load { .. }
| Store { .. }
| Trap { .. }

35
cranelift/codegen/src/write.rs
View File

@@ -418,12 +418,7 @@ pub fn write_operands(w: &mut dyn Write, dfg: &DataFlowGraph, inst: Inst) -> fmt
}
IntCompare { cond, args, .. } => write!(w, " {} {}, {}", cond, args[0], args[1]),
IntCompareImm { cond, arg, imm, .. } => write!(w, " {} {}, {}", cond, arg, imm),
IntCond { cond, arg, .. } => write!(w, " {} {}", cond, arg),
FloatCompare { cond, args, .. } => write!(w, " {} {}, {}", cond, args[0], args[1]),
FloatCond { cond, arg, .. } => write!(w, " {} {}", cond, arg),
IntSelect { cond, args, .. } => {
write!(w, " {} {}, {}, {}", cond, args[0], args[1], args[2])
}
Jump {
destination,
ref args,
@@ -441,36 +436,6 @@ pub fn write_operands(w: &mut dyn Write, dfg: &DataFlowGraph, inst: Inst) -> fmt
write!(w, " {}, {}", args[0], destination)?;
write_block_args(w, &args[1..])
}
BranchInt {
cond,
destination,
ref args,
..
} => {
let args = args.as_slice(pool);
write!(w, " {} {}, {}", cond, args[0], destination)?;
write_block_args(w, &args[1..])
}
BranchFloat {
cond,
destination,
ref args,
..
} => {
let args = args.as_slice(pool);
write!(w, " {} {}, {}", cond, args[0], destination)?;
write_block_args(w, &args[1..])
}
BranchIcmp {
cond,
destination,
ref args,
..
} => {
let args = args.as_slice(pool);
write!(w, " {} {}, {}, {}", cond, args[0], args[1], destination)?;
write_block_args(w, &args[2..])
}
BranchTable {
arg,
destination,