T0b1/wasmtime
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Rework x64 addressing-mode lowering to be slightly more flexible. (#4080)

Browse Source 
This PR refactors the x64 backend address-mode lowering to use an
incremental-build approach, where it considers each node in a tree of
`iadd`s that feed into a load/store address and, at each step, builds
the best possible `Amode`. It will combine an arbitrary number of
constant offsets (an extension beyond the current rules), and can
capture a left-shifted (scaled) index in any position of the tree
(another extension).

This doesn't have any measurable performance improvement on our Wasm
benchmarks in Sightglass, unfortunately, because the IR lowered from
wasm32 will do address computation in 32 bits and then `uextend` it to
add to the 64-bit heap base. We can't quite lift the 32-bit adds to 64
bits because this loses the wraparound semantics.

(We could label adds as "expected not to overflow", and allow *those* to
be lifted to 64 bit operations; wasm32 heap address computation should
fit this.  This is `add nuw` (no unsigned wrap) in LLVM IR terms. That's
likely my next step.)

Nevertheless, (i) this generalizes the cases we can handle, which should
be a good thing, all other things being equal (and in this case, no
compile time impact was measured); and (ii) might benefit non-Wasm
frontends.
This commit is contained in:
Chris Fallin
2022-05-02 16:20:39 -07:00
committed by GitHub
parent 61dc38c065
commit f85047b084
13 changed files with 1416 additions and 742 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest
View File

@@ -1,4 +1,4 @@
src/clif.isle 443b34b797fc8ace
src/prelude.isle a7915a6b88310eb5
src/prelude.isle 97c4b6eebbab9f05
src/isa/aarch64/inst.isle 21a43af20be377d2
src/isa/aarch64/lower.isle 75ad8450963e3829

85
cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.rs generated
View File

@@ -21,6 +21,12 @@ pub trait Context {
fn unpack_value_array_3(&mut self, arg0: &ValueArray3) -> (Value, Value, Value);
fn pack_value_array_3(&mut self, arg0: Value, arg1: Value, arg2: Value) -> ValueArray3;
fn u32_add(&mut self, arg0: u32, arg1: u32) -> u32;
fn s32_add_fallible(&mut self, arg0: u32, arg1: u32) -> Option<u32>;
fn u32_nonnegative(&mut self, arg0: u32) -> Option<u32>;
fn offset32(&mut self, arg0: Offset32) -> Option<u32>;
fn u32_lteq(&mut self, arg0: u32, arg1: u32) -> Option<Unit>;
fn simm32(&mut self, arg0: Imm64) -> Option<u32>;
fn uimm8(&mut self, arg0: Imm64) -> Option<u8>;
fn u8_and(&mut self, arg0: u8, arg1: u8) -> u8;
fn value_reg(&mut self, arg0: Reg) -> ValueRegs;
fn value_regs(&mut self, arg0: Reg, arg1: Reg) -> ValueRegs;
@@ -32,19 +38,22 @@ pub trait Context {
fn output_builder_push(&mut self, arg0: &InstOutputBuilder, arg1: ValueRegs) -> Unit;
fn output_builder_finish(&mut self, arg0: &InstOutputBuilder) -> InstOutput;
fn temp_writable_reg(&mut self, arg0: Type) -> WritableReg;
fn invalid_reg_etor(&mut self, arg0: Reg) -> Option<()>;
fn invalid_reg(&mut self) -> Reg;
fn valid_reg(&mut self, arg0: Reg) -> Option<()>;
fn put_in_reg(&mut self, arg0: Value) -> Reg;
fn put_in_regs(&mut self, arg0: Value) -> ValueRegs;
fn ensure_in_vreg(&mut self, arg0: Reg, arg1: Type) -> Reg;
fn value_regs_get(&mut self, arg0: ValueRegs, arg1: usize) -> Reg;
fn u8_as_u64(&mut self, arg0: u8) -> u64;
fn u16_as_u64(&mut self, arg0: u16) -> u64;
fn u32_as_u64(&mut self, arg0: u32) -> u64;
fn i64_as_u64(&mut self, arg0: i64) -> u64;
fn u64_add(&mut self, arg0: u64, arg1: u64) -> u64;
fn u64_sub(&mut self, arg0: u64, arg1: u64) -> u64;
fn u64_and(&mut self, arg0: u64, arg1: u64) -> u64;
fn ty_bits(&mut self, arg0: Type) -> u8;
fn u8_as_u32(&mut self, arg0: u8) -> Option<u32>;
fn u8_as_u64(&mut self, arg0: u8) -> Option<u64>;
fn u16_as_u64(&mut self, arg0: u16) -> Option<u64>;
fn u32_as_u64(&mut self, arg0: u32) -> Option<u64>;
fn i64_as_u64(&mut self, arg0: i64) -> Option<u64>;
fn u64_add(&mut self, arg0: u64, arg1: u64) -> Option<u64>;
fn u64_sub(&mut self, arg0: u64, arg1: u64) -> Option<u64>;
fn u64_and(&mut self, arg0: u64, arg1: u64) -> Option<u64>;
fn ty_bits(&mut self, arg0: Type) -> Option<u8>;
fn ty_bits_u16(&mut self, arg0: Type) -> u16;
fn ty_bits_u64(&mut self, arg0: Type) -> u64;
fn ty_mask(&mut self, arg0: Type) -> u64;
@@ -137,14 +146,14 @@ pub trait Context {
fn rotr_opposite_amount(&mut self, arg0: Type, arg1: ImmShift) -> ImmShift;
}
/// Internal type SideEffectNoResult: defined at src/prelude.isle line 412.
/// Internal type SideEffectNoResult: defined at src/prelude.isle line 447.
#[derive(Clone, Debug)]
pub enum SideEffectNoResult {
Inst { inst: MInst },
Inst2 { inst1: MInst, inst2: MInst },
}
/// Internal type ProducesFlags: defined at src/prelude.isle line 439.
/// Internal type ProducesFlags: defined at src/prelude.isle line 474.
#[derive(Clone, Debug)]
pub enum ProducesFlags {
ProducesFlagsSideEffect { inst: MInst },
@@ -152,7 +161,7 @@ pub enum ProducesFlags {
ProducesFlagsReturnsResultWithConsumer { inst: MInst, result: Reg },
}
/// Internal type ConsumesFlags: defined at src/prelude.isle line 450.
/// Internal type ConsumesFlags: defined at src/prelude.isle line 485.
#[derive(Clone, Debug)]
pub enum ConsumesFlags {
ConsumesFlagsReturnsResultWithProducer {
@@ -1049,7 +1058,7 @@ pub enum AtomicRMWLoopOp {
// Generated as internal constructor for term output_reg.
pub fn constructor_output_reg<C: Context>(ctx: &mut C, arg0: Reg) -> Option<InstOutput> {
let pattern0_0 = arg0;
// Rule at src/prelude.isle line 86.
// Rule at src/prelude.isle line 113.
let expr0_0 = C::value_reg(ctx, pattern0_0);
let expr1_0 = C::output(ctx, expr0_0);
return Some(expr1_0);
@@ -1058,7 +1067,7 @@ pub fn constructor_output_reg<C: Context>(ctx: &mut C, arg0: Reg) -> Option<Inst
// Generated as internal constructor for term output_value.
pub fn constructor_output_value<C: Context>(ctx: &mut C, arg0: Value) -> Option<InstOutput> {
let pattern0_0 = arg0;
// Rule at src/prelude.isle line 90.
// Rule at src/prelude.isle line 117.
let expr0_0 = C::put_in_regs(ctx, pattern0_0);
let expr1_0 = C::output(ctx, expr0_0);
return Some(expr1_0);
@@ -1067,7 +1076,7 @@ pub fn constructor_output_value<C: Context>(ctx: &mut C, arg0: Value) -> Option<
// Generated as internal constructor for term temp_reg.
pub fn constructor_temp_reg<C: Context>(ctx: &mut C, arg0: Type) -> Option<Reg> {
let pattern0_0 = arg0;
// Rule at src/prelude.isle line 110.
// Rule at src/prelude.isle line 137.
let expr0_0 = C::temp_writable_reg(ctx, pattern0_0);
let expr1_0 = C::writable_reg_to_reg(ctx, expr0_0);
return Some(expr1_0);
@@ -1076,7 +1085,7 @@ pub fn constructor_temp_reg<C: Context>(ctx: &mut C, arg0: Type) -> Option<Reg>
// Generated as internal constructor for term lo_reg.
pub fn constructor_lo_reg<C: Context>(ctx: &mut C, arg0: Value) -> Option<Reg> {
let pattern0_0 = arg0;
// Rule at src/prelude.isle line 150.
// Rule at src/prelude.isle line 182.
let expr0_0 = C::put_in_regs(ctx, pattern0_0);
let expr1_0: usize = 0;
let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0);
@@ -1093,7 +1102,7 @@ pub fn constructor_side_effect<C: Context>(
&SideEffectNoResult::Inst {
inst: ref pattern1_0,
} => {
// Rule at src/prelude.isle line 420.
// Rule at src/prelude.isle line 455.
let expr0_0 = C::emit(ctx, pattern1_0);
let expr1_0 = C::output_none(ctx);
return Some(expr1_0);
@@ -1102,7 +1111,7 @@ pub fn constructor_side_effect<C: Context>(
inst1: ref pattern1_0,
inst2: ref pattern1_1,
} => {
// Rule at src/prelude.isle line 423.
// Rule at src/prelude.isle line 458.
let expr0_0 = C::emit(ctx, pattern1_0);
let expr1_0 = C::emit(ctx, pattern1_1);
let expr2_0 = C::output_none(ctx);
@@ -1129,7 +1138,7 @@ pub fn constructor_side_effect_concat<C: Context>(
inst: ref pattern3_0,
} = pattern2_0
{
// Rule at src/prelude.isle line 429.
// Rule at src/prelude.isle line 464.
let expr0_0 = SideEffectNoResult::Inst2 {
inst1: pattern1_0.clone(),
inst2: pattern3_0.clone(),
@@ -1151,7 +1160,7 @@ pub fn constructor_produces_flags_get_reg<C: Context>(
result: pattern1_1,
} = pattern0_0
{
// Rule at src/prelude.isle line 466.
// Rule at src/prelude.isle line 501.
return Some(pattern1_1);
}
return None;
@@ -1168,7 +1177,7 @@ pub fn constructor_produces_flags_ignore<C: Context>(
inst: ref pattern1_0,
result: pattern1_1,
} => {
// Rule at src/prelude.isle line 471.
// Rule at src/prelude.isle line 506.
let expr0_0 = ProducesFlags::ProducesFlagsSideEffect {
inst: pattern1_0.clone(),
};
@@ -1178,7 +1187,7 @@ pub fn constructor_produces_flags_ignore<C: Context>(
inst: ref pattern1_0,
result: pattern1_1,
} => {
// Rule at src/prelude.isle line 473.
// Rule at src/prelude.isle line 508.
let expr0_0 = ProducesFlags::ProducesFlagsSideEffect {
inst: pattern1_0.clone(),
};
@@ -1207,7 +1216,7 @@ pub fn constructor_consumes_flags_concat<C: Context>(
result: pattern3_1,
} = pattern2_0
{
// Rule at src/prelude.isle line 480.
// Rule at src/prelude.isle line 515.
let expr0_0 = C::value_regs(ctx, pattern1_1, pattern3_1);
let expr1_0 = ConsumesFlags::ConsumesFlagsTwiceReturnsValueRegs {
inst1: pattern1_0.clone(),
@@ -1237,7 +1246,7 @@ pub fn constructor_with_flags<C: Context>(
inst: ref pattern3_0,
result: pattern3_1,
} => {
// Rule at src/prelude.isle line 505.
// Rule at src/prelude.isle line 540.
let expr0_0 = C::emit(ctx, pattern1_0);
let expr1_0 = C::emit(ctx, pattern3_0);
let expr2_0 = C::value_reg(ctx, pattern3_1);
@@ -1248,7 +1257,7 @@ pub fn constructor_with_flags<C: Context>(
inst2: ref pattern3_1,
result: pattern3_2,
} => {
// Rule at src/prelude.isle line 511.
// Rule at src/prelude.isle line 546.
let expr0_0 = C::emit(ctx, pattern1_0);
let expr1_0 = C::emit(ctx, pattern3_0);
let expr2_0 = C::emit(ctx, pattern3_1);
@@ -1261,7 +1270,7 @@ pub fn constructor_with_flags<C: Context>(
inst4: ref pattern3_3,
result: pattern3_4,
} => {
// Rule at src/prelude.isle line 523.
// Rule at src/prelude.isle line 558.
let expr0_0 = C::emit(ctx, pattern1_0);
let expr1_0 = C::emit(ctx, pattern3_0);
let expr2_0 = C::emit(ctx, pattern3_1);
@@ -1282,7 +1291,7 @@ pub fn constructor_with_flags<C: Context>(
result: pattern3_1,
} = pattern2_0
{
// Rule at src/prelude.isle line 499.
// Rule at src/prelude.isle line 534.
let expr0_0 = C::emit(ctx, pattern1_0);
let expr1_0 = C::emit(ctx, pattern3_0);
let expr2_0 = C::value_regs(ctx, pattern1_1, pattern3_1);
@@ -1302,7 +1311,7 @@ pub fn constructor_with_flags_reg<C: Context>(
) -> Option<Reg> {
let pattern0_0 = arg0;
let pattern1_0 = arg1;
// Rule at src/prelude.isle line 540.
// Rule at src/prelude.isle line 575.
let expr0_0 = constructor_with_flags(ctx, pattern0_0, pattern1_0)?;
let expr1_0: usize = 0;
let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0);
@@ -3154,7 +3163,7 @@ pub fn constructor_put_in_reg_sext32<C: Context>(ctx: &mut C, arg0: Value) -> Op
// Rule at src/isa/aarch64/inst.isle line 1895.
let expr0_0 = C::put_in_reg(ctx, pattern0_0);
let expr1_0: bool = true;
let expr2_0 = C::ty_bits(ctx, pattern2_0);
let expr2_0 = C::ty_bits(ctx, pattern2_0)?;
let expr3_0: u8 = 32;
let expr4_0 = constructor_extend(ctx, expr0_0, expr1_0, expr2_0, expr3_0)?;
return Some(expr4_0);
@@ -3180,7 +3189,7 @@ pub fn constructor_put_in_reg_zext32<C: Context>(ctx: &mut C, arg0: Value) -> Op
// Rule at src/isa/aarch64/inst.isle line 1904.
let expr0_0 = C::put_in_reg(ctx, pattern0_0);
let expr1_0: bool = false;
let expr2_0 = C::ty_bits(ctx, pattern2_0);
let expr2_0 = C::ty_bits(ctx, pattern2_0)?;
let expr3_0: u8 = 32;
let expr4_0 = constructor_extend(ctx, expr0_0, expr1_0, expr2_0, expr3_0)?;
return Some(expr4_0);
@@ -3201,7 +3210,7 @@ pub fn constructor_put_in_reg_sext64<C: Context>(ctx: &mut C, arg0: Value) -> Op
// Rule at src/isa/aarch64/inst.isle line 1913.
let expr0_0 = C::put_in_reg(ctx, pattern0_0);
let expr1_0: bool = true;
let expr2_0 = C::ty_bits(ctx, pattern2_0);
let expr2_0 = C::ty_bits(ctx, pattern2_0)?;
let expr3_0: u8 = 64;
let expr4_0 = constructor_extend(ctx, expr0_0, expr1_0, expr2_0, expr3_0)?;
return Some(expr4_0);
@@ -3222,7 +3231,7 @@ pub fn constructor_put_in_reg_zext64<C: Context>(ctx: &mut C, arg0: Value) -> Op
// Rule at src/isa/aarch64/inst.isle line 1921.
let expr0_0 = C::put_in_reg(ctx, pattern0_0);
let expr1_0: bool = false;
let expr2_0 = C::ty_bits(ctx, pattern2_0);
let expr2_0 = C::ty_bits(ctx, pattern2_0)?;
let expr3_0: u8 = 64;
let expr4_0 = constructor_extend(ctx, expr0_0, expr1_0, expr2_0, expr3_0)?;
return Some(expr4_0);
@@ -5936,7 +5945,7 @@ pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<InstOutp
let expr3_0 = C::zero_reg(ctx);
let expr4_0 = constructor_madd(ctx, expr2_0, expr0_0, expr1_0, expr3_0)?;
let expr5_0: Type = I64;
let expr6_0 = C::ty_bits(ctx, pattern3_0);
let expr6_0 = C::ty_bits(ctx, pattern3_0)?;
let expr7_0 = C::imm_shift_from_u8(ctx, expr6_0);
let expr8_0 = constructor_lsr_imm(ctx, expr5_0, expr4_0, expr7_0)?;
let expr9_0 = C::value_reg(ctx, expr8_0);
@@ -5952,7 +5961,7 @@ pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<InstOutp
let expr3_0 = C::zero_reg(ctx);
let expr4_0 = constructor_madd(ctx, expr2_0, expr0_0, expr1_0, expr3_0)?;
let expr5_0: Type = I64;
let expr6_0 = C::ty_bits(ctx, pattern3_0);
let expr6_0 = C::ty_bits(ctx, pattern3_0)?;
let expr7_0 = C::imm_shift_from_u8(ctx, expr6_0);
let expr8_0 = constructor_asr_imm(ctx, expr5_0, expr4_0, expr7_0)?;
let expr9_0 = constructor_output_reg(ctx, expr8_0)?;
@@ -6628,8 +6637,8 @@ pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<InstOutp
// Rule at src/isa/aarch64/lower.isle line 485.
let expr0_0 = C::put_in_reg(ctx, pattern5_1);
let expr1_0: bool = false;
let expr2_0 = C::ty_bits(ctx, pattern7_0);
let expr3_0 = C::ty_bits(ctx, pattern3_0);
let expr2_0 = C::ty_bits(ctx, pattern7_0)?;
let expr3_0 = C::ty_bits(ctx, pattern3_0)?;
let expr4_0 =
constructor_extend(ctx, expr0_0, expr1_0, expr2_0, expr3_0)?;
let expr5_0 = constructor_output_reg(ctx, expr4_0)?;
@@ -6667,8 +6676,8 @@ pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<InstOutp
// Rule at src/isa/aarch64/lower.isle line 517.
let expr0_0 = C::put_in_reg(ctx, pattern5_1);
let expr1_0: bool = true;
let expr2_0 = C::ty_bits(ctx, pattern7_0);
let expr3_0 = C::ty_bits(ctx, pattern3_0);
let expr2_0 = C::ty_bits(ctx, pattern7_0)?;
let expr3_0 = C::ty_bits(ctx, pattern3_0)?;
let expr4_0 =
constructor_extend(ctx, expr0_0, expr1_0, expr2_0, expr3_0)?;
let expr5_0 = constructor_output_reg(ctx, expr4_0)?;
@@ -7250,7 +7259,7 @@ pub fn constructor_small_rotr<C: Context>(
let expr1_0 = C::rotr_mask(ctx, pattern0_0);
let expr2_0 = constructor_and_imm(ctx, expr0_0, pattern2_0, expr1_0)?;
let expr3_0: Type = I32;
let expr4_0 = C::ty_bits(ctx, pattern0_0);
let expr4_0 = C::ty_bits(ctx, pattern0_0)?;
let expr5_0 = C::u8_into_imm12(ctx, expr4_0);
let expr6_0 = constructor_sub_imm(ctx, expr3_0, expr2_0, expr5_0)?;
let expr7_0: Type = I32;

7
cranelift/codegen/src/isa/s390x/lower/isle.rs
View File

@@ -248,7 +248,10 @@ where
let inst = self.lower_ctx.dfg().value_def(val).inst()?;
let constant = self.lower_ctx.get_constant(inst)?;
let ty = self.lower_ctx.output_ty(inst, 0);
Some(super::sign_extend_to_u64(constant, self.ty_bits(ty)))
Some(super::sign_extend_to_u64(
constant,
self.ty_bits(ty).unwrap(),
))
}
#[inline]
@@ -327,7 +330,7 @@ where
#[inline]
fn mask_amt_imm(&mut self, ty: Type, amt: i64) -> u8 {
let mask = self.ty_bits(ty) - 1;
let mask = self.ty_bits(ty).unwrap() - 1;
(amt as u8) & mask
}

2
cranelift/codegen/src/isa/s390x/lower/isle/generated_code.manifest
View File

@@ -1,4 +1,4 @@
src/clif.isle 443b34b797fc8ace
src/prelude.isle a7915a6b88310eb5
src/prelude.isle 97c4b6eebbab9f05
src/isa/s390x/inst.isle 36c2500563cdd4e6
src/isa/s390x/lower.isle e5c946ab8a265b77

75
cranelift/codegen/src/isa/s390x/lower/isle/generated_code.rs generated
View File

@@ -21,6 +21,12 @@ pub trait Context {
fn unpack_value_array_3(&mut self, arg0: &ValueArray3) -> (Value, Value, Value);
fn pack_value_array_3(&mut self, arg0: Value, arg1: Value, arg2: Value) -> ValueArray3;
fn u32_add(&mut self, arg0: u32, arg1: u32) -> u32;
fn s32_add_fallible(&mut self, arg0: u32, arg1: u32) -> Option<u32>;
fn u32_nonnegative(&mut self, arg0: u32) -> Option<u32>;
fn offset32(&mut self, arg0: Offset32) -> Option<u32>;
fn u32_lteq(&mut self, arg0: u32, arg1: u32) -> Option<Unit>;
fn simm32(&mut self, arg0: Imm64) -> Option<u32>;
fn uimm8(&mut self, arg0: Imm64) -> Option<u8>;
fn u8_and(&mut self, arg0: u8, arg1: u8) -> u8;
fn value_reg(&mut self, arg0: Reg) -> ValueRegs;
fn value_regs(&mut self, arg0: Reg, arg1: Reg) -> ValueRegs;
@@ -32,19 +38,22 @@ pub trait Context {
fn output_builder_push(&mut self, arg0: &InstOutputBuilder, arg1: ValueRegs) -> Unit;
fn output_builder_finish(&mut self, arg0: &InstOutputBuilder) -> InstOutput;
fn temp_writable_reg(&mut self, arg0: Type) -> WritableReg;
fn invalid_reg_etor(&mut self, arg0: Reg) -> Option<()>;
fn invalid_reg(&mut self) -> Reg;
fn valid_reg(&mut self, arg0: Reg) -> Option<()>;
fn put_in_reg(&mut self, arg0: Value) -> Reg;
fn put_in_regs(&mut self, arg0: Value) -> ValueRegs;
fn ensure_in_vreg(&mut self, arg0: Reg, arg1: Type) -> Reg;
fn value_regs_get(&mut self, arg0: ValueRegs, arg1: usize) -> Reg;
fn u8_as_u64(&mut self, arg0: u8) -> u64;
fn u16_as_u64(&mut self, arg0: u16) -> u64;
fn u32_as_u64(&mut self, arg0: u32) -> u64;
fn i64_as_u64(&mut self, arg0: i64) -> u64;
fn u64_add(&mut self, arg0: u64, arg1: u64) -> u64;
fn u64_sub(&mut self, arg0: u64, arg1: u64) -> u64;
fn u64_and(&mut self, arg0: u64, arg1: u64) -> u64;
fn ty_bits(&mut self, arg0: Type) -> u8;
fn u8_as_u32(&mut self, arg0: u8) -> Option<u32>;
fn u8_as_u64(&mut self, arg0: u8) -> Option<u64>;
fn u16_as_u64(&mut self, arg0: u16) -> Option<u64>;
fn u32_as_u64(&mut self, arg0: u32) -> Option<u64>;
fn i64_as_u64(&mut self, arg0: i64) -> Option<u64>;
fn u64_add(&mut self, arg0: u64, arg1: u64) -> Option<u64>;
fn u64_sub(&mut self, arg0: u64, arg1: u64) -> Option<u64>;
fn u64_and(&mut self, arg0: u64, arg1: u64) -> Option<u64>;
fn ty_bits(&mut self, arg0: Type) -> Option<u8>;
fn ty_bits_u16(&mut self, arg0: Type) -> u16;
fn ty_bits_u64(&mut self, arg0: Type) -> u64;
fn ty_mask(&mut self, arg0: Type) -> u64;
@@ -156,14 +165,14 @@ pub trait Context {
fn same_reg(&mut self, arg0: WritableReg, arg1: Reg) -> Option<Reg>;
}
/// Internal type SideEffectNoResult: defined at src/prelude.isle line 412.
/// Internal type SideEffectNoResult: defined at src/prelude.isle line 447.
#[derive(Clone, Debug)]
pub enum SideEffectNoResult {
Inst { inst: MInst },
Inst2 { inst1: MInst, inst2: MInst },
}
/// Internal type ProducesFlags: defined at src/prelude.isle line 439.
/// Internal type ProducesFlags: defined at src/prelude.isle line 474.
#[derive(Clone, Debug)]
pub enum ProducesFlags {
ProducesFlagsSideEffect { inst: MInst },
@@ -171,7 +180,7 @@ pub enum ProducesFlags {
ProducesFlagsReturnsResultWithConsumer { inst: MInst, result: Reg },
}
/// Internal type ConsumesFlags: defined at src/prelude.isle line 450.
/// Internal type ConsumesFlags: defined at src/prelude.isle line 485.
#[derive(Clone, Debug)]
pub enum ConsumesFlags {
ConsumesFlagsReturnsResultWithProducer {
@@ -917,7 +926,7 @@ pub enum ProducesBool {
// Generated as internal constructor for term output_reg.
pub fn constructor_output_reg<C: Context>(ctx: &mut C, arg0: Reg) -> Option<InstOutput> {
let pattern0_0 = arg0;
// Rule at src/prelude.isle line 86.
// Rule at src/prelude.isle line 113.
let expr0_0 = C::value_reg(ctx, pattern0_0);
let expr1_0 = C::output(ctx, expr0_0);
return Some(expr1_0);
@@ -926,7 +935,7 @@ pub fn constructor_output_reg<C: Context>(ctx: &mut C, arg0: Reg) -> Option<Inst
// Generated as internal constructor for term output_value.
pub fn constructor_output_value<C: Context>(ctx: &mut C, arg0: Value) -> Option<InstOutput> {
let pattern0_0 = arg0;
// Rule at src/prelude.isle line 90.
// Rule at src/prelude.isle line 117.
let expr0_0 = C::put_in_regs(ctx, pattern0_0);
let expr1_0 = C::output(ctx, expr0_0);
return Some(expr1_0);
@@ -935,7 +944,7 @@ pub fn constructor_output_value<C: Context>(ctx: &mut C, arg0: Value) -> Option<
// Generated as internal constructor for term temp_reg.
pub fn constructor_temp_reg<C: Context>(ctx: &mut C, arg0: Type) -> Option<Reg> {
let pattern0_0 = arg0;
// Rule at src/prelude.isle line 110.
// Rule at src/prelude.isle line 137.
let expr0_0 = C::temp_writable_reg(ctx, pattern0_0);
let expr1_0 = C::writable_reg_to_reg(ctx, expr0_0);
return Some(expr1_0);
@@ -944,7 +953,7 @@ pub fn constructor_temp_reg<C: Context>(ctx: &mut C, arg0: Type) -> Option<Reg>
// Generated as internal constructor for term lo_reg.
pub fn constructor_lo_reg<C: Context>(ctx: &mut C, arg0: Value) -> Option<Reg> {
let pattern0_0 = arg0;
// Rule at src/prelude.isle line 150.
// Rule at src/prelude.isle line 182.
let expr0_0 = C::put_in_regs(ctx, pattern0_0);
let expr1_0: usize = 0;
let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0);
@@ -961,7 +970,7 @@ pub fn constructor_side_effect<C: Context>(
&SideEffectNoResult::Inst {
inst: ref pattern1_0,
} => {
// Rule at src/prelude.isle line 420.
// Rule at src/prelude.isle line 455.
let expr0_0 = C::emit(ctx, pattern1_0);
let expr1_0 = C::output_none(ctx);
return Some(expr1_0);
@@ -970,7 +979,7 @@ pub fn constructor_side_effect<C: Context>(
inst1: ref pattern1_0,
inst2: ref pattern1_1,
} => {
// Rule at src/prelude.isle line 423.
// Rule at src/prelude.isle line 458.
let expr0_0 = C::emit(ctx, pattern1_0);
let expr1_0 = C::emit(ctx, pattern1_1);
let expr2_0 = C::output_none(ctx);
@@ -997,7 +1006,7 @@ pub fn constructor_side_effect_concat<C: Context>(
inst: ref pattern3_0,
} = pattern2_0
{
// Rule at src/prelude.isle line 429.
// Rule at src/prelude.isle line 464.
let expr0_0 = SideEffectNoResult::Inst2 {
inst1: pattern1_0.clone(),
inst2: pattern3_0.clone(),
@@ -1019,7 +1028,7 @@ pub fn constructor_produces_flags_get_reg<C: Context>(
result: pattern1_1,
} = pattern0_0
{
// Rule at src/prelude.isle line 466.
// Rule at src/prelude.isle line 501.
return Some(pattern1_1);
}
return None;
@@ -1036,7 +1045,7 @@ pub fn constructor_produces_flags_ignore<C: Context>(
inst: ref pattern1_0,
result: pattern1_1,
} => {
// Rule at src/prelude.isle line 471.
// Rule at src/prelude.isle line 506.
let expr0_0 = ProducesFlags::ProducesFlagsSideEffect {
inst: pattern1_0.clone(),
};
@@ -1046,7 +1055,7 @@ pub fn constructor_produces_flags_ignore<C: Context>(
inst: ref pattern1_0,
result: pattern1_1,
} => {
// Rule at src/prelude.isle line 473.
// Rule at src/prelude.isle line 508.
let expr0_0 = ProducesFlags::ProducesFlagsSideEffect {
inst: pattern1_0.clone(),
};
@@ -1075,7 +1084,7 @@ pub fn constructor_consumes_flags_concat<C: Context>(
result: pattern3_1,
} = pattern2_0
{
// Rule at src/prelude.isle line 480.
// Rule at src/prelude.isle line 515.
let expr0_0 = C::value_regs(ctx, pattern1_1, pattern3_1);
let expr1_0 = ConsumesFlags::ConsumesFlagsTwiceReturnsValueRegs {
inst1: pattern1_0.clone(),
@@ -1105,7 +1114,7 @@ pub fn constructor_with_flags<C: Context>(
inst: ref pattern3_0,
result: pattern3_1,
} => {
// Rule at src/prelude.isle line 505.
// Rule at src/prelude.isle line 540.
let expr0_0 = C::emit(ctx, pattern1_0);
let expr1_0 = C::emit(ctx, pattern3_0);
let expr2_0 = C::value_reg(ctx, pattern3_1);
@@ -1116,7 +1125,7 @@ pub fn constructor_with_flags<C: Context>(
inst2: ref pattern3_1,
result: pattern3_2,
} => {
// Rule at src/prelude.isle line 511.
// Rule at src/prelude.isle line 546.
let expr0_0 = C::emit(ctx, pattern1_0);
let expr1_0 = C::emit(ctx, pattern3_0);
let expr2_0 = C::emit(ctx, pattern3_1);
@@ -1129,7 +1138,7 @@ pub fn constructor_with_flags<C: Context>(
inst4: ref pattern3_3,
result: pattern3_4,
} => {
// Rule at src/prelude.isle line 523.
// Rule at src/prelude.isle line 558.
let expr0_0 = C::emit(ctx, pattern1_0);
let expr1_0 = C::emit(ctx, pattern3_0);
let expr2_0 = C::emit(ctx, pattern3_1);
@@ -1150,7 +1159,7 @@ pub fn constructor_with_flags<C: Context>(
result: pattern3_1,
} = pattern2_0
{
// Rule at src/prelude.isle line 499.
// Rule at src/prelude.isle line 534.
let expr0_0 = C::emit(ctx, pattern1_0);
let expr1_0 = C::emit(ctx, pattern3_0);
let expr2_0 = C::value_regs(ctx, pattern1_1, pattern3_1);
@@ -1170,7 +1179,7 @@ pub fn constructor_with_flags_reg<C: Context>(
) -> Option<Reg> {
let pattern0_0 = arg0;
let pattern1_0 = arg1;
// Rule at src/prelude.isle line 540.
// Rule at src/prelude.isle line 575.
let expr0_0 = constructor_with_flags(ctx, pattern0_0, pattern1_0)?;
let expr1_0: usize = 0;
let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0);
@@ -3610,7 +3619,7 @@ pub fn constructor_emit_zext32_reg<C: Context>(
let pattern2_0 = arg2;
// Rule at src/isa/s390x/inst.isle line 1990.
let expr0_0: bool = false;
let expr1_0 = C::ty_bits(ctx, pattern1_0);
let expr1_0 = C::ty_bits(ctx, pattern1_0)?;
let expr2_0: u8 = 32;
let expr3_0 = MInst::Extend {
rd: pattern0_0,
@@ -3635,7 +3644,7 @@ pub fn constructor_emit_sext32_reg<C: Context>(
let pattern2_0 = arg2;
// Rule at src/isa/s390x/inst.isle line 1996.
let expr0_0: bool = true;
let expr1_0 = C::ty_bits(ctx, pattern1_0);
let expr1_0 = C::ty_bits(ctx, pattern1_0)?;
let expr2_0: u8 = 32;
let expr3_0 = MInst::Extend {
rd: pattern0_0,
@@ -3660,7 +3669,7 @@ pub fn constructor_emit_zext64_reg<C: Context>(
let pattern2_0 = arg2;
// Rule at src/isa/s390x/inst.isle line 2002.
let expr0_0: bool = false;
let expr1_0 = C::ty_bits(ctx, pattern1_0);
let expr1_0 = C::ty_bits(ctx, pattern1_0)?;
let expr2_0: u8 = 64;
let expr3_0 = MInst::Extend {
rd: pattern0_0,
@@ -3685,7 +3694,7 @@ pub fn constructor_emit_sext64_reg<C: Context>(
let pattern2_0 = arg2;
// Rule at src/isa/s390x/inst.isle line 2008.
let expr0_0: bool = true;
let expr1_0 = C::ty_bits(ctx, pattern1_0);
let expr1_0 = C::ty_bits(ctx, pattern1_0)?;
let expr2_0: u8 = 64;
let expr3_0 = MInst::Extend {
rd: pattern0_0,
@@ -11522,7 +11531,7 @@ pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<InstOutp
let expr2_0: Type = I32;
let expr3_0 = constructor_mul_reg(ctx, expr2_0, expr0_0, expr1_0)?;
let expr4_0: Type = I32;
let expr5_0 = C::ty_bits(ctx, pattern3_0);
let expr5_0 = C::ty_bits(ctx, pattern3_0)?;
let expr6_0 = constructor_lshr_imm(ctx, expr4_0, expr3_0, expr5_0)?;
let expr7_0 = constructor_output_reg(ctx, expr6_0)?;
return Some(expr7_0);
@@ -11535,7 +11544,7 @@ pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<InstOutp
let expr2_0: Type = I32;
let expr3_0 = constructor_mul_reg(ctx, expr2_0, expr0_0, expr1_0)?;
let expr4_0: Type = I32;
let expr5_0 = C::ty_bits(ctx, pattern3_0);
let expr5_0 = C::ty_bits(ctx, pattern3_0)?;
let expr6_0 = constructor_ashr_imm(ctx, expr4_0, expr3_0, expr5_0)?;
let expr7_0 = constructor_output_reg(ctx, expr6_0)?;
return Some(expr7_0);

243
cranelift/codegen/src/isa/x64/inst.isle
View File

@@ -765,7 +765,28 @@
(decl amode_to_synthetic_amode (Amode) SyntheticAmode)
(extern constructor amode_to_synthetic_amode amode_to_synthetic_amode)
(type Amode extern (enum))
;; An `Amode` represents a possible addressing mode that can be used
;; in instructions. These denote a 64-bit value only.
(type Amode (enum
;; Immediate sign-extended and a register
(ImmReg (simm32 u32)
(base Reg)
(flags MemFlags))
;; Sign-extend-32-to-64(simm32) + base + (index << shift)
(ImmRegRegShift (simm32 u32)
(base Gpr)
(index Gpr)
(shift u8)
(flags MemFlags))
;; Sign-extend-32-to-64(immediate) + RIP (instruction
;; pointer). The appropriate relocation is emitted so
;; that the resulting immediate makes this Amode refer to
;; the given MachLabel.
(RipRelative (target MachLabel))))
;; Some Amode constructor helpers.
(decl amode_with_flags (Amode MemFlags) Amode)
(extern constructor amode_with_flags amode_with_flags)
@@ -784,69 +805,185 @@
(rule (amode_imm_reg_reg_shift_flags offset base index shift flags)
(amode_with_flags (amode_imm_reg_reg_shift offset base index shift) flags))
;; A helper to check if a shift amount (the `Value`) is both constant and
;; less-than or equal to 3; this is needed since x64 can only shift addresses
;; using two bits.
(decl const_shift_lt_eq_3 (u8) Value)
(extern extractor const_shift_lt_eq_3 const_shift_lt_eq_3 )
;; A helper to both check that the `Imm64` and `Offset32` values sum to less
;; than 32-bits AND return this summed `u32` value. Also, the `Imm64` will be
;; zero-extended from `Type` up to 64 bits. This is useful for `to_amode`.
(decl pure sum_extend_fits_in_32_bits (Type Imm64 Offset32) u32)
(extern constructor sum_extend_fits_in_32_bits sum_extend_fits_in_32_bits)
;; To generate an address for a memory access, we can pattern-match various CLIF
;; sub-trees to x64's complex addressing modes (`Amode`). In pseudo-code:
;;;; Amode lowering ;;;;
;; To generate an address for a memory access, we can pattern-match
;; various CLIF sub-trees to x64's complex addressing modes (`Amode`).
;;
;; if address matches iadd(a, b):
;; if either a or b:
;; matches (ishl c with shift amount <= 3):
;; amode(base + offset + (c << amount))
;; matches (iconst c where c + offset will fit in 32 bits):
;; amode(base + eval(c + offset))
;; matches (uextend (iconst c) where c + offset will fit in 32 bits):
;; amode(base + eval(c + offset))
;; else:
;; amode(a + offset + (b << 0))
;; else:
;; amode(base + offset)
;; Information about available addressing modes is available in
;; Intel's Software Developer's Manual, volume 2, section 2.1.5,
;; "Addressing-Mode Encoding of ModR/M and SIB Bytes."
;;
;; The rules for `to_amode` correspond to a subset of the possible addressing
;; modes available by tweaking the SIB byte, the MOD bits, and the size of the
;; displacement (i.e., offset). More information is available in Intel's
;; Software Developer's Manual, volume 2, section 2.1.5, "Addressing-Mode
;; Encoding of ModR/M and SIB Bytes."
;; The general strategy to build an `Amode` is to traverse over the
;; input expression's addends, recursively deconstructing a tree of
;; `iadd` operators that add up parts of the address, updating the
;; `Amode` in an incremental fashion as we add in each piece.
;;
;; We start with an "immediate + register" form that encapsulates the
;; load/store's built-in `Offset32` and `invalid_reg` as the
;; register. This is given by `amode_initial`. Then we add `Value`s
;; one at a time with `amode_add`. (Why start with `invalid_reg` at
;; all? Because we don't want to special-case the first input and
;; duplicate rules; this lets us use the "add a value" logic even for
;; the first value.)
;;
;; It is always valid to use `amode_add` to add the one single
;; `address` input to the load/store (i.e., the `Value` given to
;; `to_amode`). In the fallback case, this is what we do. Then we get
;; an `Amode.ImmReg` with the `Offset32` and `Value` below and nothing
;; else; this always works and is not *that* bad.
;;
;; But we can often do better. The toplevel rule for `iadd` below will
;; turn an `(amode_add amode (iadd a b))` into two invocations of
;; `amode_add`, for each operand of the `iadd`. This is what allows us
;; to handle sums of many parts.
;;
;; Then we "just" need to work out how we can incorporate a new
;; component into an existing addressing mode:
;;
;; - Case 1: When we have an `ImmReg` and the register is
;; `invalid_reg` (the initial `Amode` above), we can put the new
;; addend into a register and insert it into the `ImmReg`.
;;
;; - Case 2: When we have an `ImmReg` with a valid register already,
;; and we have another register to add, we can transition to an
;; `ImmRegRegShift`.
;;
;; - Case 3: When we're adding an `ishl`, we can refine the above rule
;; and use the built-in multiplier of 1, 2, 4, 8 to implement a
;; left-shift by 0, 1, 2, 3.
;;
;; - Case 4: When we are adding another constant offset, we can fold
;; it into the existing offset, as long as the sum still fits into
;; the signed 32-bit field.
;;
;; - Case 5: And as a general fallback, we can generate a new `add`
;; instruction and add the new addend to an existing component of
;; the `Amode`.
(decl to_amode (MemFlags Value Offset32) Amode)
;; ...matches (ishl c ...)
(rule (to_amode flags (iadd (ishl src (const_shift_lt_eq_3 amt)) base) offset)
(amode_imm_reg_reg_shift_flags offset (put_in_gpr base) (put_in_gpr src) amt flags))
(rule (to_amode flags (iadd base (ishl src (const_shift_lt_eq_3 amt))) offset)
(amode_imm_reg_reg_shift_flags offset (put_in_gpr base) (put_in_gpr src) amt flags))
;; ...matches (iconst c ...); note how this matching pattern uses an in-out
;; extractor to check that the offset and constant value (`c`, the in
;; parameter), when summed will fit into x64's 32-bit displacement, returned as
;; `sum` (the out parameter). The syntax for this could be improved (TODO).
(rule (to_amode flags (iadd (iconst c) base) offset)
(if-let sum (sum_extend_fits_in_32_bits $I64 c offset))
(amode_imm_reg_flags sum (put_in_gpr base) flags))
(rule (to_amode flags (iadd base (iconst c)) offset)
(if-let sum (sum_extend_fits_in_32_bits $I64 c offset))
(amode_imm_reg_flags sum (put_in_gpr base) flags))
;; ...matches (uextend(iconst c) ...); see notes above.
(rule (to_amode flags (iadd (has_type ty (uextend (iconst c))) base) offset)
(if-let sum (sum_extend_fits_in_32_bits $I64 c offset))
(amode_imm_reg_flags sum (put_in_gpr base) flags))
(rule (to_amode flags (iadd base (has_type ty (uextend (iconst c)))) offset)
(if-let sum (sum_extend_fits_in_32_bits $I64 c offset))
(amode_imm_reg_flags sum (put_in_gpr base) flags))
;; ...else only matches (iadd(a b))
(rule (to_amode flags (iadd base index) offset)
(amode_imm_reg_reg_shift_flags offset (put_in_gpr base) (put_in_gpr index) 0 flags))
;; ...else
;; Initial step in amode processing: create an ImmReg with
;; (invalid_reg) and encapsulating the flags and offset from the
;; load/store.
(decl amode_initial (MemFlags Offset32) Amode)
(rule (amode_initial flags (offset32 off))
(Amode.ImmReg off (invalid_reg) flags))
;; One step in amode processing: take an existing amode and add
;; another value to it.
(decl amode_add (Amode Value) Amode)
;; -- Top-level driver: pull apart the addends.
;;
;; Any amode can absorb an `iadd` by absorbing first the LHS of the
;; add, then the RHS.
;;
;; Priority 2 to take this above fallbacks and ensure we traverse the
;; `iadd` tree fully.
(rule 2 (amode_add amode (iadd x y))
(let ((amode1 Amode (amode_add amode x))
(amode2 Amode (amode_add amode1 y)))
amode2))
;; -- Case 1 (adding a register to the initial Amode with invalid_reg).
;;
;; An Amode.ImmReg with invalid_reg (initial state) can absorb a
;; register as the base register.
(rule (amode_add (Amode.ImmReg off (invalid_reg) flags) value)
(Amode.ImmReg off value flags))
;; -- Case 2 (adding a register to an Amode with a register already).
;;
;; An Amode.ImmReg can absorb another register as the index register.
(rule (amode_add (Amode.ImmReg off base flags) value)
(if-let (valid_reg) base)
;; Shift of 0 --> base + 1*value.
(Amode.ImmRegRegShift off base value 0 flags))
;; -- Case 3 (adding a shifted value to an Amode).
;;
;; An Amode.ImmReg can absorb a shift of another register as the index register.
;;
;; Priority 2 to take these rules above generic case.
(rule 2 (amode_add (Amode.ImmReg off base flags) (ishl index (iconst (uimm8 shift))))
(if-let (valid_reg) base)
(if (u32_lteq (u8_as_u32 shift) 3))
(Amode.ImmRegRegShift off base index shift flags))
(rule 2 (amode_add (Amode.ImmReg off base flags) (uextend (ishl index (iconst (uimm8 shift)))))
(if-let (valid_reg) base)
(if (u32_lteq (u8_as_u32 shift) 3))
(Amode.ImmRegRegShift off base (extend_to_gpr index $I64 (ExtendKind.Zero)) shift flags))
;; Same, but with a uextend of a shift of a 32-bit add. This is valid
;; because we know our lowering of a narrower-than-64-bit `iadd` will
;; always write the full register width, so we can effectively ignore
;; the `uextend` and look through it to the `ishl`.
;;
;; Priority 2 to take this case above generic rules.
(rule 2 (amode_add (Amode.ImmReg off base flags)
(uextend (ishl index @ (iadd _ _) (iconst (uimm8 shift)))))
(if-let (valid_reg) base)
(if (u32_lteq (u8_as_u32 shift) 3))
(Amode.ImmRegRegShift off base index shift flags))
;; -- Case 4 (absorbing constant offsets).
;;
;; An Amode can absorb a constant (i64, or extended i32) as long as
;; the sum still fits in the signed-32-bit offset.
;;
;; Priority 3 in order to take this option above the fallback
;; (immediate in register). Two rules, for imm+reg and
;; imm+reg+scale*reg cases.
(rule 3 (amode_add (Amode.ImmReg off base flags)
(iconst (simm32 c)))
(if-let sum (s32_add_fallible off c))
(Amode.ImmReg sum base flags))
(rule 3 (amode_add (Amode.ImmRegRegShift off base index shift flags)
(iconst (simm32 c)))
(if-let sum (s32_add_fallible off c))
(Amode.ImmRegRegShift sum base index shift flags))
;; Likewise for a zero-extended i32 const, as long as the constant
;; wasn't negative. (Why nonnegative? Because adding a
;; non-sign-extended negative to a 64-bit address is not the same as
;; adding in simm32-space.)
(rule 3 (amode_add (Amode.ImmReg off base flags)
(uextend (iconst (simm32 (u32_nonnegative c)))))
(if-let sum (s32_add_fallible off c))
(Amode.ImmReg sum base flags))
(rule 3 (amode_add (Amode.ImmRegRegShift off base index shift flags)
(uextend (iconst (simm32 (u32_nonnegative c)))))
(if-let sum (s32_add_fallible off c))
(Amode.ImmRegRegShift sum base index shift flags))
;; Likewise for a sign-extended i32 const.
(rule 3 (amode_add (Amode.ImmReg off base flags)
(sextend (iconst (simm32 c))))
(if-let sum (s32_add_fallible off c))
(Amode.ImmReg sum base flags))
(rule 3 (amode_add (Amode.ImmRegRegShift off base index shift flags)
(sextend (iconst (simm32 c))))
(if-let sum (s32_add_fallible off c))
(Amode.ImmRegRegShift sum base index shift flags))
;; -- Case 5 (fallback to add a new value to an imm+reg+scale*reg).
;;
;; An Amode.ImmRegRegShift can absorb any other value by creating a
;; new add instruction and replacing the base with
;; (base+value).
(rule (amode_add (Amode.ImmRegRegShift off base index shift flags) value)
(let ((sum Gpr (x64_add $I64 base value)))
(Amode.ImmRegRegShift off sum index shift flags)))
;; Finally, define the toplevel `to_amode`.
(rule (to_amode flags base offset)
(amode_imm_reg_flags offset (put_in_gpr base) flags))
(amode_add (amode_initial flags offset) base))
;; Offsetting an Amode. Used when we need to do consecutive
;; loads/stores to adjacent addresses.

27
cranelift/codegen/src/isa/x64/inst/args.rs
View File

@@ -249,30 +249,11 @@ newtype_of_reg!(
|reg| reg.class() == RegClass::Float
);
/// A possible addressing mode (amode) that can be used in instructions.
/// These denote a 64-bit value only.
#[derive(Clone, Debug)]
pub enum Amode {
/// Immediate sign-extended and a Register.
ImmReg {
simm32: u32,
base: Reg,
flags: MemFlags,
},
// N.B.: `Amode` is defined in `inst.isle`. We add some convenience
// constructors here.
/// sign-extend-32-to-64(Immediate) + Register1 + (Register2 << Shift)
ImmRegRegShift {
simm32: u32,
base: Gpr,
index: Gpr,
shift: u8, /* 0 .. 3 only */
flags: MemFlags,
},
/// sign-extend-32-to-64(Immediate) + RIP (instruction pointer).
/// To wit: not supported in 32-bits mode.
RipRelative { target: MachLabel },
}
// Re-export the type from the ISLE generated code.
pub use crate::isa::x64::lower::isle::generated_code::Amode;
impl Amode {
pub(crate) fn imm_reg(simm32: u32, base: Reg) -> Self {

9
cranelift/codegen/src/isa/x64/lower/isle.rs
View File

@@ -312,15 +312,6 @@ where
amode.clone().into()
}
#[inline]
fn const_shift_lt_eq_3(&mut self, shift_amount: Value) -> Option<u8> {
let input = self.lower_ctx.get_value_as_source_or_const(shift_amount);
match input.constant {
Some(shift_amount) if shift_amount <= 3 => Some(shift_amount as u8),
_ => None,
}
}
#[inline]
fn writable_gpr_to_reg(&mut self, r: WritableGpr) -> WritableReg {
r.to_writable_reg()

4
cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest
View File

@@ -1,4 +1,4 @@
src/clif.isle 443b34b797fc8ace
src/prelude.isle a7915a6b88310eb5
src/isa/x64/inst.isle 65f15f51eefe0ce3
src/prelude.isle 97c4b6eebbab9f05
src/isa/x64/inst.isle a7f86254b89a7136
src/isa/x64/lower.isle 4c567e9157f84afb

1431
cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs generated
View File

File diff suppressed because it is too large Load Diff

102
cranelift/codegen/src/machinst/isle.rs
View File

@@ -110,6 +110,26 @@ macro_rules! isle_prelude_methods {
Reg::invalid_sentinel()
}
#[inline]
fn invalid_reg_etor(&mut self, reg: Reg) -> Option<()> {
use crate::machinst::valueregs::InvalidSentinel;
if reg.is_invalid_sentinel() {
Some(())
} else {
None
}
}
#[inline]
fn valid_reg(&mut self, reg: Reg) -> Option<()> {
use crate::machinst::valueregs::InvalidSentinel;
if !reg.is_invalid_sentinel() {
Some(())
} else {
None
}
}
#[inline]
fn put_in_reg(&mut self, val: Value) -> Reg {
self.lower_ctx.put_value_in_regs(val).only_reg().unwrap()
@@ -131,44 +151,49 @@ macro_rules! isle_prelude_methods {
}
#[inline]
fn u8_as_u64(&mut self, x: u8) -> u64 {
x.into()
fn u8_as_u32(&mut self, x: u8) -> Option<u32> {
Some(x.into())
}
#[inline]
fn u16_as_u64(&mut self, x: u16) -> u64 {
x.into()
fn u8_as_u64(&mut self, x: u8) -> Option<u64> {
Some(x.into())
}
#[inline]
fn u32_as_u64(&mut self, x: u32) -> u64 {
x.into()
fn u16_as_u64(&mut self, x: u16) -> Option<u64> {
Some(x.into())
}
#[inline]
fn i64_as_u64(&mut self, x: i64) -> u64 {
x as u64
fn u32_as_u64(&mut self, x: u32) -> Option<u64> {
Some(x.into())
}
#[inline]
fn u64_add(&mut self, x: u64, y: u64) -> u64 {
x.wrapping_add(y)
fn i64_as_u64(&mut self, x: i64) -> Option<u64> {
Some(x as u64)
}
#[inline]
fn u64_sub(&mut self, x: u64, y: u64) -> u64 {
x.wrapping_sub(y)
fn u64_add(&mut self, x: u64, y: u64) -> Option<u64> {
Some(x.wrapping_add(y))
}
#[inline]
fn u64_and(&mut self, x: u64, y: u64) -> u64 {
x & y
fn u64_sub(&mut self, x: u64, y: u64) -> Option<u64> {
Some(x.wrapping_sub(y))
}
#[inline]
fn ty_bits(&mut self, ty: Type) -> u8 {
fn u64_and(&mut self, x: u64, y: u64) -> Option<u64> {
Some(x & y)
}
#[inline]
fn ty_bits(&mut self, ty: Type) -> Option<u8> {
use std::convert::TryInto;
ty.bits().try_into().unwrap()
Some(ty.bits().try_into().unwrap())
}
#[inline]
@@ -454,6 +479,51 @@ macro_rules! isle_prelude_methods {
a.wrapping_add(b)
}
#[inline]
fn s32_add_fallible(&mut self, a: u32, b: u32) -> Option<u32> {
let a = a as i32;
let b = b as i32;
a.checked_add(b).map(|sum| sum as u32)
}
#[inline]
fn u32_nonnegative(&mut self, x: u32) -> Option<u32> {
if (x as i32) >= 0 {
Some(x)
} else {
None
}
}
#[inline]
fn u32_lteq(&mut self, a: u32, b: u32) -> Option<()> {
if a <= b {
Some(())
} else {
None
}
}
#[inline]
fn simm32(&mut self, x: Imm64) -> Option<u32> {
let x64: i64 = x.into();
let x32: i32 = x64.try_into().ok()?;
Some(x32 as u32)
}
#[inline]
fn uimm8(&mut self, x: Imm64) -> Option<u8> {
let x64: i64 = x.into();
let x8: u8 = x64.try_into().ok()?;
Some(x8)
}
#[inline]
fn offset32(&mut self, x: Offset32) -> Option<u32> {
let x: i32 = x.into();
Some(x as u32)
}
#[inline]
fn u8_and(&mut self, a: u8, b: u8) -> u8 {
a & b

53
cranelift/codegen/src/prelude.isle
View File

@@ -46,6 +46,33 @@
(decl u32_add (u32 u32) u32)
(extern constructor u32_add u32_add)
;; Pure/fallible constructor that tries to add two `u32`s, interpreted
;; as signed values, and fails to match on overflow.
(decl pure s32_add_fallible (u32 u32) u32)
(extern constructor s32_add_fallible s32_add_fallible)
;; Extractor that matches a `u32` only if non-negative.
(decl u32_nonnegative (u32) u32)
(extern extractor u32_nonnegative u32_nonnegative)
;; Extractor that pulls apart an Offset32 into a u32 with the raw
;; signed-32-bit twos-complement bits.
(decl offset32 (u32) Offset32)
(extern extractor offset32 offset32)
;; Pure/fallible constructor that tests if one u32 is less than or
;; equal to another.
(decl pure u32_lteq (u32 u32) Unit)
(extern constructor u32_lteq u32_lteq)
;; Get a signed 32-bit immediate in an u32 from an Imm64, if possible.
(decl simm32 (u32) Imm64)
(extern extractor simm32 simm32)
;; Get an unsigned 8-bit immediate in a u8 from an Imm64, if possible.
(decl uimm8 (u8) Imm64)
(extern extractor uimm8 uimm8)
(decl u8_and (u8 u8) u8)
(extern constructor u8_and u8_and)
@@ -110,9 +137,14 @@
(rule (temp_reg ty)
(writable_reg_to_reg (temp_writable_reg ty)))
;; Get the invalid register.
;; Get or match the invalid register.
(decl invalid_reg () Reg)
(extern constructor invalid_reg invalid_reg)
(extern extractor invalid_reg invalid_reg_etor)
;; Match any register but the invalid register.
(decl valid_reg () Reg)
(extern extractor valid_reg valid_reg)
;; Put the given value into a register.
;;
@@ -163,27 +195,30 @@
;;;; Primitive Type Conversions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(decl u8_as_u64 (u8) u64)
(decl pure u8_as_u32 (u8) u32)
(extern constructor u8_as_u32 u8_as_u32)
(decl pure u8_as_u64 (u8) u64)
(extern constructor u8_as_u64 u8_as_u64)
(decl u16_as_u64 (u16) u64)
(decl pure u16_as_u64 (u16) u64)
(extern constructor u16_as_u64 u16_as_u64)
(decl u32_as_u64 (u32) u64)
(decl pure u32_as_u64 (u32) u64)
(extern constructor u32_as_u64 u32_as_u64)
(decl i64_as_u64 (i64) u64)
(decl pure i64_as_u64 (i64) u64)
(extern constructor i64_as_u64 i64_as_u64)
;;;; Primitive Arithmetic ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(decl u64_add (u64 u64) u64)
(decl pure u64_add (u64 u64) u64)
(extern constructor u64_add u64_add)
(decl u64_sub (u64 u64) u64)
(decl pure u64_sub (u64 u64) u64)
(extern constructor u64_sub u64_sub)
(decl u64_and (u64 u64) u64)
(decl pure u64_and (u64 u64) u64)
(extern constructor u64_and u64_and)
;;;; `cranelift_codegen::ir::Type` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -221,7 +256,7 @@
(extern const $F64X2 Type)
;; Get the bit width of a given type.
(decl ty_bits (Type) u8)
(decl pure ty_bits (Type) u8)
(extern constructor ty_bits ty_bits)
;; Get the bit width of a given type.

94
cranelift/filetests/filetests/isa/x64/amode-opt.clif
View File

@@ -65,3 +65,97 @@ block0(v0: i64):
; popq %rbp
; ret
function %amode_reg_reg_imm(i64, i64) -> i64 {
block0(v0: i64, v1: i64):
v2 = iadd v0, v1
v3 = iconst.i64 256
v4 = iadd v2, v3
v5 = load.i64 v4+64
return v5
}
; pushq %rbp
; movq %rsp, %rbp
; block0:
; movq 320(%rdi,%rsi,1), %rax
; movq %rbp, %rsp
; popq %rbp
; ret
function %amode_reg_reg_imm_negative(i64, i64) -> i64 {
block0(v0: i64, v1: i64):
v2 = iadd v0, v1
v3 = iconst.i64 -1
v4 = iadd v2, v3
v5 = load.i64 v4
return v5
}
; pushq %rbp
; movq %rsp, %rbp
; block0:
; movq -1(%rdi,%rsi,1), %rax
; movq %rbp, %rsp
; popq %rbp
; ret
function %amode_reg_reg_imm_scaled(i64, i64) -> i64 {
block0(v0: i64, v1: i64):
v2 = iconst.i64 -1
v3 = iadd v0, v2
v4 = ishl_imm v1, 3
v5 = iadd v3, v4
v6 = load.i64 v5
return v6
}
; pushq %rbp
; movq %rsp, %rbp
; block0:
; movq -1(%rdi,%rsi,8), %rax
; movq %rbp, %rsp
; popq %rbp
; ret
function %amode_reg_reg_imm_uext_scaled(i64, i32) -> i64 {
block0(v0: i64, v1: i32):
v2 = iconst.i64 -1
v3 = iadd v0, v2
v4 = ishl_imm v1, 3
v5 = uextend.i64 v4
v6 = iadd v3, v5
v7 = load.i64 v6
return v7
}
; pushq %rbp
; movq %rsp, %rbp
; block0:
; movl %esi, %r8d
; movq -1(%rdi,%r8,8), %rax
; movq %rbp, %rsp
; popq %rbp
; ret
function %amode_reg_reg_imm_uext_scaled_add(i64, i32, i32) -> i64 {
block0(v0: i64, v1: i32, v2: i32):
v3 = iconst.i64 -1
v4 = iadd v0, v3
v5 = iadd v1, v2
v6 = ishl_imm v5, 2
v7 = uextend.i64 v6
v8 = iadd v4, v7
v9 = load.i64 v8
return v9
}
; pushq %rbp
; movq %rsp, %rbp
; block0:
; addl %esi, %edx, %esi
; movq -1(%rdi,%rsi,4), %rax
; movq %rbp, %rsp
; popq %rbp
; ret