T0b1/wasmtime
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

x64: use constant pool for u64 constants rather than movabs. (#4088)

Browse Source 
* Allow emitting u64 constants into constant pool.

* Use constant pool for constants on x64 that do not fit in a simm32 and are needed as a RegMem or RegMemImm.

* Fix rip-relative addressing bug in pinsrd emission.
This commit is contained in:
Chris Fallin
2022-05-10 09:21:05 -07:00
committed by GitHub
parent d3087487ea
commit eb435f3057
16 changed files with 483 additions and 383 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
cranelift/codegen/src/isa/aarch64/lower/isle.rs
View File

@@ -23,7 +23,7 @@ use crate::{
isa::aarch64::inst::args::{ShiftOp, ShiftOpShiftImm}, isa::aarch64::inst::args::{ShiftOp, ShiftOpShiftImm},
isa::aarch64::lower::{is_valid_atomic_transaction_ty, writable_xreg, xreg}, isa::aarch64::lower::{is_valid_atomic_transaction_ty, writable_xreg, xreg},
isa::unwind::UnwindInst, isa::unwind::UnwindInst,
machinst::{ty_bits, InsnOutput, LowerCtx}, machinst::{ty_bits, InsnOutput, LowerCtx, VCodeConstant, VCodeConstantData},
}; };
use std::boxed::Box; use std::boxed::Box;
use std::convert::TryFrom; use std::convert::TryFrom;

2
cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest
View File

@@ -1,4 +1,4 @@
src/clif.isle 443b34b797fc8ace src/clif.isle 443b34b797fc8ace
src/prelude.isle 97c4b6eebbab9f05 src/prelude.isle e6c91b0115343ab9
src/isa/aarch64/inst.isle 21a43af20be377d2 src/isa/aarch64/inst.isle 21a43af20be377d2
src/isa/aarch64/lower.isle 75ad8450963e3829 src/isa/aarch64/lower.isle 75ad8450963e3829

31
cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.rs generated
View File

@@ -90,6 +90,7 @@ pub trait Context {
fn def_inst(&mut self, arg0: Value) -> Option<Inst>; fn def_inst(&mut self, arg0: Value) -> Option<Inst>;
fn offset32_to_u32(&mut self, arg0: Offset32) -> u32; fn offset32_to_u32(&mut self, arg0: Offset32) -> u32;
fn emit(&mut self, arg0: &MInst) -> Unit; fn emit(&mut self, arg0: &MInst) -> Unit;
fn emit_u64_le_const(&mut self, arg0: u64) -> VCodeConstant;
fn trap_code_division_by_zero(&mut self) -> TrapCode; fn trap_code_division_by_zero(&mut self) -> TrapCode;
fn trap_code_integer_overflow(&mut self) -> TrapCode; fn trap_code_integer_overflow(&mut self) -> TrapCode;
fn trap_code_bad_conversion_to_integer(&mut self) -> TrapCode; fn trap_code_bad_conversion_to_integer(&mut self) -> TrapCode;
@@ -146,14 +147,14 @@ pub trait Context {
fn rotr_opposite_amount(&mut self, arg0: Type, arg1: ImmShift) -> ImmShift; fn rotr_opposite_amount(&mut self, arg0: Type, arg1: ImmShift) -> ImmShift;
} }
/// Internal type SideEffectNoResult: defined at src/prelude.isle line 447. /// Internal type SideEffectNoResult: defined at src/prelude.isle line 457.
#[derive(Clone, Debug)] #[derive(Clone, Debug)]
pub enum SideEffectNoResult { pub enum SideEffectNoResult {
Inst { inst: MInst }, Inst { inst: MInst },
Inst2 { inst1: MInst, inst2: MInst }, Inst2 { inst1: MInst, inst2: MInst },
} }
/// Internal type ProducesFlags: defined at src/prelude.isle line 474. /// Internal type ProducesFlags: defined at src/prelude.isle line 484.
#[derive(Clone, Debug)] #[derive(Clone, Debug)]
pub enum ProducesFlags { pub enum ProducesFlags {
ProducesFlagsSideEffect { inst: MInst }, ProducesFlagsSideEffect { inst: MInst },
@@ -161,7 +162,7 @@ pub enum ProducesFlags {
ProducesFlagsReturnsResultWithConsumer { inst: MInst, result: Reg }, ProducesFlagsReturnsResultWithConsumer { inst: MInst, result: Reg },
} }
/// Internal type ConsumesFlags: defined at src/prelude.isle line 485. /// Internal type ConsumesFlags: defined at src/prelude.isle line 495.
#[derive(Clone, Debug)] #[derive(Clone, Debug)]
pub enum ConsumesFlags { pub enum ConsumesFlags {
ConsumesFlagsReturnsResultWithProducer { ConsumesFlagsReturnsResultWithProducer {
@@ -1102,7 +1103,7 @@ pub fn constructor_side_effect<C: Context>(
&SideEffectNoResult::Inst { &SideEffectNoResult::Inst {
inst: ref pattern1_0, inst: ref pattern1_0,
} => { } => {
// Rule at src/prelude.isle line 455. // Rule at src/prelude.isle line 465.
let expr0_0 = C::emit(ctx, pattern1_0); let expr0_0 = C::emit(ctx, pattern1_0);
let expr1_0 = C::output_none(ctx); let expr1_0 = C::output_none(ctx);
return Some(expr1_0); return Some(expr1_0);
@@ -1111,7 +1112,7 @@ pub fn constructor_side_effect<C: Context>(
inst1: ref pattern1_0, inst1: ref pattern1_0,
inst2: ref pattern1_1, inst2: ref pattern1_1,
} => { } => {
// Rule at src/prelude.isle line 458. // Rule at src/prelude.isle line 468.
let expr0_0 = C::emit(ctx, pattern1_0); let expr0_0 = C::emit(ctx, pattern1_0);
let expr1_0 = C::emit(ctx, pattern1_1); let expr1_0 = C::emit(ctx, pattern1_1);
let expr2_0 = C::output_none(ctx); let expr2_0 = C::output_none(ctx);
@@ -1138,7 +1139,7 @@ pub fn constructor_side_effect_concat<C: Context>(
inst: ref pattern3_0, inst: ref pattern3_0,
} = pattern2_0 } = pattern2_0
{ {
// Rule at src/prelude.isle line 464. // Rule at src/prelude.isle line 474.
let expr0_0 = SideEffectNoResult::Inst2 { let expr0_0 = SideEffectNoResult::Inst2 {
inst1: pattern1_0.clone(), inst1: pattern1_0.clone(),
inst2: pattern3_0.clone(), inst2: pattern3_0.clone(),
@@ -1160,7 +1161,7 @@ pub fn constructor_produces_flags_get_reg<C: Context>(
result: pattern1_1, result: pattern1_1,
} = pattern0_0 } = pattern0_0
{ {
// Rule at src/prelude.isle line 501. // Rule at src/prelude.isle line 511.
return Some(pattern1_1); return Some(pattern1_1);
} }
return None; return None;
@@ -1177,7 +1178,7 @@ pub fn constructor_produces_flags_ignore<C: Context>(
inst: ref pattern1_0, inst: ref pattern1_0,
result: pattern1_1, result: pattern1_1,
} => { } => {
// Rule at src/prelude.isle line 506. // Rule at src/prelude.isle line 516.
let expr0_0 = ProducesFlags::ProducesFlagsSideEffect { let expr0_0 = ProducesFlags::ProducesFlagsSideEffect {
inst: pattern1_0.clone(), inst: pattern1_0.clone(),
}; };
@@ -1187,7 +1188,7 @@ pub fn constructor_produces_flags_ignore<C: Context>(
inst: ref pattern1_0, inst: ref pattern1_0,
result: pattern1_1, result: pattern1_1,
} => { } => {
// Rule at src/prelude.isle line 508. // Rule at src/prelude.isle line 518.
let expr0_0 = ProducesFlags::ProducesFlagsSideEffect { let expr0_0 = ProducesFlags::ProducesFlagsSideEffect {
inst: pattern1_0.clone(), inst: pattern1_0.clone(),
}; };
@@ -1216,7 +1217,7 @@ pub fn constructor_consumes_flags_concat<C: Context>(
result: pattern3_1, result: pattern3_1,
} = pattern2_0 } = pattern2_0
{ {
// Rule at src/prelude.isle line 515. // Rule at src/prelude.isle line 525.
let expr0_0 = C::value_regs(ctx, pattern1_1, pattern3_1); let expr0_0 = C::value_regs(ctx, pattern1_1, pattern3_1);
let expr1_0 = ConsumesFlags::ConsumesFlagsTwiceReturnsValueRegs { let expr1_0 = ConsumesFlags::ConsumesFlagsTwiceReturnsValueRegs {
inst1: pattern1_0.clone(), inst1: pattern1_0.clone(),
@@ -1246,7 +1247,7 @@ pub fn constructor_with_flags<C: Context>(
inst: ref pattern3_0, inst: ref pattern3_0,
result: pattern3_1, result: pattern3_1,
} => { } => {
// Rule at src/prelude.isle line 540. // Rule at src/prelude.isle line 550.
let expr0_0 = C::emit(ctx, pattern1_0); let expr0_0 = C::emit(ctx, pattern1_0);
let expr1_0 = C::emit(ctx, pattern3_0); let expr1_0 = C::emit(ctx, pattern3_0);
let expr2_0 = C::value_reg(ctx, pattern3_1); let expr2_0 = C::value_reg(ctx, pattern3_1);
@@ -1257,7 +1258,7 @@ pub fn constructor_with_flags<C: Context>(
inst2: ref pattern3_1, inst2: ref pattern3_1,
result: pattern3_2, result: pattern3_2,
} => { } => {
// Rule at src/prelude.isle line 546. // Rule at src/prelude.isle line 556.
let expr0_0 = C::emit(ctx, pattern1_0); let expr0_0 = C::emit(ctx, pattern1_0);
let expr1_0 = C::emit(ctx, pattern3_0); let expr1_0 = C::emit(ctx, pattern3_0);
let expr2_0 = C::emit(ctx, pattern3_1); let expr2_0 = C::emit(ctx, pattern3_1);
@@ -1270,7 +1271,7 @@ pub fn constructor_with_flags<C: Context>(
inst4: ref pattern3_3, inst4: ref pattern3_3,
result: pattern3_4, result: pattern3_4,
} => { } => {
// Rule at src/prelude.isle line 558. // Rule at src/prelude.isle line 568.
let expr0_0 = C::emit(ctx, pattern1_0); let expr0_0 = C::emit(ctx, pattern1_0);
let expr1_0 = C::emit(ctx, pattern3_0); let expr1_0 = C::emit(ctx, pattern3_0);
let expr2_0 = C::emit(ctx, pattern3_1); let expr2_0 = C::emit(ctx, pattern3_1);
@@ -1291,7 +1292,7 @@ pub fn constructor_with_flags<C: Context>(
result: pattern3_1, result: pattern3_1,
} = pattern2_0 } = pattern2_0
{ {
// Rule at src/prelude.isle line 534. // Rule at src/prelude.isle line 544.
let expr0_0 = C::emit(ctx, pattern1_0); let expr0_0 = C::emit(ctx, pattern1_0);
let expr1_0 = C::emit(ctx, pattern3_0); let expr1_0 = C::emit(ctx, pattern3_0);
let expr2_0 = C::value_regs(ctx, pattern1_1, pattern3_1); let expr2_0 = C::value_regs(ctx, pattern1_1, pattern3_1);
@@ -1311,7 +1312,7 @@ pub fn constructor_with_flags_reg<C: Context>(
) -> Option<Reg> { ) -> Option<Reg> {
let pattern0_0 = arg0; let pattern0_0 = arg0;
let pattern1_0 = arg1; let pattern1_0 = arg1;
// Rule at src/prelude.isle line 575. // Rule at src/prelude.isle line 585.
let expr0_0 = constructor_with_flags(ctx, pattern0_0, pattern1_0)?; let expr0_0 = constructor_with_flags(ctx, pattern0_0, pattern1_0)?;
let expr1_0: usize = 0; let expr1_0: usize = 0;
let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0); let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0);

2
cranelift/codegen/src/isa/s390x/lower/isle.rs
View File

@@ -17,7 +17,7 @@ use crate::{
StackSlot, TrapCode, Value, ValueList, StackSlot, TrapCode, Value, ValueList,
}, },
isa::unwind::UnwindInst, isa::unwind::UnwindInst,
machinst::{InsnOutput, LowerCtx}, machinst::{InsnOutput, LowerCtx, VCodeConstant, VCodeConstantData},
}; };
use std::boxed::Box; use std::boxed::Box;
use std::cell::Cell; use std::cell::Cell;

2
cranelift/codegen/src/isa/s390x/lower/isle/generated_code.manifest
View File

@@ -1,4 +1,4 @@
src/clif.isle 443b34b797fc8ace src/clif.isle 443b34b797fc8ace
src/prelude.isle 97c4b6eebbab9f05 src/prelude.isle e6c91b0115343ab9
src/isa/s390x/inst.isle 36c2500563cdd4e6 src/isa/s390x/inst.isle 36c2500563cdd4e6
src/isa/s390x/lower.isle e5c946ab8a265b77 src/isa/s390x/lower.isle e5c946ab8a265b77

31
cranelift/codegen/src/isa/s390x/lower/isle/generated_code.rs generated
View File

@@ -90,6 +90,7 @@ pub trait Context {
fn def_inst(&mut self, arg0: Value) -> Option<Inst>; fn def_inst(&mut self, arg0: Value) -> Option<Inst>;
fn offset32_to_u32(&mut self, arg0: Offset32) -> u32; fn offset32_to_u32(&mut self, arg0: Offset32) -> u32;
fn emit(&mut self, arg0: &MInst) -> Unit; fn emit(&mut self, arg0: &MInst) -> Unit;
fn emit_u64_le_const(&mut self, arg0: u64) -> VCodeConstant;
fn trap_code_division_by_zero(&mut self) -> TrapCode; fn trap_code_division_by_zero(&mut self) -> TrapCode;
fn trap_code_integer_overflow(&mut self) -> TrapCode; fn trap_code_integer_overflow(&mut self) -> TrapCode;
fn trap_code_bad_conversion_to_integer(&mut self) -> TrapCode; fn trap_code_bad_conversion_to_integer(&mut self) -> TrapCode;
@@ -165,14 +166,14 @@ pub trait Context {
fn same_reg(&mut self, arg0: WritableReg, arg1: Reg) -> Option<Reg>; fn same_reg(&mut self, arg0: WritableReg, arg1: Reg) -> Option<Reg>;
} }
/// Internal type SideEffectNoResult: defined at src/prelude.isle line 447. /// Internal type SideEffectNoResult: defined at src/prelude.isle line 457.
#[derive(Clone, Debug)] #[derive(Clone, Debug)]
pub enum SideEffectNoResult { pub enum SideEffectNoResult {
Inst { inst: MInst }, Inst { inst: MInst },
Inst2 { inst1: MInst, inst2: MInst }, Inst2 { inst1: MInst, inst2: MInst },
} }
/// Internal type ProducesFlags: defined at src/prelude.isle line 474. /// Internal type ProducesFlags: defined at src/prelude.isle line 484.
#[derive(Clone, Debug)] #[derive(Clone, Debug)]
pub enum ProducesFlags { pub enum ProducesFlags {
ProducesFlagsSideEffect { inst: MInst }, ProducesFlagsSideEffect { inst: MInst },
@@ -180,7 +181,7 @@ pub enum ProducesFlags {
ProducesFlagsReturnsResultWithConsumer { inst: MInst, result: Reg }, ProducesFlagsReturnsResultWithConsumer { inst: MInst, result: Reg },
} }
/// Internal type ConsumesFlags: defined at src/prelude.isle line 485. /// Internal type ConsumesFlags: defined at src/prelude.isle line 495.
#[derive(Clone, Debug)] #[derive(Clone, Debug)]
pub enum ConsumesFlags { pub enum ConsumesFlags {
ConsumesFlagsReturnsResultWithProducer { ConsumesFlagsReturnsResultWithProducer {
@@ -970,7 +971,7 @@ pub fn constructor_side_effect<C: Context>(
&SideEffectNoResult::Inst { &SideEffectNoResult::Inst {
inst: ref pattern1_0, inst: ref pattern1_0,
} => { } => {
// Rule at src/prelude.isle line 455. // Rule at src/prelude.isle line 465.
let expr0_0 = C::emit(ctx, pattern1_0); let expr0_0 = C::emit(ctx, pattern1_0);
let expr1_0 = C::output_none(ctx); let expr1_0 = C::output_none(ctx);
return Some(expr1_0); return Some(expr1_0);
@@ -979,7 +980,7 @@ pub fn constructor_side_effect<C: Context>(
inst1: ref pattern1_0, inst1: ref pattern1_0,
inst2: ref pattern1_1, inst2: ref pattern1_1,
} => { } => {
// Rule at src/prelude.isle line 458. // Rule at src/prelude.isle line 468.
let expr0_0 = C::emit(ctx, pattern1_0); let expr0_0 = C::emit(ctx, pattern1_0);
let expr1_0 = C::emit(ctx, pattern1_1); let expr1_0 = C::emit(ctx, pattern1_1);
let expr2_0 = C::output_none(ctx); let expr2_0 = C::output_none(ctx);
@@ -1006,7 +1007,7 @@ pub fn constructor_side_effect_concat<C: Context>(
inst: ref pattern3_0, inst: ref pattern3_0,
} = pattern2_0 } = pattern2_0
{ {
// Rule at src/prelude.isle line 464. // Rule at src/prelude.isle line 474.
let expr0_0 = SideEffectNoResult::Inst2 { let expr0_0 = SideEffectNoResult::Inst2 {
inst1: pattern1_0.clone(), inst1: pattern1_0.clone(),
inst2: pattern3_0.clone(), inst2: pattern3_0.clone(),
@@ -1028,7 +1029,7 @@ pub fn constructor_produces_flags_get_reg<C: Context>(
result: pattern1_1, result: pattern1_1,
} = pattern0_0 } = pattern0_0
{ {
// Rule at src/prelude.isle line 501. // Rule at src/prelude.isle line 511.
return Some(pattern1_1); return Some(pattern1_1);
} }
return None; return None;
@@ -1045,7 +1046,7 @@ pub fn constructor_produces_flags_ignore<C: Context>(
inst: ref pattern1_0, inst: ref pattern1_0,
result: pattern1_1, result: pattern1_1,
} => { } => {
// Rule at src/prelude.isle line 506. // Rule at src/prelude.isle line 516.
let expr0_0 = ProducesFlags::ProducesFlagsSideEffect { let expr0_0 = ProducesFlags::ProducesFlagsSideEffect {
inst: pattern1_0.clone(), inst: pattern1_0.clone(),
}; };
@@ -1055,7 +1056,7 @@ pub fn constructor_produces_flags_ignore<C: Context>(
inst: ref pattern1_0, inst: ref pattern1_0,
result: pattern1_1, result: pattern1_1,
} => { } => {
// Rule at src/prelude.isle line 508. // Rule at src/prelude.isle line 518.
let expr0_0 = ProducesFlags::ProducesFlagsSideEffect { let expr0_0 = ProducesFlags::ProducesFlagsSideEffect {
inst: pattern1_0.clone(), inst: pattern1_0.clone(),
}; };
@@ -1084,7 +1085,7 @@ pub fn constructor_consumes_flags_concat<C: Context>(
result: pattern3_1, result: pattern3_1,
} = pattern2_0 } = pattern2_0
{ {
// Rule at src/prelude.isle line 515. // Rule at src/prelude.isle line 525.
let expr0_0 = C::value_regs(ctx, pattern1_1, pattern3_1); let expr0_0 = C::value_regs(ctx, pattern1_1, pattern3_1);
let expr1_0 = ConsumesFlags::ConsumesFlagsTwiceReturnsValueRegs { let expr1_0 = ConsumesFlags::ConsumesFlagsTwiceReturnsValueRegs {
inst1: pattern1_0.clone(), inst1: pattern1_0.clone(),
@@ -1114,7 +1115,7 @@ pub fn constructor_with_flags<C: Context>(
inst: ref pattern3_0, inst: ref pattern3_0,
result: pattern3_1, result: pattern3_1,
} => { } => {
// Rule at src/prelude.isle line 540. // Rule at src/prelude.isle line 550.
let expr0_0 = C::emit(ctx, pattern1_0); let expr0_0 = C::emit(ctx, pattern1_0);
let expr1_0 = C::emit(ctx, pattern3_0); let expr1_0 = C::emit(ctx, pattern3_0);
let expr2_0 = C::value_reg(ctx, pattern3_1); let expr2_0 = C::value_reg(ctx, pattern3_1);
@@ -1125,7 +1126,7 @@ pub fn constructor_with_flags<C: Context>(
inst2: ref pattern3_1, inst2: ref pattern3_1,
result: pattern3_2, result: pattern3_2,
} => { } => {
// Rule at src/prelude.isle line 546. // Rule at src/prelude.isle line 556.
let expr0_0 = C::emit(ctx, pattern1_0); let expr0_0 = C::emit(ctx, pattern1_0);
let expr1_0 = C::emit(ctx, pattern3_0); let expr1_0 = C::emit(ctx, pattern3_0);
let expr2_0 = C::emit(ctx, pattern3_1); let expr2_0 = C::emit(ctx, pattern3_1);
@@ -1138,7 +1139,7 @@ pub fn constructor_with_flags<C: Context>(
inst4: ref pattern3_3, inst4: ref pattern3_3,
result: pattern3_4, result: pattern3_4,
} => { } => {
// Rule at src/prelude.isle line 558. // Rule at src/prelude.isle line 568.
let expr0_0 = C::emit(ctx, pattern1_0); let expr0_0 = C::emit(ctx, pattern1_0);
let expr1_0 = C::emit(ctx, pattern3_0); let expr1_0 = C::emit(ctx, pattern3_0);
let expr2_0 = C::emit(ctx, pattern3_1); let expr2_0 = C::emit(ctx, pattern3_1);
@@ -1159,7 +1160,7 @@ pub fn constructor_with_flags<C: Context>(
result: pattern3_1, result: pattern3_1,
} = pattern2_0 } = pattern2_0
{ {
// Rule at src/prelude.isle line 534. // Rule at src/prelude.isle line 544.
let expr0_0 = C::emit(ctx, pattern1_0); let expr0_0 = C::emit(ctx, pattern1_0);
let expr1_0 = C::emit(ctx, pattern3_0); let expr1_0 = C::emit(ctx, pattern3_0);
let expr2_0 = C::value_regs(ctx, pattern1_1, pattern3_1); let expr2_0 = C::value_regs(ctx, pattern1_1, pattern3_1);
@@ -1179,7 +1180,7 @@ pub fn constructor_with_flags_reg<C: Context>(
) -> Option<Reg> { ) -> Option<Reg> {
let pattern0_0 = arg0; let pattern0_0 = arg0;
let pattern1_0 = arg1; let pattern1_0 = arg1;
// Rule at src/prelude.isle line 575. // Rule at src/prelude.isle line 585.
let expr0_0 = constructor_with_flags(ctx, pattern0_0, pattern1_0)?; let expr0_0 = constructor_with_flags(ctx, pattern0_0, pattern1_0)?;
let expr1_0: usize = 0; let expr1_0: usize = 0;
let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0); let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0);

12
cranelift/codegen/src/isa/x64/encoding/rex.rs
View File

@@ -284,6 +284,7 @@ pub(crate) fn emit_std_enc_mem(
enc_g: u8, enc_g: u8,
mem_e: &Amode, mem_e: &Amode,
rex: RexFlags, rex: RexFlags,
bytes_at_end: u8,
) { ) {
// General comment for this function: the registers in `mem_e` must be // General comment for this function: the registers in `mem_e` must be
// 64-bit integer registers, because they are part of an address // 64-bit integer registers, because they are part of an address
@@ -413,7 +414,14 @@ pub(crate) fn emit_std_enc_mem(
let offset = sink.cur_offset(); let offset = sink.cur_offset();
sink.use_label_at_offset(offset, *target, LabelUse::JmpRel32); sink.use_label_at_offset(offset, *target, LabelUse::JmpRel32);
sink.put4(0); // N.B.: some instructions (XmmRmRImm format for example)
// have bytes *after* the RIP-relative offset. The
// addressed location is relative to the end of the
// instruction, but the relocation is nominally relative
// to the end of the u32 field. So, to compensate for
// this, we emit a negative extra offset in the u32 field
// initially, and the relocation will add to it.
sink.put4(-(bytes_at_end as i32) as u32);
} }
} }
} }
@@ -466,6 +474,7 @@ pub(crate) fn emit_std_reg_mem(
reg_g: Reg, reg_g: Reg,
mem_e: &Amode, mem_e: &Amode,
rex: RexFlags, rex: RexFlags,
bytes_at_end: u8,
) { ) {
let enc_g = reg_enc(reg_g); let enc_g = reg_enc(reg_g);
emit_std_enc_mem( emit_std_enc_mem(
@@ -478,6 +487,7 @@ pub(crate) fn emit_std_reg_mem(
enc_g, enc_g,
mem_e, mem_e,
rex, rex,
bytes_at_end,
); );
} }

2
cranelift/codegen/src/isa/x64/inst.isle
View File

@@ -497,8 +497,6 @@
Size32 Size32
Size64)) Size64))
(type VCodeConstant (primitive VCodeConstant))
(type FenceKind extern (type FenceKind extern
(enum MFence (enum MFence
LFence LFence

37
cranelift/codegen/src/isa/x64/inst/emit.rs
View File

@@ -184,6 +184,7 @@ pub(crate) fn emit(
reg_g, reg_g,
&amode, &amode,
rex, rex,
0,
); );
} }
@@ -246,6 +247,7 @@ pub(crate) fn emit(
reg_g, reg_g,
&amode, &amode,
rex, rex,
0,
); );
} }
@@ -299,6 +301,7 @@ pub(crate) fn emit(
enc_g, enc_g,
&src1_dst, &src1_dst,
RexFlags::from(*size), RexFlags::from(*size),
0,
); );
} }
@@ -343,6 +346,7 @@ pub(crate) fn emit(
dst, dst,
&amode, &amode,
rex_flags, rex_flags,
0,
); );
} }
} }
@@ -437,6 +441,7 @@ pub(crate) fn emit(
subopcode, subopcode,
&amode, &amode,
RexFlags::from(*size), RexFlags::from(*size),
0,
); );
} }
} }
@@ -475,7 +480,7 @@ pub(crate) fn emit(
RegMem::Mem { addr: src } => { RegMem::Mem { addr: src } => {
let amode = src.finalize(state, sink).with_allocs(allocs); let amode = src.finalize(state, sink).with_allocs(allocs);
emit_std_enc_mem( emit_std_enc_mem(
sink, state, info, prefix, 0xF7, 1, subopcode, &amode, rex_flags, sink, state, info, prefix, 0xF7, 1, subopcode, &amode, rex_flags, 0,
); );
} }
} }
@@ -751,6 +756,7 @@ pub(crate) fn emit(
dst, dst,
src, src,
rex_flags, rex_flags,
0,
) )
} }
} }
@@ -770,6 +776,7 @@ pub(crate) fn emit(
dst, dst,
src, src,
RexFlags::set_w(), RexFlags::set_w(),
0,
) )
} }
@@ -787,6 +794,7 @@ pub(crate) fn emit(
dst, dst,
&amode, &amode,
RexFlags::set_w(), RexFlags::set_w(),
0,
); );
} }
@@ -849,6 +857,7 @@ pub(crate) fn emit(
dst, dst,
src, src,
rex_flags, rex_flags,
0,
) )
} }
} }
@@ -877,7 +886,7 @@ pub(crate) fn emit(
// 16-bit: MOV r16, r/m16 is 66 (REX.W==0) 89 /r // 16-bit: MOV r16, r/m16 is 66 (REX.W==0) 89 /r
// 32-bit: MOV r32, r/m32 is (REX.W==0) 89 /r // 32-bit: MOV r32, r/m32 is (REX.W==0) 89 /r
// 64-bit: MOV r64, r/m64 is (REX.W==1) 89 /r // 64-bit: MOV r64, r/m64 is (REX.W==1) 89 /r
emit_std_reg_mem(sink, state, info, prefix, opcode, 1, src, dst, rex); emit_std_reg_mem(sink, state, info, prefix, opcode, 1, src, dst, rex, 0);
} }
Inst::ShiftR { Inst::ShiftR {
@@ -997,6 +1006,7 @@ pub(crate) fn emit(
dst, dst,
addr, addr,
rex, rex,
0,
); );
} }
RegMemImm::Imm { .. } => unreachable!(), RegMemImm::Imm { .. } => unreachable!(),
@@ -1052,7 +1062,7 @@ pub(crate) fn emit(
(OperandSize::Size8, false) => 0x84, (OperandSize::Size8, false) => 0x84,
(_, false) => 0x85, (_, false) => 0x85,
}; };
emit_std_reg_mem(sink, state, info, prefix, opcode, 1, reg_g, addr, rex); emit_std_reg_mem(sink, state, info, prefix, opcode, 1, reg_g, addr, rex, 0);
} }
RegMemImm::Imm { simm32 } => { RegMemImm::Imm { simm32 } => {
@@ -1126,7 +1136,9 @@ pub(crate) fn emit(
} }
RegMem::Mem { addr } => { RegMem::Mem { addr } => {
let addr = &addr.finalize(state, sink).with_allocs(allocs); let addr = &addr.finalize(state, sink).with_allocs(allocs);
emit_std_reg_mem(sink, state, info, prefix, opcode, 2, dst, addr, rex_flags); emit_std_reg_mem(
sink, state, info, prefix, opcode, 2, dst, addr, rex_flags, 0,
);
} }
} }
} }
@@ -1190,6 +1202,7 @@ pub(crate) fn emit(
6, /*subopcode*/ 6, /*subopcode*/
addr, addr,
RexFlags::clear_w(), RexFlags::clear_w(),
0,
); );
} }
@@ -1266,6 +1279,7 @@ pub(crate) fn emit(
2, /*subopcode*/ 2, /*subopcode*/
addr, addr,
RexFlags::clear_w(), RexFlags::clear_w(),
0,
); );
} }
} }
@@ -1367,6 +1381,7 @@ pub(crate) fn emit(
4, /*subopcode*/ 4, /*subopcode*/
addr, addr,
RexFlags::clear_w(), RexFlags::clear_w(),
0,
); );
} }
} }
@@ -1543,6 +1558,7 @@ pub(crate) fn emit(
reg_g, reg_g,
addr, addr,
rex, rex,
0,
); );
} }
}; };
@@ -1701,7 +1717,9 @@ pub(crate) fn emit(
} }
RegMem::Mem { addr } => { RegMem::Mem { addr } => {
let addr = &addr.finalize(state, sink); let addr = &addr.finalize(state, sink);
emit_std_reg_mem(sink, state, info, prefix, opcode, length, reg_g, addr, rex); emit_std_reg_mem(
sink, state, info, prefix, opcode, length, reg_g, addr, rex, 0,
);
} }
} }
} }
@@ -1897,7 +1915,8 @@ pub(crate) fn emit(
!regs_swapped, !regs_swapped,
"No existing way to encode a mem argument in the ModRM r/m field." "No existing way to encode a mem argument in the ModRM r/m field."
); );
emit_std_reg_mem(sink, state, info, prefix, opcode, len, dst, addr, rex); // N.B.: bytes_at_end == 1, because of the `imm` byte below.
emit_std_reg_mem(sink, state, info, prefix, opcode, len, dst, addr, rex, 1);
} }
} }
sink.put1(*imm); sink.put1(*imm);
@@ -1940,6 +1959,7 @@ pub(crate) fn emit(
src, src,
dst, dst,
RexFlags::clear_w(), RexFlags::clear_w(),
0,
); );
} }
@@ -1993,7 +2013,7 @@ pub(crate) fn emit(
} }
RegMem::Mem { addr } => { RegMem::Mem { addr } => {
let addr = &addr.finalize(state, sink); let addr = &addr.finalize(state, sink);
emit_std_reg_mem(sink, state, info, prefix, opcode, 2, reg_g, addr, rex); emit_std_reg_mem(sink, state, info, prefix, opcode, 2, reg_g, addr, rex, 0);
} }
} }
} }
@@ -2016,7 +2036,7 @@ pub(crate) fn emit(
} }
RegMem::Mem { addr } => { RegMem::Mem { addr } => {
let addr = &addr.finalize(state, sink); let addr = &addr.finalize(state, sink);
emit_std_reg_mem(sink, state, info, prefix, opcode, len, dst, addr, rex); emit_std_reg_mem(sink, state, info, prefix, opcode, len, dst, addr, rex, 0);
} }
} }
} }
@@ -2633,6 +2653,7 @@ pub(crate) fn emit(
replacement, replacement,
&amode, &amode,
rex, rex,
0,
); );
} }

14
cranelift/codegen/src/isa/x64/lower/isle.rs
View File

@@ -78,10 +78,11 @@ where
return imm.to_reg_mem_imm(); return imm.to_reg_mem_imm();
} }
// Generate constants fresh at each use to minimize long-range // A load from the constant pool is better than a
// rematerialization into a register, because it reduces
// register pressure. // register pressure.
let ty = self.value_type(val); let vcode_constant = self.emit_u64_le_const(c);
return RegMemImm::reg(generated_code::constructor_imm(self, ty, c).unwrap()); return RegMemImm::mem(SyntheticAmode::ConstantOffset(vcode_constant));
} }
if let InputSourceInst::UniqueUse(src_insn, 0) = inputs.inst { if let InputSourceInst::UniqueUse(src_insn, 0) = inputs.inst {
@@ -99,10 +100,11 @@ where
let inputs = self.lower_ctx.get_value_as_source_or_const(val); let inputs = self.lower_ctx.get_value_as_source_or_const(val);
if let Some(c) = inputs.constant { if let Some(c) = inputs.constant {
// Generate constants fresh at each use to minimize long-range // A load from the constant pool is better than a
// rematerialization into a register, because it reduces
// register pressure. // register pressure.
let ty = self.value_type(val); let vcode_constant = self.emit_u64_le_const(c);
return RegMem::reg(generated_code::constructor_imm(self, ty, c).unwrap()); return RegMem::mem(SyntheticAmode::ConstantOffset(vcode_constant));
} }
if let InputSourceInst::UniqueUse(src_insn, 0) = inputs.inst { if let InputSourceInst::UniqueUse(src_insn, 0) = inputs.inst {

4
cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest
View File

@@ -1,4 +1,4 @@
src/clif.isle 443b34b797fc8ace src/clif.isle 443b34b797fc8ace
src/prelude.isle 97c4b6eebbab9f05 src/prelude.isle e6c91b0115343ab9
src/isa/x64/inst.isle a7f86254b89a7136 src/isa/x64/inst.isle 833710d359126637
src/isa/x64/lower.isle 4c567e9157f84afb src/isa/x64/lower.isle 4c567e9157f84afb

651
cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs generated
View File

File diff suppressed because it is too large Load Diff

6
cranelift/codegen/src/machinst/isle.rs
View File

@@ -539,6 +539,12 @@ macro_rules! isle_prelude_methods {
let offset: i32 = offset.into(); let offset: i32 = offset.into();
offset as u32 offset as u32
} }
#[inline]
fn emit_u64_le_const(&mut self, value: u64) -> VCodeConstant {
let data = VCodeConstantData::U64(value.to_le_bytes());
self.lower_ctx.use_constant(data)
}
}; };
} }

25
cranelift/codegen/src/machinst/vcode.rs
View File

@@ -31,6 +31,7 @@ use regalloc2::{
use alloc::boxed::Box; use alloc::boxed::Box;
use alloc::vec::Vec; use alloc::vec::Vec;
use cranelift_entity::{entity_impl, Keys, PrimaryMap}; use cranelift_entity::{entity_impl, Keys, PrimaryMap};
use std::collections::hash_map::Entry;
use std::collections::HashMap; use std::collections::HashMap;
use std::fmt; use std::fmt;
@@ -1288,6 +1289,7 @@ pub struct VCodeConstants {
constants: PrimaryMap<VCodeConstant, VCodeConstantData>, constants: PrimaryMap<VCodeConstant, VCodeConstantData>,
pool_uses: HashMap<Constant, VCodeConstant>, pool_uses: HashMap<Constant, VCodeConstant>,
well_known_uses: HashMap<*const [u8], VCodeConstant>, well_known_uses: HashMap<*const [u8], VCodeConstant>,
u64s: HashMap<[u8; 8], VCodeConstant>,
} }
impl VCodeConstants { impl VCodeConstants {
/// Initialize the structure with the expected number of constants. /// Initialize the structure with the expected number of constants.
@@ -1296,6 +1298,7 @@ impl VCodeConstants {
constants: PrimaryMap::with_capacity(expected_num_constants), constants: PrimaryMap::with_capacity(expected_num_constants),
pool_uses: HashMap::with_capacity(expected_num_constants), pool_uses: HashMap::with_capacity(expected_num_constants),
well_known_uses: HashMap::new(), well_known_uses: HashMap::new(),
u64s: HashMap::new(),
} }
} }
@@ -1315,16 +1318,23 @@ impl VCodeConstants {
Some(&vcode_constant) => vcode_constant, Some(&vcode_constant) => vcode_constant,
}, },
VCodeConstantData::WellKnown(data_ref) => { VCodeConstantData::WellKnown(data_ref) => {
match self.well_known_uses.get(&(data_ref as *const [u8])) { match self.well_known_uses.entry(data_ref as *const [u8]) {
None => { Entry::Vacant(v) => {
let vcode_constant = self.constants.push(data); let vcode_constant = self.constants.push(data);
self.well_known_uses v.insert(vcode_constant);
.insert(data_ref as *const [u8], vcode_constant);
vcode_constant vcode_constant
} }
Some(&vcode_constant) => vcode_constant, Entry::Occupied(o) => *o.get(),
} }
} }
VCodeConstantData::U64(value) => match self.u64s.entry(value) {
Entry::Vacant(v) => {
let vcode_constant = self.constants.push(data);
v.insert(vcode_constant);
vcode_constant
}
Entry::Occupied(o) => *o.get(),
},
} }
} }
@@ -1361,6 +1371,10 @@ pub enum VCodeConstantData {
/// A constant value generated during lowering; the value may depend on the instruction context /// A constant value generated during lowering; the value may depend on the instruction context
/// which makes it difficult to de-duplicate--if possible, use other variants. /// which makes it difficult to de-duplicate--if possible, use other variants.
Generated(ConstantData), Generated(ConstantData),
/// A constant of at most 64 bits. These are deduplicated as
/// well. Stored as a fixed-size array of `u8` so that we do not
/// encounter endianness problems when cross-compiling.
U64([u8; 8]),
} }
impl VCodeConstantData { impl VCodeConstantData {
/// Retrieve the constant data as a byte slice. /// Retrieve the constant data as a byte slice.
@@ -1368,6 +1382,7 @@ impl VCodeConstantData {
match self { match self {
VCodeConstantData::Pool(_, d) | VCodeConstantData::Generated(d) => d.as_slice(), VCodeConstantData::Pool(_, d) | VCodeConstantData::Generated(d) => d.as_slice(),
VCodeConstantData::WellKnown(d) => d, VCodeConstantData::WellKnown(d) => d,
VCodeConstantData::U64(value) => &value[..],
} }
} }

10
cranelift/codegen/src/prelude.isle
View File

@@ -442,6 +442,16 @@
(decl emit (MInst) Unit) (decl emit (MInst) Unit)
(extern constructor emit emit) (extern constructor emit emit)
;; Constant pool emission.
(type VCodeConstant (primitive VCodeConstant))
;; Add a u64 little-endian constant to the in-memory constant pool and
;; return a VCodeConstant index that refers to it. This is
;; side-effecting but idempotent (constants are deduplicated).
(decl emit_u64_le_const (u64) VCodeConstant)
(extern constructor emit_u64_le_const emit_u64_le_const)
;;;; Helpers for Side-Effectful Instructions Without Results ;;;;;;;;;;;;;;;;;;; ;;;; Helpers for Side-Effectful Instructions Without Results ;;;;;;;;;;;;;;;;;;;
(type SideEffectNoResult (enum (type SideEffectNoResult (enum

35
cranelift/filetests/filetests/isa/x64/immediates.clif Normal file
View File

@@ -0,0 +1,35 @@
test compile precise-output
target x86_64
function %f0(i64, i64) {
block0(v0: i64, v1: i64):
v2 = iconst.i64 0xffff_eeee_dddd_cccc
v3 = iadd.i64 v0, v2
store v3, v1
v4 = isub.i64 v0, v2
store v4, v1
v5 = band.i64 v0, v2
store v5, v1
v6 = bor.i64 v0, v2
store v6, v1
return
}
; pushq %rbp
; movq %rsp, %rbp
; block0:
; movq %rdi, %r11
; addq %r11, const(VCodeConstant(0)), %r11
; movq %r11, 0(%rsi)
; movq %rdi, %rax
; subq %rax, const(VCodeConstant(0)), %rax
; movq %rax, 0(%rsi)
; movq %rdi, %rax
; andq %rax, const(VCodeConstant(0)), %rax
; movq %rax, 0(%rsi)
; orq %rdi, const(VCodeConstant(0)), %rdi
; movq %rdi, 0(%rsi)
; movq %rbp, %rsp
; popq %rbp
; ret