x64: use constant pool for u64 constants rather than movabs. (#4088)

* Allow emitting u64 constants into constant pool.

* Use constant pool for constants on x64 that do not fit in a simm32 and are needed as a RegMem or RegMemImm.

* Fix rip-relative addressing bug in pinsrd emission.
This commit is contained in:
Chris Fallin
2022-05-10 09:21:05 -07:00
committed by GitHub
parent d3087487ea
commit eb435f3057
16 changed files with 483 additions and 383 deletions

View File

@@ -23,7 +23,7 @@ use crate::{
isa::aarch64::inst::args::{ShiftOp, ShiftOpShiftImm},
isa::aarch64::lower::{is_valid_atomic_transaction_ty, writable_xreg, xreg},
isa::unwind::UnwindInst,
machinst::{ty_bits, InsnOutput, LowerCtx},
machinst::{ty_bits, InsnOutput, LowerCtx, VCodeConstant, VCodeConstantData},
};
use std::boxed::Box;
use std::convert::TryFrom;

View File

@@ -1,4 +1,4 @@
src/clif.isle 443b34b797fc8ace
src/prelude.isle 97c4b6eebbab9f05
src/prelude.isle e6c91b0115343ab9
src/isa/aarch64/inst.isle 21a43af20be377d2
src/isa/aarch64/lower.isle 75ad8450963e3829

View File

@@ -90,6 +90,7 @@ pub trait Context {
fn def_inst(&mut self, arg0: Value) -> Option<Inst>;
fn offset32_to_u32(&mut self, arg0: Offset32) -> u32;
fn emit(&mut self, arg0: &MInst) -> Unit;
fn emit_u64_le_const(&mut self, arg0: u64) -> VCodeConstant;
fn trap_code_division_by_zero(&mut self) -> TrapCode;
fn trap_code_integer_overflow(&mut self) -> TrapCode;
fn trap_code_bad_conversion_to_integer(&mut self) -> TrapCode;
@@ -146,14 +147,14 @@ pub trait Context {
fn rotr_opposite_amount(&mut self, arg0: Type, arg1: ImmShift) -> ImmShift;
}
/// Internal type SideEffectNoResult: defined at src/prelude.isle line 447.
/// Internal type SideEffectNoResult: defined at src/prelude.isle line 457.
#[derive(Clone, Debug)]
pub enum SideEffectNoResult {
Inst { inst: MInst },
Inst2 { inst1: MInst, inst2: MInst },
}
/// Internal type ProducesFlags: defined at src/prelude.isle line 474.
/// Internal type ProducesFlags: defined at src/prelude.isle line 484.
#[derive(Clone, Debug)]
pub enum ProducesFlags {
ProducesFlagsSideEffect { inst: MInst },
@@ -161,7 +162,7 @@ pub enum ProducesFlags {
ProducesFlagsReturnsResultWithConsumer { inst: MInst, result: Reg },
}
/// Internal type ConsumesFlags: defined at src/prelude.isle line 485.
/// Internal type ConsumesFlags: defined at src/prelude.isle line 495.
#[derive(Clone, Debug)]
pub enum ConsumesFlags {
ConsumesFlagsReturnsResultWithProducer {
@@ -1102,7 +1103,7 @@ pub fn constructor_side_effect<C: Context>(
&SideEffectNoResult::Inst {
inst: ref pattern1_0,
} => {
// Rule at src/prelude.isle line 455.
// Rule at src/prelude.isle line 465.
let expr0_0 = C::emit(ctx, pattern1_0);
let expr1_0 = C::output_none(ctx);
return Some(expr1_0);
@@ -1111,7 +1112,7 @@ pub fn constructor_side_effect<C: Context>(
inst1: ref pattern1_0,
inst2: ref pattern1_1,
} => {
// Rule at src/prelude.isle line 458.
// Rule at src/prelude.isle line 468.
let expr0_0 = C::emit(ctx, pattern1_0);
let expr1_0 = C::emit(ctx, pattern1_1);
let expr2_0 = C::output_none(ctx);
@@ -1138,7 +1139,7 @@ pub fn constructor_side_effect_concat<C: Context>(
inst: ref pattern3_0,
} = pattern2_0
{
// Rule at src/prelude.isle line 464.
// Rule at src/prelude.isle line 474.
let expr0_0 = SideEffectNoResult::Inst2 {
inst1: pattern1_0.clone(),
inst2: pattern3_0.clone(),
@@ -1160,7 +1161,7 @@ pub fn constructor_produces_flags_get_reg<C: Context>(
result: pattern1_1,
} = pattern0_0
{
// Rule at src/prelude.isle line 501.
// Rule at src/prelude.isle line 511.
return Some(pattern1_1);
}
return None;
@@ -1177,7 +1178,7 @@ pub fn constructor_produces_flags_ignore<C: Context>(
inst: ref pattern1_0,
result: pattern1_1,
} => {
// Rule at src/prelude.isle line 506.
// Rule at src/prelude.isle line 516.
let expr0_0 = ProducesFlags::ProducesFlagsSideEffect {
inst: pattern1_0.clone(),
};
@@ -1187,7 +1188,7 @@ pub fn constructor_produces_flags_ignore<C: Context>(
inst: ref pattern1_0,
result: pattern1_1,
} => {
// Rule at src/prelude.isle line 508.
// Rule at src/prelude.isle line 518.
let expr0_0 = ProducesFlags::ProducesFlagsSideEffect {
inst: pattern1_0.clone(),
};
@@ -1216,7 +1217,7 @@ pub fn constructor_consumes_flags_concat<C: Context>(
result: pattern3_1,
} = pattern2_0
{
// Rule at src/prelude.isle line 515.
// Rule at src/prelude.isle line 525.
let expr0_0 = C::value_regs(ctx, pattern1_1, pattern3_1);
let expr1_0 = ConsumesFlags::ConsumesFlagsTwiceReturnsValueRegs {
inst1: pattern1_0.clone(),
@@ -1246,7 +1247,7 @@ pub fn constructor_with_flags<C: Context>(
inst: ref pattern3_0,
result: pattern3_1,
} => {
// Rule at src/prelude.isle line 540.
// Rule at src/prelude.isle line 550.
let expr0_0 = C::emit(ctx, pattern1_0);
let expr1_0 = C::emit(ctx, pattern3_0);
let expr2_0 = C::value_reg(ctx, pattern3_1);
@@ -1257,7 +1258,7 @@ pub fn constructor_with_flags<C: Context>(
inst2: ref pattern3_1,
result: pattern3_2,
} => {
// Rule at src/prelude.isle line 546.
// Rule at src/prelude.isle line 556.
let expr0_0 = C::emit(ctx, pattern1_0);
let expr1_0 = C::emit(ctx, pattern3_0);
let expr2_0 = C::emit(ctx, pattern3_1);
@@ -1270,7 +1271,7 @@ pub fn constructor_with_flags<C: Context>(
inst4: ref pattern3_3,
result: pattern3_4,
} => {
// Rule at src/prelude.isle line 558.
// Rule at src/prelude.isle line 568.
let expr0_0 = C::emit(ctx, pattern1_0);
let expr1_0 = C::emit(ctx, pattern3_0);
let expr2_0 = C::emit(ctx, pattern3_1);
@@ -1291,7 +1292,7 @@ pub fn constructor_with_flags<C: Context>(
result: pattern3_1,
} = pattern2_0
{
// Rule at src/prelude.isle line 534.
// Rule at src/prelude.isle line 544.
let expr0_0 = C::emit(ctx, pattern1_0);
let expr1_0 = C::emit(ctx, pattern3_0);
let expr2_0 = C::value_regs(ctx, pattern1_1, pattern3_1);
@@ -1311,7 +1312,7 @@ pub fn constructor_with_flags_reg<C: Context>(
) -> Option<Reg> {
let pattern0_0 = arg0;
let pattern1_0 = arg1;
// Rule at src/prelude.isle line 575.
// Rule at src/prelude.isle line 585.
let expr0_0 = constructor_with_flags(ctx, pattern0_0, pattern1_0)?;
let expr1_0: usize = 0;
let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0);

View File

@@ -17,7 +17,7 @@ use crate::{
StackSlot, TrapCode, Value, ValueList,
},
isa::unwind::UnwindInst,
machinst::{InsnOutput, LowerCtx},
machinst::{InsnOutput, LowerCtx, VCodeConstant, VCodeConstantData},
};
use std::boxed::Box;
use std::cell::Cell;

View File

@@ -1,4 +1,4 @@
src/clif.isle 443b34b797fc8ace
src/prelude.isle 97c4b6eebbab9f05
src/prelude.isle e6c91b0115343ab9
src/isa/s390x/inst.isle 36c2500563cdd4e6
src/isa/s390x/lower.isle e5c946ab8a265b77

View File

@@ -90,6 +90,7 @@ pub trait Context {
fn def_inst(&mut self, arg0: Value) -> Option<Inst>;
fn offset32_to_u32(&mut self, arg0: Offset32) -> u32;
fn emit(&mut self, arg0: &MInst) -> Unit;
fn emit_u64_le_const(&mut self, arg0: u64) -> VCodeConstant;
fn trap_code_division_by_zero(&mut self) -> TrapCode;
fn trap_code_integer_overflow(&mut self) -> TrapCode;
fn trap_code_bad_conversion_to_integer(&mut self) -> TrapCode;
@@ -165,14 +166,14 @@ pub trait Context {
fn same_reg(&mut self, arg0: WritableReg, arg1: Reg) -> Option<Reg>;
}
/// Internal type SideEffectNoResult: defined at src/prelude.isle line 447.
/// Internal type SideEffectNoResult: defined at src/prelude.isle line 457.
#[derive(Clone, Debug)]
pub enum SideEffectNoResult {
Inst { inst: MInst },
Inst2 { inst1: MInst, inst2: MInst },
}
/// Internal type ProducesFlags: defined at src/prelude.isle line 474.
/// Internal type ProducesFlags: defined at src/prelude.isle line 484.
#[derive(Clone, Debug)]
pub enum ProducesFlags {
ProducesFlagsSideEffect { inst: MInst },
@@ -180,7 +181,7 @@ pub enum ProducesFlags {
ProducesFlagsReturnsResultWithConsumer { inst: MInst, result: Reg },
}
/// Internal type ConsumesFlags: defined at src/prelude.isle line 485.
/// Internal type ConsumesFlags: defined at src/prelude.isle line 495.
#[derive(Clone, Debug)]
pub enum ConsumesFlags {
ConsumesFlagsReturnsResultWithProducer {
@@ -970,7 +971,7 @@ pub fn constructor_side_effect<C: Context>(
&SideEffectNoResult::Inst {
inst: ref pattern1_0,
} => {
// Rule at src/prelude.isle line 455.
// Rule at src/prelude.isle line 465.
let expr0_0 = C::emit(ctx, pattern1_0);
let expr1_0 = C::output_none(ctx);
return Some(expr1_0);
@@ -979,7 +980,7 @@ pub fn constructor_side_effect<C: Context>(
inst1: ref pattern1_0,
inst2: ref pattern1_1,
} => {
// Rule at src/prelude.isle line 458.
// Rule at src/prelude.isle line 468.
let expr0_0 = C::emit(ctx, pattern1_0);
let expr1_0 = C::emit(ctx, pattern1_1);
let expr2_0 = C::output_none(ctx);
@@ -1006,7 +1007,7 @@ pub fn constructor_side_effect_concat<C: Context>(
inst: ref pattern3_0,
} = pattern2_0
{
// Rule at src/prelude.isle line 464.
// Rule at src/prelude.isle line 474.
let expr0_0 = SideEffectNoResult::Inst2 {
inst1: pattern1_0.clone(),
inst2: pattern3_0.clone(),
@@ -1028,7 +1029,7 @@ pub fn constructor_produces_flags_get_reg<C: Context>(
result: pattern1_1,
} = pattern0_0
{
// Rule at src/prelude.isle line 501.
// Rule at src/prelude.isle line 511.
return Some(pattern1_1);
}
return None;
@@ -1045,7 +1046,7 @@ pub fn constructor_produces_flags_ignore<C: Context>(
inst: ref pattern1_0,
result: pattern1_1,
} => {
// Rule at src/prelude.isle line 506.
// Rule at src/prelude.isle line 516.
let expr0_0 = ProducesFlags::ProducesFlagsSideEffect {
inst: pattern1_0.clone(),
};
@@ -1055,7 +1056,7 @@ pub fn constructor_produces_flags_ignore<C: Context>(
inst: ref pattern1_0,
result: pattern1_1,
} => {
// Rule at src/prelude.isle line 508.
// Rule at src/prelude.isle line 518.
let expr0_0 = ProducesFlags::ProducesFlagsSideEffect {
inst: pattern1_0.clone(),
};
@@ -1084,7 +1085,7 @@ pub fn constructor_consumes_flags_concat<C: Context>(
result: pattern3_1,
} = pattern2_0
{
// Rule at src/prelude.isle line 515.
// Rule at src/prelude.isle line 525.
let expr0_0 = C::value_regs(ctx, pattern1_1, pattern3_1);
let expr1_0 = ConsumesFlags::ConsumesFlagsTwiceReturnsValueRegs {
inst1: pattern1_0.clone(),
@@ -1114,7 +1115,7 @@ pub fn constructor_with_flags<C: Context>(
inst: ref pattern3_0,
result: pattern3_1,
} => {
// Rule at src/prelude.isle line 540.
// Rule at src/prelude.isle line 550.
let expr0_0 = C::emit(ctx, pattern1_0);
let expr1_0 = C::emit(ctx, pattern3_0);
let expr2_0 = C::value_reg(ctx, pattern3_1);
@@ -1125,7 +1126,7 @@ pub fn constructor_with_flags<C: Context>(
inst2: ref pattern3_1,
result: pattern3_2,
} => {
// Rule at src/prelude.isle line 546.
// Rule at src/prelude.isle line 556.
let expr0_0 = C::emit(ctx, pattern1_0);
let expr1_0 = C::emit(ctx, pattern3_0);
let expr2_0 = C::emit(ctx, pattern3_1);
@@ -1138,7 +1139,7 @@ pub fn constructor_with_flags<C: Context>(
inst4: ref pattern3_3,
result: pattern3_4,
} => {
// Rule at src/prelude.isle line 558.
// Rule at src/prelude.isle line 568.
let expr0_0 = C::emit(ctx, pattern1_0);
let expr1_0 = C::emit(ctx, pattern3_0);
let expr2_0 = C::emit(ctx, pattern3_1);
@@ -1159,7 +1160,7 @@ pub fn constructor_with_flags<C: Context>(
result: pattern3_1,
} = pattern2_0
{
// Rule at src/prelude.isle line 534.
// Rule at src/prelude.isle line 544.
let expr0_0 = C::emit(ctx, pattern1_0);
let expr1_0 = C::emit(ctx, pattern3_0);
let expr2_0 = C::value_regs(ctx, pattern1_1, pattern3_1);
@@ -1179,7 +1180,7 @@ pub fn constructor_with_flags_reg<C: Context>(
) -> Option<Reg> {
let pattern0_0 = arg0;
let pattern1_0 = arg1;
// Rule at src/prelude.isle line 575.
// Rule at src/prelude.isle line 585.
let expr0_0 = constructor_with_flags(ctx, pattern0_0, pattern1_0)?;
let expr1_0: usize = 0;
let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0);

View File

@@ -284,6 +284,7 @@ pub(crate) fn emit_std_enc_mem(
enc_g: u8,
mem_e: &Amode,
rex: RexFlags,
bytes_at_end: u8,
) {
// General comment for this function: the registers in `mem_e` must be
// 64-bit integer registers, because they are part of an address
@@ -413,7 +414,14 @@ pub(crate) fn emit_std_enc_mem(
let offset = sink.cur_offset();
sink.use_label_at_offset(offset, *target, LabelUse::JmpRel32);
sink.put4(0);
// N.B.: some instructions (XmmRmRImm format for example)
// have bytes *after* the RIP-relative offset. The
// addressed location is relative to the end of the
// instruction, but the relocation is nominally relative
// to the end of the u32 field. So, to compensate for
// this, we emit a negative extra offset in the u32 field
// initially, and the relocation will add to it.
sink.put4(-(bytes_at_end as i32) as u32);
}
}
}
@@ -466,6 +474,7 @@ pub(crate) fn emit_std_reg_mem(
reg_g: Reg,
mem_e: &Amode,
rex: RexFlags,
bytes_at_end: u8,
) {
let enc_g = reg_enc(reg_g);
emit_std_enc_mem(
@@ -478,6 +487,7 @@ pub(crate) fn emit_std_reg_mem(
enc_g,
mem_e,
rex,
bytes_at_end,
);
}

View File

@@ -497,8 +497,6 @@
Size32
Size64))
(type VCodeConstant (primitive VCodeConstant))
(type FenceKind extern
(enum MFence
LFence

View File

@@ -184,6 +184,7 @@ pub(crate) fn emit(
reg_g,
&amode,
rex,
0,
);
}
@@ -246,6 +247,7 @@ pub(crate) fn emit(
reg_g,
&amode,
rex,
0,
);
}
@@ -299,6 +301,7 @@ pub(crate) fn emit(
enc_g,
&src1_dst,
RexFlags::from(*size),
0,
);
}
@@ -343,6 +346,7 @@ pub(crate) fn emit(
dst,
&amode,
rex_flags,
0,
);
}
}
@@ -437,6 +441,7 @@ pub(crate) fn emit(
subopcode,
&amode,
RexFlags::from(*size),
0,
);
}
}
@@ -475,7 +480,7 @@ pub(crate) fn emit(
RegMem::Mem { addr: src } => {
let amode = src.finalize(state, sink).with_allocs(allocs);
emit_std_enc_mem(
sink, state, info, prefix, 0xF7, 1, subopcode, &amode, rex_flags,
sink, state, info, prefix, 0xF7, 1, subopcode, &amode, rex_flags, 0,
);
}
}
@@ -751,6 +756,7 @@ pub(crate) fn emit(
dst,
src,
rex_flags,
0,
)
}
}
@@ -770,6 +776,7 @@ pub(crate) fn emit(
dst,
src,
RexFlags::set_w(),
0,
)
}
@@ -787,6 +794,7 @@ pub(crate) fn emit(
dst,
&amode,
RexFlags::set_w(),
0,
);
}
@@ -849,6 +857,7 @@ pub(crate) fn emit(
dst,
src,
rex_flags,
0,
)
}
}
@@ -877,7 +886,7 @@ pub(crate) fn emit(
// 16-bit: MOV r16, r/m16 is 66 (REX.W==0) 89 /r
// 32-bit: MOV r32, r/m32 is (REX.W==0) 89 /r
// 64-bit: MOV r64, r/m64 is (REX.W==1) 89 /r
emit_std_reg_mem(sink, state, info, prefix, opcode, 1, src, dst, rex);
emit_std_reg_mem(sink, state, info, prefix, opcode, 1, src, dst, rex, 0);
}
Inst::ShiftR {
@@ -997,6 +1006,7 @@ pub(crate) fn emit(
dst,
addr,
rex,
0,
);
}
RegMemImm::Imm { .. } => unreachable!(),
@@ -1052,7 +1062,7 @@ pub(crate) fn emit(
(OperandSize::Size8, false) => 0x84,
(_, false) => 0x85,
};
emit_std_reg_mem(sink, state, info, prefix, opcode, 1, reg_g, addr, rex);
emit_std_reg_mem(sink, state, info, prefix, opcode, 1, reg_g, addr, rex, 0);
}
RegMemImm::Imm { simm32 } => {
@@ -1126,7 +1136,9 @@ pub(crate) fn emit(
}
RegMem::Mem { addr } => {
let addr = &addr.finalize(state, sink).with_allocs(allocs);
emit_std_reg_mem(sink, state, info, prefix, opcode, 2, dst, addr, rex_flags);
emit_std_reg_mem(
sink, state, info, prefix, opcode, 2, dst, addr, rex_flags, 0,
);
}
}
}
@@ -1190,6 +1202,7 @@ pub(crate) fn emit(
6, /*subopcode*/
addr,
RexFlags::clear_w(),
0,
);
}
@@ -1266,6 +1279,7 @@ pub(crate) fn emit(
2, /*subopcode*/
addr,
RexFlags::clear_w(),
0,
);
}
}
@@ -1367,6 +1381,7 @@ pub(crate) fn emit(
4, /*subopcode*/
addr,
RexFlags::clear_w(),
0,
);
}
}
@@ -1543,6 +1558,7 @@ pub(crate) fn emit(
reg_g,
addr,
rex,
0,
);
}
};
@@ -1701,7 +1717,9 @@ pub(crate) fn emit(
}
RegMem::Mem { addr } => {
let addr = &addr.finalize(state, sink);
emit_std_reg_mem(sink, state, info, prefix, opcode, length, reg_g, addr, rex);
emit_std_reg_mem(
sink, state, info, prefix, opcode, length, reg_g, addr, rex, 0,
);
}
}
}
@@ -1897,7 +1915,8 @@ pub(crate) fn emit(
!regs_swapped,
"No existing way to encode a mem argument in the ModRM r/m field."
);
emit_std_reg_mem(sink, state, info, prefix, opcode, len, dst, addr, rex);
// N.B.: bytes_at_end == 1, because of the `imm` byte below.
emit_std_reg_mem(sink, state, info, prefix, opcode, len, dst, addr, rex, 1);
}
}
sink.put1(*imm);
@@ -1940,6 +1959,7 @@ pub(crate) fn emit(
src,
dst,
RexFlags::clear_w(),
0,
);
}
@@ -1993,7 +2013,7 @@ pub(crate) fn emit(
}
RegMem::Mem { addr } => {
let addr = &addr.finalize(state, sink);
emit_std_reg_mem(sink, state, info, prefix, opcode, 2, reg_g, addr, rex);
emit_std_reg_mem(sink, state, info, prefix, opcode, 2, reg_g, addr, rex, 0);
}
}
}
@@ -2016,7 +2036,7 @@ pub(crate) fn emit(
}
RegMem::Mem { addr } => {
let addr = &addr.finalize(state, sink);
emit_std_reg_mem(sink, state, info, prefix, opcode, len, dst, addr, rex);
emit_std_reg_mem(sink, state, info, prefix, opcode, len, dst, addr, rex, 0);
}
}
}
@@ -2633,6 +2653,7 @@ pub(crate) fn emit(
replacement,
&amode,
rex,
0,
);
}

View File

@@ -78,10 +78,11 @@ where
return imm.to_reg_mem_imm();
}
// Generate constants fresh at each use to minimize long-range
// A load from the constant pool is better than a
// rematerialization into a register, because it reduces
// register pressure.
let ty = self.value_type(val);
return RegMemImm::reg(generated_code::constructor_imm(self, ty, c).unwrap());
let vcode_constant = self.emit_u64_le_const(c);
return RegMemImm::mem(SyntheticAmode::ConstantOffset(vcode_constant));
}
if let InputSourceInst::UniqueUse(src_insn, 0) = inputs.inst {
@@ -99,10 +100,11 @@ where
let inputs = self.lower_ctx.get_value_as_source_or_const(val);
if let Some(c) = inputs.constant {
// Generate constants fresh at each use to minimize long-range
// A load from the constant pool is better than a
// rematerialization into a register, because it reduces
// register pressure.
let ty = self.value_type(val);
return RegMem::reg(generated_code::constructor_imm(self, ty, c).unwrap());
let vcode_constant = self.emit_u64_le_const(c);
return RegMem::mem(SyntheticAmode::ConstantOffset(vcode_constant));
}
if let InputSourceInst::UniqueUse(src_insn, 0) = inputs.inst {

View File

@@ -1,4 +1,4 @@
src/clif.isle 443b34b797fc8ace
src/prelude.isle 97c4b6eebbab9f05
src/isa/x64/inst.isle a7f86254b89a7136
src/prelude.isle e6c91b0115343ab9
src/isa/x64/inst.isle 833710d359126637
src/isa/x64/lower.isle 4c567e9157f84afb

File diff suppressed because it is too large Load Diff