T0b1/wasmtime
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

x64: use constant pool for u64 constants rather than movabs. (#4088)

Browse Source 
* Allow emitting u64 constants into constant pool.

* Use constant pool for constants on x64 that do not fit in a simm32 and are needed as a RegMem or RegMemImm.

* Fix rip-relative addressing bug in pinsrd emission.
This commit is contained in:
Chris Fallin
2022-05-10 09:21:05 -07:00
committed by GitHub
parent d3087487ea
commit eb435f3057
16 changed files with 483 additions and 383 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
cranelift/codegen/src/isa/x64/encoding/rex.rs
View File

@@ -284,6 +284,7 @@ pub(crate) fn emit_std_enc_mem(
enc_g: u8,
mem_e: &Amode,
rex: RexFlags,
bytes_at_end: u8,
) {
// General comment for this function: the registers in `mem_e` must be
// 64-bit integer registers, because they are part of an address
@@ -413,7 +414,14 @@ pub(crate) fn emit_std_enc_mem(
let offset = sink.cur_offset();
sink.use_label_at_offset(offset, *target, LabelUse::JmpRel32);
sink.put4(0);
// N.B.: some instructions (XmmRmRImm format for example)
// have bytes *after* the RIP-relative offset. The
// addressed location is relative to the end of the
// instruction, but the relocation is nominally relative
// to the end of the u32 field. So, to compensate for
// this, we emit a negative extra offset in the u32 field
// initially, and the relocation will add to it.
sink.put4(-(bytes_at_end as i32) as u32);
}
}
}
@@ -466,6 +474,7 @@ pub(crate) fn emit_std_reg_mem(
reg_g: Reg,
mem_e: &Amode,
rex: RexFlags,
bytes_at_end: u8,
) {
let enc_g = reg_enc(reg_g);
emit_std_enc_mem(
@@ -478,6 +487,7 @@ pub(crate) fn emit_std_reg_mem(
enc_g,
mem_e,
rex,
bytes_at_end,
);
}

2
cranelift/codegen/src/isa/x64/inst.isle
View File

@@ -497,8 +497,6 @@
Size32
Size64))
(type VCodeConstant (primitive VCodeConstant))
(type FenceKind extern
(enum MFence
LFence

37
cranelift/codegen/src/isa/x64/inst/emit.rs
View File

@@ -184,6 +184,7 @@ pub(crate) fn emit(
reg_g,
&amode,
rex,
0,
);
}
@@ -246,6 +247,7 @@ pub(crate) fn emit(
reg_g,
&amode,
rex,
0,
);
}
@@ -299,6 +301,7 @@ pub(crate) fn emit(
enc_g,
&src1_dst,
RexFlags::from(*size),
0,
);
}
@@ -343,6 +346,7 @@ pub(crate) fn emit(
dst,
&amode,
rex_flags,
0,
);
}
}
@@ -437,6 +441,7 @@ pub(crate) fn emit(
subopcode,
&amode,
RexFlags::from(*size),
0,
);
}
}
@@ -475,7 +480,7 @@ pub(crate) fn emit(
RegMem::Mem { addr: src } => {
let amode = src.finalize(state, sink).with_allocs(allocs);
emit_std_enc_mem(
sink, state, info, prefix, 0xF7, 1, subopcode, &amode, rex_flags,
sink, state, info, prefix, 0xF7, 1, subopcode, &amode, rex_flags, 0,
);
}
}
@@ -751,6 +756,7 @@ pub(crate) fn emit(
dst,
src,
rex_flags,
0,
)
}
}
@@ -770,6 +776,7 @@ pub(crate) fn emit(
dst,
src,
RexFlags::set_w(),
0,
)
}
@@ -787,6 +794,7 @@ pub(crate) fn emit(
dst,
&amode,
RexFlags::set_w(),
0,
);
}
@@ -849,6 +857,7 @@ pub(crate) fn emit(
dst,
src,
rex_flags,
0,
)
}
}
@@ -877,7 +886,7 @@ pub(crate) fn emit(
// 16-bit: MOV r16, r/m16 is 66 (REX.W==0) 89 /r
// 32-bit: MOV r32, r/m32 is (REX.W==0) 89 /r
// 64-bit: MOV r64, r/m64 is (REX.W==1) 89 /r
emit_std_reg_mem(sink, state, info, prefix, opcode, 1, src, dst, rex);
emit_std_reg_mem(sink, state, info, prefix, opcode, 1, src, dst, rex, 0);
}
Inst::ShiftR {
@@ -997,6 +1006,7 @@ pub(crate) fn emit(
dst,
addr,
rex,
0,
);
}
RegMemImm::Imm { .. } => unreachable!(),
@@ -1052,7 +1062,7 @@ pub(crate) fn emit(
(OperandSize::Size8, false) => 0x84,
(_, false) => 0x85,
};
emit_std_reg_mem(sink, state, info, prefix, opcode, 1, reg_g, addr, rex);
emit_std_reg_mem(sink, state, info, prefix, opcode, 1, reg_g, addr, rex, 0);
}
RegMemImm::Imm { simm32 } => {
@@ -1126,7 +1136,9 @@ pub(crate) fn emit(
}
RegMem::Mem { addr } => {
let addr = &addr.finalize(state, sink).with_allocs(allocs);
emit_std_reg_mem(sink, state, info, prefix, opcode, 2, dst, addr, rex_flags);
emit_std_reg_mem(
sink, state, info, prefix, opcode, 2, dst, addr, rex_flags, 0,
);
}
}
}
@@ -1190,6 +1202,7 @@ pub(crate) fn emit(
6, /*subopcode*/
addr,
RexFlags::clear_w(),
0,
);
}
@@ -1266,6 +1279,7 @@ pub(crate) fn emit(
2, /*subopcode*/
addr,
RexFlags::clear_w(),
0,
);
}
}
@@ -1367,6 +1381,7 @@ pub(crate) fn emit(
4, /*subopcode*/
addr,
RexFlags::clear_w(),
0,
);
}
}
@@ -1543,6 +1558,7 @@ pub(crate) fn emit(
reg_g,
addr,
rex,
0,
);
}
};
@@ -1701,7 +1717,9 @@ pub(crate) fn emit(
}
RegMem::Mem { addr } => {
let addr = &addr.finalize(state, sink);
emit_std_reg_mem(sink, state, info, prefix, opcode, length, reg_g, addr, rex);
emit_std_reg_mem(
sink, state, info, prefix, opcode, length, reg_g, addr, rex, 0,
);
}
}
}
@@ -1897,7 +1915,8 @@ pub(crate) fn emit(
!regs_swapped,
"No existing way to encode a mem argument in the ModRM r/m field."
);
emit_std_reg_mem(sink, state, info, prefix, opcode, len, dst, addr, rex);
// N.B.: bytes_at_end == 1, because of the `imm` byte below.
emit_std_reg_mem(sink, state, info, prefix, opcode, len, dst, addr, rex, 1);
}
}
sink.put1(*imm);
@@ -1940,6 +1959,7 @@ pub(crate) fn emit(
src,
dst,
RexFlags::clear_w(),
0,
);
}
@@ -1993,7 +2013,7 @@ pub(crate) fn emit(
}
RegMem::Mem { addr } => {
let addr = &addr.finalize(state, sink);
emit_std_reg_mem(sink, state, info, prefix, opcode, 2, reg_g, addr, rex);
emit_std_reg_mem(sink, state, info, prefix, opcode, 2, reg_g, addr, rex, 0);
}
}
}
@@ -2016,7 +2036,7 @@ pub(crate) fn emit(
}
RegMem::Mem { addr } => {
let addr = &addr.finalize(state, sink);
emit_std_reg_mem(sink, state, info, prefix, opcode, len, dst, addr, rex);
emit_std_reg_mem(sink, state, info, prefix, opcode, len, dst, addr, rex, 0);
}
}
}
@@ -2633,6 +2653,7 @@ pub(crate) fn emit(
replacement,
&amode,
rex,
0,
);
}

14
cranelift/codegen/src/isa/x64/lower/isle.rs
View File

@@ -78,10 +78,11 @@ where
return imm.to_reg_mem_imm();
}
// Generate constants fresh at each use to minimize long-range
// A load from the constant pool is better than a
// rematerialization into a register, because it reduces
// register pressure.
let ty = self.value_type(val);
return RegMemImm::reg(generated_code::constructor_imm(self, ty, c).unwrap());
let vcode_constant = self.emit_u64_le_const(c);
return RegMemImm::mem(SyntheticAmode::ConstantOffset(vcode_constant));
}
if let InputSourceInst::UniqueUse(src_insn, 0) = inputs.inst {
@@ -99,10 +100,11 @@ where
let inputs = self.lower_ctx.get_value_as_source_or_const(val);
if let Some(c) = inputs.constant {
// Generate constants fresh at each use to minimize long-range
// A load from the constant pool is better than a
// rematerialization into a register, because it reduces
// register pressure.
let ty = self.value_type(val);
return RegMem::reg(generated_code::constructor_imm(self, ty, c).unwrap());
let vcode_constant = self.emit_u64_le_const(c);
return RegMem::mem(SyntheticAmode::ConstantOffset(vcode_constant));
}
if let InputSourceInst::UniqueUse(src_insn, 0) = inputs.inst {

4
cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest
View File

@@ -1,4 +1,4 @@
src/clif.isle 443b34b797fc8ace
src/prelude.isle 97c4b6eebbab9f05
src/isa/x64/inst.isle a7f86254b89a7136
src/prelude.isle e6c91b0115343ab9
src/isa/x64/inst.isle 833710d359126637
src/isa/x64/lower.isle 4c567e9157f84afb

651
cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs generated
View File

File diff suppressed because it is too large Load Diff