T0b1/wasmtime
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[machinst x64]: lower remaining lane operations--any_true, all_true, splat

Browse Source
This commit is contained in:
Andrew Brown
2020-09-29 14:42:53 -07:00
parent 4565582f02
commit 50b9399006
4 changed files with 165 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
cranelift/codegen/src/isa/x64/inst/args.rs
View File

@@ -459,6 +459,7 @@ pub enum SseOpcode {
Psubd,
Psubq,
Psubw,
Ptest,
Pxor,
Rcpss,
Roundss,
@@ -606,6 +607,7 @@ impl SseOpcode {
| SseOpcode::Pminuw
| SseOpcode::Pminud
| SseOpcode::Pmulld
| SseOpcode::Ptest
| SseOpcode::Roundss
| SseOpcode::Roundsd => SSE41,
@@ -734,6 +736,7 @@ impl fmt::Debug for SseOpcode {
SseOpcode::Psubd => "psubd",
SseOpcode::Psubq => "psubq",
SseOpcode::Psubw => "psubw",
SseOpcode::Ptest => "ptest",
SseOpcode::Pxor => "pxor",
SseOpcode::Rcpss => "rcpss",
SseOpcode::Roundss => "roundss",

16
cranelift/codegen/src/isa/x64/inst/emit.rs
View File

@@ -2003,6 +2003,11 @@ pub(crate) fn emit(
sink.bind_label(constant_end_label);
}
Inst::XmmFakeDef { .. } => {
// This instruction format only exists to declare a register as a `def`; no code is
// emitted.
}
Inst::Xmm_Mov_R_M {
op,
src,
@@ -2087,19 +2092,20 @@ pub(crate) fn emit(
Inst::XMM_Cmp_RM_R { op, src, dst } => {
let rex = RexFlags::clear_w();
let (prefix, opcode) = match op {
SseOpcode::Ucomisd => (LegacyPrefixes::_66, 0x0F2E),
SseOpcode::Ucomiss => (LegacyPrefixes::None, 0x0F2E),
let (prefix, opcode, len) = match op {
SseOpcode::Ptest => (LegacyPrefixes::_66, 0x0F3817, 3),
SseOpcode::Ucomisd => (LegacyPrefixes::_66, 0x0F2E, 2),
SseOpcode::Ucomiss => (LegacyPrefixes::None, 0x0F2E, 2),
_ => unimplemented!("Emit xmm cmp rm r"),
};
match src {
RegMem::Reg { reg } => {
emit_std_reg_reg(sink, prefix, opcode, 2, *dst, *reg, rex);
emit_std_reg_reg(sink, prefix, opcode, len, *dst, *reg, rex);
}
RegMem::Mem { addr } => {
let addr = &addr.finalize(state);
emit_std_reg_mem(sink, prefix, opcode, 2, *dst, addr, rex);
emit_std_reg_mem(sink, prefix, opcode, len, *dst, addr, rex);
}
}
}

19
cranelift/codegen/src/isa/x64/inst/mod.rs
View File

@@ -342,6 +342,10 @@ pub enum Inst {
is64: bool,
},
/// Provides a way to tell the register allocator that the upcoming sequence of instructions
/// will overwrite `dst` so it should be considered as a `def`; use with care.
XmmFakeDef { dst: Writable<Reg> },
// =====================================
// Control flow instructions.
/// Direct call: call simm32.
@@ -640,6 +644,11 @@ impl Inst {
Inst::XMM_RM_R { op, src, dst }
}
pub(crate) fn xmm_fake_def(dst: Writable<Reg>) -> Self {
debug_assert!(dst.to_reg().get_class() == RegClass::V128);
Inst::XmmFakeDef { dst }
}
pub(crate) fn xmm_mov_r_m(
op: SseOpcode,
src: Reg,
@@ -1324,6 +1333,12 @@ impl ShowWithRRU for Inst {
dst.show_rru(mb_rru),
),
Inst::XmmFakeDef { dst } => format!(
"{} {}",
ljustify("fake_def".into()),
dst.show_rru(mb_rru),
),
Inst::XmmLoadConstSeq { val, dst, .. } => {
format!("load_const ${:?}, {}", val, dst.show_rru(mb_rru),)
}
@@ -1754,6 +1769,7 @@ fn x64_get_regs(inst: &Inst, collector: &mut RegUsageCollector) {
collector.add_mod(*dst);
}
}
Inst::XmmFakeDef { dst } => collector.add_def(*dst),
Inst::XmmLoadConstSeq { dst, .. } => collector.add_def(*dst),
Inst::XmmMinMaxSeq { lhs, rhs_dst, .. } => {
collector.add_use(*lhs);
@@ -2088,6 +2104,9 @@ fn x64_map_regs<RUM: RegUsageMapper>(inst: &mut Inst, mapper: &RUM) {
src.map_uses(mapper);
map_mod(mapper, dst);
}
Inst::XmmFakeDef { ref mut dst, .. } => {
map_def(mapper, dst);
}
Inst::XmmLoadConstSeq { ref mut dst, .. } => {
map_def(mapper, dst);
}

132
cranelift/codegen/src/isa/x64/lower.rs
View File

@@ -2945,6 +2945,138 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
}
}
Opcode::Splat => {
let ty = ty.unwrap();
assert_eq!(ty.bits(), 128);
let src_ty = ctx.input_ty(insn, 0);
assert!(src_ty.bits() < 128);
let src = input_to_reg_mem(ctx, inputs[0]);
let dst = get_output_reg(ctx, outputs[0]);
fn emit_insert_lane<C: LowerCtx<I = Inst>>(
ctx: &mut C,
src: RegMem,
dst: Writable<Reg>,
lane: u8,
ty: Type,
) {
if !ty.is_float() {
let (sse_op, is64) = match ty.lane_bits() {
8 => (SseOpcode::Pinsrb, false),
16 => (SseOpcode::Pinsrw, false),
32 => (SseOpcode::Pinsrd, false),
64 => (SseOpcode::Pinsrd, true),
_ => panic!("Unable to insertlane for lane size: {}", ty.lane_bits()),
};
ctx.emit(Inst::xmm_rm_r_imm(sse_op, src, dst, lane, is64));
} else if ty == types::F32 {
let sse_op = SseOpcode::Insertps;
// Insert 32-bits from replacement (at index 00, bits 7:8) to vector (lane
// shifted into bits 5:6).
let lane = 0b00_00_00_00 | lane << 4;
ctx.emit(Inst::xmm_rm_r_imm(sse_op, src, dst, lane, false));
} else if ty == types::F64 {
let sse_op = match lane {
// Move the lowest quadword in replacement to vector without changing
// the upper bits.
0 => SseOpcode::Movsd,
// Move the low 64 bits of replacement vector to the high 64 bits of the
// vector.
1 => SseOpcode::Movlhps,
_ => unreachable!(),
};
// Here we use the `xmm_rm_r` encoding because it correctly tells the register
// allocator how we are using `dst`: we are using `dst` as a `mod` whereas other
// encoding formats like `xmm_unary_rm_r` treat it as a `def`.
ctx.emit(Inst::xmm_rm_r(sse_op, src, dst));
}
};
// We know that splat will overwrite all of the lanes of `dst` but it takes several
// instructions to do so. Because of the multiple instructions, there is no good way to
// declare `dst` a `def` except with the following pseudo-instruction.
ctx.emit(Inst::xmm_fake_def(dst));
match ty.lane_bits() {
8 => {
emit_insert_lane(ctx, src, dst, 0, ty.lane_type());
// Initialize a register with all 0s.
let tmp = ctx.alloc_tmp(RegClass::V128, ty);
ctx.emit(Inst::xmm_rm_r(SseOpcode::Pxor, RegMem::from(tmp), tmp));
// Shuffle the lowest byte lane to all other lanes.
ctx.emit(Inst::xmm_rm_r(SseOpcode::Pshufb, RegMem::from(tmp), dst))
}
16 => {
emit_insert_lane(ctx, src.clone(), dst, 0, ty.lane_type());
emit_insert_lane(ctx, src, dst, 1, ty.lane_type());
// Shuffle the lowest two lanes to all other lanes.
ctx.emit(Inst::xmm_rm_r_imm(
SseOpcode::Pshufd,
RegMem::from(dst),
dst,
0,
false,
))
}
32 => {
emit_insert_lane(ctx, src, dst, 0, ty.lane_type());
// Shuffle the lowest lane to all other lanes.
ctx.emit(Inst::xmm_rm_r_imm(
SseOpcode::Pshufd,
RegMem::from(dst),
dst,
0,
false,
))
}
64 => {
emit_insert_lane(ctx, src.clone(), dst, 0, ty.lane_type());
emit_insert_lane(ctx, src, dst, 1, ty.lane_type());
}
_ => panic!("Invalid type to splat: {}", ty),
}
}
Opcode::VanyTrue => {
let dst = get_output_reg(ctx, outputs[0]);
let src_ty = ctx.input_ty(insn, 0);
assert_eq!(src_ty.bits(), 128);
let src = put_input_in_reg(ctx, inputs[0]);
// Set the ZF if the result is all zeroes.
ctx.emit(Inst::xmm_cmp_rm_r(SseOpcode::Ptest, RegMem::reg(src), src));
// If the ZF is not set, place a 1 in `dst`.
ctx.emit(Inst::setcc(CC::NZ, dst));
}
Opcode::VallTrue => {
let ty = ty.unwrap();
let dst = get_output_reg(ctx, outputs[0]);
let src_ty = ctx.input_ty(insn, 0);
assert_eq!(src_ty.bits(), 128);
let src = input_to_reg_mem(ctx, inputs[0]);
let eq = |ty: Type| match ty.lane_bits() {
8 => SseOpcode::Pcmpeqb,
16 => SseOpcode::Pcmpeqw,
32 => SseOpcode::Pcmpeqd,
64 => SseOpcode::Pcmpeqq,
_ => panic!("Unable to find an instruction for {} for type: {}", op, ty),
};
// Initialize a register with all 0s.
let tmp = ctx.alloc_tmp(RegClass::V128, ty);
ctx.emit(Inst::xmm_rm_r(SseOpcode::Pxor, RegMem::from(tmp), tmp));
// Compare to see what lanes are filled with all 1s.
ctx.emit(Inst::xmm_rm_r(eq(src_ty), src, tmp));
// Set the ZF if the result is all zeroes.
ctx.emit(Inst::xmm_cmp_rm_r(
SseOpcode::Ptest,
RegMem::from(tmp),
tmp.to_reg(),
));
// If the ZF is set, place a 1 in `dst`.
ctx.emit(Inst::setcc(CC::Z, dst));
}
Opcode::IaddImm
| Opcode::ImulImm
| Opcode::UdivImm