T0b1/wasmtime
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #3690 from fitzgen/a-bunch-more-isle

Browse Source 
cranelift: Port a bunch more lowerings to ISLE on x64
This commit is contained in:
Nick Fitzgerald
2022-01-13 18:08:31 -08:00
committed by GitHub
parent 7d943f68c5 a052285340
commit df37074218
12 changed files with 846 additions and 910 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest
View File

@@ -1,4 +1,4 @@
src/clif.isle f176ef3bba99365
src/prelude.isle 7b911d3b894ae17
src/prelude.isle 22dd5ff133398960
src/isa/aarch64/inst.isle 5fa80451697b084f
src/isa/aarch64/lower.isle 2d2e1e076a0c8a23

39
cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.rs generated
View File

@@ -24,6 +24,7 @@ pub trait Context {
fn u8_and(&mut self, arg0: u8, arg1: u8) -> u8;
fn value_reg(&mut self, arg0: Reg) -> ValueRegs;
fn value_regs(&mut self, arg0: Reg, arg1: Reg) -> ValueRegs;
fn value_regs_invalid(&mut self) -> ValueRegs;
fn temp_writable_reg(&mut self, arg0: Type) -> WritableReg;
fn invalid_reg(&mut self) -> Reg;
fn put_in_reg(&mut self, arg0: Value) -> Reg;
@@ -92,13 +93,19 @@ pub trait Context {
fn rotr_opposite_amount(&mut self, arg0: Type, arg1: ImmShift) -> ImmShift;
}
/// Internal type ProducesFlags: defined at src/prelude.isle line 277.
/// Internal type SideEffectNoResult: defined at src/prelude.isle line 279.
#[derive(Clone, Debug)]
pub enum SideEffectNoResult {
Inst { inst: MInst },
}
/// Internal type ProducesFlags: defined at src/prelude.isle line 292.
#[derive(Clone, Debug)]
pub enum ProducesFlags {
ProducesFlags { inst: MInst, result: Reg },
}
/// Internal type ConsumesFlags: defined at src/prelude.isle line 280.
/// Internal type ConsumesFlags: defined at src/prelude.isle line 295.
#[derive(Clone, Debug)]
pub enum ConsumesFlags {
ConsumesFlags { inst: MInst, result: Reg },
@@ -978,7 +985,7 @@ pub enum AtomicRMWOp {
// Generated as internal constructor for term temp_reg.
pub fn constructor_temp_reg<C: Context>(ctx: &mut C, arg0: Type) -> Option<Reg> {
let pattern0_0 = arg0;
// Rule at src/prelude.isle line 66.
// Rule at src/prelude.isle line 70.
let expr0_0 = C::temp_writable_reg(ctx, pattern0_0);
let expr1_0 = C::writable_reg_to_reg(ctx, expr0_0);
return Some(expr1_0);
@@ -987,13 +994,31 @@ pub fn constructor_temp_reg<C: Context>(ctx: &mut C, arg0: Type) -> Option<Reg>
// Generated as internal constructor for term lo_reg.
pub fn constructor_lo_reg<C: Context>(ctx: &mut C, arg0: Value) -> Option<Reg> {
let pattern0_0 = arg0;
// Rule at src/prelude.isle line 101.
// Rule at src/prelude.isle line 105.
let expr0_0 = C::put_in_regs(ctx, pattern0_0);
let expr1_0: usize = 0;
let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0);
return Some(expr2_0);
}
// Generated as internal constructor for term value_regs_none.
pub fn constructor_value_regs_none<C: Context>(
ctx: &mut C,
arg0: &SideEffectNoResult,
) -> Option<ValueRegs> {
let pattern0_0 = arg0;
if let &SideEffectNoResult::Inst {
inst: ref pattern1_0,
} = pattern0_0
{
// Rule at src/prelude.isle line 284.
let expr0_0 = C::emit(ctx, &pattern1_0);
let expr1_0 = C::value_regs_invalid(ctx);
return Some(expr1_0);
}
return None;
}
// Generated as internal constructor for term with_flags.
pub fn constructor_with_flags<C: Context>(
ctx: &mut C,
@@ -1012,7 +1037,7 @@ pub fn constructor_with_flags<C: Context>(
result: pattern3_1,
} = pattern2_0
{
// Rule at src/prelude.isle line 290.
// Rule at src/prelude.isle line 305.
let expr0_0 = C::emit(ctx, &pattern1_0);
let expr1_0 = C::emit(ctx, &pattern3_0);
let expr2_0 = C::value_regs(ctx, pattern1_1, pattern3_1);
@@ -1040,7 +1065,7 @@ pub fn constructor_with_flags_1<C: Context>(
result: pattern3_1,
} = pattern2_0
{
// Rule at src/prelude.isle line 298.
// Rule at src/prelude.isle line 313.
let expr0_0 = C::emit(ctx, &pattern1_0);
let expr1_0 = C::emit(ctx, &pattern3_0);
return Some(pattern3_1);
@@ -1074,7 +1099,7 @@ pub fn constructor_with_flags_2<C: Context>(
result: pattern5_1,
} = pattern4_0
{
// Rule at src/prelude.isle line 308.
// Rule at src/prelude.isle line 323.
let expr0_0 = C::emit(ctx, &pattern1_0);
let expr1_0 = C::emit(ctx, &pattern5_0);
let expr2_0 = C::emit(ctx, &pattern3_0);

38
cranelift/codegen/src/isa/x64/inst.isle
View File

@@ -4,6 +4,7 @@
(type MInst extern
(enum (Nop (len u8))
(Ud2 (trap_code TrapCode))
(AluRmiR (size OperandSize)
(op AluRmiROpcode)
(src1 Reg)
@@ -72,9 +73,11 @@
(Not (size OperandSize)
(src Reg)
(dst WritableReg))
(LoadEffectiveAddress (addr SyntheticAmode)
(Neg (size OperandSize)
(src Reg)
(dst WritableReg))
))
(LoadEffectiveAddress (addr SyntheticAmode)
(dst WritableReg))))
(type OperandSize extern
(enum Size8
@@ -697,8 +700,8 @@
;;
;; Use `m_` prefix (short for "mach inst") to disambiguate with the ISLE-builtin
;; `and` operator.
(decl m_and (Type Reg RegMemImm) Reg)
(rule (m_and ty src1 src2)
(decl x64_and (Type Reg RegMemImm) Reg)
(rule (x64_and ty src1 src2)
(alu_rmi_r ty
(AluRmiROpcode.And)
src1
@@ -724,7 +727,7 @@
(decl imm (Type u64) Reg)
;; Integer immediates.
(rule (imm ty simm64)
(rule (imm (fits_in_64 ty) simm64)
(let ((dst WritableReg (temp_writable_reg ty))
(size OperandSize (operand_size_of_type_32_64 ty))
(_ Unit (emit (MInst.Imm size simm64 dst))))
@@ -749,7 +752,7 @@
(writable_reg_to_reg dst)))
;; Special case for integer zero immediates: turn them into an `xor r, r`.
(rule (imm ty 0)
(rule (imm (fits_in_64 ty) 0)
(let ((wr WritableReg (temp_writable_reg ty))
(r Reg (writable_reg_to_reg wr))
(size OperandSize (operand_size_of_type_32_64 ty))
@@ -807,10 +810,16 @@
;; Helper for creating `rotl` instructions (prefixed with "m_", short for "mach
;; inst", to disambiguate this from clif's `rotl`).
(decl m_rotl (Type Reg Imm8Reg) Reg)
(rule (m_rotl ty src1 src2)
(decl x64_rotl (Type Reg Imm8Reg) Reg)
(rule (x64_rotl ty src1 src2)
(shift_r ty (ShiftKind.RotateLeft) src1 src2))
;; Helper for creating `rotr` instructions (prefixed with "m_", short for "mach
;; inst", to disambiguate this from clif's `rotr`).
(decl x64_rotr (Type Reg Imm8Reg) Reg)
(rule (x64_rotr ty src1 src2)
(shift_r ty (ShiftKind.RotateRight) src1 src2))
;; Helper for creating `shl` instructions.
(decl shl (Type Reg Imm8Reg) Reg)
(rule (shl ty src1 src2)
@@ -1423,8 +1432,21 @@
(_ Unit (emit (MInst.Not size src dst))))
(writable_reg_to_reg dst)))
;; Helper for creating `neg` instructions.
(decl neg (Type Reg) Reg)
(rule (neg ty src)
(let ((dst WritableReg (temp_writable_reg ty))
(size OperandSize (operand_size_of_type_32_64 ty))
(_ Unit (emit (MInst.Neg size src dst))))
(writable_reg_to_reg dst)))
(decl lea (SyntheticAmode) Reg)
(rule (lea addr)
(let ((dst WritableReg (temp_writable_reg $I64))
(_ Unit (emit (MInst.LoadEffectiveAddress addr dst))))
(writable_reg_to_reg dst)))
;; Helper for creating `ud2` instructions.
(decl ud2 (TrapCode) SideEffectNoResult)
(rule (ud2 code)
(SideEffectNoResult.Inst (MInst.Ud2 code)))

11
cranelift/codegen/src/isa/x64/inst/emit_tests.rs
View File

@@ -16,6 +16,17 @@ use super::*;
use crate::isa::x64;
use alloc::vec::Vec;
impl Inst {
fn neg(size: OperandSize, src: Writable<Reg>) -> Inst {
debug_assert_eq!(src.to_reg().get_class(), RegClass::I64);
Inst::Neg {
size,
src: src.to_reg(),
dst: src,
}
}
}
#[test]
fn test_x64_emit() {
let rax = regs::rax();

9
cranelift/codegen/src/isa/x64/inst/mod.rs
View File

@@ -668,15 +668,6 @@ impl Inst {
}
}
pub(crate) fn neg(size: OperandSize, src: Writable<Reg>) -> Inst {
debug_assert_eq!(src.to_reg().get_class(), RegClass::I64);
Inst::Neg {
size,
src: src.to_reg(),
dst: src,
}
}
pub(crate) fn div(size: OperandSize, signed: bool, divisor: RegMem) -> Inst {
divisor.assert_regclass_is(RegClass::I64);
Inst::Div {

99
cranelift/codegen/src/isa/x64/lower.isle
View File

@@ -326,7 +326,7 @@
;; And two registers.
(rule (lower (has_type (fits_in_64 ty) (band x y)))
(value_reg (m_and ty
(value_reg (x64_and ty
(put_in_reg x)
(RegMemImm.Reg (put_in_reg y)))))
@@ -334,13 +334,13 @@
(rule (lower (has_type (fits_in_64 ty)
(band x (sinkable_load y))))
(value_reg (m_and ty
(value_reg (x64_and ty
(put_in_reg x)
(sink_load y))))
(rule (lower (has_type (fits_in_64 ty)
(band (sinkable_load x) y)))
(value_reg (m_and ty
(value_reg (x64_and ty
(put_in_reg y)
(sink_load x))))
@@ -348,13 +348,13 @@
(rule (lower (has_type (fits_in_64 ty)
(band x (simm32_from_value y))))
(value_reg (m_and ty
(value_reg (x64_and ty
(put_in_reg x)
y)))
(rule (lower (has_type (fits_in_64 ty)
(band (simm32_from_value x) y)))
(value_reg (m_and ty
(value_reg (x64_and ty
(put_in_reg y)
x)))
@@ -378,8 +378,8 @@
(y_regs ValueRegs (put_in_regs y))
(y_lo Reg (value_regs_get y_regs 0))
(y_hi Reg (value_regs_get y_regs 1)))
(value_regs (m_and $I64 x_lo (RegMemImm.Reg y_lo))
(m_and $I64 x_hi (RegMemImm.Reg y_hi)))))
(value_regs (x64_and $I64 x_lo (RegMemImm.Reg y_lo))
(x64_and $I64 x_hi (RegMemImm.Reg y_hi)))))
(rule (lower (has_type $B128 (band x y)))
;; Booleans are always `0` or `1`, so we only need to do the `and` on the
@@ -389,7 +389,7 @@
(x_lo Reg (value_regs_get x_regs 0))
(x_hi Reg (value_regs_get x_regs 1))
(y_lo Reg (lo_reg y)))
(value_regs (m_and $I64 x_lo (RegMemImm.Reg y_lo))
(value_regs (x64_and $I64 x_lo (RegMemImm.Reg y_lo))
x_hi)))
;;;; Rules for `bor` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -832,11 +832,11 @@
(rule (lower (has_type (ty_8_or_16 ty) (rotl src amt)))
(let ((amt_ Reg (extend_to_reg amt $I32 (ExtendKind.Zero))))
(value_reg (m_rotl ty (put_in_reg src) (Imm8Reg.Reg amt_)))))
(value_reg (x64_rotl ty (put_in_reg src) (Imm8Reg.Reg amt_)))))
(rule (lower (has_type (ty_8_or_16 ty)
(rotl src (u64_from_iconst amt))))
(value_reg (m_rotl ty
(value_reg (x64_rotl ty
(put_in_reg src)
(const_to_type_masked_imm8 amt ty))))
@@ -847,11 +847,11 @@
;; NB: Only the low bits of `amt` matter since we logically mask the
;; shift amount to the value's bit width.
(let ((amt_ Reg (lo_reg amt)))
(value_reg (m_rotl ty (put_in_reg src) (Imm8Reg.Reg amt_)))))
(value_reg (x64_rotl ty (put_in_reg src) (Imm8Reg.Reg amt_)))))
(rule (lower (has_type (ty_32_or_64 ty)
(rotl src (u64_from_iconst amt))))
(value_reg (m_rotl ty
(value_reg (x64_rotl ty
(put_in_reg src)
(const_to_type_masked_imm8 amt ty))))
@@ -865,6 +865,71 @@
(or_i128 (shl_i128 src_ amt_)
(shr_i128 src_ (sub $I64 (imm $I64 128) (RegMemImm.Reg amt_))))))
;;;; Rules for `rotr` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; `i16` and `i8`: we need to extend the shift amount, or mask the
;; constant.
(rule (lower (has_type (ty_8_or_16 ty) (rotr src amt)))
(let ((amt_ Reg (extend_to_reg amt $I32 (ExtendKind.Zero))))
(value_reg (x64_rotr ty (put_in_reg src) (Imm8Reg.Reg amt_)))))
(rule (lower (has_type (ty_8_or_16 ty)
(rotr src (u64_from_iconst amt))))
(value_reg (x64_rotr ty
(put_in_reg src)
(const_to_type_masked_imm8 amt ty))))
;; `i64` and `i32`: we can rely on x86's rotate-amount masking since
;; we operate on the whole register.
(rule (lower (has_type (ty_32_or_64 ty) (rotr src amt)))
;; NB: Only the low bits of `amt` matter since we logically mask the
;; shift amount to the value's bit width.
(let ((amt_ Reg (lo_reg amt)))
(value_reg (x64_rotr ty (put_in_reg src) (Imm8Reg.Reg amt_)))))
(rule (lower (has_type (ty_32_or_64 ty)
(rotr src (u64_from_iconst amt))))
(value_reg (x64_rotr ty
(put_in_reg src)
(const_to_type_masked_imm8 amt ty))))
;; `i128`.
(rule (lower (has_type $I128 (rotr src amt)))
(let ((src_ ValueRegs (put_in_regs src))
;; NB: Only the low bits of `amt` matter since we logically mask the
;; rotation amount to the value's bit width.
(amt_ Reg (lo_reg amt)))
(or_i128 (shr_i128 src_ amt_)
(shl_i128 src_ (sub $I64 (imm $I64 128) (RegMemImm.Reg amt_))))))
;;;; Rules for `ineg` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; `i64` and smaller.
(rule (lower (has_type (fits_in_64 ty) (ineg x)))
(value_reg (neg ty (put_in_reg x))))
;; SSE.
(rule (lower (has_type $I8X16 (ineg x)))
(value_reg (psubb (imm $I8X16 0)
(put_in_reg_mem x))))
(rule (lower (has_type $I16X8 (ineg x)))
(value_reg (psubw (imm $I16X8 0)
(put_in_reg_mem x))))
(rule (lower (has_type $I32X4 (ineg x)))
(value_reg (psubd (imm $I32X4 0)
(put_in_reg_mem x))))
(rule (lower (has_type $I64X2 (ineg x)))
(value_reg (psubq (imm $I64X2 0)
(put_in_reg_mem x))))
;;;; Rules for `avg_round` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(rule (lower (has_type (multi_lane 8 16)
@@ -1367,3 +1432,13 @@
(rule (lower (has_type $I32X4 (umin x y)))
(value_reg (pminud (put_in_reg x) (put_in_reg_mem y))))
;;;; Rules for `trap` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(rule (lower (trap code))
(value_regs_none (ud2 code)))
;;;; Rules for `resumable_trap` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(rule (lower (resumable_trap code))
(value_regs_none (ud2 code)))

457
cranelift/codegen/src/isa/x64/lower.rs
View File

@@ -877,316 +877,6 @@ fn emit_bitrev<C: LowerCtx<I = Inst>>(ctx: &mut C, src: Reg, dst: Writable<Reg>,
ctx.emit(Inst::gen_move(dst, tmp0.to_reg(), types::I64));
}
fn emit_shl_i128<C: LowerCtx<I = Inst>>(
ctx: &mut C,
src: ValueRegs<Reg>,
dst: ValueRegs<Writable<Reg>>,
amt_src: Reg,
) {
let src_lo = src.regs()[0];
let src_hi = src.regs()[1];
let dst_lo = dst.regs()[0];
let dst_hi = dst.regs()[1];
// mov tmp1, src_lo
// shl tmp1, amt_src
// mov tmp2, src_hi
// shl tmp2, amt_src
// mov amt, 64
// sub amt, amt_src
// mov tmp3, src_lo
// shr tmp3, amt
// xor dst_lo, dst_lo
// test amt_src, 127
// cmovz tmp3, dst_lo
// or tmp3, tmp2
// mov amt, amt_src
// and amt, 64
// cmovz dst_hi, tmp3
// cmovz dst_lo, tmp1
// cmovnz dst_hi, tmp1
let tmp1 = ctx.alloc_tmp(types::I64).only_reg().unwrap();
let tmp2 = ctx.alloc_tmp(types::I64).only_reg().unwrap();
let tmp3 = ctx.alloc_tmp(types::I64).only_reg().unwrap();
let amt = ctx.alloc_tmp(types::I64).only_reg().unwrap();
ctx.emit(Inst::gen_move(tmp1, src_lo, types::I64));
ctx.emit(Inst::gen_move(
Writable::from_reg(regs::rcx()),
amt_src,
types::I64,
));
ctx.emit(Inst::shift_r(
OperandSize::Size64,
ShiftKind::ShiftLeft,
None,
tmp1,
));
ctx.emit(Inst::gen_move(tmp2, src_hi, types::I64));
ctx.emit(Inst::gen_move(
Writable::from_reg(regs::rcx()),
amt_src,
types::I64,
));
ctx.emit(Inst::shift_r(
OperandSize::Size64,
ShiftKind::ShiftLeft,
None,
tmp2,
));
ctx.emit(Inst::imm(OperandSize::Size64, 64, amt));
ctx.emit(Inst::alu_rmi_r(
OperandSize::Size64,
AluRmiROpcode::Sub,
RegMemImm::reg(amt_src),
amt,
));
ctx.emit(Inst::gen_move(tmp3, src_lo, types::I64));
ctx.emit(Inst::gen_move(
Writable::from_reg(regs::rcx()),
amt.to_reg(),
types::I64,
));
ctx.emit(Inst::shift_r(
OperandSize::Size64,
ShiftKind::ShiftRightLogical,
None,
tmp3,
));
ctx.emit(Inst::alu_rmi_r(
OperandSize::Size64,
AluRmiROpcode::Xor,
RegMemImm::reg(dst_lo.to_reg()),
dst_lo,
));
ctx.emit(Inst::test_rmi_r(
OperandSize::Size64,
RegMemImm::imm(127),
amt_src,
));
ctx.emit(Inst::cmove(
OperandSize::Size64,
CC::Z,
RegMem::reg(dst_lo.to_reg()),
tmp3,
));
ctx.emit(Inst::alu_rmi_r(
OperandSize::Size64,
AluRmiROpcode::Or,
RegMemImm::reg(tmp2.to_reg()),
tmp3,
));
// This isn't semantically necessary, but it keeps the
// register allocator happy, because it cannot otherwise
// infer that cmovz + cmovnz always defines dst_hi.
ctx.emit(Inst::alu_rmi_r(
OperandSize::Size64,
AluRmiROpcode::Xor,
RegMemImm::reg(dst_hi.to_reg()),
dst_hi,
));
ctx.emit(Inst::gen_move(amt, amt_src, types::I64));
ctx.emit(Inst::alu_rmi_r(
OperandSize::Size64,
AluRmiROpcode::And,
RegMemImm::imm(64),
amt,
));
ctx.emit(Inst::cmove(
OperandSize::Size64,
CC::Z,
RegMem::reg(tmp3.to_reg()),
dst_hi,
));
ctx.emit(Inst::cmove(
OperandSize::Size64,
CC::Z,
RegMem::reg(tmp1.to_reg()),
dst_lo,
));
ctx.emit(Inst::cmove(
OperandSize::Size64,
CC::NZ,
RegMem::reg(tmp1.to_reg()),
dst_hi,
));
}
fn emit_shr_i128<C: LowerCtx<I = Inst>>(
ctx: &mut C,
src: ValueRegs<Reg>,
dst: ValueRegs<Writable<Reg>>,
amt_src: Reg,
is_signed: bool,
) {
let src_lo = src.regs()[0];
let src_hi = src.regs()[1];
let dst_lo = dst.regs()[0];
let dst_hi = dst.regs()[1];
// mov tmp1, src_hi
// {u,s}shr tmp1, amt_src
// mov tmp2, src_lo
// ushr tmp2, amt_src
// mov amt, 64
// sub amt, amt_src
// mov tmp3, src_hi
// shl tmp3, amt
// xor dst_lo, dst_lo
// test amt_src, 127
// cmovz tmp3, dst_lo
// or tmp3, tmp2
// if is_signed:
// mov dst_hi, src_hi
// sshr dst_hi, 63 // get the sign bit
// else:
// xor dst_hi, dst_hi
// mov amt, amt_src
// and amt, 64
// cmovz dst_hi, tmp1
// cmovz dst_lo, tmp3
// cmovnz dst_lo, tmp1
let tmp1 = ctx.alloc_tmp(types::I64).only_reg().unwrap();
let tmp2 = ctx.alloc_tmp(types::I64).only_reg().unwrap();
let tmp3 = ctx.alloc_tmp(types::I64).only_reg().unwrap();
let amt = ctx.alloc_tmp(types::I64).only_reg().unwrap();
let shift_kind = if is_signed {
ShiftKind::ShiftRightArithmetic
} else {
ShiftKind::ShiftRightLogical
};
ctx.emit(Inst::gen_move(tmp1, src_hi, types::I64));
ctx.emit(Inst::gen_move(
Writable::from_reg(regs::rcx()),
amt_src,
types::I64,
));
ctx.emit(Inst::shift_r(OperandSize::Size64, shift_kind, None, tmp1));
ctx.emit(Inst::gen_move(tmp2, src_lo, types::I64));
ctx.emit(Inst::gen_move(
Writable::from_reg(regs::rcx()),
amt_src,
types::I64,
));
// N.B.: right-shift of *lower* half is *always* unsigned (its MSB is not a sign bit).
ctx.emit(Inst::shift_r(
OperandSize::Size64,
ShiftKind::ShiftRightLogical,
None,
tmp2,
));
ctx.emit(Inst::imm(OperandSize::Size64, 64, amt));
ctx.emit(Inst::alu_rmi_r(
OperandSize::Size64,
AluRmiROpcode::Sub,
RegMemImm::reg(amt_src),
amt,
));
ctx.emit(Inst::gen_move(tmp3, src_hi, types::I64));
ctx.emit(Inst::gen_move(
Writable::from_reg(regs::rcx()),
amt.to_reg(),
types::I64,
));
ctx.emit(Inst::shift_r(
OperandSize::Size64,
ShiftKind::ShiftLeft,
None,
tmp3,
));
ctx.emit(Inst::alu_rmi_r(
OperandSize::Size64,
AluRmiROpcode::Xor,
RegMemImm::reg(dst_lo.to_reg()),
dst_lo,
));
ctx.emit(Inst::test_rmi_r(
OperandSize::Size64,
RegMemImm::imm(127),
amt_src,
));
ctx.emit(Inst::cmove(
OperandSize::Size64,
CC::Z,
RegMem::reg(dst_lo.to_reg()),
tmp3,
));
ctx.emit(Inst::alu_rmi_r(
OperandSize::Size64,
AluRmiROpcode::Or,
RegMemImm::reg(tmp2.to_reg()),
tmp3,
));
if is_signed {
ctx.emit(Inst::gen_move(dst_hi, src_hi, types::I64));
ctx.emit(Inst::shift_r(
OperandSize::Size64,
ShiftKind::ShiftRightArithmetic,
Some(63),
dst_hi,
));
} else {
ctx.emit(Inst::alu_rmi_r(
OperandSize::Size64,
AluRmiROpcode::Xor,
RegMemImm::reg(dst_hi.to_reg()),
dst_hi,
));
}
// This isn't semantically necessary, but it keeps the
// register allocator happy, because it cannot otherwise
// infer that cmovz + cmovnz always defines dst_lo.
ctx.emit(Inst::alu_rmi_r(
OperandSize::Size64,
AluRmiROpcode::Xor,
RegMemImm::reg(dst_lo.to_reg()),
dst_lo,
));
ctx.emit(Inst::gen_move(amt, amt_src, types::I64));
ctx.emit(Inst::alu_rmi_r(
OperandSize::Size64,
AluRmiROpcode::And,
RegMemImm::imm(64),
amt,
));
ctx.emit(Inst::cmove(
OperandSize::Size64,
CC::Z,
RegMem::reg(tmp1.to_reg()),
dst_hi,
));
ctx.emit(Inst::cmove(
OperandSize::Size64,
CC::Z,
RegMem::reg(tmp3.to_reg()),
dst_lo,
));
ctx.emit(Inst::cmove(
OperandSize::Size64,
CC::NZ,
RegMem::reg(tmp1.to_reg()),
dst_lo,
));
}
fn make_libcall_sig<C: LowerCtx<I = Inst>>(
ctx: &mut C,
insn: IRInst,
@@ -1541,142 +1231,12 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
| Opcode::Vselect
| Opcode::Ushr
| Opcode::Sshr
| Opcode::Ishl => implemented_in_isle(ctx),
Opcode::Rotl | Opcode::Rotr => {
let dst_ty = ctx.output_ty(insn, 0);
debug_assert_eq!(ctx.input_ty(insn, 0), dst_ty);
if !dst_ty.is_vector() && dst_ty.bits() <= 64 {
if op != Opcode::Rotr {
implemented_in_isle(ctx);
}
// Scalar shifts on x86 have various encodings:
// - shift by one bit, e.g. `SAL r/m8, 1` (not used here)
// - shift by an immediate amount, e.g. `SAL r/m8, imm8`
// - shift by a dynamic amount but only from the CL register, e.g. `SAL r/m8, CL`.
// This implementation uses the last two encoding methods.
let (size, lhs) = match dst_ty {
types::I8 | types::I16 => match op {
Opcode::Rotr => (
OperandSize::from_ty(dst_ty),
put_input_in_reg(ctx, inputs[0]),
),
_ => unreachable!(),
},
types::I32 | types::I64 => (
OperandSize::from_ty(dst_ty),
put_input_in_reg(ctx, inputs[0]),
),
_ => unreachable!("unhandled output type for shift/rotates: {}", dst_ty),
};
let (count, rhs) =
if let Some(cst) = ctx.get_input_as_source_or_const(insn, 1).constant {
// Mask count, according to Cranelift's semantics.
let cst = (cst as u8) & (dst_ty.bits() as u8 - 1);
(Some(cst), None)
} else {
// We can ignore upper registers if shift amount is multi-reg, because we
// are taking the shift amount mod 2^(lhs_width) anyway.
(None, Some(put_input_in_regs(ctx, inputs[1]).regs()[0]))
};
let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
let shift_kind = match op {
Opcode::Rotr => ShiftKind::RotateRight,
_ => unreachable!(),
};
let w_rcx = Writable::from_reg(regs::rcx());
ctx.emit(Inst::mov_r_r(OperandSize::Size64, lhs, dst));
if count.is_none() {
ctx.emit(Inst::mov_r_r(OperandSize::Size64, rhs.unwrap(), w_rcx));
}
ctx.emit(Inst::shift_r(size, shift_kind, count, dst));
} else if dst_ty == types::I128 {
let amt_src = put_input_in_regs(ctx, inputs[1]).regs()[0];
let src = put_input_in_regs(ctx, inputs[0]);
let dst = get_output_reg(ctx, outputs[0]);
match op {
Opcode::Rotr => {
// (mov tmp, src)
// (ushr.i128 tmp, amt)
// (mov dst, src)
// (shl.i128 dst, 128-amt)
// (or dst, tmp)
let tmp = ctx.alloc_tmp(types::I128);
emit_shr_i128(ctx, src, tmp, amt_src, /* is_signed = */ false);
let inv_amt = ctx.alloc_tmp(types::I64).only_reg().unwrap();
ctx.emit(Inst::imm(OperandSize::Size64, 128, inv_amt));
ctx.emit(Inst::alu_rmi_r(
OperandSize::Size64,
AluRmiROpcode::Sub,
RegMemImm::reg(amt_src),
inv_amt,
));
emit_shl_i128(ctx, src, dst, inv_amt.to_reg());
ctx.emit(Inst::alu_rmi_r(
OperandSize::Size64,
AluRmiROpcode::Or,
RegMemImm::reg(tmp.regs()[0].to_reg()),
dst.regs()[0],
));
ctx.emit(Inst::alu_rmi_r(
OperandSize::Size64,
AluRmiROpcode::Or,
RegMemImm::reg(tmp.regs()[1].to_reg()),
dst.regs()[1],
));
}
_ => unreachable!(),
}
} else {
implemented_in_isle(ctx);
}
}
Opcode::Ineg => {
let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
let ty = ty.unwrap();
if ty.is_vector() {
// Zero's out a register and then does a packed subtraction
// of the input from the register.
let src = input_to_reg_mem(ctx, inputs[0]);
let tmp = ctx.alloc_tmp(types::I32X4).only_reg().unwrap();
let subtract_opcode = match ty {
types::I8X16 => SseOpcode::Psubb,
types::I16X8 => SseOpcode::Psubw,
types::I32X4 => SseOpcode::Psubd,
types::I64X2 => SseOpcode::Psubq,
_ => panic!("Unsupported type for Ineg instruction, found {}", ty),
};
// Note we must zero out a tmp instead of using the destination register since
// the desitnation could be an alias for the source input register
ctx.emit(Inst::xmm_rm_r(
SseOpcode::Pxor,
RegMem::reg(tmp.to_reg()),
tmp,
));
ctx.emit(Inst::xmm_rm_r(subtract_opcode, src, tmp));
ctx.emit(Inst::xmm_unary_rm_r(
SseOpcode::Movapd,
RegMem::reg(tmp.to_reg()),
dst,
));
} else {
let src = put_input_in_reg(ctx, inputs[0]);
ctx.emit(Inst::gen_move(dst, src, ty));
ctx.emit(Inst::neg(OperandSize::from_ty(ty), dst));
}
}
| Opcode::Ishl
| Opcode::Rotl
| Opcode::Rotr
| Opcode::Ineg
| Opcode::Trap
| Opcode::ResumableTrap => implemented_in_isle(ctx),
Opcode::Clz => {
let orig_ty = ty.unwrap();
@@ -2811,11 +2371,6 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
ctx.emit(Inst::Hlt);
}
Opcode::Trap | Opcode::ResumableTrap => {
let trap_code = ctx.data(insn).trap_code().unwrap();
ctx.emit_safepoint(Inst::Ud2 { trap_code });
}
Opcode::Trapif | Opcode::Trapff => {
let trap_code = ctx.data(insn).trap_code().unwrap();

6
cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest
View File

@@ -1,4 +1,4 @@
src/clif.isle f176ef3bba99365
src/prelude.isle 7b911d3b894ae17
src/isa/x64/inst.isle 41304d8ef6f7d816
src/isa/x64/lower.isle 4689585f55f41438
src/prelude.isle 22dd5ff133398960
src/isa/x64/inst.isle 61004acbb1289816
src/isa/x64/lower.isle 82db7f7d47ac7809

761
cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs generated
View File

File diff suppressed because it is too large Load Diff

5
cranelift/codegen/src/machinst/isle.rs
View File

@@ -48,6 +48,11 @@ macro_rules! isle_prelude_methods {
ValueRegs::two(r1, r2)
}
#[inline]
fn value_regs_invalid(&mut self) -> ValueRegs {
ValueRegs::invalid()
}
#[inline]
fn temp_writable_reg(&mut self, ty: Type) -> WritableReg {
let value_regs = self.lower_ctx.alloc_tmp(ty);

15
cranelift/codegen/src/prelude.isle
View File

@@ -57,6 +57,10 @@
(decl value_regs (Reg Reg) ValueRegs)
(extern constructor value_regs value_regs)
;; Construct an empty `ValueRegs` containing only invalid register sentinels.
(decl value_regs_invalid () ValueRegs)
(extern constructor value_regs_invalid value_regs_invalid)
;; Get a temporary register for writing.
(decl temp_writable_reg (Type) WritableReg)
(extern constructor temp_writable_reg temp_writable_reg)
@@ -270,6 +274,17 @@
(extractor (u64_from_iconst x)
(def_inst (iconst (u64_from_imm64 x))))
;;;; Helpers for Side-Effectful Instructions Without Results ;;;;;;;;;;;;;;;;;;;
(type SideEffectNoResult (enum (Inst (inst MInst))))
;; Create an empty `ValueRegs`, but do emit the given side-effectful
;; instruction.
(decl value_regs_none (SideEffectNoResult) ValueRegs)
(rule (value_regs_none (SideEffectNoResult.Inst inst))
(let ((_ Unit (emit inst)))
(value_regs_invalid)))
;;;; Helpers for Working with Flags ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Newtype wrapper around `MInst` for instructions that are used for their

94
cranelift/filetests/filetests/isa/x64/i128.clif
View File

@@ -1270,56 +1270,52 @@ block0(v0: i128, v1: i128):
; Entry block: 0
; Block 0:
; (original IR block: block0)
; (instruction range: 0 .. 50)
; (instruction range: 0 .. 46)
; Inst 0: pushq %rbp
; Inst 1: movq %rsp, %rbp
; Inst 2: movq %rsi, %rax
; Inst 3: movq %rax, %r9
; Inst 4: movq %rdx, %rcx
; Inst 5: shrq %cl, %r9
; Inst 6: movq %rdi, %rsi
; Inst 7: movq %rdx, %rcx
; Inst 8: shrq %cl, %rsi
; Inst 9: movl $64, %ecx
; Inst 10: subq %rdx, %rcx
; Inst 11: movq %rax, %r10
; Inst 12: shlq %cl, %r10
; Inst 13: xorq %rcx, %rcx
; Inst 14: testq $127, %rdx
; Inst 15: cmovzq %rcx, %r10
; Inst 16: orq %rsi, %r10
; Inst 17: xorq %rsi, %rsi
; Inst 18: xorq %r8, %r8
; Inst 19: movq %rdx, %rcx
; Inst 20: andq $64, %rcx
; Inst 21: cmovzq %r9, %rsi
; Inst 22: cmovzq %r10, %r8
; Inst 23: cmovnzq %r9, %r8
; Inst 24: movl $128, %r9d
; Inst 25: subq %rdx, %r9
; Inst 26: movq %rdi, %rdx
; Inst 27: movq %r9, %rcx
; Inst 28: shlq %cl, %rdx
; Inst 29: movq %r9, %rcx
; Inst 30: shlq %cl, %rax
; Inst 31: movl $64, %ecx
; Inst 32: subq %r9, %rcx
; Inst 33: shrq %cl, %rdi
; Inst 34: xorq %rcx, %rcx
; Inst 35: testq $127, %r9
; Inst 36: cmovzq %rcx, %rdi
; Inst 37: orq %rax, %rdi
; Inst 38: xorq %rax, %rax
; Inst 39: andq $64, %r9
; Inst 40: cmovzq %rdi, %rax
; Inst 41: cmovzq %rdx, %rcx
; Inst 42: cmovnzq %rdx, %rax
; Inst 43: orq %r8, %rcx
; Inst 44: orq %rsi, %rax
; Inst 45: movq %rax, %rdx
; Inst 46: movq %rcx, %rax
; Inst 47: movq %rbp, %rsp
; Inst 48: popq %rbp
; Inst 49: ret
; Inst 2: movq %rdi, %rax
; Inst 3: movq %rdx, %rcx
; Inst 4: shrq %cl, %rax
; Inst 5: movq %rsi, %r8
; Inst 6: movq %rdx, %rcx
; Inst 7: shrq %cl, %r8
; Inst 8: movl $64, %ecx
; Inst 9: subq %rdx, %rcx
; Inst 10: movq %rsi, %r9
; Inst 11: shlq %cl, %r9
; Inst 12: xorq %rcx, %rcx
; Inst 13: testq $127, %rdx
; Inst 14: cmovzq %rcx, %r9
; Inst 15: movq %r9, %rcx
; Inst 16: orq %rax, %rcx
; Inst 17: xorq %rax, %rax
; Inst 18: testq $64, %rdx
; Inst 19: cmovzq %r8, %rax
; Inst 20: cmovzq %rcx, %r8
; Inst 21: movl $128, %r9d
; Inst 22: subq %rdx, %r9
; Inst 23: movq %rdi, %rdx
; Inst 24: movq %r9, %rcx
; Inst 25: shlq %cl, %rdx
; Inst 26: movq %r9, %rcx
; Inst 27: shlq %cl, %rsi
; Inst 28: movl $64, %ecx
; Inst 29: subq %r9, %rcx
; Inst 30: shrq %cl, %rdi
; Inst 31: xorq %rcx, %rcx
; Inst 32: testq $127, %r9
; Inst 33: cmovzq %rcx, %rdi
; Inst 34: orq %rsi, %rdi
; Inst 35: testq $64, %r9
; Inst 36: movq %rdx, %rsi
; Inst 37: cmovzq %rdi, %rsi
; Inst 38: cmovzq %rdx, %rcx
; Inst 39: orq %rcx, %r8
; Inst 40: orq %rsi, %rax
; Inst 41: movq %rax, %rdx
; Inst 42: movq %r8, %rax
; Inst 43: movq %rbp, %rsp
; Inst 44: popq %rbp
; Inst 45: ret
; }}