T0b1/wasmtime
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

ISLE: port extend/reduce opcodes on x64. (#3849)

Browse Source
This commit is contained in:
Chris Fallin
2022-02-28 11:49:28 -08:00
committed by GitHub
parent dd9c86a58c
commit 90a081a731
10 changed files with 515 additions and 190 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest
View File

@@ -1,4 +1,4 @@
src/clif.isle 9ea75a6f790b5c03
src/prelude.isle 6b0160bfcac86902
src/prelude.isle b2bc986bcbbbb77
src/isa/aarch64/inst.isle 3678d0a37bdb4cff
src/isa/aarch64/lower.isle 90accbfcadaea46d

30
cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.rs generated
View File

@@ -54,6 +54,8 @@ pub trait Context {
fn fits_in_64(&mut self, arg0: Type) -> Option<Type>;
fn ty_32_or_64(&mut self, arg0: Type) -> Option<Type>;
fn ty_8_or_16(&mut self, arg0: Type) -> Option<Type>;
fn ty_int_bool_64(&mut self, arg0: Type) -> Option<Type>;
fn ty_int_bool_128(&mut self, arg0: Type) -> Option<Type>;
fn vec128(&mut self, arg0: Type) -> Option<Type>;
fn not_i64x2(&mut self, arg0: Type) -> Option<()>;
fn value_list_slice(&mut self, arg0: ValueList) -> ValueSlice;
@@ -116,13 +118,13 @@ pub trait Context {
fn rotr_opposite_amount(&mut self, arg0: Type, arg1: ImmShift) -> ImmShift;
}
/// Internal type SideEffectNoResult: defined at src/prelude.isle line 385.
/// Internal type SideEffectNoResult: defined at src/prelude.isle line 393.
#[derive(Clone, Debug)]
pub enum SideEffectNoResult {
Inst { inst: MInst },
}
/// Internal type ProducesFlags: defined at src/prelude.isle line 407.
/// Internal type ProducesFlags: defined at src/prelude.isle line 415.
#[derive(Clone, Debug)]
pub enum ProducesFlags {
ProducesFlagsSideEffect { inst: MInst },
@@ -130,7 +132,7 @@ pub enum ProducesFlags {
ProducesFlagsReturnsResultWithConsumer { inst: MInst, result: Reg },
}
/// Internal type ConsumesFlags: defined at src/prelude.isle line 418.
/// Internal type ConsumesFlags: defined at src/prelude.isle line 426.
#[derive(Clone, Debug)]
pub enum ConsumesFlags {
ConsumesFlagsReturnsResultWithProducer {
@@ -1063,7 +1065,7 @@ pub fn constructor_side_effect<C: Context>(
inst: ref pattern1_0,
} = pattern0_0
{
// Rule at src/prelude.isle line 390.
// Rule at src/prelude.isle line 398.
let expr0_0 = C::emit(ctx, pattern1_0);
let expr1_0 = C::output_none(ctx);
return Some(expr1_0);
@@ -1081,7 +1083,7 @@ pub fn constructor_safepoint<C: Context>(
inst: ref pattern1_0,
} = pattern0_0
{
// Rule at src/prelude.isle line 396.
// Rule at src/prelude.isle line 404.
let expr0_0 = C::emit_safepoint(ctx, pattern1_0);
let expr1_0 = C::output_none(ctx);
return Some(expr1_0);
@@ -1100,7 +1102,7 @@ pub fn constructor_produces_flags_get_reg<C: Context>(
result: pattern1_1,
} = pattern0_0
{
// Rule at src/prelude.isle line 434.
// Rule at src/prelude.isle line 442.
return Some(pattern1_1);
}
return None;
@@ -1117,7 +1119,7 @@ pub fn constructor_produces_flags_ignore<C: Context>(
inst: ref pattern1_0,
result: pattern1_1,
} => {
// Rule at src/prelude.isle line 439.
// Rule at src/prelude.isle line 447.
let expr0_0 = ProducesFlags::ProducesFlagsSideEffect {
inst: pattern1_0.clone(),
};
@@ -1127,7 +1129,7 @@ pub fn constructor_produces_flags_ignore<C: Context>(
inst: ref pattern1_0,
result: pattern1_1,
} => {
// Rule at src/prelude.isle line 441.
// Rule at src/prelude.isle line 449.
let expr0_0 = ProducesFlags::ProducesFlagsSideEffect {
inst: pattern1_0.clone(),
};
@@ -1156,7 +1158,7 @@ pub fn constructor_consumes_flags_concat<C: Context>(
result: pattern3_1,
} = pattern2_0
{
// Rule at src/prelude.isle line 448.
// Rule at src/prelude.isle line 456.
let expr0_0 = C::value_regs(ctx, pattern1_1, pattern3_1);
let expr1_0 = ConsumesFlags::ConsumesFlagsTwiceReturnsValueRegs {
inst1: pattern1_0.clone(),
@@ -1186,7 +1188,7 @@ pub fn constructor_with_flags<C: Context>(
inst: ref pattern3_0,
result: pattern3_1,
} => {
// Rule at src/prelude.isle line 473.
// Rule at src/prelude.isle line 481.
let expr0_0 = C::emit(ctx, pattern1_0);
let expr1_0 = C::emit(ctx, pattern3_0);
let expr2_0 = C::value_reg(ctx, pattern3_1);
@@ -1197,7 +1199,7 @@ pub fn constructor_with_flags<C: Context>(
inst2: ref pattern3_1,
result: pattern3_2,
} => {
// Rule at src/prelude.isle line 479.
// Rule at src/prelude.isle line 487.
let expr0_0 = C::emit(ctx, pattern1_0);
let expr1_0 = C::emit(ctx, pattern3_0);
let expr2_0 = C::emit(ctx, pattern3_1);
@@ -1210,7 +1212,7 @@ pub fn constructor_with_flags<C: Context>(
inst4: ref pattern3_3,
result: pattern3_4,
} => {
// Rule at src/prelude.isle line 491.
// Rule at src/prelude.isle line 499.
let expr0_0 = C::emit(ctx, pattern1_0);
let expr1_0 = C::emit(ctx, pattern3_0);
let expr2_0 = C::emit(ctx, pattern3_1);
@@ -1231,7 +1233,7 @@ pub fn constructor_with_flags<C: Context>(
result: pattern3_1,
} = pattern2_0
{
// Rule at src/prelude.isle line 467.
// Rule at src/prelude.isle line 475.
let expr0_0 = C::emit(ctx, pattern1_0);
let expr1_0 = C::emit(ctx, pattern3_0);
let expr2_0 = C::value_regs(ctx, pattern1_1, pattern3_1);
@@ -1251,7 +1253,7 @@ pub fn constructor_with_flags_reg<C: Context>(
) -> Option<Reg> {
let pattern0_0 = arg0;
let pattern1_0 = arg1;
// Rule at src/prelude.isle line 508.
// Rule at src/prelude.isle line 516.
let expr0_0 = constructor_with_flags(ctx, pattern0_0, pattern1_0)?;
let expr1_0: usize = 0;
let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0);

2
cranelift/codegen/src/isa/s390x/lower/isle/generated_code.manifest
View File

@@ -1,4 +1,4 @@
src/clif.isle 9ea75a6f790b5c03
src/prelude.isle 6b0160bfcac86902
src/prelude.isle b2bc986bcbbbb77
src/isa/s390x/inst.isle d91a16074ab186a8
src/isa/s390x/lower.isle 1cc5a12adc8c75f9

30
cranelift/codegen/src/isa/s390x/lower/isle/generated_code.rs generated
View File

@@ -54,6 +54,8 @@ pub trait Context {
fn fits_in_64(&mut self, arg0: Type) -> Option<Type>;
fn ty_32_or_64(&mut self, arg0: Type) -> Option<Type>;
fn ty_8_or_16(&mut self, arg0: Type) -> Option<Type>;
fn ty_int_bool_64(&mut self, arg0: Type) -> Option<Type>;
fn ty_int_bool_128(&mut self, arg0: Type) -> Option<Type>;
fn vec128(&mut self, arg0: Type) -> Option<Type>;
fn not_i64x2(&mut self, arg0: Type) -> Option<()>;
fn value_list_slice(&mut self, arg0: ValueList) -> ValueSlice;
@@ -150,13 +152,13 @@ pub trait Context {
fn same_reg(&mut self, arg0: Reg, arg1: WritableReg) -> Option<()>;
}
/// Internal type SideEffectNoResult: defined at src/prelude.isle line 385.
/// Internal type SideEffectNoResult: defined at src/prelude.isle line 393.
#[derive(Clone, Debug)]
pub enum SideEffectNoResult {
Inst { inst: MInst },
}
/// Internal type ProducesFlags: defined at src/prelude.isle line 407.
/// Internal type ProducesFlags: defined at src/prelude.isle line 415.
#[derive(Clone, Debug)]
pub enum ProducesFlags {
ProducesFlagsSideEffect { inst: MInst },
@@ -164,7 +166,7 @@ pub enum ProducesFlags {
ProducesFlagsReturnsResultWithConsumer { inst: MInst, result: Reg },
}
/// Internal type ConsumesFlags: defined at src/prelude.isle line 418.
/// Internal type ConsumesFlags: defined at src/prelude.isle line 426.
#[derive(Clone, Debug)]
pub enum ConsumesFlags {
ConsumesFlagsReturnsResultWithProducer {
@@ -954,7 +956,7 @@ pub fn constructor_side_effect<C: Context>(
inst: ref pattern1_0,
} = pattern0_0
{
// Rule at src/prelude.isle line 390.
// Rule at src/prelude.isle line 398.
let expr0_0 = C::emit(ctx, pattern1_0);
let expr1_0 = C::output_none(ctx);
return Some(expr1_0);
@@ -972,7 +974,7 @@ pub fn constructor_safepoint<C: Context>(
inst: ref pattern1_0,
} = pattern0_0
{
// Rule at src/prelude.isle line 396.
// Rule at src/prelude.isle line 404.
let expr0_0 = C::emit_safepoint(ctx, pattern1_0);
let expr1_0 = C::output_none(ctx);
return Some(expr1_0);
@@ -991,7 +993,7 @@ pub fn constructor_produces_flags_get_reg<C: Context>(
result: pattern1_1,
} = pattern0_0
{
// Rule at src/prelude.isle line 434.
// Rule at src/prelude.isle line 442.
return Some(pattern1_1);
}
return None;
@@ -1008,7 +1010,7 @@ pub fn constructor_produces_flags_ignore<C: Context>(
inst: ref pattern1_0,
result: pattern1_1,
} => {
// Rule at src/prelude.isle line 439.
// Rule at src/prelude.isle line 447.
let expr0_0 = ProducesFlags::ProducesFlagsSideEffect {
inst: pattern1_0.clone(),
};
@@ -1018,7 +1020,7 @@ pub fn constructor_produces_flags_ignore<C: Context>(
inst: ref pattern1_0,
result: pattern1_1,
} => {
// Rule at src/prelude.isle line 441.
// Rule at src/prelude.isle line 449.
let expr0_0 = ProducesFlags::ProducesFlagsSideEffect {
inst: pattern1_0.clone(),
};
@@ -1047,7 +1049,7 @@ pub fn constructor_consumes_flags_concat<C: Context>(
result: pattern3_1,
} = pattern2_0
{
// Rule at src/prelude.isle line 448.
// Rule at src/prelude.isle line 456.
let expr0_0 = C::value_regs(ctx, pattern1_1, pattern3_1);
let expr1_0 = ConsumesFlags::ConsumesFlagsTwiceReturnsValueRegs {
inst1: pattern1_0.clone(),
@@ -1077,7 +1079,7 @@ pub fn constructor_with_flags<C: Context>(
inst: ref pattern3_0,
result: pattern3_1,
} => {
// Rule at src/prelude.isle line 473.
// Rule at src/prelude.isle line 481.
let expr0_0 = C::emit(ctx, pattern1_0);
let expr1_0 = C::emit(ctx, pattern3_0);
let expr2_0 = C::value_reg(ctx, pattern3_1);
@@ -1088,7 +1090,7 @@ pub fn constructor_with_flags<C: Context>(
inst2: ref pattern3_1,
result: pattern3_2,
} => {
// Rule at src/prelude.isle line 479.
// Rule at src/prelude.isle line 487.
let expr0_0 = C::emit(ctx, pattern1_0);
let expr1_0 = C::emit(ctx, pattern3_0);
let expr2_0 = C::emit(ctx, pattern3_1);
@@ -1101,7 +1103,7 @@ pub fn constructor_with_flags<C: Context>(
inst4: ref pattern3_3,
result: pattern3_4,
} => {
// Rule at src/prelude.isle line 491.
// Rule at src/prelude.isle line 499.
let expr0_0 = C::emit(ctx, pattern1_0);
let expr1_0 = C::emit(ctx, pattern3_0);
let expr2_0 = C::emit(ctx, pattern3_1);
@@ -1122,7 +1124,7 @@ pub fn constructor_with_flags<C: Context>(
result: pattern3_1,
} = pattern2_0
{
// Rule at src/prelude.isle line 467.
// Rule at src/prelude.isle line 475.
let expr0_0 = C::emit(ctx, pattern1_0);
let expr1_0 = C::emit(ctx, pattern3_0);
let expr2_0 = C::value_regs(ctx, pattern1_1, pattern3_1);
@@ -1142,7 +1144,7 @@ pub fn constructor_with_flags_reg<C: Context>(
) -> Option<Reg> {
let pattern0_0 = arg0;
let pattern1_0 = arg1;
// Rule at src/prelude.isle line 508.
// Rule at src/prelude.isle line 516.
let expr0_0 = constructor_with_flags(ctx, pattern0_0, pattern1_0)?;
let expr1_0: usize = 0;
let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0);

127
cranelift/codegen/src/isa/x64/lower.isle
View File

@@ -1859,3 +1859,130 @@
(cmp_imm (OperandSize.Size64) 0xffffffff src) ;; simm32 0xffff_ffff is sign-extended to -1.
(setcc (CC.Z))))
;; Rules for `uextend` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; T -> T is a no-op.
(rule (lower (has_type ty (uextend src @ (value_type =ty))))
src)
;; I64 -> I128.
(rule (lower (has_type $I128 (uextend src @ (value_type $I64))))
(value_regs src (imm $I64 0)))
;; I{8,16,32} -> I128.
(rule (lower (has_type $I128 (uextend src @ (value_type (fits_in_32 src_ty)))))
(value_regs (extend_to_gpr src $I64 (ExtendKind.Zero)) (imm $I64 0)))
;; I{8,16,32} -> I64.
(rule (lower (has_type $I64 (uextend src @ (value_type (fits_in_32 src_ty)))))
(extend_to_gpr src $I64 (ExtendKind.Zero)))
;; I8 -> I{16,32}, I16 -> I32.
(rule (lower (has_type (fits_in_32 dst_ty) (uextend src @ (value_type (fits_in_32 src_ty)))))
(extend_to_gpr src $I32 (ExtendKind.Zero)))
;; I32 -> I64 with op that produces a zero-extended value in a register.
;;
;; As a particular x64 extra-pattern matching opportunity, all the ALU
;; opcodes on 32-bits will zero-extend the upper 32-bits, so we can
;; even not generate a zero-extended move in this case.
;;
;; (Note that we unfortunately can't factor out the
;; insts-that-zero-upper-32 pattern into a separate extractor until we
;; can write internal extractors with multiple rules; and we'd rather
;; keep these here than write an external extractor containing bits of
;; the instruction pattern.s)
(rule (lower (has_type $I64
(uextend src @ (iadd _ _))))
src)
(rule (lower (has_type $I64
(uextend src @ (iadd_ifcout _ _))))
src)
(rule (lower (has_type $I64
(uextend src @ (isub _ _))))
src)
(rule (lower (has_type $I64
(uextend src @ (imul _ _))))
src)
(rule (lower (has_type $I64
(uextend src @ (band _ _))))
src)
(rule (lower (has_type $I64
(uextend src @ (bor _ _))))
src)
(rule (lower (has_type $I64
(uextend src @ (bxor _ _))))
src)
(rule (lower (has_type $I64
(uextend src @ (ishl _ _))))
src)
(rule (lower (has_type $I64
(uextend src @ (ushr _ _))))
src)
(rule (lower (has_type $I64
(uextend src @ (uload32 _ _ _))))
src)
;; Rules for `sextend` / `bextend` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(decl generic_sextend (Value Type Type) InstOutput)
;; T -> T is a no-op.
(rule (generic_sextend src ty =ty)
src)
;; Produce upper 64 bits sign-extended from lower 64: shift right by
;; 63 bits to spread the sign bit across the result.
(decl spread_sign_bit (Gpr) Gpr)
(rule (spread_sign_bit src)
(sar $I64 src (Imm8Reg.Imm8 63)))
;; I64 -> I128.
(rule (generic_sextend src (ty_int_bool_64 _) (ty_int_bool_128 _))
(value_regs src (spread_sign_bit src)))
;; I{8,16,32} -> I128.
(rule (generic_sextend src (fits_in_32 src_ty) (ty_int_bool_128 _))
(let ((lo Gpr (extend_to_gpr src $I64 (ExtendKind.Sign)))
(hi Gpr (spread_sign_bit lo)))
(value_regs lo hi)))
;; I{8,16,32} -> I64.
(rule (generic_sextend src (fits_in_32 src_ty) (ty_int_bool_64 _))
(extend_to_gpr src $I64 (ExtendKind.Sign)))
;; I8 -> I{16,32}, I16 -> I32.
(rule (generic_sextend src (fits_in_32 src_ty) (fits_in_32 dst_ty))
(extend_to_gpr src $I32 (ExtendKind.Sign)))
(rule (lower
(has_type dst_ty
(sextend src @ (value_type src_ty))))
(generic_sextend src src_ty dst_ty))
;; Bools are stored as 0/-1 so extends must sign-extend as well.
(rule (lower
(has_type dst_ty
(bextend src @ (value_type src_ty))))
(generic_sextend src src_ty dst_ty))
;; Rules for `ireduce` / `breduce` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; T -> T is always a no-op, even I128 -> I128.
(rule (lower (has_type ty (ireduce src @ (value_type =ty))))
src)
;; T -> I{64,32,16,8}: We can simply pass through the value: values
;; are always stored with high bits undefined, so we can just leave
;; them be.
(rule (lower (has_type (fits_in_64 ty) (ireduce src)))
(value_regs_get_gpr src 0))
;; Likewise for breduce.
(rule (lower (has_type ty (breduce src @ (value_type =ty))))
src)
(rule (lower (has_type (fits_in_64 ty) (breduce src)))
(value_regs_get_gpr src 0))

141
cranelift/codegen/src/isa/x64/lower.rs
View File

@@ -71,25 +71,6 @@ fn matches_input<C: LowerCtx<I = Inst>>(
})
}
/// Returns whether the given specified `input` is a result produced by an instruction with any of
/// the opcodes specified in `ops`.
fn matches_input_any<C: LowerCtx<I = Inst>>(
ctx: &mut C,
input: InsnInput,
ops: &[Opcode],
) -> Option<IRInst> {
let inputs = ctx.get_input_as_source_or_const(input.insn, input.input);
inputs.inst.and_then(|(src_inst, _)| {
let data = ctx.data(src_inst);
for &op in ops {
if data.opcode() == op {
return Some(src_inst);
}
}
None
})
}
/// Emits instruction(s) to generate the given 64-bit constant value into a newly-allocated
/// temporary register, returning that register.
fn generate_constant<C: LowerCtx<I = Inst>>(ctx: &mut C, ty: Type, c: u64) -> ValueRegs<Reg> {
@@ -917,122 +898,12 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
| Opcode::Popcnt
| Opcode::Bitrev
| Opcode::IsNull
| Opcode::IsInvalid => implemented_in_isle(ctx),
Opcode::Uextend | Opcode::Sextend | Opcode::Breduce | Opcode::Bextend | Opcode::Ireduce => {
let src_ty = ctx.input_ty(insn, 0);
let dst_ty = ctx.output_ty(insn, 0);
if src_ty == types::I128 {
assert!(dst_ty.bits() <= 64);
assert!(op == Opcode::Ireduce);
let src = put_input_in_regs(ctx, inputs[0]);
let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
ctx.emit(Inst::gen_move(dst, src.regs()[0], types::I64));
} else if dst_ty == types::I128 {
assert!(src_ty.bits() <= 64);
let src = put_input_in_reg(ctx, inputs[0]);
let dst = get_output_reg(ctx, outputs[0]);
assert!(op == Opcode::Uextend || op == Opcode::Sextend);
// Extend to 64 bits first.
let ext_mode = ExtMode::new(src_ty.bits(), /* dst bits = */ 64);
if let Some(ext_mode) = ext_mode {
if op == Opcode::Sextend {
ctx.emit(Inst::movsx_rm_r(ext_mode, RegMem::reg(src), dst.regs()[0]));
} else {
ctx.emit(Inst::movzx_rm_r(ext_mode, RegMem::reg(src), dst.regs()[0]));
}
} else {
ctx.emit(Inst::mov64_rm_r(RegMem::reg(src), dst.regs()[0]));
}
// Now generate the top 64 bits.
if op == Opcode::Sextend {
// Sign-extend: move dst[0] into dst[1] and arithmetic-shift right by 63 bits
// to spread the sign bit across all bits.
ctx.emit(Inst::gen_move(
dst.regs()[1],
dst.regs()[0].to_reg(),
types::I64,
));
ctx.emit(Inst::shift_r(
OperandSize::Size64,
ShiftKind::ShiftRightArithmetic,
Some(63),
dst.regs()[1],
));
} else {
// Zero-extend: just zero the top word.
ctx.emit(Inst::alu_rmi_r(
OperandSize::Size64,
AluRmiROpcode::Xor,
RegMemImm::reg(dst.regs()[1].to_reg()),
dst.regs()[1],
));
}
} else {
// Sextend requires a sign-extended move, but all the other opcodes are simply a move
// from a zero-extended source. Here is why this works, in each case:
//
// - Breduce, Bextend: changing width of a boolean. We
// represent a bool as a 0 or -1, so Breduce can mask, while
// Bextend must sign-extend.
//
// - Ireduce: changing width of an integer. Smaller ints are stored with undefined
// high-order bits, so we can simply do a copy.
let is_sextend = match op {
Opcode::Sextend | Opcode::Bextend => true,
_ => false,
};
if src_ty == types::I32 && dst_ty == types::I64 && !is_sextend {
// As a particular x64 extra-pattern matching opportunity, all the ALU opcodes on
// 32-bits will zero-extend the upper 32-bits, so we can even not generate a
// zero-extended move in this case.
// TODO add loads and shifts here.
if let Some(_) = matches_input_any(
ctx,
inputs[0],
&[
Opcode::Iadd,
Opcode::IaddIfcout,
Opcode::Isub,
Opcode::Imul,
Opcode::Band,
Opcode::Bor,
Opcode::Bxor,
],
) {
let src = put_input_in_reg(ctx, inputs[0]);
let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
ctx.emit(Inst::gen_move(dst, src, types::I64));
return Ok(());
}
}
let src = input_to_reg_mem(ctx, inputs[0]);
let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
let ext_mode = ExtMode::new(src_ty.bits(), dst_ty.bits());
assert_eq!(
src_ty.bits() < dst_ty.bits(),
ext_mode.is_some(),
"unexpected extension: {} -> {}",
src_ty,
dst_ty
);
if let Some(ext_mode) = ext_mode {
if is_sextend {
ctx.emit(Inst::movsx_rm_r(ext_mode, src, dst));
} else {
ctx.emit(Inst::movzx_rm_r(ext_mode, src, dst));
}
} else {
ctx.emit(Inst::mov64_rm_r(src, dst));
}
}
}
| Opcode::IsInvalid
| Opcode::Uextend
| Opcode::Sextend
| Opcode::Breduce
| Opcode::Bextend
| Opcode::Ireduce => implemented_in_isle(ctx),
Opcode::Bint => {
// Booleans are stored as all-zeroes (0) or all-ones (-1). We AND

4
cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest
View File

@@ -1,4 +1,4 @@
src/clif.isle 9ea75a6f790b5c03
src/prelude.isle 6b0160bfcac86902
src/prelude.isle b2bc986bcbbbb77
src/isa/x64/inst.isle 67eb719e568c2a81
src/isa/x64/lower.isle 142626fe062fd7d7
src/isa/x64/lower.isle 2d06b233fb3a1e1c

329
cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs generated
View File

@@ -54,6 +54,8 @@ pub trait Context {
fn fits_in_64(&mut self, arg0: Type) -> Option<Type>;
fn ty_32_or_64(&mut self, arg0: Type) -> Option<Type>;
fn ty_8_or_16(&mut self, arg0: Type) -> Option<Type>;
fn ty_int_bool_64(&mut self, arg0: Type) -> Option<Type>;
fn ty_int_bool_128(&mut self, arg0: Type) -> Option<Type>;
fn vec128(&mut self, arg0: Type) -> Option<Type>;
fn not_i64x2(&mut self, arg0: Type) -> Option<()>;
fn value_list_slice(&mut self, arg0: ValueList) -> ValueSlice;
@@ -147,13 +149,13 @@ pub trait Context {
fn popcount_low_mask(&mut self) -> VCodeConstant;
}
/// Internal type SideEffectNoResult: defined at src/prelude.isle line 385.
/// Internal type SideEffectNoResult: defined at src/prelude.isle line 393.
#[derive(Clone, Debug)]
pub enum SideEffectNoResult {
Inst { inst: MInst },
}
/// Internal type ProducesFlags: defined at src/prelude.isle line 407.
/// Internal type ProducesFlags: defined at src/prelude.isle line 415.
#[derive(Clone, Debug)]
pub enum ProducesFlags {
ProducesFlagsSideEffect { inst: MInst },
@@ -161,7 +163,7 @@ pub enum ProducesFlags {
ProducesFlagsReturnsResultWithConsumer { inst: MInst, result: Reg },
}
/// Internal type ConsumesFlags: defined at src/prelude.isle line 418.
/// Internal type ConsumesFlags: defined at src/prelude.isle line 426.
#[derive(Clone, Debug)]
pub enum ConsumesFlags {
ConsumesFlagsReturnsResultWithProducer {
@@ -552,7 +554,7 @@ pub fn constructor_side_effect<C: Context>(
inst: ref pattern1_0,
} = pattern0_0
{
// Rule at src/prelude.isle line 390.
// Rule at src/prelude.isle line 398.
let expr0_0 = C::emit(ctx, pattern1_0);
let expr1_0 = C::output_none(ctx);
return Some(expr1_0);
@@ -570,7 +572,7 @@ pub fn constructor_safepoint<C: Context>(
inst: ref pattern1_0,
} = pattern0_0
{
// Rule at src/prelude.isle line 396.
// Rule at src/prelude.isle line 404.
let expr0_0 = C::emit_safepoint(ctx, pattern1_0);
let expr1_0 = C::output_none(ctx);
return Some(expr1_0);
@@ -589,7 +591,7 @@ pub fn constructor_produces_flags_get_reg<C: Context>(
result: pattern1_1,
} = pattern0_0
{
// Rule at src/prelude.isle line 434.
// Rule at src/prelude.isle line 442.
return Some(pattern1_1);
}
return None;
@@ -606,7 +608,7 @@ pub fn constructor_produces_flags_ignore<C: Context>(
inst: ref pattern1_0,
result: pattern1_1,
} => {
// Rule at src/prelude.isle line 439.
// Rule at src/prelude.isle line 447.
let expr0_0 = ProducesFlags::ProducesFlagsSideEffect {
inst: pattern1_0.clone(),
};
@@ -616,7 +618,7 @@ pub fn constructor_produces_flags_ignore<C: Context>(
inst: ref pattern1_0,
result: pattern1_1,
} => {
// Rule at src/prelude.isle line 441.
// Rule at src/prelude.isle line 449.
let expr0_0 = ProducesFlags::ProducesFlagsSideEffect {
inst: pattern1_0.clone(),
};
@@ -645,7 +647,7 @@ pub fn constructor_consumes_flags_concat<C: Context>(
result: pattern3_1,
} = pattern2_0
{
// Rule at src/prelude.isle line 448.
// Rule at src/prelude.isle line 456.
let expr0_0 = C::value_regs(ctx, pattern1_1, pattern3_1);
let expr1_0 = ConsumesFlags::ConsumesFlagsTwiceReturnsValueRegs {
inst1: pattern1_0.clone(),
@@ -675,7 +677,7 @@ pub fn constructor_with_flags<C: Context>(
inst: ref pattern3_0,
result: pattern3_1,
} => {
// Rule at src/prelude.isle line 473.
// Rule at src/prelude.isle line 481.
let expr0_0 = C::emit(ctx, pattern1_0);
let expr1_0 = C::emit(ctx, pattern3_0);
let expr2_0 = C::value_reg(ctx, pattern3_1);
@@ -686,7 +688,7 @@ pub fn constructor_with_flags<C: Context>(
inst2: ref pattern3_1,
result: pattern3_2,
} => {
// Rule at src/prelude.isle line 479.
// Rule at src/prelude.isle line 487.
let expr0_0 = C::emit(ctx, pattern1_0);
let expr1_0 = C::emit(ctx, pattern3_0);
let expr2_0 = C::emit(ctx, pattern3_1);
@@ -699,7 +701,7 @@ pub fn constructor_with_flags<C: Context>(
inst4: ref pattern3_3,
result: pattern3_4,
} => {
// Rule at src/prelude.isle line 491.
// Rule at src/prelude.isle line 499.
let expr0_0 = C::emit(ctx, pattern1_0);
let expr1_0 = C::emit(ctx, pattern3_0);
let expr2_0 = C::emit(ctx, pattern3_1);
@@ -720,7 +722,7 @@ pub fn constructor_with_flags<C: Context>(
result: pattern3_1,
} = pattern2_0
{
// Rule at src/prelude.isle line 467.
// Rule at src/prelude.isle line 475.
let expr0_0 = C::emit(ctx, pattern1_0);
let expr1_0 = C::emit(ctx, pattern3_0);
let expr2_0 = C::value_regs(ctx, pattern1_1, pattern3_1);
@@ -740,7 +742,7 @@ pub fn constructor_with_flags_reg<C: Context>(
) -> Option<Reg> {
let pattern0_0 = arg0;
let pattern1_0 = arg1;
// Rule at src/prelude.isle line 508.
// Rule at src/prelude.isle line 516.
let expr0_0 = constructor_with_flags(ctx, pattern0_0, pattern1_0)?;
let expr1_0: usize = 0;
let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0);
@@ -4260,7 +4262,8 @@ pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<InstOutp
arg: pattern5_1,
} = &pattern4_0
{
if let &Opcode::Bitrev = pattern5_0 {
match pattern5_0 {
&Opcode::Bitrev => {
// Rule at src/isa/x64/lower.isle line 1780.
let expr0_0: Type = I64;
let expr1_0 = constructor_put_in_gpr(ctx, pattern5_1)?;
@@ -4268,6 +4271,118 @@ pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<InstOutp
let expr3_0 = constructor_output_gpr(ctx, expr2_0)?;
return Some(expr3_0);
}
&Opcode::Uextend => {
if let Some(pattern7_0) = C::def_inst(ctx, pattern5_1) {
let pattern8_0 = C::inst_data(ctx, pattern7_0);
match &pattern8_0 {
&InstructionData::Binary {
opcode: ref pattern9_0,
args: ref pattern9_1,
} => {
match pattern9_0 {
&Opcode::Iadd => {
let (pattern11_0, pattern11_1) =
C::unpack_value_array_2(ctx, pattern9_1);
// Rule at src/isa/x64/lower.isle line 1896.
let expr0_0 =
constructor_output_value(ctx, pattern5_1)?;
return Some(expr0_0);
}
&Opcode::Isub => {
let (pattern11_0, pattern11_1) =
C::unpack_value_array_2(ctx, pattern9_1);
// Rule at src/isa/x64/lower.isle line 1902.
let expr0_0 =
constructor_output_value(ctx, pattern5_1)?;
return Some(expr0_0);
}
&Opcode::Imul => {
let (pattern11_0, pattern11_1) =
C::unpack_value_array_2(ctx, pattern9_1);
// Rule at src/isa/x64/lower.isle line 1905.
let expr0_0 =
constructor_output_value(ctx, pattern5_1)?;
return Some(expr0_0);
}
&Opcode::IaddIfcout => {
let (pattern11_0, pattern11_1) =
C::unpack_value_array_2(ctx, pattern9_1);
// Rule at src/isa/x64/lower.isle line 1899.
let expr0_0 =
constructor_output_value(ctx, pattern5_1)?;
return Some(expr0_0);
}
&Opcode::Band => {
let (pattern11_0, pattern11_1) =
C::unpack_value_array_2(ctx, pattern9_1);
// Rule at src/isa/x64/lower.isle line 1908.
let expr0_0 =
constructor_output_value(ctx, pattern5_1)?;
return Some(expr0_0);
}
&Opcode::Bor => {
let (pattern11_0, pattern11_1) =
C::unpack_value_array_2(ctx, pattern9_1);
// Rule at src/isa/x64/lower.isle line 1911.
let expr0_0 =
constructor_output_value(ctx, pattern5_1)?;
return Some(expr0_0);
}
&Opcode::Bxor => {
let (pattern11_0, pattern11_1) =
C::unpack_value_array_2(ctx, pattern9_1);
// Rule at src/isa/x64/lower.isle line 1914.
let expr0_0 =
constructor_output_value(ctx, pattern5_1)?;
return Some(expr0_0);
}
&Opcode::Ishl => {
let (pattern11_0, pattern11_1) =
C::unpack_value_array_2(ctx, pattern9_1);
// Rule at src/isa/x64/lower.isle line 1917.
let expr0_0 =
constructor_output_value(ctx, pattern5_1)?;
return Some(expr0_0);
}
&Opcode::Ushr => {
let (pattern11_0, pattern11_1) =
C::unpack_value_array_2(ctx, pattern9_1);
// Rule at src/isa/x64/lower.isle line 1920.
let expr0_0 =
constructor_output_value(ctx, pattern5_1)?;
return Some(expr0_0);
}
_ => {}
}
}
&InstructionData::Load {
opcode: ref pattern9_0,
arg: pattern9_1,
flags: pattern9_2,
offset: pattern9_3,
} => {
if let &Opcode::Uload32 = pattern9_0 {
// Rule at src/isa/x64/lower.isle line 1923.
let expr0_0 = constructor_output_value(ctx, pattern5_1)?;
return Some(expr0_0);
}
}
_ => {}
}
}
let pattern7_0 = C::value_type(ctx, pattern5_1);
if let Some(pattern8_0) = C::fits_in_32(ctx, pattern7_0) {
// Rule at src/isa/x64/lower.isle line 1878.
let expr0_0: Type = I64;
let expr1_0 = ExtendKind::Zero;
let expr2_0 =
constructor_extend_to_gpr(ctx, pattern5_1, expr0_0, &expr1_0)?;
let expr3_0 = constructor_output_gpr(ctx, expr2_0)?;
return Some(expr3_0);
}
}
_ => {}
}
}
}
if pattern2_0 == I128 {
@@ -4630,6 +4745,33 @@ pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<InstOutp
let expr18_0 = C::output(ctx, expr17_0);
return Some(expr18_0);
}
&Opcode::Uextend => {
let pattern7_0 = C::value_type(ctx, pattern5_1);
if pattern7_0 == I64 {
// Rule at src/isa/x64/lower.isle line 1870.
let expr0_0 = C::put_in_reg(ctx, pattern5_1);
let expr1_0: Type = I64;
let expr2_0: u64 = 0;
let expr3_0 = constructor_imm(ctx, expr1_0, expr2_0)?;
let expr4_0 = C::value_regs(ctx, expr0_0, expr3_0);
let expr5_0 = C::output(ctx, expr4_0);
return Some(expr5_0);
}
if let Some(pattern8_0) = C::fits_in_32(ctx, pattern7_0) {
// Rule at src/isa/x64/lower.isle line 1874.
let expr0_0: Type = I64;
let expr1_0 = ExtendKind::Zero;
let expr2_0 =
constructor_extend_to_gpr(ctx, pattern5_1, expr0_0, &expr1_0)?;
let expr3_0 = C::gpr_to_reg(ctx, expr2_0);
let expr4_0: Type = I64;
let expr5_0: u64 = 0;
let expr6_0 = constructor_imm(ctx, expr4_0, expr5_0)?;
let expr7_0 = C::value_regs(ctx, expr3_0, expr6_0);
let expr8_0 = C::output(ctx, expr7_0);
return Some(expr8_0);
}
}
_ => {}
}
}
@@ -5390,6 +5532,52 @@ pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<InstOutp
}
}
}
&InstructionData::Unary {
opcode: ref pattern4_0,
arg: pattern4_1,
} => {
match pattern4_0 {
&Opcode::Breduce => {
let pattern6_0 = C::value_type(ctx, pattern4_1);
if pattern6_0 == pattern2_0 {
// Rule at src/isa/x64/lower.isle line 1984.
let expr0_0 = constructor_output_value(ctx, pattern4_1)?;
return Some(expr0_0);
}
}
&Opcode::Bextend => {
let pattern6_0 = C::value_type(ctx, pattern4_1);
// Rule at src/isa/x64/lower.isle line 1965.
let expr0_0 =
constructor_generic_sextend(ctx, pattern4_1, pattern6_0, pattern2_0)?;
return Some(expr0_0);
}
&Opcode::Ireduce => {
let pattern6_0 = C::value_type(ctx, pattern4_1);
if pattern6_0 == pattern2_0 {
// Rule at src/isa/x64/lower.isle line 1973.
let expr0_0 = constructor_output_value(ctx, pattern4_1)?;
return Some(expr0_0);
}
}
&Opcode::Uextend => {
let pattern6_0 = C::value_type(ctx, pattern4_1);
if pattern6_0 == pattern2_0 {
// Rule at src/isa/x64/lower.isle line 1866.
let expr0_0 = constructor_output_value(ctx, pattern4_1)?;
return Some(expr0_0);
}
}
&Opcode::Sextend => {
let pattern6_0 = C::value_type(ctx, pattern4_1);
// Rule at src/isa/x64/lower.isle line 1959.
let expr0_0 =
constructor_generic_sextend(ctx, pattern4_1, pattern6_0, pattern2_0)?;
return Some(expr0_0);
}
_ => {}
}
}
_ => {}
}
if let Some(()) = C::avx512vl_enabled(ctx, pattern2_0) {
@@ -6568,6 +6756,27 @@ pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<InstOutp
_ => {}
}
}
if let Some(pattern3_0) = C::fits_in_32(ctx, pattern2_0) {
let pattern4_0 = C::inst_data(ctx, pattern0_0);
if let &InstructionData::Unary {
opcode: ref pattern5_0,
arg: pattern5_1,
} = &pattern4_0
{
if let &Opcode::Uextend = pattern5_0 {
let pattern7_0 = C::value_type(ctx, pattern5_1);
if let Some(pattern8_0) = C::fits_in_32(ctx, pattern7_0) {
// Rule at src/isa/x64/lower.isle line 1882.
let expr0_0: Type = I32;
let expr1_0 = ExtendKind::Zero;
let expr2_0 =
constructor_extend_to_gpr(ctx, pattern5_1, expr0_0, &expr1_0)?;
let expr3_0 = constructor_output_gpr(ctx, expr2_0)?;
return Some(expr3_0);
}
}
}
}
if let Some(pattern3_0) = C::fits_in_64(ctx, pattern2_0) {
let pattern4_0 = C::inst_data(ctx, pattern0_0);
match &pattern4_0 {
@@ -6993,6 +7202,22 @@ pub fn constructor_lower<C: Context>(ctx: &mut C, arg0: Inst) -> Option<InstOutp
let expr2_0 = constructor_output_gpr(ctx, expr1_0)?;
return Some(expr2_0);
}
&Opcode::Breduce => {
// Rule at src/isa/x64/lower.isle line 1987.
let expr0_0 = C::put_in_regs(ctx, pattern5_1);
let expr1_0: usize = 0;
let expr2_0 = constructor_value_regs_get_gpr(ctx, expr0_0, expr1_0)?;
let expr3_0 = constructor_output_gpr(ctx, expr2_0)?;
return Some(expr3_0);
}
&Opcode::Ireduce => {
// Rule at src/isa/x64/lower.isle line 1979.
let expr0_0 = C::put_in_regs(ctx, pattern5_1);
let expr1_0: usize = 0;
let expr2_0 = constructor_value_regs_get_gpr(ctx, expr0_0, expr1_0)?;
let expr3_0 = constructor_output_gpr(ctx, expr2_0)?;
return Some(expr3_0);
}
_ => {}
}
}
@@ -8192,3 +8417,77 @@ pub fn constructor_do_bitrev64<C: Context>(ctx: &mut C, arg0: Type, arg1: Gpr) -
}
return None;
}
// Generated as internal constructor for term generic_sextend.
pub fn constructor_generic_sextend<C: Context>(
ctx: &mut C,
arg0: Value,
arg1: Type,
arg2: Type,
) -> Option<InstOutput> {
let pattern0_0 = arg0;
let pattern1_0 = arg1;
if let Some(pattern2_0) = C::fits_in_32(ctx, pattern1_0) {
let pattern3_0 = arg2;
if let Some(pattern4_0) = C::fits_in_32(ctx, pattern3_0) {
// Rule at src/isa/x64/lower.isle line 1956.
let expr0_0: Type = I32;
let expr1_0 = ExtendKind::Sign;
let expr2_0 = constructor_extend_to_gpr(ctx, pattern0_0, expr0_0, &expr1_0)?;
let expr3_0 = constructor_output_gpr(ctx, expr2_0)?;
return Some(expr3_0);
}
if let Some(pattern4_0) = C::ty_int_bool_64(ctx, pattern3_0) {
// Rule at src/isa/x64/lower.isle line 1952.
let expr0_0: Type = I64;
let expr1_0 = ExtendKind::Sign;
let expr2_0 = constructor_extend_to_gpr(ctx, pattern0_0, expr0_0, &expr1_0)?;
let expr3_0 = constructor_output_gpr(ctx, expr2_0)?;
return Some(expr3_0);
}
if let Some(pattern4_0) = C::ty_int_bool_128(ctx, pattern3_0) {
// Rule at src/isa/x64/lower.isle line 1946.
let expr0_0: Type = I64;
let expr1_0 = ExtendKind::Sign;
let expr2_0 = constructor_extend_to_gpr(ctx, pattern0_0, expr0_0, &expr1_0)?;
let expr3_0 = constructor_spread_sign_bit(ctx, expr2_0)?;
let expr4_0 = C::gpr_to_reg(ctx, expr2_0);
let expr5_0 = C::gpr_to_reg(ctx, expr3_0);
let expr6_0 = C::value_regs(ctx, expr4_0, expr5_0);
let expr7_0 = C::output(ctx, expr6_0);
return Some(expr7_0);
}
}
if let Some(pattern2_0) = C::ty_int_bool_64(ctx, pattern1_0) {
let pattern3_0 = arg2;
if let Some(pattern4_0) = C::ty_int_bool_128(ctx, pattern3_0) {
// Rule at src/isa/x64/lower.isle line 1942.
let expr0_0 = C::put_in_reg(ctx, pattern0_0);
let expr1_0 = constructor_put_in_gpr(ctx, pattern0_0)?;
let expr2_0 = constructor_spread_sign_bit(ctx, expr1_0)?;
let expr3_0 = C::gpr_to_reg(ctx, expr2_0);
let expr4_0 = C::value_regs(ctx, expr0_0, expr3_0);
let expr5_0 = C::output(ctx, expr4_0);
return Some(expr5_0);
}
}
let pattern2_0 = arg2;
if pattern2_0 == pattern1_0 {
// Rule at src/isa/x64/lower.isle line 1932.
let expr0_0 = constructor_output_value(ctx, pattern0_0)?;
return Some(expr0_0);
}
return None;
}
// Generated as internal constructor for term spread_sign_bit.
pub fn constructor_spread_sign_bit<C: Context>(ctx: &mut C, arg0: Gpr) -> Option<Gpr> {
let pattern0_0 = arg0;
// Rule at src/isa/x64/lower.isle line 1938.
let expr0_0: Type = I64;
let expr1_0: u8 = 63;
let expr2_0 = Imm8Reg::Imm8 { imm: expr1_0 };
let expr3_0 = C::imm8_reg_to_imm8_gpr(ctx, &expr2_0);
let expr4_0 = constructor_sar(ctx, expr0_0, pattern0_0, &expr3_0)?;
return Some(expr4_0);
}

16
cranelift/codegen/src/machinst/isle.rs
View File

@@ -237,6 +237,22 @@ macro_rules! isle_prelude_methods {
}
}
#[inline]
fn ty_int_bool_64(&mut self, ty: Type) -> Option<Type> {
match ty {
I64 | B64 => Some(ty),
_ => None,
}
}
#[inline]
fn ty_int_bool_128(&mut self, ty: Type) -> Option<Type> {
match ty {
I128 | B128 => Some(ty),
_ => None,
}
}
fn vec128(&mut self, ty: Type) -> Option<Type> {
if ty.is_vector() && ty.bits() == 128 {
Some(ty)

12
cranelift/codegen/src/prelude.isle
View File

@@ -253,14 +253,22 @@
(decl fits_in_64 (Type) Type)
(extern extractor fits_in_64 fits_in_64)
;; An extractor that maches 32- and 64-bit types only.
;; An extractor that matches 32- and 64-bit types only.
(decl ty_32_or_64 (Type) Type)
(extern extractor ty_32_or_64 ty_32_or_64)
;; An extractor that maches 8- and 16-bit types only.
;; An extractor that matches 8- and 16-bit types only.
(decl ty_8_or_16 (Type) Type)
(extern extractor ty_8_or_16 ty_8_or_16)
;; An extractor that matches I64 or B64.
(decl ty_int_bool_64 (Type) Type)
(extern extractor ty_int_bool_64 ty_int_bool_64)
;; An extractor that matches I128 or B128.
(decl ty_int_bool_128 (Type) Type)
(extern extractor ty_int_bool_128 ty_int_bool_128)
;; An extractor that only matches 128-bit vector types.
(decl vec128 (Type) Type)
(extern extractor vec128 vec128)