T0b1/wasmtime
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

x64: Migrate {s,u}{div,rem} to ISLE (#6008)

Browse Source 
* x64: Add precise-output tests for div traps

This adds a suite of `*.clif` files which are intended to test the
`avoid_div_traps=true` compilation of the `{s,u}{div,rem}` instructions.

* x64: Remove conditional regalloc in `Div` instruction

Move the 8-bit `Div` logic into a dedicated `Div8` instruction to avoid
having conditionally-used registers with respect to regalloc.

* x64: Migrate non-trapping, `udiv`/`urem` to ISLE

* x64: Port checked `udiv` to ISLE

* x64: Migrate urem entirely to ISLE

* x64: Use `test` instead of `cmp` to compare-to-zero

* x64: Port `sdiv` lowering to ISLE

* x64: Port `srem` lowering to ISLE

* Tidy up regalloc behavior and fix tests

* Update docs and winch

* Review comments

* Reword again

* More refactoring test fixes

* More test fixes
This commit is contained in:
Alex Crichton
2023-03-13 20:44:06 -05:00
committed by GitHub
parent 188f712025
commit 5c1b468648
52 changed files with 2178 additions and 835 deletions
Download Patch File Download Diff File Expand all files Collapse all files

206
cranelift/codegen/src/isa/x64/inst/mod.rs
View File

@@ -71,13 +71,17 @@ impl Inst {
| Inst::Bswap { .. }
| Inst::CallKnown { .. }
| Inst::CallUnknown { .. }
| Inst::CheckedDivOrRemSeq { .. }
| Inst::CheckedSRemSeq { .. }
| Inst::CheckedSRemSeq8 { .. }
| Inst::ValidateSdivDivisor { .. }
| Inst::ValidateSdivDivisor64 { .. }
| Inst::Cmove { .. }
| Inst::CmpRmiR { .. }
| Inst::CvtFloatToSintSeq { .. }
| Inst::CvtFloatToUintSeq { .. }
| Inst::CvtUint64ToFloatSeq { .. }
| Inst::Div { .. }
| Inst::Div8 { .. }
| Inst::Fence { .. }
| Inst::Hlt
| Inst::Imm { .. }
@@ -220,7 +224,7 @@ impl Inst {
pub(crate) fn div(
size: OperandSize,
signed: bool,
sign: DivSignedness,
divisor: RegMem,
dividend_lo: Gpr,
dividend_hi: Gpr,
@@ -230,7 +234,7 @@ impl Inst {
divisor.assert_regclass_is(RegClass::Int);
Inst::Div {
size,
signed,
sign,
divisor: GprMem::new(divisor).unwrap(),
dividend_lo,
dividend_hi,
@@ -239,36 +243,21 @@ impl Inst {
}
}
pub(crate) fn checked_div_or_rem_seq(
kind: DivOrRemKind,
size: OperandSize,
divisor: Reg,
dividend_lo: Gpr,
dividend_hi: Gpr,
dst_quotient: WritableGpr,
dst_remainder: WritableGpr,
tmp: Option<Writable<Reg>>,
pub(crate) fn div8(
sign: DivSignedness,
divisor: RegMem,
dividend: Gpr,
dst: WritableGpr,
) -> Inst {
debug_assert!(divisor.class() == RegClass::Int);
debug_assert!(tmp
.map(|tmp| tmp.to_reg().class() == RegClass::Int)
.unwrap_or(true));
Inst::CheckedDivOrRemSeq {
kind,
size,
divisor: Gpr::new(divisor).unwrap(),
dividend_lo,
dividend_hi,
dst_quotient,
dst_remainder,
tmp: tmp.map(|tmp| WritableGpr::from_writable_reg(tmp).unwrap()),
divisor.assert_regclass_is(RegClass::Int);
Inst::Div8 {
sign,
divisor: GprMem::new(divisor).unwrap(),
dividend,
dst,
}
}
pub(crate) fn sign_extend_data(size: OperandSize, src: Gpr, dst: WritableGpr) -> Inst {
Inst::SignExtendData { size, src, dst }
}
pub(crate) fn imm(dst_size: OperandSize, simm64: u64, dst: Writable<Reg>) -> Inst {
debug_assert!(dst_size.is_one_of(&[OperandSize::Size32, OperandSize::Size64]));
debug_assert!(dst.to_reg().class() == RegClass::Int);
@@ -780,33 +769,25 @@ impl PrettyPrint for Inst {
Inst::Div {
size,
signed,
sign,
divisor,
dividend_lo,
dividend_hi,
dst_quotient,
dst_remainder,
} => {
let divisor = divisor.pretty_print(size.to_bytes(), allocs);
let dividend_lo = pretty_print_reg(dividend_lo.to_reg(), size.to_bytes(), allocs);
let dividend_hi = pretty_print_reg(dividend_hi.to_reg(), size.to_bytes(), allocs);
let dst_quotient =
pretty_print_reg(dst_quotient.to_reg().to_reg(), size.to_bytes(), allocs);
let dst_remainder = if size.to_bits() > 8 {
pretty_print_reg(dst_remainder.to_reg().to_reg(), size.to_bytes(), allocs)
} else {
"(none)".to_string()
};
let dividend_hi = if size.to_bits() > 8 {
pretty_print_reg(dividend_hi.to_reg(), size.to_bytes(), allocs)
} else {
"(none)".to_string()
};
let divisor = divisor.pretty_print(size.to_bytes(), allocs);
let dst_remainder =
pretty_print_reg(dst_remainder.to_reg().to_reg(), size.to_bytes(), allocs);
format!(
"{} {}, {}, {}, {}, {}",
ljustify(if *signed {
"idiv".to_string()
} else {
"div".into()
ljustify(match sign {
DivSignedness::Signed => "idiv".to_string(),
DivSignedness::Unsigned => "div".to_string(),
}),
dividend_lo,
dividend_hi,
@@ -816,6 +797,24 @@ impl PrettyPrint for Inst {
)
}
Inst::Div8 {
sign,
divisor,
dividend,
dst,
} => {
let divisor = divisor.pretty_print(1, allocs);
let dividend = pretty_print_reg(dividend.to_reg(), 1, allocs);
let dst = pretty_print_reg(dst.to_reg().to_reg(), 1, allocs);
format!(
"{} {dividend}, {divisor}, {dst}",
ljustify(match sign {
DivSignedness::Signed => "idiv".to_string(),
DivSignedness::Unsigned => "div".to_string(),
}),
)
}
Inst::MulHi {
size,
signed,
@@ -842,43 +841,59 @@ impl PrettyPrint for Inst {
)
}
Inst::CheckedDivOrRemSeq {
kind,
Inst::CheckedSRemSeq {
size,
divisor,
dividend_lo,
dividend_hi,
dst_quotient,
dst_remainder,
tmp,
} => {
let divisor = pretty_print_reg(divisor.to_reg(), size.to_bytes(), allocs);
let dividend_lo = pretty_print_reg(dividend_lo.to_reg(), size.to_bytes(), allocs);
let dividend_hi = pretty_print_reg(dividend_hi.to_reg(), size.to_bytes(), allocs);
let divisor = pretty_print_reg(divisor.to_reg(), size.to_bytes(), allocs);
let dst_quotient =
pretty_print_reg(dst_quotient.to_reg().to_reg(), size.to_bytes(), allocs);
let dst_remainder =
pretty_print_reg(dst_remainder.to_reg().to_reg(), size.to_bytes(), allocs);
let tmp = tmp
.map(|tmp| pretty_print_reg(tmp.to_reg().to_reg(), size.to_bytes(), allocs))
.unwrap_or("(none)".to_string());
format!(
"{} {}, {}, {}, {}, {}, tmp={}",
match kind {
DivOrRemKind::SignedDiv => "sdiv_seq",
DivOrRemKind::UnsignedDiv => "udiv_seq",
DivOrRemKind::SignedRem => "srem_seq",
DivOrRemKind::UnsignedRem => "urem_seq",
},
dividend_lo,
dividend_hi,
divisor,
dst_quotient,
dst_remainder,
tmp,
"checked_srem_seq {dividend_lo}, {dividend_hi}, \
{divisor}, {dst_quotient}, {dst_remainder}",
)
}
Inst::CheckedSRemSeq8 {
divisor,
dividend,
dst,
} => {
let divisor = pretty_print_reg(divisor.to_reg(), 1, allocs);
let dividend = pretty_print_reg(dividend.to_reg(), 1, allocs);
let dst = pretty_print_reg(dst.to_reg().to_reg(), 1, allocs);
format!("checked_srem_seq {dividend}, {divisor}, {dst}")
}
Inst::ValidateSdivDivisor {
dividend,
divisor,
size,
} => {
let dividend = pretty_print_reg(dividend.to_reg(), size.to_bytes(), allocs);
let divisor = pretty_print_reg(divisor.to_reg(), size.to_bytes(), allocs);
format!("validate_sdiv_divisor {dividend}, {divisor}")
}
Inst::ValidateSdivDivisor64 {
dividend,
divisor,
tmp,
} => {
let dividend = pretty_print_reg(dividend.to_reg(), 8, allocs);
let divisor = pretty_print_reg(divisor.to_reg(), 8, allocs);
let tmp = pretty_print_reg(tmp.to_reg().to_reg(), 8, allocs);
format!("validate_sdiv_divisor {dividend}, {divisor} {tmp}")
}
Inst::SignExtendData { size, src, dst } => {
let src = pretty_print_reg(src.to_reg(), size.to_bytes(), allocs);
let dst = pretty_print_reg(dst.to_reg().to_reg(), size.to_bytes(), allocs);
@@ -1857,21 +1872,37 @@ fn x64_get_operands<F: Fn(VReg) -> VReg>(inst: &Inst, collector: &mut OperandCol
collector.reg_reuse_def(dst.to_writable_reg(), 0);
}
Inst::Div {
divisor,
dividend_lo,
dividend_hi,
dst_quotient,
dst_remainder,
size,
..
}
| Inst::CheckedSRemSeq {
dividend_lo,
dividend_hi,
dst_quotient,
dst_remainder,
..
} => {
collector.reg_fixed_use(dividend_lo.to_reg(), regs::rax());
collector.reg_fixed_def(dst_quotient.to_writable_reg(), regs::rax());
if size.to_bits() > 8 {
collector.reg_fixed_def(dst_remainder.to_writable_reg(), regs::rdx());
collector.reg_fixed_use(dividend_hi.to_reg(), regs::rdx());
match inst {
Inst::Div { divisor, .. } => divisor.get_operands(collector),
Inst::CheckedSRemSeq { divisor, .. } => collector.reg_use(divisor.to_reg()),
_ => {}
}
divisor.get_operands(collector);
collector.reg_fixed_use(dividend_lo.to_reg(), regs::rax());
collector.reg_fixed_use(dividend_hi.to_reg(), regs::rdx());
collector.reg_fixed_def(dst_quotient.to_writable_reg(), regs::rax());
collector.reg_fixed_def(dst_remainder.to_writable_reg(), regs::rdx());
}
Inst::Div8 { dividend, dst, .. } | Inst::CheckedSRemSeq8 { dividend, dst, .. } => {
match inst {
Inst::Div8 { divisor, .. } => divisor.get_operands(collector),
Inst::CheckedSRemSeq8 { divisor, .. } => collector.reg_use(divisor.to_reg()),
_ => {}
}
collector.reg_fixed_use(dividend.to_reg(), regs::rax());
collector.reg_fixed_def(dst.to_writable_reg(), regs::rax());
}
Inst::MulHi {
src1,
@@ -1885,25 +1916,20 @@ fn x64_get_operands<F: Fn(VReg) -> VReg>(inst: &Inst, collector: &mut OperandCol
collector.reg_fixed_def(dst_hi.to_writable_reg(), regs::rdx());
src2.get_operands(collector);
}
Inst::CheckedDivOrRemSeq {
divisor,
dividend_lo,
dividend_hi,
dst_quotient,
dst_remainder,
tmp,
..
Inst::ValidateSdivDivisor {
dividend, divisor, ..
} => {
collector.reg_fixed_use(dividend_lo.to_reg(), regs::rax());
collector.reg_fixed_use(dividend_hi.to_reg(), regs::rdx());
collector.reg_use(divisor.to_reg());
collector.reg_fixed_def(dst_quotient.to_writable_reg(), regs::rax());
collector.reg_fixed_def(dst_remainder.to_writable_reg(), regs::rdx());
if let Some(tmp) = tmp {
// Early def so that the temporary register does not
// conflict with inputs or outputs.
collector.reg_early_def(tmp.to_writable_reg());
}
collector.reg_use(dividend.to_reg());
}
Inst::ValidateSdivDivisor64 {
dividend,
divisor,
tmp,
} => {
collector.reg_use(divisor.to_reg());
collector.reg_use(dividend.to_reg());
collector.reg_early_def(tmp.to_writable_reg());
}
Inst::SignExtendData { size, src, dst } => {
match size {