Remove MachInst::gen_constant (#5427)
* aarch64: constant generation cleanup Add support for MOVZ and MOVN generation via ISLE. Handle f32const, f64const, and nop instructions via ISLE. No longer call Inst::gen_constant from lower.rs. * riscv64: constant generation cleanup Handle f32const, f64const, and nop instructions via ISLE. * s390x: constant generation cleanup Fix rule priorities for "imm" term. Only handle 32-bit stack offsets; no longer use load_constant64. * x64: constant generation cleanup No longer call Inst::gen_constant from lower.rs or abi.rs. * Refactor LowerBackend::lower to return InstOutput No longer write to the per-insn output registers; instead, return an InstOutput vector of temp registers holding the outputs. This will allow calling LowerBackend::lower multiple times for the same instruction, e.g. to rematerialize constants. When emitting the primary copy of the instruction during lowering, writing to the per-insn registers is now done in lower_clif_block. As a result, the ISLE lower_common routine is no longer needed. In addition, the InsnOutput type and all code related to it can be removed as well. * Refactor IsleContext to hold a LowerBackend reference Remove the "triple", "flags", and "isa_flags" fields that are copied from LowerBackend to each IsleContext, and instead just hold a reference to LowerBackend in IsleContext. This will allow calling LowerBackend::lower from within callbacks in src/machinst/isle.rs, e.g. to rematerialize constants. To avoid having to pass LowerBackend references through multiple functions, eliminate the lower_insn_to_regs subroutines in those targets that still have them, and just inline into the main lower routine. This also eliminates lower_inst.rs on aarch64 and riscv64. Replace all accesses to the removed IsleContext fields by going through the LowerBackend reference. * Remove MachInst::gen_constant This addresses the problem described in issue https://github.com/bytecodealliance/wasmtime/issues/4426 that targets currently have to duplicate code to emit constants between the ISLE logic and the gen_constant callback. After the various cleanups in earlier patches in this series, the only remaining user of get_constant is put_value_in_regs in Lower. This can now be removed, and instead constant rematerialization can be performed in the put_in_regs ISLE callback by simply directly calling LowerBackend::lower on the instruction defining the constant (using a different output register). Since the check for egraph mode is now no longer performed in put_value_in_regs, the Lower::flags member becomes obsolete. Care needs to be taken that other calls directly to the Lower::put_value_in_regs routine now handle the fact that no more rematerialization is performed. All such calls in target code already historically handle constants themselves. The remaining call site in the ISLE gen_call_common helper can be redirected to the ISLE put_in_regs callback. The existing target implementations of gen_constant are then unused and can be removed. (In some target there may still be further opportunities to remove duplication between ISLE and some local Rust code - this can be left to future patches.)
This commit is contained in:
Ulrich Weigand
@@ -1607,6 +1607,12 @@
|
||||
|
||||
;; Extractor helpers for various immmediate constants ;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
|
||||
(decl pure partial move_wide_const_from_u64 (Type u64) MoveWideConst)
|
||||
(extern constructor move_wide_const_from_u64 move_wide_const_from_u64)
|
||||
|
||||
(decl pure partial move_wide_const_from_inverted_u64 (Type u64) MoveWideConst)
|
||||
(extern constructor move_wide_const_from_inverted_u64 move_wide_const_from_inverted_u64)
|
||||
|
||||
(decl pure partial imm_logic_from_u64 (Type u64) ImmLogic)
|
||||
(extern constructor imm_logic_from_u64 imm_logic_from_u64)
|
||||
|
||||
@@ -2747,6 +2753,15 @@
|
||||
;; such as `I8` are either sign- or zero-extended.
|
||||
(decl imm (Type ImmExtend u64) Reg)
|
||||
|
||||
;; Move wide immediate instructions; to simplify, we only match when we
|
||||
;; are zero-extending the value.
|
||||
(rule 3 (imm (integral_ty ty) (ImmExtend.Zero) k)
|
||||
(if-let n (move_wide_const_from_u64 ty k))
|
||||
(movz n (operand_size ty)))
|
||||
(rule 2 (imm (integral_ty (ty_32_or_64 ty)) (ImmExtend.Zero) k)
|
||||
(if-let n (move_wide_const_from_inverted_u64 ty k))
|
||||
(movn n (operand_size ty)))
|
||||
|
||||
;; Weird logical-instruction immediate in ORI using zero register; to simplify,
|
||||
;; we only match when we are zero-extending the value.
|
||||
(rule 1 (imm (integral_ty ty) (ImmExtend.Zero) k)
|
||||
@@ -2940,6 +2955,11 @@
|
||||
(let ((_ Unit (sink_inst x)))
|
||||
(amode ty addr offset)))
|
||||
|
||||
;; Lower a constant f32.
|
||||
(decl constant_f32 (u64) Reg)
|
||||
;; TODO: Port lower_constant_f32() to ISLE.
|
||||
(extern constructor constant_f32 constant_f32)
|
||||
|
||||
;; Lower a constant f64.
|
||||
(decl constant_f64 (u64) Reg)
|
||||
;; TODO: Port lower_constant_f64() to ISLE.
|
||||
|
||||
@@ -234,27 +234,6 @@ impl Inst {
|
||||
}
|
||||
}
|
||||
|
||||
/// Create instructions that load a 128-bit constant.
|
||||
pub fn load_constant128<F: FnMut(Type) -> Writable<Reg>>(
|
||||
to_regs: ValueRegs<Writable<Reg>>,
|
||||
value: u128,
|
||||
mut alloc_tmp: F,
|
||||
) -> SmallVec<[Inst; 4]> {
|
||||
assert_eq!(to_regs.len(), 2, "Expected to load i128 into two registers");
|
||||
|
||||
let lower = value as u64;
|
||||
let upper = (value >> 64) as u64;
|
||||
|
||||
let lower_reg = to_regs.regs()[0];
|
||||
let upper_reg = to_regs.regs()[1];
|
||||
|
||||
let mut load_ins = Inst::load_constant(lower_reg, lower, &mut alloc_tmp);
|
||||
let load_upper = Inst::load_constant(upper_reg, upper, &mut alloc_tmp);
|
||||
|
||||
load_ins.extend(load_upper.into_iter());
|
||||
load_ins
|
||||
}
|
||||
|
||||
/// Create instructions that load a 32-bit floating-point constant.
|
||||
pub fn load_fp_constant32<F: FnMut(Type) -> Writable<Reg>>(
|
||||
rd: Writable<Reg>,
|
||||
@@ -1233,24 +1212,6 @@ impl MachInst for Inst {
|
||||
}
|
||||
}
|
||||
|
||||
fn gen_constant<F: FnMut(Type) -> Writable<Reg>>(
|
||||
to_regs: ValueRegs<Writable<Reg>>,
|
||||
value: u128,
|
||||
ty: Type,
|
||||
mut alloc_tmp: F,
|
||||
) -> SmallVec<[Inst; 4]> {
|
||||
let to_reg = to_regs.only_reg();
|
||||
match ty {
|
||||
F64 => Inst::load_fp_constant64(to_reg.unwrap(), value as u64, alloc_tmp),
|
||||
F32 => Inst::load_fp_constant32(to_reg.unwrap(), value as u32, alloc_tmp),
|
||||
I8 | I16 | I32 | I64 | R32 | R64 => {
|
||||
Inst::load_constant(to_reg.unwrap(), value as u64, &mut alloc_tmp)
|
||||
}
|
||||
I128 => Inst::load_constant128(to_regs, value, alloc_tmp),
|
||||
_ => panic!("Cannot generate constant for type: {}", ty),
|
||||
}
|
||||
}
|
||||
|
||||
fn gen_dummy_use(reg: Reg) -> Inst {
|
||||
Inst::DummyUse { reg }
|
||||
}
|
||||
|
||||
@@ -24,6 +24,21 @@
|
||||
(rule (lower (has_type ty (null)))
|
||||
(imm ty (ImmExtend.Zero) 0))
|
||||
|
||||
;;;; Rules for `f32const` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
|
||||
(rule (lower (f32const (u64_from_ieee32 n)))
|
||||
(constant_f32 n))
|
||||
|
||||
;;;; Rules for `f64const` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
|
||||
(rule (lower (f64const (u64_from_ieee64 n)))
|
||||
(constant_f64 n))
|
||||
|
||||
;;;; Rules for `nop` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
|
||||
(rule (lower (nop))
|
||||
(invalid_reg))
|
||||
|
||||
;;;; Rules for `iadd` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
|
||||
;; `i64` and smaller
|
||||
|
||||
@@ -7,7 +7,6 @@
|
||||
//!
|
||||
//! - Floating-point immediates (FIMM instruction).
|
||||
|
||||
use super::lower_inst;
|
||||
use crate::ir::condcodes::{FloatCC, IntCC};
|
||||
use crate::ir::types::*;
|
||||
use crate::ir::Inst as IRInst;
|
||||
@@ -16,6 +15,7 @@ use crate::isa::aarch64::inst::*;
|
||||
use crate::isa::aarch64::AArch64Backend;
|
||||
use crate::machinst::lower::*;
|
||||
use crate::machinst::{Reg, Writable};
|
||||
use crate::CodegenError;
|
||||
use crate::CodegenResult;
|
||||
use crate::{machinst::*, trace};
|
||||
use smallvec::{smallvec, SmallVec};
|
||||
@@ -43,27 +43,6 @@ impl NarrowValueMode {
|
||||
}
|
||||
}
|
||||
|
||||
/// Emits instruction(s) to generate the given constant value into newly-allocated
|
||||
/// temporary registers, returning these registers.
|
||||
fn generate_constant(ctx: &mut Lower<Inst>, ty: Type, c: u128) -> ValueRegs<Reg> {
|
||||
let from_bits = ty_bits(ty);
|
||||
let masked = if from_bits < 128 {
|
||||
c & ((1u128 << from_bits) - 1)
|
||||
} else {
|
||||
c
|
||||
};
|
||||
|
||||
let cst_copy = ctx.alloc_tmp(ty);
|
||||
for inst in Inst::gen_constant(cst_copy, masked, ty, |ty| {
|
||||
ctx.alloc_tmp(ty).only_reg().unwrap()
|
||||
})
|
||||
.into_iter()
|
||||
{
|
||||
ctx.emit(inst);
|
||||
}
|
||||
non_writable_value_regs(cst_copy)
|
||||
}
|
||||
|
||||
/// Extends a register according to `narrow_mode`.
|
||||
/// If extended, the value is always extended to 64 bits, for simplicity.
|
||||
fn extend_reg(
|
||||
@@ -112,7 +91,20 @@ fn lower_value_to_regs(ctx: &mut Lower<Inst>, value: Value) -> (ValueRegs<Reg>,
|
||||
|
||||
let in_regs = if let Some(c) = inputs.constant {
|
||||
// Generate constants fresh at each use to minimize long-range register pressure.
|
||||
generate_constant(ctx, ty, c as u128)
|
||||
let from_bits = ty_bits(ty);
|
||||
let c = if from_bits < 64 {
|
||||
c & ((1u64 << from_bits) - 1)
|
||||
} else {
|
||||
c
|
||||
};
|
||||
match ty {
|
||||
I8 | I16 | I32 | I64 | R32 | R64 => {
|
||||
let cst_copy = ctx.alloc_tmp(ty);
|
||||
lower_constant_u64(ctx, cst_copy.only_reg().unwrap(), c);
|
||||
non_writable_value_regs(cst_copy)
|
||||
}
|
||||
_ => unreachable!(), // Only used for addresses.
|
||||
}
|
||||
} else {
|
||||
ctx.put_value_in_regs(value)
|
||||
};
|
||||
@@ -754,8 +746,227 @@ pub(crate) fn maybe_value_multi(
|
||||
impl LowerBackend for AArch64Backend {
|
||||
type MInst = Inst;
|
||||
|
||||
fn lower(&self, ctx: &mut Lower<Inst>, ir_inst: IRInst) -> CodegenResult<()> {
|
||||
lower_inst::lower_insn_to_regs(ctx, ir_inst, &self.triple, &self.flags, &self.isa_flags)
|
||||
fn lower(&self, ctx: &mut Lower<Inst>, ir_inst: IRInst) -> CodegenResult<InstOutput> {
|
||||
if let Some(temp_regs) = super::lower::isle::lower(ctx, self, ir_inst) {
|
||||
return Ok(temp_regs);
|
||||
}
|
||||
|
||||
let op = ctx.data(ir_inst).opcode();
|
||||
let ty = if ctx.num_outputs(ir_inst) > 0 {
|
||||
Some(ctx.output_ty(ir_inst, 0))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
match op {
|
||||
Opcode::Iconst
|
||||
| Opcode::Null
|
||||
| Opcode::F32const
|
||||
| Opcode::F64const
|
||||
| Opcode::GetFramePointer
|
||||
| Opcode::GetStackPointer
|
||||
| Opcode::GetReturnAddress
|
||||
| Opcode::Iadd
|
||||
| Opcode::Isub
|
||||
| Opcode::UaddSat
|
||||
| Opcode::SaddSat
|
||||
| Opcode::UsubSat
|
||||
| Opcode::SsubSat
|
||||
| Opcode::Ineg
|
||||
| Opcode::Imul
|
||||
| Opcode::Umulhi
|
||||
| Opcode::Smulhi
|
||||
| Opcode::Udiv
|
||||
| Opcode::Sdiv
|
||||
| Opcode::Urem
|
||||
| Opcode::Srem
|
||||
| Opcode::Uextend
|
||||
| Opcode::Sextend
|
||||
| Opcode::Bnot
|
||||
| Opcode::Band
|
||||
| Opcode::Bor
|
||||
| Opcode::Bxor
|
||||
| Opcode::BandNot
|
||||
| Opcode::BorNot
|
||||
| Opcode::BxorNot
|
||||
| Opcode::Ishl
|
||||
| Opcode::Ushr
|
||||
| Opcode::Sshr
|
||||
| Opcode::Rotr
|
||||
| Opcode::Rotl
|
||||
| Opcode::Bitrev
|
||||
| Opcode::Clz
|
||||
| Opcode::Cls
|
||||
| Opcode::Ctz
|
||||
| Opcode::Bswap
|
||||
| Opcode::Popcnt
|
||||
| Opcode::Load
|
||||
| Opcode::Uload8
|
||||
| Opcode::Sload8
|
||||
| Opcode::Uload16
|
||||
| Opcode::Sload16
|
||||
| Opcode::Uload32
|
||||
| Opcode::Sload32
|
||||
| Opcode::Sload8x8
|
||||
| Opcode::Uload8x8
|
||||
| Opcode::Sload16x4
|
||||
| Opcode::Uload16x4
|
||||
| Opcode::Sload32x2
|
||||
| Opcode::Uload32x2
|
||||
| Opcode::Store
|
||||
| Opcode::Istore8
|
||||
| Opcode::Istore16
|
||||
| Opcode::Istore32
|
||||
| Opcode::StackAddr
|
||||
| Opcode::DynamicStackAddr
|
||||
| Opcode::AtomicRmw
|
||||
| Opcode::AtomicCas
|
||||
| Opcode::AtomicLoad
|
||||
| Opcode::AtomicStore
|
||||
| Opcode::Fence
|
||||
| Opcode::Nop
|
||||
| Opcode::Select
|
||||
| Opcode::SelectSpectreGuard
|
||||
| Opcode::Bitselect
|
||||
| Opcode::Vselect
|
||||
| Opcode::IsNull
|
||||
| Opcode::IsInvalid
|
||||
| Opcode::Ireduce
|
||||
| Opcode::Bmask
|
||||
| Opcode::Bitcast
|
||||
| Opcode::Return
|
||||
| Opcode::Icmp
|
||||
| Opcode::Fcmp
|
||||
| Opcode::Debugtrap
|
||||
| Opcode::Trap
|
||||
| Opcode::ResumableTrap
|
||||
| Opcode::FuncAddr
|
||||
| Opcode::SymbolValue
|
||||
| Opcode::Call
|
||||
| Opcode::CallIndirect
|
||||
| Opcode::GetPinnedReg
|
||||
| Opcode::SetPinnedReg
|
||||
| Opcode::Vconst
|
||||
| Opcode::Extractlane
|
||||
| Opcode::Insertlane
|
||||
| Opcode::Splat
|
||||
| Opcode::ScalarToVector
|
||||
| Opcode::VallTrue
|
||||
| Opcode::VanyTrue
|
||||
| Opcode::VhighBits
|
||||
| Opcode::Shuffle
|
||||
| Opcode::Swizzle
|
||||
| Opcode::Isplit
|
||||
| Opcode::Iconcat
|
||||
| Opcode::Smax
|
||||
| Opcode::Umax
|
||||
| Opcode::Umin
|
||||
| Opcode::Smin
|
||||
| Opcode::IaddPairwise
|
||||
| Opcode::WideningPairwiseDotProductS
|
||||
| Opcode::Fadd
|
||||
| Opcode::Fsub
|
||||
| Opcode::Fmul
|
||||
| Opcode::Fdiv
|
||||
| Opcode::Fmin
|
||||
| Opcode::Fmax
|
||||
| Opcode::FminPseudo
|
||||
| Opcode::FmaxPseudo
|
||||
| Opcode::Sqrt
|
||||
| Opcode::Fneg
|
||||
| Opcode::Fabs
|
||||
| Opcode::Fpromote
|
||||
| Opcode::Fdemote
|
||||
| Opcode::Ceil
|
||||
| Opcode::Floor
|
||||
| Opcode::Trunc
|
||||
| Opcode::Nearest
|
||||
| Opcode::Fma
|
||||
| Opcode::Fcopysign
|
||||
| Opcode::FcvtToUint
|
||||
| Opcode::FcvtToSint
|
||||
| Opcode::FcvtFromUint
|
||||
| Opcode::FcvtFromSint
|
||||
| Opcode::FcvtToUintSat
|
||||
| Opcode::FcvtToSintSat
|
||||
| Opcode::UaddOverflowTrap
|
||||
| Opcode::IaddCout
|
||||
| Opcode::Iabs
|
||||
| Opcode::AvgRound
|
||||
| Opcode::Snarrow
|
||||
| Opcode::Unarrow
|
||||
| Opcode::Uunarrow
|
||||
| Opcode::SwidenLow
|
||||
| Opcode::SwidenHigh
|
||||
| Opcode::UwidenLow
|
||||
| Opcode::UwidenHigh
|
||||
| Opcode::TlsValue
|
||||
| Opcode::SqmulRoundSat
|
||||
| Opcode::FcvtLowFromSint
|
||||
| Opcode::FvpromoteLow
|
||||
| Opcode::Fvdemote
|
||||
| Opcode::ExtractVector => {
|
||||
unreachable!(
|
||||
"implemented in ISLE: inst = `{}`, type = `{:?}`",
|
||||
ctx.dfg().display_inst(ir_inst),
|
||||
ty
|
||||
);
|
||||
}
|
||||
|
||||
Opcode::StackLoad
|
||||
| Opcode::StackStore
|
||||
| Opcode::DynamicStackStore
|
||||
| Opcode::DynamicStackLoad => {
|
||||
panic!("Direct stack memory access not supported; should not be used by Wasm");
|
||||
}
|
||||
Opcode::HeapLoad | Opcode::HeapStore | Opcode::HeapAddr => {
|
||||
panic!("heap access instructions should have been removed by legalization!");
|
||||
}
|
||||
Opcode::TableAddr => {
|
||||
panic!("table_addr should have been removed by legalization!");
|
||||
}
|
||||
Opcode::Trapz | Opcode::Trapnz | Opcode::ResumableTrapnz => {
|
||||
panic!(
|
||||
"trapz / trapnz / resumable_trapnz should have been removed by legalization!"
|
||||
);
|
||||
}
|
||||
Opcode::GlobalValue => {
|
||||
panic!("global_value should have been removed by legalization!");
|
||||
}
|
||||
Opcode::Jump | Opcode::Brz | Opcode::Brnz | Opcode::BrTable => {
|
||||
panic!("Branch opcode reached non-branch lowering logic!");
|
||||
}
|
||||
Opcode::IaddImm
|
||||
| Opcode::ImulImm
|
||||
| Opcode::UdivImm
|
||||
| Opcode::SdivImm
|
||||
| Opcode::UremImm
|
||||
| Opcode::SremImm
|
||||
| Opcode::IrsubImm
|
||||
| Opcode::IaddCin
|
||||
| Opcode::IaddCarry
|
||||
| Opcode::IsubBin
|
||||
| Opcode::IsubBout
|
||||
| Opcode::IsubBorrow
|
||||
| Opcode::BandImm
|
||||
| Opcode::BorImm
|
||||
| Opcode::BxorImm
|
||||
| Opcode::RotlImm
|
||||
| Opcode::RotrImm
|
||||
| Opcode::IshlImm
|
||||
| Opcode::UshrImm
|
||||
| Opcode::SshrImm
|
||||
| Opcode::IcmpImm => {
|
||||
panic!("ALU+imm and ALU+carry ops should not appear here!");
|
||||
}
|
||||
|
||||
Opcode::Vconcat | Opcode::Vsplit => {
|
||||
return Err(CodegenError::Unsupported(format!(
|
||||
"Unimplemented lowering: {}",
|
||||
op
|
||||
)));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn lower_branch_group(
|
||||
@@ -776,14 +987,8 @@ impl LowerBackend for AArch64Backend {
|
||||
assert!(op1 == Opcode::Jump);
|
||||
}
|
||||
|
||||
if let Ok(()) = super::lower::isle::lower_branch(
|
||||
ctx,
|
||||
&self.triple,
|
||||
&self.flags,
|
||||
&self.isa_flags,
|
||||
branches[0],
|
||||
targets,
|
||||
) {
|
||||
if let Some(temp_regs) = super::lower::isle::lower_branch(ctx, self, branches[0], targets) {
|
||||
assert!(temp_regs.len() == 0);
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
|
||||
@@ -18,10 +18,9 @@ use super::{
|
||||
use crate::ir::condcodes;
|
||||
use crate::isa::aarch64::inst::{FPULeftShiftImm, FPURightShiftImm};
|
||||
use crate::isa::aarch64::lower::{lower_address, lower_pair_address, lower_splat_const};
|
||||
use crate::isa::aarch64::settings::Flags as IsaFlags;
|
||||
use crate::isa::aarch64::AArch64Backend;
|
||||
use crate::machinst::valueregs;
|
||||
use crate::machinst::{isle::*, InputSourceInst};
|
||||
use crate::settings::Flags;
|
||||
use crate::{
|
||||
binemit::CodeOffset,
|
||||
ir::{
|
||||
@@ -32,7 +31,7 @@ use crate::{
|
||||
isa::aarch64::inst::args::{ShiftOp, ShiftOpShiftImm},
|
||||
isa::unwind::UnwindInst,
|
||||
machinst::{
|
||||
abi::ArgPair, ty_bits, InsnOutput, Lower, MachInst, VCodeConstant, VCodeConstantData,
|
||||
abi::ArgPair, ty_bits, InstOutput, Lower, MachInst, VCodeConstant, VCodeConstantData,
|
||||
},
|
||||
};
|
||||
use crate::{isle_common_prelude_methods, isle_lower_prelude_methods};
|
||||
@@ -40,7 +39,6 @@ use regalloc2::PReg;
|
||||
use std::boxed::Box;
|
||||
use std::convert::TryFrom;
|
||||
use std::vec::Vec;
|
||||
use target_lexicon::Triple;
|
||||
|
||||
type BoxCallInfo = Box<CallInfo>;
|
||||
type BoxCallIndInfo = Box<CallIndInfo>;
|
||||
@@ -52,40 +50,25 @@ type VecArgPair = Vec<ArgPair>;
|
||||
/// The main entry point for lowering with ISLE.
|
||||
pub(crate) fn lower(
|
||||
lower_ctx: &mut Lower<MInst>,
|
||||
triple: &Triple,
|
||||
flags: &Flags,
|
||||
isa_flags: &IsaFlags,
|
||||
outputs: &[InsnOutput],
|
||||
backend: &AArch64Backend,
|
||||
inst: Inst,
|
||||
) -> Result<(), ()> {
|
||||
lower_common(
|
||||
lower_ctx,
|
||||
triple,
|
||||
flags,
|
||||
isa_flags,
|
||||
outputs,
|
||||
inst,
|
||||
|cx, insn| generated_code::constructor_lower(cx, insn),
|
||||
)
|
||||
) -> Option<InstOutput> {
|
||||
// TODO: reuse the ISLE context across lowerings so we can reuse its
|
||||
// internal heap allocations.
|
||||
let mut isle_ctx = IsleContext { lower_ctx, backend };
|
||||
generated_code::constructor_lower(&mut isle_ctx, inst)
|
||||
}
|
||||
|
||||
pub(crate) fn lower_branch(
|
||||
lower_ctx: &mut Lower<MInst>,
|
||||
triple: &Triple,
|
||||
flags: &Flags,
|
||||
isa_flags: &IsaFlags,
|
||||
backend: &AArch64Backend,
|
||||
branch: Inst,
|
||||
targets: &[MachLabel],
|
||||
) -> Result<(), ()> {
|
||||
lower_common(
|
||||
lower_ctx,
|
||||
triple,
|
||||
flags,
|
||||
isa_flags,
|
||||
&[],
|
||||
branch,
|
||||
|cx, insn| generated_code::constructor_lower_branch(cx, insn, &targets.to_vec()),
|
||||
)
|
||||
) -> Option<InstOutput> {
|
||||
// TODO: reuse the ISLE context across lowerings so we can reuse its
|
||||
// internal heap allocations.
|
||||
let mut isle_ctx = IsleContext { lower_ctx, backend };
|
||||
generated_code::constructor_lower_branch(&mut isle_ctx, branch, &targets.to_vec())
|
||||
}
|
||||
|
||||
pub struct ExtendedValue {
|
||||
@@ -93,16 +76,16 @@ pub struct ExtendedValue {
|
||||
extend: ExtendOp,
|
||||
}
|
||||
|
||||
impl IsleContext<'_, '_, MInst, Flags, IsaFlags, 6> {
|
||||
impl IsleContext<'_, '_, MInst, AArch64Backend> {
|
||||
isle_prelude_method_helpers!(AArch64Caller);
|
||||
}
|
||||
|
||||
impl Context for IsleContext<'_, '_, MInst, Flags, IsaFlags, 6> {
|
||||
impl Context for IsleContext<'_, '_, MInst, AArch64Backend> {
|
||||
isle_lower_prelude_methods!();
|
||||
isle_prelude_caller_methods!(crate::isa::aarch64::abi::AArch64MachineDeps, AArch64Caller);
|
||||
|
||||
fn sign_return_address_disabled(&mut self) -> Option<()> {
|
||||
if self.isa_flags.sign_return_address() {
|
||||
if self.backend.isa_flags.sign_return_address() {
|
||||
None
|
||||
} else {
|
||||
Some(())
|
||||
@@ -110,13 +93,27 @@ impl Context for IsleContext<'_, '_, MInst, Flags, IsaFlags, 6> {
|
||||
}
|
||||
|
||||
fn use_lse(&mut self, _: Inst) -> Option<()> {
|
||||
if self.isa_flags.has_lse() {
|
||||
if self.backend.isa_flags.has_lse() {
|
||||
Some(())
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
fn move_wide_const_from_u64(&mut self, ty: Type, n: u64) -> Option<MoveWideConst> {
|
||||
let bits = ty.bits();
|
||||
let n = if bits < 64 {
|
||||
n & !(u64::MAX << bits)
|
||||
} else {
|
||||
n
|
||||
};
|
||||
MoveWideConst::maybe_from_u64(n)
|
||||
}
|
||||
|
||||
fn move_wide_const_from_inverted_u64(&mut self, ty: Type, n: u64) -> Option<MoveWideConst> {
|
||||
self.move_wide_const_from_u64(ty, !n)
|
||||
}
|
||||
|
||||
fn imm_logic_from_u64(&mut self, ty: Type, n: u64) -> Option<ImmLogic> {
|
||||
ImmLogic::maybe_from_u64(n, ty)
|
||||
}
|
||||
@@ -523,6 +520,14 @@ impl Context for IsleContext<'_, '_, MInst, Flags, IsaFlags, 6> {
|
||||
lower_pair_address(self.lower_ctx, addr, offset as i32)
|
||||
}
|
||||
|
||||
fn constant_f32(&mut self, value: u64) -> Reg {
|
||||
let rd = self.temp_writable_reg(I8X16);
|
||||
|
||||
lower_constant_f32(self.lower_ctx, rd, f32::from_bits(value as u32));
|
||||
|
||||
rd.to_reg()
|
||||
}
|
||||
|
||||
fn constant_f64(&mut self, value: u64) -> Reg {
|
||||
let rd = self.temp_writable_reg(I8X16);
|
||||
|
||||
|
||||
@@ -1,306 +0,0 @@
|
||||
//! Lower a single Cranelift instruction into vcode.
|
||||
|
||||
use crate::ir::Inst as IRInst;
|
||||
use crate::ir::Opcode;
|
||||
use crate::isa::aarch64::inst::*;
|
||||
use crate::isa::aarch64::settings as aarch64_settings;
|
||||
use crate::machinst::lower::*;
|
||||
use crate::machinst::*;
|
||||
use crate::settings::Flags;
|
||||
use crate::{CodegenError, CodegenResult};
|
||||
use target_lexicon::Triple;
|
||||
|
||||
/// Actually codegen an instruction's results into registers.
|
||||
pub(crate) fn lower_insn_to_regs(
|
||||
ctx: &mut Lower<Inst>,
|
||||
insn: IRInst,
|
||||
triple: &Triple,
|
||||
flags: &Flags,
|
||||
isa_flags: &aarch64_settings::Flags,
|
||||
) -> CodegenResult<()> {
|
||||
let op = ctx.data(insn).opcode();
|
||||
let outputs = insn_outputs(ctx, insn);
|
||||
let ty = if outputs.len() > 0 {
|
||||
Some(ctx.output_ty(insn, 0))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
if let Ok(()) = super::lower::isle::lower(ctx, triple, flags, isa_flags, &outputs, insn) {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let implemented_in_isle = |ctx: &mut Lower<Inst>| -> ! {
|
||||
unreachable!(
|
||||
"implemented in ISLE: inst = `{}`, type = `{:?}`",
|
||||
ctx.dfg().display_inst(insn),
|
||||
ty
|
||||
);
|
||||
};
|
||||
|
||||
match op {
|
||||
Opcode::Iconst | Opcode::Null => implemented_in_isle(ctx),
|
||||
|
||||
Opcode::F32const => {
|
||||
let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
|
||||
let val = ctx.get_constant(insn).unwrap();
|
||||
for inst in
|
||||
Inst::load_fp_constant32(rd, val as u32, |ty| ctx.alloc_tmp(ty).only_reg().unwrap())
|
||||
{
|
||||
ctx.emit(inst);
|
||||
}
|
||||
}
|
||||
|
||||
Opcode::F64const => {
|
||||
let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
|
||||
let val = ctx.get_constant(insn).unwrap();
|
||||
for inst in
|
||||
Inst::load_fp_constant64(rd, val, |ty| ctx.alloc_tmp(ty).only_reg().unwrap())
|
||||
{
|
||||
ctx.emit(inst);
|
||||
}
|
||||
}
|
||||
|
||||
Opcode::GetFramePointer | Opcode::GetStackPointer | Opcode::GetReturnAddress => {
|
||||
implemented_in_isle(ctx)
|
||||
}
|
||||
|
||||
Opcode::Iadd => implemented_in_isle(ctx),
|
||||
Opcode::Isub => implemented_in_isle(ctx),
|
||||
Opcode::UaddSat | Opcode::SaddSat | Opcode::UsubSat | Opcode::SsubSat => {
|
||||
implemented_in_isle(ctx)
|
||||
}
|
||||
|
||||
Opcode::Ineg => implemented_in_isle(ctx),
|
||||
|
||||
Opcode::Imul => implemented_in_isle(ctx),
|
||||
|
||||
Opcode::Umulhi | Opcode::Smulhi => implemented_in_isle(ctx),
|
||||
|
||||
Opcode::Udiv | Opcode::Sdiv | Opcode::Urem | Opcode::Srem => implemented_in_isle(ctx),
|
||||
|
||||
Opcode::Uextend | Opcode::Sextend => implemented_in_isle(ctx),
|
||||
|
||||
Opcode::Bnot => implemented_in_isle(ctx),
|
||||
|
||||
Opcode::Band
|
||||
| Opcode::Bor
|
||||
| Opcode::Bxor
|
||||
| Opcode::BandNot
|
||||
| Opcode::BorNot
|
||||
| Opcode::BxorNot => implemented_in_isle(ctx),
|
||||
|
||||
Opcode::Ishl | Opcode::Ushr | Opcode::Sshr => implemented_in_isle(ctx),
|
||||
|
||||
Opcode::Rotr | Opcode::Rotl => implemented_in_isle(ctx),
|
||||
|
||||
Opcode::Bitrev | Opcode::Clz | Opcode::Cls | Opcode::Ctz => implemented_in_isle(ctx),
|
||||
|
||||
Opcode::Bswap => implemented_in_isle(ctx),
|
||||
|
||||
Opcode::Popcnt => implemented_in_isle(ctx),
|
||||
|
||||
Opcode::Load
|
||||
| Opcode::Uload8
|
||||
| Opcode::Sload8
|
||||
| Opcode::Uload16
|
||||
| Opcode::Sload16
|
||||
| Opcode::Uload32
|
||||
| Opcode::Sload32
|
||||
| Opcode::Sload8x8
|
||||
| Opcode::Uload8x8
|
||||
| Opcode::Sload16x4
|
||||
| Opcode::Uload16x4
|
||||
| Opcode::Sload32x2
|
||||
| Opcode::Uload32x2 => implemented_in_isle(ctx),
|
||||
|
||||
Opcode::Store | Opcode::Istore8 | Opcode::Istore16 | Opcode::Istore32 => {
|
||||
implemented_in_isle(ctx)
|
||||
}
|
||||
|
||||
Opcode::StackAddr => implemented_in_isle(ctx),
|
||||
|
||||
Opcode::DynamicStackAddr => implemented_in_isle(ctx),
|
||||
|
||||
Opcode::AtomicRmw => implemented_in_isle(ctx),
|
||||
|
||||
Opcode::AtomicCas => implemented_in_isle(ctx),
|
||||
|
||||
Opcode::AtomicLoad => implemented_in_isle(ctx),
|
||||
|
||||
Opcode::AtomicStore => implemented_in_isle(ctx),
|
||||
|
||||
Opcode::Fence => implemented_in_isle(ctx),
|
||||
|
||||
Opcode::StackLoad
|
||||
| Opcode::StackStore
|
||||
| Opcode::DynamicStackStore
|
||||
| Opcode::DynamicStackLoad => {
|
||||
panic!("Direct stack memory access not supported; should not be used by Wasm");
|
||||
}
|
||||
|
||||
Opcode::HeapLoad | Opcode::HeapStore | Opcode::HeapAddr => {
|
||||
panic!("heap access instructions should have been removed by legalization!");
|
||||
}
|
||||
|
||||
Opcode::TableAddr => {
|
||||
panic!("table_addr should have been removed by legalization!");
|
||||
}
|
||||
|
||||
Opcode::Nop => {
|
||||
// Nothing.
|
||||
}
|
||||
|
||||
Opcode::Select => implemented_in_isle(ctx),
|
||||
|
||||
Opcode::SelectSpectreGuard => implemented_in_isle(ctx),
|
||||
|
||||
Opcode::Bitselect | Opcode::Vselect => implemented_in_isle(ctx),
|
||||
|
||||
Opcode::IsNull | Opcode::IsInvalid => implemented_in_isle(ctx),
|
||||
|
||||
Opcode::Ireduce => implemented_in_isle(ctx),
|
||||
|
||||
Opcode::Bmask => implemented_in_isle(ctx),
|
||||
|
||||
Opcode::Bitcast => implemented_in_isle(ctx),
|
||||
|
||||
Opcode::Return => implemented_in_isle(ctx),
|
||||
|
||||
Opcode::Icmp => implemented_in_isle(ctx),
|
||||
|
||||
Opcode::Fcmp => implemented_in_isle(ctx),
|
||||
|
||||
Opcode::Debugtrap => implemented_in_isle(ctx),
|
||||
|
||||
Opcode::Trap | Opcode::ResumableTrap => implemented_in_isle(ctx),
|
||||
|
||||
Opcode::Trapz | Opcode::Trapnz | Opcode::ResumableTrapnz => {
|
||||
panic!("trapz / trapnz / resumable_trapnz should have been removed by legalization!");
|
||||
}
|
||||
|
||||
Opcode::FuncAddr => implemented_in_isle(ctx),
|
||||
|
||||
Opcode::GlobalValue => {
|
||||
panic!("global_value should have been removed by legalization!");
|
||||
}
|
||||
|
||||
Opcode::SymbolValue => implemented_in_isle(ctx),
|
||||
|
||||
Opcode::Call | Opcode::CallIndirect => implemented_in_isle(ctx),
|
||||
|
||||
Opcode::GetPinnedReg | Opcode::SetPinnedReg => implemented_in_isle(ctx),
|
||||
|
||||
Opcode::Jump | Opcode::Brz | Opcode::Brnz | Opcode::BrTable => {
|
||||
panic!("Branch opcode reached non-branch lowering logic!");
|
||||
}
|
||||
|
||||
Opcode::Vconst => implemented_in_isle(ctx),
|
||||
|
||||
Opcode::Extractlane => implemented_in_isle(ctx),
|
||||
|
||||
Opcode::Insertlane => implemented_in_isle(ctx),
|
||||
|
||||
Opcode::Splat => implemented_in_isle(ctx),
|
||||
|
||||
Opcode::ScalarToVector => implemented_in_isle(ctx),
|
||||
|
||||
Opcode::VallTrue | Opcode::VanyTrue => implemented_in_isle(ctx),
|
||||
|
||||
Opcode::VhighBits => implemented_in_isle(ctx),
|
||||
|
||||
Opcode::Shuffle => implemented_in_isle(ctx),
|
||||
|
||||
Opcode::Swizzle => implemented_in_isle(ctx),
|
||||
|
||||
Opcode::Isplit => implemented_in_isle(ctx),
|
||||
|
||||
Opcode::Iconcat => implemented_in_isle(ctx),
|
||||
|
||||
Opcode::Smax | Opcode::Umax | Opcode::Umin | Opcode::Smin => implemented_in_isle(ctx),
|
||||
|
||||
Opcode::IaddPairwise => implemented_in_isle(ctx),
|
||||
|
||||
Opcode::WideningPairwiseDotProductS => implemented_in_isle(ctx),
|
||||
|
||||
Opcode::Fadd | Opcode::Fsub | Opcode::Fmul | Opcode::Fdiv | Opcode::Fmin | Opcode::Fmax => {
|
||||
implemented_in_isle(ctx)
|
||||
}
|
||||
|
||||
Opcode::FminPseudo | Opcode::FmaxPseudo => implemented_in_isle(ctx),
|
||||
|
||||
Opcode::Sqrt | Opcode::Fneg | Opcode::Fabs | Opcode::Fpromote | Opcode::Fdemote => {
|
||||
implemented_in_isle(ctx)
|
||||
}
|
||||
|
||||
Opcode::Ceil | Opcode::Floor | Opcode::Trunc | Opcode::Nearest => implemented_in_isle(ctx),
|
||||
|
||||
Opcode::Fma => implemented_in_isle(ctx),
|
||||
|
||||
Opcode::Fcopysign => implemented_in_isle(ctx),
|
||||
|
||||
Opcode::FcvtToUint | Opcode::FcvtToSint => implemented_in_isle(ctx),
|
||||
|
||||
Opcode::FcvtFromUint | Opcode::FcvtFromSint => implemented_in_isle(ctx),
|
||||
|
||||
Opcode::FcvtToUintSat | Opcode::FcvtToSintSat => implemented_in_isle(ctx),
|
||||
|
||||
Opcode::UaddOverflowTrap => implemented_in_isle(ctx),
|
||||
|
||||
Opcode::IaddCout => implemented_in_isle(ctx),
|
||||
|
||||
Opcode::IaddImm
|
||||
| Opcode::ImulImm
|
||||
| Opcode::UdivImm
|
||||
| Opcode::SdivImm
|
||||
| Opcode::UremImm
|
||||
| Opcode::SremImm
|
||||
| Opcode::IrsubImm
|
||||
| Opcode::IaddCin
|
||||
| Opcode::IaddCarry
|
||||
| Opcode::IsubBin
|
||||
| Opcode::IsubBout
|
||||
| Opcode::IsubBorrow
|
||||
| Opcode::BandImm
|
||||
| Opcode::BorImm
|
||||
| Opcode::BxorImm
|
||||
| Opcode::RotlImm
|
||||
| Opcode::RotrImm
|
||||
| Opcode::IshlImm
|
||||
| Opcode::UshrImm
|
||||
| Opcode::SshrImm
|
||||
| Opcode::IcmpImm => {
|
||||
panic!("ALU+imm and ALU+carry ops should not appear here!");
|
||||
}
|
||||
|
||||
Opcode::Iabs => implemented_in_isle(ctx),
|
||||
Opcode::AvgRound => implemented_in_isle(ctx),
|
||||
|
||||
Opcode::Snarrow | Opcode::Unarrow | Opcode::Uunarrow => implemented_in_isle(ctx),
|
||||
|
||||
Opcode::SwidenLow | Opcode::SwidenHigh | Opcode::UwidenLow | Opcode::UwidenHigh => {
|
||||
implemented_in_isle(ctx)
|
||||
}
|
||||
|
||||
Opcode::TlsValue => implemented_in_isle(ctx),
|
||||
|
||||
Opcode::SqmulRoundSat => implemented_in_isle(ctx),
|
||||
|
||||
Opcode::FcvtLowFromSint => implemented_in_isle(ctx),
|
||||
|
||||
Opcode::FvpromoteLow => implemented_in_isle(ctx),
|
||||
|
||||
Opcode::Fvdemote => implemented_in_isle(ctx),
|
||||
|
||||
Opcode::ExtractVector => implemented_in_isle(ctx),
|
||||
|
||||
Opcode::Vconcat | Opcode::Vsplit => {
|
||||
return Err(CodegenError::Unsupported(format!(
|
||||
"Unimplemented lowering: {}",
|
||||
op
|
||||
)));
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -21,7 +21,6 @@ use target_lexicon::{Aarch64Architecture, Architecture, OperatingSystem, Triple}
|
||||
mod abi;
|
||||
pub(crate) mod inst;
|
||||
mod lower;
|
||||
mod lower_inst;
|
||||
mod settings;
|
||||
|
||||
use inst::create_reg_env;
|
||||
@@ -267,11 +266,11 @@ mod test {
|
||||
// on it to update:
|
||||
// > aarch64-linux-gnu-objdump -b binary -D <file> -m aarch64
|
||||
//
|
||||
// 0: d2824682 mov x2, #0x1234 // #4660
|
||||
// 0: 52824682 mov w2, #0x1234 // #4660
|
||||
// 4: 0b020000 add w0, w0, w2
|
||||
// 8: d65f03c0 ret
|
||||
|
||||
let golden = vec![130, 70, 130, 210, 0, 0, 2, 11, 192, 3, 95, 214];
|
||||
let golden = vec![130, 70, 130, 82, 0, 0, 2, 11, 192, 3, 95, 214];
|
||||
|
||||
assert_eq!(code, &golden[..]);
|
||||
}
|
||||
@@ -325,24 +324,24 @@ mod test {
|
||||
// on it to update:
|
||||
// > aarch64-linux-gnu-objdump -b binary -D <file> -m aarch64
|
||||
//
|
||||
// 0: d2824689 mov x9, #0x1234 // #4660
|
||||
// 0: 52824689 mov w9, #0x1234 // #4660
|
||||
// 4: 0b09000b add w11, w0, w9
|
||||
// 8: 2a0b03ea mov w10, w11
|
||||
// c: b50000aa cbnz x10, 0x20
|
||||
// 10: d282468c mov x12, #0x1234 // #4660
|
||||
// 10: 5282468c mov w12, #0x1234 // #4660
|
||||
// 14: 0b0c016e add w14, w11, w12
|
||||
// 18: 2a0e03ed mov w13, w14
|
||||
// 1c: b5ffffad cbnz x13, 0x10
|
||||
// 20: 2a0b03e0 mov w0, w11
|
||||
// 24: b5ffff60 cbnz x0, 0x10
|
||||
// 28: d2824681 mov x1, #0x1234 // #4660
|
||||
// 28: 52824681 mov w1, #0x1234 // #4660
|
||||
// 2c: 4b010160 sub w0, w11, w1
|
||||
// 30: d65f03c0 ret
|
||||
|
||||
let golden = vec![
|
||||
137, 70, 130, 210, 11, 0, 9, 11, 234, 3, 11, 42, 170, 0, 0, 181, 140, 70, 130, 210,
|
||||
110, 1, 12, 11, 237, 3, 14, 42, 173, 255, 255, 181, 224, 3, 11, 42, 96, 255, 255, 181,
|
||||
129, 70, 130, 210, 96, 1, 1, 75, 192, 3, 95, 214,
|
||||
137, 70, 130, 82, 11, 0, 9, 11, 234, 3, 11, 42, 170, 0, 0, 181, 140, 70, 130, 82, 110,
|
||||
1, 12, 11, 237, 3, 14, 42, 173, 255, 255, 181, 224, 3, 11, 42, 96, 255, 255, 181, 129,
|
||||
70, 130, 82, 96, 1, 1, 75, 192, 3, 95, 214,
|
||||
];
|
||||
|
||||
assert_eq!(code, &golden[..]);
|
||||
@@ -412,18 +411,17 @@ mod test {
|
||||
// 1c: d61f00e0 br x7
|
||||
// 20: 00000010 udf #16
|
||||
// 24: 00000018 udf #24
|
||||
// 28: d2800060 mov x0, #0x3 // #3
|
||||
// 28: 52800060 mov w0, #0x3 // #3
|
||||
// 2c: d65f03c0 ret
|
||||
// 30: d2800020 mov x0, #0x1 // #1
|
||||
// 30: 52800020 mov w0, #0x1 // #1
|
||||
// 34: d65f03c0 ret
|
||||
// 38: d2800040 mov x0, #0x2 // #2
|
||||
// 38: 52800040 mov w0, #0x2 // #2
|
||||
// 3c: d65f03c0 ret
|
||||
|
||||
let golden = vec![
|
||||
31, 8, 0, 113, 34, 1, 0, 84, 232, 35, 128, 154, 159, 34, 3, 213, 135, 0, 0, 16, 232,
|
||||
88, 168, 184, 231, 0, 8, 139, 224, 0, 31, 214, 16, 0, 0, 0, 24, 0, 0, 0, 96, 0, 128,
|
||||
210, 192, 3, 95, 214, 32, 0, 128, 210, 192, 3, 95, 214, 64, 0, 128, 210, 192, 3, 95,
|
||||
214,
|
||||
82, 192, 3, 95, 214, 32, 0, 128, 82, 192, 3, 95, 214, 64, 0, 128, 82, 192, 3, 95, 214,
|
||||
];
|
||||
|
||||
assert_eq!(code, &golden[..]);
|
||||
|
||||
Reference in New Issue
Block a user