T0b1/wasmtime
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Cranelift AArch64: Migrate AMode to ISLE (#4832)

Browse Source 
Copyright (c) 2022, Arm Limited.

Co-authored-by: Chris Fallin <chris@cfallin.org>
This commit is contained in:
Anton Kirilov
2022-09-02 01:24:46 +01:00
committed by GitHub
parent 385bd0cbf8
commit d2e19b8d74
9 changed files with 684 additions and 329 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
cranelift/codegen/src/isa/aarch64/abi.rs
View File

@@ -34,9 +34,9 @@ static STACK_ARG_RET_SIZE_LIMIT: u64 = 128 * 1024 * 1024;
impl Into<AMode> for StackAMode { impl Into<AMode> for StackAMode {
fn into(self) -> AMode { fn into(self) -> AMode {
match self { match self {
StackAMode::FPOffset(off, ty) => AMode::FPOffset(off, ty), StackAMode::FPOffset(off, ty) => AMode::FPOffset { off, ty },
StackAMode::NominalSPOffset(off, ty) => AMode::NominalSPOffset(off, ty), StackAMode::NominalSPOffset(off, ty) => AMode::NominalSPOffset { off, ty },
StackAMode::SPOffset(off, ty) => AMode::SPOffset(off, ty), StackAMode::SPOffset(off, ty) => AMode::SPOffset { off, ty },
} }
} }
} }
@@ -462,12 +462,20 @@ impl ABIMachineSpec for AArch64MachineDeps {
} }
fn gen_load_base_offset(into_reg: Writable<Reg>, base: Reg, offset: i32, ty: Type) -> Inst { fn gen_load_base_offset(into_reg: Writable<Reg>, base: Reg, offset: i32, ty: Type) -> Inst {
let mem = AMode::RegOffset(base, offset as i64, ty); let mem = AMode::RegOffset {
rn: base,
off: offset as i64,
ty,
};
Inst::gen_load(into_reg, mem, ty, MemFlags::trusted()) Inst::gen_load(into_reg, mem, ty, MemFlags::trusted())
} }
fn gen_store_base_offset(base: Reg, offset: i32, from_reg: Reg, ty: Type) -> Inst { fn gen_store_base_offset(base: Reg, offset: i32, from_reg: Reg, ty: Type) -> Inst {
let mem = AMode::RegOffset(base, offset as i64, ty); let mem = AMode::RegOffset {
rn: base,
off: offset as i64,
ty,
};
Inst::gen_store(mem, from_reg, ty, MemFlags::trusted()) Inst::gen_store(mem, from_reg, ty, MemFlags::trusted())
} }
@@ -674,7 +682,9 @@ impl ABIMachineSpec for AArch64MachineDeps {
// str rd, [sp, #-16]! // str rd, [sp, #-16]!
insts.push(Inst::Store64 { insts.push(Inst::Store64 {
rd, rd,
mem: AMode::SPPreIndexed(SImm9::maybe_from_i64(-clobber_offset_change).unwrap()), mem: AMode::SPPreIndexed {
simm9: SImm9::maybe_from_i64(-clobber_offset_change).unwrap(),
},
flags: MemFlags::trusted(), flags: MemFlags::trusted(),
}); });
@@ -728,7 +738,9 @@ impl ABIMachineSpec for AArch64MachineDeps {
let store_vec_reg = |rd| Inst::FpuStore64 { let store_vec_reg = |rd| Inst::FpuStore64 {
rd, rd,
mem: AMode::SPPreIndexed(SImm9::maybe_from_i64(-clobber_offset_change).unwrap()), mem: AMode::SPPreIndexed {
simm9: SImm9::maybe_from_i64(-clobber_offset_change).unwrap(),
},
flags: MemFlags::trusted(), flags: MemFlags::trusted(),
}; };
let iter = clobbered_vec.chunks_exact(2); let iter = clobbered_vec.chunks_exact(2);
@@ -821,7 +833,9 @@ impl ABIMachineSpec for AArch64MachineDeps {
let load_vec_reg = |rd| Inst::FpuLoad64 { let load_vec_reg = |rd| Inst::FpuLoad64 {
rd, rd,
mem: AMode::SPPostIndexed(SImm9::maybe_from_i64(16).unwrap()), mem: AMode::SPPostIndexed {
simm9: SImm9::maybe_from_i64(16).unwrap(),
},
flags: MemFlags::trusted(), flags: MemFlags::trusted(),
}; };
let load_vec_reg_pair = |rt, rt2| Inst::FpuLoadP64 { let load_vec_reg_pair = |rt, rt2| Inst::FpuLoadP64 {
@@ -877,7 +891,9 @@ impl ABIMachineSpec for AArch64MachineDeps {
// ldr rd, [sp], #16 // ldr rd, [sp], #16
insts.push(Inst::ULoad64 { insts.push(Inst::ULoad64 {
rd, rd,
mem: AMode::SPPostIndexed(SImm9::maybe_from_i64(16).unwrap()), mem: AMode::SPPostIndexed {
simm9: SImm9::maybe_from_i64(16).unwrap(),
},
flags: MemFlags::trusted(), flags: MemFlags::trusted(),
}); });
} }

150
cranelift/codegen/src/isa/aarch64/inst.isle
View File

@@ -1010,7 +1010,108 @@
(Cls) (Cls)
)) ))
(type AMode extern (enum)) (type MemLabel extern (enum))
(type SImm9 extern (enum))
(type UImm12Scaled extern (enum))
;; An addressing mode specified for a load/store operation.
(type AMode
(enum
;;
;; Real ARM64 addressing modes:
;;
;; "post-indexed" mode as per AArch64 docs: postincrement reg after
;; address computation.
;; Specialized here to SP so we don't have to emit regalloc metadata.
(SPPostIndexed
(simm9 SImm9))
;; "pre-indexed" mode as per AArch64 docs: preincrement reg before
;; address computation.
;; Specialized here to SP so we don't have to emit regalloc metadata.
(SPPreIndexed
(simm9 SImm9))
;; N.B.: RegReg, RegScaled, and RegScaledExtended all correspond to
;; what the ISA calls the "register offset" addressing mode. We split
;; out several options here for more ergonomic codegen.
;;
;; Register plus register offset.
(RegReg
(rn Reg)
(rm Reg))
;; Register plus register offset, scaled by type's size.
(RegScaled
(rn Reg)
(rm Reg)
(ty Type))
;; Register plus register offset, scaled by type's size, with index
;; sign- or zero-extended first.
(RegScaledExtended
(rn Reg)
(rm Reg)
(ty Type)
(extendop ExtendOp))
;; Register plus register offset, with index sign- or zero-extended
;; first.
(RegExtended
(rn Reg)
(rm Reg)
(extendop ExtendOp))
;; Unscaled signed 9-bit immediate offset from reg.
(Unscaled
(rn Reg)
(simm9 SImm9))
;; Scaled (by size of a type) unsigned 12-bit immediate offset from reg.
(UnsignedOffset
(rn Reg)
(uimm12 UImm12Scaled))
;; virtual addressing modes that are lowered at emission time:
;;
;; Reference to a "label": e.g., a symbol.
(Label
(label MemLabel))
;; Arbitrary offset from a register. Converted to generation of large
;; offsets with multiple instructions as necessary during code emission.
(RegOffset
(rn Reg)
(off i64)
(ty Type))
;; Offset from the stack pointer.
(SPOffset
(off i64)
(ty Type))
;; Offset from the frame pointer.
(FPOffset
(off i64)
(ty Type))
;; Offset from the "nominal stack pointer", which is where the real SP is
;; just after stack and spill slots are allocated in the function prologue.
;; At emission time, this is converted to `SPOffset` with a fixup added to
;; the offset constant. The fixup is a running value that is tracked as
;; emission iterates through instructions in linear order, and can be
;; adjusted up and down with [Inst::VirtualSPOffsetAdj].
;;
;; The standard ABI is in charge of handling this (by emitting the
;; adjustment meta-instructions). It maintains the invariant that "nominal
;; SP" is where the actual SP is after the function prologue and before
;; clobber pushes. See the diagram in the documentation for
;; [crate::isa::aarch64::abi](the ABI module) for more details.
(NominalSPOffset
(off i64)
(ty Type))
))
(type PairAMode extern (enum)) (type PairAMode extern (enum))
(type FPUOpRI extern (enum)) (type FPUOpRI extern (enum))
(type FPUOpRIMod extern (enum)) (type FPUOpRIMod extern (enum))
@@ -1554,6 +1655,12 @@
(decl valid_atomic_transaction (Type) Type) (decl valid_atomic_transaction (Type) Type)
(extern extractor valid_atomic_transaction valid_atomic_transaction) (extern extractor valid_atomic_transaction valid_atomic_transaction)
(decl pure is_zero_simm9 (SImm9) Unit)
(extern constructor is_zero_simm9 is_zero_simm9)
(decl pure is_zero_uimm12 (UImm12Scaled) Unit)
(extern constructor is_zero_uimm12 is_zero_uimm12)
;; Helper to go directly from a `Value`, when it's an `iconst`, to an `Imm12`. ;; Helper to go directly from a `Value`, when it's an `iconst`, to an `Imm12`.
(decl imm12_from_value (Imm12) Value) (decl imm12_from_value (Imm12) Value)
(extractor (extractor
@@ -1586,25 +1693,21 @@
(decl cond_br_cond (Cond) CondBrKind) (decl cond_br_cond (Cond) CondBrKind)
(extern constructor cond_br_cond cond_br_cond) (extern constructor cond_br_cond cond_br_cond)
;; Lower the address of a load or a store.
(decl amode (Type Value u32) AMode)
;; TODO: Port lower_address() to ISLE.
(extern constructor amode amode)
(decl pair_amode (Value u32) PairAMode) (decl pair_amode (Value u32) PairAMode)
(extern constructor pair_amode pair_amode) (extern constructor pair_amode pair_amode)
;; Matches an `AMode` that is just a register.
(decl pure amode_is_reg (AMode) Reg)
;; TODO: Implement in ISLE.
(extern constructor amode_is_reg amode_is_reg)
;; Instruction creation helpers ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; Instruction creation helpers ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Helper for creating the zero register. ;; Helper for creating the zero register.
(decl zero_reg () Reg) (decl zero_reg () Reg)
(extern constructor zero_reg zero_reg) (extern constructor zero_reg zero_reg)
(decl fp_reg () Reg)
(extern constructor fp_reg fp_reg)
(decl stack_reg () Reg)
(extern constructor stack_reg stack_reg)
(decl writable_zero_reg () WritableReg) (decl writable_zero_reg () WritableReg)
(extern constructor writable_zero_reg writable_zero_reg) (extern constructor writable_zero_reg writable_zero_reg)
@@ -2678,14 +2781,33 @@
;; Helper for emitting `MInst.LoadAddr` instructions. ;; Helper for emitting `MInst.LoadAddr` instructions.
(decl load_addr (AMode) Reg) (decl load_addr (AMode) Reg)
(rule (load_addr (AMode.UnsignedOffset r imm))
(if (is_zero_uimm12 imm))
r)
(rule (load_addr (AMode.Unscaled r imm))
(if (is_zero_simm9 imm))
r)
(rule (load_addr (AMode.RegOffset r 0 _)) r)
(rule (load_addr (AMode.FPOffset 0 _)) (fp_reg))
(rule (load_addr (AMode.SPOffset 0 _)) (stack_reg))
(rule (load_addr addr) (rule (load_addr addr)
(let ((dst WritableReg (temp_writable_reg $I64)) (let ((dst WritableReg (temp_writable_reg $I64))
(_ Unit (emit (MInst.LoadAddr dst addr)))) (_ Unit (emit (MInst.LoadAddr dst addr))))
dst)) dst))
(rule (load_addr addr) ;; Lower the address of a load or a store.
(if-let addr_reg (amode_is_reg addr)) (decl amode (Type Value u32) AMode)
addr_reg) ;; TODO: Port lower_address() to ISLE.
(extern constructor amode amode)
(decl sink_load_into_amode (Type Inst) AMode)
(rule (sink_load_into_amode ty x @ (load _ addr offset))
(let ((_ Unit (sink_inst x)))
(amode ty addr offset)))
;; Lower a constant f64. ;; Lower a constant f64.
(decl constant_f64 (u64) Reg) (decl constant_f64 (u64) Reg)

214
cranelift/codegen/src/isa/aarch64/inst/args.rs
View File

@@ -115,122 +115,75 @@ pub enum MemLabel {
PCRel(i32), PCRel(i32),
} }
/// An addressing mode specified for a load/store operation.
#[derive(Clone, Debug)]
pub enum AMode {
//
// Real ARM64 addressing modes:
//
/// "post-indexed" mode as per AArch64 docs: postincrement reg after address computation.
/// Specialized here to SP so we don't have to emit regalloc metadata.
SPPostIndexed(SImm9),
/// "pre-indexed" mode as per AArch64 docs: preincrement reg before address computation.
/// Specialized here to SP so we don't have to emit regalloc metadata.
SPPreIndexed(SImm9),
// N.B.: RegReg, RegScaled, and RegScaledExtended all correspond to
// what the ISA calls the "register offset" addressing mode. We split out
// several options here for more ergonomic codegen.
/// Register plus register offset.
RegReg(Reg, Reg),
#[allow(dead_code)]
/// Register plus register offset, scaled by type's size.
RegScaled(Reg, Reg, Type),
/// Register plus register offset, scaled by type's size, with index sign- or zero-extended
/// first.
RegScaledExtended(Reg, Reg, Type, ExtendOp),
/// Register plus register offset, with index sign- or zero-extended first.
RegExtended(Reg, Reg, ExtendOp),
/// Unscaled signed 9-bit immediate offset from reg.
Unscaled(Reg, SImm9),
/// Scaled (by size of a type) unsigned 12-bit immediate offset from reg.
UnsignedOffset(Reg, UImm12Scaled),
//
// virtual addressing modes that are lowered at emission time:
//
/// Reference to a "label": e.g., a symbol.
Label(MemLabel),
/// Arbitrary offset from a register. Converted to generation of large
/// offsets with multiple instructions as necessary during code emission.
RegOffset(Reg, i64, Type),
/// Offset from the stack pointer.
SPOffset(i64, Type),
/// Offset from the frame pointer.
FPOffset(i64, Type),
/// Offset from the "nominal stack pointer", which is where the real SP is
/// just after stack and spill slots are allocated in the function prologue.
/// At emission time, this is converted to `SPOffset` with a fixup added to
/// the offset constant. The fixup is a running value that is tracked as
/// emission iterates through instructions in linear order, and can be
/// adjusted up and down with [Inst::VirtualSPOffsetAdj].
///
/// The standard ABI is in charge of handling this (by emitting the
/// adjustment meta-instructions). It maintains the invariant that "nominal
/// SP" is where the actual SP is after the function prologue and before
/// clobber pushes. See the diagram in the documentation for
/// [crate::isa::aarch64::abi](the ABI module) for more details.
NominalSPOffset(i64, Type),
}
impl AMode { impl AMode {
/// Memory reference using an address in a register. /// Memory reference using an address in a register.
pub fn reg(reg: Reg) -> AMode { pub fn reg(reg: Reg) -> AMode {
// Use UnsignedOffset rather than Unscaled to use ldr rather than ldur. // Use UnsignedOffset rather than Unscaled to use ldr rather than ldur.
// This also does not use PostIndexed / PreIndexed as they update the register. // This also does not use PostIndexed / PreIndexed as they update the register.
AMode::UnsignedOffset(reg, UImm12Scaled::zero(I64)) AMode::UnsignedOffset {
rn: reg,
uimm12: UImm12Scaled::zero(I64),
}
} }
/// Memory reference using `reg1 + sizeof(ty) * reg2` as an address, with `reg2` sign- or /// Memory reference using `reg1 + sizeof(ty) * reg2` as an address, with `reg2` sign- or
/// zero-extended as per `op`. /// zero-extended as per `op`.
pub fn reg_plus_reg_scaled_extended(reg1: Reg, reg2: Reg, ty: Type, op: ExtendOp) -> AMode { pub fn reg_plus_reg_scaled_extended(reg1: Reg, reg2: Reg, ty: Type, op: ExtendOp) -> AMode {
AMode::RegScaledExtended(reg1, reg2, ty, op) AMode::RegScaledExtended {
} rn: reg1,
rm: reg2,
/// Does the address resolve to just a register value, with no offset or ty,
/// other computation? extendop: op,
pub fn is_reg(&self) -> Option<Reg> {
match self {
&AMode::UnsignedOffset(r, uimm12) if uimm12.value() == 0 => Some(r),
&AMode::Unscaled(r, imm9) if imm9.value() == 0 => Some(r),
&AMode::RegOffset(r, off, _) if off == 0 => Some(r),
&AMode::FPOffset(off, _) if off == 0 => Some(fp_reg()),
&AMode::SPOffset(off, _) if off == 0 => Some(stack_reg()),
_ => None,
} }
} }
pub fn with_allocs(&self, allocs: &mut AllocationConsumer<'_>) -> Self { pub fn with_allocs(&self, allocs: &mut AllocationConsumer<'_>) -> Self {
// This should match `memarg_operands()`. // This should match `memarg_operands()`.
match self { match self {
&AMode::Unscaled(reg, imm9) => AMode::Unscaled(allocs.next(reg), imm9), &AMode::Unscaled { rn, simm9 } => AMode::Unscaled {
&AMode::UnsignedOffset(r, uimm12) => AMode::UnsignedOffset(allocs.next(r), uimm12), rn: allocs.next(rn),
&AMode::RegReg(r1, r2) => AMode::RegReg(allocs.next(r1), allocs.next(r2)), simm9,
&AMode::RegScaled(r1, r2, ty) => AMode::RegScaled(allocs.next(r1), allocs.next(r2), ty), },
&AMode::RegScaledExtended(r1, r2, ty, ext) => { &AMode::UnsignedOffset { rn, uimm12 } => AMode::UnsignedOffset {
AMode::RegScaledExtended(allocs.next(r1), allocs.next(r2), ty, ext) rn: allocs.next(rn),
} uimm12,
&AMode::RegExtended(r1, r2, ext) => { },
AMode::RegExtended(allocs.next(r1), allocs.next(r2), ext) &AMode::RegReg { rn, rm } => AMode::RegReg {
} rn: allocs.next(rn),
// Note that SP is not managed by regalloc, so there is no register to report in the rm: allocs.next(rm),
// pre/post-indexed amodes. },
&AMode::RegOffset(r, off, ty) => AMode::RegOffset(allocs.next(r), off, ty), &AMode::RegScaled { rn, rm, ty } => AMode::RegScaled {
&AMode::SPPreIndexed(..) rn: allocs.next(rn),
| &AMode::SPPostIndexed(..) rm: allocs.next(rm),
| &AMode::FPOffset(..) ty,
| &AMode::SPOffset(..) },
| &AMode::NominalSPOffset(..) &AMode::RegScaledExtended {
| AMode::Label(..) => self.clone(), rn,
rm,
ty,
extendop,
} => AMode::RegScaledExtended {
rn: allocs.next(rn),
rm: allocs.next(rm),
ty,
extendop,
},
&AMode::RegExtended { rn, rm, extendop } => AMode::RegExtended {
rn: allocs.next(rn),
rm: allocs.next(rm),
extendop,
},
&AMode::RegOffset { rn, off, ty } => AMode::RegOffset {
rn: allocs.next(rn),
off,
ty,
},
&AMode::SPPreIndexed { .. }
| &AMode::SPPostIndexed { .. }
| &AMode::FPOffset { .. }
| &AMode::SPOffset { .. }
| &AMode::NominalSPOffset { .. }
| AMode::Label { .. } => self.clone(),
} }
} }
} }
@@ -418,8 +371,8 @@ fn shift_for_type(ty: Type) -> usize {
impl PrettyPrint for AMode { impl PrettyPrint for AMode {
fn pretty_print(&self, _: u8, allocs: &mut AllocationConsumer<'_>) -> String { fn pretty_print(&self, _: u8, allocs: &mut AllocationConsumer<'_>) -> String {
match self { match self {
&AMode::Unscaled(reg, simm9) => { &AMode::Unscaled { rn, simm9 } => {
let reg = pretty_print_reg(reg, allocs); let reg = pretty_print_reg(rn, allocs);
if simm9.value != 0 { if simm9.value != 0 {
let simm9 = simm9.pretty_print(8, allocs); let simm9 = simm9.pretty_print(8, allocs);
format!("[{}, {}]", reg, simm9) format!("[{}, {}]", reg, simm9)
@@ -427,8 +380,8 @@ impl PrettyPrint for AMode {
format!("[{}]", reg) format!("[{}]", reg)
} }
} }
&AMode::UnsignedOffset(reg, uimm12) => { &AMode::UnsignedOffset { rn, uimm12 } => {
let reg = pretty_print_reg(reg, allocs); let reg = pretty_print_reg(rn, allocs);
if uimm12.value != 0 { if uimm12.value != 0 {
let uimm12 = uimm12.pretty_print(8, allocs); let uimm12 = uimm12.pretty_print(8, allocs);
format!("[{}, {}]", reg, uimm12) format!("[{}, {}]", reg, uimm12)
@@ -436,53 +389,58 @@ impl PrettyPrint for AMode {
format!("[{}]", reg) format!("[{}]", reg)
} }
} }
&AMode::RegReg(r1, r2) => { &AMode::RegReg { rn, rm } => {
let r1 = pretty_print_reg(r1, allocs); let r1 = pretty_print_reg(rn, allocs);
let r2 = pretty_print_reg(r2, allocs); let r2 = pretty_print_reg(rm, allocs);
format!("[{}, {}]", r1, r2) format!("[{}, {}]", r1, r2)
} }
&AMode::RegScaled(r1, r2, ty) => { &AMode::RegScaled { rn, rm, ty } => {
let r1 = pretty_print_reg(r1, allocs); let r1 = pretty_print_reg(rn, allocs);
let r2 = pretty_print_reg(r2, allocs); let r2 = pretty_print_reg(rm, allocs);
let shift = shift_for_type(ty); let shift = shift_for_type(ty);
format!("[{}, {}, LSL #{}]", r1, r2, shift) format!("[{}, {}, LSL #{}]", r1, r2, shift)
} }
&AMode::RegScaledExtended(r1, r2, ty, op) => { &AMode::RegScaledExtended {
rn,
rm,
ty,
extendop,
} => {
let shift = shift_for_type(ty); let shift = shift_for_type(ty);
let size = match op { let size = match extendop {
ExtendOp::SXTW | ExtendOp::UXTW => OperandSize::Size32, ExtendOp::SXTW | ExtendOp::UXTW => OperandSize::Size32,
_ => OperandSize::Size64, _ => OperandSize::Size64,
}; };
let r1 = pretty_print_reg(r1, allocs); let r1 = pretty_print_reg(rn, allocs);
let r2 = pretty_print_ireg(r2, size, allocs); let r2 = pretty_print_ireg(rm, size, allocs);
let op = op.pretty_print(0, allocs); let op = extendop.pretty_print(0, allocs);
format!("[{}, {}, {} #{}]", r1, r2, op, shift) format!("[{}, {}, {} #{}]", r1, r2, op, shift)
} }
&AMode::RegExtended(r1, r2, op) => { &AMode::RegExtended { rn, rm, extendop } => {
let size = match op { let size = match extendop {
ExtendOp::SXTW | ExtendOp::UXTW => OperandSize::Size32, ExtendOp::SXTW | ExtendOp::UXTW => OperandSize::Size32,
_ => OperandSize::Size64, _ => OperandSize::Size64,
}; };
let r1 = pretty_print_reg(r1, allocs); let r1 = pretty_print_reg(rn, allocs);
let r2 = pretty_print_ireg(r2, size, allocs); let r2 = pretty_print_ireg(rm, size, allocs);
let op = op.pretty_print(0, allocs); let op = extendop.pretty_print(0, allocs);
format!("[{}, {}, {}]", r1, r2, op) format!("[{}, {}, {}]", r1, r2, op)
} }
&AMode::Label(ref label) => label.pretty_print(0, allocs), &AMode::Label { ref label } => label.pretty_print(0, allocs),
&AMode::SPPreIndexed(simm9) => { &AMode::SPPreIndexed { simm9 } => {
let simm9 = simm9.pretty_print(8, allocs); let simm9 = simm9.pretty_print(8, allocs);
format!("[sp, {}]!", simm9) format!("[sp, {}]!", simm9)
} }
&AMode::SPPostIndexed(simm9) => { &AMode::SPPostIndexed { simm9 } => {
let simm9 = simm9.pretty_print(8, allocs); let simm9 = simm9.pretty_print(8, allocs);
format!("[sp], {}", simm9) format!("[sp], {}", simm9)
} }
// Eliminated by `mem_finalize()`. // Eliminated by `mem_finalize()`.
&AMode::SPOffset(..) &AMode::SPOffset { .. }
| &AMode::FPOffset(..) | &AMode::FPOffset { .. }
| &AMode::NominalSPOffset(..) | &AMode::NominalSPOffset { .. }
| &AMode::RegOffset(..) => { | &AMode::RegOffset { .. } => {
panic!("Unexpected pseudo mem-arg mode (stack-offset or generic reg-offset)!") panic!("Unexpected pseudo mem-arg mode: {:?}", self)
} }
} }
} }

163
cranelift/codegen/src/isa/aarch64/inst/emit.rs
View File

@@ -28,18 +28,18 @@ pub fn mem_finalize(
state: &EmitState, state: &EmitState,
) -> (SmallVec<[Inst; 4]>, AMode) { ) -> (SmallVec<[Inst; 4]>, AMode) {
match mem { match mem {
&AMode::RegOffset(_, off, ty) &AMode::RegOffset { off, ty, .. }
| &AMode::SPOffset(off, ty) | &AMode::SPOffset { off, ty }
| &AMode::FPOffset(off, ty) | &AMode::FPOffset { off, ty }
| &AMode::NominalSPOffset(off, ty) => { | &AMode::NominalSPOffset { off, ty } => {
let basereg = match mem { let basereg = match mem {
&AMode::RegOffset(reg, _, _) => reg, &AMode::RegOffset { rn, .. } => rn,
&AMode::SPOffset(..) | &AMode::NominalSPOffset(..) => stack_reg(), &AMode::SPOffset { .. } | &AMode::NominalSPOffset { .. } => stack_reg(),
&AMode::FPOffset(..) => fp_reg(), &AMode::FPOffset { .. } => fp_reg(),
_ => unreachable!(), _ => unreachable!(),
}; };
let adj = match mem { let adj = match mem {
&AMode::NominalSPOffset(..) => { &AMode::NominalSPOffset { .. } => {
trace!( trace!(
"mem_finalize: nominal SP offset {} + adj {} -> {}", "mem_finalize: nominal SP offset {} + adj {} -> {}",
off, off,
@@ -53,10 +53,13 @@ pub fn mem_finalize(
let off = off + adj; let off = off + adj;
if let Some(simm9) = SImm9::maybe_from_i64(off) { if let Some(simm9) = SImm9::maybe_from_i64(off) {
let mem = AMode::Unscaled(basereg, simm9); let mem = AMode::Unscaled { rn: basereg, simm9 };
(smallvec![], mem) (smallvec![], mem)
} else if let Some(uimm12s) = UImm12Scaled::maybe_from_i64(off, ty) { } else if let Some(uimm12) = UImm12Scaled::maybe_from_i64(off, ty) {
let mem = AMode::UnsignedOffset(basereg, uimm12s); let mem = AMode::UnsignedOffset {
rn: basereg,
uimm12,
};
(smallvec![], mem) (smallvec![], mem)
} else { } else {
let tmp = writable_spilltmp_reg(); let tmp = writable_spilltmp_reg();
@@ -78,9 +81,14 @@ pub fn mem_finalize(
} }
} }
&AMode::Label(ref label) => { &AMode::Label { ref label } => {
let off = memlabel_finalize(insn_off, label); let off = memlabel_finalize(insn_off, label);
(smallvec![], AMode::Label(MemLabel::PCRel(off))) (
smallvec![],
AMode::Label {
label: MemLabel::PCRel(off),
},
)
} }
_ => (smallvec![], mem.clone()), _ => (smallvec![], mem.clone()),
@@ -977,40 +985,41 @@ impl MachInstEmit for Inst {
} }
match &mem { match &mem {
&AMode::Unscaled(reg, simm9) => { &AMode::Unscaled { rn, simm9 } => {
let reg = allocs.next(reg); let reg = allocs.next(rn);
sink.put4(enc_ldst_simm9(op, simm9, 0b00, reg, rd)); sink.put4(enc_ldst_simm9(op, simm9, 0b00, reg, rd));
} }
&AMode::UnsignedOffset(reg, uimm12scaled) => { &AMode::UnsignedOffset { rn, uimm12 } => {
let reg = allocs.next(reg); let reg = allocs.next(rn);
if uimm12scaled.value() != 0 { if uimm12.value() != 0 {
assert_eq!(bits, ty_bits(uimm12scaled.scale_ty())); assert_eq!(bits, ty_bits(uimm12.scale_ty()));
} }
sink.put4(enc_ldst_uimm12(op, uimm12scaled, reg, rd)); sink.put4(enc_ldst_uimm12(op, uimm12, reg, rd));
} }
&AMode::RegReg(r1, r2) => { &AMode::RegReg { rn, rm } => {
let r1 = allocs.next(r1); let r1 = allocs.next(rn);
let r2 = allocs.next(r2); let r2 = allocs.next(rm);
sink.put4(enc_ldst_reg( sink.put4(enc_ldst_reg(
op, r1, r2, /* scaled = */ false, /* extendop = */ None, rd, op, r1, r2, /* scaled = */ false, /* extendop = */ None, rd,
)); ));
} }
&AMode::RegScaled(r1, r2, ty) | &AMode::RegScaledExtended(r1, r2, ty, _) => { &AMode::RegScaled { rn, rm, ty }
let r1 = allocs.next(r1); | &AMode::RegScaledExtended { rn, rm, ty, .. } => {
let r2 = allocs.next(r2); let r1 = allocs.next(rn);
let r2 = allocs.next(rm);
assert_eq!(bits, ty_bits(ty)); assert_eq!(bits, ty_bits(ty));
let extendop = match &mem { let extendop = match &mem {
&AMode::RegScaled(..) => None, &AMode::RegScaled { .. } => None,
&AMode::RegScaledExtended(_, _, _, op) => Some(op), &AMode::RegScaledExtended { extendop, .. } => Some(extendop),
_ => unreachable!(), _ => unreachable!(),
}; };
sink.put4(enc_ldst_reg( sink.put4(enc_ldst_reg(
op, r1, r2, /* scaled = */ true, extendop, rd, op, r1, r2, /* scaled = */ true, extendop, rd,
)); ));
} }
&AMode::RegExtended(r1, r2, extendop) => { &AMode::RegExtended { rn, rm, extendop } => {
let r1 = allocs.next(r1); let r1 = allocs.next(rn);
let r2 = allocs.next(r2); let r2 = allocs.next(rm);
sink.put4(enc_ldst_reg( sink.put4(enc_ldst_reg(
op, op,
r1, r1,
@@ -1020,7 +1029,7 @@ impl MachInstEmit for Inst {
rd, rd,
)); ));
} }
&AMode::Label(ref label) => { &AMode::Label { ref label } => {
let offset = match label { let offset = match label {
// cast i32 to u32 (two's-complement) // cast i32 to u32 (two's-complement)
&MemLabel::PCRel(off) => off as u32, &MemLabel::PCRel(off) => off as u32,
@@ -1048,19 +1057,21 @@ impl MachInstEmit for Inst {
_ => panic!("Unspported size for LDR from constant pool!"), _ => panic!("Unspported size for LDR from constant pool!"),
} }
} }
&AMode::SPPreIndexed(simm9) => { &AMode::SPPreIndexed { simm9 } => {
let reg = stack_reg(); let reg = stack_reg();
sink.put4(enc_ldst_simm9(op, simm9, 0b11, reg, rd)); sink.put4(enc_ldst_simm9(op, simm9, 0b11, reg, rd));
} }
&AMode::SPPostIndexed(simm9) => { &AMode::SPPostIndexed { simm9 } => {
let reg = stack_reg(); let reg = stack_reg();
sink.put4(enc_ldst_simm9(op, simm9, 0b01, reg, rd)); sink.put4(enc_ldst_simm9(op, simm9, 0b01, reg, rd));
} }
// Eliminated by `mem_finalize()` above. // Eliminated by `mem_finalize()` above.
&AMode::SPOffset(..) | &AMode::FPOffset(..) | &AMode::NominalSPOffset(..) => { &AMode::SPOffset { .. }
panic!("Should not see stack-offset here!") | &AMode::FPOffset { .. }
| &AMode::NominalSPOffset { .. }
| &AMode::RegOffset { .. } => {
panic!("Should not see {:?} here!", mem)
} }
&AMode::RegOffset(..) => panic!("SHould not see generic reg-offset here!"),
} }
} }
@@ -1097,39 +1108,39 @@ impl MachInstEmit for Inst {
} }
match &mem { match &mem {
&AMode::Unscaled(reg, simm9) => { &AMode::Unscaled { rn, simm9 } => {
let reg = allocs.next(reg); let reg = allocs.next(rn);
sink.put4(enc_ldst_simm9(op, simm9, 0b00, reg, rd)); sink.put4(enc_ldst_simm9(op, simm9, 0b00, reg, rd));
} }
&AMode::UnsignedOffset(reg, uimm12scaled) => { &AMode::UnsignedOffset { rn, uimm12 } => {
let reg = allocs.next(reg); let reg = allocs.next(rn);
if uimm12scaled.value() != 0 { if uimm12.value() != 0 {
assert_eq!(bits, ty_bits(uimm12scaled.scale_ty())); assert_eq!(bits, ty_bits(uimm12.scale_ty()));
} }
sink.put4(enc_ldst_uimm12(op, uimm12scaled, reg, rd)); sink.put4(enc_ldst_uimm12(op, uimm12, reg, rd));
} }
&AMode::RegReg(r1, r2) => { &AMode::RegReg { rn, rm } => {
let r1 = allocs.next(r1); let r1 = allocs.next(rn);
let r2 = allocs.next(r2); let r2 = allocs.next(rm);
sink.put4(enc_ldst_reg( sink.put4(enc_ldst_reg(
op, r1, r2, /* scaled = */ false, /* extendop = */ None, rd, op, r1, r2, /* scaled = */ false, /* extendop = */ None, rd,
)); ));
} }
&AMode::RegScaled(r1, r2, _ty) | &AMode::RegScaledExtended(r1, r2, _ty, _) => { &AMode::RegScaled { rn, rm, .. } | &AMode::RegScaledExtended { rn, rm, .. } => {
let r1 = allocs.next(r1); let r1 = allocs.next(rn);
let r2 = allocs.next(r2); let r2 = allocs.next(rm);
let extendop = match &mem { let extendop = match &mem {
&AMode::RegScaled(..) => None, &AMode::RegScaled { .. } => None,
&AMode::RegScaledExtended(_, _, _, op) => Some(op), &AMode::RegScaledExtended { extendop, .. } => Some(extendop),
_ => unreachable!(), _ => unreachable!(),
}; };
sink.put4(enc_ldst_reg( sink.put4(enc_ldst_reg(
op, r1, r2, /* scaled = */ true, extendop, rd, op, r1, r2, /* scaled = */ true, extendop, rd,
)); ));
} }
&AMode::RegExtended(r1, r2, extendop) => { &AMode::RegExtended { rn, rm, extendop } => {
let r1 = allocs.next(r1); let r1 = allocs.next(rn);
let r2 = allocs.next(r2); let r2 = allocs.next(rm);
sink.put4(enc_ldst_reg( sink.put4(enc_ldst_reg(
op, op,
r1, r1,
@@ -1139,22 +1150,24 @@ impl MachInstEmit for Inst {
rd, rd,
)); ));
} }
&AMode::Label(..) => { &AMode::Label { .. } => {
panic!("Store to a MemLabel not implemented!"); panic!("Store to a MemLabel not implemented!");
} }
&AMode::SPPreIndexed(simm9) => { &AMode::SPPreIndexed { simm9 } => {
let reg = stack_reg(); let reg = stack_reg();
sink.put4(enc_ldst_simm9(op, simm9, 0b11, reg, rd)); sink.put4(enc_ldst_simm9(op, simm9, 0b11, reg, rd));
} }
&AMode::SPPostIndexed(simm9) => { &AMode::SPPostIndexed { simm9 } => {
let reg = stack_reg(); let reg = stack_reg();
sink.put4(enc_ldst_simm9(op, simm9, 0b01, reg, rd)); sink.put4(enc_ldst_simm9(op, simm9, 0b01, reg, rd));
} }
// Eliminated by `mem_finalize()` above. // Eliminated by `mem_finalize()` above.
&AMode::SPOffset(..) | &AMode::FPOffset(..) | &AMode::NominalSPOffset(..) => { &AMode::SPOffset { .. }
panic!("Should not see stack-offset here!") | &AMode::FPOffset { .. }
| &AMode::NominalSPOffset { .. }
| &AMode::RegOffset { .. } => {
panic!("Should not see {:?} here!", mem)
} }
&AMode::RegOffset(..) => panic!("SHould not see generic reg-offset here!"),
} }
} }
@@ -2216,7 +2229,9 @@ impl MachInstEmit for Inst {
let rd = allocs.next_writable(rd); let rd = allocs.next_writable(rd);
let inst = Inst::FpuLoad64 { let inst = Inst::FpuLoad64 {
rd, rd,
mem: AMode::Label(MemLabel::PCRel(8)), mem: AMode::Label {
label: MemLabel::PCRel(8),
},
flags: MemFlags::trusted(), flags: MemFlags::trusted(),
}; };
inst.emit(&[], sink, emit_info, state); inst.emit(&[], sink, emit_info, state);
@@ -2230,7 +2245,9 @@ impl MachInstEmit for Inst {
let rd = allocs.next_writable(rd); let rd = allocs.next_writable(rd);
let inst = Inst::FpuLoad128 { let inst = Inst::FpuLoad128 {
rd, rd,
mem: AMode::Label(MemLabel::PCRel(8)), mem: AMode::Label {
label: MemLabel::PCRel(8),
},
flags: MemFlags::trusted(), flags: MemFlags::trusted(),
}; };
inst.emit(&[], sink, emit_info, state); inst.emit(&[], sink, emit_info, state);
@@ -3159,7 +3176,9 @@ impl MachInstEmit for Inst {
let rd = allocs.next_writable(rd); let rd = allocs.next_writable(rd);
let inst = Inst::ULoad64 { let inst = Inst::ULoad64 {
rd, rd,
mem: AMode::Label(MemLabel::PCRel(8)), mem: AMode::Label {
label: MemLabel::PCRel(8),
},
flags: MemFlags::trusted(), flags: MemFlags::trusted(),
}; };
inst.emit(&[], sink, emit_info, state); inst.emit(&[], sink, emit_info, state);
@@ -3179,17 +3198,17 @@ impl MachInstEmit for Inst {
} }
let (reg, index_reg, offset) = match mem { let (reg, index_reg, offset) = match mem {
AMode::RegExtended(r, idx, extendop) => { AMode::RegExtended { rn, rm, extendop } => {
let r = allocs.next(r); let r = allocs.next(rn);
(r, Some((idx, extendop)), 0) (r, Some((rm, extendop)), 0)
} }
AMode::Unscaled(r, simm9) => { AMode::Unscaled { rn, simm9 } => {
let r = allocs.next(r); let r = allocs.next(rn);
(r, None, simm9.value()) (r, None, simm9.value())
} }
AMode::UnsignedOffset(r, uimm12scaled) => { AMode::UnsignedOffset { rn, uimm12 } => {
let r = allocs.next(r); let r = allocs.next(rn);
(r, None, uimm12scaled.value() as i32) (r, None, uimm12.value() as i32)
} }
_ => panic!("Unsupported case for LoadAddr: {:?}", mem), _ => panic!("Unsupported case for LoadAddr: {:?}", mem),
}; };

305
cranelift/codegen/src/isa/aarch64/inst/emit_tests.rs
View File

@@ -1371,7 +1371,10 @@ fn test_aarch64_binemit() {
insns.push(( insns.push((
Inst::ULoad8 { Inst::ULoad8 {
rd: writable_xreg(1), rd: writable_xreg(1),
mem: AMode::Unscaled(xreg(2), simm9_zero()), mem: AMode::Unscaled {
rn: xreg(2),
simm9: simm9_zero(),
},
flags: MemFlags::trusted(), flags: MemFlags::trusted(),
}, },
"41004038", "41004038",
@@ -1380,7 +1383,10 @@ fn test_aarch64_binemit() {
insns.push(( insns.push((
Inst::ULoad8 { Inst::ULoad8 {
rd: writable_xreg(1), rd: writable_xreg(1),
mem: AMode::UnsignedOffset(xreg(2), UImm12Scaled::zero(I8)), mem: AMode::UnsignedOffset {
rn: xreg(2),
uimm12: UImm12Scaled::zero(I8),
},
flags: MemFlags::trusted(), flags: MemFlags::trusted(),
}, },
"41004039", "41004039",
@@ -1389,7 +1395,10 @@ fn test_aarch64_binemit() {
insns.push(( insns.push((
Inst::ULoad8 { Inst::ULoad8 {
rd: writable_xreg(1), rd: writable_xreg(1),
mem: AMode::RegReg(xreg(2), xreg(5)), mem: AMode::RegReg {
rn: xreg(2),
rm: xreg(5),
},
flags: MemFlags::trusted(), flags: MemFlags::trusted(),
}, },
"41686538", "41686538",
@@ -1398,7 +1407,10 @@ fn test_aarch64_binemit() {
insns.push(( insns.push((
Inst::SLoad8 { Inst::SLoad8 {
rd: writable_xreg(1), rd: writable_xreg(1),
mem: AMode::Unscaled(xreg(2), simm9_zero()), mem: AMode::Unscaled {
rn: xreg(2),
simm9: simm9_zero(),
},
flags: MemFlags::trusted(), flags: MemFlags::trusted(),
}, },
"41008038", "41008038",
@@ -1407,7 +1419,10 @@ fn test_aarch64_binemit() {
insns.push(( insns.push((
Inst::SLoad8 { Inst::SLoad8 {
rd: writable_xreg(1), rd: writable_xreg(1),
mem: AMode::UnsignedOffset(xreg(2), UImm12Scaled::maybe_from_i64(63, I8).unwrap()), mem: AMode::UnsignedOffset {
rn: xreg(2),
uimm12: UImm12Scaled::maybe_from_i64(63, I8).unwrap(),
},
flags: MemFlags::trusted(), flags: MemFlags::trusted(),
}, },
"41FC8039", "41FC8039",
@@ -1416,7 +1431,10 @@ fn test_aarch64_binemit() {
insns.push(( insns.push((
Inst::SLoad8 { Inst::SLoad8 {
rd: writable_xreg(1), rd: writable_xreg(1),
mem: AMode::RegReg(xreg(2), xreg(5)), mem: AMode::RegReg {
rn: xreg(2),
rm: xreg(5),
},
flags: MemFlags::trusted(), flags: MemFlags::trusted(),
}, },
"4168A538", "4168A538",
@@ -1425,7 +1443,10 @@ fn test_aarch64_binemit() {
insns.push(( insns.push((
Inst::ULoad16 { Inst::ULoad16 {
rd: writable_xreg(1), rd: writable_xreg(1),
mem: AMode::Unscaled(xreg(2), SImm9::maybe_from_i64(5).unwrap()), mem: AMode::Unscaled {
rn: xreg(2),
simm9: SImm9::maybe_from_i64(5).unwrap(),
},
flags: MemFlags::trusted(), flags: MemFlags::trusted(),
}, },
"41504078", "41504078",
@@ -1434,7 +1455,10 @@ fn test_aarch64_binemit() {
insns.push(( insns.push((
Inst::ULoad16 { Inst::ULoad16 {
rd: writable_xreg(1), rd: writable_xreg(1),
mem: AMode::UnsignedOffset(xreg(2), UImm12Scaled::maybe_from_i64(8, I16).unwrap()), mem: AMode::UnsignedOffset {
rn: xreg(2),
uimm12: UImm12Scaled::maybe_from_i64(8, I16).unwrap(),
},
flags: MemFlags::trusted(), flags: MemFlags::trusted(),
}, },
"41104079", "41104079",
@@ -1443,7 +1467,11 @@ fn test_aarch64_binemit() {
insns.push(( insns.push((
Inst::ULoad16 { Inst::ULoad16 {
rd: writable_xreg(1), rd: writable_xreg(1),
mem: AMode::RegScaled(xreg(2), xreg(3), I16), mem: AMode::RegScaled {
rn: xreg(2),
rm: xreg(3),
ty: I16,
},
flags: MemFlags::trusted(), flags: MemFlags::trusted(),
}, },
"41786378", "41786378",
@@ -1452,7 +1480,10 @@ fn test_aarch64_binemit() {
insns.push(( insns.push((
Inst::SLoad16 { Inst::SLoad16 {
rd: writable_xreg(1), rd: writable_xreg(1),
mem: AMode::Unscaled(xreg(2), simm9_zero()), mem: AMode::Unscaled {
rn: xreg(2),
simm9: simm9_zero(),
},
flags: MemFlags::trusted(), flags: MemFlags::trusted(),
}, },
"41008078", "41008078",
@@ -1461,7 +1492,10 @@ fn test_aarch64_binemit() {
insns.push(( insns.push((
Inst::SLoad16 { Inst::SLoad16 {
rd: writable_xreg(28), rd: writable_xreg(28),
mem: AMode::UnsignedOffset(xreg(20), UImm12Scaled::maybe_from_i64(24, I16).unwrap()), mem: AMode::UnsignedOffset {
rn: xreg(20),
uimm12: UImm12Scaled::maybe_from_i64(24, I16).unwrap(),
},
flags: MemFlags::trusted(), flags: MemFlags::trusted(),
}, },
"9C328079", "9C328079",
@@ -1470,7 +1504,11 @@ fn test_aarch64_binemit() {
insns.push(( insns.push((
Inst::SLoad16 { Inst::SLoad16 {
rd: writable_xreg(28), rd: writable_xreg(28),
mem: AMode::RegScaled(xreg(20), xreg(20), I16), mem: AMode::RegScaled {
rn: xreg(20),
rm: xreg(20),
ty: I16,
},
flags: MemFlags::trusted(), flags: MemFlags::trusted(),
}, },
"9C7AB478", "9C7AB478",
@@ -1479,7 +1517,10 @@ fn test_aarch64_binemit() {
insns.push(( insns.push((
Inst::ULoad32 { Inst::ULoad32 {
rd: writable_xreg(1), rd: writable_xreg(1),
mem: AMode::Unscaled(xreg(2), simm9_zero()), mem: AMode::Unscaled {
rn: xreg(2),
simm9: simm9_zero(),
},
flags: MemFlags::trusted(), flags: MemFlags::trusted(),
}, },
"410040B8", "410040B8",
@@ -1488,7 +1529,10 @@ fn test_aarch64_binemit() {
insns.push(( insns.push((
Inst::ULoad32 { Inst::ULoad32 {
rd: writable_xreg(12), rd: writable_xreg(12),
mem: AMode::UnsignedOffset(xreg(0), UImm12Scaled::maybe_from_i64(204, I32).unwrap()), mem: AMode::UnsignedOffset {
rn: xreg(0),
uimm12: UImm12Scaled::maybe_from_i64(204, I32).unwrap(),
},
flags: MemFlags::trusted(), flags: MemFlags::trusted(),
}, },
"0CCC40B9", "0CCC40B9",
@@ -1497,7 +1541,11 @@ fn test_aarch64_binemit() {
insns.push(( insns.push((
Inst::ULoad32 { Inst::ULoad32 {
rd: writable_xreg(1), rd: writable_xreg(1),
mem: AMode::RegScaled(xreg(2), xreg(12), I32), mem: AMode::RegScaled {
rn: xreg(2),
rm: xreg(12),
ty: I32,
},
flags: MemFlags::trusted(), flags: MemFlags::trusted(),
}, },
"41786CB8", "41786CB8",
@@ -1506,7 +1554,10 @@ fn test_aarch64_binemit() {
insns.push(( insns.push((
Inst::SLoad32 { Inst::SLoad32 {
rd: writable_xreg(1), rd: writable_xreg(1),
mem: AMode::Unscaled(xreg(2), simm9_zero()), mem: AMode::Unscaled {
rn: xreg(2),
simm9: simm9_zero(),
},
flags: MemFlags::trusted(), flags: MemFlags::trusted(),
}, },
"410080B8", "410080B8",
@@ -1515,7 +1566,10 @@ fn test_aarch64_binemit() {
insns.push(( insns.push((
Inst::SLoad32 { Inst::SLoad32 {
rd: writable_xreg(12), rd: writable_xreg(12),
mem: AMode::UnsignedOffset(xreg(1), UImm12Scaled::maybe_from_i64(16380, I32).unwrap()), mem: AMode::UnsignedOffset {
rn: xreg(1),
uimm12: UImm12Scaled::maybe_from_i64(16380, I32).unwrap(),
},
flags: MemFlags::trusted(), flags: MemFlags::trusted(),
}, },
"2CFCBFB9", "2CFCBFB9",
@@ -1524,7 +1578,11 @@ fn test_aarch64_binemit() {
insns.push(( insns.push((
Inst::SLoad32 { Inst::SLoad32 {
rd: writable_xreg(1), rd: writable_xreg(1),
mem: AMode::RegScaled(xreg(5), xreg(1), I32), mem: AMode::RegScaled {
rn: xreg(5),
rm: xreg(1),
ty: I32,
},
flags: MemFlags::trusted(), flags: MemFlags::trusted(),
}, },
"A178A1B8", "A178A1B8",
@@ -1533,7 +1591,10 @@ fn test_aarch64_binemit() {
insns.push(( insns.push((
Inst::ULoad64 { Inst::ULoad64 {
rd: writable_xreg(1), rd: writable_xreg(1),
mem: AMode::Unscaled(xreg(2), simm9_zero()), mem: AMode::Unscaled {
rn: xreg(2),
simm9: simm9_zero(),
},
flags: MemFlags::trusted(), flags: MemFlags::trusted(),
}, },
"410040F8", "410040F8",
@@ -1542,7 +1603,10 @@ fn test_aarch64_binemit() {
insns.push(( insns.push((
Inst::ULoad64 { Inst::ULoad64 {
rd: writable_xreg(1), rd: writable_xreg(1),
mem: AMode::Unscaled(xreg(2), SImm9::maybe_from_i64(-256).unwrap()), mem: AMode::Unscaled {
rn: xreg(2),
simm9: SImm9::maybe_from_i64(-256).unwrap(),
},
flags: MemFlags::trusted(), flags: MemFlags::trusted(),
}, },
"410050F8", "410050F8",
@@ -1551,7 +1615,10 @@ fn test_aarch64_binemit() {
insns.push(( insns.push((
Inst::ULoad64 { Inst::ULoad64 {
rd: writable_xreg(1), rd: writable_xreg(1),
mem: AMode::Unscaled(xreg(2), SImm9::maybe_from_i64(255).unwrap()), mem: AMode::Unscaled {
rn: xreg(2),
simm9: SImm9::maybe_from_i64(255).unwrap(),
},
flags: MemFlags::trusted(), flags: MemFlags::trusted(),
}, },
"41F04FF8", "41F04FF8",
@@ -1560,7 +1627,10 @@ fn test_aarch64_binemit() {
insns.push(( insns.push((
Inst::ULoad64 { Inst::ULoad64 {
rd: writable_xreg(1), rd: writable_xreg(1),
mem: AMode::UnsignedOffset(xreg(2), UImm12Scaled::maybe_from_i64(32760, I64).unwrap()), mem: AMode::UnsignedOffset {
rn: xreg(2),
uimm12: UImm12Scaled::maybe_from_i64(32760, I64).unwrap(),
},
flags: MemFlags::trusted(), flags: MemFlags::trusted(),
}, },
"41FC7FF9", "41FC7FF9",
@@ -1569,7 +1639,10 @@ fn test_aarch64_binemit() {
insns.push(( insns.push((
Inst::ULoad64 { Inst::ULoad64 {
rd: writable_xreg(1), rd: writable_xreg(1),
mem: AMode::RegReg(xreg(2), xreg(3)), mem: AMode::RegReg {
rn: xreg(2),
rm: xreg(3),
},
flags: MemFlags::trusted(), flags: MemFlags::trusted(),
}, },
"416863F8", "416863F8",
@@ -1578,7 +1651,11 @@ fn test_aarch64_binemit() {
insns.push(( insns.push((
Inst::ULoad64 { Inst::ULoad64 {
rd: writable_xreg(1), rd: writable_xreg(1),
mem: AMode::RegScaled(xreg(2), xreg(3), I64), mem: AMode::RegScaled {
rn: xreg(2),
rm: xreg(3),
ty: I64,
},
flags: MemFlags::trusted(), flags: MemFlags::trusted(),
}, },
"417863F8", "417863F8",
@@ -1587,7 +1664,12 @@ fn test_aarch64_binemit() {
insns.push(( insns.push((
Inst::ULoad64 { Inst::ULoad64 {
rd: writable_xreg(1), rd: writable_xreg(1),
mem: AMode::RegScaledExtended(xreg(2), xreg(3), I64, ExtendOp::SXTW), mem: AMode::RegScaledExtended {
rn: xreg(2),
rm: xreg(3),
ty: I64,
extendop: ExtendOp::SXTW,
},
flags: MemFlags::trusted(), flags: MemFlags::trusted(),
}, },
"41D863F8", "41D863F8",
@@ -1596,7 +1678,11 @@ fn test_aarch64_binemit() {
insns.push(( insns.push((
Inst::ULoad64 { Inst::ULoad64 {
rd: writable_xreg(1), rd: writable_xreg(1),
mem: AMode::RegExtended(xreg(2), xreg(3), ExtendOp::SXTW), mem: AMode::RegExtended {
rn: xreg(2),
rm: xreg(3),
extendop: ExtendOp::SXTW,
},
flags: MemFlags::trusted(), flags: MemFlags::trusted(),
}, },
"41C863F8", "41C863F8",
@@ -1605,7 +1691,9 @@ fn test_aarch64_binemit() {
insns.push(( insns.push((
Inst::ULoad64 { Inst::ULoad64 {
rd: writable_xreg(1), rd: writable_xreg(1),
mem: AMode::Label(MemLabel::PCRel(64)), mem: AMode::Label {
label: MemLabel::PCRel(64),
},
flags: MemFlags::trusted(), flags: MemFlags::trusted(),
}, },
"01020058", "01020058",
@@ -1614,7 +1702,9 @@ fn test_aarch64_binemit() {
insns.push(( insns.push((
Inst::ULoad64 { Inst::ULoad64 {
rd: writable_xreg(1), rd: writable_xreg(1),
mem: AMode::SPPreIndexed(SImm9::maybe_from_i64(16).unwrap()), mem: AMode::SPPreIndexed {
simm9: SImm9::maybe_from_i64(16).unwrap(),
},
flags: MemFlags::trusted(), flags: MemFlags::trusted(),
}, },
"E10F41F8", "E10F41F8",
@@ -1623,7 +1713,9 @@ fn test_aarch64_binemit() {
insns.push(( insns.push((
Inst::ULoad64 { Inst::ULoad64 {
rd: writable_xreg(1), rd: writable_xreg(1),
mem: AMode::SPPostIndexed(SImm9::maybe_from_i64(16).unwrap()), mem: AMode::SPPostIndexed {
simm9: SImm9::maybe_from_i64(16).unwrap(),
},
flags: MemFlags::trusted(), flags: MemFlags::trusted(),
}, },
"E10741F8", "E10741F8",
@@ -1632,7 +1724,7 @@ fn test_aarch64_binemit() {
insns.push(( insns.push((
Inst::ULoad64 { Inst::ULoad64 {
rd: writable_xreg(1), rd: writable_xreg(1),
mem: AMode::FPOffset(32768, I8), mem: AMode::FPOffset { off: 32768, ty: I8 },
flags: MemFlags::trusted(), flags: MemFlags::trusted(),
}, },
"100090D2B063308B010240F9", "100090D2B063308B010240F9",
@@ -1641,7 +1733,10 @@ fn test_aarch64_binemit() {
insns.push(( insns.push((
Inst::ULoad64 { Inst::ULoad64 {
rd: writable_xreg(1), rd: writable_xreg(1),
mem: AMode::FPOffset(-32768, I8), mem: AMode::FPOffset {
off: -32768,
ty: I8,
},
flags: MemFlags::trusted(), flags: MemFlags::trusted(),
}, },
"F0FF8F92B063308B010240F9", "F0FF8F92B063308B010240F9",
@@ -1650,7 +1745,10 @@ fn test_aarch64_binemit() {
insns.push(( insns.push((
Inst::ULoad64 { Inst::ULoad64 {
rd: writable_xreg(1), rd: writable_xreg(1),
mem: AMode::FPOffset(1048576, I8), // 2^20 mem: AMode::FPOffset {
off: 1048576,
ty: I8,
}, // 2^20
flags: MemFlags::trusted(), flags: MemFlags::trusted(),
}, },
"1002A0D2B063308B010240F9", "1002A0D2B063308B010240F9",
@@ -1659,7 +1757,10 @@ fn test_aarch64_binemit() {
insns.push(( insns.push((
Inst::ULoad64 { Inst::ULoad64 {
rd: writable_xreg(1), rd: writable_xreg(1),
mem: AMode::FPOffset(1048576 + 1, I8), // 2^20 + 1 mem: AMode::FPOffset {
off: 1048576 + 1,
ty: I8,
}, // 2^20 + 1
flags: MemFlags::trusted(), flags: MemFlags::trusted(),
}, },
"300080521002A072B063308B010240F9", "300080521002A072B063308B010240F9",
@@ -1669,7 +1770,11 @@ fn test_aarch64_binemit() {
insns.push(( insns.push((
Inst::ULoad64 { Inst::ULoad64 {
rd: writable_xreg(1), rd: writable_xreg(1),
mem: AMode::RegOffset(xreg(7), 8, I64), mem: AMode::RegOffset {
rn: xreg(7),
off: 8,
ty: I64,
},
flags: MemFlags::trusted(), flags: MemFlags::trusted(),
}, },
"E18040F8", "E18040F8",
@@ -1679,7 +1784,11 @@ fn test_aarch64_binemit() {
insns.push(( insns.push((
Inst::ULoad64 { Inst::ULoad64 {
rd: writable_xreg(1), rd: writable_xreg(1),
mem: AMode::RegOffset(xreg(7), 1024, I64), mem: AMode::RegOffset {
rn: xreg(7),
off: 1024,
ty: I64,
},
flags: MemFlags::trusted(), flags: MemFlags::trusted(),
}, },
"E10042F9", "E10042F9",
@@ -1689,7 +1798,11 @@ fn test_aarch64_binemit() {
insns.push(( insns.push((
Inst::ULoad64 { Inst::ULoad64 {
rd: writable_xreg(1), rd: writable_xreg(1),
mem: AMode::RegOffset(xreg(7), 1048576, I64), mem: AMode::RegOffset {
rn: xreg(7),
off: 1048576,
ty: I64,
},
flags: MemFlags::trusted(), flags: MemFlags::trusted(),
}, },
"1002A0D2F060308B010240F9", "1002A0D2F060308B010240F9",
@@ -1699,7 +1812,10 @@ fn test_aarch64_binemit() {
insns.push(( insns.push((
Inst::Store8 { Inst::Store8 {
rd: xreg(1), rd: xreg(1),
mem: AMode::Unscaled(xreg(2), simm9_zero()), mem: AMode::Unscaled {
rn: xreg(2),
simm9: simm9_zero(),
},
flags: MemFlags::trusted(), flags: MemFlags::trusted(),
}, },
"41000038", "41000038",
@@ -1708,7 +1824,10 @@ fn test_aarch64_binemit() {
insns.push(( insns.push((
Inst::Store8 { Inst::Store8 {
rd: xreg(1), rd: xreg(1),
mem: AMode::UnsignedOffset(xreg(2), UImm12Scaled::maybe_from_i64(4095, I8).unwrap()), mem: AMode::UnsignedOffset {
rn: xreg(2),
uimm12: UImm12Scaled::maybe_from_i64(4095, I8).unwrap(),
},
flags: MemFlags::trusted(), flags: MemFlags::trusted(),
}, },
"41FC3F39", "41FC3F39",
@@ -1717,7 +1836,10 @@ fn test_aarch64_binemit() {
insns.push(( insns.push((
Inst::Store16 { Inst::Store16 {
rd: xreg(1), rd: xreg(1),
mem: AMode::Unscaled(xreg(2), simm9_zero()), mem: AMode::Unscaled {
rn: xreg(2),
simm9: simm9_zero(),
},
flags: MemFlags::trusted(), flags: MemFlags::trusted(),
}, },
"41000078", "41000078",
@@ -1726,7 +1848,10 @@ fn test_aarch64_binemit() {
insns.push(( insns.push((
Inst::Store16 { Inst::Store16 {
rd: xreg(1), rd: xreg(1),
mem: AMode::UnsignedOffset(xreg(2), UImm12Scaled::maybe_from_i64(8190, I16).unwrap()), mem: AMode::UnsignedOffset {
rn: xreg(2),
uimm12: UImm12Scaled::maybe_from_i64(8190, I16).unwrap(),
},
flags: MemFlags::trusted(), flags: MemFlags::trusted(),
}, },
"41FC3F79", "41FC3F79",
@@ -1735,7 +1860,10 @@ fn test_aarch64_binemit() {
insns.push(( insns.push((
Inst::Store32 { Inst::Store32 {
rd: xreg(1), rd: xreg(1),
mem: AMode::Unscaled(xreg(2), simm9_zero()), mem: AMode::Unscaled {
rn: xreg(2),
simm9: simm9_zero(),
},
flags: MemFlags::trusted(), flags: MemFlags::trusted(),
}, },
"410000B8", "410000B8",
@@ -1744,7 +1872,10 @@ fn test_aarch64_binemit() {
insns.push(( insns.push((
Inst::Store32 { Inst::Store32 {
rd: xreg(1), rd: xreg(1),
mem: AMode::UnsignedOffset(xreg(2), UImm12Scaled::maybe_from_i64(16380, I32).unwrap()), mem: AMode::UnsignedOffset {
rn: xreg(2),
uimm12: UImm12Scaled::maybe_from_i64(16380, I32).unwrap(),
},
flags: MemFlags::trusted(), flags: MemFlags::trusted(),
}, },
"41FC3FB9", "41FC3FB9",
@@ -1753,7 +1884,10 @@ fn test_aarch64_binemit() {
insns.push(( insns.push((
Inst::Store64 { Inst::Store64 {
rd: xreg(1), rd: xreg(1),
mem: AMode::Unscaled(xreg(2), simm9_zero()), mem: AMode::Unscaled {
rn: xreg(2),
simm9: simm9_zero(),
},
flags: MemFlags::trusted(), flags: MemFlags::trusted(),
}, },
"410000F8", "410000F8",
@@ -1762,7 +1896,10 @@ fn test_aarch64_binemit() {
insns.push(( insns.push((
Inst::Store64 { Inst::Store64 {
rd: xreg(1), rd: xreg(1),
mem: AMode::UnsignedOffset(xreg(2), UImm12Scaled::maybe_from_i64(32760, I64).unwrap()), mem: AMode::UnsignedOffset {
rn: xreg(2),
uimm12: UImm12Scaled::maybe_from_i64(32760, I64).unwrap(),
},
flags: MemFlags::trusted(), flags: MemFlags::trusted(),
}, },
"41FC3FF9", "41FC3FF9",
@@ -1771,7 +1908,10 @@ fn test_aarch64_binemit() {
insns.push(( insns.push((
Inst::Store64 { Inst::Store64 {
rd: xreg(1), rd: xreg(1),
mem: AMode::RegReg(xreg(2), xreg(3)), mem: AMode::RegReg {
rn: xreg(2),
rm: xreg(3),
},
flags: MemFlags::trusted(), flags: MemFlags::trusted(),
}, },
"416823F8", "416823F8",
@@ -1780,7 +1920,11 @@ fn test_aarch64_binemit() {
insns.push(( insns.push((
Inst::Store64 { Inst::Store64 {
rd: xreg(1), rd: xreg(1),
mem: AMode::RegScaled(xreg(2), xreg(3), I64), mem: AMode::RegScaled {
rn: xreg(2),
rm: xreg(3),
ty: I64,
},
flags: MemFlags::trusted(), flags: MemFlags::trusted(),
}, },
"417823F8", "417823F8",
@@ -1789,7 +1933,12 @@ fn test_aarch64_binemit() {
insns.push(( insns.push((
Inst::Store64 { Inst::Store64 {
rd: xreg(1), rd: xreg(1),
mem: AMode::RegScaledExtended(xreg(2), xreg(3), I64, ExtendOp::UXTW), mem: AMode::RegScaledExtended {
rn: xreg(2),
rm: xreg(3),
ty: I64,
extendop: ExtendOp::UXTW,
},
flags: MemFlags::trusted(), flags: MemFlags::trusted(),
}, },
"415823F8", "415823F8",
@@ -1798,7 +1947,11 @@ fn test_aarch64_binemit() {
insns.push(( insns.push((
Inst::Store64 { Inst::Store64 {
rd: xreg(1), rd: xreg(1),
mem: AMode::RegExtended(xreg(2), xreg(3), ExtendOp::UXTW), mem: AMode::RegExtended {
rn: xreg(2),
rm: xreg(3),
extendop: ExtendOp::UXTW,
},
flags: MemFlags::trusted(), flags: MemFlags::trusted(),
}, },
"414823F8", "414823F8",
@@ -1807,7 +1960,9 @@ fn test_aarch64_binemit() {
insns.push(( insns.push((
Inst::Store64 { Inst::Store64 {
rd: xreg(1), rd: xreg(1),
mem: AMode::SPPreIndexed(SImm9::maybe_from_i64(16).unwrap()), mem: AMode::SPPreIndexed {
simm9: SImm9::maybe_from_i64(16).unwrap(),
},
flags: MemFlags::trusted(), flags: MemFlags::trusted(),
}, },
"E10F01F8", "E10F01F8",
@@ -1816,7 +1971,9 @@ fn test_aarch64_binemit() {
insns.push(( insns.push((
Inst::Store64 { Inst::Store64 {
rd: xreg(1), rd: xreg(1),
mem: AMode::SPPostIndexed(SImm9::maybe_from_i64(16).unwrap()), mem: AMode::SPPostIndexed {
simm9: SImm9::maybe_from_i64(16).unwrap(),
},
flags: MemFlags::trusted(), flags: MemFlags::trusted(),
}, },
"E10701F8", "E10701F8",
@@ -6397,7 +6554,11 @@ fn test_aarch64_binemit() {
insns.push(( insns.push((
Inst::FpuLoad32 { Inst::FpuLoad32 {
rd: writable_vreg(16), rd: writable_vreg(16),
mem: AMode::RegScaled(xreg(8), xreg(9), F32), mem: AMode::RegScaled {
rn: xreg(8),
rm: xreg(9),
ty: F32,
},
flags: MemFlags::trusted(), flags: MemFlags::trusted(),
}, },
"107969BC", "107969BC",
@@ -6407,7 +6568,11 @@ fn test_aarch64_binemit() {
insns.push(( insns.push((
Inst::FpuLoad64 { Inst::FpuLoad64 {
rd: writable_vreg(16), rd: writable_vreg(16),
mem: AMode::RegScaled(xreg(8), xreg(9), F64), mem: AMode::RegScaled {
rn: xreg(8),
rm: xreg(9),
ty: F64,
},
flags: MemFlags::trusted(), flags: MemFlags::trusted(),
}, },
"107969FC", "107969FC",
@@ -6417,7 +6582,11 @@ fn test_aarch64_binemit() {
insns.push(( insns.push((
Inst::FpuLoad128 { Inst::FpuLoad128 {
rd: writable_vreg(16), rd: writable_vreg(16),
mem: AMode::RegScaled(xreg(8), xreg(9), I128), mem: AMode::RegScaled {
rn: xreg(8),
rm: xreg(9),
ty: I128,
},
flags: MemFlags::trusted(), flags: MemFlags::trusted(),
}, },
"1079E93C", "1079E93C",
@@ -6427,7 +6596,9 @@ fn test_aarch64_binemit() {
insns.push(( insns.push((
Inst::FpuLoad32 { Inst::FpuLoad32 {
rd: writable_vreg(16), rd: writable_vreg(16),
mem: AMode::Label(MemLabel::PCRel(8)), mem: AMode::Label {
label: MemLabel::PCRel(8),
},
flags: MemFlags::trusted(), flags: MemFlags::trusted(),
}, },
"5000001C", "5000001C",
@@ -6437,7 +6608,9 @@ fn test_aarch64_binemit() {
insns.push(( insns.push((
Inst::FpuLoad64 { Inst::FpuLoad64 {
rd: writable_vreg(16), rd: writable_vreg(16),
mem: AMode::Label(MemLabel::PCRel(8)), mem: AMode::Label {
label: MemLabel::PCRel(8),
},
flags: MemFlags::trusted(), flags: MemFlags::trusted(),
}, },
"5000005C", "5000005C",
@@ -6447,7 +6620,9 @@ fn test_aarch64_binemit() {
insns.push(( insns.push((
Inst::FpuLoad128 { Inst::FpuLoad128 {
rd: writable_vreg(16), rd: writable_vreg(16),
mem: AMode::Label(MemLabel::PCRel(8)), mem: AMode::Label {
label: MemLabel::PCRel(8),
},
flags: MemFlags::trusted(), flags: MemFlags::trusted(),
}, },
"5000009C", "5000009C",
@@ -6457,7 +6632,11 @@ fn test_aarch64_binemit() {
insns.push(( insns.push((
Inst::FpuStore32 { Inst::FpuStore32 {
rd: vreg(16), rd: vreg(16),
mem: AMode::RegScaled(xreg(8), xreg(9), F32), mem: AMode::RegScaled {
rn: xreg(8),
rm: xreg(9),
ty: F32,
},
flags: MemFlags::trusted(), flags: MemFlags::trusted(),
}, },
"107929BC", "107929BC",
@@ -6467,7 +6646,11 @@ fn test_aarch64_binemit() {
insns.push(( insns.push((
Inst::FpuStore64 { Inst::FpuStore64 {
rd: vreg(16), rd: vreg(16),
mem: AMode::RegScaled(xreg(8), xreg(9), F64), mem: AMode::RegScaled {
rn: xreg(8),
rm: xreg(9),
ty: F64,
},
flags: MemFlags::trusted(), flags: MemFlags::trusted(),
}, },
"107929FC", "107929FC",
@@ -6477,7 +6660,11 @@ fn test_aarch64_binemit() {
insns.push(( insns.push((
Inst::FpuStore128 { Inst::FpuStore128 {
rd: vreg(16), rd: vreg(16),
mem: AMode::RegScaled(xreg(8), xreg(9), I128), mem: AMode::RegScaled {
rn: xreg(8),
rm: xreg(9),
ty: I128,
},
flags: MemFlags::trusted(), flags: MemFlags::trusted(),
}, },
"1079A93C", "1079A93C",

42
cranelift/codegen/src/isa/aarch64/inst/mod.rs
View File

@@ -36,7 +36,7 @@ mod emit_tests;
// Instructions (top level): definition // Instructions (top level): definition
pub use crate::isa::aarch64::lower::isle::generated_code::{ pub use crate::isa::aarch64::lower::isle::generated_code::{
ALUOp, ALUOp3, APIKey, AtomicRMWLoopOp, AtomicRMWOp, BitOp, FPUOp1, FPUOp2, FPUOp3, ALUOp, ALUOp3, AMode, APIKey, AtomicRMWLoopOp, AtomicRMWOp, BitOp, FPUOp1, FPUOp2, FPUOp3,
FpuRoundMode, FpuToIntOp, IntToFpuOp, MInst as Inst, MoveWideOp, VecALUModOp, VecALUOp, FpuRoundMode, FpuToIntOp, IntToFpuOp, MInst as Inst, MoveWideOp, VecALUModOp, VecALUOp,
VecExtendOp, VecLanesOp, VecMisc2, VecPairOp, VecRRLongOp, VecRRNarrowOp, VecRRPairLongOp, VecExtendOp, VecLanesOp, VecMisc2, VecPairOp, VecRRLongOp, VecRRNarrowOp, VecRRPairLongOp,
VecRRRLongModOp, VecRRRLongOp, VecShiftImmModOp, VecShiftImmOp, VecRRRLongModOp, VecRRRLongOp, VecShiftImmModOp, VecShiftImmOp,
@@ -546,22 +546,22 @@ impl Inst {
fn memarg_operands<F: Fn(VReg) -> VReg>(memarg: &AMode, collector: &mut OperandCollector<'_, F>) { fn memarg_operands<F: Fn(VReg) -> VReg>(memarg: &AMode, collector: &mut OperandCollector<'_, F>) {
// This should match `AMode::with_allocs()`. // This should match `AMode::with_allocs()`.
match memarg { match memarg {
&AMode::Unscaled(reg, ..) | &AMode::UnsignedOffset(reg, ..) => { &AMode::Unscaled { rn, .. } | &AMode::UnsignedOffset { rn, .. } => {
collector.reg_use(reg); collector.reg_use(rn);
} }
&AMode::RegReg(r1, r2, ..) &AMode::RegReg { rn, rm, .. }
| &AMode::RegScaled(r1, r2, ..) | &AMode::RegScaled { rn, rm, .. }
| &AMode::RegScaledExtended(r1, r2, ..) | &AMode::RegScaledExtended { rn, rm, .. }
| &AMode::RegExtended(r1, r2, ..) => { | &AMode::RegExtended { rn, rm, .. } => {
collector.reg_use(r1); collector.reg_use(rn);
collector.reg_use(r2); collector.reg_use(rm);
} }
&AMode::Label(..) => {} &AMode::Label { .. } => {}
&AMode::SPPreIndexed(..) | &AMode::SPPostIndexed(..) => {} &AMode::SPPreIndexed { .. } | &AMode::SPPostIndexed { .. } => {}
&AMode::FPOffset(..) => {} &AMode::FPOffset { .. } => {}
&AMode::SPOffset(..) | &AMode::NominalSPOffset(..) => {} &AMode::SPOffset { .. } | &AMode::NominalSPOffset { .. } => {}
&AMode::RegOffset(r, ..) => { &AMode::RegOffset { rn, .. } => {
collector.reg_use(r); collector.reg_use(rn);
} }
} }
} }
@@ -1446,7 +1446,7 @@ impl Inst {
| &Inst::SLoad32 { rd, ref mem, .. } | &Inst::SLoad32 { rd, ref mem, .. }
| &Inst::ULoad64 { rd, ref mem, .. } => { | &Inst::ULoad64 { rd, ref mem, .. } => {
let is_unscaled = match &mem { let is_unscaled = match &mem {
&AMode::Unscaled(..) => true, &AMode::Unscaled { .. } => true,
_ => false, _ => false,
}; };
let (op, size) = match (self, is_unscaled) { let (op, size) = match (self, is_unscaled) {
@@ -1479,7 +1479,7 @@ impl Inst {
| &Inst::Store32 { rd, ref mem, .. } | &Inst::Store32 { rd, ref mem, .. }
| &Inst::Store64 { rd, ref mem, .. } => { | &Inst::Store64 { rd, ref mem, .. } => {
let is_unscaled = match &mem { let is_unscaled = match &mem {
&AMode::Unscaled(..) => true, &AMode::Unscaled { .. } => true,
_ => false, _ => false,
}; };
let (op, size) = match (self, is_unscaled) { let (op, size) = match (self, is_unscaled) {
@@ -2736,11 +2736,9 @@ impl Inst {
); );
} }
let (reg, index_reg, offset) = match mem { let (reg, index_reg, offset) = match mem {
AMode::RegExtended(r, idx, extendop) => (r, Some((idx, extendop)), 0), AMode::RegExtended { rn, rm, extendop } => (rn, Some((rm, extendop)), 0),
AMode::Unscaled(r, simm9) => (r, None, simm9.value()), AMode::Unscaled { rn, simm9 } => (rn, None, simm9.value()),
AMode::UnsignedOffset(r, uimm12scaled) => { AMode::UnsignedOffset { rn, uimm12 } => (rn, None, uimm12.value() as i32),
(r, None, uimm12scaled.value() as i32)
}
_ => panic!("Unsupported case for LoadAddr: {:?}", mem), _ => panic!("Unsupported case for LoadAddr: {:?}", mem),
}; };
let abs_offset = if offset < 0 { let abs_offset = if offset < 0 {

5
cranelift/codegen/src/isa/aarch64/lower.isle
View File

@@ -1772,10 +1772,9 @@
(rule (lower (has_type ty (splat (ireduce (iconst (u64_from_imm64 n)))))) (rule (lower (has_type ty (splat (ireduce (iconst (u64_from_imm64 n))))))
(splat_const n (vector_size ty))) (splat_const n (vector_size ty)))
(rule (lower (has_type ty (splat x @ (load flags addr offset)))) (rule (lower (has_type ty (splat x @ (load flags _ _))))
(if-let mem_op (is_sinkable_inst x)) (if-let mem_op (is_sinkable_inst x))
(let ((_ Unit (sink_inst mem_op)) (let ((addr AMode (sink_load_into_amode (lane_type ty) mem_op))
(addr AMode (amode (lane_type ty) addr offset))
(address Reg (load_addr addr))) (address Reg (load_addr addr)))
(ld1r address (vector_size ty) flags))) (ld1r address (vector_size ty) flags)))

56
cranelift/codegen/src/isa/aarch64/lower.rs
View File

@@ -658,16 +658,24 @@ pub(crate) fn lower_address(
if addends32.len() > 0 { if addends32.len() > 0 {
let (reg32, extendop) = addends32.pop().unwrap(); let (reg32, extendop) = addends32.pop().unwrap();
let reg64 = addends64.pop().unwrap(); let reg64 = addends64.pop().unwrap();
AMode::RegExtended(reg64, reg32, extendop) AMode::RegExtended {
rn: reg64,
rm: reg32,
extendop,
}
} else if offset > 0 && offset < 0x1000 { } else if offset > 0 && offset < 0x1000 {
let reg64 = addends64.pop().unwrap(); let reg64 = addends64.pop().unwrap();
let off = offset; let off = offset;
offset = 0; offset = 0;
AMode::RegOffset(reg64, off, elem_ty) AMode::RegOffset {
rn: reg64,
off,
ty: elem_ty,
}
} else if addends64.len() >= 2 { } else if addends64.len() >= 2 {
let reg1 = addends64.pop().unwrap(); let reg1 = addends64.pop().unwrap();
let reg2 = addends64.pop().unwrap(); let reg2 = addends64.pop().unwrap();
AMode::RegReg(reg1, reg2) AMode::RegReg { rn: reg1, rm: reg2 }
} else { } else {
let reg1 = addends64.pop().unwrap(); let reg1 = addends64.pop().unwrap();
AMode::reg(reg1) AMode::reg(reg1)
@@ -691,7 +699,11 @@ pub(crate) fn lower_address(
to_bits: 64, to_bits: 64,
}); });
if let Some((reg2, extendop)) = addends32.pop() { if let Some((reg2, extendop)) = addends32.pop() {
AMode::RegExtended(tmp.to_reg(), reg2, extendop) AMode::RegExtended {
rn: tmp.to_reg(),
rm: reg2,
extendop,
}
} else { } else {
AMode::reg(tmp.to_reg()) AMode::reg(tmp.to_reg())
} }
@@ -716,12 +728,36 @@ pub(crate) fn lower_address(
// Allocate the temp and shoehorn it into the AMode. // Allocate the temp and shoehorn it into the AMode.
let addr = ctx.alloc_tmp(I64).only_reg().unwrap(); let addr = ctx.alloc_tmp(I64).only_reg().unwrap();
let (reg, memarg) = match memarg { let (reg, memarg) = match memarg {
AMode::RegExtended(r1, r2, extendop) => { AMode::RegExtended { rn, rm, extendop } => (
(r1, AMode::RegExtended(addr.to_reg(), r2, extendop)) rn,
} AMode::RegExtended {
AMode::RegOffset(r, off, ty) => (r, AMode::RegOffset(addr.to_reg(), off, ty)), rn: addr.to_reg(),
AMode::RegReg(r1, r2) => (r2, AMode::RegReg(addr.to_reg(), r1)), rm,
AMode::UnsignedOffset(r, imm) => (r, AMode::UnsignedOffset(addr.to_reg(), imm)), extendop,
},
),
AMode::RegOffset { rn, off, ty } => (
rn,
AMode::RegOffset {
rn: addr.to_reg(),
off,
ty,
},
),
AMode::RegReg { rn, rm } => (
rm,
AMode::RegReg {
rn: addr.to_reg(),
rm: rn,
},
),
AMode::UnsignedOffset { rn, uimm12 } => (
rn,
AMode::UnsignedOffset {
rn: addr.to_reg(),
uimm12,
},
),
_ => unreachable!(), _ => unreachable!(),
}; };

40
cranelift/codegen/src/isa/aarch64/lower/isle.rs
View File

@@ -6,12 +6,12 @@ use generated_code::Context;
// Types that the generated ISLE code uses via `use super::*`. // Types that the generated ISLE code uses via `use super::*`.
use super::{ use super::{
lower_constant_f128, lower_constant_f32, lower_constant_f64, lower_fp_condcode, fp_reg, lower_constant_f128, lower_constant_f32, lower_constant_f64, lower_fp_condcode,
writable_zero_reg, zero_reg, AMode, ASIMDFPModImm, ASIMDMovModImm, BranchTarget, CallIndInfo, stack_reg, writable_zero_reg, zero_reg, AMode, ASIMDFPModImm, ASIMDMovModImm, BranchTarget,
CallInfo, Cond, CondBrKind, ExtendOp, FPUOpRI, FPUOpRIMod, FloatCC, Imm12, ImmLogic, ImmShift, CallIndInfo, CallInfo, Cond, CondBrKind, ExtendOp, FPUOpRI, FPUOpRIMod, FloatCC, Imm12,
Inst as MInst, IntCC, JTSequenceInfo, MachLabel, MoveWideConst, MoveWideOp, NarrowValueMode, ImmLogic, ImmShift, Inst as MInst, IntCC, JTSequenceInfo, MachLabel, MemLabel, MoveWideConst,
Opcode, OperandSize, PairAMode, Reg, ScalarSize, ShiftOpAndAmt, UImm5, VecMisc2, VectorSize, MoveWideOp, NarrowValueMode, Opcode, OperandSize, PairAMode, Reg, SImm9, ScalarSize,
NZCV, ShiftOpAndAmt, UImm12Scaled, UImm5, VecMisc2, VectorSize, NZCV,
}; };
use crate::ir::condcodes; use crate::ir::condcodes;
use crate::isa::aarch64::inst::{FPULeftShiftImm, FPURightShiftImm}; use crate::isa::aarch64::inst::{FPULeftShiftImm, FPURightShiftImm};
@@ -151,6 +151,22 @@ impl Context for IsleContext<'_, '_, MInst, Flags, IsaFlags, 6> {
} }
} }
fn is_zero_simm9(&mut self, imm: &SImm9) -> Option<()> {
if imm.value() == 0 {
Some(())
} else {
None
}
}
fn is_zero_uimm12(&mut self, imm: &UImm12Scaled) -> Option<()> {
if imm.value() == 0 {
Some(())
} else {
None
}
}
/// This is target-word-size dependent. And it excludes booleans and reftypes. /// This is target-word-size dependent. And it excludes booleans and reftypes.
fn valid_atomic_transaction(&mut self, ty: Type) -> Option<Type> { fn valid_atomic_transaction(&mut self, ty: Type) -> Option<Type> {
match ty { match ty {
@@ -293,6 +309,14 @@ impl Context for IsleContext<'_, '_, MInst, Flags, IsaFlags, 6> {
zero_reg() zero_reg()
} }
fn stack_reg(&mut self) -> Reg {
stack_reg()
}
fn fp_reg(&mut self) -> Reg {
fp_reg()
}
fn extended_value_from_value(&mut self, val: Value) -> Option<ExtendedValue> { fn extended_value_from_value(&mut self, val: Value) -> Option<ExtendedValue> {
let (val, extend) = let (val, extend) =
super::get_as_extended_value(self.lower_ctx, val, NarrowValueMode::None)?; super::get_as_extended_value(self.lower_ctx, val, NarrowValueMode::None)?;
@@ -481,10 +505,6 @@ impl Context for IsleContext<'_, '_, MInst, Flags, IsaFlags, 6> {
lower_pair_address(self.lower_ctx, addr, offset as i32) lower_pair_address(self.lower_ctx, addr, offset as i32)
} }
fn amode_is_reg(&mut self, address: &AMode) -> Option<Reg> {
address.is_reg()
}
fn constant_f64(&mut self, value: u64) -> Reg { fn constant_f64(&mut self, value: u64) -> Reg {
let rd = self.temp_writable_reg(I8X16); let rd = self.temp_writable_reg(I8X16);