T0b1/wasmtime
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Cranelift AArch64: Migrate AMode to ISLE (#4832)

Browse Source 
Copyright (c) 2022, Arm Limited.

Co-authored-by: Chris Fallin <chris@cfallin.org>
This commit is contained in:
Anton Kirilov
2022-09-02 01:24:46 +01:00
committed by GitHub
parent 385bd0cbf8
commit d2e19b8d74
9 changed files with 684 additions and 329 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
cranelift/codegen/src/isa/aarch64/abi.rs
View File

@@ -34,9 +34,9 @@ static STACK_ARG_RET_SIZE_LIMIT: u64 = 128 * 1024 * 1024;
impl Into<AMode> for StackAMode {
fn into(self) -> AMode {
match self {
StackAMode::FPOffset(off, ty) => AMode::FPOffset(off, ty),
StackAMode::NominalSPOffset(off, ty) => AMode::NominalSPOffset(off, ty),
StackAMode::SPOffset(off, ty) => AMode::SPOffset(off, ty),
StackAMode::FPOffset(off, ty) => AMode::FPOffset { off, ty },
StackAMode::NominalSPOffset(off, ty) => AMode::NominalSPOffset { off, ty },
StackAMode::SPOffset(off, ty) => AMode::SPOffset { off, ty },
}
}
}
@@ -462,12 +462,20 @@ impl ABIMachineSpec for AArch64MachineDeps {
}
fn gen_load_base_offset(into_reg: Writable<Reg>, base: Reg, offset: i32, ty: Type) -> Inst {
let mem = AMode::RegOffset(base, offset as i64, ty);
let mem = AMode::RegOffset {
rn: base,
off: offset as i64,
ty,
};
Inst::gen_load(into_reg, mem, ty, MemFlags::trusted())
}
fn gen_store_base_offset(base: Reg, offset: i32, from_reg: Reg, ty: Type) -> Inst {
let mem = AMode::RegOffset(base, offset as i64, ty);
let mem = AMode::RegOffset {
rn: base,
off: offset as i64,
ty,
};
Inst::gen_store(mem, from_reg, ty, MemFlags::trusted())
}
@@ -674,7 +682,9 @@ impl ABIMachineSpec for AArch64MachineDeps {
// str rd, [sp, #-16]!
insts.push(Inst::Store64 {
rd,
mem: AMode::SPPreIndexed(SImm9::maybe_from_i64(-clobber_offset_change).unwrap()),
mem: AMode::SPPreIndexed {
simm9: SImm9::maybe_from_i64(-clobber_offset_change).unwrap(),
},
flags: MemFlags::trusted(),
});
@@ -728,7 +738,9 @@ impl ABIMachineSpec for AArch64MachineDeps {
let store_vec_reg = |rd| Inst::FpuStore64 {
rd,
mem: AMode::SPPreIndexed(SImm9::maybe_from_i64(-clobber_offset_change).unwrap()),
mem: AMode::SPPreIndexed {
simm9: SImm9::maybe_from_i64(-clobber_offset_change).unwrap(),
},
flags: MemFlags::trusted(),
};
let iter = clobbered_vec.chunks_exact(2);
@@ -821,7 +833,9 @@ impl ABIMachineSpec for AArch64MachineDeps {
let load_vec_reg = |rd| Inst::FpuLoad64 {
rd,
mem: AMode::SPPostIndexed(SImm9::maybe_from_i64(16).unwrap()),
mem: AMode::SPPostIndexed {
simm9: SImm9::maybe_from_i64(16).unwrap(),
},
flags: MemFlags::trusted(),
};
let load_vec_reg_pair = |rt, rt2| Inst::FpuLoadP64 {
@@ -877,7 +891,9 @@ impl ABIMachineSpec for AArch64MachineDeps {
// ldr rd, [sp], #16
insts.push(Inst::ULoad64 {
rd,
mem: AMode::SPPostIndexed(SImm9::maybe_from_i64(16).unwrap()),
mem: AMode::SPPostIndexed {
simm9: SImm9::maybe_from_i64(16).unwrap(),
},
flags: MemFlags::trusted(),
});
}

150
cranelift/codegen/src/isa/aarch64/inst.isle
View File

@@ -1010,7 +1010,108 @@
(Cls)
))
(type AMode extern (enum))
(type MemLabel extern (enum))
(type SImm9 extern (enum))
(type UImm12Scaled extern (enum))
;; An addressing mode specified for a load/store operation.
(type AMode
(enum
;;
;; Real ARM64 addressing modes:
;;
;; "post-indexed" mode as per AArch64 docs: postincrement reg after
;; address computation.
;; Specialized here to SP so we don't have to emit regalloc metadata.
(SPPostIndexed
(simm9 SImm9))
;; "pre-indexed" mode as per AArch64 docs: preincrement reg before
;; address computation.
;; Specialized here to SP so we don't have to emit regalloc metadata.
(SPPreIndexed
(simm9 SImm9))
;; N.B.: RegReg, RegScaled, and RegScaledExtended all correspond to
;; what the ISA calls the "register offset" addressing mode. We split
;; out several options here for more ergonomic codegen.
;;
;; Register plus register offset.
(RegReg
(rn Reg)
(rm Reg))
;; Register plus register offset, scaled by type's size.
(RegScaled
(rn Reg)
(rm Reg)
(ty Type))
;; Register plus register offset, scaled by type's size, with index
;; sign- or zero-extended first.
(RegScaledExtended
(rn Reg)
(rm Reg)
(ty Type)
(extendop ExtendOp))
;; Register plus register offset, with index sign- or zero-extended
;; first.
(RegExtended
(rn Reg)
(rm Reg)
(extendop ExtendOp))
;; Unscaled signed 9-bit immediate offset from reg.
(Unscaled
(rn Reg)
(simm9 SImm9))
;; Scaled (by size of a type) unsigned 12-bit immediate offset from reg.
(UnsignedOffset
(rn Reg)
(uimm12 UImm12Scaled))
;; virtual addressing modes that are lowered at emission time:
;;
;; Reference to a "label": e.g., a symbol.
(Label
(label MemLabel))
;; Arbitrary offset from a register. Converted to generation of large
;; offsets with multiple instructions as necessary during code emission.
(RegOffset
(rn Reg)
(off i64)
(ty Type))
;; Offset from the stack pointer.
(SPOffset
(off i64)
(ty Type))
;; Offset from the frame pointer.
(FPOffset
(off i64)
(ty Type))
;; Offset from the "nominal stack pointer", which is where the real SP is
;; just after stack and spill slots are allocated in the function prologue.
;; At emission time, this is converted to `SPOffset` with a fixup added to
;; the offset constant. The fixup is a running value that is tracked as
;; emission iterates through instructions in linear order, and can be
;; adjusted up and down with [Inst::VirtualSPOffsetAdj].
;;
;; The standard ABI is in charge of handling this (by emitting the
;; adjustment meta-instructions). It maintains the invariant that "nominal
;; SP" is where the actual SP is after the function prologue and before
;; clobber pushes. See the diagram in the documentation for
;; [crate::isa::aarch64::abi](the ABI module) for more details.
(NominalSPOffset
(off i64)
(ty Type))
))
(type PairAMode extern (enum))
(type FPUOpRI extern (enum))
(type FPUOpRIMod extern (enum))
@@ -1554,6 +1655,12 @@
(decl valid_atomic_transaction (Type) Type)
(extern extractor valid_atomic_transaction valid_atomic_transaction)
(decl pure is_zero_simm9 (SImm9) Unit)
(extern constructor is_zero_simm9 is_zero_simm9)
(decl pure is_zero_uimm12 (UImm12Scaled) Unit)
(extern constructor is_zero_uimm12 is_zero_uimm12)
;; Helper to go directly from a `Value`, when it's an `iconst`, to an `Imm12`.
(decl imm12_from_value (Imm12) Value)
(extractor
@@ -1586,25 +1693,21 @@
(decl cond_br_cond (Cond) CondBrKind)
(extern constructor cond_br_cond cond_br_cond)
;; Lower the address of a load or a store.
(decl amode (Type Value u32) AMode)
;; TODO: Port lower_address() to ISLE.
(extern constructor amode amode)
(decl pair_amode (Value u32) PairAMode)
(extern constructor pair_amode pair_amode)
;; Matches an `AMode` that is just a register.
(decl pure amode_is_reg (AMode) Reg)
;; TODO: Implement in ISLE.
(extern constructor amode_is_reg amode_is_reg)
;; Instruction creation helpers ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Helper for creating the zero register.
(decl zero_reg () Reg)
(extern constructor zero_reg zero_reg)
(decl fp_reg () Reg)
(extern constructor fp_reg fp_reg)
(decl stack_reg () Reg)
(extern constructor stack_reg stack_reg)
(decl writable_zero_reg () WritableReg)
(extern constructor writable_zero_reg writable_zero_reg)
@@ -2678,14 +2781,33 @@
;; Helper for emitting `MInst.LoadAddr` instructions.
(decl load_addr (AMode) Reg)
(rule (load_addr (AMode.UnsignedOffset r imm))
(if (is_zero_uimm12 imm))
r)
(rule (load_addr (AMode.Unscaled r imm))
(if (is_zero_simm9 imm))
r)
(rule (load_addr (AMode.RegOffset r 0 _)) r)
(rule (load_addr (AMode.FPOffset 0 _)) (fp_reg))
(rule (load_addr (AMode.SPOffset 0 _)) (stack_reg))
(rule (load_addr addr)
(let ((dst WritableReg (temp_writable_reg $I64))
(_ Unit (emit (MInst.LoadAddr dst addr))))
dst))
(rule (load_addr addr)
(if-let addr_reg (amode_is_reg addr))
addr_reg)
;; Lower the address of a load or a store.
(decl amode (Type Value u32) AMode)
;; TODO: Port lower_address() to ISLE.
(extern constructor amode amode)
(decl sink_load_into_amode (Type Inst) AMode)
(rule (sink_load_into_amode ty x @ (load _ addr offset))
(let ((_ Unit (sink_inst x)))
(amode ty addr offset)))
;; Lower a constant f64.
(decl constant_f64 (u64) Reg)

214
cranelift/codegen/src/isa/aarch64/inst/args.rs
View File

@@ -115,122 +115,75 @@ pub enum MemLabel {
PCRel(i32),
}
/// An addressing mode specified for a load/store operation.
#[derive(Clone, Debug)]
pub enum AMode {
//
// Real ARM64 addressing modes:
//
/// "post-indexed" mode as per AArch64 docs: postincrement reg after address computation.
/// Specialized here to SP so we don't have to emit regalloc metadata.
SPPostIndexed(SImm9),
/// "pre-indexed" mode as per AArch64 docs: preincrement reg before address computation.
/// Specialized here to SP so we don't have to emit regalloc metadata.
SPPreIndexed(SImm9),
// N.B.: RegReg, RegScaled, and RegScaledExtended all correspond to
// what the ISA calls the "register offset" addressing mode. We split out
// several options here for more ergonomic codegen.
/// Register plus register offset.
RegReg(Reg, Reg),
#[allow(dead_code)]
/// Register plus register offset, scaled by type's size.
RegScaled(Reg, Reg, Type),
/// Register plus register offset, scaled by type's size, with index sign- or zero-extended
/// first.
RegScaledExtended(Reg, Reg, Type, ExtendOp),
/// Register plus register offset, with index sign- or zero-extended first.
RegExtended(Reg, Reg, ExtendOp),
/// Unscaled signed 9-bit immediate offset from reg.
Unscaled(Reg, SImm9),
/// Scaled (by size of a type) unsigned 12-bit immediate offset from reg.
UnsignedOffset(Reg, UImm12Scaled),
//
// virtual addressing modes that are lowered at emission time:
//
/// Reference to a "label": e.g., a symbol.
Label(MemLabel),
/// Arbitrary offset from a register. Converted to generation of large
/// offsets with multiple instructions as necessary during code emission.
RegOffset(Reg, i64, Type),
/// Offset from the stack pointer.
SPOffset(i64, Type),
/// Offset from the frame pointer.
FPOffset(i64, Type),
/// Offset from the "nominal stack pointer", which is where the real SP is
/// just after stack and spill slots are allocated in the function prologue.
/// At emission time, this is converted to `SPOffset` with a fixup added to
/// the offset constant. The fixup is a running value that is tracked as
/// emission iterates through instructions in linear order, and can be
/// adjusted up and down with [Inst::VirtualSPOffsetAdj].
///
/// The standard ABI is in charge of handling this (by emitting the
/// adjustment meta-instructions). It maintains the invariant that "nominal
/// SP" is where the actual SP is after the function prologue and before
/// clobber pushes. See the diagram in the documentation for
/// [crate::isa::aarch64::abi](the ABI module) for more details.
NominalSPOffset(i64, Type),
}
impl AMode {
/// Memory reference using an address in a register.
pub fn reg(reg: Reg) -> AMode {
// Use UnsignedOffset rather than Unscaled to use ldr rather than ldur.
// This also does not use PostIndexed / PreIndexed as they update the register.
AMode::UnsignedOffset(reg, UImm12Scaled::zero(I64))
AMode::UnsignedOffset {
rn: reg,
uimm12: UImm12Scaled::zero(I64),
}
}
/// Memory reference using `reg1 + sizeof(ty) * reg2` as an address, with `reg2` sign- or
/// zero-extended as per `op`.
pub fn reg_plus_reg_scaled_extended(reg1: Reg, reg2: Reg, ty: Type, op: ExtendOp) -> AMode {
AMode::RegScaledExtended(reg1, reg2, ty, op)
}
/// Does the address resolve to just a register value, with no offset or
/// other computation?
pub fn is_reg(&self) -> Option<Reg> {
match self {
&AMode::UnsignedOffset(r, uimm12) if uimm12.value() == 0 => Some(r),
&AMode::Unscaled(r, imm9) if imm9.value() == 0 => Some(r),
&AMode::RegOffset(r, off, _) if off == 0 => Some(r),
&AMode::FPOffset(off, _) if off == 0 => Some(fp_reg()),
&AMode::SPOffset(off, _) if off == 0 => Some(stack_reg()),
_ => None,
AMode::RegScaledExtended {
rn: reg1,
rm: reg2,
ty,
extendop: op,
}
}
pub fn with_allocs(&self, allocs: &mut AllocationConsumer<'_>) -> Self {
// This should match `memarg_operands()`.
match self {
&AMode::Unscaled(reg, imm9) => AMode::Unscaled(allocs.next(reg), imm9),
&AMode::UnsignedOffset(r, uimm12) => AMode::UnsignedOffset(allocs.next(r), uimm12),
&AMode::RegReg(r1, r2) => AMode::RegReg(allocs.next(r1), allocs.next(r2)),
&AMode::RegScaled(r1, r2, ty) => AMode::RegScaled(allocs.next(r1), allocs.next(r2), ty),
&AMode::RegScaledExtended(r1, r2, ty, ext) => {
AMode::RegScaledExtended(allocs.next(r1), allocs.next(r2), ty, ext)
}
&AMode::RegExtended(r1, r2, ext) => {
AMode::RegExtended(allocs.next(r1), allocs.next(r2), ext)
}
// Note that SP is not managed by regalloc, so there is no register to report in the
// pre/post-indexed amodes.
&AMode::RegOffset(r, off, ty) => AMode::RegOffset(allocs.next(r), off, ty),
&AMode::SPPreIndexed(..)
| &AMode::SPPostIndexed(..)
| &AMode::FPOffset(..)
| &AMode::SPOffset(..)
| &AMode::NominalSPOffset(..)
| AMode::Label(..) => self.clone(),
&AMode::Unscaled { rn, simm9 } => AMode::Unscaled {
rn: allocs.next(rn),
simm9,
},
&AMode::UnsignedOffset { rn, uimm12 } => AMode::UnsignedOffset {
rn: allocs.next(rn),
uimm12,
},
&AMode::RegReg { rn, rm } => AMode::RegReg {
rn: allocs.next(rn),
rm: allocs.next(rm),
},
&AMode::RegScaled { rn, rm, ty } => AMode::RegScaled {
rn: allocs.next(rn),
rm: allocs.next(rm),
ty,
},
&AMode::RegScaledExtended {
rn,
rm,
ty,
extendop,
} => AMode::RegScaledExtended {
rn: allocs.next(rn),
rm: allocs.next(rm),
ty,
extendop,
},
&AMode::RegExtended { rn, rm, extendop } => AMode::RegExtended {
rn: allocs.next(rn),
rm: allocs.next(rm),
extendop,
},
&AMode::RegOffset { rn, off, ty } => AMode::RegOffset {
rn: allocs.next(rn),
off,
ty,
},
&AMode::SPPreIndexed { .. }
| &AMode::SPPostIndexed { .. }
| &AMode::FPOffset { .. }
| &AMode::SPOffset { .. }
| &AMode::NominalSPOffset { .. }
| AMode::Label { .. } => self.clone(),
}
}
}
@@ -418,8 +371,8 @@ fn shift_for_type(ty: Type) -> usize {
impl PrettyPrint for AMode {
fn pretty_print(&self, _: u8, allocs: &mut AllocationConsumer<'_>) -> String {
match self {
&AMode::Unscaled(reg, simm9) => {
let reg = pretty_print_reg(reg, allocs);
&AMode::Unscaled { rn, simm9 } => {
let reg = pretty_print_reg(rn, allocs);
if simm9.value != 0 {
let simm9 = simm9.pretty_print(8, allocs);
format!("[{}, {}]", reg, simm9)
@@ -427,8 +380,8 @@ impl PrettyPrint for AMode {
format!("[{}]", reg)
}
}
&AMode::UnsignedOffset(reg, uimm12) => {
let reg = pretty_print_reg(reg, allocs);
&AMode::UnsignedOffset { rn, uimm12 } => {
let reg = pretty_print_reg(rn, allocs);
if uimm12.value != 0 {
let uimm12 = uimm12.pretty_print(8, allocs);
format!("[{}, {}]", reg, uimm12)
@@ -436,53 +389,58 @@ impl PrettyPrint for AMode {
format!("[{}]", reg)
}
}
&AMode::RegReg(r1, r2) => {
let r1 = pretty_print_reg(r1, allocs);
let r2 = pretty_print_reg(r2, allocs);
&AMode::RegReg { rn, rm } => {
let r1 = pretty_print_reg(rn, allocs);
let r2 = pretty_print_reg(rm, allocs);
format!("[{}, {}]", r1, r2)
}
&AMode::RegScaled(r1, r2, ty) => {
let r1 = pretty_print_reg(r1, allocs);
let r2 = pretty_print_reg(r2, allocs);
&AMode::RegScaled { rn, rm, ty } => {
let r1 = pretty_print_reg(rn, allocs);
let r2 = pretty_print_reg(rm, allocs);
let shift = shift_for_type(ty);
format!("[{}, {}, LSL #{}]", r1, r2, shift)
}
&AMode::RegScaledExtended(r1, r2, ty, op) => {
&AMode::RegScaledExtended {
rn,
rm,
ty,
extendop,
} => {
let shift = shift_for_type(ty);
let size = match op {
let size = match extendop {
ExtendOp::SXTW | ExtendOp::UXTW => OperandSize::Size32,
_ => OperandSize::Size64,
};
let r1 = pretty_print_reg(r1, allocs);
let r2 = pretty_print_ireg(r2, size, allocs);
let op = op.pretty_print(0, allocs);
let r1 = pretty_print_reg(rn, allocs);
let r2 = pretty_print_ireg(rm, size, allocs);
let op = extendop.pretty_print(0, allocs);
format!("[{}, {}, {} #{}]", r1, r2, op, shift)
}
&AMode::RegExtended(r1, r2, op) => {
let size = match op {
&AMode::RegExtended { rn, rm, extendop } => {
let size = match extendop {
ExtendOp::SXTW | ExtendOp::UXTW => OperandSize::Size32,
_ => OperandSize::Size64,
};
let r1 = pretty_print_reg(r1, allocs);
let r2 = pretty_print_ireg(r2, size, allocs);
let op = op.pretty_print(0, allocs);
let r1 = pretty_print_reg(rn, allocs);
let r2 = pretty_print_ireg(rm, size, allocs);
let op = extendop.pretty_print(0, allocs);
format!("[{}, {}, {}]", r1, r2, op)
}
&AMode::Label(ref label) => label.pretty_print(0, allocs),
&AMode::SPPreIndexed(simm9) => {
&AMode::Label { ref label } => label.pretty_print(0, allocs),
&AMode::SPPreIndexed { simm9 } => {
let simm9 = simm9.pretty_print(8, allocs);
format!("[sp, {}]!", simm9)
}
&AMode::SPPostIndexed(simm9) => {
&AMode::SPPostIndexed { simm9 } => {
let simm9 = simm9.pretty_print(8, allocs);
format!("[sp], {}", simm9)
}
// Eliminated by `mem_finalize()`.
&AMode::SPOffset(..)
| &AMode::FPOffset(..)
| &AMode::NominalSPOffset(..)
| &AMode::RegOffset(..) => {
panic!("Unexpected pseudo mem-arg mode (stack-offset or generic reg-offset)!")
&AMode::SPOffset { .. }
| &AMode::FPOffset { .. }
| &AMode::NominalSPOffset { .. }
| &AMode::RegOffset { .. } => {
panic!("Unexpected pseudo mem-arg mode: {:?}", self)
}
}
}

163
cranelift/codegen/src/isa/aarch64/inst/emit.rs
View File

@@ -28,18 +28,18 @@ pub fn mem_finalize(
state: &EmitState,
) -> (SmallVec<[Inst; 4]>, AMode) {
match mem {
&AMode::RegOffset(_, off, ty)
| &AMode::SPOffset(off, ty)
| &AMode::FPOffset(off, ty)
| &AMode::NominalSPOffset(off, ty) => {
&AMode::RegOffset { off, ty, .. }
| &AMode::SPOffset { off, ty }
| &AMode::FPOffset { off, ty }
| &AMode::NominalSPOffset { off, ty } => {
let basereg = match mem {
&AMode::RegOffset(reg, _, _) => reg,
&AMode::SPOffset(..) | &AMode::NominalSPOffset(..) => stack_reg(),
&AMode::FPOffset(..) => fp_reg(),
&AMode::RegOffset { rn, .. } => rn,
&AMode::SPOffset { .. } | &AMode::NominalSPOffset { .. } => stack_reg(),
&AMode::FPOffset { .. } => fp_reg(),
_ => unreachable!(),
};
let adj = match mem {
&AMode::NominalSPOffset(..) => {
&AMode::NominalSPOffset { .. } => {
trace!(
"mem_finalize: nominal SP offset {} + adj {} -> {}",
off,
@@ -53,10 +53,13 @@ pub fn mem_finalize(
let off = off + adj;
if let Some(simm9) = SImm9::maybe_from_i64(off) {
let mem = AMode::Unscaled(basereg, simm9);
let mem = AMode::Unscaled { rn: basereg, simm9 };
(smallvec![], mem)
} else if let Some(uimm12s) = UImm12Scaled::maybe_from_i64(off, ty) {
let mem = AMode::UnsignedOffset(basereg, uimm12s);
} else if let Some(uimm12) = UImm12Scaled::maybe_from_i64(off, ty) {
let mem = AMode::UnsignedOffset {
rn: basereg,
uimm12,
};
(smallvec![], mem)
} else {
let tmp = writable_spilltmp_reg();
@@ -78,9 +81,14 @@ pub fn mem_finalize(
}
}
&AMode::Label(ref label) => {
&AMode::Label { ref label } => {
let off = memlabel_finalize(insn_off, label);
(smallvec![], AMode::Label(MemLabel::PCRel(off)))
(
smallvec![],
AMode::Label {
label: MemLabel::PCRel(off),
},
)
}
_ => (smallvec![], mem.clone()),
@@ -977,40 +985,41 @@ impl MachInstEmit for Inst {
}
match &mem {
&AMode::Unscaled(reg, simm9) => {
let reg = allocs.next(reg);
&AMode::Unscaled { rn, simm9 } => {
let reg = allocs.next(rn);
sink.put4(enc_ldst_simm9(op, simm9, 0b00, reg, rd));
}
&AMode::UnsignedOffset(reg, uimm12scaled) => {
let reg = allocs.next(reg);
if uimm12scaled.value() != 0 {
assert_eq!(bits, ty_bits(uimm12scaled.scale_ty()));
&AMode::UnsignedOffset { rn, uimm12 } => {
let reg = allocs.next(rn);
if uimm12.value() != 0 {
assert_eq!(bits, ty_bits(uimm12.scale_ty()));
}
sink.put4(enc_ldst_uimm12(op, uimm12scaled, reg, rd));
sink.put4(enc_ldst_uimm12(op, uimm12, reg, rd));
}
&AMode::RegReg(r1, r2) => {
let r1 = allocs.next(r1);
let r2 = allocs.next(r2);
&AMode::RegReg { rn, rm } => {
let r1 = allocs.next(rn);
let r2 = allocs.next(rm);
sink.put4(enc_ldst_reg(
op, r1, r2, /* scaled = */ false, /* extendop = */ None, rd,
));
}
&AMode::RegScaled(r1, r2, ty) | &AMode::RegScaledExtended(r1, r2, ty, _) => {
let r1 = allocs.next(r1);
let r2 = allocs.next(r2);
&AMode::RegScaled { rn, rm, ty }
| &AMode::RegScaledExtended { rn, rm, ty, .. } => {
let r1 = allocs.next(rn);
let r2 = allocs.next(rm);
assert_eq!(bits, ty_bits(ty));
let extendop = match &mem {
&AMode::RegScaled(..) => None,
&AMode::RegScaledExtended(_, _, _, op) => Some(op),
&AMode::RegScaled { .. } => None,
&AMode::RegScaledExtended { extendop, .. } => Some(extendop),
_ => unreachable!(),
};
sink.put4(enc_ldst_reg(
op, r1, r2, /* scaled = */ true, extendop, rd,
));
}
&AMode::RegExtended(r1, r2, extendop) => {
let r1 = allocs.next(r1);
let r2 = allocs.next(r2);
&AMode::RegExtended { rn, rm, extendop } => {
let r1 = allocs.next(rn);
let r2 = allocs.next(rm);
sink.put4(enc_ldst_reg(
op,
r1,
@@ -1020,7 +1029,7 @@ impl MachInstEmit for Inst {
rd,
));
}
&AMode::Label(ref label) => {
&AMode::Label { ref label } => {
let offset = match label {
// cast i32 to u32 (two's-complement)
&MemLabel::PCRel(off) => off as u32,
@@ -1048,19 +1057,21 @@ impl MachInstEmit for Inst {
_ => panic!("Unspported size for LDR from constant pool!"),
}
}
&AMode::SPPreIndexed(simm9) => {
&AMode::SPPreIndexed { simm9 } => {
let reg = stack_reg();
sink.put4(enc_ldst_simm9(op, simm9, 0b11, reg, rd));
}
&AMode::SPPostIndexed(simm9) => {
&AMode::SPPostIndexed { simm9 } => {
let reg = stack_reg();
sink.put4(enc_ldst_simm9(op, simm9, 0b01, reg, rd));
}
// Eliminated by `mem_finalize()` above.
&AMode::SPOffset(..) | &AMode::FPOffset(..) | &AMode::NominalSPOffset(..) => {
panic!("Should not see stack-offset here!")
&AMode::SPOffset { .. }
| &AMode::FPOffset { .. }
| &AMode::NominalSPOffset { .. }
| &AMode::RegOffset { .. } => {
panic!("Should not see {:?} here!", mem)
}
&AMode::RegOffset(..) => panic!("SHould not see generic reg-offset here!"),
}
}
@@ -1097,39 +1108,39 @@ impl MachInstEmit for Inst {
}
match &mem {
&AMode::Unscaled(reg, simm9) => {
let reg = allocs.next(reg);
&AMode::Unscaled { rn, simm9 } => {
let reg = allocs.next(rn);
sink.put4(enc_ldst_simm9(op, simm9, 0b00, reg, rd));
}
&AMode::UnsignedOffset(reg, uimm12scaled) => {
let reg = allocs.next(reg);
if uimm12scaled.value() != 0 {
assert_eq!(bits, ty_bits(uimm12scaled.scale_ty()));
&AMode::UnsignedOffset { rn, uimm12 } => {
let reg = allocs.next(rn);
if uimm12.value() != 0 {
assert_eq!(bits, ty_bits(uimm12.scale_ty()));
}
sink.put4(enc_ldst_uimm12(op, uimm12scaled, reg, rd));
sink.put4(enc_ldst_uimm12(op, uimm12, reg, rd));
}
&AMode::RegReg(r1, r2) => {
let r1 = allocs.next(r1);
let r2 = allocs.next(r2);
&AMode::RegReg { rn, rm } => {
let r1 = allocs.next(rn);
let r2 = allocs.next(rm);
sink.put4(enc_ldst_reg(
op, r1, r2, /* scaled = */ false, /* extendop = */ None, rd,
));
}
&AMode::RegScaled(r1, r2, _ty) | &AMode::RegScaledExtended(r1, r2, _ty, _) => {
let r1 = allocs.next(r1);
let r2 = allocs.next(r2);
&AMode::RegScaled { rn, rm, .. } | &AMode::RegScaledExtended { rn, rm, .. } => {
let r1 = allocs.next(rn);
let r2 = allocs.next(rm);
let extendop = match &mem {
&AMode::RegScaled(..) => None,
&AMode::RegScaledExtended(_, _, _, op) => Some(op),
&AMode::RegScaled { .. } => None,
&AMode::RegScaledExtended { extendop, .. } => Some(extendop),
_ => unreachable!(),
};
sink.put4(enc_ldst_reg(
op, r1, r2, /* scaled = */ true, extendop, rd,
));
}
&AMode::RegExtended(r1, r2, extendop) => {
let r1 = allocs.next(r1);
let r2 = allocs.next(r2);
&AMode::RegExtended { rn, rm, extendop } => {
let r1 = allocs.next(rn);
let r2 = allocs.next(rm);
sink.put4(enc_ldst_reg(
op,
r1,
@@ -1139,22 +1150,24 @@ impl MachInstEmit for Inst {
rd,
));
}
&AMode::Label(..) => {
&AMode::Label { .. } => {
panic!("Store to a MemLabel not implemented!");
}
&AMode::SPPreIndexed(simm9) => {
&AMode::SPPreIndexed { simm9 } => {
let reg = stack_reg();
sink.put4(enc_ldst_simm9(op, simm9, 0b11, reg, rd));
}
&AMode::SPPostIndexed(simm9) => {
&AMode::SPPostIndexed { simm9 } => {
let reg = stack_reg();
sink.put4(enc_ldst_simm9(op, simm9, 0b01, reg, rd));
}
// Eliminated by `mem_finalize()` above.
&AMode::SPOffset(..) | &AMode::FPOffset(..) | &AMode::NominalSPOffset(..) => {
panic!("Should not see stack-offset here!")
&AMode::SPOffset { .. }
| &AMode::FPOffset { .. }
| &AMode::NominalSPOffset { .. }
| &AMode::RegOffset { .. } => {
panic!("Should not see {:?} here!", mem)
}
&AMode::RegOffset(..) => panic!("SHould not see generic reg-offset here!"),
}
}
@@ -2216,7 +2229,9 @@ impl MachInstEmit for Inst {
let rd = allocs.next_writable(rd);
let inst = Inst::FpuLoad64 {
rd,
mem: AMode::Label(MemLabel::PCRel(8)),
mem: AMode::Label {
label: MemLabel::PCRel(8),
},
flags: MemFlags::trusted(),
};
inst.emit(&[], sink, emit_info, state);
@@ -2230,7 +2245,9 @@ impl MachInstEmit for Inst {
let rd = allocs.next_writable(rd);
let inst = Inst::FpuLoad128 {
rd,
mem: AMode::Label(MemLabel::PCRel(8)),
mem: AMode::Label {
label: MemLabel::PCRel(8),
},
flags: MemFlags::trusted(),
};
inst.emit(&[], sink, emit_info, state);
@@ -3159,7 +3176,9 @@ impl MachInstEmit for Inst {
let rd = allocs.next_writable(rd);
let inst = Inst::ULoad64 {
rd,
mem: AMode::Label(MemLabel::PCRel(8)),
mem: AMode::Label {
label: MemLabel::PCRel(8),
},
flags: MemFlags::trusted(),
};
inst.emit(&[], sink, emit_info, state);
@@ -3179,17 +3198,17 @@ impl MachInstEmit for Inst {
}
let (reg, index_reg, offset) = match mem {
AMode::RegExtended(r, idx, extendop) => {
let r = allocs.next(r);
(r, Some((idx, extendop)), 0)
AMode::RegExtended { rn, rm, extendop } => {
let r = allocs.next(rn);
(r, Some((rm, extendop)), 0)
}
AMode::Unscaled(r, simm9) => {
let r = allocs.next(r);
AMode::Unscaled { rn, simm9 } => {
let r = allocs.next(rn);
(r, None, simm9.value())
}
AMode::UnsignedOffset(r, uimm12scaled) => {
let r = allocs.next(r);
(r, None, uimm12scaled.value() as i32)
AMode::UnsignedOffset { rn, uimm12 } => {
let r = allocs.next(rn);
(r, None, uimm12.value() as i32)
}
_ => panic!("Unsupported case for LoadAddr: {:?}", mem),
};

305
cranelift/codegen/src/isa/aarch64/inst/emit_tests.rs
View File

@@ -1371,7 +1371,10 @@ fn test_aarch64_binemit() {
insns.push((
Inst::ULoad8 {
rd: writable_xreg(1),
mem: AMode::Unscaled(xreg(2), simm9_zero()),
mem: AMode::Unscaled {
rn: xreg(2),
simm9: simm9_zero(),
},
flags: MemFlags::trusted(),
},
"41004038",
@@ -1380,7 +1383,10 @@ fn test_aarch64_binemit() {
insns.push((
Inst::ULoad8 {
rd: writable_xreg(1),
mem: AMode::UnsignedOffset(xreg(2), UImm12Scaled::zero(I8)),
mem: AMode::UnsignedOffset {
rn: xreg(2),
uimm12: UImm12Scaled::zero(I8),
},
flags: MemFlags::trusted(),
},
"41004039",
@@ -1389,7 +1395,10 @@ fn test_aarch64_binemit() {
insns.push((
Inst::ULoad8 {
rd: writable_xreg(1),
mem: AMode::RegReg(xreg(2), xreg(5)),
mem: AMode::RegReg {
rn: xreg(2),
rm: xreg(5),
},
flags: MemFlags::trusted(),
},
"41686538",
@@ -1398,7 +1407,10 @@ fn test_aarch64_binemit() {
insns.push((
Inst::SLoad8 {
rd: writable_xreg(1),
mem: AMode::Unscaled(xreg(2), simm9_zero()),
mem: AMode::Unscaled {
rn: xreg(2),
simm9: simm9_zero(),
},
flags: MemFlags::trusted(),
},
"41008038",
@@ -1407,7 +1419,10 @@ fn test_aarch64_binemit() {
insns.push((
Inst::SLoad8 {
rd: writable_xreg(1),
mem: AMode::UnsignedOffset(xreg(2), UImm12Scaled::maybe_from_i64(63, I8).unwrap()),
mem: AMode::UnsignedOffset {
rn: xreg(2),
uimm12: UImm12Scaled::maybe_from_i64(63, I8).unwrap(),
},
flags: MemFlags::trusted(),
},
"41FC8039",
@@ -1416,7 +1431,10 @@ fn test_aarch64_binemit() {
insns.push((
Inst::SLoad8 {
rd: writable_xreg(1),
mem: AMode::RegReg(xreg(2), xreg(5)),
mem: AMode::RegReg {
rn: xreg(2),
rm: xreg(5),
},
flags: MemFlags::trusted(),
},
"4168A538",
@@ -1425,7 +1443,10 @@ fn test_aarch64_binemit() {
insns.push((
Inst::ULoad16 {
rd: writable_xreg(1),
mem: AMode::Unscaled(xreg(2), SImm9::maybe_from_i64(5).unwrap()),
mem: AMode::Unscaled {
rn: xreg(2),
simm9: SImm9::maybe_from_i64(5).unwrap(),
},
flags: MemFlags::trusted(),
},
"41504078",
@@ -1434,7 +1455,10 @@ fn test_aarch64_binemit() {
insns.push((
Inst::ULoad16 {
rd: writable_xreg(1),
mem: AMode::UnsignedOffset(xreg(2), UImm12Scaled::maybe_from_i64(8, I16).unwrap()),
mem: AMode::UnsignedOffset {
rn: xreg(2),
uimm12: UImm12Scaled::maybe_from_i64(8, I16).unwrap(),
},
flags: MemFlags::trusted(),
},
"41104079",
@@ -1443,7 +1467,11 @@ fn test_aarch64_binemit() {
insns.push((
Inst::ULoad16 {
rd: writable_xreg(1),
mem: AMode::RegScaled(xreg(2), xreg(3), I16),
mem: AMode::RegScaled {
rn: xreg(2),
rm: xreg(3),
ty: I16,
},
flags: MemFlags::trusted(),
},
"41786378",
@@ -1452,7 +1480,10 @@ fn test_aarch64_binemit() {
insns.push((
Inst::SLoad16 {
rd: writable_xreg(1),
mem: AMode::Unscaled(xreg(2), simm9_zero()),
mem: AMode::Unscaled {
rn: xreg(2),
simm9: simm9_zero(),
},
flags: MemFlags::trusted(),
},
"41008078",
@@ -1461,7 +1492,10 @@ fn test_aarch64_binemit() {
insns.push((
Inst::SLoad16 {
rd: writable_xreg(28),
mem: AMode::UnsignedOffset(xreg(20), UImm12Scaled::maybe_from_i64(24, I16).unwrap()),
mem: AMode::UnsignedOffset {
rn: xreg(20),
uimm12: UImm12Scaled::maybe_from_i64(24, I16).unwrap(),
},
flags: MemFlags::trusted(),
},
"9C328079",
@@ -1470,7 +1504,11 @@ fn test_aarch64_binemit() {
insns.push((
Inst::SLoad16 {
rd: writable_xreg(28),
mem: AMode::RegScaled(xreg(20), xreg(20), I16),
mem: AMode::RegScaled {
rn: xreg(20),
rm: xreg(20),
ty: I16,
},
flags: MemFlags::trusted(),
},
"9C7AB478",
@@ -1479,7 +1517,10 @@ fn test_aarch64_binemit() {
insns.push((
Inst::ULoad32 {
rd: writable_xreg(1),
mem: AMode::Unscaled(xreg(2), simm9_zero()),
mem: AMode::Unscaled {
rn: xreg(2),
simm9: simm9_zero(),
},
flags: MemFlags::trusted(),
},
"410040B8",
@@ -1488,7 +1529,10 @@ fn test_aarch64_binemit() {
insns.push((
Inst::ULoad32 {
rd: writable_xreg(12),
mem: AMode::UnsignedOffset(xreg(0), UImm12Scaled::maybe_from_i64(204, I32).unwrap()),
mem: AMode::UnsignedOffset {
rn: xreg(0),
uimm12: UImm12Scaled::maybe_from_i64(204, I32).unwrap(),
},
flags: MemFlags::trusted(),
},
"0CCC40B9",
@@ -1497,7 +1541,11 @@ fn test_aarch64_binemit() {
insns.push((
Inst::ULoad32 {
rd: writable_xreg(1),
mem: AMode::RegScaled(xreg(2), xreg(12), I32),
mem: AMode::RegScaled {
rn: xreg(2),
rm: xreg(12),
ty: I32,
},
flags: MemFlags::trusted(),
},
"41786CB8",
@@ -1506,7 +1554,10 @@ fn test_aarch64_binemit() {
insns.push((
Inst::SLoad32 {
rd: writable_xreg(1),
mem: AMode::Unscaled(xreg(2), simm9_zero()),
mem: AMode::Unscaled {
rn: xreg(2),
simm9: simm9_zero(),
},
flags: MemFlags::trusted(),
},
"410080B8",
@@ -1515,7 +1566,10 @@ fn test_aarch64_binemit() {
insns.push((
Inst::SLoad32 {
rd: writable_xreg(12),
mem: AMode::UnsignedOffset(xreg(1), UImm12Scaled::maybe_from_i64(16380, I32).unwrap()),
mem: AMode::UnsignedOffset {
rn: xreg(1),
uimm12: UImm12Scaled::maybe_from_i64(16380, I32).unwrap(),
},
flags: MemFlags::trusted(),
},
"2CFCBFB9",
@@ -1524,7 +1578,11 @@ fn test_aarch64_binemit() {
insns.push((
Inst::SLoad32 {
rd: writable_xreg(1),
mem: AMode::RegScaled(xreg(5), xreg(1), I32),
mem: AMode::RegScaled {
rn: xreg(5),
rm: xreg(1),
ty: I32,
},
flags: MemFlags::trusted(),
},
"A178A1B8",
@@ -1533,7 +1591,10 @@ fn test_aarch64_binemit() {
insns.push((
Inst::ULoad64 {
rd: writable_xreg(1),
mem: AMode::Unscaled(xreg(2), simm9_zero()),
mem: AMode::Unscaled {
rn: xreg(2),
simm9: simm9_zero(),
},
flags: MemFlags::trusted(),
},
"410040F8",
@@ -1542,7 +1603,10 @@ fn test_aarch64_binemit() {
insns.push((
Inst::ULoad64 {
rd: writable_xreg(1),
mem: AMode::Unscaled(xreg(2), SImm9::maybe_from_i64(-256).unwrap()),
mem: AMode::Unscaled {
rn: xreg(2),
simm9: SImm9::maybe_from_i64(-256).unwrap(),
},
flags: MemFlags::trusted(),
},
"410050F8",
@@ -1551,7 +1615,10 @@ fn test_aarch64_binemit() {
insns.push((
Inst::ULoad64 {
rd: writable_xreg(1),
mem: AMode::Unscaled(xreg(2), SImm9::maybe_from_i64(255).unwrap()),
mem: AMode::Unscaled {
rn: xreg(2),
simm9: SImm9::maybe_from_i64(255).unwrap(),
},
flags: MemFlags::trusted(),
},
"41F04FF8",
@@ -1560,7 +1627,10 @@ fn test_aarch64_binemit() {
insns.push((
Inst::ULoad64 {
rd: writable_xreg(1),
mem: AMode::UnsignedOffset(xreg(2), UImm12Scaled::maybe_from_i64(32760, I64).unwrap()),
mem: AMode::UnsignedOffset {
rn: xreg(2),
uimm12: UImm12Scaled::maybe_from_i64(32760, I64).unwrap(),
},
flags: MemFlags::trusted(),
},
"41FC7FF9",
@@ -1569,7 +1639,10 @@ fn test_aarch64_binemit() {
insns.push((
Inst::ULoad64 {
rd: writable_xreg(1),
mem: AMode::RegReg(xreg(2), xreg(3)),
mem: AMode::RegReg {
rn: xreg(2),
rm: xreg(3),
},
flags: MemFlags::trusted(),
},
"416863F8",
@@ -1578,7 +1651,11 @@ fn test_aarch64_binemit() {
insns.push((
Inst::ULoad64 {
rd: writable_xreg(1),
mem: AMode::RegScaled(xreg(2), xreg(3), I64),
mem: AMode::RegScaled {
rn: xreg(2),
rm: xreg(3),
ty: I64,
},
flags: MemFlags::trusted(),
},
"417863F8",
@@ -1587,7 +1664,12 @@ fn test_aarch64_binemit() {
insns.push((
Inst::ULoad64 {
rd: writable_xreg(1),
mem: AMode::RegScaledExtended(xreg(2), xreg(3), I64, ExtendOp::SXTW),
mem: AMode::RegScaledExtended {
rn: xreg(2),
rm: xreg(3),
ty: I64,
extendop: ExtendOp::SXTW,
},
flags: MemFlags::trusted(),
},
"41D863F8",
@@ -1596,7 +1678,11 @@ fn test_aarch64_binemit() {
insns.push((
Inst::ULoad64 {
rd: writable_xreg(1),
mem: AMode::RegExtended(xreg(2), xreg(3), ExtendOp::SXTW),
mem: AMode::RegExtended {
rn: xreg(2),
rm: xreg(3),
extendop: ExtendOp::SXTW,
},
flags: MemFlags::trusted(),
},
"41C863F8",
@@ -1605,7 +1691,9 @@ fn test_aarch64_binemit() {
insns.push((
Inst::ULoad64 {
rd: writable_xreg(1),
mem: AMode::Label(MemLabel::PCRel(64)),
mem: AMode::Label {
label: MemLabel::PCRel(64),
},
flags: MemFlags::trusted(),
},
"01020058",
@@ -1614,7 +1702,9 @@ fn test_aarch64_binemit() {
insns.push((
Inst::ULoad64 {
rd: writable_xreg(1),
mem: AMode::SPPreIndexed(SImm9::maybe_from_i64(16).unwrap()),
mem: AMode::SPPreIndexed {
simm9: SImm9::maybe_from_i64(16).unwrap(),
},
flags: MemFlags::trusted(),
},
"E10F41F8",
@@ -1623,7 +1713,9 @@ fn test_aarch64_binemit() {
insns.push((
Inst::ULoad64 {
rd: writable_xreg(1),
mem: AMode::SPPostIndexed(SImm9::maybe_from_i64(16).unwrap()),
mem: AMode::SPPostIndexed {
simm9: SImm9::maybe_from_i64(16).unwrap(),
},
flags: MemFlags::trusted(),
},
"E10741F8",
@@ -1632,7 +1724,7 @@ fn test_aarch64_binemit() {
insns.push((
Inst::ULoad64 {
rd: writable_xreg(1),
mem: AMode::FPOffset(32768, I8),
mem: AMode::FPOffset { off: 32768, ty: I8 },
flags: MemFlags::trusted(),
},
"100090D2B063308B010240F9",
@@ -1641,7 +1733,10 @@ fn test_aarch64_binemit() {
insns.push((
Inst::ULoad64 {
rd: writable_xreg(1),
mem: AMode::FPOffset(-32768, I8),
mem: AMode::FPOffset {
off: -32768,
ty: I8,
},
flags: MemFlags::trusted(),
},
"F0FF8F92B063308B010240F9",
@@ -1650,7 +1745,10 @@ fn test_aarch64_binemit() {
insns.push((
Inst::ULoad64 {
rd: writable_xreg(1),
mem: AMode::FPOffset(1048576, I8), // 2^20
mem: AMode::FPOffset {
off: 1048576,
ty: I8,
}, // 2^20
flags: MemFlags::trusted(),
},
"1002A0D2B063308B010240F9",
@@ -1659,7 +1757,10 @@ fn test_aarch64_binemit() {
insns.push((
Inst::ULoad64 {
rd: writable_xreg(1),
mem: AMode::FPOffset(1048576 + 1, I8), // 2^20 + 1
mem: AMode::FPOffset {
off: 1048576 + 1,
ty: I8,
}, // 2^20 + 1
flags: MemFlags::trusted(),
},
"300080521002A072B063308B010240F9",
@@ -1669,7 +1770,11 @@ fn test_aarch64_binemit() {
insns.push((
Inst::ULoad64 {
rd: writable_xreg(1),
mem: AMode::RegOffset(xreg(7), 8, I64),
mem: AMode::RegOffset {
rn: xreg(7),
off: 8,
ty: I64,
},
flags: MemFlags::trusted(),
},
"E18040F8",
@@ -1679,7 +1784,11 @@ fn test_aarch64_binemit() {
insns.push((
Inst::ULoad64 {
rd: writable_xreg(1),
mem: AMode::RegOffset(xreg(7), 1024, I64),
mem: AMode::RegOffset {
rn: xreg(7),
off: 1024,
ty: I64,
},
flags: MemFlags::trusted(),
},
"E10042F9",
@@ -1689,7 +1798,11 @@ fn test_aarch64_binemit() {
insns.push((
Inst::ULoad64 {
rd: writable_xreg(1),
mem: AMode::RegOffset(xreg(7), 1048576, I64),
mem: AMode::RegOffset {
rn: xreg(7),
off: 1048576,
ty: I64,
},
flags: MemFlags::trusted(),
},
"1002A0D2F060308B010240F9",
@@ -1699,7 +1812,10 @@ fn test_aarch64_binemit() {
insns.push((
Inst::Store8 {
rd: xreg(1),
mem: AMode::Unscaled(xreg(2), simm9_zero()),
mem: AMode::Unscaled {
rn: xreg(2),
simm9: simm9_zero(),
},
flags: MemFlags::trusted(),
},
"41000038",
@@ -1708,7 +1824,10 @@ fn test_aarch64_binemit() {
insns.push((
Inst::Store8 {
rd: xreg(1),
mem: AMode::UnsignedOffset(xreg(2), UImm12Scaled::maybe_from_i64(4095, I8).unwrap()),
mem: AMode::UnsignedOffset {
rn: xreg(2),
uimm12: UImm12Scaled::maybe_from_i64(4095, I8).unwrap(),
},
flags: MemFlags::trusted(),
},
"41FC3F39",
@@ -1717,7 +1836,10 @@ fn test_aarch64_binemit() {
insns.push((
Inst::Store16 {
rd: xreg(1),
mem: AMode::Unscaled(xreg(2), simm9_zero()),
mem: AMode::Unscaled {
rn: xreg(2),
simm9: simm9_zero(),
},
flags: MemFlags::trusted(),
},
"41000078",
@@ -1726,7 +1848,10 @@ fn test_aarch64_binemit() {
insns.push((
Inst::Store16 {
rd: xreg(1),
mem: AMode::UnsignedOffset(xreg(2), UImm12Scaled::maybe_from_i64(8190, I16).unwrap()),
mem: AMode::UnsignedOffset {
rn: xreg(2),
uimm12: UImm12Scaled::maybe_from_i64(8190, I16).unwrap(),
},
flags: MemFlags::trusted(),
},
"41FC3F79",
@@ -1735,7 +1860,10 @@ fn test_aarch64_binemit() {
insns.push((
Inst::Store32 {
rd: xreg(1),
mem: AMode::Unscaled(xreg(2), simm9_zero()),
mem: AMode::Unscaled {
rn: xreg(2),
simm9: simm9_zero(),
},
flags: MemFlags::trusted(),
},
"410000B8",
@@ -1744,7 +1872,10 @@ fn test_aarch64_binemit() {
insns.push((
Inst::Store32 {
rd: xreg(1),
mem: AMode::UnsignedOffset(xreg(2), UImm12Scaled::maybe_from_i64(16380, I32).unwrap()),
mem: AMode::UnsignedOffset {
rn: xreg(2),
uimm12: UImm12Scaled::maybe_from_i64(16380, I32).unwrap(),
},
flags: MemFlags::trusted(),
},
"41FC3FB9",
@@ -1753,7 +1884,10 @@ fn test_aarch64_binemit() {
insns.push((
Inst::Store64 {
rd: xreg(1),
mem: AMode::Unscaled(xreg(2), simm9_zero()),
mem: AMode::Unscaled {
rn: xreg(2),
simm9: simm9_zero(),
},
flags: MemFlags::trusted(),
},
"410000F8",
@@ -1762,7 +1896,10 @@ fn test_aarch64_binemit() {
insns.push((
Inst::Store64 {
rd: xreg(1),
mem: AMode::UnsignedOffset(xreg(2), UImm12Scaled::maybe_from_i64(32760, I64).unwrap()),
mem: AMode::UnsignedOffset {
rn: xreg(2),
uimm12: UImm12Scaled::maybe_from_i64(32760, I64).unwrap(),
},
flags: MemFlags::trusted(),
},
"41FC3FF9",
@@ -1771,7 +1908,10 @@ fn test_aarch64_binemit() {
insns.push((
Inst::Store64 {
rd: xreg(1),
mem: AMode::RegReg(xreg(2), xreg(3)),
mem: AMode::RegReg {
rn: xreg(2),
rm: xreg(3),
},
flags: MemFlags::trusted(),
},
"416823F8",
@@ -1780,7 +1920,11 @@ fn test_aarch64_binemit() {
insns.push((
Inst::Store64 {
rd: xreg(1),
mem: AMode::RegScaled(xreg(2), xreg(3), I64),
mem: AMode::RegScaled {
rn: xreg(2),
rm: xreg(3),
ty: I64,
},
flags: MemFlags::trusted(),
},
"417823F8",
@@ -1789,7 +1933,12 @@ fn test_aarch64_binemit() {
insns.push((
Inst::Store64 {
rd: xreg(1),
mem: AMode::RegScaledExtended(xreg(2), xreg(3), I64, ExtendOp::UXTW),
mem: AMode::RegScaledExtended {
rn: xreg(2),
rm: xreg(3),
ty: I64,
extendop: ExtendOp::UXTW,
},
flags: MemFlags::trusted(),
},
"415823F8",
@@ -1798,7 +1947,11 @@ fn test_aarch64_binemit() {
insns.push((
Inst::Store64 {
rd: xreg(1),
mem: AMode::RegExtended(xreg(2), xreg(3), ExtendOp::UXTW),
mem: AMode::RegExtended {
rn: xreg(2),
rm: xreg(3),
extendop: ExtendOp::UXTW,
},
flags: MemFlags::trusted(),
},
"414823F8",
@@ -1807,7 +1960,9 @@ fn test_aarch64_binemit() {
insns.push((
Inst::Store64 {
rd: xreg(1),
mem: AMode::SPPreIndexed(SImm9::maybe_from_i64(16).unwrap()),
mem: AMode::SPPreIndexed {
simm9: SImm9::maybe_from_i64(16).unwrap(),
},
flags: MemFlags::trusted(),
},
"E10F01F8",
@@ -1816,7 +1971,9 @@ fn test_aarch64_binemit() {
insns.push((
Inst::Store64 {
rd: xreg(1),
mem: AMode::SPPostIndexed(SImm9::maybe_from_i64(16).unwrap()),
mem: AMode::SPPostIndexed {
simm9: SImm9::maybe_from_i64(16).unwrap(),
},
flags: MemFlags::trusted(),
},
"E10701F8",
@@ -6397,7 +6554,11 @@ fn test_aarch64_binemit() {
insns.push((
Inst::FpuLoad32 {
rd: writable_vreg(16),
mem: AMode::RegScaled(xreg(8), xreg(9), F32),
mem: AMode::RegScaled {
rn: xreg(8),
rm: xreg(9),
ty: F32,
},
flags: MemFlags::trusted(),
},
"107969BC",
@@ -6407,7 +6568,11 @@ fn test_aarch64_binemit() {
insns.push((
Inst::FpuLoad64 {
rd: writable_vreg(16),
mem: AMode::RegScaled(xreg(8), xreg(9), F64),
mem: AMode::RegScaled {
rn: xreg(8),
rm: xreg(9),
ty: F64,
},
flags: MemFlags::trusted(),
},
"107969FC",
@@ -6417,7 +6582,11 @@ fn test_aarch64_binemit() {
insns.push((
Inst::FpuLoad128 {
rd: writable_vreg(16),
mem: AMode::RegScaled(xreg(8), xreg(9), I128),
mem: AMode::RegScaled {
rn: xreg(8),
rm: xreg(9),
ty: I128,
},
flags: MemFlags::trusted(),
},
"1079E93C",
@@ -6427,7 +6596,9 @@ fn test_aarch64_binemit() {
insns.push((
Inst::FpuLoad32 {
rd: writable_vreg(16),
mem: AMode::Label(MemLabel::PCRel(8)),
mem: AMode::Label {
label: MemLabel::PCRel(8),
},
flags: MemFlags::trusted(),
},
"5000001C",
@@ -6437,7 +6608,9 @@ fn test_aarch64_binemit() {
insns.push((
Inst::FpuLoad64 {
rd: writable_vreg(16),
mem: AMode::Label(MemLabel::PCRel(8)),
mem: AMode::Label {
label: MemLabel::PCRel(8),
},
flags: MemFlags::trusted(),
},
"5000005C",
@@ -6447,7 +6620,9 @@ fn test_aarch64_binemit() {
insns.push((
Inst::FpuLoad128 {
rd: writable_vreg(16),
mem: AMode::Label(MemLabel::PCRel(8)),
mem: AMode::Label {
label: MemLabel::PCRel(8),
},
flags: MemFlags::trusted(),
},
"5000009C",
@@ -6457,7 +6632,11 @@ fn test_aarch64_binemit() {
insns.push((
Inst::FpuStore32 {
rd: vreg(16),
mem: AMode::RegScaled(xreg(8), xreg(9), F32),
mem: AMode::RegScaled {
rn: xreg(8),
rm: xreg(9),
ty: F32,
},
flags: MemFlags::trusted(),
},
"107929BC",
@@ -6467,7 +6646,11 @@ fn test_aarch64_binemit() {
insns.push((
Inst::FpuStore64 {
rd: vreg(16),
mem: AMode::RegScaled(xreg(8), xreg(9), F64),
mem: AMode::RegScaled {
rn: xreg(8),
rm: xreg(9),
ty: F64,
},
flags: MemFlags::trusted(),
},
"107929FC",
@@ -6477,7 +6660,11 @@ fn test_aarch64_binemit() {
insns.push((
Inst::FpuStore128 {
rd: vreg(16),
mem: AMode::RegScaled(xreg(8), xreg(9), I128),
mem: AMode::RegScaled {
rn: xreg(8),
rm: xreg(9),
ty: I128,
},
flags: MemFlags::trusted(),
},
"1079A93C",

42
cranelift/codegen/src/isa/aarch64/inst/mod.rs
View File

@@ -36,7 +36,7 @@ mod emit_tests;
// Instructions (top level): definition
pub use crate::isa::aarch64::lower::isle::generated_code::{
ALUOp, ALUOp3, APIKey, AtomicRMWLoopOp, AtomicRMWOp, BitOp, FPUOp1, FPUOp2, FPUOp3,
ALUOp, ALUOp3, AMode, APIKey, AtomicRMWLoopOp, AtomicRMWOp, BitOp, FPUOp1, FPUOp2, FPUOp3,
FpuRoundMode, FpuToIntOp, IntToFpuOp, MInst as Inst, MoveWideOp, VecALUModOp, VecALUOp,
VecExtendOp, VecLanesOp, VecMisc2, VecPairOp, VecRRLongOp, VecRRNarrowOp, VecRRPairLongOp,
VecRRRLongModOp, VecRRRLongOp, VecShiftImmModOp, VecShiftImmOp,
@@ -546,22 +546,22 @@ impl Inst {
fn memarg_operands<F: Fn(VReg) -> VReg>(memarg: &AMode, collector: &mut OperandCollector<'_, F>) {
// This should match `AMode::with_allocs()`.
match memarg {
&AMode::Unscaled(reg, ..) | &AMode::UnsignedOffset(reg, ..) => {
collector.reg_use(reg);
&AMode::Unscaled { rn, .. } | &AMode::UnsignedOffset { rn, .. } => {
collector.reg_use(rn);
}
&AMode::RegReg(r1, r2, ..)
| &AMode::RegScaled(r1, r2, ..)
| &AMode::RegScaledExtended(r1, r2, ..)
| &AMode::RegExtended(r1, r2, ..) => {
collector.reg_use(r1);
collector.reg_use(r2);
&AMode::RegReg { rn, rm, .. }
| &AMode::RegScaled { rn, rm, .. }
| &AMode::RegScaledExtended { rn, rm, .. }
| &AMode::RegExtended { rn, rm, .. } => {
collector.reg_use(rn);
collector.reg_use(rm);
}
&AMode::Label(..) => {}
&AMode::SPPreIndexed(..) | &AMode::SPPostIndexed(..) => {}
&AMode::FPOffset(..) => {}
&AMode::SPOffset(..) | &AMode::NominalSPOffset(..) => {}
&AMode::RegOffset(r, ..) => {
collector.reg_use(r);
&AMode::Label { .. } => {}
&AMode::SPPreIndexed { .. } | &AMode::SPPostIndexed { .. } => {}
&AMode::FPOffset { .. } => {}
&AMode::SPOffset { .. } | &AMode::NominalSPOffset { .. } => {}
&AMode::RegOffset { rn, .. } => {
collector.reg_use(rn);
}
}
}
@@ -1446,7 +1446,7 @@ impl Inst {
| &Inst::SLoad32 { rd, ref mem, .. }
| &Inst::ULoad64 { rd, ref mem, .. } => {
let is_unscaled = match &mem {
&AMode::Unscaled(..) => true,
&AMode::Unscaled { .. } => true,
_ => false,
};
let (op, size) = match (self, is_unscaled) {
@@ -1479,7 +1479,7 @@ impl Inst {
| &Inst::Store32 { rd, ref mem, .. }
| &Inst::Store64 { rd, ref mem, .. } => {
let is_unscaled = match &mem {
&AMode::Unscaled(..) => true,
&AMode::Unscaled { .. } => true,
_ => false,
};
let (op, size) = match (self, is_unscaled) {
@@ -2736,11 +2736,9 @@ impl Inst {
);
}
let (reg, index_reg, offset) = match mem {
AMode::RegExtended(r, idx, extendop) => (r, Some((idx, extendop)), 0),
AMode::Unscaled(r, simm9) => (r, None, simm9.value()),
AMode::UnsignedOffset(r, uimm12scaled) => {
(r, None, uimm12scaled.value() as i32)
}
AMode::RegExtended { rn, rm, extendop } => (rn, Some((rm, extendop)), 0),
AMode::Unscaled { rn, simm9 } => (rn, None, simm9.value()),
AMode::UnsignedOffset { rn, uimm12 } => (rn, None, uimm12.value() as i32),
_ => panic!("Unsupported case for LoadAddr: {:?}", mem),
};
let abs_offset = if offset < 0 {

9
cranelift/codegen/src/isa/aarch64/lower.isle
View File

@@ -1772,12 +1772,11 @@
(rule (lower (has_type ty (splat (ireduce (iconst (u64_from_imm64 n))))))
(splat_const n (vector_size ty)))
(rule (lower (has_type ty (splat x @ (load flags addr offset))))
(rule (lower (has_type ty (splat x @ (load flags _ _))))
(if-let mem_op (is_sinkable_inst x))
(let ((_ Unit (sink_inst mem_op))
(addr AMode (amode (lane_type ty) addr offset))
(address Reg (load_addr addr)))
(ld1r address (vector_size ty) flags)))
(let ((addr AMode (sink_load_into_amode (lane_type ty) mem_op))
(address Reg (load_addr addr)))
(ld1r address (vector_size ty) flags)))
;;;; Rules for `AtomicLoad` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(rule (lower (has_type (valid_atomic_transaction ty) (atomic_load flags addr)))

56
cranelift/codegen/src/isa/aarch64/lower.rs
View File

@@ -658,16 +658,24 @@ pub(crate) fn lower_address(
if addends32.len() > 0 {
let (reg32, extendop) = addends32.pop().unwrap();
let reg64 = addends64.pop().unwrap();
AMode::RegExtended(reg64, reg32, extendop)
AMode::RegExtended {
rn: reg64,
rm: reg32,
extendop,
}
} else if offset > 0 && offset < 0x1000 {
let reg64 = addends64.pop().unwrap();
let off = offset;
offset = 0;
AMode::RegOffset(reg64, off, elem_ty)
AMode::RegOffset {
rn: reg64,
off,
ty: elem_ty,
}
} else if addends64.len() >= 2 {
let reg1 = addends64.pop().unwrap();
let reg2 = addends64.pop().unwrap();
AMode::RegReg(reg1, reg2)
AMode::RegReg { rn: reg1, rm: reg2 }
} else {
let reg1 = addends64.pop().unwrap();
AMode::reg(reg1)
@@ -691,7 +699,11 @@ pub(crate) fn lower_address(
to_bits: 64,
});
if let Some((reg2, extendop)) = addends32.pop() {
AMode::RegExtended(tmp.to_reg(), reg2, extendop)
AMode::RegExtended {
rn: tmp.to_reg(),
rm: reg2,
extendop,
}
} else {
AMode::reg(tmp.to_reg())
}
@@ -716,12 +728,36 @@ pub(crate) fn lower_address(
// Allocate the temp and shoehorn it into the AMode.
let addr = ctx.alloc_tmp(I64).only_reg().unwrap();
let (reg, memarg) = match memarg {
AMode::RegExtended(r1, r2, extendop) => {
(r1, AMode::RegExtended(addr.to_reg(), r2, extendop))
}
AMode::RegOffset(r, off, ty) => (r, AMode::RegOffset(addr.to_reg(), off, ty)),
AMode::RegReg(r1, r2) => (r2, AMode::RegReg(addr.to_reg(), r1)),
AMode::UnsignedOffset(r, imm) => (r, AMode::UnsignedOffset(addr.to_reg(), imm)),
AMode::RegExtended { rn, rm, extendop } => (
rn,
AMode::RegExtended {
rn: addr.to_reg(),
rm,
extendop,
},
),
AMode::RegOffset { rn, off, ty } => (
rn,
AMode::RegOffset {
rn: addr.to_reg(),
off,
ty,
},
),
AMode::RegReg { rn, rm } => (
rm,
AMode::RegReg {
rn: addr.to_reg(),
rm: rn,
},
),
AMode::UnsignedOffset { rn, uimm12 } => (
rn,
AMode::UnsignedOffset {
rn: addr.to_reg(),
uimm12,
},
),
_ => unreachable!(),
};

40
cranelift/codegen/src/isa/aarch64/lower/isle.rs
View File

@@ -6,12 +6,12 @@ use generated_code::Context;
// Types that the generated ISLE code uses via `use super::*`.
use super::{
lower_constant_f128, lower_constant_f32, lower_constant_f64, lower_fp_condcode,
writable_zero_reg, zero_reg, AMode, ASIMDFPModImm, ASIMDMovModImm, BranchTarget, CallIndInfo,
CallInfo, Cond, CondBrKind, ExtendOp, FPUOpRI, FPUOpRIMod, FloatCC, Imm12, ImmLogic, ImmShift,
Inst as MInst, IntCC, JTSequenceInfo, MachLabel, MoveWideConst, MoveWideOp, NarrowValueMode,
Opcode, OperandSize, PairAMode, Reg, ScalarSize, ShiftOpAndAmt, UImm5, VecMisc2, VectorSize,
NZCV,
fp_reg, lower_constant_f128, lower_constant_f32, lower_constant_f64, lower_fp_condcode,
stack_reg, writable_zero_reg, zero_reg, AMode, ASIMDFPModImm, ASIMDMovModImm, BranchTarget,
CallIndInfo, CallInfo, Cond, CondBrKind, ExtendOp, FPUOpRI, FPUOpRIMod, FloatCC, Imm12,
ImmLogic, ImmShift, Inst as MInst, IntCC, JTSequenceInfo, MachLabel, MemLabel, MoveWideConst,
MoveWideOp, NarrowValueMode, Opcode, OperandSize, PairAMode, Reg, SImm9, ScalarSize,
ShiftOpAndAmt, UImm12Scaled, UImm5, VecMisc2, VectorSize, NZCV,
};
use crate::ir::condcodes;
use crate::isa::aarch64::inst::{FPULeftShiftImm, FPURightShiftImm};
@@ -151,6 +151,22 @@ impl Context for IsleContext<'_, '_, MInst, Flags, IsaFlags, 6> {
}
}
fn is_zero_simm9(&mut self, imm: &SImm9) -> Option<()> {
if imm.value() == 0 {
Some(())
} else {
None
}
}
fn is_zero_uimm12(&mut self, imm: &UImm12Scaled) -> Option<()> {
if imm.value() == 0 {
Some(())
} else {
None
}
}
/// This is target-word-size dependent. And it excludes booleans and reftypes.
fn valid_atomic_transaction(&mut self, ty: Type) -> Option<Type> {
match ty {
@@ -293,6 +309,14 @@ impl Context for IsleContext<'_, '_, MInst, Flags, IsaFlags, 6> {
zero_reg()
}
fn stack_reg(&mut self) -> Reg {
stack_reg()
}
fn fp_reg(&mut self) -> Reg {
fp_reg()
}
fn extended_value_from_value(&mut self, val: Value) -> Option<ExtendedValue> {
let (val, extend) =
super::get_as_extended_value(self.lower_ctx, val, NarrowValueMode::None)?;
@@ -481,10 +505,6 @@ impl Context for IsleContext<'_, '_, MInst, Flags, IsaFlags, 6> {
lower_pair_address(self.lower_ctx, addr, offset as i32)
}
fn amode_is_reg(&mut self, address: &AMode) -> Option<Reg> {
address.is_reg()
}
fn constant_f64(&mut self, value: u64) -> Reg {
let rd = self.temp_writable_reg(I8X16);