T0b1/wasmtime
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

s390x: Migrate branches and traps to ISLE

Browse Source 
In order to migrate branches to ISLE, we define a second entry
point `lower_branch` which gets the list of branch targets as
additional argument.

This requires a small change to `lower_common`: the `isle_lower`
callback argument is changed from a function pointer to a closure.
This allows passing the extra argument via a closure.

Traps make use of the recently added facility to emit safepoints
from ISLE, but are otherwise straightforward.
This commit is contained in:
Ulrich Weigand
2022-01-25 18:15:32 +01:00
parent cd6b73fc90
commit 36369a6f35
7 changed files with 1485 additions and 1500 deletions
Download Patch File Download Diff File Expand all files Collapse all files

71
cranelift/codegen/src/isa/s390x/inst.isle
View File

@@ -679,7 +679,6 @@
(type BoxCallInfo (primitive BoxCallInfo))
(type BoxCallIndInfo (primitive BoxCallIndInfo))
(type MachLabel (primitive MachLabel))
(type VecMachLabel (primitive VecMachLabel))
(type BoxJTSequenceInfo (primitive BoxJTSequenceInfo))
(type BoxExternalName (primitive BoxExternalName))
(type ValueLabel (primitive ValueLabel))
@@ -1040,6 +1039,19 @@
(extern extractor unsigned unsigned)
;; Helpers for machine label vectors ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; VecMachLabel needs to be passed by reference, so it cannot be
;; declared as primitive type. Declare as extern enum instead.
(type VecMachLabel extern (enum))
(decl vec_length_minus1 (VecMachLabel) u32)
(extern constructor vec_length_minus1 vec_length_minus1)
(decl vec_element (VecMachLabel u8) MachLabel)
(extern constructor vec_element vec_element)
;; Helpers for memory arguments ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Accessors for `RelocDistance`.
@@ -1284,6 +1296,8 @@
;; implementation detail by helpers that preserve the SSA facade themselves.
(decl emit (MInst) Unit)
(extern constructor emit emit)
(decl emit_safepoint (MInst) Unit)
(extern constructor emit_safepoint emit_safepoint)
;; Helper for emitting `MInst.AluRRR` instructions.
(decl alu_rrr (Type ALUOp Reg Reg) Reg)
@@ -1695,6 +1709,26 @@
(_ Unit (emit (MInst.LoadAddr dst mem))))
(writable_reg_to_reg dst)))
;; Helper for emitting `MInst.Jump` instructions.
(decl jump_impl (MachLabel) SideEffectNoResult)
(rule (jump_impl target)
(SideEffectNoResult.Inst (MInst.Jump target)))
;; Helper for emitting `MInst.CondBr` instructions.
(decl cond_br (MachLabel MachLabel Cond) SideEffectNoResult)
(rule (cond_br taken not_taken cond)
(SideEffectNoResult.Inst (MInst.CondBr taken not_taken cond)))
;; Helper for emitting `MInst.OneWayCondBr` instructions.
(decl oneway_cond_br (MachLabel Cond) SideEffectNoResult)
(rule (oneway_cond_br dest cond)
(SideEffectNoResult.Inst (MInst.OneWayCondBr dest cond)))
;; Helper for emitting `MInst.JTSequence` instructions.
(decl jt_sequence (Reg VecMachLabel) SideEffectNoResult)
(rule (jt_sequence ridx targets)
(SideEffectNoResult.Inst (MInst.JTSequence ridx targets)))
;; Emit a `ProducesFlags` instruction when the flags are not actually needed.
(decl drop_flags (ProducesFlags) Reg)
(rule (drop_flags (ProducesFlags.ProducesFlags inst result))
@@ -2149,6 +2183,23 @@
src imm cond trap_code))))
(invalid_reg)))
(decl trap_impl (TrapCode) SideEffectNoResult)
(rule (trap_impl trap_code)
(SideEffectNoResult.Inst (MInst.Trap trap_code)))
(decl trap_if_impl (Cond TrapCode) SideEffectNoResult)
(rule (trap_if_impl cond trap_code)
(SideEffectNoResult.Inst (MInst.TrapIf cond trap_code)))
(decl debugtrap_impl () SideEffectNoResult)
(rule (debugtrap_impl)
(SideEffectNoResult.Inst (MInst.Debugtrap)))
(decl safepoint (SideEffectNoResult) ValueRegs)
(rule (safepoint (SideEffectNoResult.Inst inst))
(let ((_ Unit (emit_safepoint inst)))
(value_regs_invalid)))
;; Helpers for handling boolean conditions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -2201,6 +2252,24 @@
(rule (lower_bool $B32 cond) (select_bool_imm $B32 cond -1 0))
(rule (lower_bool $B64 cond) (select_bool_imm $B64 cond -1 0))
;; Emit a conditional branch based on a boolean condition.
(decl cond_br_bool (ProducesBool MachLabel MachLabel) SideEffectNoResult)
(rule (cond_br_bool (ProducesBool.ProducesBool producer cond) taken not_taken)
(let ((_ Unit (emit_producer producer)))
(cond_br taken not_taken cond)))
;; Emit a one-way conditional branch based on a boolean condition.
(decl oneway_cond_br_bool (ProducesBool MachLabel) SideEffectNoResult)
(rule (oneway_cond_br_bool (ProducesBool.ProducesBool producer cond) dest)
(let ((_ Unit (emit_producer producer)))
(oneway_cond_br dest cond)))
;; Emit a conditional trap based on a boolean condition.
(decl trap_if_bool (ProducesBool TrapCode) SideEffectNoResult)
(rule (trap_if_bool (ProducesBool.ProducesBool producer cond) trap_code)
(let ((_ Unit (emit_producer producer)))
(trap_if_impl cond trap_code)))
;; Helpers for generating `clz` instructions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

153
cranelift/codegen/src/isa/s390x/lower.isle
View File

@@ -4,6 +4,11 @@
;; register(s) within which the lowered instruction's result values live.
(decl lower (Inst) ValueRegs)
;; A variant of the main lowering constructor term, used for branches.
;; The only difference is that it gets an extra argument holding a vector
;; of branch targets to be used.
(decl lower_branch (Inst VecMachLabel) ValueRegs)
;;;; Rules for `iconst` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -1778,3 +1783,151 @@
(value_reg (select_bool_reg ty (icmp_val $false int_cc x y)
(put_in_reg val_true) (put_in_reg val_false))))
;;;; Rules for `jump` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Unconditional branch. The target is found as first (and only) element in
;; the list of the current block's branch targets passed as `targets`.
(rule (lower_branch (jump _ _) targets)
(value_regs_none (jump_impl (vec_element targets 0))))
;;;; Rules for `br_table` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Jump table. `targets` contains the default target followed by the
;; list of branch targets per index value.
(rule (lower_branch (br_table val_idx _ _) targets)
(let ((idx Reg (put_in_reg_zext64 val_idx))
;; Bounds-check the index and branch to default.
;; This is an internal branch that is not a terminator insn.
;; Instead, the default target is listed a potential target
;; in the final JTSequence, which is the block terminator.
(cond ProducesBool
(bool (icmpu_uimm32 $I64 idx (vec_length_minus1 targets))
(intcc_as_cond (IntCC.UnsignedGreaterThanOrEqual))))
(_ ValueRegs (value_regs_none (oneway_cond_br_bool cond
(vec_element targets 0)))))
;; Scale the index by the element size, and then emit the
;; compound instruction that does:
;;
;; larl %r1, <jt-base>
;; agf %r1, 0(%r1, %rScaledIndex)
;; br %r1
;; [jt entries]
;;
;; This must be *one* instruction in the vcode because
;; we cannot allow regalloc to insert any spills/fills
;; in the middle of the sequence; otherwise, the LARL's
;; PC-rel offset to the jumptable would be incorrect.
;; (The alternative is to introduce a relocation pass
;; for inlined jumptables, which is much worse, IMHO.)
(value_regs_none (jt_sequence (lshl_imm $I64 idx 2) targets))))
;;;; Rules for `brz` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Two-way conditional branch on zero. `targets` contains:
;; - element 0: target if the condition is true (i.e. value is zero)
;; - element 1: target if the condition is false (i.e. value is nonzero)
(rule (lower_branch (brz val_cond _ _) targets)
(value_regs_none (cond_br_bool (invert_bool (value_nonzero val_cond))
(vec_element targets 0)
(vec_element targets 1))))
;;;; Rules for `brnz` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Two-way conditional branch on nonzero. `targets` contains:
;; - element 0: target if the condition is true (i.e. value is nonzero)
;; - element 1: target if the condition is false (i.e. value is zero)
(rule (lower_branch (brnz val_cond _ _) targets)
(value_regs_none (cond_br_bool (value_nonzero val_cond)
(vec_element targets 0)
(vec_element targets 1))))
;;;; Rules for `brif` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Similarly to `selectif_spectre_guard`, we only recognize specific patterns
;; generated by common code here. Others will fail to lower.
(rule (lower_branch (brif int_cc (def_inst (ifcmp x y)) _ _) targets)
(value_regs_none (cond_br_bool (icmp_val $false int_cc x y)
(vec_element targets 0)
(vec_element targets 1))))
;;;; Rules for `trap` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(rule (lower (trap trap_code))
(safepoint (trap_impl trap_code)))
;;;; Rules for `resumable_trap` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(rule (lower (resumable_trap trap_code))
(safepoint (trap_impl trap_code)))
;;;; Rules for `trapz` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(rule (lower (trapz val trap_code))
(safepoint (trap_if_bool (invert_bool (value_nonzero val)) trap_code)))
;;;; Rules for `trapnz` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(rule (lower (trapnz val trap_code))
(safepoint (trap_if_bool (value_nonzero val) trap_code)))
;;;; Rules for `resumable_trapnz` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(rule (lower (resumable_trapnz val trap_code))
(safepoint (trap_if_bool (value_nonzero val) trap_code)))
;;;; Rules for `debugtrap` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(rule (lower (debugtrap))
(value_regs_none (debugtrap_impl)))
;;;; Rules for `trapif` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Similarly to `selectif_spectre_guard`, we only recognize specific patterns
;; generated by common code here. Others will fail to lower.
;; Recognize the case of `ifcmp` feeding into `trapif`. Directly generate
;; the desired comparison here; there is no separate `ifcmp` lowering.
(rule (lower (trapif int_cc (def_inst (ifcmp x y)) trap_code))
(safepoint (trap_if_bool (icmp_val $false int_cc x y) trap_code)))
;; Recognize the case of `iadd_ifcout` feeding into `trapif`. Note that
;; in the case, the `iadd_ifcout` is generated by a separate lowering
;; (in order to properly handle the register output of that instruction.)
;;
;; The flags must not have been clobbered by any other instruction between the
;; iadd_ifcout and this instruction, as verified by the CLIF validator; so we
;; can simply rely on the condition code here.
;;
;; IaddIfcout is implemented via a ADD LOGICAL instruction, which sets the
;; the condition code as follows:
;; 0 Result zero; no carry
;; 1 Result not zero; no carry
;; 2 Result zero; carry
;; 3 Result not zero; carry
;; This means "carry" corresponds to condition code 2 or 3, i.e.
;; a condition mask of 2 | 1.
;;
;; As this does not match any of the encodings used with a normal integer
;; comparsion, this cannot be represented by any IntCC value. We need to
;; remap the IntCC::UnsignedGreaterThan value that we have here as result
;; of the unsigned_add_overflow_condition call to the correct mask.
(rule (lower (trapif (IntCC.UnsignedGreaterThan)
(def_inst (iadd_ifcout x y)) trap_code))
(value_regs_none (trap_if_impl (mask_as_cond 3) trap_code)))

794
cranelift/codegen/src/isa/s390x/lower.rs
View File

@@ -2,7 +2,7 @@
use crate::ir::condcodes::IntCC;
use crate::ir::Inst as IRInst;
use crate::ir::{types, Endianness, MemFlags, Opcode, Type};
use crate::ir::{MemFlags, Opcode};
use crate::isa::s390x::abi::*;
use crate::isa::s390x::inst::*;
use crate::isa::s390x::settings as s390x_settings;
@@ -11,255 +11,14 @@ use crate::machinst::lower::*;
use crate::machinst::*;
use crate::settings::Flags;
use crate::CodegenResult;
use alloc::boxed::Box;
use core::convert::TryFrom;
use regalloc::{Reg, Writable};
use regalloc::Reg;
use smallvec::SmallVec;
pub mod isle;
//=============================================================================
// Helpers for instruction lowering.
fn choose_32_64<T: Copy>(ty: Type, op32: T, op64: T) -> T {
let bits = ty_bits(ty);
if bits <= 32 {
op32
} else if bits == 64 {
op64
} else {
panic!("choose_32_64 on > 64 bits!")
}
}
//============================================================================
// Lowering: convert instruction inputs to forms that we can use.
/// Lower an instruction input to a 64-bit constant, if possible.
fn input_matches_const<C: LowerCtx<I = Inst>>(ctx: &mut C, input: InsnInput) -> Option<u64> {
let input = ctx.get_input_as_source_or_const(input.insn, input.input);
input.constant
}
/// Lower an instruction input to a 64-bit signed constant, if possible.
fn input_matches_sconst<C: LowerCtx<I = Inst>>(ctx: &mut C, input: InsnInput) -> Option<i64> {
if let Some(imm) = input_matches_const(ctx, input) {
let ty = ctx.input_ty(input.insn, input.input);
Some(sign_extend_to_u64(imm, ty_bits(ty) as u8) as i64)
} else {
None
}
}
/// Lower an instruction input to a 16-bit signed constant, if possible.
fn input_matches_simm16<C: LowerCtx<I = Inst>>(ctx: &mut C, input: InsnInput) -> Option<i16> {
if let Some(imm_value) = input_matches_sconst(ctx, input) {
if let Ok(imm) = i16::try_from(imm_value) {
return Some(imm);
}
}
None
}
/// Lower an instruction input to a 32-bit signed constant, if possible.
fn input_matches_simm32<C: LowerCtx<I = Inst>>(ctx: &mut C, input: InsnInput) -> Option<i32> {
if let Some(imm_value) = input_matches_sconst(ctx, input) {
if let Ok(imm) = i32::try_from(imm_value) {
return Some(imm);
}
}
None
}
/// Lower an instruction input to a 32-bit unsigned constant, if possible.
fn input_matches_uimm32<C: LowerCtx<I = Inst>>(ctx: &mut C, input: InsnInput) -> Option<u32> {
if let Some(imm_value) = input_matches_const(ctx, input) {
if let Ok(imm) = u32::try_from(imm_value) {
return Some(imm);
}
}
None
}
/// Checks for an instance of `op` feeding the given input.
fn input_matches_insn<C: LowerCtx<I = Inst>>(
c: &mut C,
input: InsnInput,
op: Opcode,
) -> Option<IRInst> {
let inputs = c.get_input_as_source_or_const(input.insn, input.input);
if let Some((src_inst, _)) = inputs.inst {
let data = c.data(src_inst);
if data.opcode() == op {
return Some(src_inst);
}
}
None
}
/// Checks for an instance of `op` feeding the given input, possibly via a conversion `conv` (e.g.,
/// Bint or a bitcast).
fn input_matches_insn_via_conv<C: LowerCtx<I = Inst>>(
c: &mut C,
input: InsnInput,
op: Opcode,
conv: Opcode,
) -> Option<IRInst> {
let inputs = c.get_input_as_source_or_const(input.insn, input.input);
if let Some((src_inst, _)) = inputs.inst {
let data = c.data(src_inst);
if data.opcode() == op {
return Some(src_inst);
}
if data.opcode() == conv {
let inputs = c.get_input_as_source_or_const(src_inst, 0);
if let Some((src_inst, _)) = inputs.inst {
let data = c.data(src_inst);
if data.opcode() == op {
return Some(src_inst);
}
}
}
}
None
}
fn input_matches_load_insn<C: LowerCtx<I = Inst>>(
ctx: &mut C,
input: InsnInput,
op: Opcode,
) -> Option<MemArg> {
if let Some(insn) = input_matches_insn(ctx, input, op) {
let inputs: SmallVec<[InsnInput; 4]> = (0..ctx.num_inputs(insn))
.map(|i| InsnInput { insn, input: i })
.collect();
let off = ctx.data(insn).load_store_offset().unwrap();
let flags = ctx.memflags(insn).unwrap();
let endianness = flags.endianness(Endianness::Big);
if endianness == Endianness::Big {
let mem = lower_address(ctx, &inputs[..], off, flags);
ctx.sink_inst(insn);
return Some(mem);
}
}
None
}
fn input_matches_mem<C: LowerCtx<I = Inst>>(ctx: &mut C, input: InsnInput) -> Option<MemArg> {
if ty_bits(ctx.input_ty(input.insn, input.input)) >= 32 {
return input_matches_load_insn(ctx, input, Opcode::Load);
}
None
}
fn input_matches_sext16_mem<C: LowerCtx<I = Inst>>(
ctx: &mut C,
input: InsnInput,
) -> Option<MemArg> {
if ty_bits(ctx.input_ty(input.insn, input.input)) == 16 {
return input_matches_load_insn(ctx, input, Opcode::Load);
}
if ty_bits(ctx.input_ty(input.insn, input.input)) >= 32 {
return input_matches_load_insn(ctx, input, Opcode::Sload16);
}
None
}
fn input_matches_sext32_mem<C: LowerCtx<I = Inst>>(
ctx: &mut C,
input: InsnInput,
) -> Option<MemArg> {
if ty_bits(ctx.input_ty(input.insn, input.input)) > 32 {
return input_matches_load_insn(ctx, input, Opcode::Sload32);
}
None
}
fn input_matches_sext32_reg<C: LowerCtx<I = Inst>>(ctx: &mut C, input: InsnInput) -> Option<Reg> {
if let Some(insn) = input_matches_insn(ctx, input, Opcode::Sextend) {
if ty_bits(ctx.input_ty(insn, 0)) == 32 {
let reg = put_input_in_reg(ctx, InsnInput { insn, input: 0 }, NarrowValueMode::None);
return Some(reg);
}
}
None
}
fn input_matches_uext32_reg<C: LowerCtx<I = Inst>>(ctx: &mut C, input: InsnInput) -> Option<Reg> {
if let Some(insn) = input_matches_insn(ctx, input, Opcode::Uextend) {
if ty_bits(ctx.input_ty(insn, 0)) == 32 {
let reg = put_input_in_reg(ctx, InsnInput { insn, input: 0 }, NarrowValueMode::None);
return Some(reg);
}
}
None
}
fn input_matches_uext16_mem<C: LowerCtx<I = Inst>>(
ctx: &mut C,
input: InsnInput,
) -> Option<MemArg> {
if ty_bits(ctx.input_ty(input.insn, input.input)) == 16 {
return input_matches_load_insn(ctx, input, Opcode::Load);
}
if ty_bits(ctx.input_ty(input.insn, input.input)) >= 32 {
return input_matches_load_insn(ctx, input, Opcode::Uload16);
}
None
}
fn input_matches_uext32_mem<C: LowerCtx<I = Inst>>(
ctx: &mut C,
input: InsnInput,
) -> Option<MemArg> {
if ty_bits(ctx.input_ty(input.insn, input.input)) > 32 {
return input_matches_load_insn(ctx, input, Opcode::Uload32);
}
None
}
//============================================================================
// Lowering: force instruction input into a register
/// How to handle narrow values loaded into registers; see note on `narrow_mode`
/// parameter to `put_input_in_*` below.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum NarrowValueMode {
None,
/// Zero-extend to 32 bits if original is < 32 bits.
ZeroExtend32,
/// Sign-extend to 32 bits if original is < 32 bits.
SignExtend32,
/// Zero-extend to 64 bits if original is < 64 bits.
ZeroExtend64,
/// Sign-extend to 64 bits if original is < 64 bits.
SignExtend64,
}
fn extend_memory_to_reg<C: LowerCtx<I = Inst>>(
ctx: &mut C,
mem: MemArg,
from_ty: Type,
to_ty: Type,
signed: bool,
) -> Reg {
let rd = ctx.alloc_tmp(to_ty).only_reg().unwrap();
ctx.emit(match (signed, ty_bits(to_ty), ty_bits(from_ty)) {
(false, 32, 8) => Inst::Load32ZExt8 { rd, mem },
(false, 32, 16) => Inst::Load32ZExt16 { rd, mem },
(true, 32, 8) => Inst::Load32SExt8 { rd, mem },
(true, 32, 16) => Inst::Load32SExt16 { rd, mem },
(false, 64, 8) => Inst::Load64ZExt8 { rd, mem },
(false, 64, 16) => Inst::Load64ZExt16 { rd, mem },
(false, 64, 32) => Inst::Load64ZExt32 { rd, mem },
(true, 64, 8) => Inst::Load64SExt8 { rd, mem },
(true, 64, 16) => Inst::Load64SExt16 { rd, mem },
(true, 64, 32) => Inst::Load64SExt32 { rd, mem },
_ => panic!("Unsupported size in load"),
});
rd.to_reg()
}
/// Sign-extend the low `from_bits` bits of `value` to a full u64.
fn sign_extend_to_u64(value: u64, from_bits: u8) -> u64 {
assert!(from_bits <= 64);
@@ -270,169 +29,11 @@ fn sign_extend_to_u64(value: u64, from_bits: u8) -> u64 {
}
}
/// Zero-extend the low `from_bits` bits of `value` to a full u64.
fn zero_extend_to_u64(value: u64, from_bits: u8) -> u64 {
assert!(from_bits <= 64);
if from_bits >= 64 {
value
} else {
value & ((1u64 << from_bits) - 1)
}
}
/// Lower an instruction input to a reg.
///
/// The given register will be extended appropriately, according to
/// `narrow_mode` and the input's type.
fn put_input_in_reg<C: LowerCtx<I = Inst>>(
ctx: &mut C,
input: InsnInput,
narrow_mode: NarrowValueMode,
) -> Reg {
let signed = match narrow_mode {
NarrowValueMode::SignExtend32 | NarrowValueMode::SignExtend64 => true,
NarrowValueMode::ZeroExtend32 | NarrowValueMode::ZeroExtend64 => false,
_ => false,
};
let ty = ctx.input_ty(input.insn, input.input);
let from_bits = ty_bits(ty) as u8;
let ext_ty = match narrow_mode {
NarrowValueMode::None => ty,
NarrowValueMode::ZeroExtend32 | NarrowValueMode::SignExtend32 => types::I32,
NarrowValueMode::ZeroExtend64 | NarrowValueMode::SignExtend64 => types::I64,
};
let to_bits = ty_bits(ext_ty) as u8;
assert!(to_bits >= from_bits);
if let Some(c) = input_matches_const(ctx, input) {
let extended = if from_bits == to_bits {
c
} else if signed {
sign_extend_to_u64(c, from_bits)
} else {
zero_extend_to_u64(c, from_bits)
};
let masked = zero_extend_to_u64(extended, to_bits);
// Generate constants fresh at each use to minimize long-range register pressure.
let to_reg = ctx.alloc_tmp(ext_ty).only_reg().unwrap();
for inst in Inst::gen_constant(ValueRegs::one(to_reg), masked as u128, ext_ty, |ty| {
ctx.alloc_tmp(ty).only_reg().unwrap()
})
.into_iter()
{
ctx.emit(inst);
}
to_reg.to_reg()
} else if to_bits == from_bits {
ctx.put_input_in_regs(input.insn, input.input)
.only_reg()
.unwrap()
} else if let Some(mem) = input_matches_load_insn(ctx, input, Opcode::Load) {
extend_memory_to_reg(ctx, mem, ty, ext_ty, signed)
} else {
let rd = ctx.alloc_tmp(ext_ty).only_reg().unwrap();
let rn = ctx
.put_input_in_regs(input.insn, input.input)
.only_reg()
.unwrap();
ctx.emit(Inst::Extend {
rd,
rn,
signed,
from_bits,
to_bits,
});
rd.to_reg()
}
}
//============================================================================
// Lowering: addressing mode support. Takes instruction directly, rather
// than an `InsnInput`, to do more introspection.
/// Lower the address of a load or store.
fn lower_address<C: LowerCtx<I = Inst>>(
ctx: &mut C,
addends: &[InsnInput],
offset: i32,
flags: MemFlags,
) -> MemArg {
// Handle one reg and offset.
if addends.len() == 1 {
if offset == 0 {
if let Some(add) = input_matches_insn(ctx, addends[0], Opcode::Iadd) {
debug_assert_eq!(ctx.output_ty(add, 0), types::I64);
let add_inputs = &[
InsnInput {
insn: add,
input: 0,
},
InsnInput {
insn: add,
input: 1,
},
];
let ra = put_input_in_reg(ctx, add_inputs[0], NarrowValueMode::None);
let rb = put_input_in_reg(ctx, add_inputs[1], NarrowValueMode::None);
return MemArg::reg_plus_reg(ra, rb, flags);
}
}
if let Some(symbol) = input_matches_insn(ctx, addends[0], Opcode::SymbolValue) {
let (extname, dist, ext_offset) = ctx.symbol_value(symbol).unwrap();
let ext_offset = ext_offset + i64::from(offset);
if dist == RelocDistance::Near && (ext_offset & 1) == 0 {
if let Ok(offset) = i32::try_from(ext_offset) {
return MemArg::Symbol {
name: Box::new(extname.clone()),
offset,
flags,
};
}
}
}
let reg = put_input_in_reg(ctx, addends[0], NarrowValueMode::None);
return MemArg::reg_plus_off(reg, offset as i64, flags);
}
// Handle two regs and a zero offset.
if addends.len() == 2 && offset == 0 {
let ra = put_input_in_reg(ctx, addends[0], NarrowValueMode::None);
let rb = put_input_in_reg(ctx, addends[1], NarrowValueMode::None);
return MemArg::reg_plus_reg(ra, rb, flags);
}
// Otherwise, generate add instructions.
let addr = ctx.alloc_tmp(types::I64).only_reg().unwrap();
// Get the const into a reg.
lower_constant_u64(ctx, addr.clone(), offset as u64);
// Add each addend to the address.
for addend in addends {
let reg = put_input_in_reg(ctx, *addend, NarrowValueMode::None);
ctx.emit(Inst::AluRRR {
alu_op: ALUOp::Add64,
rd: addr.clone(),
rn: addr.to_reg(),
rm: reg.clone(),
});
}
MemArg::reg(addr.to_reg(), flags)
}
//============================================================================
// Lowering: generating constants.
fn lower_constant_u64<C: LowerCtx<I = Inst>>(ctx: &mut C, rd: Writable<Reg>, value: u64) {
for inst in Inst::load_constant64(rd, value) {
ctx.emit(inst);
}
fn put_input_in_reg<C: LowerCtx<I = Inst>>(ctx: &mut C, input: InsnInput) -> Reg {
ctx.put_input_in_regs(input.insn, input.input)
.only_reg()
.unwrap()
}
//=============================================================================
@@ -459,151 +60,6 @@ pub fn condcode_is_signed(cc: IntCC) -> bool {
}
}
fn lower_icmp_to_flags<C: LowerCtx<I = Inst>>(
ctx: &mut C,
insn: IRInst,
is_signed: bool,
may_sink_memory: bool,
) {
let ty = ctx.input_ty(insn, 0);
let bits = ty_bits(ty);
let narrow_mode = match (bits <= 32, is_signed) {
(true, true) => NarrowValueMode::SignExtend32,
(true, false) => NarrowValueMode::ZeroExtend32,
(false, true) => NarrowValueMode::SignExtend64,
(false, false) => NarrowValueMode::ZeroExtend64,
};
let inputs = [InsnInput { insn, input: 0 }, InsnInput { insn, input: 1 }];
let ty = ctx.input_ty(insn, 0);
let rn = put_input_in_reg(ctx, inputs[0], narrow_mode);
if is_signed {
let op = choose_32_64(ty, CmpOp::CmpS32, CmpOp::CmpS64);
// Try matching immedate operand.
if let Some(imm) = input_matches_simm16(ctx, inputs[1]) {
return ctx.emit(Inst::CmpRSImm16 { op, rn, imm });
}
if let Some(imm) = input_matches_simm32(ctx, inputs[1]) {
return ctx.emit(Inst::CmpRSImm32 { op, rn, imm });
}
// If sinking memory loads is allowed, try matching memory operand.
if may_sink_memory {
if let Some(mem) = input_matches_mem(ctx, inputs[1]) {
return ctx.emit(Inst::CmpRX { op, rn, mem });
}
if let Some(mem) = input_matches_sext16_mem(ctx, inputs[1]) {
let op = choose_32_64(ty, CmpOp::CmpS32Ext16, CmpOp::CmpS64Ext16);
return ctx.emit(Inst::CmpRX { op, rn, mem });
}
if let Some(mem) = input_matches_sext32_mem(ctx, inputs[1]) {
return ctx.emit(Inst::CmpRX {
op: CmpOp::CmpS64Ext32,
rn,
mem,
});
}
}
// Try matching sign-extension in register.
if let Some(rm) = input_matches_sext32_reg(ctx, inputs[1]) {
return ctx.emit(Inst::CmpRR {
op: CmpOp::CmpS64Ext32,
rn,
rm,
});
}
// If no special case matched above, fall back to a register compare.
let rm = put_input_in_reg(ctx, inputs[1], narrow_mode);
return ctx.emit(Inst::CmpRR { op, rn, rm });
} else {
let op = choose_32_64(ty, CmpOp::CmpL32, CmpOp::CmpL64);
// Try matching immedate operand.
if let Some(imm) = input_matches_uimm32(ctx, inputs[1]) {
return ctx.emit(Inst::CmpRUImm32 { op, rn, imm });
}
// If sinking memory loads is allowed, try matching memory operand.
if may_sink_memory {
if let Some(mem) = input_matches_mem(ctx, inputs[1]) {
return ctx.emit(Inst::CmpRX { op, rn, mem });
}
if let Some(mem) = input_matches_uext16_mem(ctx, inputs[1]) {
match &mem {
&MemArg::Symbol { .. } => {
let op = choose_32_64(ty, CmpOp::CmpL32Ext16, CmpOp::CmpL64Ext16);
return ctx.emit(Inst::CmpRX { op, rn, mem });
}
_ => {
let reg_ty = choose_32_64(ty, types::I32, types::I64);
let rm = extend_memory_to_reg(ctx, mem, types::I16, reg_ty, false);
return ctx.emit(Inst::CmpRR { op, rn, rm });
}
}
}
if let Some(mem) = input_matches_uext32_mem(ctx, inputs[1]) {
return ctx.emit(Inst::CmpRX {
op: CmpOp::CmpL64Ext32,
rn,
mem,
});
}
}
// Try matching zero-extension in register.
if let Some(rm) = input_matches_uext32_reg(ctx, inputs[1]) {
return ctx.emit(Inst::CmpRR {
op: CmpOp::CmpL64Ext32,
rn,
rm,
});
}
// If no special case matched above, fall back to a register compare.
let rm = put_input_in_reg(ctx, inputs[1], narrow_mode);
return ctx.emit(Inst::CmpRR { op, rn, rm });
}
}
fn lower_fcmp_to_flags<C: LowerCtx<I = Inst>>(ctx: &mut C, insn: IRInst) {
let ty = ctx.input_ty(insn, 0);
let bits = ty_bits(ty);
let inputs = [InsnInput { insn, input: 0 }, InsnInput { insn, input: 1 }];
let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
let rm = put_input_in_reg(ctx, inputs[1], NarrowValueMode::None);
match bits {
32 => {
ctx.emit(Inst::FpuCmp32 { rn, rm });
}
64 => {
ctx.emit(Inst::FpuCmp64 { rn, rm });
}
_ => panic!("Unknown float size"),
}
}
fn lower_boolean_to_flags<C: LowerCtx<I = Inst>>(ctx: &mut C, input: InsnInput) -> Cond {
if let Some(icmp_insn) = input_matches_insn_via_conv(ctx, input, Opcode::Icmp, Opcode::Bint) {
// FIXME: If the Icmp (and Bint) only have a single use, we can still allow sinking memory
let may_sink_memory = false;
let condcode = ctx.data(icmp_insn).cond_code().unwrap();
let is_signed = condcode_is_signed(condcode);
lower_icmp_to_flags(ctx, icmp_insn, is_signed, may_sink_memory);
Cond::from_intcc(condcode)
} else if let Some(fcmp_insn) =
input_matches_insn_via_conv(ctx, input, Opcode::Fcmp, Opcode::Bint)
{
let condcode = ctx.data(fcmp_insn).fp_cond_code().unwrap();
lower_fcmp_to_flags(ctx, fcmp_insn);
Cond::from_floatcc(condcode)
} else {
let ty = ctx.input_ty(input.insn, input.input);
let narrow_mode = if ty.bits() < 32 {
NarrowValueMode::ZeroExtend32
} else {
NarrowValueMode::None
};
let rn = put_input_in_reg(ctx, input, narrow_mode);
let op = choose_32_64(ty, CmpOp::CmpS32, CmpOp::CmpS64);
ctx.emit(Inst::CmpRSImm16 { op, rn, imm: 0 });
Cond::from_intcc(IntCC::NotEqual)
}
}
//============================================================================
// Lowering: main entry point for lowering a instruction
@@ -728,6 +184,13 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
| Opcode::IsInvalid
| Opcode::Select
| Opcode::SelectifSpectreGuard
| Opcode::Trap
| Opcode::ResumableTrap
| Opcode::Trapz
| Opcode::Trapnz
| Opcode::ResumableTrapnz
| Opcode::Trapif
| Opcode::Debugtrap
| Opcode::StackAddr
| Opcode::FuncAddr
| Opcode::SymbolValue => implemented_in_isle(),
@@ -765,63 +228,6 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
unimplemented!("Pinned register support not implemented!");
}
Opcode::Trap | Opcode::ResumableTrap => {
let trap_code = ctx.data(insn).trap_code().unwrap();
ctx.emit_safepoint(Inst::Trap { trap_code })
}
Opcode::Trapz | Opcode::Trapnz | Opcode::ResumableTrapnz => {
let cond = lower_boolean_to_flags(ctx, inputs[0]);
let negated = op == Opcode::Trapz;
let cond = if negated { cond.invert() } else { cond };
let trap_code = ctx.data(insn).trap_code().unwrap();
ctx.emit_safepoint(Inst::TrapIf { trap_code, cond });
}
Opcode::Trapif => {
let condcode = ctx.data(insn).cond_code().unwrap();
let mut cond = Cond::from_intcc(condcode);
let is_signed = condcode_is_signed(condcode);
let cmp_insn = ctx
.get_input_as_source_or_const(inputs[0].insn, inputs[0].input)
.inst
.unwrap()
.0;
if ctx.data(cmp_insn).opcode() == Opcode::IaddIfcout {
// The flags must not have been clobbered by any other instruction between the
// iadd_ifcout and this instruction, as verified by the CLIF validator; so we
// can simply rely on the condition code here.
//
// IaddIfcout is implemented via a ADD LOGICAL instruction, which sets the
// the condition code as follows:
// 0 Result zero; no carry
// 1 Result not zero; no carry
// 2 Result zero; carry
// 3 Result not zero; carry
// This means "carry" corresponds to condition code 2 or 3, i.e.
// a condition mask of 2 | 1.
//
// As this does not match any of the encodings used with a normal integer
// comparsion, this cannot be represented by any IntCC value. We need to
// remap the IntCC::UnsignedGreaterThan value that we have here as result
// of the unsigned_add_overflow_condition call to the correct mask.
assert!(condcode == IntCC::UnsignedGreaterThan);
cond = Cond::from_mask(2 | 1);
} else {
// Verification ensures that the input is always a single-def ifcmp
debug_assert_eq!(ctx.data(cmp_insn).opcode(), Opcode::Ifcmp);
lower_icmp_to_flags(ctx, cmp_insn, is_signed, true);
}
let trap_code = ctx.data(insn).trap_code().unwrap();
ctx.emit_safepoint(Inst::TrapIf { trap_code, cond });
}
Opcode::Debugtrap => {
ctx.emit(Inst::Debugtrap);
}
Opcode::Call | Opcode::CallIndirect => {
let caller_conv = ctx.abi().call_conv();
let (mut abi, inputs) = match op {
@@ -837,7 +243,7 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
)
}
Opcode::CallIndirect => {
let ptr = put_input_in_reg(ctx, inputs[0], NarrowValueMode::ZeroExtend64);
let ptr = put_input_in_reg(ctx, inputs[0]);
let sig = ctx.call_sig(insn).unwrap();
assert!(inputs.len() - 1 == sig.params.len());
assert!(outputs.len() == sig.returns.len());
@@ -851,7 +257,7 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
assert!(inputs.len() == abi.num_args());
for (i, input) in inputs.iter().enumerate() {
let arg_reg = put_input_in_reg(ctx, *input, NarrowValueMode::None);
let arg_reg = put_input_in_reg(ctx, *input);
abi.emit_copy_regs_to_arg(ctx, i, ValueRegs::one(arg_reg));
}
abi.emit_call(ctx);
@@ -864,7 +270,7 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
Opcode::FallthroughReturn | Opcode::Return => {
for (i, input) in inputs.iter().enumerate() {
let reg = put_input_in_reg(ctx, *input, NarrowValueMode::None);
let reg = put_input_in_reg(ctx, *input);
let retval_reg = ctx.retval(i).only_reg().unwrap();
let ty = ctx.input_ty(insn, i);
ctx.emit(Inst::gen_move(retval_reg, reg, ty));
@@ -995,146 +401,6 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
Ok(())
}
//============================================================================
// Lowering: main entry point for lowering a branch group
fn lower_branch<C: LowerCtx<I = Inst>>(
ctx: &mut C,
branches: &[IRInst],
targets: &[MachLabel],
) -> CodegenResult<()> {
// A block should end with at most two branches. The first may be a
// conditional branch; a conditional branch can be followed only by an
// unconditional branch or fallthrough. Otherwise, if only one branch,
// it may be an unconditional branch, a fallthrough, a return, or a
// trap. These conditions are verified by `is_ebb_basic()` during the
// verifier pass.
assert!(branches.len() <= 2);
if branches.len() == 2 {
// Must be a conditional branch followed by an unconditional branch.
let op0 = ctx.data(branches[0]).opcode();
let op1 = ctx.data(branches[1]).opcode();
assert!(op1 == Opcode::Jump);
let taken = targets[0];
let not_taken = targets[1];
match op0 {
Opcode::Brz | Opcode::Brnz => {
let flag_input = InsnInput {
insn: branches[0],
input: 0,
};
let cond = lower_boolean_to_flags(ctx, flag_input);
let negated = op0 == Opcode::Brz;
let cond = if negated { cond.invert() } else { cond };
ctx.emit(Inst::CondBr {
taken,
not_taken,
cond,
});
}
Opcode::Brif => {
let condcode = ctx.data(branches[0]).cond_code().unwrap();
let cond = Cond::from_intcc(condcode);
let is_signed = condcode_is_signed(condcode);
// Verification ensures that the input is always a single-def ifcmp.
let cmp_insn = ctx
.get_input_as_source_or_const(branches[0], 0)
.inst
.unwrap()
.0;
debug_assert_eq!(ctx.data(cmp_insn).opcode(), Opcode::Ifcmp);
lower_icmp_to_flags(ctx, cmp_insn, is_signed, true);
ctx.emit(Inst::CondBr {
taken,
not_taken,
cond,
});
}
Opcode::Brff => unreachable!(),
_ => unimplemented!(),
}
} else {
// Must be an unconditional branch or an indirect branch.
let op = ctx.data(branches[0]).opcode();
match op {
Opcode::Jump => {
assert!(branches.len() == 1);
ctx.emit(Inst::Jump { dest: targets[0] });
}
Opcode::BrTable => {
let jt_size = targets.len() - 1;
assert!(jt_size <= std::u32::MAX as usize);
// Load up jump table element index.
let ridx = put_input_in_reg(
ctx,
InsnInput {
insn: branches[0],
input: 0,
},
NarrowValueMode::ZeroExtend64,
);
// Bounds-check index and branch to default.
// This is an internal branch that is not a terminator insn.
// Instead, the default target is listed a potential target
// in the final JTSequence, which is the block terminator.
ctx.emit(Inst::CmpRUImm32 {
op: CmpOp::CmpL64,
rn: ridx,
imm: jt_size as u32,
});
ctx.emit(Inst::OneWayCondBr {
target: targets[0],
cond: Cond::from_intcc(IntCC::UnsignedGreaterThanOrEqual),
});
// Compute index scaled by entry size.
let rtmp = ctx.alloc_tmp(types::I64).only_reg().unwrap();
ctx.emit(Inst::ShiftRR {
shift_op: ShiftOp::LShL64,
rd: rtmp,
rn: ridx,
shift_imm: 2,
shift_reg: zero_reg(),
});
// Emit the compound instruction that does:
//
// larl %r1, <jt-base>
// agf %r1, 0(%r1, %rTmp)
// br %r1
// [jt entries]
//
// This must be *one* instruction in the vcode because
// we cannot allow regalloc to insert any spills/fills
// in the middle of the sequence; otherwise, the ADR's
// PC-rel offset to the jumptable would be incorrect.
// (The alternative is to introduce a relocation pass
// for inlined jumptables, which is much worse, IMHO.)
ctx.emit(Inst::JTSequence {
ridx: rtmp.to_reg(),
targets: targets.to_vec(),
});
}
_ => panic!("Unknown branch type!"),
}
}
Ok(())
}
//=============================================================================
// Lowering-backend trait implementation.
@@ -1151,6 +417,32 @@ impl LowerBackend for S390xBackend {
branches: &[IRInst],
targets: &[MachLabel],
) -> CodegenResult<()> {
lower_branch(ctx, branches, targets)
// A block should end with at most two branches. The first may be a
// conditional branch; a conditional branch can be followed only by an
// unconditional branch or fallthrough. Otherwise, if only one branch,
// it may be an unconditional branch, a fallthrough, a return, or a
// trap. These conditions are verified by `is_ebb_basic()` during the
// verifier pass.
assert!(branches.len() <= 2);
if branches.len() == 2 {
let op1 = ctx.data(branches[1]).opcode();
assert!(op1 == Opcode::Jump);
}
// Lower the first branch in ISLE. This will automatically handle
// the second branch (if any) by emitting a two-way conditional branch.
if let Ok(()) = super::lower::isle::lower_branch(
ctx,
&self.flags,
&self.isa_flags,
branches[0],
targets,
) {
return Ok(());
}
unreachable!(
"implemented in ISLE: branch = `{}`",
ctx.dfg().display_inst(branches[0]),
);
}
}

37
cranelift/codegen/src/isa/s390x/lower/isle.rs
View File

@@ -51,6 +51,28 @@ where
)
}
/// The main entry point for branch lowering with ISLE.
pub(crate) fn lower_branch<C>(
lower_ctx: &mut C,
flags: &Flags,
isa_flags: &IsaFlags,
branch: Inst,
targets: &[MachLabel],
) -> Result<(), ()>
where
C: LowerCtx<I = MInst>,
{
lower_common(
lower_ctx,
flags,
isa_flags,
&[],
branch,
|cx, insn| generated_code::constructor_lower_branch(cx, insn, &targets.to_vec()),
s390x_map_regs,
)
}
impl<C> generated_code::Context for IsleContext<'_, C, Flags, IsaFlags, 6>
where
C: LowerCtx<I = MInst>,
@@ -369,6 +391,16 @@ where
}
}
#[inline]
fn vec_length_minus1(&mut self, vec: &VecMachLabel) -> u32 {
u32::try_from(vec.len()).unwrap() - 1
}
#[inline]
fn vec_element(&mut self, vec: &VecMachLabel, index: u8) -> MachLabel {
vec[usize::from(index)]
}
#[inline]
fn reloc_distance_near(&mut self, dist: &RelocDistance) -> Option<()> {
if *dist == RelocDistance::Near {
@@ -471,4 +503,9 @@ where
fn emit(&mut self, inst: &MInst) -> Unit {
self.emitted_insts.push((inst.clone(), false));
}
#[inline]
fn emit_safepoint(&mut self, inst: &MInst) -> Unit {
self.emitted_insts.push((inst.clone(), true));
}
}

4
cranelift/codegen/src/isa/s390x/lower/isle/generated_code.manifest
View File

@@ -1,4 +1,4 @@
src/clif.isle 9ea75a6f790b5c03
src/prelude.isle 51d2aef2566c1c96
src/isa/s390x/inst.isle 17b77476355c4509
src/isa/s390x/lower.isle a0e21a567040bc33
src/isa/s390x/inst.isle d7bfd05fb4d4a66d
src/isa/s390x/lower.isle 57dcc39cbab2d1c6

1921
cranelift/codegen/src/isa/s390x/lower/isle/generated_code.rs generated
View File

File diff suppressed because it is too large Load Diff

5
cranelift/codegen/src/machinst/isle.rs
View File

@@ -312,18 +312,19 @@ where
/// The `isle_lower` argument here is an ISLE-generated function for `lower` and
/// then this function otherwise handles register mapping and such around the
/// lowering.
pub(crate) fn lower_common<C, F, I, const N: usize>(
pub(crate) fn lower_common<C, F, I, IF, const N: usize>(
lower_ctx: &mut C,
flags: &F,
isa_flags: &I,
outputs: &[InsnOutput],
inst: Inst,
isle_lower: fn(&mut IsleContext<'_, C, F, I, N>, Inst) -> Option<ValueRegs>,
isle_lower: IF,
map_regs: fn(&mut C::I, &RegRenamer),
) -> Result<(), ()>
where
C: LowerCtx,
[(C::I, bool); N]: smallvec::Array<Item = (C::I, bool)>,
IF: Fn(&mut IsleContext<'_, C, F, I, N>, Inst) -> Option<ValueRegs>,
{
// TODO: reuse the ISLE context across lowerings so we can reuse its
// internal heap allocations.