T0b1/wasmtime
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

s390x: Migrate branches and traps to ISLE

Browse Source 
In order to migrate branches to ISLE, we define a second entry
point `lower_branch` which gets the list of branch targets as
additional argument.

This requires a small change to `lower_common`: the `isle_lower`
callback argument is changed from a function pointer to a closure.
This allows passing the extra argument via a closure.

Traps make use of the recently added facility to emit safepoints
from ISLE, but are otherwise straightforward.
This commit is contained in:
Ulrich Weigand
2022-01-25 18:15:32 +01:00
parent cd6b73fc90
commit 36369a6f35
7 changed files with 1485 additions and 1500 deletions
Download Patch File Download Diff File Expand all files Collapse all files

71
cranelift/codegen/src/isa/s390x/inst.isle
View File

@@ -679,7 +679,6 @@
(type BoxCallInfo (primitive BoxCallInfo)) (type BoxCallInfo (primitive BoxCallInfo))
(type BoxCallIndInfo (primitive BoxCallIndInfo)) (type BoxCallIndInfo (primitive BoxCallIndInfo))
(type MachLabel (primitive MachLabel)) (type MachLabel (primitive MachLabel))
(type VecMachLabel (primitive VecMachLabel))
(type BoxJTSequenceInfo (primitive BoxJTSequenceInfo)) (type BoxJTSequenceInfo (primitive BoxJTSequenceInfo))
(type BoxExternalName (primitive BoxExternalName)) (type BoxExternalName (primitive BoxExternalName))
(type ValueLabel (primitive ValueLabel)) (type ValueLabel (primitive ValueLabel))
@@ -1040,6 +1039,19 @@
(extern extractor unsigned unsigned) (extern extractor unsigned unsigned)
;; Helpers for machine label vectors ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; VecMachLabel needs to be passed by reference, so it cannot be
;; declared as primitive type. Declare as extern enum instead.
(type VecMachLabel extern (enum))
(decl vec_length_minus1 (VecMachLabel) u32)
(extern constructor vec_length_minus1 vec_length_minus1)
(decl vec_element (VecMachLabel u8) MachLabel)
(extern constructor vec_element vec_element)
;; Helpers for memory arguments ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; Helpers for memory arguments ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Accessors for `RelocDistance`. ;; Accessors for `RelocDistance`.
@@ -1284,6 +1296,8 @@
;; implementation detail by helpers that preserve the SSA facade themselves. ;; implementation detail by helpers that preserve the SSA facade themselves.
(decl emit (MInst) Unit) (decl emit (MInst) Unit)
(extern constructor emit emit) (extern constructor emit emit)
(decl emit_safepoint (MInst) Unit)
(extern constructor emit_safepoint emit_safepoint)
;; Helper for emitting `MInst.AluRRR` instructions. ;; Helper for emitting `MInst.AluRRR` instructions.
(decl alu_rrr (Type ALUOp Reg Reg) Reg) (decl alu_rrr (Type ALUOp Reg Reg) Reg)
@@ -1695,6 +1709,26 @@
(_ Unit (emit (MInst.LoadAddr dst mem)))) (_ Unit (emit (MInst.LoadAddr dst mem))))
(writable_reg_to_reg dst))) (writable_reg_to_reg dst)))
;; Helper for emitting `MInst.Jump` instructions.
(decl jump_impl (MachLabel) SideEffectNoResult)
(rule (jump_impl target)
(SideEffectNoResult.Inst (MInst.Jump target)))
;; Helper for emitting `MInst.CondBr` instructions.
(decl cond_br (MachLabel MachLabel Cond) SideEffectNoResult)
(rule (cond_br taken not_taken cond)
(SideEffectNoResult.Inst (MInst.CondBr taken not_taken cond)))
;; Helper for emitting `MInst.OneWayCondBr` instructions.
(decl oneway_cond_br (MachLabel Cond) SideEffectNoResult)
(rule (oneway_cond_br dest cond)
(SideEffectNoResult.Inst (MInst.OneWayCondBr dest cond)))
;; Helper for emitting `MInst.JTSequence` instructions.
(decl jt_sequence (Reg VecMachLabel) SideEffectNoResult)
(rule (jt_sequence ridx targets)
(SideEffectNoResult.Inst (MInst.JTSequence ridx targets)))
;; Emit a `ProducesFlags` instruction when the flags are not actually needed. ;; Emit a `ProducesFlags` instruction when the flags are not actually needed.
(decl drop_flags (ProducesFlags) Reg) (decl drop_flags (ProducesFlags) Reg)
(rule (drop_flags (ProducesFlags.ProducesFlags inst result)) (rule (drop_flags (ProducesFlags.ProducesFlags inst result))
@@ -2149,6 +2183,23 @@
src imm cond trap_code)))) src imm cond trap_code))))
(invalid_reg))) (invalid_reg)))
(decl trap_impl (TrapCode) SideEffectNoResult)
(rule (trap_impl trap_code)
(SideEffectNoResult.Inst (MInst.Trap trap_code)))
(decl trap_if_impl (Cond TrapCode) SideEffectNoResult)
(rule (trap_if_impl cond trap_code)
(SideEffectNoResult.Inst (MInst.TrapIf cond trap_code)))
(decl debugtrap_impl () SideEffectNoResult)
(rule (debugtrap_impl)
(SideEffectNoResult.Inst (MInst.Debugtrap)))
(decl safepoint (SideEffectNoResult) ValueRegs)
(rule (safepoint (SideEffectNoResult.Inst inst))
(let ((_ Unit (emit_safepoint inst)))
(value_regs_invalid)))
;; Helpers for handling boolean conditions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; Helpers for handling boolean conditions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -2201,6 +2252,24 @@
(rule (lower_bool $B32 cond) (select_bool_imm $B32 cond -1 0)) (rule (lower_bool $B32 cond) (select_bool_imm $B32 cond -1 0))
(rule (lower_bool $B64 cond) (select_bool_imm $B64 cond -1 0)) (rule (lower_bool $B64 cond) (select_bool_imm $B64 cond -1 0))
;; Emit a conditional branch based on a boolean condition.
(decl cond_br_bool (ProducesBool MachLabel MachLabel) SideEffectNoResult)
(rule (cond_br_bool (ProducesBool.ProducesBool producer cond) taken not_taken)
(let ((_ Unit (emit_producer producer)))
(cond_br taken not_taken cond)))
;; Emit a one-way conditional branch based on a boolean condition.
(decl oneway_cond_br_bool (ProducesBool MachLabel) SideEffectNoResult)
(rule (oneway_cond_br_bool (ProducesBool.ProducesBool producer cond) dest)
(let ((_ Unit (emit_producer producer)))
(oneway_cond_br dest cond)))
;; Emit a conditional trap based on a boolean condition.
(decl trap_if_bool (ProducesBool TrapCode) SideEffectNoResult)
(rule (trap_if_bool (ProducesBool.ProducesBool producer cond) trap_code)
(let ((_ Unit (emit_producer producer)))
(trap_if_impl cond trap_code)))
;; Helpers for generating `clz` instructions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; Helpers for generating `clz` instructions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

153
cranelift/codegen/src/isa/s390x/lower.isle
View File

@@ -4,6 +4,11 @@
;; register(s) within which the lowered instruction's result values live. ;; register(s) within which the lowered instruction's result values live.
(decl lower (Inst) ValueRegs) (decl lower (Inst) ValueRegs)
;; A variant of the main lowering constructor term, used for branches.
;; The only difference is that it gets an extra argument holding a vector
;; of branch targets to be used.
(decl lower_branch (Inst VecMachLabel) ValueRegs)
;;;; Rules for `iconst` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;;; Rules for `iconst` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -1778,3 +1783,151 @@
(value_reg (select_bool_reg ty (icmp_val $false int_cc x y) (value_reg (select_bool_reg ty (icmp_val $false int_cc x y)
(put_in_reg val_true) (put_in_reg val_false)))) (put_in_reg val_true) (put_in_reg val_false))))
;;;; Rules for `jump` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Unconditional branch. The target is found as first (and only) element in
;; the list of the current block's branch targets passed as `targets`.
(rule (lower_branch (jump _ _) targets)
(value_regs_none (jump_impl (vec_element targets 0))))
;;;; Rules for `br_table` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Jump table. `targets` contains the default target followed by the
;; list of branch targets per index value.
(rule (lower_branch (br_table val_idx _ _) targets)
(let ((idx Reg (put_in_reg_zext64 val_idx))
;; Bounds-check the index and branch to default.
;; This is an internal branch that is not a terminator insn.
;; Instead, the default target is listed a potential target
;; in the final JTSequence, which is the block terminator.
(cond ProducesBool
(bool (icmpu_uimm32 $I64 idx (vec_length_minus1 targets))
(intcc_as_cond (IntCC.UnsignedGreaterThanOrEqual))))
(_ ValueRegs (value_regs_none (oneway_cond_br_bool cond
(vec_element targets 0)))))
;; Scale the index by the element size, and then emit the
;; compound instruction that does:
;;
;; larl %r1, <jt-base>
;; agf %r1, 0(%r1, %rScaledIndex)
;; br %r1
;; [jt entries]
;;
;; This must be *one* instruction in the vcode because
;; we cannot allow regalloc to insert any spills/fills
;; in the middle of the sequence; otherwise, the LARL's
;; PC-rel offset to the jumptable would be incorrect.
;; (The alternative is to introduce a relocation pass
;; for inlined jumptables, which is much worse, IMHO.)
(value_regs_none (jt_sequence (lshl_imm $I64 idx 2) targets))))
;;;; Rules for `brz` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Two-way conditional branch on zero. `targets` contains:
;; - element 0: target if the condition is true (i.e. value is zero)
;; - element 1: target if the condition is false (i.e. value is nonzero)
(rule (lower_branch (brz val_cond _ _) targets)
(value_regs_none (cond_br_bool (invert_bool (value_nonzero val_cond))
(vec_element targets 0)
(vec_element targets 1))))
;;;; Rules for `brnz` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Two-way conditional branch on nonzero. `targets` contains:
;; - element 0: target if the condition is true (i.e. value is nonzero)
;; - element 1: target if the condition is false (i.e. value is zero)
(rule (lower_branch (brnz val_cond _ _) targets)
(value_regs_none (cond_br_bool (value_nonzero val_cond)
(vec_element targets 0)
(vec_element targets 1))))
;;;; Rules for `brif` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Similarly to `selectif_spectre_guard`, we only recognize specific patterns
;; generated by common code here. Others will fail to lower.
(rule (lower_branch (brif int_cc (def_inst (ifcmp x y)) _ _) targets)
(value_regs_none (cond_br_bool (icmp_val $false int_cc x y)
(vec_element targets 0)
(vec_element targets 1))))
;;;; Rules for `trap` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(rule (lower (trap trap_code))
(safepoint (trap_impl trap_code)))
;;;; Rules for `resumable_trap` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(rule (lower (resumable_trap trap_code))
(safepoint (trap_impl trap_code)))
;;;; Rules for `trapz` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(rule (lower (trapz val trap_code))
(safepoint (trap_if_bool (invert_bool (value_nonzero val)) trap_code)))
;;;; Rules for `trapnz` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(rule (lower (trapnz val trap_code))
(safepoint (trap_if_bool (value_nonzero val) trap_code)))
;;;; Rules for `resumable_trapnz` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(rule (lower (resumable_trapnz val trap_code))
(safepoint (trap_if_bool (value_nonzero val) trap_code)))
;;;; Rules for `debugtrap` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(rule (lower (debugtrap))
(value_regs_none (debugtrap_impl)))
;;;; Rules for `trapif` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Similarly to `selectif_spectre_guard`, we only recognize specific patterns
;; generated by common code here. Others will fail to lower.
;; Recognize the case of `ifcmp` feeding into `trapif`. Directly generate
;; the desired comparison here; there is no separate `ifcmp` lowering.
(rule (lower (trapif int_cc (def_inst (ifcmp x y)) trap_code))
(safepoint (trap_if_bool (icmp_val $false int_cc x y) trap_code)))
;; Recognize the case of `iadd_ifcout` feeding into `trapif`. Note that
;; in the case, the `iadd_ifcout` is generated by a separate lowering
;; (in order to properly handle the register output of that instruction.)
;;
;; The flags must not have been clobbered by any other instruction between the
;; iadd_ifcout and this instruction, as verified by the CLIF validator; so we
;; can simply rely on the condition code here.
;;
;; IaddIfcout is implemented via a ADD LOGICAL instruction, which sets the
;; the condition code as follows:
;; 0 Result zero; no carry
;; 1 Result not zero; no carry
;; 2 Result zero; carry
;; 3 Result not zero; carry
;; This means "carry" corresponds to condition code 2 or 3, i.e.
;; a condition mask of 2 | 1.
;;
;; As this does not match any of the encodings used with a normal integer
;; comparsion, this cannot be represented by any IntCC value. We need to
;; remap the IntCC::UnsignedGreaterThan value that we have here as result
;; of the unsigned_add_overflow_condition call to the correct mask.
(rule (lower (trapif (IntCC.UnsignedGreaterThan)
(def_inst (iadd_ifcout x y)) trap_code))
(value_regs_none (trap_if_impl (mask_as_cond 3) trap_code)))

788
cranelift/codegen/src/isa/s390x/lower.rs
View File

@@ -2,7 +2,7 @@
use crate::ir::condcodes::IntCC; use crate::ir::condcodes::IntCC;
use crate::ir::Inst as IRInst; use crate::ir::Inst as IRInst;
use crate::ir::{types, Endianness, MemFlags, Opcode, Type}; use crate::ir::{MemFlags, Opcode};
use crate::isa::s390x::abi::*; use crate::isa::s390x::abi::*;
use crate::isa::s390x::inst::*; use crate::isa::s390x::inst::*;
use crate::isa::s390x::settings as s390x_settings; use crate::isa::s390x::settings as s390x_settings;
@@ -11,255 +11,14 @@ use crate::machinst::lower::*;
use crate::machinst::*; use crate::machinst::*;
use crate::settings::Flags; use crate::settings::Flags;
use crate::CodegenResult; use crate::CodegenResult;
use alloc::boxed::Box; use regalloc::Reg;
use core::convert::TryFrom;
use regalloc::{Reg, Writable};
use smallvec::SmallVec; use smallvec::SmallVec;
pub mod isle; pub mod isle;
//=============================================================================
// Helpers for instruction lowering.
fn choose_32_64<T: Copy>(ty: Type, op32: T, op64: T) -> T {
let bits = ty_bits(ty);
if bits <= 32 {
op32
} else if bits == 64 {
op64
} else {
panic!("choose_32_64 on > 64 bits!")
}
}
//============================================================================
// Lowering: convert instruction inputs to forms that we can use.
/// Lower an instruction input to a 64-bit constant, if possible.
fn input_matches_const<C: LowerCtx<I = Inst>>(ctx: &mut C, input: InsnInput) -> Option<u64> {
let input = ctx.get_input_as_source_or_const(input.insn, input.input);
input.constant
}
/// Lower an instruction input to a 64-bit signed constant, if possible.
fn input_matches_sconst<C: LowerCtx<I = Inst>>(ctx: &mut C, input: InsnInput) -> Option<i64> {
if let Some(imm) = input_matches_const(ctx, input) {
let ty = ctx.input_ty(input.insn, input.input);
Some(sign_extend_to_u64(imm, ty_bits(ty) as u8) as i64)
} else {
None
}
}
/// Lower an instruction input to a 16-bit signed constant, if possible.
fn input_matches_simm16<C: LowerCtx<I = Inst>>(ctx: &mut C, input: InsnInput) -> Option<i16> {
if let Some(imm_value) = input_matches_sconst(ctx, input) {
if let Ok(imm) = i16::try_from(imm_value) {
return Some(imm);
}
}
None
}
/// Lower an instruction input to a 32-bit signed constant, if possible.
fn input_matches_simm32<C: LowerCtx<I = Inst>>(ctx: &mut C, input: InsnInput) -> Option<i32> {
if let Some(imm_value) = input_matches_sconst(ctx, input) {
if let Ok(imm) = i32::try_from(imm_value) {
return Some(imm);
}
}
None
}
/// Lower an instruction input to a 32-bit unsigned constant, if possible.
fn input_matches_uimm32<C: LowerCtx<I = Inst>>(ctx: &mut C, input: InsnInput) -> Option<u32> {
if let Some(imm_value) = input_matches_const(ctx, input) {
if let Ok(imm) = u32::try_from(imm_value) {
return Some(imm);
}
}
None
}
/// Checks for an instance of `op` feeding the given input.
fn input_matches_insn<C: LowerCtx<I = Inst>>(
c: &mut C,
input: InsnInput,
op: Opcode,
) -> Option<IRInst> {
let inputs = c.get_input_as_source_or_const(input.insn, input.input);
if let Some((src_inst, _)) = inputs.inst {
let data = c.data(src_inst);
if data.opcode() == op {
return Some(src_inst);
}
}
None
}
/// Checks for an instance of `op` feeding the given input, possibly via a conversion `conv` (e.g.,
/// Bint or a bitcast).
fn input_matches_insn_via_conv<C: LowerCtx<I = Inst>>(
c: &mut C,
input: InsnInput,
op: Opcode,
conv: Opcode,
) -> Option<IRInst> {
let inputs = c.get_input_as_source_or_const(input.insn, input.input);
if let Some((src_inst, _)) = inputs.inst {
let data = c.data(src_inst);
if data.opcode() == op {
return Some(src_inst);
}
if data.opcode() == conv {
let inputs = c.get_input_as_source_or_const(src_inst, 0);
if let Some((src_inst, _)) = inputs.inst {
let data = c.data(src_inst);
if data.opcode() == op {
return Some(src_inst);
}
}
}
}
None
}
fn input_matches_load_insn<C: LowerCtx<I = Inst>>(
ctx: &mut C,
input: InsnInput,
op: Opcode,
) -> Option<MemArg> {
if let Some(insn) = input_matches_insn(ctx, input, op) {
let inputs: SmallVec<[InsnInput; 4]> = (0..ctx.num_inputs(insn))
.map(|i| InsnInput { insn, input: i })
.collect();
let off = ctx.data(insn).load_store_offset().unwrap();
let flags = ctx.memflags(insn).unwrap();
let endianness = flags.endianness(Endianness::Big);
if endianness == Endianness::Big {
let mem = lower_address(ctx, &inputs[..], off, flags);
ctx.sink_inst(insn);
return Some(mem);
}
}
None
}
fn input_matches_mem<C: LowerCtx<I = Inst>>(ctx: &mut C, input: InsnInput) -> Option<MemArg> {
if ty_bits(ctx.input_ty(input.insn, input.input)) >= 32 {
return input_matches_load_insn(ctx, input, Opcode::Load);
}
None
}
fn input_matches_sext16_mem<C: LowerCtx<I = Inst>>(
ctx: &mut C,
input: InsnInput,
) -> Option<MemArg> {
if ty_bits(ctx.input_ty(input.insn, input.input)) == 16 {
return input_matches_load_insn(ctx, input, Opcode::Load);
}
if ty_bits(ctx.input_ty(input.insn, input.input)) >= 32 {
return input_matches_load_insn(ctx, input, Opcode::Sload16);
}
None
}
fn input_matches_sext32_mem<C: LowerCtx<I = Inst>>(
ctx: &mut C,
input: InsnInput,
) -> Option<MemArg> {
if ty_bits(ctx.input_ty(input.insn, input.input)) > 32 {
return input_matches_load_insn(ctx, input, Opcode::Sload32);
}
None
}
fn input_matches_sext32_reg<C: LowerCtx<I = Inst>>(ctx: &mut C, input: InsnInput) -> Option<Reg> {
if let Some(insn) = input_matches_insn(ctx, input, Opcode::Sextend) {
if ty_bits(ctx.input_ty(insn, 0)) == 32 {
let reg = put_input_in_reg(ctx, InsnInput { insn, input: 0 }, NarrowValueMode::None);
return Some(reg);
}
}
None
}
fn input_matches_uext32_reg<C: LowerCtx<I = Inst>>(ctx: &mut C, input: InsnInput) -> Option<Reg> {
if let Some(insn) = input_matches_insn(ctx, input, Opcode::Uextend) {
if ty_bits(ctx.input_ty(insn, 0)) == 32 {
let reg = put_input_in_reg(ctx, InsnInput { insn, input: 0 }, NarrowValueMode::None);
return Some(reg);
}
}
None
}
fn input_matches_uext16_mem<C: LowerCtx<I = Inst>>(
ctx: &mut C,
input: InsnInput,
) -> Option<MemArg> {
if ty_bits(ctx.input_ty(input.insn, input.input)) == 16 {
return input_matches_load_insn(ctx, input, Opcode::Load);
}
if ty_bits(ctx.input_ty(input.insn, input.input)) >= 32 {
return input_matches_load_insn(ctx, input, Opcode::Uload16);
}
None
}
fn input_matches_uext32_mem<C: LowerCtx<I = Inst>>(
ctx: &mut C,
input: InsnInput,
) -> Option<MemArg> {
if ty_bits(ctx.input_ty(input.insn, input.input)) > 32 {
return input_matches_load_insn(ctx, input, Opcode::Uload32);
}
None
}
//============================================================================ //============================================================================
// Lowering: force instruction input into a register // Lowering: force instruction input into a register
/// How to handle narrow values loaded into registers; see note on `narrow_mode`
/// parameter to `put_input_in_*` below.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum NarrowValueMode {
None,
/// Zero-extend to 32 bits if original is < 32 bits.
ZeroExtend32,
/// Sign-extend to 32 bits if original is < 32 bits.
SignExtend32,
/// Zero-extend to 64 bits if original is < 64 bits.
ZeroExtend64,
/// Sign-extend to 64 bits if original is < 64 bits.
SignExtend64,
}
fn extend_memory_to_reg<C: LowerCtx<I = Inst>>(
ctx: &mut C,
mem: MemArg,
from_ty: Type,
to_ty: Type,
signed: bool,
) -> Reg {
let rd = ctx.alloc_tmp(to_ty).only_reg().unwrap();
ctx.emit(match (signed, ty_bits(to_ty), ty_bits(from_ty)) {
(false, 32, 8) => Inst::Load32ZExt8 { rd, mem },
(false, 32, 16) => Inst::Load32ZExt16 { rd, mem },
(true, 32, 8) => Inst::Load32SExt8 { rd, mem },
(true, 32, 16) => Inst::Load32SExt16 { rd, mem },
(false, 64, 8) => Inst::Load64ZExt8 { rd, mem },
(false, 64, 16) => Inst::Load64ZExt16 { rd, mem },
(false, 64, 32) => Inst::Load64ZExt32 { rd, mem },
(true, 64, 8) => Inst::Load64SExt8 { rd, mem },
(true, 64, 16) => Inst::Load64SExt16 { rd, mem },
(true, 64, 32) => Inst::Load64SExt32 { rd, mem },
_ => panic!("Unsupported size in load"),
});
rd.to_reg()
}
/// Sign-extend the low `from_bits` bits of `value` to a full u64. /// Sign-extend the low `from_bits` bits of `value` to a full u64.
fn sign_extend_to_u64(value: u64, from_bits: u8) -> u64 { fn sign_extend_to_u64(value: u64, from_bits: u8) -> u64 {
assert!(from_bits <= 64); assert!(from_bits <= 64);
@@ -270,169 +29,11 @@ fn sign_extend_to_u64(value: u64, from_bits: u8) -> u64 {
} }
} }
/// Zero-extend the low `from_bits` bits of `value` to a full u64.
fn zero_extend_to_u64(value: u64, from_bits: u8) -> u64 {
assert!(from_bits <= 64);
if from_bits >= 64 {
value
} else {
value & ((1u64 << from_bits) - 1)
}
}
/// Lower an instruction input to a reg. /// Lower an instruction input to a reg.
/// fn put_input_in_reg<C: LowerCtx<I = Inst>>(ctx: &mut C, input: InsnInput) -> Reg {
/// The given register will be extended appropriately, according to
/// `narrow_mode` and the input's type.
fn put_input_in_reg<C: LowerCtx<I = Inst>>(
ctx: &mut C,
input: InsnInput,
narrow_mode: NarrowValueMode,
) -> Reg {
let signed = match narrow_mode {
NarrowValueMode::SignExtend32 | NarrowValueMode::SignExtend64 => true,
NarrowValueMode::ZeroExtend32 | NarrowValueMode::ZeroExtend64 => false,
_ => false,
};
let ty = ctx.input_ty(input.insn, input.input);
let from_bits = ty_bits(ty) as u8;
let ext_ty = match narrow_mode {
NarrowValueMode::None => ty,
NarrowValueMode::ZeroExtend32 | NarrowValueMode::SignExtend32 => types::I32,
NarrowValueMode::ZeroExtend64 | NarrowValueMode::SignExtend64 => types::I64,
};
let to_bits = ty_bits(ext_ty) as u8;
assert!(to_bits >= from_bits);
if let Some(c) = input_matches_const(ctx, input) {
let extended = if from_bits == to_bits {
c
} else if signed {
sign_extend_to_u64(c, from_bits)
} else {
zero_extend_to_u64(c, from_bits)
};
let masked = zero_extend_to_u64(extended, to_bits);
// Generate constants fresh at each use to minimize long-range register pressure.
let to_reg = ctx.alloc_tmp(ext_ty).only_reg().unwrap();
for inst in Inst::gen_constant(ValueRegs::one(to_reg), masked as u128, ext_ty, |ty| {
ctx.alloc_tmp(ty).only_reg().unwrap()
})
.into_iter()
{
ctx.emit(inst);
}
to_reg.to_reg()
} else if to_bits == from_bits {
ctx.put_input_in_regs(input.insn, input.input) ctx.put_input_in_regs(input.insn, input.input)
.only_reg() .only_reg()
.unwrap() .unwrap()
} else if let Some(mem) = input_matches_load_insn(ctx, input, Opcode::Load) {
extend_memory_to_reg(ctx, mem, ty, ext_ty, signed)
} else {
let rd = ctx.alloc_tmp(ext_ty).only_reg().unwrap();
let rn = ctx
.put_input_in_regs(input.insn, input.input)
.only_reg()
.unwrap();
ctx.emit(Inst::Extend {
rd,
rn,
signed,
from_bits,
to_bits,
});
rd.to_reg()
}
}
//============================================================================
// Lowering: addressing mode support. Takes instruction directly, rather
// than an `InsnInput`, to do more introspection.
/// Lower the address of a load or store.
fn lower_address<C: LowerCtx<I = Inst>>(
ctx: &mut C,
addends: &[InsnInput],
offset: i32,
flags: MemFlags,
) -> MemArg {
// Handle one reg and offset.
if addends.len() == 1 {
if offset == 0 {
if let Some(add) = input_matches_insn(ctx, addends[0], Opcode::Iadd) {
debug_assert_eq!(ctx.output_ty(add, 0), types::I64);
let add_inputs = &[
InsnInput {
insn: add,
input: 0,
},
InsnInput {
insn: add,
input: 1,
},
];
let ra = put_input_in_reg(ctx, add_inputs[0], NarrowValueMode::None);
let rb = put_input_in_reg(ctx, add_inputs[1], NarrowValueMode::None);
return MemArg::reg_plus_reg(ra, rb, flags);
}
}
if let Some(symbol) = input_matches_insn(ctx, addends[0], Opcode::SymbolValue) {
let (extname, dist, ext_offset) = ctx.symbol_value(symbol).unwrap();
let ext_offset = ext_offset + i64::from(offset);
if dist == RelocDistance::Near && (ext_offset & 1) == 0 {
if let Ok(offset) = i32::try_from(ext_offset) {
return MemArg::Symbol {
name: Box::new(extname.clone()),
offset,
flags,
};
}
}
}
let reg = put_input_in_reg(ctx, addends[0], NarrowValueMode::None);
return MemArg::reg_plus_off(reg, offset as i64, flags);
}
// Handle two regs and a zero offset.
if addends.len() == 2 && offset == 0 {
let ra = put_input_in_reg(ctx, addends[0], NarrowValueMode::None);
let rb = put_input_in_reg(ctx, addends[1], NarrowValueMode::None);
return MemArg::reg_plus_reg(ra, rb, flags);
}
// Otherwise, generate add instructions.
let addr = ctx.alloc_tmp(types::I64).only_reg().unwrap();
// Get the const into a reg.
lower_constant_u64(ctx, addr.clone(), offset as u64);
// Add each addend to the address.
for addend in addends {
let reg = put_input_in_reg(ctx, *addend, NarrowValueMode::None);
ctx.emit(Inst::AluRRR {
alu_op: ALUOp::Add64,
rd: addr.clone(),
rn: addr.to_reg(),
rm: reg.clone(),
});
}
MemArg::reg(addr.to_reg(), flags)
}
//============================================================================
// Lowering: generating constants.
fn lower_constant_u64<C: LowerCtx<I = Inst>>(ctx: &mut C, rd: Writable<Reg>, value: u64) {
for inst in Inst::load_constant64(rd, value) {
ctx.emit(inst);
}
} }
//============================================================================= //=============================================================================
@@ -459,151 +60,6 @@ pub fn condcode_is_signed(cc: IntCC) -> bool {
} }
} }
fn lower_icmp_to_flags<C: LowerCtx<I = Inst>>(
ctx: &mut C,
insn: IRInst,
is_signed: bool,
may_sink_memory: bool,
) {
let ty = ctx.input_ty(insn, 0);
let bits = ty_bits(ty);
let narrow_mode = match (bits <= 32, is_signed) {
(true, true) => NarrowValueMode::SignExtend32,
(true, false) => NarrowValueMode::ZeroExtend32,
(false, true) => NarrowValueMode::SignExtend64,
(false, false) => NarrowValueMode::ZeroExtend64,
};
let inputs = [InsnInput { insn, input: 0 }, InsnInput { insn, input: 1 }];
let ty = ctx.input_ty(insn, 0);
let rn = put_input_in_reg(ctx, inputs[0], narrow_mode);
if is_signed {
let op = choose_32_64(ty, CmpOp::CmpS32, CmpOp::CmpS64);
// Try matching immedate operand.
if let Some(imm) = input_matches_simm16(ctx, inputs[1]) {
return ctx.emit(Inst::CmpRSImm16 { op, rn, imm });
}
if let Some(imm) = input_matches_simm32(ctx, inputs[1]) {
return ctx.emit(Inst::CmpRSImm32 { op, rn, imm });
}
// If sinking memory loads is allowed, try matching memory operand.
if may_sink_memory {
if let Some(mem) = input_matches_mem(ctx, inputs[1]) {
return ctx.emit(Inst::CmpRX { op, rn, mem });
}
if let Some(mem) = input_matches_sext16_mem(ctx, inputs[1]) {
let op = choose_32_64(ty, CmpOp::CmpS32Ext16, CmpOp::CmpS64Ext16);
return ctx.emit(Inst::CmpRX { op, rn, mem });
}
if let Some(mem) = input_matches_sext32_mem(ctx, inputs[1]) {
return ctx.emit(Inst::CmpRX {
op: CmpOp::CmpS64Ext32,
rn,
mem,
});
}
}
// Try matching sign-extension in register.
if let Some(rm) = input_matches_sext32_reg(ctx, inputs[1]) {
return ctx.emit(Inst::CmpRR {
op: CmpOp::CmpS64Ext32,
rn,
rm,
});
}
// If no special case matched above, fall back to a register compare.
let rm = put_input_in_reg(ctx, inputs[1], narrow_mode);
return ctx.emit(Inst::CmpRR { op, rn, rm });
} else {
let op = choose_32_64(ty, CmpOp::CmpL32, CmpOp::CmpL64);
// Try matching immedate operand.
if let Some(imm) = input_matches_uimm32(ctx, inputs[1]) {
return ctx.emit(Inst::CmpRUImm32 { op, rn, imm });
}
// If sinking memory loads is allowed, try matching memory operand.
if may_sink_memory {
if let Some(mem) = input_matches_mem(ctx, inputs[1]) {
return ctx.emit(Inst::CmpRX { op, rn, mem });
}
if let Some(mem) = input_matches_uext16_mem(ctx, inputs[1]) {
match &mem {
&MemArg::Symbol { .. } => {
let op = choose_32_64(ty, CmpOp::CmpL32Ext16, CmpOp::CmpL64Ext16);
return ctx.emit(Inst::CmpRX { op, rn, mem });
}
_ => {
let reg_ty = choose_32_64(ty, types::I32, types::I64);
let rm = extend_memory_to_reg(ctx, mem, types::I16, reg_ty, false);
return ctx.emit(Inst::CmpRR { op, rn, rm });
}
}
}
if let Some(mem) = input_matches_uext32_mem(ctx, inputs[1]) {
return ctx.emit(Inst::CmpRX {
op: CmpOp::CmpL64Ext32,
rn,
mem,
});
}
}
// Try matching zero-extension in register.
if let Some(rm) = input_matches_uext32_reg(ctx, inputs[1]) {
return ctx.emit(Inst::CmpRR {
op: CmpOp::CmpL64Ext32,
rn,
rm,
});
}
// If no special case matched above, fall back to a register compare.
let rm = put_input_in_reg(ctx, inputs[1], narrow_mode);
return ctx.emit(Inst::CmpRR { op, rn, rm });
}
}
fn lower_fcmp_to_flags<C: LowerCtx<I = Inst>>(ctx: &mut C, insn: IRInst) {
let ty = ctx.input_ty(insn, 0);
let bits = ty_bits(ty);
let inputs = [InsnInput { insn, input: 0 }, InsnInput { insn, input: 1 }];
let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
let rm = put_input_in_reg(ctx, inputs[1], NarrowValueMode::None);
match bits {
32 => {
ctx.emit(Inst::FpuCmp32 { rn, rm });
}
64 => {
ctx.emit(Inst::FpuCmp64 { rn, rm });
}
_ => panic!("Unknown float size"),
}
}
fn lower_boolean_to_flags<C: LowerCtx<I = Inst>>(ctx: &mut C, input: InsnInput) -> Cond {
if let Some(icmp_insn) = input_matches_insn_via_conv(ctx, input, Opcode::Icmp, Opcode::Bint) {
// FIXME: If the Icmp (and Bint) only have a single use, we can still allow sinking memory
let may_sink_memory = false;
let condcode = ctx.data(icmp_insn).cond_code().unwrap();
let is_signed = condcode_is_signed(condcode);
lower_icmp_to_flags(ctx, icmp_insn, is_signed, may_sink_memory);
Cond::from_intcc(condcode)
} else if let Some(fcmp_insn) =
input_matches_insn_via_conv(ctx, input, Opcode::Fcmp, Opcode::Bint)
{
let condcode = ctx.data(fcmp_insn).fp_cond_code().unwrap();
lower_fcmp_to_flags(ctx, fcmp_insn);
Cond::from_floatcc(condcode)
} else {
let ty = ctx.input_ty(input.insn, input.input);
let narrow_mode = if ty.bits() < 32 {
NarrowValueMode::ZeroExtend32
} else {
NarrowValueMode::None
};
let rn = put_input_in_reg(ctx, input, narrow_mode);
let op = choose_32_64(ty, CmpOp::CmpS32, CmpOp::CmpS64);
ctx.emit(Inst::CmpRSImm16 { op, rn, imm: 0 });
Cond::from_intcc(IntCC::NotEqual)
}
}
//============================================================================ //============================================================================
// Lowering: main entry point for lowering a instruction // Lowering: main entry point for lowering a instruction
@@ -728,6 +184,13 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
| Opcode::IsInvalid | Opcode::IsInvalid
| Opcode::Select | Opcode::Select
| Opcode::SelectifSpectreGuard | Opcode::SelectifSpectreGuard
| Opcode::Trap
| Opcode::ResumableTrap
| Opcode::Trapz
| Opcode::Trapnz
| Opcode::ResumableTrapnz
| Opcode::Trapif
| Opcode::Debugtrap
| Opcode::StackAddr | Opcode::StackAddr
| Opcode::FuncAddr | Opcode::FuncAddr
| Opcode::SymbolValue => implemented_in_isle(), | Opcode::SymbolValue => implemented_in_isle(),
@@ -765,63 +228,6 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
unimplemented!("Pinned register support not implemented!"); unimplemented!("Pinned register support not implemented!");
} }
Opcode::Trap | Opcode::ResumableTrap => {
let trap_code = ctx.data(insn).trap_code().unwrap();
ctx.emit_safepoint(Inst::Trap { trap_code })
}
Opcode::Trapz | Opcode::Trapnz | Opcode::ResumableTrapnz => {
let cond = lower_boolean_to_flags(ctx, inputs[0]);
let negated = op == Opcode::Trapz;
let cond = if negated { cond.invert() } else { cond };
let trap_code = ctx.data(insn).trap_code().unwrap();
ctx.emit_safepoint(Inst::TrapIf { trap_code, cond });
}
Opcode::Trapif => {
let condcode = ctx.data(insn).cond_code().unwrap();
let mut cond = Cond::from_intcc(condcode);
let is_signed = condcode_is_signed(condcode);
let cmp_insn = ctx
.get_input_as_source_or_const(inputs[0].insn, inputs[0].input)
.inst
.unwrap()
.0;
if ctx.data(cmp_insn).opcode() == Opcode::IaddIfcout {
// The flags must not have been clobbered by any other instruction between the
// iadd_ifcout and this instruction, as verified by the CLIF validator; so we
// can simply rely on the condition code here.
//
// IaddIfcout is implemented via a ADD LOGICAL instruction, which sets the
// the condition code as follows:
// 0 Result zero; no carry
// 1 Result not zero; no carry
// 2 Result zero; carry
// 3 Result not zero; carry
// This means "carry" corresponds to condition code 2 or 3, i.e.
// a condition mask of 2 | 1.
//
// As this does not match any of the encodings used with a normal integer
// comparsion, this cannot be represented by any IntCC value. We need to
// remap the IntCC::UnsignedGreaterThan value that we have here as result
// of the unsigned_add_overflow_condition call to the correct mask.
assert!(condcode == IntCC::UnsignedGreaterThan);
cond = Cond::from_mask(2 | 1);
} else {
// Verification ensures that the input is always a single-def ifcmp
debug_assert_eq!(ctx.data(cmp_insn).opcode(), Opcode::Ifcmp);
lower_icmp_to_flags(ctx, cmp_insn, is_signed, true);
}
let trap_code = ctx.data(insn).trap_code().unwrap();
ctx.emit_safepoint(Inst::TrapIf { trap_code, cond });
}
Opcode::Debugtrap => {
ctx.emit(Inst::Debugtrap);
}
Opcode::Call | Opcode::CallIndirect => { Opcode::Call | Opcode::CallIndirect => {
let caller_conv = ctx.abi().call_conv(); let caller_conv = ctx.abi().call_conv();
let (mut abi, inputs) = match op { let (mut abi, inputs) = match op {
@@ -837,7 +243,7 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
) )
} }
Opcode::CallIndirect => { Opcode::CallIndirect => {
let ptr = put_input_in_reg(ctx, inputs[0], NarrowValueMode::ZeroExtend64); let ptr = put_input_in_reg(ctx, inputs[0]);
let sig = ctx.call_sig(insn).unwrap(); let sig = ctx.call_sig(insn).unwrap();
assert!(inputs.len() - 1 == sig.params.len()); assert!(inputs.len() - 1 == sig.params.len());
assert!(outputs.len() == sig.returns.len()); assert!(outputs.len() == sig.returns.len());
@@ -851,7 +257,7 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
assert!(inputs.len() == abi.num_args()); assert!(inputs.len() == abi.num_args());
for (i, input) in inputs.iter().enumerate() { for (i, input) in inputs.iter().enumerate() {
let arg_reg = put_input_in_reg(ctx, *input, NarrowValueMode::None); let arg_reg = put_input_in_reg(ctx, *input);
abi.emit_copy_regs_to_arg(ctx, i, ValueRegs::one(arg_reg)); abi.emit_copy_regs_to_arg(ctx, i, ValueRegs::one(arg_reg));
} }
abi.emit_call(ctx); abi.emit_call(ctx);
@@ -864,7 +270,7 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
Opcode::FallthroughReturn | Opcode::Return => { Opcode::FallthroughReturn | Opcode::Return => {
for (i, input) in inputs.iter().enumerate() { for (i, input) in inputs.iter().enumerate() {
let reg = put_input_in_reg(ctx, *input, NarrowValueMode::None); let reg = put_input_in_reg(ctx, *input);
let retval_reg = ctx.retval(i).only_reg().unwrap(); let retval_reg = ctx.retval(i).only_reg().unwrap();
let ty = ctx.input_ty(insn, i); let ty = ctx.input_ty(insn, i);
ctx.emit(Inst::gen_move(retval_reg, reg, ty)); ctx.emit(Inst::gen_move(retval_reg, reg, ty));
@@ -995,146 +401,6 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
Ok(()) Ok(())
} }
//============================================================================
// Lowering: main entry point for lowering a branch group
fn lower_branch<C: LowerCtx<I = Inst>>(
ctx: &mut C,
branches: &[IRInst],
targets: &[MachLabel],
) -> CodegenResult<()> {
// A block should end with at most two branches. The first may be a
// conditional branch; a conditional branch can be followed only by an
// unconditional branch or fallthrough. Otherwise, if only one branch,
// it may be an unconditional branch, a fallthrough, a return, or a
// trap. These conditions are verified by `is_ebb_basic()` during the
// verifier pass.
assert!(branches.len() <= 2);
if branches.len() == 2 {
// Must be a conditional branch followed by an unconditional branch.
let op0 = ctx.data(branches[0]).opcode();
let op1 = ctx.data(branches[1]).opcode();
assert!(op1 == Opcode::Jump);
let taken = targets[0];
let not_taken = targets[1];
match op0 {
Opcode::Brz | Opcode::Brnz => {
let flag_input = InsnInput {
insn: branches[0],
input: 0,
};
let cond = lower_boolean_to_flags(ctx, flag_input);
let negated = op0 == Opcode::Brz;
let cond = if negated { cond.invert() } else { cond };
ctx.emit(Inst::CondBr {
taken,
not_taken,
cond,
});
}
Opcode::Brif => {
let condcode = ctx.data(branches[0]).cond_code().unwrap();
let cond = Cond::from_intcc(condcode);
let is_signed = condcode_is_signed(condcode);
// Verification ensures that the input is always a single-def ifcmp.
let cmp_insn = ctx
.get_input_as_source_or_const(branches[0], 0)
.inst
.unwrap()
.0;
debug_assert_eq!(ctx.data(cmp_insn).opcode(), Opcode::Ifcmp);
lower_icmp_to_flags(ctx, cmp_insn, is_signed, true);
ctx.emit(Inst::CondBr {
taken,
not_taken,
cond,
});
}
Opcode::Brff => unreachable!(),
_ => unimplemented!(),
}
} else {
// Must be an unconditional branch or an indirect branch.
let op = ctx.data(branches[0]).opcode();
match op {
Opcode::Jump => {
assert!(branches.len() == 1);
ctx.emit(Inst::Jump { dest: targets[0] });
}
Opcode::BrTable => {
let jt_size = targets.len() - 1;
assert!(jt_size <= std::u32::MAX as usize);
// Load up jump table element index.
let ridx = put_input_in_reg(
ctx,
InsnInput {
insn: branches[0],
input: 0,
},
NarrowValueMode::ZeroExtend64,
);
// Bounds-check index and branch to default.
// This is an internal branch that is not a terminator insn.
// Instead, the default target is listed a potential target
// in the final JTSequence, which is the block terminator.
ctx.emit(Inst::CmpRUImm32 {
op: CmpOp::CmpL64,
rn: ridx,
imm: jt_size as u32,
});
ctx.emit(Inst::OneWayCondBr {
target: targets[0],
cond: Cond::from_intcc(IntCC::UnsignedGreaterThanOrEqual),
});
// Compute index scaled by entry size.
let rtmp = ctx.alloc_tmp(types::I64).only_reg().unwrap();
ctx.emit(Inst::ShiftRR {
shift_op: ShiftOp::LShL64,
rd: rtmp,
rn: ridx,
shift_imm: 2,
shift_reg: zero_reg(),
});
// Emit the compound instruction that does:
//
// larl %r1, <jt-base>
// agf %r1, 0(%r1, %rTmp)
// br %r1
// [jt entries]
//
// This must be *one* instruction in the vcode because
// we cannot allow regalloc to insert any spills/fills
// in the middle of the sequence; otherwise, the ADR's
// PC-rel offset to the jumptable would be incorrect.
// (The alternative is to introduce a relocation pass
// for inlined jumptables, which is much worse, IMHO.)
ctx.emit(Inst::JTSequence {
ridx: rtmp.to_reg(),
targets: targets.to_vec(),
});
}
_ => panic!("Unknown branch type!"),
}
}
Ok(())
}
//============================================================================= //=============================================================================
// Lowering-backend trait implementation. // Lowering-backend trait implementation.
@@ -1151,6 +417,32 @@ impl LowerBackend for S390xBackend {
branches: &[IRInst], branches: &[IRInst],
targets: &[MachLabel], targets: &[MachLabel],
) -> CodegenResult<()> { ) -> CodegenResult<()> {
lower_branch(ctx, branches, targets) // A block should end with at most two branches. The first may be a
// conditional branch; a conditional branch can be followed only by an
// unconditional branch or fallthrough. Otherwise, if only one branch,
// it may be an unconditional branch, a fallthrough, a return, or a
// trap. These conditions are verified by `is_ebb_basic()` during the
// verifier pass.
assert!(branches.len() <= 2);
if branches.len() == 2 {
let op1 = ctx.data(branches[1]).opcode();
assert!(op1 == Opcode::Jump);
}
// Lower the first branch in ISLE. This will automatically handle
// the second branch (if any) by emitting a two-way conditional branch.
if let Ok(()) = super::lower::isle::lower_branch(
ctx,
&self.flags,
&self.isa_flags,
branches[0],
targets,
) {
return Ok(());
}
unreachable!(
"implemented in ISLE: branch = `{}`",
ctx.dfg().display_inst(branches[0]),
);
} }
} }

37
cranelift/codegen/src/isa/s390x/lower/isle.rs
View File

@@ -51,6 +51,28 @@ where
) )
} }
/// The main entry point for branch lowering with ISLE.
pub(crate) fn lower_branch<C>(
lower_ctx: &mut C,
flags: &Flags,
isa_flags: &IsaFlags,
branch: Inst,
targets: &[MachLabel],
) -> Result<(), ()>
where
C: LowerCtx<I = MInst>,
{
lower_common(
lower_ctx,
flags,
isa_flags,
&[],
branch,
|cx, insn| generated_code::constructor_lower_branch(cx, insn, &targets.to_vec()),
s390x_map_regs,
)
}
impl<C> generated_code::Context for IsleContext<'_, C, Flags, IsaFlags, 6> impl<C> generated_code::Context for IsleContext<'_, C, Flags, IsaFlags, 6>
where where
C: LowerCtx<I = MInst>, C: LowerCtx<I = MInst>,
@@ -369,6 +391,16 @@ where
} }
} }
#[inline]
fn vec_length_minus1(&mut self, vec: &VecMachLabel) -> u32 {
u32::try_from(vec.len()).unwrap() - 1
}
#[inline]
fn vec_element(&mut self, vec: &VecMachLabel, index: u8) -> MachLabel {
vec[usize::from(index)]
}
#[inline] #[inline]
fn reloc_distance_near(&mut self, dist: &RelocDistance) -> Option<()> { fn reloc_distance_near(&mut self, dist: &RelocDistance) -> Option<()> {
if *dist == RelocDistance::Near { if *dist == RelocDistance::Near {
@@ -471,4 +503,9 @@ where
fn emit(&mut self, inst: &MInst) -> Unit { fn emit(&mut self, inst: &MInst) -> Unit {
self.emitted_insts.push((inst.clone(), false)); self.emitted_insts.push((inst.clone(), false));
} }
#[inline]
fn emit_safepoint(&mut self, inst: &MInst) -> Unit {
self.emitted_insts.push((inst.clone(), true));
}
} }

4
cranelift/codegen/src/isa/s390x/lower/isle/generated_code.manifest
View File

@@ -1,4 +1,4 @@
src/clif.isle 9ea75a6f790b5c03 src/clif.isle 9ea75a6f790b5c03
src/prelude.isle 51d2aef2566c1c96 src/prelude.isle 51d2aef2566c1c96
src/isa/s390x/inst.isle 17b77476355c4509 src/isa/s390x/inst.isle d7bfd05fb4d4a66d
src/isa/s390x/lower.isle a0e21a567040bc33 src/isa/s390x/lower.isle 57dcc39cbab2d1c6

1921
cranelift/codegen/src/isa/s390x/lower/isle/generated_code.rs generated
View File

File diff suppressed because it is too large Load Diff

5
cranelift/codegen/src/machinst/isle.rs
View File

@@ -312,18 +312,19 @@ where
/// The `isle_lower` argument here is an ISLE-generated function for `lower` and /// The `isle_lower` argument here is an ISLE-generated function for `lower` and
/// then this function otherwise handles register mapping and such around the /// then this function otherwise handles register mapping and such around the
/// lowering. /// lowering.
pub(crate) fn lower_common<C, F, I, const N: usize>( pub(crate) fn lower_common<C, F, I, IF, const N: usize>(
lower_ctx: &mut C, lower_ctx: &mut C,
flags: &F, flags: &F,
isa_flags: &I, isa_flags: &I,
outputs: &[InsnOutput], outputs: &[InsnOutput],
inst: Inst, inst: Inst,
isle_lower: fn(&mut IsleContext<'_, C, F, I, N>, Inst) -> Option<ValueRegs>, isle_lower: IF,
map_regs: fn(&mut C::I, &RegRenamer), map_regs: fn(&mut C::I, &RegRenamer),
) -> Result<(), ()> ) -> Result<(), ()>
where where
C: LowerCtx, C: LowerCtx,
[(C::I, bool); N]: smallvec::Array<Item = (C::I, bool)>, [(C::I, bool); N]: smallvec::Array<Item = (C::I, bool)>,
IF: Fn(&mut IsleContext<'_, C, F, I, N>, Inst) -> Option<ValueRegs>,
{ {
// TODO: reuse the ISLE context across lowerings so we can reuse its // TODO: reuse the ISLE context across lowerings so we can reuse its
// internal heap allocations. // internal heap allocations.