s390x: Add z14 support

* Add support for processor features (including auto-detection). * Move base architecture set requirement back to z14. * Add z15 feature sets and re-enable z15-specific code generation when required features are available.
2021-06-16 14:58:21 +02:00
parent 5140fd251a
commit def54fb1fa
15 changed files with 895 additions and 117 deletions
--- a/cranelift/codegen/src/isa/s390x/inst/emit.rs
+++ b/cranelift/codegen/src/isa/s390x/inst/emit.rs
@@ -5,6 +5,7 @@ use crate::ir::condcodes::IntCC;
 use crate::ir::MemFlags;
 use crate::ir::{SourceLoc, TrapCode};
 use crate::isa::s390x::inst::*;
+use crate::isa::s390x::settings as s390x_settings;
 use core::convert::TryFrom;
 use log::debug;
 use regalloc::{Reg, RegClass};
@@ -905,17 +906,20 @@ impl EmitState {
 }

 /// Constant state used during function compilation.
-pub struct EmitInfo(settings::Flags);
+pub struct EmitInfo {
+    flags: settings::Flags,
+    isa_flags: s390x_settings::Flags,
+}

 impl EmitInfo {
-    pub(crate) fn new(flags: settings::Flags) -> Self {
-        Self(flags)
+    pub(crate) fn new(flags: settings::Flags, isa_flags: s390x_settings::Flags) -> Self {
+        Self { flags, isa_flags }
    }
 }

 impl MachInstEmitInfo for EmitInfo {
    fn flags(&self) -> &settings::Flags {
-        &self.0
+        &self.flags
    }
 }

@@ -924,6 +928,25 @@ impl MachInstEmit for Inst {
    type Info = EmitInfo;

    fn emit(&self, sink: &mut MachBuffer<Inst>, emit_info: &Self::Info, state: &mut EmitState) {
+        // Verify that we can emit this Inst in the current ISA
+        let matches_isa_flags = |iset_requirement: &InstructionSet| -> bool {
+            match iset_requirement {
+                // Baseline ISA is z14
+                InstructionSet::Base => true,
+                // Miscellaneous-Instruction-Extensions Facility 2 (z15)
+                InstructionSet::MIE2 => emit_info.isa_flags.has_mie2(),
+                // Vector-Enhancements Facility 2 (z15)
+                InstructionSet::VXRS_EXT2 => emit_info.isa_flags.has_vxrs_ext2(),
+            }
+        };
+        let isa_requirements = self.available_in_isa();
+        if !matches_isa_flags(&isa_requirements) {
+            panic!(
+                "Cannot emit inst '{:?}' for target; failed to match ISA requirements: {:?}",
+                self, isa_requirements
+            )
+        }
+
        // N.B.: we *must* not exceed the "worst-case size" used to compute
        // where to insert islands, except when islands are explicitly triggered
        // (with an `EmitIsland`). We check this in debug builds. This is `mut`
--- a/cranelift/codegen/src/isa/s390x/inst/emit_tests.rs
+++ b/cranelift/codegen/src/isa/s390x/inst/emit_tests.rs
@@ -1,5 +1,6 @@
 use crate::ir::MemFlags;
 use crate::isa::s390x::inst::*;
+use crate::isa::s390x::settings as s390x_settings;
 use crate::isa::test_utils;
 use crate::settings;
 use alloc::vec::Vec;
@@ -7767,8 +7768,14 @@ fn test_s390x_binemit() {
    ));

    let flags = settings::Flags::new(settings::builder());
+
+    use crate::settings::Configurable;
+    let mut isa_flag_builder = s390x_settings::builder();
+    isa_flag_builder.enable("arch13").unwrap();
+    let isa_flags = s390x_settings::Flags::new(&flags, isa_flag_builder);
+
    let rru = create_reg_universe(&flags);
-    let emit_info = EmitInfo::new(flags);
+    let emit_info = EmitInfo::new(flags, isa_flags);
    for (insn, expected_encoding, expected_printing) in insns {
        println!(
            "S390x: {:?}, {}, {}",
--- a/cranelift/codegen/src/isa/s390x/inst/mod.rs
+++ b/cranelift/codegen/src/isa/s390x/inst/mod.rs
@@ -34,6 +34,18 @@ mod emit_tests;
 //=============================================================================
 // Instructions (top level): definition

+/// Supported instruction sets
+#[allow(non_camel_case_types)]
+#[derive(Debug)]
+pub(crate) enum InstructionSet {
+    /// Baseline ISA for cranelift is z14.
+    Base,
+    /// Miscellaneous-Instruction-Extensions Facility 2 (z15)
+    MIE2,
+    /// Vector-Enhancements Facility 2 (z15)
+    VXRS_EXT2,
+}
+
 /// An ALU operation. This can be paired with several instruction formats
 /// below (see `Inst`) in any combination.
 #[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
@@ -70,6 +82,17 @@ pub enum ALUOp {
    XorNot64,
 }

+impl ALUOp {
+    pub(crate) fn available_from(&self) -> InstructionSet {
+        match self {
+            ALUOp::AndNot32 | ALUOp::AndNot64 => InstructionSet::MIE2,
+            ALUOp::OrrNot32 | ALUOp::OrrNot64 => InstructionSet::MIE2,
+            ALUOp::XorNot32 | ALUOp::XorNot64 => InstructionSet::MIE2,
+            _ => InstructionSet::Base,
+        }
+    }
+}
+
 #[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
 pub enum UnaryOp {
    Abs32,
@@ -82,6 +105,15 @@ pub enum UnaryOp {
    PopcntReg,
 }

+impl UnaryOp {
+    pub(crate) fn available_from(&self) -> InstructionSet {
+        match self {
+            UnaryOp::PopcntReg => InstructionSet::MIE2,
+            _ => InstructionSet::Base,
+        }
+    }
+}
+
 #[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
 pub enum ShiftOp {
    RotL32,
@@ -941,18 +973,6 @@ pub enum Inst {
    },
 }

-fn count_zero_half_words(mut value: u64) -> usize {
-    let mut count = 0;
-    for _ in 0..4 {
-        if value & 0xffff == 0 {
-            count += 1;
-        }
-        value >>= 16;
-    }
-
-    count
-}
-
 #[test]
 fn inst_size_test() {
    // This test will help with unintentionally growing the size
@@ -961,6 +981,135 @@ fn inst_size_test() {
 }

 impl Inst {
+    /// Retrieve the ISA feature set in which the instruction is available.
+    fn available_in_isa(&self) -> InstructionSet {
+        match self {
+            // These instructions are part of the baseline ISA for cranelift (z14)
+            Inst::Nop0
+            | Inst::Nop2
+            | Inst::AluRRSImm16 { .. }
+            | Inst::AluRR { .. }
+            | Inst::AluRX { .. }
+            | Inst::AluRSImm16 { .. }
+            | Inst::AluRSImm32 { .. }
+            | Inst::AluRUImm32 { .. }
+            | Inst::AluRUImm16Shifted { .. }
+            | Inst::AluRUImm32Shifted { .. }
+            | Inst::ShiftRR { .. }
+            | Inst::SMulWide { .. }
+            | Inst::UMulWide { .. }
+            | Inst::SDivMod32 { .. }
+            | Inst::SDivMod64 { .. }
+            | Inst::UDivMod32 { .. }
+            | Inst::UDivMod64 { .. }
+            | Inst::Flogr { .. }
+            | Inst::CmpRR { .. }
+            | Inst::CmpRX { .. }
+            | Inst::CmpRSImm16 { .. }
+            | Inst::CmpRSImm32 { .. }
+            | Inst::CmpRUImm32 { .. }
+            | Inst::CmpTrapRR { .. }
+            | Inst::CmpTrapRSImm16 { .. }
+            | Inst::CmpTrapRUImm16 { .. }
+            | Inst::AtomicRmw { .. }
+            | Inst::AtomicCas32 { .. }
+            | Inst::AtomicCas64 { .. }
+            | Inst::Fence
+            | Inst::Load32 { .. }
+            | Inst::Load32ZExt8 { .. }
+            | Inst::Load32SExt8 { .. }
+            | Inst::Load32ZExt16 { .. }
+            | Inst::Load32SExt16 { .. }
+            | Inst::Load64 { .. }
+            | Inst::Load64ZExt8 { .. }
+            | Inst::Load64SExt8 { .. }
+            | Inst::Load64ZExt16 { .. }
+            | Inst::Load64SExt16 { .. }
+            | Inst::Load64ZExt32 { .. }
+            | Inst::Load64SExt32 { .. }
+            | Inst::LoadRev16 { .. }
+            | Inst::LoadRev32 { .. }
+            | Inst::LoadRev64 { .. }
+            | Inst::Store8 { .. }
+            | Inst::Store16 { .. }
+            | Inst::Store32 { .. }
+            | Inst::Store64 { .. }
+            | Inst::StoreImm8 { .. }
+            | Inst::StoreImm16 { .. }
+            | Inst::StoreImm32SExt16 { .. }
+            | Inst::StoreImm64SExt16 { .. }
+            | Inst::StoreRev16 { .. }
+            | Inst::StoreRev32 { .. }
+            | Inst::StoreRev64 { .. }
+            | Inst::LoadMultiple64 { .. }
+            | Inst::StoreMultiple64 { .. }
+            | Inst::Mov32 { .. }
+            | Inst::Mov64 { .. }
+            | Inst::Mov32Imm { .. }
+            | Inst::Mov32SImm16 { .. }
+            | Inst::Mov64SImm16 { .. }
+            | Inst::Mov64SImm32 { .. }
+            | Inst::Mov64UImm16Shifted { .. }
+            | Inst::Mov64UImm32Shifted { .. }
+            | Inst::Insert64UImm16Shifted { .. }
+            | Inst::Insert64UImm32Shifted { .. }
+            | Inst::Extend { .. }
+            | Inst::CMov32 { .. }
+            | Inst::CMov64 { .. }
+            | Inst::CMov32SImm16 { .. }
+            | Inst::CMov64SImm16 { .. }
+            | Inst::FpuMove32 { .. }
+            | Inst::FpuMove64 { .. }
+            | Inst::FpuCMov32 { .. }
+            | Inst::FpuCMov64 { .. }
+            | Inst::MovToFpr { .. }
+            | Inst::MovFromFpr { .. }
+            | Inst::FpuRR { .. }
+            | Inst::FpuRRR { .. }
+            | Inst::FpuRRRR { .. }
+            | Inst::FpuCopysign { .. }
+            | Inst::FpuCmp32 { .. }
+            | Inst::FpuCmp64 { .. }
+            | Inst::FpuLoad32 { .. }
+            | Inst::FpuStore32 { .. }
+            | Inst::FpuLoad64 { .. }
+            | Inst::FpuStore64 { .. }
+            | Inst::LoadFpuConst32 { .. }
+            | Inst::LoadFpuConst64 { .. }
+            | Inst::FpuToInt { .. }
+            | Inst::IntToFpu { .. }
+            | Inst::FpuRound { .. }
+            | Inst::FpuVecRRR { .. }
+            | Inst::Call { .. }
+            | Inst::CallInd { .. }
+            | Inst::Ret { .. }
+            | Inst::EpiloguePlaceholder
+            | Inst::Jump { .. }
+            | Inst::CondBr { .. }
+            | Inst::TrapIf { .. }
+            | Inst::OneWayCondBr { .. }
+            | Inst::IndirectBr { .. }
+            | Inst::Debugtrap
+            | Inst::Trap { .. }
+            | Inst::JTSequence { .. }
+            | Inst::LoadExtNameFar { .. }
+            | Inst::LoadAddr { .. }
+            | Inst::VirtualSPOffsetAdj { .. }
+            | Inst::ValueLabelMarker { .. }
+            | Inst::Unwind { .. } => InstructionSet::Base,
+
+            // These depend on the opcode
+            Inst::AluRRR { alu_op, .. } => alu_op.available_from(),
+            Inst::UnaryRR { op, .. } => op.available_from(),
+
+            // These are all part of VXRS_EXT2
+            Inst::FpuLoadRev32 { .. }
+            | Inst::FpuStoreRev32 { .. }
+            | Inst::FpuLoadRev64 { .. }
+            | Inst::FpuStoreRev64 { .. } => InstructionSet::VXRS_EXT2,
+        }
+    }
+
    /// Create a 64-bit move instruction.
    pub fn mov64(to_reg: Writable<Reg>, from_reg: Reg) -> Inst {
        assert!(to_reg.to_reg().get_class() == from_reg.get_class());
--- a/cranelift/codegen/src/isa/s390x/lower.rs
+++ b/cranelift/codegen/src/isa/s390x/lower.rs
@@ -5,6 +5,7 @@ use crate::ir::Inst as IRInst;
 use crate::ir::{types, Endianness, InstructionData, MemFlags, Opcode, TrapCode, Type};
 use crate::isa::s390x::abi::*;
 use crate::isa::s390x::inst::*;
+use crate::isa::s390x::settings as s390x_settings;
 use crate::isa::s390x::S390xBackend;
 use crate::machinst::lower::*;
 use crate::machinst::*;
@@ -548,6 +549,70 @@ fn lower_constant_f64<C: LowerCtx<I = Inst>>(ctx: &mut C, rd: Writable<Reg>, val
    ctx.emit(Inst::load_fp_constant64(rd, value));
 }

+//============================================================================
+// Lowering: miscellaneous helpers.
+
+/// Emit code to invert the value of type ty in register rd.
+fn lower_bnot<C: LowerCtx<I = Inst>>(ctx: &mut C, ty: Type, rd: Writable<Reg>) {
+    let alu_op = choose_32_64(ty, ALUOp::Xor32, ALUOp::Xor64);
+    ctx.emit(Inst::AluRUImm32Shifted {
+        alu_op,
+        rd,
+        imm: UImm32Shifted::maybe_from_u64(0xffff_ffff).unwrap(),
+    });
+    if ty_bits(ty) > 32 {
+        ctx.emit(Inst::AluRUImm32Shifted {
+            alu_op,
+            rd,
+            imm: UImm32Shifted::maybe_from_u64(0xffff_ffff_0000_0000).unwrap(),
+        });
+    }
+}
+
+/// Emit code to bitcast between integer and floating-point values.
+fn lower_bitcast<C: LowerCtx<I = Inst>>(
+    ctx: &mut C,
+    rd: Writable<Reg>,
+    output_ty: Type,
+    rn: Reg,
+    input_ty: Type,
+) {
+    match (input_ty, output_ty) {
+        (types::I64, types::F64) => {
+            ctx.emit(Inst::MovToFpr { rd, rn });
+        }
+        (types::F64, types::I64) => {
+            ctx.emit(Inst::MovFromFpr { rd, rn });
+        }
+        (types::I32, types::F32) => {
+            let tmp = ctx.alloc_tmp(types::I64).only_reg().unwrap();
+            ctx.emit(Inst::ShiftRR {
+                shift_op: ShiftOp::LShL64,
+                rd: tmp,
+                rn,
+                shift_imm: SImm20::maybe_from_i64(32).unwrap(),
+                shift_reg: None,
+            });
+            ctx.emit(Inst::MovToFpr {
+                rd,
+                rn: tmp.to_reg(),
+            });
+        }
+        (types::F32, types::I32) => {
+            let tmp = ctx.alloc_tmp(types::I64).only_reg().unwrap();
+            ctx.emit(Inst::MovFromFpr { rd: tmp, rn });
+            ctx.emit(Inst::ShiftRR {
+                shift_op: ShiftOp::LShR64,
+                rd,
+                rn: tmp.to_reg(),
+                shift_imm: SImm20::maybe_from_i64(32).unwrap(),
+                shift_reg: None,
+            });
+        }
+        _ => unreachable!("invalid bitcast from {:?} to {:?}", input_ty, output_ty),
+    }
+}
+
 //=============================================================================
 // Lowering: comparisons

@@ -760,6 +825,7 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
    ctx: &mut C,
    insn: IRInst,
    flags: &Flags,
+    isa_flags: &s390x_settings::Flags,
 ) -> CodegenResult<()> {
    let op = ctx.data(insn).opcode();
    let inputs: SmallVec<[InsnInput; 4]> = (0..ctx.num_inputs(insn))
@@ -1447,15 +1513,19 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(

        Opcode::Bnot => {
            let ty = ty.unwrap();
-            let alu_op = choose_32_64(ty, ALUOp::OrrNot32, ALUOp::OrrNot64);
            let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
            let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
-            ctx.emit(Inst::AluRRR {
-                alu_op,
-                rd,
-                rn,
-                rm: rn,
-            });
+            if isa_flags.has_mie2() {
+                ctx.emit(Inst::AluRRR {
+                    alu_op: choose_32_64(ty, ALUOp::OrrNot32, ALUOp::OrrNot64),
+                    rd,
+                    rn,
+                    rm: rn,
+                });
+            } else {
+                ctx.emit(Inst::gen_move(rd, rn, ty));
+                lower_bnot(ctx, ty, rd);
+            }
        }

        Opcode::Band => {
@@ -1517,16 +1587,22 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(

        Opcode::BandNot | Opcode::BorNot | Opcode::BxorNot => {
            let ty = ty.unwrap();
-            let alu_op = match op {
-                Opcode::BandNot => choose_32_64(ty, ALUOp::AndNot32, ALUOp::AndNot64),
-                Opcode::BorNot => choose_32_64(ty, ALUOp::OrrNot32, ALUOp::OrrNot64),
-                Opcode::BxorNot => choose_32_64(ty, ALUOp::XorNot32, ALUOp::XorNot64),
+            let alu_op = match (op, isa_flags.has_mie2()) {
+                (Opcode::BandNot, true) => choose_32_64(ty, ALUOp::AndNot32, ALUOp::AndNot64),
+                (Opcode::BorNot, true) => choose_32_64(ty, ALUOp::OrrNot32, ALUOp::OrrNot64),
+                (Opcode::BxorNot, true) => choose_32_64(ty, ALUOp::XorNot32, ALUOp::XorNot64),
+                (Opcode::BandNot, false) => choose_32_64(ty, ALUOp::And32, ALUOp::And64),
+                (Opcode::BorNot, false) => choose_32_64(ty, ALUOp::Orr32, ALUOp::Orr64),
+                (Opcode::BxorNot, false) => choose_32_64(ty, ALUOp::Xor32, ALUOp::Xor64),
                _ => unreachable!(),
            };
            let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap();
            let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
            let rm = put_input_in_reg(ctx, inputs[1], NarrowValueMode::None);
            ctx.emit(Inst::AluRRR { alu_op, rd, rn, rm });
+            if !isa_flags.has_mie2() {
+                lower_bnot(ctx, ty, rd);
+            }
        }

        Opcode::Bitselect => {
@@ -1542,12 +1618,22 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
                rn,
                rm: rcond,
            });
-            ctx.emit(Inst::AluRRR {
-                alu_op: choose_32_64(ty, ALUOp::AndNot32, ALUOp::AndNot64),
-                rd,
-                rn: rm,
-                rm: rcond,
-            });
+            if isa_flags.has_mie2() {
+                ctx.emit(Inst::AluRRR {
+                    alu_op: choose_32_64(ty, ALUOp::AndNot32, ALUOp::AndNot64),
+                    rd,
+                    rn: rm,
+                    rm: rcond,
+                });
+            } else {
+                ctx.emit(Inst::AluRRR {
+                    alu_op: choose_32_64(ty, ALUOp::And32, ALUOp::And64),
+                    rd,
+                    rn: rm,
+                    rm: rcond,
+                });
+                lower_bnot(ctx, ty, rd);
+            }
            ctx.emit(Inst::AluRRR {
                alu_op: choose_32_64(ty, ALUOp::Orr32, ALUOp::Orr64),
                rd,
@@ -1804,13 +1890,45 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
                    rd,
                    rn,
                });
-            } else {
+            } else if isa_flags.has_mie2() {
                let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::ZeroExtend64);
                ctx.emit(Inst::UnaryRR {
                    op: UnaryOp::PopcntReg,
                    rd,
                    rn,
                });
+            } else {
+                let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
+                ctx.emit(Inst::UnaryRR {
+                    op: UnaryOp::PopcntByte,
+                    rd,
+                    rn,
+                });
+                let tmp = ctx.alloc_tmp(types::I64).only_reg().unwrap();
+                let mut shift = ty_bits(ty) as u8;
+                while shift > 8 {
+                    shift = shift / 2;
+                    ctx.emit(Inst::ShiftRR {
+                        shift_op: choose_32_64(ty, ShiftOp::LShL32, ShiftOp::LShL64),
+                        rd: tmp,
+                        rn: rd.to_reg(),
+                        shift_imm: SImm20::maybe_from_i64(shift.into()).unwrap(),
+                        shift_reg: None,
+                    });
+                    ctx.emit(Inst::AluRR {
+                        alu_op: choose_32_64(ty, ALUOp::Add32, ALUOp::Add64),
+                        rd,
+                        rm: tmp.to_reg(),
+                    });
+                }
+                let shift = ty_bits(ty) as u8 - 8;
+                ctx.emit(Inst::ShiftRR {
+                    shift_op: choose_32_64(ty, ShiftOp::LShR32, ShiftOp::LShR64),
+                    rd,
+                    rn: rd.to_reg(),
+                    shift_imm: SImm20::maybe_from_i64(shift.into()).unwrap(),
+                    shift_reg: None,
+                });
            }
        }

@@ -2027,40 +2145,7 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
            let rn = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
            let input_ty = ctx.input_ty(insn, 0);
            let output_ty = ctx.output_ty(insn, 0);
-            match (input_ty, output_ty) {
-                (types::I64, types::F64) => {
-                    ctx.emit(Inst::MovToFpr { rd, rn });
-                }
-                (types::F64, types::I64) => {
-                    ctx.emit(Inst::MovFromFpr { rd, rn });
-                }
-                (types::I32, types::F32) => {
-                    let tmp = ctx.alloc_tmp(types::I64).only_reg().unwrap();
-                    ctx.emit(Inst::ShiftRR {
-                        shift_op: ShiftOp::LShL64,
-                        rd: tmp,
-                        rn,
-                        shift_imm: SImm20::maybe_from_i64(32).unwrap(),
-                        shift_reg: None,
-                    });
-                    ctx.emit(Inst::MovToFpr {
-                        rd,
-                        rn: tmp.to_reg(),
-                    });
-                }
-                (types::F32, types::I32) => {
-                    let tmp = ctx.alloc_tmp(types::I64).only_reg().unwrap();
-                    ctx.emit(Inst::MovFromFpr { rd: tmp, rn });
-                    ctx.emit(Inst::ShiftRR {
-                        shift_op: ShiftOp::LShR64,
-                        rd,
-                        rn: tmp.to_reg(),
-                        shift_imm: SImm20::maybe_from_i64(32).unwrap(),
-                        shift_reg: None,
-                    });
-                }
-                _ => unreachable!("invalid bitcast from {:?} to {:?}", input_ty, output_ty),
-            }
+            lower_bitcast(ctx, rd, output_ty, rn, input_ty);
        }

        Opcode::Load
@@ -2130,21 +2215,18 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
                    (64, 32, true, _) => Inst::Load64SExt32 { rd, mem },
                    _ => panic!("Unsupported size in load"),
                });
-            } else {
-                ctx.emit(match (ext_bits, from_bits, sign_extend, is_float) {
-                    (32, 32, _, true) => Inst::FpuLoadRev32 { rd, mem },
-                    (64, 64, _, true) => Inst::FpuLoadRev64 { rd, mem },
-                    (_, 16, _, false) => Inst::LoadRev16 { rd, mem },
-                    (_, 32, _, false) => Inst::LoadRev32 { rd, mem },
-                    (_, 64, _, false) => Inst::LoadRev64 { rd, mem },
-                    (32, 8, false, _) => Inst::Load32ZExt8 { rd, mem },
-                    (32, 8, true, _) => Inst::Load32SExt8 { rd, mem },
-                    (64, 8, false, _) => Inst::Load64ZExt8 { rd, mem },
-                    (64, 8, true, _) => Inst::Load64SExt8 { rd, mem },
+            } else if !is_float {
+                ctx.emit(match (ext_bits, from_bits, sign_extend) {
+                    (_, 16, _) => Inst::LoadRev16 { rd, mem },
+                    (_, 32, _) => Inst::LoadRev32 { rd, mem },
+                    (_, 64, _) => Inst::LoadRev64 { rd, mem },
+                    (32, 8, false) => Inst::Load32ZExt8 { rd, mem },
+                    (32, 8, true) => Inst::Load32SExt8 { rd, mem },
+                    (64, 8, false) => Inst::Load64ZExt8 { rd, mem },
+                    (64, 8, true) => Inst::Load64SExt8 { rd, mem },
                    _ => panic!("Unsupported size in load"),
                });
                if to_bits > from_bits && from_bits > 8 {
-                    assert!(is_float == false);
                    ctx.emit(Inst::Extend {
                        rd,
                        rn: rd.to_reg(),
@@ -2153,6 +2235,26 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(
                        to_bits: to_bits as u8,
                    });
                }
+            } else if isa_flags.has_vxrs_ext2() {
+                ctx.emit(match from_bits {
+                    32 => Inst::FpuLoadRev32 { rd, mem },
+                    64 => Inst::FpuLoadRev64 { rd, mem },
+                    _ => panic!("Unsupported size in load"),
+                });
+            } else {
+                match from_bits {
+                    32 => {
+                        let tmp = ctx.alloc_tmp(types::I32).only_reg().unwrap();
+                        ctx.emit(Inst::LoadRev32 { rd: tmp, mem });
+                        lower_bitcast(ctx, rd, elem_ty, tmp.to_reg(), types::I32);
+                    }
+                    64 => {
+                        let tmp = ctx.alloc_tmp(types::I64).only_reg().unwrap();
+                        ctx.emit(Inst::LoadRev64 { rd: tmp, mem });
+                        lower_bitcast(ctx, rd, elem_ty, tmp.to_reg(), types::I64);
+                    }
+                    _ => panic!("Unsupported size in load"),
+                }
            }
        }

@@ -2179,13 +2281,39 @@ fn lower_insn_to_regs<C: LowerCtx<I = Inst>>(

            if ty_is_float(elem_ty) {
                let rd = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None);
-                ctx.emit(match (endianness, ty_bits(elem_ty)) {
-                    (Endianness::Big, 32) => Inst::FpuStore32 { rd, mem },
-                    (Endianness::Big, 64) => Inst::FpuStore64 { rd, mem },
-                    (Endianness::Little, 32) => Inst::FpuStoreRev32 { rd, mem },
-                    (Endianness::Little, 64) => Inst::FpuStoreRev64 { rd, mem },
-                    _ => panic!("Unsupported size in store"),
-                });
+                if endianness == Endianness::Big {
+                    ctx.emit(match ty_bits(elem_ty) {
+                        32 => Inst::FpuStore32 { rd, mem },
+                        64 => Inst::FpuStore64 { rd, mem },
+                        _ => panic!("Unsupported size in store"),
+                    });
+                } else if isa_flags.has_vxrs_ext2() {
+                    ctx.emit(match ty_bits(elem_ty) {
+                        32 => Inst::FpuStoreRev32 { rd, mem },
+                        64 => Inst::FpuStoreRev64 { rd, mem },
+                        _ => panic!("Unsupported size in store"),
+                    });
+                } else {
+                    match ty_bits(elem_ty) {
+                        32 => {
+                            let tmp = ctx.alloc_tmp(types::I32).only_reg().unwrap();
+                            lower_bitcast(ctx, tmp, types::I32, rd, elem_ty);
+                            ctx.emit(Inst::StoreRev32 {
+                                rd: tmp.to_reg(),
+                                mem,
+                            });
+                        }
+                        64 => {
+                            let tmp = ctx.alloc_tmp(types::I64).only_reg().unwrap();
+                            lower_bitcast(ctx, tmp, types::I64, rd, elem_ty);
+                            ctx.emit(Inst::StoreRev64 {
+                                rd: tmp.to_reg(),
+                                mem,
+                            });
+                        }
+                        _ => panic!("Unsupported size in load"),
+                    }
+                }
            } else if ty_bits(elem_ty) <= 16 {
                if let Some(imm) = input_matches_const(ctx, inputs[0]) {
                    ctx.emit(match (endianness, ty_bits(elem_ty)) {
@@ -2980,7 +3108,7 @@ impl LowerBackend for S390xBackend {
    type MInst = Inst;

    fn lower<C: LowerCtx<I = Inst>>(&self, ctx: &mut C, ir_inst: IRInst) -> CodegenResult<()> {
-        lower_insn_to_regs(ctx, ir_inst, &self.flags)
+        lower_insn_to_regs(ctx, ir_inst, &self.flags, &self.isa_flags)
    }

    fn lower_branch_group<C: LowerCtx<I = Inst>>(
--- a/cranelift/codegen/src/isa/s390x/mod.rs
+++ b/cranelift/codegen/src/isa/s390x/mod.rs
@@ -57,7 +57,7 @@ impl S390xBackend {
        func: &Function,
        flags: shared_settings::Flags,
    ) -> CodegenResult<VCode<inst::Inst>> {
-        let emit_info = EmitInfo::new(flags.clone());
+        let emit_info = EmitInfo::new(flags.clone(), self.isa_flags.clone());
        let abi = Box::new(abi::S390xABICallee::new(func, flags)?);
        compile::compile::<S390xBackend>(func, self, abi, emit_info)
    }