diff --git a/cranelift/codegen/src/isa/x64/inst/emit.rs b/cranelift/codegen/src/isa/x64/inst/emit.rs index fb68a8b63c..3b37581489 100644 --- a/cranelift/codegen/src/isa/x64/inst/emit.rs +++ b/cranelift/codegen/src/isa/x64/inst/emit.rs @@ -1133,7 +1133,7 @@ pub(crate) fn emit( } Inst::Shift_R { - is_64, + size, kind, num_bits, dst, @@ -1147,25 +1147,39 @@ pub(crate) fn emit( ShiftKind::ShiftRightArithmetic => 7, }; - let rex = if *is_64 { - RexFlags::set_w() - } else { - RexFlags::clear_w() - }; - match num_bits { None => { + let (opcode, prefix, rex_flags) = match size { + 1 => (0xD2, LegacyPrefixes::None, RexFlags::clear_w()), + 2 => (0xD3, LegacyPrefixes::_66, RexFlags::clear_w()), + 4 => (0xD3, LegacyPrefixes::None, RexFlags::clear_w()), + 8 => (0xD3, LegacyPrefixes::None, RexFlags::set_w()), + _ => unreachable!("{}", size), + }; + + // SHL/SHR/SAR %cl, reg8 is (REX.W==0) D2 /subopcode + // SHL/SHR/SAR %cl, reg16 is 66 (REX.W==0) D3 /subopcode // SHL/SHR/SAR %cl, reg32 is (REX.W==0) D3 /subopcode // SHL/SHR/SAR %cl, reg64 is (REX.W==1) D3 /subopcode - emit_std_enc_enc(sink, LegacyPrefixes::None, 0xD3, 1, subopcode, enc_dst, rex); + emit_std_enc_enc(sink, prefix, opcode, 1, subopcode, enc_dst, rex_flags); } Some(num_bits) => { + let (opcode, prefix, rex_flags) = match size { + 1 => (0xC0, LegacyPrefixes::None, RexFlags::clear_w()), + 2 => (0xC1, LegacyPrefixes::_66, RexFlags::clear_w()), + 4 => (0xC1, LegacyPrefixes::None, RexFlags::clear_w()), + 8 => (0xC1, LegacyPrefixes::None, RexFlags::set_w()), + _ => unreachable!("{}", size), + }; + + // SHL/SHR/SAR $ib, reg8 is (REX.W==0) C0 /subopcode + // SHL/SHR/SAR $ib, reg16 is 66 (REX.W==0) C1 /subopcode // SHL/SHR/SAR $ib, reg32 is (REX.W==0) C1 /subopcode ib // SHL/SHR/SAR $ib, reg64 is (REX.W==1) C1 /subopcode ib // When the shift amount is 1, there's an even shorter encoding, but we don't // bother with that nicety here. - emit_std_enc_enc(sink, LegacyPrefixes::None, 0xC1, 1, subopcode, enc_dst, rex); + emit_std_enc_enc(sink, prefix, opcode, 1, subopcode, enc_dst, rex_flags); sink.put1(*num_bits); } } @@ -2054,12 +2068,7 @@ pub(crate) fn emit( inst.emit(sink, flags, state); // tmp_gpr1 := src >> 1 - let inst = Inst::shift_r( - /*is_64*/ true, - ShiftKind::ShiftRightLogical, - Some(1), - *tmp_gpr1, - ); + let inst = Inst::shift_r(8, ShiftKind::ShiftRightLogical, Some(1), *tmp_gpr1); inst.emit(sink, flags, state); let inst = Inst::gen_move(*tmp_gpr2, src.to_reg(), types::I64); diff --git a/cranelift/codegen/src/isa/x64/inst/emit_tests.rs b/cranelift/codegen/src/isa/x64/inst/emit_tests.rs index d4d187765b..8a65d3c729 100644 --- a/cranelift/codegen/src/isa/x64/inst/emit_tests.rs +++ b/cranelift/codegen/src/isa/x64/inst/emit_tests.rs @@ -2365,130 +2365,150 @@ fn test_x64_emit() { // ======================================================== // Shift_R insns.push(( - Inst::shift_r(false, ShiftKind::ShiftLeft, None, w_rdi), + Inst::shift_r(4, ShiftKind::ShiftLeft, None, w_rdi), "D3E7", "shll %cl, %edi", )); insns.push(( - Inst::shift_r(false, ShiftKind::ShiftLeft, None, w_r12), + Inst::shift_r(4, ShiftKind::ShiftLeft, None, w_r12), "41D3E4", "shll %cl, %r12d", )); insns.push(( - Inst::shift_r(false, ShiftKind::ShiftLeft, Some(2), w_r8), + Inst::shift_r(4, ShiftKind::ShiftLeft, Some(2), w_r8), "41C1E002", "shll $2, %r8d", )); insns.push(( - Inst::shift_r(false, ShiftKind::ShiftLeft, Some(31), w_r13), + Inst::shift_r(4, ShiftKind::ShiftLeft, Some(31), w_r13), "41C1E51F", "shll $31, %r13d", )); insns.push(( - Inst::shift_r(true, ShiftKind::ShiftLeft, None, w_r13), + Inst::shift_r(8, ShiftKind::ShiftLeft, None, w_r13), "49D3E5", "shlq %cl, %r13", )); insns.push(( - Inst::shift_r(true, ShiftKind::ShiftLeft, None, w_rdi), + Inst::shift_r(8, ShiftKind::ShiftLeft, None, w_rdi), "48D3E7", "shlq %cl, %rdi", )); insns.push(( - Inst::shift_r(true, ShiftKind::ShiftLeft, Some(2), w_r8), + Inst::shift_r(8, ShiftKind::ShiftLeft, Some(2), w_r8), "49C1E002", "shlq $2, %r8", )); insns.push(( - Inst::shift_r(true, ShiftKind::ShiftLeft, Some(3), w_rbx), + Inst::shift_r(8, ShiftKind::ShiftLeft, Some(3), w_rbx), "48C1E303", "shlq $3, %rbx", )); insns.push(( - Inst::shift_r(true, ShiftKind::ShiftLeft, Some(63), w_r13), + Inst::shift_r(8, ShiftKind::ShiftLeft, Some(63), w_r13), "49C1E53F", "shlq $63, %r13", )); insns.push(( - Inst::shift_r(false, ShiftKind::ShiftRightLogical, None, w_rdi), + Inst::shift_r(4, ShiftKind::ShiftRightLogical, None, w_rdi), "D3EF", "shrl %cl, %edi", )); insns.push(( - Inst::shift_r(false, ShiftKind::ShiftRightLogical, Some(2), w_r8), + Inst::shift_r(4, ShiftKind::ShiftRightLogical, Some(2), w_r8), "41C1E802", "shrl $2, %r8d", )); insns.push(( - Inst::shift_r(false, ShiftKind::ShiftRightLogical, Some(31), w_r13), + Inst::shift_r(4, ShiftKind::ShiftRightLogical, Some(31), w_r13), "41C1ED1F", "shrl $31, %r13d", )); insns.push(( - Inst::shift_r(true, ShiftKind::ShiftRightLogical, None, w_rdi), + Inst::shift_r(8, ShiftKind::ShiftRightLogical, None, w_rdi), "48D3EF", "shrq %cl, %rdi", )); insns.push(( - Inst::shift_r(true, ShiftKind::ShiftRightLogical, Some(2), w_r8), + Inst::shift_r(8, ShiftKind::ShiftRightLogical, Some(2), w_r8), "49C1E802", "shrq $2, %r8", )); insns.push(( - Inst::shift_r(true, ShiftKind::ShiftRightLogical, Some(63), w_r13), + Inst::shift_r(8, ShiftKind::ShiftRightLogical, Some(63), w_r13), "49C1ED3F", "shrq $63, %r13", )); insns.push(( - Inst::shift_r(false, ShiftKind::ShiftRightArithmetic, None, w_rdi), + Inst::shift_r(4, ShiftKind::ShiftRightArithmetic, None, w_rdi), "D3FF", "sarl %cl, %edi", )); insns.push(( - Inst::shift_r(false, ShiftKind::ShiftRightArithmetic, Some(2), w_r8), + Inst::shift_r(4, ShiftKind::ShiftRightArithmetic, Some(2), w_r8), "41C1F802", "sarl $2, %r8d", )); insns.push(( - Inst::shift_r(false, ShiftKind::ShiftRightArithmetic, Some(31), w_r13), + Inst::shift_r(4, ShiftKind::ShiftRightArithmetic, Some(31), w_r13), "41C1FD1F", "sarl $31, %r13d", )); insns.push(( - Inst::shift_r(true, ShiftKind::ShiftRightArithmetic, None, w_rdi), + Inst::shift_r(8, ShiftKind::ShiftRightArithmetic, None, w_rdi), "48D3FF", "sarq %cl, %rdi", )); insns.push(( - Inst::shift_r(true, ShiftKind::ShiftRightArithmetic, Some(2), w_r8), + Inst::shift_r(8, ShiftKind::ShiftRightArithmetic, Some(2), w_r8), "49C1F802", "sarq $2, %r8", )); insns.push(( - Inst::shift_r(true, ShiftKind::ShiftRightArithmetic, Some(63), w_r13), + Inst::shift_r(8, ShiftKind::ShiftRightArithmetic, Some(63), w_r13), "49C1FD3F", "sarq $63, %r13", )); insns.push(( - Inst::shift_r(true, ShiftKind::RotateLeft, None, w_r8), + Inst::shift_r(8, ShiftKind::RotateLeft, None, w_r8), "49D3C0", "rolq %cl, %r8", )); insns.push(( - Inst::shift_r(false, ShiftKind::RotateLeft, Some(3), w_r9), + Inst::shift_r(4, ShiftKind::RotateLeft, Some(3), w_r9), "41C1C103", "roll $3, %r9d", )); insns.push(( - Inst::shift_r(false, ShiftKind::RotateRight, None, w_rsi), + Inst::shift_r(4, ShiftKind::RotateRight, None, w_rsi), "D3CE", "rorl %cl, %esi", )); insns.push(( - Inst::shift_r(true, ShiftKind::RotateRight, Some(5), w_r15), + Inst::shift_r(8, ShiftKind::RotateRight, Some(5), w_r15), "49C1CF05", "rorq $5, %r15", )); + insns.push(( + Inst::shift_r(1, ShiftKind::RotateRight, None, w_rsi), + "D2CE", + "rorb %cl, %sil", + )); + insns.push(( + Inst::shift_r(1, ShiftKind::RotateRight, Some(5), w_r15), + "41C0CF05", + "rorb $5, %r15b", + )); + insns.push(( + Inst::shift_r(2, ShiftKind::RotateRight, None, w_rsi), + "66D3CE", + "rorw %cl, %si", + )); + insns.push(( + Inst::shift_r(2, ShiftKind::RotateRight, Some(5), w_r15), + "6641C1CF05", + "rorw $5, %r15w", + )); // ======================================================== // CmpRMIR diff --git a/cranelift/codegen/src/isa/x64/inst/mod.rs b/cranelift/codegen/src/isa/x64/inst/mod.rs index 682f6e1876..7fb1597f4d 100644 --- a/cranelift/codegen/src/isa/x64/inst/mod.rs +++ b/cranelift/codegen/src/isa/x64/inst/mod.rs @@ -166,9 +166,9 @@ pub enum Inst { srcloc: Option, }, - /// Arithmetic shifts: (shl shr sar) (l q) imm reg. + /// Arithmetic shifts: (shl shr sar) (b w l q) imm reg. Shift_R { - is_64: bool, + size: u8, // 1, 2, 4 or 8 kind: ShiftKind, /// shift count: Some(0 .. #bits-in-type - 1), or None to mean "%cl". num_bits: Option, @@ -892,19 +892,20 @@ impl Inst { } pub(crate) fn shift_r( - is_64: bool, + size: u8, kind: ShiftKind, num_bits: Option, dst: Writable, ) -> Inst { + debug_assert!(size == 8 || size == 4 || size == 2 || size == 1); debug_assert!(if let Some(num_bits) = num_bits { - num_bits < if is_64 { 64 } else { 32 } + num_bits < size * 8 } else { true }); debug_assert!(dst.to_reg().get_class() == RegClass::I64); Inst::Shift_R { - is_64, + size, kind, num_bits, dst, @@ -1511,22 +1512,22 @@ impl ShowWithRRU for Inst { ), Inst::Shift_R { - is_64, + size, kind, num_bits, dst, } => match num_bits { None => format!( "{} %cl, {}", - ljustify2(kind.to_string(), suffixLQ(*is_64)), - show_ireg_sized(dst.to_reg(), mb_rru, sizeLQ(*is_64)) + ljustify2(kind.to_string(), suffixBWLQ(*size)), + show_ireg_sized(dst.to_reg(), mb_rru, *size) ), Some(num_bits) => format!( "{} ${}, {}", - ljustify2(kind.to_string(), suffixLQ(*is_64)), + ljustify2(kind.to_string(), suffixBWLQ(*size)), num_bits, - show_ireg_sized(dst.to_reg(), mb_rru, sizeLQ(*is_64)) + show_ireg_sized(dst.to_reg(), mb_rru, *size) ), }, diff --git a/cranelift/codegen/src/isa/x64/lower.rs b/cranelift/codegen/src/isa/x64/lower.rs index 39f718eaeb..f9cb49ded9 100644 --- a/cranelift/codegen/src/isa/x64/lower.rs +++ b/cranelift/codegen/src/isa/x64/lower.rs @@ -634,15 +634,23 @@ fn lower_insn_to_regs>( let dst_ty = ctx.output_ty(insn, 0); debug_assert_eq!(ctx.input_ty(insn, 0), dst_ty); - let lhs = match dst_ty { + let (size, lhs) = match dst_ty { types::I8 | types::I16 => match op { - Opcode::Ishl => input_to_reg(ctx, inputs[0]), - Opcode::Ushr => extend_input_to_reg(ctx, inputs[0], ExtSpec::ZeroExtendTo32), - Opcode::Sshr => extend_input_to_reg(ctx, inputs[0], ExtSpec::SignExtendTo32), - Opcode::Rotl | Opcode::Rotr => unimplemented!("rotl/rotr.i8/i16"), + Opcode::Ishl => (4, input_to_reg(ctx, inputs[0])), + Opcode::Ushr => ( + 4, + extend_input_to_reg(ctx, inputs[0], ExtSpec::ZeroExtendTo32), + ), + Opcode::Sshr => ( + 4, + extend_input_to_reg(ctx, inputs[0], ExtSpec::SignExtendTo32), + ), + Opcode::Rotl | Opcode::Rotr => { + (dst_ty.bytes() as u8, input_to_reg(ctx, inputs[0])) + } _ => unreachable!(), }, - types::I32 | types::I64 => input_to_reg(ctx, inputs[0]), + types::I32 | types::I64 => (dst_ty.bytes() as u8, input_to_reg(ctx, inputs[0])), _ => unreachable!("{}", dst_ty), }; @@ -669,13 +677,12 @@ fn lower_insn_to_regs>( _ => unreachable!(), }; - let is_64 = dst_ty == types::I64; let w_rcx = Writable::from_reg(regs::rcx()); ctx.emit(Inst::mov_r_r(true, lhs, dst)); if count.is_none() { ctx.emit(Inst::mov_r_r(true, rhs.unwrap(), w_rcx)); } - ctx.emit(Inst::shift_r(is_64, shift_kind, count, dst)); + ctx.emit(Inst::shift_r(size, shift_kind, count, dst)); } Opcode::Ineg => { @@ -828,7 +835,7 @@ fn lower_insn_to_regs>( // shr $1, tmp1 ctx.emit(Inst::shift_r( - is_64, + 8, ShiftKind::ShiftRightLogical, Some(1), tmp1, @@ -858,7 +865,7 @@ fn lower_insn_to_regs>( // shr $1, tmp1 ctx.emit(Inst::shift_r( - is_64, + 8, ShiftKind::ShiftRightLogical, Some(1), tmp1, @@ -882,7 +889,7 @@ fn lower_insn_to_regs>( // shr $1, tmp1 ctx.emit(Inst::shift_r( - is_64, + 8, ShiftKind::ShiftRightLogical, Some(1), tmp1, @@ -908,12 +915,7 @@ fn lower_insn_to_regs>( ctx.emit(Inst::mov64_rm_r(RegMem::reg(tmp2.to_reg()), dst, None)); // shr $4, dst - ctx.emit(Inst::shift_r( - is_64, - ShiftKind::ShiftRightLogical, - Some(4), - dst, - )); + ctx.emit(Inst::shift_r(8, ShiftKind::ShiftRightLogical, Some(4), dst)); // add tmp2, dst ctx.emit(Inst::alu_rmi_r( @@ -947,7 +949,7 @@ fn lower_insn_to_regs>( // shr $56, dst ctx.emit(Inst::shift_r( - is_64, + 8, ShiftKind::ShiftRightLogical, Some(56), dst, @@ -964,7 +966,7 @@ fn lower_insn_to_regs>( // shr $1, tmp1 ctx.emit(Inst::shift_r( - is_64, + 4, ShiftKind::ShiftRightLogical, Some(1), tmp1, @@ -991,7 +993,7 @@ fn lower_insn_to_regs>( // shr $1, tmp1 ctx.emit(Inst::shift_r( - is_64, + 4, ShiftKind::ShiftRightLogical, Some(1), tmp1, @@ -1015,7 +1017,7 @@ fn lower_insn_to_regs>( // shr $1, tmp1 ctx.emit(Inst::shift_r( - is_64, + 4, ShiftKind::ShiftRightLogical, Some(1), tmp1, @@ -1041,12 +1043,7 @@ fn lower_insn_to_regs>( ctx.emit(Inst::mov64_rm_r(RegMem::reg(tmp2.to_reg()), dst, None)); // shr $4, dst - ctx.emit(Inst::shift_r( - is_64, - ShiftKind::ShiftRightLogical, - Some(4), - dst, - )); + ctx.emit(Inst::shift_r(4, ShiftKind::ShiftRightLogical, Some(4), dst)); // add tmp2, dst ctx.emit(Inst::alu_rmi_r( @@ -1074,7 +1071,7 @@ fn lower_insn_to_regs>( // shr $24, dst ctx.emit(Inst::shift_r( - is_64, + 4, ShiftKind::ShiftRightLogical, Some(24), dst,