From 3c688458139ff23e51a91bbcf830d1f14a49a612 Mon Sep 17 00:00:00 2001 From: Yury Delendik Date: Thu, 15 Oct 2020 08:34:50 -0500 Subject: [PATCH] Cranelift: refactoring of unwind info (#2289) * factor common code * move fde/unwind emit to more abstract level * code_len -> function_size * speedup block scanning * better function_size calciulation * Rename UnwindCode enums --- cranelift/codegen/src/ir/function.rs | 2 +- cranelift/codegen/src/isa/unwind.rs | 52 ++ cranelift/codegen/src/isa/unwind/systemv.rs | 189 +++++++- cranelift/codegen/src/isa/unwind/winx64.rs | 61 +++ cranelift/codegen/src/isa/x86/abi.rs | 8 +- cranelift/codegen/src/isa/x86/unwind.rs | 459 ++++++++++++++++++ .../codegen/src/isa/x86/unwind/systemv.rs | 298 +----------- .../codegen/src/isa/x86/unwind/winx64.rs | 135 +----- 8 files changed, 813 insertions(+), 391 deletions(-) diff --git a/cranelift/codegen/src/ir/function.rs b/cranelift/codegen/src/ir/function.rs index f425eb669f..92911c8a59 100644 --- a/cranelift/codegen/src/ir/function.rs +++ b/cranelift/codegen/src/ir/function.rs @@ -94,7 +94,7 @@ pub struct Function { /// The instructions that mark the start (inclusive) of an epilogue in the function. /// /// This is used for some ABIs to generate unwind information. - pub epilogues_start: Vec, + pub epilogues_start: Vec<(Inst, Block)>, /// An optional global value which represents an expression evaluating to /// the stack limit for this function. This `GlobalValue` will be diff --git a/cranelift/codegen/src/isa/unwind.rs b/cranelift/codegen/src/isa/unwind.rs index b594720a5d..0f957d631b 100644 --- a/cranelift/codegen/src/isa/unwind.rs +++ b/cranelift/codegen/src/isa/unwind.rs @@ -14,3 +14,55 @@ pub enum UnwindInfo { /// System V ABI unwind information. SystemV(systemv::UnwindInfo), } + +pub(crate) mod input { + use crate::binemit::CodeOffset; + use alloc::vec::Vec; + #[cfg(feature = "enable-serde")] + use serde::{Deserialize, Serialize}; + + #[derive(Clone, Debug, PartialEq, Eq)] + #[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] + pub(crate) enum UnwindCode { + SaveRegister { + offset: CodeOffset, + reg: Reg, + }, + RestoreRegister { + offset: CodeOffset, + reg: Reg, + }, + SaveXmmRegister { + offset: CodeOffset, + reg: Reg, + stack_offset: u32, + }, + StackAlloc { + offset: CodeOffset, + size: u32, + }, + StackDealloc { + offset: CodeOffset, + size: u32, + }, + SetFramePointer { + offset: CodeOffset, + reg: Reg, + }, + RememberState { + offset: CodeOffset, + }, + RestoreState { + offset: CodeOffset, + }, + } + + #[derive(Clone, Debug, PartialEq, Eq)] + #[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] + pub struct UnwindInfo { + pub(crate) prologue_size: CodeOffset, + pub(crate) prologue_unwind_codes: Vec>, + pub(crate) epilogues_unwind_codes: Vec>>, + pub(crate) function_size: CodeOffset, + } +} diff --git a/cranelift/codegen/src/isa/unwind/systemv.rs b/cranelift/codegen/src/isa/unwind/systemv.rs index 190226e8ce..c3070fe1c2 100644 --- a/cranelift/codegen/src/isa/unwind/systemv.rs +++ b/cranelift/codegen/src/isa/unwind/systemv.rs @@ -1,5 +1,7 @@ //! System V ABI unwind information. +use crate::isa::{unwind::input, RegUnit}; +use crate::result::{CodegenError, CodegenResult}; use alloc::vec::Vec; use gimli::write::{Address, FrameDescriptionEntry}; use thiserror::Error; @@ -92,6 +94,14 @@ impl Into for CallFrameInstruction { } } +/// Maps UnwindInfo register to gimli's index space. +pub(crate) trait RegisterMapper { + /// Maps RegUnit. + fn map(&self, reg: RegUnit) -> Result; + /// Gets RSP in gimli's index space. + fn rsp(&self) -> Register; +} + /// Represents unwind information for a single System V ABI function. /// /// This representation is not ISA specific. @@ -103,8 +113,58 @@ pub struct UnwindInfo { } impl UnwindInfo { - pub(crate) fn new(instructions: Vec<(u32, CallFrameInstruction)>, len: u32) -> Self { - Self { instructions, len } + pub(crate) fn build<'b>( + unwind: input::UnwindInfo, + word_size: u8, + frame_register: Option, + map_reg: &'b dyn RegisterMapper, + ) -> CodegenResult { + use input::UnwindCode; + let mut builder = InstructionBuilder::new(word_size, frame_register, map_reg); + + for c in unwind.prologue_unwind_codes.iter().chain( + unwind + .epilogues_unwind_codes + .iter() + .map(|c| c.iter()) + .flatten(), + ) { + match c { + UnwindCode::SaveRegister { offset, reg } => { + builder + .push_reg(*offset, *reg) + .map_err(CodegenError::RegisterMappingError)?; + } + UnwindCode::StackAlloc { offset, size } => { + builder.adjust_sp_down_imm(*offset, *size as i64); + } + UnwindCode::StackDealloc { offset, size } => { + builder.adjust_sp_up_imm(*offset, *size as i64); + } + UnwindCode::RestoreRegister { offset, reg } => { + builder + .pop_reg(*offset, *reg) + .map_err(CodegenError::RegisterMappingError)?; + } + UnwindCode::SetFramePointer { offset, reg } => { + builder + .set_cfa_reg(*offset, *reg) + .map_err(CodegenError::RegisterMappingError)?; + } + UnwindCode::RememberState { offset } => { + builder.remember_state(*offset); + } + UnwindCode::RestoreState { offset } => { + builder.restore_state(*offset); + } + _ => {} + } + } + + let instructions = builder.instructions; + let len = unwind.function_size; + + Ok(Self { instructions, len }) } /// Converts the unwind information into a `FrameDescriptionEntry`. @@ -118,3 +178,128 @@ impl UnwindInfo { fde } } + +struct InstructionBuilder<'a> { + word_size: u8, + cfa_offset: i32, + saved_state: Option, + frame_register: Option, + map_reg: &'a dyn RegisterMapper, + instructions: Vec<(u32, CallFrameInstruction)>, +} + +impl<'a> InstructionBuilder<'a> { + fn new( + word_size: u8, + frame_register: Option, + map_reg: &'a (dyn RegisterMapper + 'a), + ) -> Self { + Self { + word_size, + cfa_offset: word_size as i32, // CFA offset starts at word size offset to account for the return address on stack + saved_state: None, + frame_register, + map_reg, + instructions: Vec::new(), + } + } + + fn push_reg(&mut self, offset: u32, reg: RegUnit) -> Result<(), RegisterMappingError> { + self.cfa_offset += self.word_size as i32; + // Update the CFA if this is the save of the frame pointer register or if a frame pointer isn't being used + // When using a frame pointer, we only need to update the CFA to account for the push of the frame pointer itself + if match self.frame_register { + Some(fp) => reg == fp, + None => true, + } { + self.instructions + .push((offset, CallFrameInstruction::CfaOffset(self.cfa_offset))); + } + + // Pushes in the prologue are register saves, so record an offset of the save + self.instructions.push(( + offset, + CallFrameInstruction::Offset(self.map_reg.map(reg)?, -self.cfa_offset), + )); + + Ok(()) + } + + fn adjust_sp_down_imm(&mut self, offset: u32, imm: i64) { + assert!(imm <= core::u32::MAX as i64); + + // Don't adjust the CFA if we're using a frame pointer + if self.frame_register.is_some() { + return; + } + + self.cfa_offset += imm as i32; + self.instructions + .push((offset, CallFrameInstruction::CfaOffset(self.cfa_offset))); + } + + fn adjust_sp_up_imm(&mut self, offset: u32, imm: i64) { + assert!(imm <= core::u32::MAX as i64); + + // Don't adjust the CFA if we're using a frame pointer + if self.frame_register.is_some() { + return; + } + + self.cfa_offset -= imm as i32; + self.instructions + .push((offset, CallFrameInstruction::CfaOffset(self.cfa_offset))); + } + + fn set_cfa_reg(&mut self, offset: u32, reg: RegUnit) -> Result<(), RegisterMappingError> { + self.instructions.push(( + offset, + CallFrameInstruction::CfaRegister(self.map_reg.map(reg)?), + )); + Ok(()) + } + + fn pop_reg(&mut self, offset: u32, reg: RegUnit) -> Result<(), RegisterMappingError> { + self.cfa_offset -= self.word_size as i32; + + // Update the CFA if this is the restore of the frame pointer register or if a frame pointer isn't being used + match self.frame_register { + Some(fp) => { + if reg == fp { + self.instructions.push(( + offset, + CallFrameInstruction::Cfa(self.map_reg.rsp(), self.cfa_offset), + )); + } + } + None => { + self.instructions + .push((offset, CallFrameInstruction::CfaOffset(self.cfa_offset))); + + // Pops in the epilogue are register restores, so record a "same value" for the register + // This isn't necessary when using a frame pointer as the CFA doesn't change for CSR restores + self.instructions.push(( + offset, + CallFrameInstruction::SameValue(self.map_reg.map(reg)?), + )); + } + }; + + Ok(()) + } + + fn remember_state(&mut self, offset: u32) { + self.saved_state = Some(self.cfa_offset); + + self.instructions + .push((offset, CallFrameInstruction::RememberState)); + } + + fn restore_state(&mut self, offset: u32) { + let cfa_offset = self.saved_state.take().unwrap(); + self.cfa_offset = cfa_offset; + + self.instructions + .push((offset, CallFrameInstruction::RestoreState)); + } +} diff --git a/cranelift/codegen/src/isa/unwind/winx64.rs b/cranelift/codegen/src/isa/unwind/winx64.rs index 02600296ff..f7d4d94c72 100644 --- a/cranelift/codegen/src/isa/unwind/winx64.rs +++ b/cranelift/codegen/src/isa/unwind/winx64.rs @@ -1,7 +1,10 @@ //! Windows x64 ABI unwind information. +use crate::isa::{unwind::input, RegUnit}; +use crate::result::{CodegenError, CodegenResult}; use alloc::vec::Vec; use byteorder::{ByteOrder, LittleEndian}; +use log::warn; #[cfg(feature = "enable-serde")] use serde::{Deserialize, Serialize}; @@ -134,6 +137,12 @@ impl UnwindCode { } } +/// Maps UnwindInfo register to Windows x64 unwind data. +pub(crate) trait RegisterMapper { + /// Maps RegUnit. + fn map(reg: RegUnit) -> u8; +} + /// Represents Windows x64 unwind information. /// /// For information about Windows x64 unwind info, see: @@ -204,4 +213,56 @@ impl UnwindInfo { .iter() .fold(0, |nodes, c| nodes + c.node_count()) } + + pub(crate) fn build( + unwind: input::UnwindInfo, + ) -> CodegenResult { + use crate::isa::unwind::input::UnwindCode as InputUnwindCode; + + let mut unwind_codes = Vec::new(); + for c in unwind.prologue_unwind_codes.iter() { + match c { + InputUnwindCode::SaveRegister { offset, reg } => { + unwind_codes.push(UnwindCode::PushRegister { + offset: ensure_unwind_offset(*offset)?, + reg: MR::map(*reg), + }); + } + InputUnwindCode::StackAlloc { offset, size } => { + unwind_codes.push(UnwindCode::StackAlloc { + offset: ensure_unwind_offset(*offset)?, + size: *size, + }); + } + InputUnwindCode::SaveXmmRegister { + offset, + reg, + stack_offset, + } => { + unwind_codes.push(UnwindCode::SaveXmm { + offset: ensure_unwind_offset(*offset)?, + reg: MR::map(*reg), + stack_offset: *stack_offset, + }); + } + _ => {} + } + } + + Ok(Self { + flags: 0, // this assumes cranelift functions have no SEH handlers + prologue_size: ensure_unwind_offset(unwind.prologue_size)?, + frame_register: None, + frame_register_offset: 0, + unwind_codes, + }) + } +} + +fn ensure_unwind_offset(offset: u32) -> CodegenResult { + if offset > 255 { + warn!("function prologues cannot exceed 255 bytes in size for Windows x64"); + return Err(CodegenError::CodeTooLarge); + } + Ok(offset as u8) } diff --git a/cranelift/codegen/src/isa/x86/abi.rs b/cranelift/codegen/src/isa/x86/abi.rs index 3a303974dc..227e2a8ee7 100644 --- a/cranelift/codegen/src/isa/x86/abi.rs +++ b/cranelift/codegen/src/isa/x86/abi.rs @@ -993,7 +993,7 @@ fn insert_common_epilogues( pos.goto_last_inst(block); if let Some(inst) = pos.current_inst() { if pos.func.dfg[inst].opcode().is_return() { - insert_common_epilogue(inst, stack_size, pos, reg_type, csrs, sp_arg_index); + insert_common_epilogue(inst, block, stack_size, pos, reg_type, csrs, sp_arg_index); } } } @@ -1003,6 +1003,7 @@ fn insert_common_epilogues( /// This is used by common calling conventions such as System V. fn insert_common_epilogue( inst: ir::Inst, + block: ir::Block, stack_size: i64, pos: &mut EncCursor, reg_type: ir::types::Type, @@ -1062,12 +1063,13 @@ fn insert_common_epilogue( assert!(csrs.iter(FPR).len() == 0); } - pos.func.epilogues_start.push( + pos.func.epilogues_start.push(( first_fpr_load .or(sp_adjust_inst) .or(first_csr_pop_inst) .unwrap_or(fp_pop_inst), - ); + block, + )); } #[cfg(feature = "unwind")] diff --git a/cranelift/codegen/src/isa/x86/unwind.rs b/cranelift/codegen/src/isa/x86/unwind.rs index 0f8a8ef927..e458111b37 100644 --- a/cranelift/codegen/src/isa/x86/unwind.rs +++ b/cranelift/codegen/src/isa/x86/unwind.rs @@ -2,3 +2,462 @@ pub mod systemv; pub mod winx64; + +use crate::ir::{Function, InstructionData, Opcode, ValueLoc}; +use crate::isa::x86::registers::{FPR, RU}; +use crate::isa::{RegUnit, TargetIsa}; +use crate::result::CodegenResult; +use alloc::vec::Vec; +use std::collections::HashMap; + +use crate::isa::unwind::input::{UnwindCode, UnwindInfo}; + +pub(crate) fn create_unwind_info( + func: &Function, + isa: &dyn TargetIsa, + frame_register: Option, +) -> CodegenResult>> { + // Find last block based on max offset. + let last_block = func + .layout + .blocks() + .max_by_key(|b| func.offsets[*b]) + .expect("at least a block"); + // Find last instruction offset + size, and make it function size. + let function_size = func + .inst_offsets(last_block, &isa.encoding_info()) + .fold(0, |_, (offset, _, size)| offset + size); + + let entry_block = func.layout.entry_block().expect("missing entry block"); + let prologue_end = func.prologue_end.unwrap(); + let epilogues_start = func + .epilogues_start + .iter() + .map(|(i, b)| (*b, *i)) + .collect::>(); + + let mut stack_size = None; + let mut prologue_size = 0; + let mut prologue_unwind_codes = Vec::new(); + let mut epilogues_unwind_codes = Vec::new(); + + // Process only entry block and blocks with epilogues. + let mut blocks = func + .epilogues_start + .iter() + .map(|(_, b)| *b) + .collect::>(); + if !blocks.contains(&entry_block) { + blocks.push(entry_block); + } + blocks.sort_by_key(|b| func.offsets[*b]); + + for block in blocks.iter() { + let mut in_prologue = block == &entry_block; + let mut in_epilogue = false; + let mut epilogue_pop_offsets = Vec::new(); + + let epilogue_start = epilogues_start.get(block); + let is_last_block = block == &last_block; + + for (offset, inst, size) in func.inst_offsets(*block, &isa.encoding_info()) { + let offset = offset + size; + + let unwind_codes; + if in_prologue { + // Check for prologue end (inclusive) + if prologue_end == inst { + in_prologue = false; + } + prologue_size += size; + unwind_codes = &mut prologue_unwind_codes; + } else if !in_epilogue && epilogue_start == Some(&inst) { + // Now in an epilogue, emit a remember state instruction if not last block + in_epilogue = true; + + epilogues_unwind_codes.push(Vec::new()); + unwind_codes = epilogues_unwind_codes.last_mut().unwrap(); + + if !is_last_block { + unwind_codes.push(UnwindCode::RememberState { offset }); + } + } else if in_epilogue { + unwind_codes = epilogues_unwind_codes.last_mut().unwrap(); + } else { + // Ignore normal instructions + continue; + } + + match func.dfg[inst] { + InstructionData::Unary { opcode, arg } => { + match opcode { + Opcode::X86Push => { + let reg = func.locations[arg].unwrap_reg(); + unwind_codes.push(UnwindCode::SaveRegister { offset, reg }); + } + Opcode::AdjustSpDown => { + let stack_size = + stack_size.expect("expected a previous stack size instruction"); + + // This is used when calling a stack check function + // We need to track the assignment to RAX which has the size of the stack + unwind_codes.push(UnwindCode::StackAlloc { + offset, + size: stack_size, + }); + } + _ => {} + } + } + InstructionData::UnaryImm { opcode, imm } => { + match opcode { + Opcode::Iconst => { + let imm: i64 = imm.into(); + assert!(imm <= core::u32::MAX as i64); + assert!(stack_size.is_none()); + + // This instruction should only appear in a prologue to pass an + // argument of the stack size to a stack check function. + // Record the stack size so we know what it is when we encounter the adjustment + // instruction (which will adjust via the register assigned to this instruction). + stack_size = Some(imm as u32); + } + Opcode::AdjustSpDownImm => { + let imm: i64 = imm.into(); + assert!(imm <= core::u32::MAX as i64); + + stack_size = Some(imm as u32); + + unwind_codes.push(UnwindCode::StackAlloc { + offset, + size: imm as u32, + }); + } + Opcode::AdjustSpUpImm => { + let imm: i64 = imm.into(); + assert!(imm <= core::u32::MAX as i64); + + stack_size = Some(imm as u32); + + unwind_codes.push(UnwindCode::StackDealloc { + offset, + size: imm as u32, + }); + } + _ => {} + } + } + InstructionData::Store { + opcode: Opcode::Store, + args: [arg1, arg2], + offset: stack_offset, + .. + } => { + if let (ValueLoc::Reg(src), ValueLoc::Reg(dst)) = + (func.locations[arg1], func.locations[arg2]) + { + // If this is a save of an FPR, record an unwind operation + // Note: the stack_offset here is relative to an adjusted SP + if dst == (RU::rsp as RegUnit) && FPR.contains(src) { + let stack_offset: i32 = stack_offset.into(); + unwind_codes.push(UnwindCode::SaveXmmRegister { + offset, + reg: src, + stack_offset: stack_offset as u32, + }); + } + } + } + InstructionData::CopySpecial { src, dst, .. } => { + if let Some(fp) = frame_register { + // Check for change in CFA register (RSP is always the starting CFA) + if src == (RU::rsp as RegUnit) && dst == fp { + unwind_codes.push(UnwindCode::SetFramePointer { offset, reg: dst }); + } + } + } + InstructionData::NullAry { opcode } => match opcode { + Opcode::X86Pop => { + epilogue_pop_offsets.push(offset); + } + _ => {} + }, + InstructionData::MultiAry { opcode, .. } if in_epilogue => match opcode { + Opcode::Return => { + let args = func.dfg.inst_args(inst); + for (i, arg) in args.iter().rev().enumerate() { + // Only walk back the args for the pop instructions encountered + if i >= epilogue_pop_offsets.len() { + break; + } + + let offset = epilogue_pop_offsets[i]; + + let reg = func.locations[*arg].unwrap_reg(); + unwind_codes.push(UnwindCode::RestoreRegister { offset, reg }); + } + epilogue_pop_offsets.clear(); + + if !is_last_block { + unwind_codes.push(UnwindCode::RestoreState { offset }); + } + + in_epilogue = false; + } + _ => {} + }, + _ => {} + }; + } + } + + Ok(Some(UnwindInfo { + prologue_size, + prologue_unwind_codes, + epilogues_unwind_codes, + function_size, + })) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::cursor::{Cursor, FuncCursor}; + use crate::ir::{ + types, AbiParam, ExternalName, InstBuilder, Signature, StackSlotData, StackSlotKind, + }; + use crate::isa::{lookup, CallConv}; + use crate::settings::{builder, Flags}; + use crate::Context; + use std::str::FromStr; + use target_lexicon::triple; + + #[test] + #[cfg_attr(feature = "x64", should_panic)] // TODO #2079 + fn test_small_alloc() { + let isa = lookup(triple!("x86_64")) + .expect("expect x86 ISA") + .finish(Flags::new(builder())); + + let mut context = Context::for_function(create_function( + CallConv::WindowsFastcall, + Some(StackSlotData::new(StackSlotKind::ExplicitSlot, 64)), + )); + + context.compile(&*isa).expect("expected compilation"); + + let unwind = create_unwind_info(&context.func, &*isa, None) + .expect("can create unwind info") + .expect("expected unwind info"); + + assert_eq!( + unwind, + UnwindInfo { + prologue_size: 9, + prologue_unwind_codes: vec![ + UnwindCode::SaveRegister { + offset: 2, + reg: RU::rbp.into(), + }, + UnwindCode::StackAlloc { + offset: 9, + size: 64 + } + ], + epilogues_unwind_codes: vec![vec![ + UnwindCode::StackDealloc { + offset: 13, + size: 64 + }, + UnwindCode::RestoreRegister { + offset: 15, + reg: RU::rbp.into() + } + ]], + function_size: 16, + } + ); + } + + #[test] + #[cfg_attr(feature = "x64", should_panic)] // TODO #2079 + fn test_medium_alloc() { + let isa = lookup(triple!("x86_64")) + .expect("expect x86 ISA") + .finish(Flags::new(builder())); + + let mut context = Context::for_function(create_function( + CallConv::WindowsFastcall, + Some(StackSlotData::new(StackSlotKind::ExplicitSlot, 10000)), + )); + + context.compile(&*isa).expect("expected compilation"); + + let unwind = create_unwind_info(&context.func, &*isa, None) + .expect("can create unwind info") + .expect("expected unwind info"); + + assert_eq!( + unwind, + UnwindInfo { + prologue_size: 27, + prologue_unwind_codes: vec![ + UnwindCode::SaveRegister { + offset: 2, + reg: RU::rbp.into(), + }, + UnwindCode::StackAlloc { + offset: 27, + size: 10000 + } + ], + epilogues_unwind_codes: vec![vec![ + UnwindCode::StackDealloc { + offset: 34, + size: 10000 + }, + UnwindCode::RestoreRegister { + offset: 36, + reg: RU::rbp.into() + } + ]], + function_size: 37, + } + ); + } + + #[test] + #[cfg_attr(feature = "x64", should_panic)] // TODO #2079 + fn test_large_alloc() { + let isa = lookup(triple!("x86_64")) + .expect("expect x86 ISA") + .finish(Flags::new(builder())); + + let mut context = Context::for_function(create_function( + CallConv::WindowsFastcall, + Some(StackSlotData::new(StackSlotKind::ExplicitSlot, 1000000)), + )); + + context.compile(&*isa).expect("expected compilation"); + + let unwind = create_unwind_info(&context.func, &*isa, None) + .expect("can create unwind info") + .expect("expected unwind info"); + + assert_eq!( + unwind, + UnwindInfo { + prologue_size: 27, + prologue_unwind_codes: vec![ + UnwindCode::SaveRegister { + offset: 2, + reg: RU::rbp.into(), + }, + UnwindCode::StackAlloc { + offset: 27, + size: 1000000 + } + ], + epilogues_unwind_codes: vec![vec![ + UnwindCode::StackDealloc { + offset: 34, + size: 1000000 + }, + UnwindCode::RestoreRegister { + offset: 36, + reg: RU::rbp.into() + } + ]], + function_size: 37, + } + ); + } + + fn create_function(call_conv: CallConv, stack_slot: Option) -> Function { + let mut func = + Function::with_name_signature(ExternalName::user(0, 0), Signature::new(call_conv)); + + let block0 = func.dfg.make_block(); + let mut pos = FuncCursor::new(&mut func); + pos.insert_block(block0); + pos.ins().return_(&[]); + + if let Some(stack_slot) = stack_slot { + func.stack_slots.push(stack_slot); + } + + func + } + + #[test] + #[cfg_attr(feature = "x64", should_panic)] // TODO #2079 + fn test_multi_return_func() { + let isa = lookup(triple!("x86_64")) + .expect("expect x86 ISA") + .finish(Flags::new(builder())); + + let mut context = Context::for_function(create_multi_return_function(CallConv::SystemV)); + + context.compile(&*isa).expect("expected compilation"); + + let unwind = create_unwind_info(&context.func, &*isa, Some(RU::rbp.into())) + .expect("can create unwind info") + .expect("expected unwind info"); + + assert_eq!( + unwind, + UnwindInfo { + prologue_size: 5, + prologue_unwind_codes: vec![ + UnwindCode::SaveRegister { + offset: 2, + reg: RU::rbp.into() + }, + UnwindCode::SetFramePointer { + offset: 5, + reg: RU::rbp.into() + } + ], + epilogues_unwind_codes: vec![ + vec![ + UnwindCode::RememberState { offset: 12 }, + UnwindCode::RestoreRegister { + offset: 12, + reg: RU::rbp.into() + }, + UnwindCode::RestoreState { offset: 13 } + ], + vec![UnwindCode::RestoreRegister { + offset: 15, + reg: RU::rbp.into() + }] + ], + function_size: 16, + } + ); + } + + fn create_multi_return_function(call_conv: CallConv) -> Function { + let mut sig = Signature::new(call_conv); + sig.params.push(AbiParam::new(types::I32)); + let mut func = Function::with_name_signature(ExternalName::user(0, 0), sig); + + let block0 = func.dfg.make_block(); + let v0 = func.dfg.append_block_param(block0, types::I32); + let block1 = func.dfg.make_block(); + let block2 = func.dfg.make_block(); + + let mut pos = FuncCursor::new(&mut func); + pos.insert_block(block0); + pos.ins().brnz(v0, block2, &[]); + pos.ins().jump(block1, &[]); + + pos.insert_block(block1); + pos.ins().return_(&[]); + + pos.insert_block(block2); + pos.ins().return_(&[]); + + func + } +} diff --git a/cranelift/codegen/src/isa/x86/unwind/systemv.rs b/cranelift/codegen/src/isa/x86/unwind/systemv.rs index e292f39b95..7bc5b283ac 100644 --- a/cranelift/codegen/src/isa/x86/unwind/systemv.rs +++ b/cranelift/codegen/src/isa/x86/unwind/systemv.rs @@ -1,13 +1,11 @@ //! Unwind information for System V ABI (x86-64). -use crate::ir::{Function, Inst, InstructionData, Opcode, Value}; +use crate::ir::Function; use crate::isa::{ - unwind::systemv::{CallFrameInstruction, RegisterMappingError, UnwindInfo}, - x86::registers::RU, + unwind::systemv::{RegisterMappingError, UnwindInfo}, CallConv, RegUnit, TargetIsa, }; -use crate::result::{CodegenError, CodegenResult}; -use alloc::vec::Vec; +use crate::result::CodegenResult; use gimli::{write::CommonInformationEntry, Encoding, Format, Register, X86_64}; /// Creates a new x86-64 common information entry (CIE). @@ -94,191 +92,6 @@ pub fn map_reg(isa: &dyn TargetIsa, reg: RegUnit) -> Result { - func: &'a Function, - isa: &'a dyn TargetIsa, - cfa_offset: i32, - frame_register: Option, - instructions: Vec<(u32, CallFrameInstruction)>, - stack_size: Option, - epilogue_pop_offsets: Vec, -} - -impl<'a> InstructionBuilder<'a> { - fn new(func: &'a Function, isa: &'a dyn TargetIsa, frame_register: Option) -> Self { - Self { - func, - isa, - cfa_offset: 8, // CFA offset starts at 8 to account to return address on stack - frame_register, - instructions: Vec::new(), - stack_size: None, - epilogue_pop_offsets: Vec::new(), - } - } - - fn push_reg(&mut self, offset: u32, arg: Value) -> Result<(), RegisterMappingError> { - self.cfa_offset += 8; - - let reg = self.func.locations[arg].unwrap_reg(); - - // Update the CFA if this is the save of the frame pointer register or if a frame pointer isn't being used - // When using a frame pointer, we only need to update the CFA to account for the push of the frame pointer itself - if match self.frame_register { - Some(fp) => reg == fp, - None => true, - } { - self.instructions - .push((offset, CallFrameInstruction::CfaOffset(self.cfa_offset))); - } - - // Pushes in the prologue are register saves, so record an offset of the save - self.instructions.push(( - offset, - CallFrameInstruction::Offset(map_reg(self.isa, reg)?.0, -self.cfa_offset), - )); - - Ok(()) - } - - fn adjust_sp_down(&mut self, offset: u32) { - // Don't adjust the CFA if we're using a frame pointer - if self.frame_register.is_some() { - return; - } - - self.cfa_offset += self - .stack_size - .expect("expected a previous stack size instruction"); - self.instructions - .push((offset, CallFrameInstruction::CfaOffset(self.cfa_offset))); - } - - fn adjust_sp_down_imm(&mut self, offset: u32, imm: i64) { - assert!(imm <= core::u32::MAX as i64); - - // Don't adjust the CFA if we're using a frame pointer - if self.frame_register.is_some() { - return; - } - - self.cfa_offset += imm as i32; - self.instructions - .push((offset, CallFrameInstruction::CfaOffset(self.cfa_offset))); - } - - fn adjust_sp_up_imm(&mut self, offset: u32, imm: i64) { - assert!(imm <= core::u32::MAX as i64); - - // Don't adjust the CFA if we're using a frame pointer - if self.frame_register.is_some() { - return; - } - - self.cfa_offset -= imm as i32; - self.instructions - .push((offset, CallFrameInstruction::CfaOffset(self.cfa_offset))); - } - - fn move_reg( - &mut self, - offset: u32, - src: RegUnit, - dst: RegUnit, - ) -> Result<(), RegisterMappingError> { - if let Some(fp) = self.frame_register { - // Check for change in CFA register (RSP is always the starting CFA) - if src == (RU::rsp as RegUnit) && dst == fp { - self.instructions.push(( - offset, - CallFrameInstruction::CfaRegister(map_reg(self.isa, dst)?.0), - )); - } - } - - Ok(()) - } - - fn prologue_imm_const(&mut self, imm: i64) { - assert!(imm <= core::u32::MAX as i64); - assert!(self.stack_size.is_none()); - - // This instruction should only appear in a prologue to pass an - // argument of the stack size to a stack check function. - // Record the stack size so we know what it is when we encounter the adjustment - // instruction (which will adjust via the register assigned to this instruction). - self.stack_size = Some(imm as i32); - } - - fn ret(&mut self, inst: Inst) -> Result<(), RegisterMappingError> { - let args = self.func.dfg.inst_args(inst); - - for (i, arg) in args.iter().rev().enumerate() { - // Only walk back the args for the pop instructions encountered - if i >= self.epilogue_pop_offsets.len() { - break; - } - - self.cfa_offset -= 8; - let reg = self.func.locations[*arg].unwrap_reg(); - - // Update the CFA if this is the restore of the frame pointer register or if a frame pointer isn't being used - match self.frame_register { - Some(fp) => { - if reg == fp { - self.instructions.push(( - self.epilogue_pop_offsets[i], - CallFrameInstruction::Cfa( - map_reg(self.isa, RU::rsp as RegUnit)?.0, - self.cfa_offset, - ), - )); - } - } - None => { - self.instructions.push(( - self.epilogue_pop_offsets[i], - CallFrameInstruction::CfaOffset(self.cfa_offset), - )); - - // Pops in the epilogue are register restores, so record a "same value" for the register - // This isn't necessary when using a frame pointer as the CFA doesn't change for CSR restores - self.instructions.push(( - self.epilogue_pop_offsets[i], - CallFrameInstruction::SameValue(map_reg(self.isa, reg)?.0), - )); - } - }; - } - - self.epilogue_pop_offsets.clear(); - - Ok(()) - } - - fn insert_pop_offset(&mut self, offset: u32) { - self.epilogue_pop_offsets.push(offset); - } - - fn remember_state(&mut self, offset: u32) { - self.instructions - .push((offset, CallFrameInstruction::RememberState)); - } - - fn restore_state(&mut self, offset: u32) { - self.instructions - .push((offset, CallFrameInstruction::RestoreState)); - } - - fn is_prologue_end(&self, inst: Inst) -> bool { - self.func.prologue_end == Some(inst) - } - - fn is_epilogue_start(&self, inst: Inst) -> bool { - self.func.epilogues_start.contains(&inst) - } -} - pub(crate) fn create_unwind_info( func: &Function, isa: &dyn TargetIsa, @@ -293,93 +106,32 @@ pub(crate) fn create_unwind_info( if func.prologue_end.is_none() || isa.name() != "x86" || isa.pointer_bits() != 64 { return Ok(None); } + const WORD_SIZE: u8 = 8; // bytes - let mut builder = InstructionBuilder::new(func, isa, frame_register); - let mut in_prologue = true; - let mut in_epilogue = false; - let mut len = 0; + let unwind = match super::create_unwind_info(func, isa, frame_register)? { + Some(u) => u, + None => { + return Ok(None); + } + }; - let mut blocks = func.layout.blocks().collect::>(); - blocks.sort_by_key(|b| func.offsets[*b]); - - for (i, block) in blocks.iter().enumerate() { - for (offset, inst, size) in func.inst_offsets(*block, &isa.encoding_info()) { - let offset = offset + size; - assert!(len <= offset); - len = offset; - - let is_last_block = i == blocks.len() - 1; - - if in_prologue { - // Check for prologue end (inclusive) - in_prologue = !builder.is_prologue_end(inst); - } else if !in_epilogue && builder.is_epilogue_start(inst) { - // Now in an epilogue, emit a remember state instruction if not last block - in_epilogue = true; - - if !is_last_block { - builder.remember_state(offset); - } - } else if !in_epilogue { - // Ignore normal instructions - continue; - } - - match builder.func.dfg[inst] { - InstructionData::Unary { opcode, arg } => match opcode { - Opcode::X86Push => { - builder - .push_reg(offset, arg) - .map_err(CodegenError::RegisterMappingError)?; - } - Opcode::AdjustSpDown => { - builder.adjust_sp_down(offset); - } - _ => {} - }, - InstructionData::CopySpecial { src, dst, .. } => { - builder - .move_reg(offset, src, dst) - .map_err(CodegenError::RegisterMappingError)?; - } - InstructionData::NullAry { opcode } => match opcode { - Opcode::X86Pop => { - builder.insert_pop_offset(offset); - } - _ => {} - }, - InstructionData::UnaryImm { opcode, imm } => match opcode { - Opcode::Iconst => { - builder.prologue_imm_const(imm.into()); - } - Opcode::AdjustSpDownImm => { - builder.adjust_sp_down_imm(offset, imm.into()); - } - Opcode::AdjustSpUpImm => { - builder.adjust_sp_up_imm(offset, imm.into()); - } - _ => {} - }, - InstructionData::MultiAry { opcode, .. } => match opcode { - Opcode::Return => { - builder - .ret(inst) - .map_err(CodegenError::RegisterMappingError)?; - - if !is_last_block { - builder.restore_state(offset); - } - - in_epilogue = false; - } - _ => {} - }, - _ => {} - }; + struct RegisterMapper<'a, 'b>(&'a (dyn TargetIsa + 'b)); + impl<'a, 'b> crate::isa::unwind::systemv::RegisterMapper for RegisterMapper<'a, 'b> { + fn map(&self, reg: RegUnit) -> Result { + Ok(map_reg(self.0, reg)?.0) + } + fn rsp(&self) -> u16 { + X86_64::RSP.0 } } + let map = RegisterMapper(isa); - Ok(Some(UnwindInfo::new(builder.instructions, len))) + Ok(Some(UnwindInfo::build( + unwind, + WORD_SIZE, + frame_register, + &map, + )?)) } #[cfg(test)] @@ -460,7 +212,7 @@ mod tests { _ => panic!("expected unwind information"), }; - assert_eq!(format!("{:?}", fde), "FrameDescriptionEntry { address: Constant(4321), length: 16, lsda: None, instructions: [(2, CfaOffset(16)), (2, Offset(Register(6), -16)), (5, CfaRegister(Register(6))), (12, RememberState), (12, Cfa(Register(7), 8)), (13, RestoreState), (15, Cfa(Register(7), 0))] }"); + assert_eq!(format!("{:?}", fde), "FrameDescriptionEntry { address: Constant(4321), length: 16, lsda: None, instructions: [(2, CfaOffset(16)), (2, Offset(Register(6), -16)), (5, CfaRegister(Register(6))), (12, RememberState), (12, Cfa(Register(7), 8)), (13, RestoreState), (15, Cfa(Register(7), 8))] }"); } fn create_multi_return_function(call_conv: CallConv) -> Function { diff --git a/cranelift/codegen/src/isa/x86/unwind/winx64.rs b/cranelift/codegen/src/isa/x86/unwind/winx64.rs index c52112d78d..98c07949a2 100644 --- a/cranelift/codegen/src/isa/x86/unwind/winx64.rs +++ b/cranelift/codegen/src/isa/x86/unwind/winx64.rs @@ -1,14 +1,9 @@ //! Unwind information for Windows x64 ABI. -use crate::ir::{Function, InstructionData, Opcode, ValueLoc}; -use crate::isa::x86::registers::{FPR, GPR, RU}; -use crate::isa::{ - unwind::winx64::{UnwindCode, UnwindInfo}, - CallConv, RegUnit, TargetIsa, -}; -use crate::result::{CodegenError, CodegenResult}; -use alloc::vec::Vec; -use log::warn; +use crate::ir::Function; +use crate::isa::x86::registers::{FPR, GPR}; +use crate::isa::{unwind::winx64::UnwindInfo, CallConv, RegUnit, TargetIsa}; +use crate::result::CodegenResult; pub(crate) fn create_unwind_info( func: &Function, @@ -19,115 +14,29 @@ pub(crate) fn create_unwind_info( return Ok(None); } - let prologue_end = func.prologue_end.unwrap(); - let entry_block = func.layout.entry_block().expect("missing entry block"); - - // Stores the stack size when SP is not adjusted via an immediate value - let mut stack_size = None; - let mut prologue_size = 0; - let mut unwind_codes = Vec::new(); - let mut found_end = false; - - for (offset, inst, size) in func.inst_offsets(entry_block, &isa.encoding_info()) { - // x64 ABI prologues cannot exceed 255 bytes in length - if (offset + size) > 255 { - warn!("function prologues cannot exceed 255 bytes in size for Windows x64"); - return Err(CodegenError::CodeTooLarge); + let unwind = match super::create_unwind_info(func, isa, None)? { + Some(u) => u, + None => { + return Ok(None); } + }; - prologue_size += size; + Ok(Some(UnwindInfo::build::(unwind)?)) +} - let unwind_offset = (offset + size) as u8; +struct RegisterMapper; - match func.dfg[inst] { - InstructionData::Unary { opcode, arg } => { - match opcode { - Opcode::X86Push => { - unwind_codes.push(UnwindCode::PushRegister { - offset: unwind_offset, - reg: GPR.index_of(func.locations[arg].unwrap_reg()) as u8, - }); - } - Opcode::AdjustSpDown => { - let stack_size = - stack_size.expect("expected a previous stack size instruction"); - - // This is used when calling a stack check function - // We need to track the assignment to RAX which has the size of the stack - unwind_codes.push(UnwindCode::StackAlloc { - offset: unwind_offset, - size: stack_size, - }); - } - _ => {} - } - } - InstructionData::UnaryImm { opcode, imm } => { - match opcode { - Opcode::Iconst => { - let imm: i64 = imm.into(); - assert!(imm <= core::u32::MAX as i64); - assert!(stack_size.is_none()); - - // This instruction should only appear in a prologue to pass an - // argument of the stack size to a stack check function. - // Record the stack size so we know what it is when we encounter the adjustment - // instruction (which will adjust via the register assigned to this instruction). - stack_size = Some(imm as u32); - } - Opcode::AdjustSpDownImm => { - let imm: i64 = imm.into(); - assert!(imm <= core::u32::MAX as i64); - - stack_size = Some(imm as u32); - - unwind_codes.push(UnwindCode::StackAlloc { - offset: unwind_offset, - size: imm as u32, - }); - } - _ => {} - } - } - InstructionData::Store { - opcode: Opcode::Store, - args: [arg1, arg2], - offset, - .. - } => { - if let (ValueLoc::Reg(src), ValueLoc::Reg(dst)) = - (func.locations[arg1], func.locations[arg2]) - { - // If this is a save of an FPR, record an unwind operation - // Note: the stack_offset here is relative to an adjusted SP - if dst == (RU::rsp as RegUnit) && FPR.contains(src) { - let offset: i32 = offset.into(); - unwind_codes.push(UnwindCode::SaveXmm { - offset: unwind_offset, - reg: src as u8, - stack_offset: offset as u32, - }); - } - } - } - _ => {} - }; - - if inst == prologue_end { - found_end = true; - break; +impl crate::isa::unwind::winx64::RegisterMapper for RegisterMapper { + fn map(reg: RegUnit) -> u8 { + if GPR.contains(reg) { + GPR.index_of(reg) as u8 + } else if FPR.contains(reg) { + // XMM register + reg as u8 + } else { + panic!() } } - - assert!(found_end); - - Ok(Some(UnwindInfo { - flags: 0, // this assumes cranelift functions have no SEH handlers - prologue_size: prologue_size as u8, - frame_register: None, - frame_register_offset: 0, - unwind_codes, - })) } #[cfg(test)] @@ -135,6 +44,8 @@ mod tests { use super::*; use crate::cursor::{Cursor, FuncCursor}; use crate::ir::{ExternalName, InstBuilder, Signature, StackSlotData, StackSlotKind}; + use crate::isa::unwind::winx64::UnwindCode; + use crate::isa::x86::registers::RU; use crate::isa::{lookup, CallConv}; use crate::settings::{builder, Flags}; use crate::Context;