diff --git a/compiler/rustc_codegen_gcc/src/lib.rs b/compiler/rustc_codegen_gcc/src/lib.rs index 5d03d2406870f..7fd9c28c43f2b 100644 --- a/compiler/rustc_codegen_gcc/src/lib.rs +++ b/compiler/rustc_codegen_gcc/src/lib.rs @@ -86,9 +86,7 @@ use rustc_codegen_ssa::back::write::{ }; use rustc_codegen_ssa::base::codegen_crate; use rustc_codegen_ssa::target_features::cfg_target_feature; -use rustc_codegen_ssa::traits::{ - CodegenBackend, ExtraBackendMethods, ThinBufferMethods, WriteBackendMethods, -}; +use rustc_codegen_ssa::traits::{CodegenBackend, ExtraBackendMethods, WriteBackendMethods}; use rustc_codegen_ssa::{CodegenResults, CompiledModule, ModuleCodegen, TargetConfig}; use rustc_data_structures::fx::FxIndexMap; use rustc_data_structures::profiling::SelfProfilerRef; @@ -423,20 +421,11 @@ unsafe impl Send for SyncContext {} // FIXME(antoyo): that shouldn't be Sync. Parallel compilation is currently disabled with "CodegenBackend::supports_parallel()". unsafe impl Sync for SyncContext {} -pub struct ThinBuffer; - -impl ThinBufferMethods for ThinBuffer { - fn data(&self) -> &[u8] { - &[] - } -} - impl WriteBackendMethods for GccCodegenBackend { type Module = GccContext; type TargetMachine = (); type ModuleBuffer = ModuleBuffer; type ThinData = (); - type ThinBuffer = ThinBuffer; fn run_and_optimize_fat_lto( cgcx: &CodegenContext, @@ -458,7 +447,7 @@ impl WriteBackendMethods for GccCodegenBackend { // FIXME(bjorn3): Limit LTO exports to these symbols _exported_symbols_for_lto: &[String], _each_linked_rlib_for_lto: &[PathBuf], - _modules: Vec<(String, Self::ThinBuffer)>, + _modules: Vec<(String, Self::ModuleBuffer)>, _cached_modules: Vec<(SerializedModule, WorkProduct)>, ) -> (Vec>, Vec) { unreachable!() @@ -502,11 +491,7 @@ impl WriteBackendMethods for GccCodegenBackend { back::write::codegen(cgcx, prof, shared_emitter, module, config) } - fn prepare_thin(_module: ModuleCodegen) -> (String, Self::ThinBuffer) { - unreachable!() - } - - fn serialize_module(_module: ModuleCodegen) -> (String, Self::ModuleBuffer) { + fn serialize_module(_module: Self::Module, _is_thin: bool) -> Self::ModuleBuffer { unimplemented!(); } } diff --git a/compiler/rustc_codegen_llvm/src/back/lto.rs b/compiler/rustc_codegen_llvm/src/back/lto.rs index 5d272d10930b1..c235437aee751 100644 --- a/compiler/rustc_codegen_llvm/src/back/lto.rs +++ b/compiler/rustc_codegen_llvm/src/back/lto.rs @@ -2,7 +2,6 @@ use std::collections::BTreeMap; use std::ffi::{CStr, CString}; use std::fs::File; use std::path::{Path, PathBuf}; -use std::ptr::NonNull; use std::sync::Arc; use std::{io, iter, slice}; @@ -187,7 +186,7 @@ pub(crate) fn run_thin( dcx: DiagCtxtHandle<'_>, exported_symbols_for_lto: &[String], each_linked_rlib_for_lto: &[PathBuf], - modules: Vec<(String, ThinBuffer)>, + modules: Vec<(String, ModuleBuffer)>, cached_modules: Vec<(SerializedModule, WorkProduct)>, ) -> (Vec>, Vec) { let (symbols_below_threshold, upstream_modules) = @@ -203,12 +202,6 @@ pub(crate) fn run_thin( thin_lto(cgcx, prof, dcx, modules, upstream_modules, cached_modules, &symbols_below_threshold) } -pub(crate) fn prepare_thin(module: ModuleCodegen) -> (String, ThinBuffer) { - let name = module.name; - let buffer = ThinBuffer::new(module.module_llvm.llmod(), true); - (name, buffer) -} - fn fat_lto( cgcx: &CodegenContext, prof: &SelfProfilerRef, @@ -297,7 +290,7 @@ fn fat_lto( // way we know of to do that is to serialize them to a string and them parse // them later. Not great but hey, that's why it's "fat" LTO, right? for module in in_memory { - let buffer = ModuleBuffer::new(module.module_llvm.llmod()); + let buffer = ModuleBuffer::new(module.module_llvm.llmod(), false); let llmod_id = CString::new(&module.name[..]).unwrap(); serialized_modules.push((SerializedModule::Local(buffer), llmod_id)); } @@ -400,7 +393,7 @@ fn thin_lto( cgcx: &CodegenContext, prof: &SelfProfilerRef, dcx: DiagCtxtHandle<'_>, - modules: Vec<(String, ThinBuffer)>, + modules: Vec<(String, ModuleBuffer)>, serialized_modules: Vec<(SerializedModule, CString)>, cached_modules: Vec<(SerializedModule, WorkProduct)>, symbols_below_threshold: &[*const libc::c_char], @@ -634,7 +627,9 @@ pub(crate) fn run_pass_manager( }; unsafe { - write::llvm_optimize(cgcx, prof, dcx, module, None, config, opt_level, opt_stage, stage); + write::llvm_optimize( + cgcx, prof, dcx, module, None, None, config, opt_level, opt_stage, stage, + ); } if cfg!(feature = "llvm_enzyme") && enable_ad && !thin { @@ -643,7 +638,7 @@ pub(crate) fn run_pass_manager( if !config.autodiff.contains(&config::AutoDiff::NoPostopt) { unsafe { write::llvm_optimize( - cgcx, prof, dcx, module, None, config, opt_level, opt_stage, stage, + cgcx, prof, dcx, module, None, None, config, opt_level, opt_stage, stage, ); } } @@ -658,31 +653,26 @@ pub(crate) fn run_pass_manager( debug!("lto done"); } -pub struct ModuleBuffer(&'static mut llvm::ModuleBuffer); - -unsafe impl Send for ModuleBuffer {} -unsafe impl Sync for ModuleBuffer {} +#[repr(transparent)] +pub(crate) struct Buffer(&'static mut llvm::Buffer); -impl ModuleBuffer { - pub(crate) fn new(m: &llvm::Module) -> ModuleBuffer { - ModuleBuffer(unsafe { llvm::LLVMRustModuleBufferCreate(m) }) - } -} +unsafe impl Send for Buffer {} +unsafe impl Sync for Buffer {} -impl ModuleBufferMethods for ModuleBuffer { - fn data(&self) -> &[u8] { +impl Buffer { + pub(crate) fn data(&self) -> &[u8] { unsafe { - let ptr = llvm::LLVMRustModuleBufferPtr(self.0); - let len = llvm::LLVMRustModuleBufferLen(self.0); + let ptr = llvm::LLVMRustBufferPtr(self.0); + let len = llvm::LLVMRustBufferLen(self.0); slice::from_raw_parts(ptr, len) } } } -impl Drop for ModuleBuffer { +impl Drop for Buffer { fn drop(&mut self) { unsafe { - llvm::LLVMRustModuleBufferFree(&mut *(self.0 as *mut _)); + llvm::LLVMRustBufferFree(&mut *(self.0 as *mut _)); } } } @@ -700,48 +690,22 @@ impl Drop for ThinData { } } -pub struct ThinBuffer(&'static mut llvm::ThinLTOBuffer); - -unsafe impl Send for ThinBuffer {} -unsafe impl Sync for ThinBuffer {} - -impl ThinBuffer { - pub(crate) fn new(m: &llvm::Module, is_thin: bool) -> ThinBuffer { - unsafe { - let buffer = llvm::LLVMRustThinLTOBufferCreate(m, is_thin); - ThinBuffer(buffer) - } - } - - pub(crate) unsafe fn from_raw_ptr(ptr: *mut llvm::ThinLTOBuffer) -> ThinBuffer { - let mut ptr = NonNull::new(ptr).unwrap(); - ThinBuffer(unsafe { ptr.as_mut() }) - } - - pub(crate) fn thin_link_data(&self) -> &[u8] { - unsafe { - let ptr = llvm::LLVMRustThinLTOBufferThinLinkDataPtr(self.0) as *const _; - let len = llvm::LLVMRustThinLTOBufferThinLinkDataLen(self.0); - slice::from_raw_parts(ptr, len) - } - } +pub struct ModuleBuffer { + data: Buffer, } -impl ThinBufferMethods for ThinBuffer { - fn data(&self) -> &[u8] { +impl ModuleBuffer { + pub(crate) fn new(m: &llvm::Module, is_thin: bool) -> ModuleBuffer { unsafe { - let ptr = llvm::LLVMRustThinLTOBufferPtr(self.0) as *const _; - let len = llvm::LLVMRustThinLTOBufferLen(self.0); - slice::from_raw_parts(ptr, len) + let buffer = llvm::LLVMRustModuleSerialize(m, is_thin); + ModuleBuffer { data: Buffer(buffer) } } } } -impl Drop for ThinBuffer { - fn drop(&mut self) { - unsafe { - llvm::LLVMRustThinLTOBufferFree(&mut *(self.0 as *mut _)); - } +impl ModuleBufferMethods for ModuleBuffer { + fn data(&self) -> &[u8] { + self.data.data() } } diff --git a/compiler/rustc_codegen_llvm/src/back/write.rs b/compiler/rustc_codegen_llvm/src/back/write.rs index 3e3ccd39e674c..d7ab1356fafe7 100644 --- a/compiler/rustc_codegen_llvm/src/back/write.rs +++ b/compiler/rustc_codegen_llvm/src/back/write.rs @@ -1,7 +1,6 @@ use std::ffi::{CStr, CString}; use std::io::{self, Write}; use std::path::{Path, PathBuf}; -use std::ptr::null_mut; use std::sync::Arc; use std::{fs, slice, str}; @@ -29,7 +28,7 @@ use rustc_target::spec::{ }; use tracing::{debug, trace}; -use crate::back::lto::ThinBuffer; +use crate::back::lto::{Buffer, ModuleBuffer}; use crate::back::owned_target_machine::OwnedTargetMachine; use crate::back::profiling::{ LlvmSelfProfiler, selfprofile_after_pass_callback, selfprofile_before_pass_callback, @@ -563,7 +562,8 @@ pub(crate) unsafe fn llvm_optimize( prof: &SelfProfilerRef, dcx: DiagCtxtHandle<'_>, module: &ModuleCodegen, - thin_lto_buffer: Option<&mut *mut llvm::ThinLTOBuffer>, + thin_lto_buffer: Option<&mut Option>, + thin_lto_summary_buffer: Option<&mut Option>, config: &ModuleConfig, opt_level: config::OptLevel, opt_stage: llvm::OptStage, @@ -786,8 +786,7 @@ pub(crate) unsafe fn llvm_optimize( config.verify_llvm_ir, config.lint_llvm_ir, thin_lto_buffer, - config.emit_thin_lto, - config.emit_thin_lto_summary, + thin_lto_summary_buffer, merge_functions, unroll_loops, vectorize_slp, @@ -933,13 +932,14 @@ pub(crate) fn optimize( // The bitcode obtained during the `codegen` phase is no longer suitable for performing LTO. // It may have undergone LTO due to ThinLocal, so we need to obtain the embedded bitcode at // this point. - let mut thin_lto_buffer = if (module.kind == ModuleKind::Regular + let (mut thin_lto_buffer, mut thin_lto_summary_buffer) = if (module.kind + == ModuleKind::Regular && config.emit_obj == EmitObj::ObjectCode(BitcodeSection::Full)) || config.emit_thin_lto_summary { - Some(null_mut()) + (Some(None), config.emit_thin_lto_summary.then_some(None)) } else { - None + (None, None) }; unsafe { llvm_optimize( @@ -948,6 +948,7 @@ pub(crate) fn optimize( dcx, module, thin_lto_buffer.as_mut(), + thin_lto_summary_buffer.as_mut(), config, opt_level, opt_stage, @@ -955,17 +956,18 @@ pub(crate) fn optimize( ) }; if let Some(thin_lto_buffer) = thin_lto_buffer { - let thin_lto_buffer = unsafe { ThinBuffer::from_raw_ptr(thin_lto_buffer) }; + let thin_lto_buffer = thin_lto_buffer.unwrap(); module.thin_lto_buffer = Some(thin_lto_buffer.data().to_vec()); let bc_summary_out = cgcx.output_filenames.temp_path_for_cgu( OutputType::ThinLinkBitcode, &module.name, cgcx.invocation_temp.as_deref(), ); - if config.emit_thin_lto_summary + if let Some(thin_lto_summary_buffer) = thin_lto_summary_buffer && let Some(thin_link_bitcode_filename) = bc_summary_out.file_name() { - let summary_data = thin_lto_buffer.thin_link_data(); + let thin_lto_summary_buffer = thin_lto_summary_buffer.unwrap(); + let summary_data = thin_lto_summary_buffer.data(); prof.artifact_size( "llvm_bitcode_summary", thin_link_bitcode_filename.to_string_lossy(), @@ -1033,7 +1035,7 @@ pub(crate) fn codegen( "LLVM_module_codegen_make_bitcode", &*module.name, ); - ThinBuffer::new(llmod, config.emit_thin_lto) + ModuleBuffer::new(llmod, cgcx.lto != Lto::Fat) }; let data = thin.data(); let _timer = prof diff --git a/compiler/rustc_codegen_llvm/src/lib.rs b/compiler/rustc_codegen_llvm/src/lib.rs index 3310fe4f43f83..373001afa1a45 100644 --- a/compiler/rustc_codegen_llvm/src/lib.rs +++ b/compiler/rustc_codegen_llvm/src/lib.rs @@ -153,7 +153,6 @@ impl WriteBackendMethods for LlvmCodegenBackend { type ModuleBuffer = back::lto::ModuleBuffer; type TargetMachine = OwnedTargetMachine; type ThinData = back::lto::ThinData; - type ThinBuffer = back::lto::ThinBuffer; fn print_pass_timings(&self) { let timings = llvm::build_string(|s| unsafe { llvm::LLVMRustPrintPassTimings(s) }).unwrap(); print!("{timings}"); @@ -193,7 +192,7 @@ impl WriteBackendMethods for LlvmCodegenBackend { dcx: DiagCtxtHandle<'_>, exported_symbols_for_lto: &[String], each_linked_rlib_for_lto: &[PathBuf], - modules: Vec<(String, Self::ThinBuffer)>, + modules: Vec<(String, Self::ModuleBuffer)>, cached_modules: Vec<(SerializedModule, WorkProduct)>, ) -> (Vec>, Vec) { back::lto::run_thin( @@ -233,11 +232,8 @@ impl WriteBackendMethods for LlvmCodegenBackend { ) -> CompiledModule { back::write::codegen(cgcx, prof, shared_emitter, module, config) } - fn prepare_thin(module: ModuleCodegen) -> (String, Self::ThinBuffer) { - back::lto::prepare_thin(module) - } - fn serialize_module(module: ModuleCodegen) -> (String, Self::ModuleBuffer) { - (module.name, back::lto::ModuleBuffer::new(module.module_llvm.llmod())) + fn serialize_module(module: Self::Module, is_thin: bool) -> Self::ModuleBuffer { + back::lto::ModuleBuffer::new(module.llmod(), is_thin) } } diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index 4956549f2ed3b..f9af42494cada 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -536,9 +536,6 @@ pub(crate) enum DiagnosticLevel { unsafe extern "C" { // LLVMRustThinLTOData pub(crate) type ThinLTOData; - - // LLVMRustThinLTOBuffer - pub(crate) type ThinLTOBuffer; } /// LLVMRustThinLTOModule @@ -848,7 +845,7 @@ bitflags! { } unsafe extern "C" { - pub(crate) type ModuleBuffer; + pub(crate) type Buffer; } pub(crate) type SelfProfileBeforePassCallback = @@ -2375,9 +2372,8 @@ unsafe extern "C" { NoPrepopulatePasses: bool, VerifyIR: bool, LintIR: bool, - ThinLTOBuffer: Option<&mut *mut ThinLTOBuffer>, - EmitThinLTO: bool, - EmitThinLTOSummary: bool, + ThinLTOBuffer: Option<&mut Option>, + ThinLTOSummaryBuffer: Option<&mut Option>, MergeFunctions: bool, UnrollLoops: bool, SLPVectorize: bool, @@ -2458,22 +2454,13 @@ unsafe extern "C" { pub(crate) fn LLVMRustSetModulePICLevel(M: &Module); pub(crate) fn LLVMRustSetModulePIELevel(M: &Module); pub(crate) fn LLVMRustSetModuleCodeModel(M: &Module, Model: CodeModel); - pub(crate) fn LLVMRustModuleBufferCreate(M: &Module) -> &'static mut ModuleBuffer; - pub(crate) fn LLVMRustModuleBufferPtr(p: &ModuleBuffer) -> *const u8; - pub(crate) fn LLVMRustModuleBufferLen(p: &ModuleBuffer) -> usize; - pub(crate) fn LLVMRustModuleBufferFree(p: &'static mut ModuleBuffer); + pub(crate) fn LLVMRustBufferPtr(p: &Buffer) -> *const u8; + pub(crate) fn LLVMRustBufferLen(p: &Buffer) -> usize; + pub(crate) fn LLVMRustBufferFree(p: &'static mut Buffer); pub(crate) fn LLVMRustModuleCost(M: &Module) -> u64; pub(crate) fn LLVMRustModuleInstructionStats(M: &Module) -> u64; - pub(crate) fn LLVMRustThinLTOBufferCreate( - M: &Module, - is_thin: bool, - ) -> &'static mut ThinLTOBuffer; - pub(crate) fn LLVMRustThinLTOBufferFree(M: &'static mut ThinLTOBuffer); - pub(crate) fn LLVMRustThinLTOBufferPtr(M: &ThinLTOBuffer) -> *const c_char; - pub(crate) fn LLVMRustThinLTOBufferLen(M: &ThinLTOBuffer) -> size_t; - pub(crate) fn LLVMRustThinLTOBufferThinLinkDataPtr(M: &ThinLTOBuffer) -> *const c_char; - pub(crate) fn LLVMRustThinLTOBufferThinLinkDataLen(M: &ThinLTOBuffer) -> size_t; + pub(crate) fn LLVMRustModuleSerialize(M: &Module, is_thin: bool) -> &'static mut Buffer; pub(crate) fn LLVMRustCreateThinLTOData( Modules: *const ThinLTOModule, NumModules: size_t, diff --git a/compiler/rustc_codegen_ssa/src/back/lto.rs b/compiler/rustc_codegen_ssa/src/back/lto.rs index 80b3b5a4d7c06..a2c951c16d28d 100644 --- a/compiler/rustc_codegen_ssa/src/back/lto.rs +++ b/compiler/rustc_codegen_ssa/src/back/lto.rs @@ -42,7 +42,7 @@ impl ThinModule { pub struct ThinShared { pub data: B::ThinData, - pub thin_buffers: Vec, + pub thin_buffers: Vec, pub serialized_modules: Vec>, pub module_names: Vec, } diff --git a/compiler/rustc_codegen_ssa/src/back/write.rs b/compiler/rustc_codegen_ssa/src/back/write.rs index 87a043fbdf245..53a7174e25398 100644 --- a/compiler/rustc_codegen_ssa/src/back/write.rs +++ b/compiler/rustc_codegen_ssa/src/back/write.rs @@ -101,7 +101,6 @@ pub struct ModuleConfig { pub emit_ir: bool, pub emit_asm: bool, pub emit_obj: EmitObj, - pub emit_thin_lto: bool, pub emit_thin_lto_summary: bool, // Miscellaneous flags. These are mostly copied from command-line @@ -212,9 +211,6 @@ impl ModuleConfig { false ), emit_obj, - // thin lto summaries prevent fat lto, so do not emit them if fat - // lto is requested. See PR #136840 for background information. - emit_thin_lto: sess.opts.unstable_opts.emit_thin_lto && sess.lto() != Lto::Fat, emit_thin_lto_summary: if_regular!( sess.opts.output_types.contains_key(&OutputType::ThinLinkBitcode), false @@ -368,7 +364,7 @@ fn generate_thin_lto_work( dcx: DiagCtxtHandle<'_>, exported_symbols_for_lto: &[String], each_linked_rlib_for_lto: &[PathBuf], - needs_thin_lto: Vec<(String, B::ThinBuffer)>, + needs_thin_lto: Vec<(String, B::ModuleBuffer)>, import_only_modules: Vec<(SerializedModule, WorkProduct)>, ) -> Vec<(ThinLtoWorkItem, u64)> { let _prof_timer = prof.generic_activity("codegen_thin_generate_lto_work"); @@ -422,7 +418,7 @@ enum MaybeLtoModules { cgcx: CodegenContext, exported_symbols_for_lto: Arc>, each_linked_rlib_file_for_lto: Vec, - needs_thin_lto: Vec<(String, ::ThinBuffer)>, + needs_thin_lto: Vec<(String, ::ModuleBuffer)>, lto_import_only_modules: Vec<(SerializedModule<::ModuleBuffer>, WorkProduct)>, }, @@ -800,7 +796,7 @@ pub(crate) enum WorkItemResult { /// The backend has finished compiling a CGU, which now needs to go through /// thin LTO. - NeedsThinLto(String, B::ThinBuffer), + NeedsThinLto(String, B::ModuleBuffer), } pub enum FatLtoInput { @@ -875,22 +871,22 @@ fn execute_optimize_work_item( WorkItemResult::Finished(module) } ComputedLtoType::Thin => { - let (name, thin_buffer) = B::prepare_thin(module); + let thin_buffer = B::serialize_module(module.module_llvm, true); if let Some(path) = bitcode { fs::write(&path, thin_buffer.data()).unwrap_or_else(|e| { panic!("Error writing pre-lto-bitcode file `{}`: {}", path.display(), e); }); } - WorkItemResult::NeedsThinLto(name, thin_buffer) + WorkItemResult::NeedsThinLto(module.name, thin_buffer) } ComputedLtoType::Fat => match bitcode { Some(path) => { - let (name, buffer) = B::serialize_module(module); + let buffer = B::serialize_module(module.module_llvm, false); fs::write(&path, buffer.data()).unwrap_or_else(|e| { panic!("Error writing pre-lto-bitcode file `{}`: {}", path.display(), e); }); WorkItemResult::NeedsFatLto(FatLtoInput::Serialized { - name, + name: module.name, buffer: SerializedModule::Local(buffer), }) } @@ -1027,7 +1023,7 @@ fn do_thin_lto( tm_factory: TargetMachineFactoryFn, exported_symbols_for_lto: Arc>, each_linked_rlib_for_lto: Vec, - needs_thin_lto: Vec<(String, ::ThinBuffer)>, + needs_thin_lto: Vec<(String, ::ModuleBuffer)>, lto_import_only_modules: Vec<( SerializedModule<::ModuleBuffer>, WorkProduct, @@ -1811,8 +1807,8 @@ fn start_executing_work( )); } else { if let Some(allocator_module) = allocator_module.take() { - let (name, thin_buffer) = B::prepare_thin(allocator_module); - needs_thin_lto.push((name, thin_buffer)); + let thin_buffer = B::serialize_module(allocator_module.module_llvm, true); + needs_thin_lto.push((allocator_module.name, thin_buffer)); } return Ok(MaybeLtoModules::ThinLto { diff --git a/compiler/rustc_codegen_ssa/src/traits/mod.rs b/compiler/rustc_codegen_ssa/src/traits/mod.rs index 6d1ac717c0b8f..f46d07ea5008e 100644 --- a/compiler/rustc_codegen_ssa/src/traits/mod.rs +++ b/compiler/rustc_codegen_ssa/src/traits/mod.rs @@ -48,7 +48,7 @@ pub use self::type_::{ ArgAbiBuilderMethods, BaseTypeCodegenMethods, DerivedTypeCodegenMethods, LayoutTypeCodegenMethods, TypeCodegenMethods, TypeMembershipCodegenMethods, }; -pub use self::write::{ModuleBufferMethods, ThinBufferMethods, WriteBackendMethods}; +pub use self::write::{ModuleBufferMethods, WriteBackendMethods}; pub trait CodegenObject = Copy + fmt::Debug; diff --git a/compiler/rustc_codegen_ssa/src/traits/write.rs b/compiler/rustc_codegen_ssa/src/traits/write.rs index 5f5d0ac5d9fc4..d33dfc1d014b5 100644 --- a/compiler/rustc_codegen_ssa/src/traits/write.rs +++ b/compiler/rustc_codegen_ssa/src/traits/write.rs @@ -15,7 +15,6 @@ pub trait WriteBackendMethods: Clone + 'static { type TargetMachine; type ModuleBuffer: ModuleBufferMethods; type ThinData: Send + Sync; - type ThinBuffer: ThinBufferMethods; /// Performs fat LTO by merging all modules into a single one, running autodiff /// if necessary and running any further optimizations @@ -37,7 +36,7 @@ pub trait WriteBackendMethods: Clone + 'static { dcx: DiagCtxtHandle<'_>, exported_symbols_for_lto: &[String], each_linked_rlib_for_lto: &[PathBuf], - modules: Vec<(String, Self::ThinBuffer)>, + modules: Vec<(String, Self::ModuleBuffer)>, cached_modules: Vec<(SerializedModule, WorkProduct)>, ) -> (Vec>, Vec); fn print_pass_timings(&self); @@ -63,12 +62,7 @@ pub trait WriteBackendMethods: Clone + 'static { module: ModuleCodegen, config: &ModuleConfig, ) -> CompiledModule; - fn prepare_thin(module: ModuleCodegen) -> (String, Self::ThinBuffer); - fn serialize_module(module: ModuleCodegen) -> (String, Self::ModuleBuffer); -} - -pub trait ThinBufferMethods: Send + Sync { - fn data(&self) -> &[u8]; + fn serialize_module(module: Self::Module, is_thin: bool) -> Self::ModuleBuffer; } pub trait ModuleBufferMethods: Send + Sync { diff --git a/compiler/rustc_interface/src/tests.rs b/compiler/rustc_interface/src/tests.rs index 5ebf898f45a34..eabc6baed94d0 100644 --- a/compiler/rustc_interface/src/tests.rs +++ b/compiler/rustc_interface/src/tests.rs @@ -796,7 +796,6 @@ fn test_unstable_options_tracking_hash() { tracked!(dwarf_version, Some(5)); tracked!(embed_metadata, false); tracked!(embed_source, true); - tracked!(emit_thin_lto, false); tracked!(emscripten_wasm_eh, false); tracked!(export_executable_symbols, true); tracked!(fewer_names, Some(true)); diff --git a/compiler/rustc_llvm/llvm-wrapper/LLVMWrapper.h b/compiler/rustc_llvm/llvm-wrapper/LLVMWrapper.h index f6598f9faf522..0cbda23f384cc 100644 --- a/compiler/rustc_llvm/llvm-wrapper/LLVMWrapper.h +++ b/compiler/rustc_llvm/llvm-wrapper/LLVMWrapper.h @@ -46,4 +46,8 @@ class RawRustStringOstream : public llvm::raw_ostream { } }; +struct LLVMRustBuffer { + std::string data; +}; + #endif // INCLUDED_RUSTC_LLVM_LLVMWRAPPER_H diff --git a/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp index 7aa4ddea78e1d..d1af0b4f13f35 100644 --- a/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp +++ b/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp @@ -87,19 +87,6 @@ extern "C" void LLVMRustTimeTraceProfilerFinish(const char *FileName) { timeTraceProfilerCleanup(); } -// This struct and various functions are sort of a hack right now, but the -// problem is that we've got in-memory LLVM modules after we generate and -// optimize all codegen-units for one compilation in rustc. To be compatible -// with the LTO support above we need to serialize the modules plus their -// ThinLTO summary into memory. -// -// This structure is basically an owned version of a serialize module, with -// a ThinLTO summary attached. -struct LLVMRustThinLTOBuffer { - std::string data; - std::string thin_link_data; -}; - extern "C" bool LLVMRustHasFeature(LLVMTargetMachineRef TM, const char *Feature) { TargetMachine *Target = unwrap(TM); @@ -566,11 +553,12 @@ extern "C" LLVMRustResult LLVMRustOptimize( LLVMModuleRef ModuleRef, LLVMTargetMachineRef TMRef, LLVMRustPassBuilderOptLevel OptLevelRust, LLVMRustOptStage OptStage, bool IsLinkerPluginLTO, bool NoPrepopulatePasses, bool VerifyIR, - bool LintIR, LLVMRustThinLTOBuffer **ThinLTOBufferRef, bool EmitThinLTO, - bool EmitThinLTOSummary, bool MergeFunctions, bool UnrollLoops, - bool SLPVectorize, bool LoopVectorize, bool DisableSimplifyLibCalls, - bool EmitLifetimeMarkers, registerEnzymeAndPassPipelineFn EnzymePtr, - bool PrintBeforeEnzyme, bool PrintAfterEnzyme, bool PrintPasses, + bool LintIR, LLVMRustBuffer **ThinLTOBufferRef, + LLVMRustBuffer **ThinLTOSummaryBufferRef, bool MergeFunctions, + bool UnrollLoops, bool SLPVectorize, bool LoopVectorize, + bool DisableSimplifyLibCalls, bool EmitLifetimeMarkers, + registerEnzymeAndPassPipelineFn EnzymePtr, bool PrintBeforeEnzyme, + bool PrintAfterEnzyme, bool PrintPasses, LLVMRustSanitizerOptions *SanitizerOptions, const char *PGOGenPath, const char *PGOUsePath, bool InstrumentCoverage, const char *InstrProfileOutput, const char *PGOSampleUsePath, @@ -808,31 +796,27 @@ extern "C" LLVMRustResult LLVMRustOptimize( } ModulePassManager MPM; - bool NeedThinLTOBufferPasses = EmitThinLTO; - auto ThinLTOBuffer = std::make_unique(); + bool NeedThinLTOBufferPasses = true; + auto ThinLTOBuffer = std::make_unique(); + auto ThinLTOSummaryBuffer = std::make_unique(); raw_string_ostream ThinLTODataOS(ThinLTOBuffer->data); - raw_string_ostream ThinLinkDataOS(ThinLTOBuffer->thin_link_data); + raw_string_ostream ThinLinkDataOS(ThinLTOSummaryBuffer->data); bool IsLTO = OptStage == LLVMRustOptStage::ThinLTO || OptStage == LLVMRustOptStage::FatLTO; if (!NoPrepopulatePasses) { + for (const auto &C : PipelineStartEPCallbacks) + PB.registerPipelineStartEPCallback(C); + for (const auto &C : OptimizerLastEPCallbacks) + PB.registerOptimizerLastEPCallback(C); + // The pre-link pipelines don't support O0 and require using // buildO0DefaultPipeline() instead. At the same time, the LTO pipelines do // support O0 and using them is required. if (OptLevel == OptimizationLevel::O0 && !IsLTO) { - for (const auto &C : PipelineStartEPCallbacks) - PB.registerPipelineStartEPCallback(C); - for (const auto &C : OptimizerLastEPCallbacks) - PB.registerOptimizerLastEPCallback(C); - // We manually schedule ThinLTOBufferPasses below, so don't pass the value // to enable it here. MPM = PB.buildO0DefaultPipeline(OptLevel); } else { - for (const auto &C : PipelineStartEPCallbacks) - PB.registerPipelineStartEPCallback(C); - for (const auto &C : OptimizerLastEPCallbacks) - PB.registerOptimizerLastEPCallback(C); - switch (OptStage) { case LLVMRustOptStage::PreLinkNoLTO: if (ThinLTOBufferRef) { @@ -840,13 +824,13 @@ extern "C" LLVMRustResult LLVMRustOptimize( // bitcode for embedding is obtained after performing // `ThinLTOPreLinkDefaultPipeline`. MPM.addPass(PB.buildThinLTOPreLinkDefaultPipeline(OptLevel)); - if (EmitThinLTO) { - MPM.addPass(ThinLTOBitcodeWriterPass( - ThinLTODataOS, EmitThinLTOSummary ? &ThinLinkDataOS : nullptr)); - } else { - MPM.addPass(BitcodeWriterPass(ThinLTODataOS)); - } + MPM.addPass(ThinLTOBitcodeWriterPass( + ThinLTODataOS, + ThinLTOSummaryBufferRef ? &ThinLinkDataOS : nullptr)); *ThinLTOBufferRef = ThinLTOBuffer.release(); + if (ThinLTOSummaryBufferRef) { + *ThinLTOSummaryBufferRef = ThinLTOSummaryBuffer.release(); + } MPM.addPass(PB.buildModuleOptimizationPipeline( OptLevel, ThinOrFullLTOPhase::None)); MPM.addPass( @@ -870,6 +854,7 @@ extern "C" LLVMRustResult LLVMRustOptimize( break; case LLVMRustOptStage::FatLTO: MPM = PB.buildLTODefaultPipeline(OptLevel, nullptr); + NeedThinLTOBufferPasses = false; break; } } @@ -897,13 +882,18 @@ extern "C" LLVMRustResult LLVMRustOptimize( } // For `-Copt-level=0`, ThinLTO, or LTO. if (ThinLTOBufferRef && *ThinLTOBufferRef == nullptr) { - if (EmitThinLTO) { + // thin lto summaries prevent fat lto, so do not emit them if fat + // lto is requested. See PR #136840 for background information. + if (OptStage != LLVMRustOptStage::PreLinkFatLTO) { MPM.addPass(ThinLTOBitcodeWriterPass( - ThinLTODataOS, EmitThinLTOSummary ? &ThinLinkDataOS : nullptr)); + ThinLTODataOS, ThinLTOSummaryBufferRef ? &ThinLinkDataOS : nullptr)); } else { MPM.addPass(BitcodeWriterPass(ThinLTODataOS)); } *ThinLTOBufferRef = ThinLTOBuffer.release(); + if (ThinLTOSummaryBufferRef) { + *ThinLTOSummaryBufferRef = ThinLTOSummaryBuffer.release(); + } } // now load "-enzyme" pass: @@ -1414,9 +1404,9 @@ extern "C" bool LLVMRustPrepareThinLTOImport(const LLVMRustThinLTOData *Data, return true; } -extern "C" LLVMRustThinLTOBuffer *LLVMRustThinLTOBufferCreate(LLVMModuleRef M, - bool is_thin) { - auto Ret = std::make_unique(); +extern "C" LLVMRustBuffer *LLVMRustModuleSerialize(LLVMModuleRef M, + bool is_thin) { + auto Ret = std::make_unique(); { auto OS = raw_string_ostream(Ret->data); { @@ -1442,30 +1432,6 @@ extern "C" LLVMRustThinLTOBuffer *LLVMRustThinLTOBufferCreate(LLVMModuleRef M, return Ret.release(); } -extern "C" void LLVMRustThinLTOBufferFree(LLVMRustThinLTOBuffer *Buffer) { - delete Buffer; -} - -extern "C" const void * -LLVMRustThinLTOBufferPtr(const LLVMRustThinLTOBuffer *Buffer) { - return Buffer->data.data(); -} - -extern "C" size_t -LLVMRustThinLTOBufferLen(const LLVMRustThinLTOBuffer *Buffer) { - return Buffer->data.length(); -} - -extern "C" const void * -LLVMRustThinLTOBufferThinLinkDataPtr(const LLVMRustThinLTOBuffer *Buffer) { - return Buffer->thin_link_data.data(); -} - -extern "C" size_t -LLVMRustThinLTOBufferThinLinkDataLen(const LLVMRustThinLTOBuffer *Buffer) { - return Buffer->thin_link_data.length(); -} - // This is what we used to parse upstream bitcode for actual ThinLTO // processing. We'll call this once per module optimized through ThinLTO, and // it'll be called concurrently on many threads. diff --git a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp index f68f4e71520b3..eabc1c94f26e9 100644 --- a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp +++ b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp @@ -1529,29 +1529,13 @@ extern "C" void LLVMRustSetDSOLocal(LLVMValueRef Global, bool is_dso_local) { unwrap(Global)->setDSOLocal(is_dso_local); } -struct LLVMRustModuleBuffer { - std::string data; -}; - -extern "C" LLVMRustModuleBuffer *LLVMRustModuleBufferCreate(LLVMModuleRef M) { - auto Ret = std::make_unique(); - { - auto OS = raw_string_ostream(Ret->data); - WriteBitcodeToFile(*unwrap(M), OS); - } - return Ret.release(); -} - -extern "C" void LLVMRustModuleBufferFree(LLVMRustModuleBuffer *Buffer) { - delete Buffer; -} +extern "C" void LLVMRustBufferFree(LLVMRustBuffer *Buffer) { delete Buffer; } -extern "C" const void * -LLVMRustModuleBufferPtr(const LLVMRustModuleBuffer *Buffer) { +extern "C" const void *LLVMRustBufferPtr(const LLVMRustBuffer *Buffer) { return Buffer->data.data(); } -extern "C" size_t LLVMRustModuleBufferLen(const LLVMRustModuleBuffer *Buffer) { +extern "C" size_t LLVMRustBufferLen(const LLVMRustBuffer *Buffer) { return Buffer->data.length(); } diff --git a/compiler/rustc_session/src/options.rs b/compiler/rustc_session/src/options.rs index 9219b5a7e8aca..a5086f2232c50 100644 --- a/compiler/rustc_session/src/options.rs +++ b/compiler/rustc_session/src/options.rs @@ -2335,8 +2335,6 @@ options! { "embed source text in DWARF debug sections (default: no)"), emit_stack_sizes: bool = (false, parse_bool, [UNTRACKED], "emit a section containing stack size metadata (default: no)"), - emit_thin_lto: bool = (true, parse_bool, [TRACKED], - "emit the bc module with thin LTO info (default: yes)"), emscripten_wasm_eh: bool = (true, parse_bool, [TRACKED], "Use WebAssembly error handling for wasm32-unknown-emscripten"), enforce_type_length_limit: bool = (false, parse_bool, [TRACKED],