Importing rustc-1.57.0
Bug: 203802952
Test: ./build.py --lto=thin
Change-Id: I45a7d6b548bf423bac860f01bd5ad978a95fe6df
diff --git a/compiler/rustc_codegen_llvm/Cargo.toml b/compiler/rustc_codegen_llvm/Cargo.toml
index 521ce34..a6a553b 100644
--- a/compiler/rustc_codegen_llvm/Cargo.toml
+++ b/compiler/rustc_codegen_llvm/Cargo.toml
@@ -1,7 +1,7 @@
[package]
name = "rustc_codegen_llvm"
version = "0.0.0"
-edition = "2018"
+edition = "2021"
[lib]
test = false
@@ -16,6 +16,7 @@
tracing = "0.1"
rustc_middle = { path = "../rustc_middle" }
rustc-demangle = "0.1.21"
+rustc_arena = { path = "../rustc_arena" }
rustc_attr = { path = "../rustc_attr" }
rustc_codegen_ssa = { path = "../rustc_codegen_ssa" }
rustc_data_structures = { path = "../rustc_data_structures" }
@@ -25,6 +26,7 @@
rustc_index = { path = "../rustc_index" }
rustc_llvm = { path = "../rustc_llvm" }
rustc_metadata = { path = "../rustc_metadata" }
+rustc_query_system = { path = "../rustc_query_system" }
rustc_session = { path = "../rustc_session" }
rustc_serialize = { path = "../rustc_serialize" }
rustc_target = { path = "../rustc_target" }
diff --git a/compiler/rustc_codegen_llvm/src/abi.rs b/compiler/rustc_codegen_llvm/src/abi.rs
index abf0ea8..dca9c1f 100644
--- a/compiler/rustc_codegen_llvm/src/abi.rs
+++ b/compiler/rustc_codegen_llvm/src/abi.rs
@@ -11,11 +11,12 @@
use rustc_codegen_ssa::traits::*;
use rustc_codegen_ssa::MemFlags;
use rustc_middle::bug;
+use rustc_middle::ty::layout::LayoutOf;
pub use rustc_middle::ty::layout::{FAT_PTR_ADDR, FAT_PTR_EXTRA};
use rustc_middle::ty::Ty;
use rustc_target::abi::call::ArgAbi;
pub use rustc_target::abi::call::*;
-use rustc_target::abi::{self, HasDataLayout, Int, LayoutOf};
+use rustc_target::abi::{self, HasDataLayout, Int};
pub use rustc_target::spec::abi::Abi;
use libc::c_uint;
@@ -510,7 +511,12 @@
}
fn apply_attrs_callsite(&self, bx: &mut Builder<'a, 'll, 'tcx>, callsite: &'ll Value) {
- // FIXME(wesleywiser, eddyb): We should apply `nounwind` and `noreturn` as appropriate to this callsite.
+ if self.ret.layout.abi.is_uninhabited() {
+ llvm::Attribute::NoReturn.apply_callsite(llvm::AttributePlace::Function, callsite);
+ }
+ if !self.can_unwind {
+ llvm::Attribute::NoUnwind.apply_callsite(llvm::AttributePlace::Function, callsite);
+ }
let mut i = 0;
let mut apply = |cx: &CodegenCx<'_, '_>, attrs: &ArgAttributes| {
@@ -520,7 +526,7 @@
};
match self.ret.mode {
PassMode::Direct(ref attrs) => {
- attrs.apply_attrs_to_callsite(llvm::AttributePlace::ReturnValue, &bx.cx, callsite);
+ attrs.apply_attrs_to_callsite(llvm::AttributePlace::ReturnValue, bx.cx, callsite);
}
PassMode::Indirect { ref attrs, extra_attrs: _, on_stack } => {
assert!(!on_stack);
@@ -535,16 +541,13 @@
}
_ => {}
}
- if let abi::Abi::Scalar(ref scalar) = self.ret.layout.abi {
+ if let abi::Abi::Scalar(scalar) = self.ret.layout.abi {
// If the value is a boolean, the range is 0..2 and that ultimately
// become 0..0 when the type becomes i1, which would be rejected
// by the LLVM verifier.
if let Int(..) = scalar.value {
- if !scalar.is_bool() {
- let range = scalar.valid_range_exclusive(bx);
- if range.start != range.end {
- bx.range_metadata(callsite, range);
- }
+ if !scalar.is_bool() && !scalar.is_always_valid(bx) {
+ bx.range_metadata(callsite, scalar.valid_range);
}
}
}
diff --git a/compiler/rustc_codegen_llvm/src/allocator.rs b/compiler/rustc_codegen_llvm/src/allocator.rs
index 2d79b73..30d91b4 100644
--- a/compiler/rustc_codegen_llvm/src/allocator.rs
+++ b/compiler/rustc_codegen_llvm/src/allocator.rs
@@ -3,19 +3,22 @@
use rustc_ast::expand::allocator::{AllocatorKind, AllocatorTy, ALLOCATOR_METHODS};
use rustc_middle::bug;
use rustc_middle::ty::TyCtxt;
+use rustc_session::config::DebugInfo;
use rustc_span::symbol::sym;
+use crate::debuginfo;
use crate::llvm::{self, False, True};
use crate::ModuleLlvm;
pub(crate) unsafe fn codegen(
tcx: TyCtxt<'_>,
- mods: &mut ModuleLlvm,
+ module_llvm: &mut ModuleLlvm,
+ module_name: &str,
kind: AllocatorKind,
has_alloc_error_handler: bool,
) {
- let llcx = &*mods.llcx;
- let llmod = mods.llmod();
+ let llcx = &*module_llvm.llcx;
+ let llmod = module_llvm.llmod();
let usize = match tcx.sess.target.pointer_width {
16 => llvm::LLVMInt16TypeInContext(llcx),
32 => llvm::LLVMInt32TypeInContext(llcx),
@@ -132,4 +135,10 @@
llvm::LLVMSetTailCall(ret, True);
llvm::LLVMBuildRetVoid(llbuilder);
llvm::LLVMDisposeBuilder(llbuilder);
+
+ if tcx.sess.opts.debuginfo != DebugInfo::None {
+ let dbg_cx = debuginfo::CrateDebugContext::new(llmod);
+ debuginfo::metadata::compile_unit_metadata(tcx, module_name, &dbg_cx);
+ dbg_cx.finalize(tcx.sess);
+ }
}
diff --git a/compiler/rustc_codegen_llvm/src/asm.rs b/compiler/rustc_codegen_llvm/src/asm.rs
index e0d3127..341a888 100644
--- a/compiler/rustc_codegen_llvm/src/asm.rs
+++ b/compiler/rustc_codegen_llvm/src/asm.rs
@@ -105,7 +105,7 @@
let r = r.unwrap();
// Again, based on how many outputs we have
- let outputs = ia.outputs.iter().zip(&outputs).filter(|&(ref o, _)| !o.is_indirect);
+ let outputs = ia.outputs.iter().zip(&outputs).filter(|&(o, _)| !o.is_indirect);
for (i, (_, &place)) in outputs.enumerate() {
let v = if num_outputs == 1 { r } else { self.extract_value(r, i as u64) };
OperandValue::Immediate(v).store(self, place);
@@ -331,7 +331,7 @@
let output_type = match &output_types[..] {
[] => self.type_void(),
[ty] => ty,
- tys => self.type_struct(&tys, false),
+ tys => self.type_struct(tys, false),
};
let dialect = match asm_arch {
InlineAsmArch::X86 | InlineAsmArch::X86_64
@@ -792,7 +792,7 @@
/// Helper function to get the LLVM type for a Scalar. Pointers are returned as
/// the equivalent integer type.
-fn llvm_asm_scalar_type(cx: &CodegenCx<'ll, 'tcx>, scalar: &Scalar) -> &'ll Type {
+fn llvm_asm_scalar_type(cx: &CodegenCx<'ll, 'tcx>, scalar: Scalar) -> &'ll Type {
match scalar.value {
Primitive::Int(Integer::I8, _) => cx.type_i8(),
Primitive::Int(Integer::I16, _) => cx.type_i16(),
@@ -812,7 +812,7 @@
reg: InlineAsmRegClass,
layout: &TyAndLayout<'tcx>,
) -> &'ll Value {
- match (reg, &layout.abi) {
+ match (reg, layout.abi) {
(InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg), Abi::Scalar(s)) => {
if let Primitive::Int(Integer::I8, _) = s.value {
let vec_ty = bx.cx.type_vector(bx.cx.type_i8(), 8);
@@ -835,7 +835,7 @@
Abi::Vector { element, count },
) if layout.size.bytes() == 8 => {
let elem_ty = llvm_asm_scalar_type(bx.cx, element);
- let vec_ty = bx.cx.type_vector(elem_ty, *count);
+ let vec_ty = bx.cx.type_vector(elem_ty, count);
let indices: Vec<_> = (0..count * 2).map(|x| bx.const_i32(x as i32)).collect();
bx.shuffle_vector(value, bx.const_undef(vec_ty), bx.const_vector(&indices))
}
@@ -890,7 +890,7 @@
reg: InlineAsmRegClass,
layout: &TyAndLayout<'tcx>,
) -> &'ll Value {
- match (reg, &layout.abi) {
+ match (reg, layout.abi) {
(InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg), Abi::Scalar(s)) => {
if let Primitive::Int(Integer::I8, _) = s.value {
bx.extract_element(value, bx.const_i32(0))
@@ -910,8 +910,8 @@
Abi::Vector { element, count },
) if layout.size.bytes() == 8 => {
let elem_ty = llvm_asm_scalar_type(bx.cx, element);
- let vec_ty = bx.cx.type_vector(elem_ty, *count * 2);
- let indices: Vec<_> = (0..*count).map(|x| bx.const_i32(x as i32)).collect();
+ let vec_ty = bx.cx.type_vector(elem_ty, count * 2);
+ let indices: Vec<_> = (0..count).map(|x| bx.const_i32(x as i32)).collect();
bx.shuffle_vector(value, bx.const_undef(vec_ty), bx.const_vector(&indices))
}
(InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_abcd), Abi::Scalar(s))
@@ -965,7 +965,7 @@
reg: InlineAsmRegClass,
layout: &TyAndLayout<'tcx>,
) -> &'ll Type {
- match (reg, &layout.abi) {
+ match (reg, layout.abi) {
(InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg), Abi::Scalar(s)) => {
if let Primitive::Int(Integer::I8, _) = s.value {
cx.type_vector(cx.type_i8(), 8)
diff --git a/compiler/rustc_codegen_llvm/src/attributes.rs b/compiler/rustc_codegen_llvm/src/attributes.rs
index 56b93f8..659cf9e 100644
--- a/compiler/rustc_codegen_llvm/src/attributes.rs
+++ b/compiler/rustc_codegen_llvm/src/attributes.rs
@@ -263,6 +263,10 @@
attributes::emit_uwtable(llfn, true);
}
+ if cx.sess().opts.debugging_opts.profile_sample_use.is_some() {
+ llvm::AddFunctionAttrString(llfn, Function, cstr!("use-sample-profile"));
+ }
+
// FIXME: none of these three functions interact with source level attributes.
set_frame_pointer_type(cx, llfn);
set_instrument_function(cx, llfn);
@@ -305,9 +309,12 @@
let mut function_features = codegen_fn_attrs
.target_features
.iter()
- .map(|f| {
+ .flat_map(|f| {
let feature = &f.as_str();
- format!("+{}", llvm_util::to_llvm_feature(cx.tcx.sess, feature))
+ llvm_util::to_llvm_feature(cx.tcx.sess, feature)
+ .into_iter()
+ .map(|f| format!("+{}", f))
+ .collect::<Vec<String>>()
})
.chain(codegen_fn_attrs.instruction_set.iter().map(|x| match x {
InstructionSetAttr::ArmA32 => "-thumb-mode".to_string(),
diff --git a/compiler/rustc_codegen_llvm/src/back/archive.rs b/compiler/rustc_codegen_llvm/src/back/archive.rs
index 6ac7093..2fb5a0f 100644
--- a/compiler/rustc_codegen_llvm/src/back/archive.rs
+++ b/compiler/rustc_codegen_llvm/src/back/archive.rs
@@ -9,18 +9,15 @@
use crate::llvm::archive_ro::{ArchiveRO, Child};
use crate::llvm::{self, ArchiveKind, LLVMMachineType, LLVMRustCOFFShortExport};
-use rustc_codegen_ssa::back::archive::{find_library, ArchiveBuilder};
-use rustc_codegen_ssa::{looks_like_rust_object_file, METADATA_FILENAME};
+use rustc_codegen_ssa::back::archive::ArchiveBuilder;
use rustc_data_structures::temp_dir::MaybeTempDir;
-use rustc_middle::middle::cstore::{DllCallingConvention, DllImport};
+use rustc_session::cstore::{DllCallingConvention, DllImport};
use rustc_session::Session;
-use rustc_span::symbol::Symbol;
struct ArchiveConfig<'a> {
pub sess: &'a Session,
pub dst: PathBuf,
pub src: Option<PathBuf>,
- pub lib_search_paths: Vec<PathBuf>,
}
/// Helper for adding many files to an archive.
@@ -54,13 +51,7 @@
}
fn archive_config<'a>(sess: &'a Session, output: &Path, input: Option<&Path>) -> ArchiveConfig<'a> {
- use rustc_codegen_ssa::back::link::archive_search_paths;
- ArchiveConfig {
- sess,
- dst: output.to_path_buf(),
- src: input.map(|p| p.to_path_buf()),
- lib_search_paths: archive_search_paths(sess),
- }
+ ArchiveConfig { sess, dst: output.to_path_buf(), src: input.map(|p| p.to_path_buf()) }
}
/// Map machine type strings to values of LLVM's MachineTypes enum.
@@ -111,57 +102,23 @@
.collect()
}
- /// Adds all of the contents of a native library to this archive. This will
- /// search in the relevant locations for a library named `name`.
- fn add_native_library(&mut self, name: Symbol, verbatim: bool) {
- let location =
- find_library(name, verbatim, &self.config.lib_search_paths, self.config.sess);
- self.add_archive(&location, |_| false).unwrap_or_else(|e| {
- self.config.sess.fatal(&format!(
- "failed to add native library {}: {}",
- location.to_string_lossy(),
- e
- ));
+ fn add_archive<F>(&mut self, archive: &Path, skip: F) -> io::Result<()>
+ where
+ F: FnMut(&str) -> bool + 'static,
+ {
+ let archive_ro = match ArchiveRO::open(archive) {
+ Ok(ar) => ar,
+ Err(e) => return Err(io::Error::new(io::ErrorKind::Other, e)),
+ };
+ if self.additions.iter().any(|ar| ar.path() == archive) {
+ return Ok(());
+ }
+ self.additions.push(Addition::Archive {
+ path: archive.to_path_buf(),
+ archive: archive_ro,
+ skip: Box::new(skip),
});
- }
-
- /// Adds all of the contents of the rlib at the specified path to this
- /// archive.
- ///
- /// This ignores adding the bytecode from the rlib, and if LTO is enabled
- /// then the object file also isn't added.
- fn add_rlib(
- &mut self,
- rlib: &Path,
- name: &str,
- lto: bool,
- skip_objects: bool,
- ) -> io::Result<()> {
- // Ignoring obj file starting with the crate name
- // as simple comparison is not enough - there
- // might be also an extra name suffix
- let obj_start = name.to_owned();
-
- self.add_archive(rlib, move |fname: &str| {
- // Ignore metadata files, no matter the name.
- if fname == METADATA_FILENAME {
- return true;
- }
-
- // Don't include Rust objects if LTO is enabled
- if lto && looks_like_rust_object_file(fname) {
- return true;
- }
-
- // Otherwise if this is *not* a rust object and we're skipping
- // objects then skip this file
- if skip_objects && (!fname.starts_with(&obj_start) || !fname.ends_with(".o")) {
- return true;
- }
-
- // ok, don't skip this
- false
- })
+ Ok(())
}
/// Adds an arbitrary file to this archive
@@ -206,13 +163,13 @@
// All import names are Rust identifiers and therefore cannot contain \0 characters.
// FIXME: when support for #[link_name] implemented, ensure that import.name values don't
// have any \0 characters
- let import_name_vector: Vec<CString> = dll_imports
+ let import_name_and_ordinal_vector: Vec<(CString, Option<u16>)> = dll_imports
.iter()
.map(|import: &DllImport| {
if self.config.sess.target.arch == "x86" {
- LlvmArchiveBuilder::i686_decorated_name(import)
+ (LlvmArchiveBuilder::i686_decorated_name(import), import.ordinal)
} else {
- CString::new(import.name.to_string()).unwrap()
+ (CString::new(import.name.to_string()).unwrap(), import.ordinal)
}
})
.collect();
@@ -227,9 +184,9 @@
dll_imports.iter().map(|import| import.name.to_string()).collect::<Vec<_>>().join(", "),
);
- let ffi_exports: Vec<LLVMRustCOFFShortExport> = import_name_vector
+ let ffi_exports: Vec<LLVMRustCOFFShortExport> = import_name_and_ordinal_vector
.iter()
- .map(|name_z| LLVMRustCOFFShortExport::from_name(name_z.as_ptr()))
+ .map(|(name_z, ordinal)| LLVMRustCOFFShortExport::new(name_z.as_ptr(), *ordinal))
.collect();
let result = unsafe {
crate::llvm::LLVMRustWriteImportLibrary(
@@ -270,25 +227,6 @@
self.src_archive.as_ref().unwrap().as_ref()
}
- fn add_archive<F>(&mut self, archive: &Path, skip: F) -> io::Result<()>
- where
- F: FnMut(&str) -> bool + 'static,
- {
- let archive_ro = match ArchiveRO::open(archive) {
- Ok(ar) => ar,
- Err(e) => return Err(io::Error::new(io::ErrorKind::Other, e)),
- };
- if self.additions.iter().any(|ar| ar.path() == archive) {
- return Ok(());
- }
- self.additions.push(Addition::Archive {
- path: archive.to_path_buf(),
- archive: archive_ro,
- skip: Box::new(skip),
- });
- Ok(())
- }
-
fn llvm_archive_kind(&self) -> Result<ArchiveKind, &str> {
let kind = &*self.config.sess.target.archive_format;
kind.parse().map_err(|_| kind)
diff --git a/compiler/rustc_codegen_llvm/src/back/lto.rs b/compiler/rustc_codegen_llvm/src/back/lto.rs
index f612785..97780de 100644
--- a/compiler/rustc_codegen_llvm/src/back/lto.rs
+++ b/compiler/rustc_codegen_llvm/src/back/lto.rs
@@ -109,7 +109,7 @@
.extend(exported_symbols[&cnum].iter().filter_map(symbol_filter));
}
- let archive = ArchiveRO::open(&path).expect("wanted an rlib");
+ let archive = ArchiveRO::open(path).expect("wanted an rlib");
let obj_files = archive
.iter()
.filter_map(|child| child.ok().and_then(|c| c.name().map(|name| (name, c))))
@@ -316,14 +316,28 @@
.generic_activity_with_arg("LLVM_fat_lto_link_module", format!("{:?}", name));
info!("linking {:?}", name);
let data = bc_decoded.data();
- linker.add(&data).map_err(|()| {
+ linker.add(data).map_err(|()| {
let msg = format!("failed to load bc of {:?}", name);
- write::llvm_err(&diag_handler, &msg)
+ write::llvm_err(diag_handler, &msg)
})?;
serialized_bitcode.push(bc_decoded);
}
drop(linker);
- save_temp_bitcode(&cgcx, &module, "lto.input");
+ save_temp_bitcode(cgcx, &module, "lto.input");
+
+ // Fat LTO also suffers from the invalid DWARF issue similar to Thin LTO.
+ // Here we rewrite all `DICompileUnit` pointers if there is only one `DICompileUnit`.
+ // This only works around the problem when codegen-units = 1.
+ // Refer to the comments in the `optimize_thin_module` function for more details.
+ let mut cu1 = ptr::null_mut();
+ let mut cu2 = ptr::null_mut();
+ unsafe { llvm::LLVMRustLTOGetDICompileUnit(llmod, &mut cu1, &mut cu2) };
+ if !cu2.is_null() {
+ let _timer =
+ cgcx.prof.generic_activity_with_arg("LLVM_fat_lto_patch_debuginfo", &*module.name);
+ unsafe { llvm::LLVMRustLTOPatchDICompileUnit(llmod, cu1) };
+ save_temp_bitcode(cgcx, &module, "fat-lto-after-patch");
+ }
// Internalize everything below threshold to help strip out more modules and such.
unsafe {
@@ -333,14 +347,14 @@
ptr as *const *const libc::c_char,
symbols_below_threshold.len() as libc::size_t,
);
- save_temp_bitcode(&cgcx, &module, "lto.after-restriction");
+ save_temp_bitcode(cgcx, &module, "lto.after-restriction");
}
if cgcx.no_landing_pads {
unsafe {
llvm::LLVMRustMarkAllFunctionsNounwind(llmod);
}
- save_temp_bitcode(&cgcx, &module, "lto.after-nounwind");
+ save_temp_bitcode(cgcx, &module, "lto.after-nounwind");
}
}
@@ -484,7 +498,7 @@
symbols_below_threshold.as_ptr(),
symbols_below_threshold.len() as u32,
)
- .ok_or_else(|| write::llvm_err(&diag_handler, "failed to prepare thin LTO context"))?;
+ .ok_or_else(|| write::llvm_err(diag_handler, "failed to prepare thin LTO context"))?;
let data = ThinData(data);
@@ -558,7 +572,7 @@
if let Some(path) = key_map_path {
if let Err(err) = curr_key_map.save_to_file(&path) {
let msg = format!("Error while writing ThinLTO key data: {}", err);
- return Err(write::llvm_err(&diag_handler, &msg));
+ return Err(write::llvm_err(diag_handler, &msg));
}
}
@@ -582,7 +596,7 @@
// tools/lto/LTOCodeGenerator.cpp
debug!("running the pass manager");
unsafe {
- if write::should_use_new_llvm_pass_manager(config) {
+ if write::should_use_new_llvm_pass_manager(cgcx, config) {
let opt_stage = if thin { llvm::OptStage::ThinLTO } else { llvm::OptStage::FatLTO };
let opt_level = config.opt_level.unwrap_or(config::OptLevel::No);
write::optimize_with_new_llvm_pass_manager(
@@ -730,8 +744,7 @@
// crates but for locally codegened modules we may be able to reuse
// that LLVM Context and Module.
let llcx = llvm::LLVMRustContextCreate(cgcx.fewer_names);
- let llmod_raw =
- parse_module(llcx, &module_name, thin_module.data(), &diag_handler)? as *const _;
+ let llmod_raw = parse_module(llcx, module_name, thin_module.data(), &diag_handler)? as *const _;
let module = ModuleCodegen {
module_llvm: ModuleLlvm { llmod_raw, llcx, tm },
name: thin_module.name().to_string(),
@@ -740,7 +753,7 @@
{
let target = &*module.module_llvm.tm;
let llmod = module.module_llvm.llmod();
- save_temp_bitcode(&cgcx, &module, "thin-lto-input");
+ save_temp_bitcode(cgcx, &module, "thin-lto-input");
// Before we do much else find the "main" `DICompileUnit` that we'll be
// using below. If we find more than one though then rustc has changed
@@ -748,7 +761,7 @@
// an error.
let mut cu1 = ptr::null_mut();
let mut cu2 = ptr::null_mut();
- llvm::LLVMRustThinLTOGetDICompileUnit(llmod, &mut cu1, &mut cu2);
+ llvm::LLVMRustLTOGetDICompileUnit(llmod, &mut cu1, &mut cu2);
if !cu2.is_null() {
let msg = "multiple source DICompileUnits found";
return Err(write::llvm_err(&diag_handler, msg));
@@ -761,7 +774,7 @@
.prof
.generic_activity_with_arg("LLVM_thin_lto_remove_landing_pads", thin_module.name());
llvm::LLVMRustMarkAllFunctionsNounwind(llmod);
- save_temp_bitcode(&cgcx, &module, "thin-lto-after-nounwind");
+ save_temp_bitcode(cgcx, &module, "thin-lto-after-nounwind");
}
// Up next comes the per-module local analyses that we do for Thin LTO.
@@ -847,7 +860,7 @@
let _timer = cgcx
.prof
.generic_activity_with_arg("LLVM_thin_lto_patch_debuginfo", thin_module.name());
- llvm::LLVMRustThinLTOPatchDICompileUnit(llmod, cu1);
+ llvm::LLVMRustLTOPatchDICompileUnit(llmod, cu1);
save_temp_bitcode(cgcx, &module, "thin-lto-after-patch");
}
@@ -933,7 +946,7 @@
llvm::LLVMRustParseBitcodeForLTO(cx, data.as_ptr(), data.len(), name.as_ptr()).ok_or_else(
|| {
let msg = "failed to parse bitcode for LTO module";
- write::llvm_err(&diag_handler, msg)
+ write::llvm_err(diag_handler, msg)
},
)
}
diff --git a/compiler/rustc_codegen_llvm/src/back/write.rs b/compiler/rustc_codegen_llvm/src/back/write.rs
index 791604a..b6d682f 100644
--- a/compiler/rustc_codegen_llvm/src/back/write.rs
+++ b/compiler/rustc_codegen_llvm/src/back/write.rs
@@ -41,7 +41,7 @@
pub fn llvm_err(handler: &rustc_errors::Handler, msg: &str) -> FatalError {
match llvm::last_error() {
Some(err) => handler.fatal(&format!("{}: {}", msg, err)),
- None => handler.fatal(&msg),
+ None => handler.fatal(msg),
}
}
@@ -96,7 +96,7 @@
None
};
let config = TargetMachineFactoryConfig { split_dwarf_file };
- target_machine_factory(&tcx.sess, tcx.backend_optimization_level(()))(config)
+ target_machine_factory(tcx.sess, tcx.backend_optimization_level(()))(config)
.unwrap_or_else(|err| llvm_err(tcx.sess.diagnostic(), &err).raise())
}
@@ -129,7 +129,8 @@
fn to_llvm_relocation_model(relocation_model: RelocModel) -> llvm::RelocModel {
match relocation_model {
RelocModel::Static => llvm::RelocModel::Static,
- RelocModel::Pic => llvm::RelocModel::PIC,
+ // LLVM doesn't have a PIE relocation model, it represents PIE as PIC with an extra attribute.
+ RelocModel::Pic | RelocModel::Pie => llvm::RelocModel::PIC,
RelocModel::DynamicNoPic => llvm::RelocModel::DynamicNoPic,
RelocModel::Ropi => llvm::RelocModel::ROPI,
RelocModel::Rwpi => llvm::RelocModel::RWPI,
@@ -369,9 +370,22 @@
.map(|path_buf| CString::new(path_buf.to_string_lossy().as_bytes()).unwrap())
}
-pub(crate) fn should_use_new_llvm_pass_manager(config: &ModuleConfig) -> bool {
- // The new pass manager is disabled by default.
- config.new_llvm_pass_manager.unwrap_or(false)
+fn get_pgo_sample_use_path(config: &ModuleConfig) -> Option<CString> {
+ config
+ .pgo_sample_use
+ .as_ref()
+ .map(|path_buf| CString::new(path_buf.to_string_lossy().as_bytes()).unwrap())
+}
+
+pub(crate) fn should_use_new_llvm_pass_manager(
+ _cgcx: &CodegenContext<LlvmCodegenBackend>,
+ config: &ModuleConfig,
+) -> bool {
+ // The new pass manager is causing significant performance issues such as #91128, and is
+ // therefore disabled in stable versions of rustc by default.
+ config
+ .new_llvm_pass_manager
+ .unwrap_or(false)
}
pub(crate) unsafe fn optimize_with_new_llvm_pass_manager(
@@ -387,6 +401,7 @@
let using_thin_buffers = opt_stage == llvm::OptStage::PreLinkThinLTO || config.bitcode_needed();
let pgo_gen_path = get_pgo_gen_path(config);
let pgo_use_path = get_pgo_use_path(config);
+ let pgo_sample_use_path = get_pgo_sample_use_path(config);
let is_lto = opt_stage == llvm::OptStage::ThinLTO || opt_stage == llvm::OptStage::FatLTO;
// Sanitizer instrumentation is only inserted during the pre-link optimization stage.
let sanitizer_options = if !is_lto {
@@ -404,13 +419,15 @@
None
};
- let llvm_selfprofiler = if cgcx.prof.llvm_recording_enabled() {
- let mut llvm_profiler = LlvmSelfProfiler::new(cgcx.prof.get_self_profiler().unwrap());
- &mut llvm_profiler as *mut _ as *mut c_void
+ let mut llvm_profiler = if cgcx.prof.llvm_recording_enabled() {
+ Some(LlvmSelfProfiler::new(cgcx.prof.get_self_profiler().unwrap()))
} else {
- std::ptr::null_mut()
+ None
};
+ let llvm_selfprofiler =
+ llvm_profiler.as_mut().map(|s| s as *mut _ as *mut c_void).unwrap_or(std::ptr::null_mut());
+
let extra_passes = config.passes.join(",");
// FIXME: NewPM doesn't provide a facility to pass custom InlineParams.
@@ -435,6 +452,8 @@
pgo_use_path.as_ref().map_or(std::ptr::null(), |s| s.as_ptr()),
config.instrument_coverage,
config.instrument_gcov,
+ pgo_sample_use_path.as_ref().map_or(std::ptr::null(), |s| s.as_ptr()),
+ config.debug_info_for_profiling,
llvm_selfprofiler,
selfprofile_before_pass_callback,
selfprofile_after_pass_callback,
@@ -468,7 +487,7 @@
}
if let Some(opt_level) = config.opt_level {
- if should_use_new_llvm_pass_manager(config) {
+ if should_use_new_llvm_pass_manager(cgcx, config) {
let opt_stage = match cgcx.lto {
Lto::Fat => llvm::OptStage::PreLinkFatLTO,
Lto::Thin | Lto::ThinLocal => llvm::OptStage::PreLinkThinLTO,
@@ -540,6 +559,9 @@
if config.instrument_coverage {
llvm::LLVMRustAddPass(mpm, find_pass("instrprof").unwrap());
}
+ if config.debug_info_for_profiling {
+ llvm::LLVMRustAddPass(mpm, find_pass("add-discriminators").unwrap());
+ }
add_sanitizer_passes(config, &mut extra_passes);
@@ -554,7 +576,7 @@
let prepare_for_thin_lto = cgcx.lto == Lto::Thin
|| cgcx.lto == Lto::ThinLocal
|| (cgcx.lto != Lto::Fat && cgcx.opts.cg.linker_plugin_lto.enabled());
- with_llvm_pmb(llmod, &config, opt_level, prepare_for_thin_lto, &mut |b| {
+ with_llvm_pmb(llmod, config, opt_level, prepare_for_thin_lto, &mut |b| {
llvm::LLVMRustAddLastExtensionPasses(
b,
extra_passes.as_ptr(),
@@ -656,9 +678,9 @@
let _timer =
cgcx.prof.generic_activity_with_arg("LLVM_link_module", format!("{:?}", module.name));
let buffer = ModuleBuffer::new(module.module_llvm.llmod());
- linker.add(&buffer.data()).map_err(|()| {
+ linker.add(buffer.data()).map_err(|()| {
let msg = format!("failed to serialize module {:?}", module.name);
- llvm_err(&diag_handler, &msg)
+ llvm_err(diag_handler, &msg)
})?;
}
drop(linker);
@@ -997,6 +1019,7 @@
let inline_threshold = config.inline_threshold;
let pgo_gen_path = get_pgo_gen_path(config);
let pgo_use_path = get_pgo_use_path(config);
+ let pgo_sample_use_path = get_pgo_sample_use_path(config);
llvm::LLVMRustConfigurePassManagerBuilder(
builder,
@@ -1007,6 +1030,7 @@
prepare_for_thin_lto,
pgo_gen_path.as_ref().map_or(ptr::null(), |s| s.as_ptr()),
pgo_use_path.as_ref().map_or(ptr::null(), |s| s.as_ptr()),
+ pgo_sample_use_path.as_ref().map_or(ptr::null(), |s| s.as_ptr()),
);
llvm::LLVMPassManagerBuilderSetSizeLevel(builder, opt_size as u32);
diff --git a/compiler/rustc_codegen_llvm/src/base.rs b/compiler/rustc_codegen_llvm/src/base.rs
index a6bdbd1..3026c2f 100644
--- a/compiler/rustc_codegen_llvm/src/base.rs
+++ b/compiler/rustc_codegen_llvm/src/base.rs
@@ -25,9 +25,9 @@
use rustc_codegen_ssa::traits::*;
use rustc_codegen_ssa::{ModuleCodegen, ModuleKind};
use rustc_data_structures::small_c_str::SmallCStr;
+use rustc_metadata::EncodedMetadata;
use rustc_middle::dep_graph;
use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrs;
-use rustc_middle::middle::cstore::EncodedMetadata;
use rustc_middle::middle::exported_symbols;
use rustc_middle::mir::mono::{Linkage, Visibility};
use rustc_middle::ty::TyCtxt;
@@ -64,7 +64,7 @@
let (metadata_llcx, metadata_llmod) = (&*llvm_module.llcx, llvm_module.llmod());
let mut compressed = rustc_metadata::METADATA_HEADER.to_vec();
- FrameEncoder::new(&mut compressed).write_all(&metadata.raw_data).unwrap();
+ FrameEncoder::new(&mut compressed).write_all(metadata.raw_data()).unwrap();
let llmeta = common::bytes_in_context(metadata_llcx, &compressed);
let llconst = common::struct_in_context(metadata_llcx, &[llmeta], false);
diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs
index 47529f7..d5deacf 100644
--- a/compiler/rustc_codegen_llvm/src/builder.rs
+++ b/compiler/rustc_codegen_llvm/src/builder.rs
@@ -15,15 +15,17 @@
use rustc_codegen_ssa::MemFlags;
use rustc_data_structures::small_c_str::SmallCStr;
use rustc_hir::def_id::DefId;
-use rustc_middle::ty::layout::TyAndLayout;
+use rustc_middle::ty::layout::{
+ FnAbiError, FnAbiOfHelpers, FnAbiRequest, LayoutError, LayoutOfHelpers, TyAndLayout,
+};
use rustc_middle::ty::{self, Ty, TyCtxt};
use rustc_span::Span;
-use rustc_target::abi::{self, Align, Size};
+use rustc_target::abi::{self, call::FnAbi, Align, Size, WrappingRange};
use rustc_target::spec::{HasTargetSpec, Target};
use std::borrow::Cow;
use std::ffi::CStr;
use std::iter;
-use std::ops::{Deref, Range};
+use std::ops::Deref;
use std::ptr;
use tracing::debug;
@@ -84,16 +86,30 @@
impl HasTargetSpec for Builder<'_, '_, 'tcx> {
#[inline]
fn target_spec(&self) -> &Target {
- &self.cx.target_spec()
+ self.cx.target_spec()
}
}
-impl abi::LayoutOf<'tcx> for Builder<'_, '_, 'tcx> {
- type Ty = Ty<'tcx>;
- type TyAndLayout = TyAndLayout<'tcx>;
+impl LayoutOfHelpers<'tcx> for Builder<'_, '_, 'tcx> {
+ type LayoutOfResult = TyAndLayout<'tcx>;
- fn layout_of(&self, ty: Ty<'tcx>) -> Self::TyAndLayout {
- self.cx.layout_of(ty)
+ #[inline]
+ fn handle_layout_err(&self, err: LayoutError<'tcx>, span: Span, ty: Ty<'tcx>) -> ! {
+ self.cx.handle_layout_err(err, span, ty)
+ }
+}
+
+impl FnAbiOfHelpers<'tcx> for Builder<'_, '_, 'tcx> {
+ type FnAbiOfResult = &'tcx FnAbi<'tcx, Ty<'tcx>>;
+
+ #[inline]
+ fn handle_fn_abi_err(
+ &self,
+ err: FnAbiError<'tcx>,
+ span: Span,
+ fn_abi_request: FnAbiRequest<'tcx>,
+ ) -> ! {
+ self.cx.handle_fn_abi_err(err, span, fn_abi_request)
}
}
@@ -382,7 +398,7 @@
val
}
}
- fn to_immediate_scalar(&mut self, val: Self::Value, scalar: &abi::Scalar) -> Self::Value {
+ fn to_immediate_scalar(&mut self, val: Self::Value, scalar: abi::Scalar) -> Self::Value {
if scalar.is_bool() {
return self.trunc(val, self.cx().type_i1());
}
@@ -460,16 +476,15 @@
fn scalar_load_metadata<'a, 'll, 'tcx>(
bx: &mut Builder<'a, 'll, 'tcx>,
load: &'ll Value,
- scalar: &abi::Scalar,
+ scalar: abi::Scalar,
) {
match scalar.value {
abi::Int(..) => {
- let range = scalar.valid_range_exclusive(bx);
- if range.start != range.end {
- bx.range_metadata(load, range);
+ if !scalar.is_always_valid(bx) {
+ bx.range_metadata(load, scalar.valid_range);
}
}
- abi::Pointer if !scalar.valid_range.contains_zero() => {
+ abi::Pointer if !scalar.valid_range.contains(0) => {
bx.nonnull_metadata(load);
}
_ => {}
@@ -489,17 +504,17 @@
}
let llval = const_llval.unwrap_or_else(|| {
let load = self.load(place.layout.llvm_type(self), place.llval, place.align);
- if let abi::Abi::Scalar(ref scalar) = place.layout.abi {
+ if let abi::Abi::Scalar(scalar) = place.layout.abi {
scalar_load_metadata(self, load, scalar);
}
load
});
OperandValue::Immediate(self.to_immediate(llval, place.layout))
- } else if let abi::Abi::ScalarPair(ref a, ref b) = place.layout.abi {
+ } else if let abi::Abi::ScalarPair(a, b) = place.layout.abi {
let b_offset = a.value.size(self).align_to(b.value.align(self).abi);
let pair_ty = place.layout.llvm_type(self);
- let mut load = |i, scalar: &abi::Scalar, align| {
+ let mut load = |i, scalar: abi::Scalar, align| {
let llptr = self.struct_gep(pair_ty, place.llval, i as u64);
let llty = place.layout.scalar_pair_element_llvm_type(self, i, false);
let load = self.load(llty, llptr, align);
@@ -555,7 +570,7 @@
next_bx
}
- fn range_metadata(&mut self, load: &'ll Value, range: Range<u128>) {
+ fn range_metadata(&mut self, load: &'ll Value, range: WrappingRange) {
if self.sess().target.arch == "amdgpu" {
// amdgpu/LLVM does something weird and thinks an i64 value is
// split into a v2i32, halving the bitwidth LLVM expects,
@@ -568,7 +583,7 @@
let llty = self.cx.val_ty(load);
let v = [
self.cx.const_uint_big(llty, range.start),
- self.cx.const_uint_big(llty, range.end),
+ self.cx.const_uint_big(llty, range.end.wrapping_add(1)),
];
llvm::LLVMSetMetadata(
@@ -813,6 +828,7 @@
}
fn fcmp(&mut self, op: RealPredicate, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
+ let op = llvm::RealPredicate::from_generic(op);
unsafe { llvm::LLVMBuildFCmp(self.llbuilder, op as c_uint, lhs, rhs, UNNAMED) }
}
diff --git a/compiler/rustc_codegen_llvm/src/callee.rs b/compiler/rustc_codegen_llvm/src/callee.rs
index bb16c90..1bc924d 100644
--- a/compiler/rustc_codegen_llvm/src/callee.rs
+++ b/compiler/rustc_codegen_llvm/src/callee.rs
@@ -4,7 +4,7 @@
//! and methods are represented as just a fn ptr and not a full
//! closure.
-use crate::abi::{FnAbi, FnAbiLlvmExt};
+use crate::abi::FnAbiLlvmExt;
use crate::attributes;
use crate::context::CodegenCx;
use crate::llvm;
@@ -12,7 +12,7 @@
use rustc_codegen_ssa::traits::*;
use tracing::debug;
-use rustc_middle::ty::layout::{FnAbiExt, HasTyCtxt};
+use rustc_middle::ty::layout::{FnAbiOf, HasTyCtxt};
use rustc_middle::ty::{self, Instance, TypeFoldable};
/// Codegens a reference to a fn/method item, monomorphizing and
@@ -42,9 +42,9 @@
sym
);
- let fn_abi = FnAbi::of_instance(cx, instance, &[]);
+ let fn_abi = cx.fn_abi_of_instance(instance, ty::List::empty());
- let llfn = if let Some(llfn) = cx.get_declared_value(&sym) {
+ let llfn = if let Some(llfn) = cx.get_declared_value(sym) {
// Create a fn pointer with the new signature.
let llptrty = fn_abi.ptr_to_llvm_type(cx);
@@ -79,7 +79,7 @@
llfn
}
} else {
- let llfn = cx.declare_fn(&sym, &fn_abi);
+ let llfn = cx.declare_fn(sym, fn_abi);
debug!("get_fn: not casting pointer!");
attributes::from_fn_attrs(cx, llfn, instance);
@@ -175,7 +175,7 @@
// should use dllimport for functions.
if cx.use_dll_storage_attrs
&& tcx.is_dllimport_foreign_item(instance_def_id)
- && tcx.sess.target.env != "gnu"
+ && !matches!(tcx.sess.target.env.as_ref(), "gnu" | "uclibc")
{
llvm::LLVMSetDLLStorageClass(llfn, llvm::DLLStorageClass::DllImport);
}
diff --git a/compiler/rustc_codegen_llvm/src/common.rs b/compiler/rustc_codegen_llvm/src/common.rs
index 5532f53..73a8d46 100644
--- a/compiler/rustc_codegen_llvm/src/common.rs
+++ b/compiler/rustc_codegen_llvm/src/common.rs
@@ -12,9 +12,10 @@
use rustc_codegen_ssa::traits::*;
use rustc_middle::bug;
use rustc_middle::mir::interpret::{Allocation, GlobalAlloc, Scalar};
-use rustc_middle::ty::{layout::TyAndLayout, ScalarInt};
+use rustc_middle::ty::layout::{LayoutOf, TyAndLayout};
+use rustc_middle::ty::ScalarInt;
use rustc_span::symbol::Symbol;
-use rustc_target::abi::{self, AddressSpace, HasDataLayout, LayoutOf, Pointer, Size};
+use rustc_target::abi::{self, AddressSpace, HasDataLayout, Pointer, Size};
use libc::{c_char, c_uint};
use tracing::debug;
@@ -227,7 +228,7 @@
})
}
- fn scalar_to_backend(&self, cv: Scalar, layout: &abi::Scalar, llty: &'ll Type) -> &'ll Value {
+ fn scalar_to_backend(&self, cv: Scalar, layout: abi::Scalar, llty: &'ll Type) -> &'ll Value {
let bitsize = if layout.is_bool() { 1 } else { layout.value.size(self).bits() };
match cv {
Scalar::Int(ScalarInt::ZST) => {
diff --git a/compiler/rustc_codegen_llvm/src/consts.rs b/compiler/rustc_codegen_llvm/src/consts.rs
index a4e4fc4..1afa6f0 100644
--- a/compiler/rustc_codegen_llvm/src/consts.rs
+++ b/compiler/rustc_codegen_llvm/src/consts.rs
@@ -15,10 +15,11 @@
Scalar as InterpScalar,
};
use rustc_middle::mir::mono::MonoItem;
+use rustc_middle::ty::layout::LayoutOf;
use rustc_middle::ty::{self, Instance, Ty};
use rustc_middle::{bug, span_bug};
use rustc_target::abi::{
- AddressSpace, Align, HasDataLayout, LayoutOf, Primitive, Scalar, Size, WrappingRange,
+ AddressSpace, Align, HasDataLayout, Primitive, Scalar, Size, WrappingRange,
};
use std::ops::Range;
use tracing::debug;
@@ -110,7 +111,7 @@
Pointer::new(alloc_id, Size::from_bytes(ptr_offset)),
&cx.tcx,
),
- &Scalar { value: Primitive::Pointer, valid_range: WrappingRange { start: 0, end: !0 } },
+ Scalar { value: Primitive::Pointer, valid_range: WrappingRange { start: 0, end: !0 } },
cx.type_i8p_ext(address_space),
));
next_offset = offset + pointer_size;
@@ -177,7 +178,7 @@
};
unsafe {
// Declare a symbol `foo` with the desired linkage.
- let g1 = cx.declare_global(&sym, llty2);
+ let g1 = cx.declare_global(sym, llty2);
llvm::LLVMRustSetLinkage(g1, base::linkage_to_llvm(linkage));
// Declare an internal global `extern_with_linkage_foo` which
@@ -187,7 +188,7 @@
// `extern_with_linkage_foo` will instead be initialized to
// zero.
let mut real_name = "_rust_extern_with_linkage_".to_string();
- real_name.push_str(&sym);
+ real_name.push_str(sym);
let g2 = cx.define_global(&real_name, llty).unwrap_or_else(|| {
cx.sess().span_fatal(
cx.tcx.def_span(span_def_id),
@@ -201,7 +202,7 @@
} else {
// Generate an external declaration.
// FIXME(nagisa): investigate whether it can be changed into define_global
- cx.declare_global(&sym, llty)
+ cx.declare_global(sym, llty)
}
}
@@ -233,7 +234,7 @@
_ => self.define_private_global(self.val_ty(cv)),
};
llvm::LLVMSetInitializer(gv, cv);
- set_global_alignment(&self, gv, align);
+ set_global_alignment(self, gv, align);
llvm::SetUnnamedAddress(gv, llvm::UnnamedAddr::Global);
gv
}
@@ -278,7 +279,7 @@
g
} else {
- check_and_apply_linkage(&self, &fn_attrs, ty, sym, def_id)
+ check_and_apply_linkage(self, fn_attrs, ty, sym, def_id)
};
// Thread-local statics in some other crate need to *always* be linked
@@ -368,7 +369,7 @@
unsafe {
let attrs = self.tcx.codegen_fn_attrs(def_id);
- let (v, alloc) = match codegen_static_initializer(&self, def_id) {
+ let (v, alloc) = match codegen_static_initializer(self, def_id) {
Ok(v) => v,
// Error has already been reported
Err(_) => return,
@@ -416,7 +417,7 @@
self.statics_to_rauw.borrow_mut().push((g, new_g));
new_g
};
- set_global_alignment(&self, g, self.align_of(ty));
+ set_global_alignment(self, g, self.align_of(ty));
llvm::LLVMSetInitializer(g, v);
if self.should_assume_dso_local(g, true) {
@@ -429,7 +430,7 @@
llvm::LLVMSetGlobalConstant(g, llvm::True);
}
- debuginfo::create_global_var_metadata(&self, def_id, g);
+ debuginfo::create_global_var_metadata(self, def_id, g);
if attrs.flags.contains(CodegenFnAttrFlags::THREAD_LOCAL) {
llvm::set_thread_local_mode(g, self.tls_model);
@@ -517,7 +518,7 @@
);
}
} else {
- base::set_link_section(g, &attrs);
+ base::set_link_section(g, attrs);
}
if attrs.flags.contains(CodegenFnAttrFlags::USED) {
diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs
index 45da18d..257a0ac 100644
--- a/compiler/rustc_codegen_llvm/src/context.rs
+++ b/compiler/rustc_codegen_llvm/src/context.rs
@@ -14,15 +14,20 @@
use rustc_data_structures::base_n;
use rustc_data_structures::fx::FxHashMap;
use rustc_data_structures::small_c_str::SmallCStr;
-use rustc_middle::bug;
use rustc_middle::mir::mono::CodegenUnit;
-use rustc_middle::ty::layout::{HasParamEnv, LayoutError, TyAndLayout};
+use rustc_middle::ty::layout::{
+ FnAbiError, FnAbiOfHelpers, FnAbiRequest, HasParamEnv, LayoutError, LayoutOfHelpers,
+ TyAndLayout,
+};
use rustc_middle::ty::{self, Instance, Ty, TyCtxt};
+use rustc_middle::{bug, span_bug};
use rustc_session::config::{CFGuard, CrateType, DebugInfo};
use rustc_session::Session;
-use rustc_span::source_map::{Span, DUMMY_SP};
+use rustc_span::source_map::Span;
use rustc_span::symbol::Symbol;
-use rustc_target::abi::{HasDataLayout, LayoutOf, PointeeInfo, Size, TargetDataLayout, VariantIdx};
+use rustc_target::abi::{
+ call::FnAbi, HasDataLayout, PointeeInfo, Size, TargetDataLayout, VariantIdx,
+};
use rustc_target::spec::{HasTargetSpec, RelocModel, Target, TlsModel};
use smallvec::SmallVec;
@@ -190,11 +195,14 @@
let llvm_target = SmallCStr::new(&sess.target.llvm_target);
llvm::LLVMRustSetNormalizedTarget(llmod, llvm_target.as_ptr());
- if sess.relocation_model() == RelocModel::Pic {
+ let reloc_model = sess.relocation_model();
+ if matches!(reloc_model, RelocModel::Pic | RelocModel::Pie) {
llvm::LLVMRustSetModulePICLevel(llmod);
// PIE is potentially more effective than PIC, but can only be used in executables.
// If all our outputs are executables, then we can relax PIC to PIE.
- if sess.crate_types().iter().all(|ty| *ty == CrateType::Executable) {
+ if reloc_model == RelocModel::Pie
+ || sess.crate_types().iter().all(|ty| *ty == CrateType::Executable)
+ {
llvm::LLVMRustSetModulePIELevel(llmod);
}
}
@@ -355,7 +363,7 @@
fn create_used_variable_impl(&self, name: &'static CStr, values: &[&'ll Value]) {
let section = cstr!("llvm.metadata");
- let array = self.const_array(&self.type_ptr_to(self.type_i8()), values);
+ let array = self.const_array(self.type_ptr_to(self.type_i8()), values);
unsafe {
let g = llvm::LLVMAddGlobal(self.llmod, self.val_ty(array), name.as_ptr());
@@ -439,7 +447,7 @@
}
fn sess(&self) -> &Session {
- &self.tcx.sess
+ self.tcx.sess
}
fn check_overflow(&self) -> bool {
@@ -835,27 +843,58 @@
}
}
-impl LayoutOf<'tcx> for CodegenCx<'ll, 'tcx> {
- type Ty = Ty<'tcx>;
- type TyAndLayout = TyAndLayout<'tcx>;
-
- fn layout_of(&self, ty: Ty<'tcx>) -> Self::TyAndLayout {
- self.spanned_layout_of(ty, DUMMY_SP)
- }
-
- fn spanned_layout_of(&self, ty: Ty<'tcx>, span: Span) -> Self::TyAndLayout {
- self.tcx.layout_of(ty::ParamEnv::reveal_all().and(ty)).unwrap_or_else(|e| {
- if let LayoutError::SizeOverflow(_) = e {
- self.sess().span_fatal(span, &e.to_string())
- } else {
- bug!("failed to get layout for `{}`: {}", ty, e)
- }
- })
- }
-}
-
impl<'tcx, 'll> HasParamEnv<'tcx> for CodegenCx<'ll, 'tcx> {
fn param_env(&self) -> ty::ParamEnv<'tcx> {
ty::ParamEnv::reveal_all()
}
}
+
+impl LayoutOfHelpers<'tcx> for CodegenCx<'ll, 'tcx> {
+ type LayoutOfResult = TyAndLayout<'tcx>;
+
+ #[inline]
+ fn handle_layout_err(&self, err: LayoutError<'tcx>, span: Span, ty: Ty<'tcx>) -> ! {
+ if let LayoutError::SizeOverflow(_) = err {
+ self.sess().span_fatal(span, &err.to_string())
+ } else {
+ span_bug!(span, "failed to get layout for `{}`: {}", ty, err)
+ }
+ }
+}
+
+impl FnAbiOfHelpers<'tcx> for CodegenCx<'ll, 'tcx> {
+ type FnAbiOfResult = &'tcx FnAbi<'tcx, Ty<'tcx>>;
+
+ #[inline]
+ fn handle_fn_abi_err(
+ &self,
+ err: FnAbiError<'tcx>,
+ span: Span,
+ fn_abi_request: FnAbiRequest<'tcx>,
+ ) -> ! {
+ if let FnAbiError::Layout(LayoutError::SizeOverflow(_)) = err {
+ self.sess().span_fatal(span, &err.to_string())
+ } else {
+ match fn_abi_request {
+ FnAbiRequest::OfFnPtr { sig, extra_args } => {
+ span_bug!(
+ span,
+ "`fn_abi_of_fn_ptr({}, {:?})` failed: {}",
+ sig,
+ extra_args,
+ err
+ );
+ }
+ FnAbiRequest::OfInstance { instance, extra_args } => {
+ span_bug!(
+ span,
+ "`fn_abi_of_instance({}, {:?})` failed: {}",
+ instance,
+ extra_args,
+ err
+ );
+ }
+ }
+ }
+ }
+}
diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs
index d2a2e73..6830864 100644
--- a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs
+++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs
@@ -73,7 +73,7 @@
mapgen.write_coverage_mapping(expressions, counter_regions, coverage_mapping_buffer);
});
debug_assert!(
- coverage_mapping_buffer.len() > 0,
+ !coverage_mapping_buffer.is_empty(),
"Every `FunctionCoverage` should have at least one counter"
);
@@ -311,8 +311,7 @@
// for each region in it's MIR.
// Convert the `HashSet` of `codegenned_def_ids` to a sortable vector, and sort them.
- let mut sorted_codegenned_def_ids: Vec<DefId> =
- codegenned_def_ids.iter().map(|def_id| *def_id).collect();
+ let mut sorted_codegenned_def_ids: Vec<DefId> = codegenned_def_ids.iter().copied().collect();
sorted_codegenned_def_ids.sort_unstable();
let mut first_covered_def_id_by_file: FxHashMap<Symbol, DefId> = FxHashMap::default();
diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs
index 019bf4a..ef11e29 100644
--- a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs
+++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs
@@ -1,6 +1,6 @@
use crate::llvm;
-use crate::abi::{Abi, FnAbi};
+use crate::abi::Abi;
use crate::builder::Builder;
use crate::common::CodegenCx;
@@ -20,7 +20,7 @@
CodeRegion, CounterValueReference, ExpressionOperandId, InjectedExpressionId, Op,
};
use rustc_middle::ty;
-use rustc_middle::ty::layout::FnAbiExt;
+use rustc_middle::ty::layout::FnAbiOf;
use rustc_middle::ty::subst::InternalSubsts;
use rustc_middle::ty::Instance;
@@ -199,9 +199,8 @@
);
let llfn = cx.declare_fn(
- &tcx.symbol_name(instance).name,
- &FnAbi::of_fn_ptr(
- cx,
+ tcx.symbol_name(instance).name,
+ cx.fn_abi_of_fn_ptr(
ty::Binder::dummy(tcx.mk_fn_sig(
iter::once(tcx.mk_unit()),
tcx.mk_unit(),
@@ -209,7 +208,7 @@
hir::Unsafety::Unsafe,
Abi::Rust,
)),
- &[],
+ ty::List::empty(),
),
);
diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/create_scope_map.rs b/compiler/rustc_codegen_llvm/src/debuginfo/create_scope_map.rs
index c2725b8..58f8573 100644
--- a/compiler/rustc_codegen_llvm/src/debuginfo/create_scope_map.rs
+++ b/compiler/rustc_codegen_llvm/src/debuginfo/create_scope_map.rs
@@ -3,12 +3,11 @@
use rustc_codegen_ssa::mir::debuginfo::{DebugScope, FunctionDebugContext};
use rustc_codegen_ssa::traits::*;
-use crate::abi::FnAbi;
use crate::common::CodegenCx;
use crate::llvm;
use crate::llvm::debuginfo::{DILocation, DIScope};
use rustc_middle::mir::{Body, SourceScope};
-use rustc_middle::ty::layout::FnAbiExt;
+use rustc_middle::ty::layout::FnAbiOf;
use rustc_middle::ty::{self, Instance};
use rustc_session::config::DebugInfo;
@@ -42,7 +41,7 @@
// Instantiate all scopes.
for idx in 0..mir.source_scopes.len() {
let scope = SourceScope::new(idx);
- make_mir_scope(cx, instance, &mir, fn_dbg_scope, &has_variables, debug_context, scope);
+ make_mir_scope(cx, instance, mir, fn_dbg_scope, &has_variables, debug_context, scope);
}
}
@@ -94,8 +93,8 @@
ty::ParamEnv::reveal_all(),
callee,
);
- let callee_fn_abi = FnAbi::of_instance(cx, callee, &[]);
- cx.dbg_scope_fn(callee, &callee_fn_abi, None)
+ let callee_fn_abi = cx.fn_abi_of_instance(callee, ty::List::empty());
+ cx.dbg_scope_fn(callee, callee_fn_abi, None)
}
None => unsafe {
llvm::LLVMRustDIBuilderCreateLexicalBlock(
diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs b/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs
index c33d35c..ae1f83d 100644
--- a/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs
+++ b/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs
@@ -59,10 +59,8 @@
}
pub fn needs_gdb_debug_scripts_section(cx: &CodegenCx<'_, '_>) -> bool {
- let omit_gdb_pretty_printer_section = cx
- .tcx
- .sess
- .contains_name(&cx.tcx.hir().krate_attrs(), sym::omit_gdb_pretty_printer_section);
+ let omit_gdb_pretty_printer_section =
+ cx.tcx.sess.contains_name(cx.tcx.hir().krate_attrs(), sym::omit_gdb_pretty_printer_section);
!omit_gdb_pretty_printer_section
&& cx.sess().opts.debuginfo != DebugInfo::None
diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs
index 346c51c..cd29f2a 100644
--- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs
+++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs
@@ -2,7 +2,7 @@
use self::RecursiveTypeDescription::*;
use super::namespace::mangled_name_of_instance;
-use super::type_names::compute_debuginfo_type_name;
+use super::type_names::{compute_debuginfo_type_name, compute_debuginfo_vtable_name};
use super::utils::{
create_DIArray, debug_context, get_namespace_for_item, is_node_local_to_unit, DIB,
};
@@ -26,18 +26,19 @@
use rustc_hir::def::CtorKind;
use rustc_hir::def_id::{DefId, LOCAL_CRATE};
use rustc_index::vec::{Idx, IndexVec};
-use rustc_middle::ich::NodeIdHashingMode;
use rustc_middle::mir::{self, GeneratorLayout};
-use rustc_middle::ty::layout::{self, IntegerExt, PrimitiveExt, TyAndLayout};
+use rustc_middle::ty::layout::{self, IntegerExt, LayoutOf, PrimitiveExt, TyAndLayout};
use rustc_middle::ty::subst::GenericArgKind;
-use rustc_middle::ty::Instance;
-use rustc_middle::ty::{self, AdtKind, GeneratorSubsts, ParamEnv, Ty, TyCtxt};
+use rustc_middle::ty::{
+ self, AdtKind, GeneratorSubsts, Instance, ParamEnv, Ty, TyCtxt, COMMON_VTABLE_ENTRIES,
+};
use rustc_middle::{bug, span_bug};
+use rustc_query_system::ich::NodeIdHashingMode;
use rustc_session::config::{self, DebugInfo};
-use rustc_span::symbol::{Interner, Symbol};
+use rustc_span::symbol::Symbol;
use rustc_span::FileNameDisplayPreference;
use rustc_span::{self, SourceFile, SourceFileHash, Span};
-use rustc_target::abi::{Abi, Align, HasDataLayout, Integer, LayoutOf, TagEncoding};
+use rustc_target::abi::{Abi, Align, HasDataLayout, Integer, TagEncoding};
use rustc_target::abi::{Int, Pointer, F32, F64};
use rustc_target::abi::{Primitive, Size, VariantIdx, Variants};
use tracing::debug;
@@ -89,8 +90,54 @@
pub const NO_SCOPE_METADATA: Option<&DIScope> = None;
-#[derive(Copy, Debug, Hash, Eq, PartialEq, Clone)]
-pub struct UniqueTypeId(Symbol);
+mod unique_type_id {
+ use super::*;
+ use rustc_arena::DroplessArena;
+
+ #[derive(Copy, Hash, Eq, PartialEq, Clone)]
+ pub(super) struct UniqueTypeId(u32);
+
+ // The `&'static str`s in this type actually point into the arena.
+ //
+ // The `FxHashMap`+`Vec` pair could be replaced by `FxIndexSet`, but #75278
+ // found that to regress performance up to 2% in some cases. This might be
+ // revisited after further improvements to `indexmap`.
+ #[derive(Default)]
+ pub(super) struct TypeIdInterner {
+ arena: DroplessArena,
+ names: FxHashMap<&'static str, UniqueTypeId>,
+ strings: Vec<&'static str>,
+ }
+
+ impl TypeIdInterner {
+ #[inline]
+ pub(super) fn intern(&mut self, string: &str) -> UniqueTypeId {
+ if let Some(&name) = self.names.get(string) {
+ return name;
+ }
+
+ let name = UniqueTypeId(self.strings.len() as u32);
+
+ // `from_utf8_unchecked` is safe since we just allocated a `&str` which is known to be
+ // UTF-8.
+ let string: &str =
+ unsafe { std::str::from_utf8_unchecked(self.arena.alloc_slice(string.as_bytes())) };
+ // It is safe to extend the arena allocation to `'static` because we only access
+ // these while the arena is still alive.
+ let string: &'static str = unsafe { &*(string as *const str) };
+ self.strings.push(string);
+ self.names.insert(string, name);
+ name
+ }
+
+ // Get the symbol as a string. `Symbol::as_str()` should be used in
+ // preference to this function.
+ pub(super) fn get(&self, symbol: UniqueTypeId) -> &str {
+ self.strings[symbol.0 as usize]
+ }
+ }
+}
+use unique_type_id::*;
/// The `TypeMap` is where the `CrateDebugContext` holds the type metadata nodes
/// created so far. The metadata nodes are indexed by `UniqueTypeId`, and, for
@@ -99,7 +146,7 @@
#[derive(Default)]
pub struct TypeMap<'ll, 'tcx> {
/// The `UniqueTypeId`s created so far.
- unique_id_interner: Interner,
+ unique_id_interner: TypeIdInterner,
/// A map from `UniqueTypeId` to debuginfo metadata for that type. This is a 1:1 mapping.
unique_id_to_metadata: FxHashMap<UniqueTypeId, &'ll DIType>,
/// A map from types to debuginfo metadata. This is an N:1 mapping.
@@ -166,8 +213,7 @@
/// Gets the string representation of a `UniqueTypeId`. This method will fail if
/// the ID is unknown.
fn get_unique_type_id_as_string(&self, unique_type_id: UniqueTypeId) -> &str {
- let UniqueTypeId(interner_key) = unique_type_id;
- self.unique_id_interner.get(interner_key)
+ self.unique_id_interner.get(unique_type_id)
}
/// Gets the `UniqueTypeId` for the given type. If the `UniqueTypeId` for the given
@@ -197,9 +243,9 @@
let unique_type_id = hasher.finish::<Fingerprint>().to_hex();
let key = self.unique_id_interner.intern(&unique_type_id);
- self.type_to_unique_id.insert(type_, UniqueTypeId(key));
+ self.type_to_unique_id.insert(type_, key);
- UniqueTypeId(key)
+ key
}
/// Gets the `UniqueTypeId` for an enum variant. Enum variants are not really
@@ -215,7 +261,7 @@
let enum_variant_type_id =
format!("{}::{}", self.get_unique_type_id_as_string(enum_type_id), variant_name);
let interner_key = self.unique_id_interner.intern(&enum_variant_type_id);
- UniqueTypeId(interner_key)
+ interner_key
}
/// Gets the unique type ID string for an enum variant part.
@@ -432,7 +478,7 @@
let signature_metadata: Vec<_> = iter::once(
// return type
match signature.output().kind() {
- ty::Tuple(ref tys) if tys.is_empty() => None,
+ ty::Tuple(tys) if tys.is_empty() => None,
_ => Some(type_metadata(cx, signature.output(), span)),
},
)
@@ -602,7 +648,7 @@
ty::Never | ty::Bool | ty::Char | ty::Int(_) | ty::Uint(_) | ty::Float(_) => {
MetadataCreationResult::new(basic_type_metadata(cx, t), false)
}
- ty::Tuple(ref elements) if elements.is_empty() => {
+ ty::Tuple(elements) if elements.is_empty() => {
MetadataCreationResult::new(basic_type_metadata(cx, t), false)
}
ty::Array(typ, _) | ty::Slice(typ) => {
@@ -701,7 +747,7 @@
.finalize(cx)
}
},
- ty::Tuple(ref elements) => {
+ ty::Tuple(elements) => {
let tys: Vec<_> = elements.iter().map(|k| k.expect_ty()).collect();
prepare_tuple_metadata(cx, t, &tys, unique_type_id, usage_site_span, NO_SCOPE_METADATA)
.finalize(cx)
@@ -887,7 +933,7 @@
let (name, encoding) = match t.kind() {
ty::Never => ("!", DW_ATE_unsigned),
- ty::Tuple(ref elements) if elements.is_empty() => ("()", DW_ATE_unsigned),
+ ty::Tuple(elements) if elements.is_empty() => ("()", DW_ATE_unsigned),
ty::Bool => ("bool", DW_ATE_boolean),
ty::Char => ("char", DW_ATE_unsigned_char),
ty::Int(int_ty) if msvc_like_names => (int_ty.msvc_basic_name(), DW_ATE_signed),
@@ -1078,7 +1124,7 @@
let gcov_cu_info = [
path_to_mdstring(debug_context.llcontext, &output_filenames.with_extension("gcno")),
- path_to_mdstring(debug_context.llcontext, &gcda_path),
+ path_to_mdstring(debug_context.llcontext, gcda_path),
cu_desc_metadata,
];
let gcov_metadata = llvm::LLVMMDNodeInContext(
@@ -1656,7 +1702,7 @@
Variants::Multiple {
tag_encoding:
TagEncoding::Niche { ref niche_variants, niche_start, dataful_variant },
- ref tag,
+ tag,
ref variants,
tag_field,
} => {
@@ -1918,17 +1964,13 @@
}
fn source_info(&self, cx: &CodegenCx<'ll, 'tcx>) -> Option<SourceInfo<'ll>> {
- match self {
- VariantInfo::Generator { def_id, variant_index, .. } => {
- let span = cx.tcx.generator_layout(*def_id).unwrap().variant_source_info
- [*variant_index]
- .span;
- if !span.is_dummy() {
- let loc = cx.lookup_debug_loc(span.lo());
- return Some(SourceInfo { file: file_metadata(cx, &loc.file), line: loc.line });
- }
+ if let VariantInfo::Generator { def_id, variant_index, .. } = self {
+ let span =
+ cx.tcx.generator_layout(*def_id).unwrap().variant_source_info[*variant_index].span;
+ if !span.is_dummy() {
+ let loc = cx.lookup_debug_loc(span.lo());
+ return Some(SourceInfo { file: file_metadata(cx, &loc.file), line: loc.line });
}
- _ => {}
}
None
}
@@ -1949,11 +1991,11 @@
let unique_type_id = debug_context(cx)
.type_map
.borrow_mut()
- .get_unique_type_id_of_enum_variant(cx, layout.ty, &variant_name);
+ .get_unique_type_id_of_enum_variant(cx, layout.ty, variant_name);
create_struct_stub(
cx,
layout.ty,
- &variant_name,
+ variant_name,
unique_type_id,
Some(containing_scope),
DIFlags::FlagZero,
@@ -2082,10 +2124,8 @@
let layout = cx.layout_of(enum_type);
- if let (
- &Abi::Scalar(_),
- &Variants::Multiple { tag_encoding: TagEncoding::Direct, ref tag, .. },
- ) = (&layout.abi, &layout.variants)
+ if let (Abi::Scalar(_), Variants::Multiple { tag_encoding: TagEncoding::Direct, tag, .. }) =
+ (layout.abi, &layout.variants)
{
return FinalMetadata(discriminant_type_metadata(tag.value));
}
@@ -2093,8 +2133,8 @@
if use_enum_fallback(cx) {
let discriminant_type_metadata = match layout.variants {
Variants::Single { .. } => None,
- Variants::Multiple { tag_encoding: TagEncoding::Niche { .. }, ref tag, .. }
- | Variants::Multiple { tag_encoding: TagEncoding::Direct, ref tag, .. } => {
+ Variants::Multiple { tag_encoding: TagEncoding::Niche { .. }, tag, .. }
+ | Variants::Multiple { tag_encoding: TagEncoding::Direct, tag, .. } => {
Some(discriminant_type_metadata(tag.value))
}
};
@@ -2146,9 +2186,7 @@
// A single-variant enum has no discriminant.
Variants::Single { .. } => None,
- Variants::Multiple {
- tag_encoding: TagEncoding::Niche { .. }, ref tag, tag_field, ..
- } => {
+ Variants::Multiple { tag_encoding: TagEncoding::Niche { .. }, tag, tag_field, .. } => {
// Find the integer type of the correct size.
let size = tag.value.size(cx);
let align = tag.value.align(cx);
@@ -2179,7 +2217,7 @@
}
}
- Variants::Multiple { tag_encoding: TagEncoding::Direct, ref tag, tag_field, .. } => {
+ Variants::Multiple { tag_encoding: TagEncoding::Direct, tag, tag_field, .. } => {
let discr_type = tag.value.to_ty(cx.tcx);
let (size, align) = cx.size_and_align_of(discr_type);
@@ -2344,7 +2382,7 @@
{
let mut composite_types_completed =
debug_context(cx).composite_types_completed.borrow_mut();
- if !composite_types_completed.insert(&composite_type_metadata) {
+ if !composite_types_completed.insert(composite_type_metadata) {
bug!(
"debuginfo::set_members_of_composite_type() - \
Already completed forward declaration re-encountered."
@@ -2554,11 +2592,45 @@
}
}
+/// Generates LLVM debuginfo for a vtable.
+fn vtable_type_metadata(
+ cx: &CodegenCx<'ll, 'tcx>,
+ ty: Ty<'tcx>,
+ poly_trait_ref: Option<ty::PolyExistentialTraitRef<'tcx>>,
+) -> &'ll DIType {
+ let tcx = cx.tcx;
+
+ let vtable_entries = if let Some(poly_trait_ref) = poly_trait_ref {
+ let trait_ref = poly_trait_ref.with_self_ty(tcx, ty);
+ let trait_ref = tcx.erase_regions(trait_ref);
+
+ tcx.vtable_entries(trait_ref)
+ } else {
+ COMMON_VTABLE_ENTRIES
+ };
+
+ // FIXME: We describe the vtable as an array of *const () pointers. The length of the array is
+ // correct - but we could create a more accurate description, e.g. by describing it
+ // as a struct where each field has a name that corresponds to the name of the method
+ // it points to.
+ // However, this is not entirely straightforward because there might be multiple
+ // methods with the same name if the vtable is for multiple traits. So for now we keep
+ // things simple instead of adding some ad-hoc disambiguation scheme.
+ let vtable_type = tcx.mk_array(tcx.mk_imm_ptr(tcx.types.unit), vtable_entries.len() as u64);
+
+ type_metadata(cx, vtable_type, rustc_span::DUMMY_SP)
+}
+
/// Creates debug information for the given vtable, which is for the
/// given type.
///
/// Adds the created metadata nodes directly to the crate's IR.
-pub fn create_vtable_metadata(cx: &CodegenCx<'ll, 'tcx>, ty: Ty<'tcx>, vtable: &'ll Value) {
+pub fn create_vtable_metadata(
+ cx: &CodegenCx<'ll, 'tcx>,
+ ty: Ty<'tcx>,
+ poly_trait_ref: Option<ty::PolyExistentialTraitRef<'tcx>>,
+ vtable: &'ll Value,
+) {
if cx.dbg_cx.is_none() {
return;
}
@@ -2568,42 +2640,16 @@
return;
}
- let type_metadata = type_metadata(cx, ty, rustc_span::DUMMY_SP);
+ let vtable_name = compute_debuginfo_vtable_name(cx.tcx, ty, poly_trait_ref);
+ let vtable_type = vtable_type_metadata(cx, ty, poly_trait_ref);
unsafe {
- // `LLVMRustDIBuilderCreateStructType()` wants an empty array. A null
- // pointer will lead to hard to trace and debug LLVM assertions
- // later on in `llvm/lib/IR/Value.cpp`.
- let empty_array = create_DIArray(DIB(cx), &[]);
- let name = "vtable";
-
- // Create a new one each time. We don't want metadata caching
- // here, because each vtable will refer to a unique containing
- // type.
- let vtable_type = llvm::LLVMRustDIBuilderCreateStructType(
- DIB(cx),
- NO_SCOPE_METADATA,
- name.as_ptr().cast(),
- name.len(),
- unknown_file_metadata(cx),
- UNKNOWN_LINE_NUMBER,
- Size::ZERO.bits(),
- cx.tcx.data_layout.pointer_align.abi.bits() as u32,
- DIFlags::FlagArtificial,
- None,
- empty_array,
- 0,
- Some(type_metadata),
- name.as_ptr().cast(),
- name.len(),
- );
-
let linkage_name = "";
llvm::LLVMRustDIBuilderCreateStaticVariable(
DIB(cx),
NO_SCOPE_METADATA,
- name.as_ptr().cast(),
- name.len(),
+ vtable_name.as_ptr().cast(),
+ vtable_name.len(),
linkage_name.as_ptr().cast(),
linkage_name.len(),
unknown_file_metadata(cx),
diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs
index 914376d..1f1bd73 100644
--- a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs
+++ b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs
@@ -25,13 +25,14 @@
use rustc_hir::def_id::{DefId, DefIdMap};
use rustc_index::vec::IndexVec;
use rustc_middle::mir;
-use rustc_middle::ty::layout::HasTyCtxt;
+use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf};
use rustc_middle::ty::subst::{GenericArgKind, SubstsRef};
use rustc_middle::ty::{self, Instance, ParamEnv, Ty, TypeFoldable};
use rustc_session::config::{self, DebugInfo};
+use rustc_session::Session;
use rustc_span::symbol::Symbol;
use rustc_span::{self, BytePos, Pos, SourceFile, SourceFileAndLine, Span};
-use rustc_target::abi::{LayoutOf, Primitive, Size};
+use rustc_target::abi::{Primitive, Size};
use libc::c_uint;
use smallvec::SmallVec;
@@ -95,45 +96,52 @@
composite_types_completed: Default::default(),
}
}
+
+ pub fn finalize(&self, sess: &Session) {
+ unsafe {
+ llvm::LLVMRustDIBuilderFinalize(self.builder);
+
+ // Debuginfo generation in LLVM by default uses a higher
+ // version of dwarf than macOS currently understands. We can
+ // instruct LLVM to emit an older version of dwarf, however,
+ // for macOS to understand. For more info see #11352
+ // This can be overridden using --llvm-opts -dwarf-version,N.
+ // Android has the same issue (#22398)
+ if let Some(version) = sess.target.dwarf_version {
+ llvm::LLVMRustAddModuleFlag(self.llmod, "Dwarf Version\0".as_ptr().cast(), version)
+ }
+
+ // Indicate that we want CodeView debug information on MSVC
+ if sess.target.is_like_msvc {
+ llvm::LLVMRustAddModuleFlag(self.llmod, "CodeView\0".as_ptr().cast(), 1)
+ }
+
+ // Prevent bitcode readers from deleting the debug info.
+ let ptr = "Debug Info Version\0".as_ptr();
+ llvm::LLVMRustAddModuleFlag(
+ self.llmod,
+ ptr.cast(),
+ llvm::LLVMRustDebugMetadataVersion(),
+ );
+ }
+ }
}
/// Creates any deferred debug metadata nodes
pub fn finalize(cx: &CodegenCx<'_, '_>) {
- if cx.dbg_cx.is_none() {
- return;
- }
+ if let Some(dbg_cx) = &cx.dbg_cx {
+ debug!("finalize");
- debug!("finalize");
-
- if gdb::needs_gdb_debug_scripts_section(cx) {
- // Add a .debug_gdb_scripts section to this compile-unit. This will
- // cause GDB to try and load the gdb_load_rust_pretty_printers.py file,
- // which activates the Rust pretty printers for binary this section is
- // contained in.
- gdb::get_or_insert_gdb_debug_scripts_section_global(cx);
- }
-
- unsafe {
- llvm::LLVMRustDIBuilderFinalize(DIB(cx));
- // Debuginfo generation in LLVM by default uses a higher
- // version of dwarf than macOS currently understands. We can
- // instruct LLVM to emit an older version of dwarf, however,
- // for macOS to understand. For more info see #11352
- // This can be overridden using --llvm-opts -dwarf-version,N.
- // Android has the same issue (#22398)
- if let Some(version) = cx.sess().target.dwarf_version {
- llvm::LLVMRustAddModuleFlag(cx.llmod, "Dwarf Version\0".as_ptr().cast(), version)
+ if gdb::needs_gdb_debug_scripts_section(cx) {
+ // Add a .debug_gdb_scripts section to this compile-unit. This will
+ // cause GDB to try and load the gdb_load_rust_pretty_printers.py file,
+ // which activates the Rust pretty printers for binary this section is
+ // contained in.
+ gdb::get_or_insert_gdb_debug_scripts_section_global(cx);
}
- // Indicate that we want CodeView debug information on MSVC
- if cx.sess().target.is_like_msvc {
- llvm::LLVMRustAddModuleFlag(cx.llmod, "CodeView\0".as_ptr().cast(), 1)
- }
-
- // Prevent bitcode readers from deleting the debug info.
- let ptr = "Debug Info Version\0".as_ptr();
- llvm::LLVMRustAddModuleFlag(cx.llmod, ptr.cast(), llvm::LLVMRustDebugMetadataVersion());
- };
+ dbg_cx.finalize(cx.sess());
+ }
}
impl DebugInfoBuilderMethods for Builder<'a, 'll, 'tcx> {
@@ -320,7 +328,7 @@
// name if necessary.
let generics = self.tcx().generics_of(enclosing_fn_def_id);
let substs = instance.substs.truncate_to(self.tcx(), generics);
- let template_parameters = get_template_parameters(self, &generics, substs, &mut name);
+ let template_parameters = get_template_parameters(self, generics, substs, &mut name);
let linkage_name = &mangled_name_of_instance(self, instance).name;
// Omit the linkage_name if it is the same as subprogram name.
@@ -542,8 +550,13 @@
unsafe { llvm::LLVMRustDIBuilderCreateDebugLocation(line, col, scope, inlined_at) }
}
- fn create_vtable_metadata(&self, ty: Ty<'tcx>, vtable: Self::Value) {
- metadata::create_vtable_metadata(self, ty, vtable)
+ fn create_vtable_metadata(
+ &self,
+ ty: Ty<'tcx>,
+ trait_ref: Option<ty::PolyExistentialTraitRef<'tcx>>,
+ vtable: Self::Value,
+ ) {
+ metadata::create_vtable_metadata(self, ty, trait_ref, vtable)
}
fn extend_scope_to_file(
@@ -551,7 +564,7 @@
scope_metadata: &'ll DIScope,
file: &rustc_span::SourceFile,
) -> &'ll DILexicalBlock {
- metadata::extend_scope_to_file(&self, scope_metadata, file)
+ metadata::extend_scope_to_file(self, scope_metadata, file)
}
fn debuginfo_finalize(&self) {
diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs
index e30c492..c43141c 100644
--- a/compiler/rustc_codegen_llvm/src/intrinsic.rs
+++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs
@@ -15,12 +15,12 @@
use rustc_codegen_ssa::mir::place::PlaceRef;
use rustc_codegen_ssa::traits::*;
use rustc_hir as hir;
-use rustc_middle::ty::layout::{FnAbiExt, HasTyCtxt};
+use rustc_middle::ty::layout::{FnAbiOf, HasTyCtxt, LayoutOf};
use rustc_middle::ty::{self, Ty};
use rustc_middle::{bug, span_bug};
use rustc_span::{sym, symbol::kw, Span, Symbol};
-use rustc_target::abi::{self, HasDataLayout, LayoutOf, Primitive};
-use rustc_target::spec::PanicStrategy;
+use rustc_target::abi::{self, HasDataLayout, Primitive};
+use rustc_target::spec::{HasTargetSpec, PanicStrategy};
use std::cmp::Ordering;
use std::iter;
@@ -71,7 +71,7 @@
sym::roundf64 => "llvm.round.f64",
_ => return None,
};
- Some(cx.get_intrinsic(&llvm_name))
+ Some(cx.get_intrinsic(llvm_name))
}
impl IntrinsicCallMethods<'tcx> for Builder<'a, 'll, 'tcx> {
@@ -96,7 +96,6 @@
let arg_tys = sig.inputs();
let ret_ty = sig.output();
let name = tcx.item_name(def_id);
- let name_str = &*name.as_str();
let llret_ty = self.layout_of(ret_ty).llvm_type(self);
let result = PlaceRef::new_sized(llresult, fn_abi.ret.layout);
@@ -133,7 +132,7 @@
}
sym::va_arg => {
match fn_abi.ret.layout.abi {
- abi::Abi::Scalar(ref scalar) => {
+ abi::Abi::Scalar(scalar) => {
match scalar.value {
Primitive::Int(..) => {
if self.cx().size_of(ret_ty).bytes() < 4 {
@@ -230,9 +229,14 @@
&[args[0].immediate(), y],
)
}
- sym::ctlz_nonzero | sym::cttz_nonzero => {
+ sym::ctlz_nonzero => {
let y = self.const_bool(true);
- let llvm_name = &format!("llvm.{}.i{}", &name_str[..4], width);
+ let llvm_name = &format!("llvm.ctlz.i{}", width);
+ self.call_intrinsic(llvm_name, &[args[0].immediate(), y])
+ }
+ sym::cttz_nonzero => {
+ let y = self.const_bool(true);
+ let llvm_name = &format!("llvm.cttz.i{}", width);
self.call_intrinsic(llvm_name, &[args[0].immediate(), y])
}
sym::ctpop => self.call_intrinsic(
@@ -353,7 +357,7 @@
return;
}
- _ if name_str.starts_with("simd_") => {
+ _ if name.as_str().starts_with("simd_") => {
match generic_simd_intrinsic(self, name, callee_ty, args, ret_ty, llret_ty, span) {
Ok(llval) => llval,
Err(()) => return,
@@ -737,9 +741,9 @@
rust_fn_sig: ty::PolyFnSig<'tcx>,
codegen: &mut dyn FnMut(Builder<'_, 'll, 'tcx>),
) -> (&'ll Type, &'ll Value) {
- let fn_abi = FnAbi::of_fn_ptr(cx, rust_fn_sig, &[]);
+ let fn_abi = cx.fn_abi_of_fn_ptr(rust_fn_sig, ty::List::empty());
let llty = fn_abi.llvm_type(cx);
- let llfn = cx.declare_fn(name, &fn_abi);
+ let llfn = cx.declare_fn(name, fn_abi);
cx.set_frame_pointer_type(llfn);
cx.apply_target_cpu_attr(llfn);
// FIXME(eddyb) find a nicer way to do this.
@@ -843,7 +847,6 @@
let sig =
tcx.normalize_erasing_late_bound_regions(ty::ParamEnv::reveal_all(), callee_ty.fn_sig(tcx));
let arg_tys = sig.inputs();
- let name_str = &*name.as_str();
if name == sym::simd_select_bitmask {
let in_ty = arg_tys[0];
@@ -917,13 +920,30 @@
));
}
- if let Some(stripped) = name_str.strip_prefix("simd_shuffle") {
- let n: u64 = stripped.parse().unwrap_or_else(|_| {
- span_bug!(span, "bad `simd_shuffle` instruction only caught in codegen?")
- });
+ if let Some(stripped) = name.as_str().strip_prefix("simd_shuffle") {
+ // If this intrinsic is the older "simd_shuffleN" form, simply parse the integer.
+ // If there is no suffix, use the index array length.
+ let n: u64 = if stripped.is_empty() {
+ // Make sure this is actually an array, since typeck only checks the length-suffixed
+ // version of this intrinsic.
+ match args[2].layout.ty.kind() {
+ ty::Array(ty, len) if matches!(ty.kind(), ty::Uint(ty::UintTy::U32)) => {
+ len.try_eval_usize(bx.cx.tcx, ty::ParamEnv::reveal_all()).unwrap_or_else(|| {
+ span_bug!(span, "could not evaluate shuffle index array length")
+ })
+ }
+ _ => return_error!(
+ "simd_shuffle index must be an array of `u32`, got `{}`",
+ args[2].layout.ty
+ ),
+ }
+ } else {
+ stripped.parse().unwrap_or_else(|_| {
+ span_bug!(span, "bad `simd_shuffle` instruction only caught in codegen?")
+ })
+ };
require_simd!(ret_ty, "return");
-
let (out_len, out_ty) = ret_ty.simd_size_and_type(bx.tcx());
require!(
out_len == n,
@@ -1139,7 +1159,7 @@
_ => return_error!("unrecognized intrinsic `{}`", name),
};
let llvm_name = &format!("llvm.{0}.v{1}{2}", intr_name, in_len, elem_ty_str);
- let f = bx.declare_cfn(&llvm_name, llvm::UnnamedAddr::No, fn_ty);
+ let f = bx.declare_cfn(llvm_name, llvm::UnnamedAddr::No, fn_ty);
let c =
bx.call(fn_ty, f, &args.iter().map(|arg| arg.immediate()).collect::<Vec<_>>(), None);
Ok(c)
@@ -1170,11 +1190,28 @@
// FIXME: use:
// https://github.com/llvm-mirror/llvm/blob/master/include/llvm/IR/Function.h#L182
// https://github.com/llvm-mirror/llvm/blob/master/include/llvm/IR/Intrinsics.h#L81
- fn llvm_vector_str(elem_ty: Ty<'_>, vec_len: u64, no_pointers: usize) -> String {
+ fn llvm_vector_str(
+ elem_ty: Ty<'_>,
+ vec_len: u64,
+ no_pointers: usize,
+ bx: &Builder<'a, 'll, 'tcx>,
+ ) -> String {
let p0s: String = "p0".repeat(no_pointers);
match *elem_ty.kind() {
- ty::Int(v) => format!("v{}{}i{}", vec_len, p0s, v.bit_width().unwrap()),
- ty::Uint(v) => format!("v{}{}i{}", vec_len, p0s, v.bit_width().unwrap()),
+ ty::Int(v) => format!(
+ "v{}{}i{}",
+ vec_len,
+ p0s,
+ // Normalize to prevent crash if v: IntTy::Isize
+ v.normalize(bx.target_spec().pointer_width).bit_width().unwrap()
+ ),
+ ty::Uint(v) => format!(
+ "v{}{}i{}",
+ vec_len,
+ p0s,
+ // Normalize to prevent crash if v: UIntTy::Usize
+ v.normalize(bx.target_spec().pointer_width).bit_width().unwrap()
+ ),
ty::Float(v) => format!("v{}{}f{}", vec_len, p0s, v.bit_width()),
_ => unreachable!(),
}
@@ -1310,11 +1347,11 @@
// Type of the vector of pointers:
let llvm_pointer_vec_ty = llvm_vector_ty(bx, underlying_ty, in_len, pointer_count);
- let llvm_pointer_vec_str = llvm_vector_str(underlying_ty, in_len, pointer_count);
+ let llvm_pointer_vec_str = llvm_vector_str(underlying_ty, in_len, pointer_count, bx);
// Type of the vector of elements:
let llvm_elem_vec_ty = llvm_vector_ty(bx, underlying_ty, in_len, pointer_count - 1);
- let llvm_elem_vec_str = llvm_vector_str(underlying_ty, in_len, pointer_count - 1);
+ let llvm_elem_vec_str = llvm_vector_str(underlying_ty, in_len, pointer_count - 1, bx);
let llvm_intrinsic =
format!("llvm.masked.gather.{}.{}", llvm_elem_vec_str, llvm_pointer_vec_str);
@@ -1438,11 +1475,11 @@
// Type of the vector of pointers:
let llvm_pointer_vec_ty = llvm_vector_ty(bx, underlying_ty, in_len, pointer_count);
- let llvm_pointer_vec_str = llvm_vector_str(underlying_ty, in_len, pointer_count);
+ let llvm_pointer_vec_str = llvm_vector_str(underlying_ty, in_len, pointer_count, bx);
// Type of the vector of elements:
let llvm_elem_vec_ty = llvm_vector_ty(bx, underlying_ty, in_len, pointer_count - 1);
- let llvm_elem_vec_str = llvm_vector_str(underlying_ty, in_len, pointer_count - 1);
+ let llvm_elem_vec_str = llvm_vector_str(underlying_ty, in_len, pointer_count - 1, bx);
let llvm_intrinsic =
format!("llvm.masked.scatter.{}.{}", llvm_elem_vec_str, llvm_pointer_vec_str);
@@ -1773,7 +1810,7 @@
let vec_ty = bx.cx.type_vector(elem_ty, in_len as u64);
let fn_ty = bx.type_func(&[vec_ty, vec_ty], vec_ty);
- let f = bx.declare_cfn(&llvm_intrinsic, llvm::UnnamedAddr::No, fn_ty);
+ let f = bx.declare_cfn(llvm_intrinsic, llvm::UnnamedAddr::No, fn_ty);
let v = bx.call(fn_ty, f, &[lhs, rhs], None);
return Ok(v);
}
diff --git a/compiler/rustc_codegen_llvm/src/lib.rs b/compiler/rustc_codegen_llvm/src/lib.rs
index 1e6e525..8f4d79e 100644
--- a/compiler/rustc_codegen_llvm/src/lib.rs
+++ b/compiler/rustc_codegen_llvm/src/lib.rs
@@ -27,8 +27,8 @@
use rustc_codegen_ssa::{CodegenResults, CompiledModule};
use rustc_data_structures::fx::FxHashMap;
use rustc_errors::{ErrorReported, FatalError, Handler};
+use rustc_metadata::EncodedMetadata;
use rustc_middle::dep_graph::{WorkProduct, WorkProductId};
-use rustc_middle::middle::cstore::EncodedMetadata;
use rustc_middle::ty::TyCtxt;
use rustc_session::config::{OptLevel, OutputFilenames, PrintRequest};
use rustc_session::Session;
@@ -92,11 +92,12 @@
fn codegen_allocator<'tcx>(
&self,
tcx: TyCtxt<'tcx>,
- mods: &mut ModuleLlvm,
+ module_llvm: &mut ModuleLlvm,
+ module_name: &str,
kind: AllocatorKind,
has_alloc_error_handler: bool,
) {
- unsafe { allocator::codegen(tcx, mods, kind, has_alloc_error_handler) }
+ unsafe { allocator::codegen(tcx, module_llvm, module_name, kind, has_alloc_error_handler) }
}
fn compile_codegen_unit(
&self,
@@ -210,9 +211,16 @@
match req {
PrintRequest::RelocationModels => {
println!("Available relocation models:");
- for name in
- &["static", "pic", "dynamic-no-pic", "ropi", "rwpi", "ropi-rwpi", "default"]
- {
+ for name in &[
+ "static",
+ "pic",
+ "pie",
+ "dynamic-no-pic",
+ "ropi",
+ "rwpi",
+ "ropi-rwpi",
+ "default",
+ ] {
println!(" {}", name);
}
println!();
@@ -331,7 +339,7 @@
unsafe {
let llcx = llvm::LLVMRustContextCreate(cgcx.fewer_names);
let llmod_raw = back::lto::parse_module(llcx, name, buffer, handler)?;
- let tm_factory_config = TargetMachineFactoryConfig::new(&cgcx, name.to_str().unwrap());
+ let tm_factory_config = TargetMachineFactoryConfig::new(cgcx, name.to_str().unwrap());
let tm = match (cgcx.tm_factory)(tm_factory_config) {
Ok(m) => m,
Err(e) => {
diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs
index 3f2ed02..4c9ae4f 100644
--- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs
+++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs
@@ -34,11 +34,18 @@
#[repr(C)]
pub struct LLVMRustCOFFShortExport {
pub name: *const c_char,
+ pub ordinal_present: bool,
+ // value of `ordinal` only important when `ordinal_present` is true
+ pub ordinal: u16,
}
impl LLVMRustCOFFShortExport {
- pub fn from_name(name: *const c_char) -> LLVMRustCOFFShortExport {
- LLVMRustCOFFShortExport { name }
+ pub fn new(name: *const c_char, ordinal: Option<u16>) -> LLVMRustCOFFShortExport {
+ LLVMRustCOFFShortExport {
+ name,
+ ordinal_present: ordinal.is_some(),
+ ordinal: ordinal.unwrap_or(0),
+ }
}
}
@@ -216,6 +223,33 @@
RealPredicateTrue = 15,
}
+impl RealPredicate {
+ pub fn from_generic(realp: rustc_codegen_ssa::common::RealPredicate) -> Self {
+ match realp {
+ rustc_codegen_ssa::common::RealPredicate::RealPredicateFalse => {
+ RealPredicate::RealPredicateFalse
+ }
+ rustc_codegen_ssa::common::RealPredicate::RealOEQ => RealPredicate::RealOEQ,
+ rustc_codegen_ssa::common::RealPredicate::RealOGT => RealPredicate::RealOGT,
+ rustc_codegen_ssa::common::RealPredicate::RealOGE => RealPredicate::RealOGE,
+ rustc_codegen_ssa::common::RealPredicate::RealOLT => RealPredicate::RealOLT,
+ rustc_codegen_ssa::common::RealPredicate::RealOLE => RealPredicate::RealOLE,
+ rustc_codegen_ssa::common::RealPredicate::RealONE => RealPredicate::RealONE,
+ rustc_codegen_ssa::common::RealPredicate::RealORD => RealPredicate::RealORD,
+ rustc_codegen_ssa::common::RealPredicate::RealUNO => RealPredicate::RealUNO,
+ rustc_codegen_ssa::common::RealPredicate::RealUEQ => RealPredicate::RealUEQ,
+ rustc_codegen_ssa::common::RealPredicate::RealUGT => RealPredicate::RealUGT,
+ rustc_codegen_ssa::common::RealPredicate::RealUGE => RealPredicate::RealUGE,
+ rustc_codegen_ssa::common::RealPredicate::RealULT => RealPredicate::RealULT,
+ rustc_codegen_ssa::common::RealPredicate::RealULE => RealPredicate::RealULE,
+ rustc_codegen_ssa::common::RealPredicate::RealUNE => RealPredicate::RealUNE,
+ rustc_codegen_ssa::common::RealPredicate::RealPredicateTrue => {
+ RealPredicate::RealPredicateTrue
+ }
+ }
+ }
+}
+
/// LLVMTypeKind
#[derive(Copy, Clone, PartialEq, Debug)]
#[repr(C)]
@@ -789,7 +823,7 @@
start_line,
start_col,
end_line,
- end_col: ((1 as u32) << 31) | end_col,
+ end_col: (1_u32 << 31) | end_col,
kind: RegionKind::GapRegion,
}
}
@@ -2176,6 +2210,7 @@
PrepareForThinLTO: bool,
PGOGenPath: *const c_char,
PGOUsePath: *const c_char,
+ PGOSampleUsePath: *const c_char,
);
pub fn LLVMRustAddLibraryInfo(
PM: &PassManager<'a>,
@@ -2210,6 +2245,8 @@
PGOUsePath: *const c_char,
InstrumentCoverage: bool,
InstrumentGCOV: bool,
+ PGOSampleUsePath: *const c_char,
+ DebugInfoForProfiling: bool,
llvm_selfprofiler: *mut c_void,
begin_callback: SelfProfileBeforePassCallback,
end_callback: SelfProfileAfterPassCallback,
@@ -2377,12 +2414,8 @@
len: usize,
out_len: &mut usize,
) -> *const u8;
- pub fn LLVMRustThinLTOGetDICompileUnit(
- M: &Module,
- CU1: &mut *mut c_void,
- CU2: &mut *mut c_void,
- );
- pub fn LLVMRustThinLTOPatchDICompileUnit(M: &Module, CU: *mut c_void);
+ pub fn LLVMRustLTOGetDICompileUnit(M: &Module, CU1: &mut *mut c_void, CU2: &mut *mut c_void);
+ pub fn LLVMRustLTOPatchDICompileUnit(M: &Module, CU: *mut c_void);
pub fn LLVMRustLinkerNew(M: &'a Module) -> &'a mut Linker<'a>;
pub fn LLVMRustLinkerAdd(
diff --git a/compiler/rustc_codegen_llvm/src/llvm_util.rs b/compiler/rustc_codegen_llvm/src/llvm_util.rs
index 3b64ec1..c2136f1 100644
--- a/compiler/rustc_codegen_llvm/src/llvm_util.rs
+++ b/compiler/rustc_codegen_llvm/src/llvm_util.rs
@@ -166,25 +166,32 @@
// Though note that Rust can also be build with an external precompiled version of LLVM
// which might lead to failures if the oldest tested / supported LLVM version
// doesn't yet support the relevant intrinsics
-pub fn to_llvm_feature<'a>(sess: &Session, s: &'a str) -> &'a str {
+pub fn to_llvm_feature<'a>(sess: &Session, s: &'a str) -> Vec<&'a str> {
let arch = if sess.target.arch == "x86_64" { "x86" } else { &*sess.target.arch };
match (arch, s) {
- ("x86", "pclmulqdq") => "pclmul",
- ("x86", "rdrand") => "rdrnd",
- ("x86", "bmi1") => "bmi",
- ("x86", "cmpxchg16b") => "cx16",
- ("x86", "avx512vaes") => "vaes",
- ("x86", "avx512gfni") => "gfni",
- ("x86", "avx512vpclmulqdq") => "vpclmulqdq",
- ("aarch64", "fp") => "fp-armv8",
- ("aarch64", "fp16") => "fullfp16",
- ("aarch64", "fhm") => "fp16fml",
- ("aarch64", "rcpc2") => "rcpc-immo",
- ("aarch64", "dpb") => "ccpp",
- ("aarch64", "dpb2") => "ccdp",
- ("aarch64", "frintts") => "fptoint",
- ("aarch64", "fcma") => "complxnum",
- (_, s) => s,
+ ("x86", "sse4.2") => {
+ if get_version() >= (14, 0, 0) {
+ vec!["sse4.2", "crc32"]
+ } else {
+ vec!["sse4.2"]
+ }
+ }
+ ("x86", "pclmulqdq") => vec!["pclmul"],
+ ("x86", "rdrand") => vec!["rdrnd"],
+ ("x86", "bmi1") => vec!["bmi"],
+ ("x86", "cmpxchg16b") => vec!["cx16"],
+ ("x86", "avx512vaes") => vec!["vaes"],
+ ("x86", "avx512gfni") => vec!["gfni"],
+ ("x86", "avx512vpclmulqdq") => vec!["vpclmulqdq"],
+ ("aarch64", "fp") => vec!["fp-armv8"],
+ ("aarch64", "fp16") => vec!["fullfp16"],
+ ("aarch64", "fhm") => vec!["fp16fml"],
+ ("aarch64", "rcpc2") => vec!["rcpc-immo"],
+ ("aarch64", "dpb") => vec!["ccpp"],
+ ("aarch64", "dpb2") => vec!["ccdp"],
+ ("aarch64", "frintts") => vec!["fptoint"],
+ ("aarch64", "fcma") => vec!["complxnum"],
+ (_, s) => vec![s],
}
}
@@ -198,9 +205,13 @@
},
)
.filter(|feature| {
- let llvm_feature = to_llvm_feature(sess, feature);
- let cstr = CString::new(llvm_feature).unwrap();
- unsafe { llvm::LLVMRustHasFeature(target_machine, cstr.as_ptr()) }
+ for llvm_feature in to_llvm_feature(sess, feature) {
+ let cstr = CString::new(llvm_feature).unwrap();
+ if unsafe { llvm::LLVMRustHasFeature(target_machine, cstr.as_ptr()) } {
+ return true;
+ }
+ }
+ false
})
.map(|feature| Symbol::intern(feature))
.collect()
@@ -253,12 +264,19 @@
let mut rustc_target_features = supported_target_features(sess)
.iter()
.filter_map(|(feature, _gate)| {
- let llvm_feature = to_llvm_feature(sess, *feature);
- // LLVM asserts that these are sorted. LLVM and Rust both use byte comparison for these strings.
- target_features.binary_search_by_key(&llvm_feature, |(f, _d)| *f).ok().map(|index| {
- let (_f, desc) = target_features.remove(index);
- (*feature, desc)
- })
+ for llvm_feature in to_llvm_feature(sess, *feature) {
+ // LLVM asserts that these are sorted. LLVM and Rust both use byte comparison for these strings.
+ match target_features.binary_search_by_key(&llvm_feature, |(f, _d)| (*f)).ok().map(
+ |index| {
+ let (_f, desc) = target_features.remove(index);
+ (*feature, desc)
+ },
+ ) {
+ Some(v) => return Some(v),
+ None => {}
+ }
+ }
+ None
})
.collect::<Vec<_>>();
rustc_target_features.extend_from_slice(&[(
@@ -280,7 +298,7 @@
for (feature, desc) in &target_features {
println!(" {1:0$} - {2}.", max_feature_len, feature, desc);
}
- if target_features.len() == 0 {
+ if target_features.is_empty() {
println!(" Target features listing is not supported by this LLVM version.");
}
println!("\nUse +feature to enable a feature, or -feature to disable it.");
@@ -373,30 +391,38 @@
let filter = |s: &str| {
if s.is_empty() {
- return None;
+ return vec![];
}
let feature = if s.starts_with('+') || s.starts_with('-') {
&s[1..]
} else {
- return Some(s.to_string());
+ return vec![s.to_string()];
};
// Rustc-specific feature requests like `+crt-static` or `-crt-static`
// are not passed down to LLVM.
if RUSTC_SPECIFIC_FEATURES.contains(&feature) {
- return None;
+ return vec![];
}
// ... otherwise though we run through `to_llvm_feature` feature when
// passing requests down to LLVM. This means that all in-language
// features also work on the command line instead of having two
// different names when the LLVM name and the Rust name differ.
- Some(format!("{}{}", &s[..1], to_llvm_feature(sess, feature)))
+ to_llvm_feature(sess, feature).iter().map(|f| format!("{}{}", &s[..1], f)).collect()
};
// Features implied by an implicit or explicit `--target`.
- features.extend(sess.target.features.split(',').filter_map(&filter));
+ features.extend(sess.target.features.split(',').flat_map(&filter));
// -Ctarget-features
- features.extend(sess.opts.cg.target_feature.split(',').filter_map(&filter));
+ features.extend(sess.opts.cg.target_feature.split(',').flat_map(&filter));
+
+ // FIXME: Move outline-atomics to target definition when earliest supported LLVM is 12.
+ if get_version() >= (12, 0, 0)
+ && sess.target.llvm_target.contains("aarch64-unknown-linux")
+ && sess.target.llvm_target != "aarch64-unknown-linux-musl"
+ {
+ features.push("+outline-atomics".to_string());
+ }
features
}
diff --git a/compiler/rustc_codegen_llvm/src/mono_item.rs b/compiler/rustc_codegen_llvm/src/mono_item.rs
index 8a8ece6..88498cf 100644
--- a/compiler/rustc_codegen_llvm/src/mono_item.rs
+++ b/compiler/rustc_codegen_llvm/src/mono_item.rs
@@ -1,4 +1,3 @@
-use crate::abi::FnAbi;
use crate::attributes;
use crate::base;
use crate::context::CodegenCx;
@@ -8,10 +7,9 @@
use rustc_hir::def_id::{DefId, LOCAL_CRATE};
pub use rustc_middle::mir::mono::MonoItem;
use rustc_middle::mir::mono::{Linkage, Visibility};
-use rustc_middle::ty::layout::FnAbiExt;
+use rustc_middle::ty::layout::{FnAbiOf, LayoutOf};
use rustc_middle::ty::{self, Instance, TypeFoldable};
use rustc_session::config::CrateType;
-use rustc_target::abi::LayoutOf;
use rustc_target::spec::RelocModel;
use tracing::debug;
@@ -54,11 +52,11 @@
) {
assert!(!instance.substs.needs_infer());
- let fn_abi = FnAbi::of_instance(self, instance, &[]);
- let lldecl = self.declare_fn(symbol_name, &fn_abi);
+ let fn_abi = self.fn_abi_of_instance(instance, ty::List::empty());
+ let lldecl = self.declare_fn(symbol_name, fn_abi);
unsafe { llvm::LLVMRustSetLinkage(lldecl, base::linkage_to_llvm(linkage)) };
let attrs = self.tcx.codegen_fn_attrs(instance.def_id());
- base::set_link_section(lldecl, &attrs);
+ base::set_link_section(lldecl, attrs);
if linkage == Linkage::LinkOnceODR || linkage == Linkage::WeakODR {
llvm::SetUniqueComdat(self.llmod, lldecl);
}
@@ -145,6 +143,8 @@
return true;
}
- return false;
+ // With pie relocation model calls of functions defined in the translation
+ // unit can use copy relocations.
+ self.tcx.sess.relocation_model() == RelocModel::Pie && !is_declaration
}
}
diff --git a/compiler/rustc_codegen_llvm/src/type_.rs b/compiler/rustc_codegen_llvm/src/type_.rs
index c7f4287..2ae0a08 100644
--- a/compiler/rustc_codegen_llvm/src/type_.rs
+++ b/compiler/rustc_codegen_llvm/src/type_.rs
@@ -248,7 +248,7 @@
}
fn ptr_to(&self, address_space: AddressSpace) -> &Type {
- unsafe { llvm::LLVMPointerType(&self, address_space.0) }
+ unsafe { llvm::LLVMPointerType(self, address_space.0) }
}
}
diff --git a/compiler/rustc_codegen_llvm/src/type_of.rs b/compiler/rustc_codegen_llvm/src/type_of.rs
index 9818905..f8c919e 100644
--- a/compiler/rustc_codegen_llvm/src/type_of.rs
+++ b/compiler/rustc_codegen_llvm/src/type_of.rs
@@ -1,15 +1,14 @@
-use crate::abi::FnAbi;
use crate::common::*;
use crate::context::TypeLowering;
use crate::type_::Type;
use rustc_codegen_ssa::traits::*;
use rustc_middle::bug;
-use rustc_middle::ty::layout::{FnAbiExt, TyAndLayout};
-use rustc_middle::ty::print::with_no_trimmed_paths;
+use rustc_middle::ty::layout::{FnAbiOf, LayoutOf, TyAndLayout};
+use rustc_middle::ty::print::{with_no_trimmed_paths, with_no_visible_paths};
use rustc_middle::ty::{self, Ty, TypeFoldable};
use rustc_target::abi::{Abi, AddressSpace, Align, FieldsShape};
use rustc_target::abi::{Int, Pointer, F32, F64};
-use rustc_target::abi::{LayoutOf, PointeeInfo, Scalar, Size, TyAbiInterface, Variants};
+use rustc_target::abi::{PointeeInfo, Scalar, Size, TyAbiInterface, Variants};
use smallvec::{smallvec, SmallVec};
use tracing::debug;
@@ -23,7 +22,7 @@
) -> &'a Type {
match layout.abi {
Abi::Scalar(_) => bug!("handled elsewhere"),
- Abi::Vector { ref element, count } => {
+ Abi::Vector { element, count } => {
let element = layout.scalar_llvm_type_at(cx, element, Size::ZERO);
return cx.type_vector(element, count);
}
@@ -44,7 +43,8 @@
// in problematically distinct types due to HRTB and subtyping (see #47638).
// ty::Dynamic(..) |
ty::Adt(..) | ty::Closure(..) | ty::Foreign(..) | ty::Generator(..) | ty::Str => {
- let mut name = with_no_trimmed_paths(|| layout.ty.to_string());
+ let mut name =
+ with_no_visible_paths(|| with_no_trimmed_paths(|| layout.ty.to_string()));
if let (&ty::Adt(def, _), &Variants::Single { index }) =
(layout.ty.kind(), &layout.variants)
{
@@ -177,7 +177,7 @@
fn scalar_llvm_type_at<'a>(
&self,
cx: &CodegenCx<'a, 'tcx>,
- scalar: &Scalar,
+ scalar: Scalar,
offset: Size,
) -> &'a Type;
fn scalar_pair_element_llvm_type<'a>(
@@ -218,7 +218,7 @@
/// of that field's type - this is useful for taking the address of
/// that field and ensuring the struct has the right alignment.
fn llvm_type<'a>(&self, cx: &CodegenCx<'a, 'tcx>) -> &'a Type {
- if let Abi::Scalar(ref scalar) = self.abi {
+ if let Abi::Scalar(scalar) = self.abi {
// Use a different cache for scalars because pointers to DSTs
// can be either fat or thin (data pointers of fat pointers).
if let Some(&llty) = cx.scalar_lltypes.borrow().get(&self.ty) {
@@ -231,7 +231,9 @@
ty::Adt(def, _) if def.is_box() => {
cx.type_ptr_to(cx.layout_of(self.ty.boxed_ty()).llvm_type(cx))
}
- ty::FnPtr(sig) => cx.fn_ptr_backend_type(&FnAbi::of_fn_ptr(cx, sig, &[])),
+ ty::FnPtr(sig) => {
+ cx.fn_ptr_backend_type(cx.fn_abi_of_fn_ptr(sig, ty::List::empty()))
+ }
_ => self.scalar_llvm_type_at(cx, scalar, Size::ZERO),
};
cx.scalar_lltypes.borrow_mut().insert(self.ty, llty);
@@ -243,7 +245,7 @@
Variants::Single { index } => Some(index),
_ => None,
};
- if let Some(ref llty) = cx.type_lowering.borrow().get(&(self.ty, variant_index)) {
+ if let Some(llty) = cx.type_lowering.borrow().get(&(self.ty, variant_index)) {
return llty.lltype;
}
@@ -268,10 +270,9 @@
};
debug!("--> mapped {:#?} to llty={:?}", self, llty);
- cx.type_lowering.borrow_mut().insert(
- (self.ty, variant_index),
- TypeLowering { lltype: llty, field_remapping: field_remapping },
- );
+ cx.type_lowering
+ .borrow_mut()
+ .insert((self.ty, variant_index), TypeLowering { lltype: llty, field_remapping });
if let Some((llty, layout)) = defer {
let (llfields, packed, new_field_remapping) = struct_llfields(cx, layout);
@@ -286,7 +287,7 @@
}
fn immediate_llvm_type<'a>(&self, cx: &CodegenCx<'a, 'tcx>) -> &'a Type {
- if let Abi::Scalar(ref scalar) = self.abi {
+ if let Abi::Scalar(scalar) = self.abi {
if scalar.is_bool() {
return cx.type_i1();
}
@@ -297,7 +298,7 @@
fn scalar_llvm_type_at<'a>(
&self,
cx: &CodegenCx<'a, 'tcx>,
- scalar: &Scalar,
+ scalar: Scalar,
offset: Size,
) -> &'a Type {
match scalar.value {
@@ -337,7 +338,7 @@
}
let (a, b) = match self.abi {
- Abi::ScalarPair(ref a, ref b) => (a, b),
+ Abi::ScalarPair(a, b) => (a, b),
_ => bug!("TyAndLayout::scalar_pair_element_llty({:?}): not applicable", self),
};
let scalar = [a, b][index];
diff --git a/compiler/rustc_codegen_llvm/src/va_arg.rs b/compiler/rustc_codegen_llvm/src/va_arg.rs
index 2208ec3..591f659 100644
--- a/compiler/rustc_codegen_llvm/src/va_arg.rs
+++ b/compiler/rustc_codegen_llvm/src/va_arg.rs
@@ -7,9 +7,9 @@
common::IntPredicate,
traits::{BaseTypeMethods, BuilderMethods, ConstMethods, DerivedTypeMethods},
};
-use rustc_middle::ty::layout::HasTyCtxt;
+use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf};
use rustc_middle::ty::Ty;
-use rustc_target::abi::{Align, Endian, HasDataLayout, LayoutOf, Size};
+use rustc_target::abi::{Align, Endian, HasDataLayout, Size};
fn round_pointer_up_to_alignment(
bx: &mut Builder<'a, 'll, 'tcx>,
@@ -125,7 +125,7 @@
// if the offset >= 0 then the value will be on the stack
let mut reg_off_v = bx.load(bx.type_i32(), reg_off, offset_align);
let use_stack = bx.icmp(IntPredicate::IntSGE, reg_off_v, zero);
- bx.cond_br(use_stack, &on_stack.llbb(), &maybe_reg.llbb());
+ bx.cond_br(use_stack, on_stack.llbb(), maybe_reg.llbb());
// The value at this point might be in a register, but there is a chance that
// it could be on the stack so we have to update the offset and then check
@@ -142,7 +142,7 @@
// Check to see if we have overflowed the registers as a result of this.
// If we have then we need to use the stack for this value
let use_stack = maybe_reg.icmp(IntPredicate::IntSGT, new_reg_off_v, zero);
- maybe_reg.cond_br(use_stack, &on_stack.llbb(), &in_reg.llbb());
+ maybe_reg.cond_br(use_stack, on_stack.llbb(), in_reg.llbb());
let top_type = bx.type_i8p();
let top = in_reg.struct_gep(va_list_ty, va_list_addr, reg_top_index);
@@ -158,17 +158,17 @@
let reg_type = layout.llvm_type(bx);
let reg_addr = in_reg.bitcast(reg_addr, bx.cx.type_ptr_to(reg_type));
let reg_value = in_reg.load(reg_type, reg_addr, layout.align.abi);
- in_reg.br(&end.llbb());
+ in_reg.br(end.llbb());
// On Stack block
let stack_value =
emit_ptr_va_arg(&mut on_stack, list, target_ty, false, Align::from_bytes(8).unwrap(), true);
- on_stack.br(&end.llbb());
+ on_stack.br(end.llbb());
let val = end.phi(
layout.immediate_llvm_type(bx),
&[reg_value, stack_value],
- &[&in_reg.llbb(), &on_stack.llbb()],
+ &[in_reg.llbb(), on_stack.llbb()],
);
*bx = end;
