From cd204e5ecb6172b7144bd146c25d29e0ed6d8e67 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Thu, 5 Jun 2025 11:33:03 +0000 Subject: [PATCH 1/6] Split out ABI handling for LLVM intrinsics --- compiler/rustc_codegen_ssa/src/mir/block.rs | 86 ++++++++++++++++++++- 1 file changed, 85 insertions(+), 1 deletion(-) diff --git a/compiler/rustc_codegen_ssa/src/mir/block.rs b/compiler/rustc_codegen_ssa/src/mir/block.rs index 0d1e27a34dc5b..207f9ade18586 100644 --- a/compiler/rustc_codegen_ssa/src/mir/block.rs +++ b/compiler/rustc_codegen_ssa/src/mir/block.rs @@ -14,7 +14,7 @@ use rustc_middle::{bug, span_bug}; use rustc_session::config::OptLevel; use rustc_span::Span; use rustc_span::source_map::Spanned; -use rustc_target::callconv::{ArgAbi, CastTarget, FnAbi, PassMode}; +use rustc_target::callconv::{ArgAbi, ArgAttributes, CastTarget, FnAbi, PassMode}; use tracing::{debug, info}; use super::operand::OperandRef; @@ -1034,6 +1034,90 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { _ => bug!("{} is not callable", callee.layout.ty), }; + if let Some(instance) = instance + && let Some(name) = bx.tcx().codegen_fn_attrs(instance.def_id()).symbol_name + && name.as_str().starts_with("llvm.") + { + let mut llargs = Vec::with_capacity(args.len()); + + let dest_ty = destination.ty(&self.mir.local_decls, bx.tcx()).ty; + let return_dest = if dest_ty.is_unit() { + ReturnDest::Nothing + } else if let Some(index) = destination.as_local() { + match self.locals[index] { + LocalRef::Place(dest) => ReturnDest::Store(dest), + LocalRef::UnsizedPlace(_) => bug!("return type must be sized"), + LocalRef::PendingOperand => { + // Handle temporary places, specifically `Operand` ones, as + // they don't have `alloca`s. + ReturnDest::DirectOperand(index) + } + LocalRef::Operand(_) => bug!("place local already assigned to"), + } + } else { + ReturnDest::Store(self.codegen_place(bx, destination.as_ref())) + }; + + for arg in args { + let op = self.codegen_operand(bx, &arg.node); + + match op.val { + ZeroSized => {} + Immediate(_) => llargs.push(op.immediate()), + Pair(a, b) => { + llargs.push(a); + llargs.push(b); + } + Ref(op_place_val) => { + let mut llval = op_place_val.llval; + // We can't use `PlaceRef::load` here because the argument + // may have a type we don't treat as immediate, but the ABI + // used for this call is passing it by-value. In that case, + // the load would just produce `OperandValue::Ref` instead + // of the `OperandValue::Immediate` we need for the call. + llval = bx.load(bx.backend_type(op.layout), llval, op_place_val.align); + if let BackendRepr::Scalar(scalar) = op.layout.backend_repr { + if scalar.is_bool() { + bx.range_metadata(llval, WrappingRange { start: 0, end: 1 }); + } + // We store bools as `i8` so we need to truncate to `i1`. + llval = bx.to_immediate_scalar(llval, scalar); + } + llargs.push(llval); + } + } + } + + let fn_ptr = bx.get_fn_addr(instance); + self.set_debug_loc(bx, source_info); + + // FIXME remove usage of fn_abi + let fn_abi = bx.fn_abi_of_instance(instance, ty::List::empty()); + assert!(!fn_abi.ret.is_indirect()); + let fn_ty = bx.fn_decl_backend_type(fn_abi); + + let llret = bx.call(fn_ty, None, None, fn_ptr, &llargs, None, None); + if self.mir[helper.bb].is_cleanup { + bx.apply_attrs_to_cleanup_callsite(llret); + } + + if let Some(target) = target { + self.store_return( + bx, + return_dest, + &ArgAbi { + layout: bx.layout_of(dest_ty), + mode: PassMode::Direct(ArgAttributes::new()), + }, + llret, + ); + return helper.funclet_br(self, bx, target, mergeable_succ); + } else { + bx.unreachable(); + return MergingSucc::False; + } + } + // FIXME(eddyb) avoid computing this if possible, when `instance` is // available - right now `sig` is only needed for getting the `abi` // and figuring out how many extra args were passed to a C-variadic `fn`. From 209d94a00f419c546104dddcb32bb9dd310b0dd9 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Thu, 18 Sep 2025 13:25:01 +0000 Subject: [PATCH 2/6] Move llvm intrinsic call to backend --- .../rustc_codegen_gcc/src/intrinsic/mod.rs | 52 +++++++++++++++++- compiler/rustc_codegen_llvm/src/intrinsic.rs | 54 ++++++++++++++++++- compiler/rustc_codegen_ssa/src/mir/block.rs | 45 ++-------------- .../rustc_codegen_ssa/src/traits/intrinsic.rs | 7 +++ 4 files changed, 114 insertions(+), 44 deletions(-) diff --git a/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs b/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs index 99a4f9b9f7e7e..c433fb42987bc 100644 --- a/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs +++ b/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs @@ -9,7 +9,7 @@ use gccjit::Type; use gccjit::{ComparisonOp, Function, FunctionType, RValue, ToRValue, UnaryOp}; #[cfg(feature = "master")] use rustc_abi::ExternAbi; -use rustc_abi::{BackendRepr, HasDataLayout}; +use rustc_abi::{BackendRepr, HasDataLayout, WrappingRange}; use rustc_codegen_ssa::MemFlags; use rustc_codegen_ssa::base::wants_msvc_seh; use rustc_codegen_ssa::common::IntPredicate; @@ -20,7 +20,7 @@ use rustc_codegen_ssa::mir::place::{PlaceRef, PlaceValue}; use rustc_codegen_ssa::traits::MiscCodegenMethods; use rustc_codegen_ssa::traits::{ ArgAbiBuilderMethods, BaseTypeCodegenMethods, BuilderMethods, ConstCodegenMethods, - IntrinsicCallBuilderMethods, + IntrinsicCallBuilderMethods, LayoutTypeCodegenMethods, }; use rustc_middle::bug; #[cfg(feature = "master")] @@ -640,6 +640,54 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tc Ok(()) } + fn codegen_llvm_intrinsic_call( + &mut self, + instance: ty::Instance<'tcx>, + args: &[OperandRef<'tcx, Self::Value>], + is_cleanup: bool, + ) -> Self::Value { + let fn_ptr = self.get_fn_addr(instance); + let fn_ty = fn_ptr.get_type(); + + let mut llargs = vec![]; + + for arg in args { + match arg.val { + OperandValue::ZeroSized => {} + OperandValue::Immediate(_) => llargs.push(arg.immediate()), + OperandValue::Pair(a, b) => { + llargs.push(a); + llargs.push(b); + } + OperandValue::Ref(op_place_val) => { + let mut llval = op_place_val.llval; + // We can't use `PlaceRef::load` here because the argument + // may have a type we don't treat as immediate, but the ABI + // used for this call is passing it by-value. In that case, + // the load would just produce `OperandValue::Ref` instead + // of the `OperandValue::Immediate` we need for the call. + llval = self.load(self.backend_type(arg.layout), llval, op_place_val.align); + if let BackendRepr::Scalar(scalar) = arg.layout.backend_repr { + if scalar.is_bool() { + self.range_metadata(llval, WrappingRange { start: 0, end: 1 }); + } + // We store bools as `i8` so we need to truncate to `i1`. + llval = self.to_immediate_scalar(llval, scalar); + } + llargs.push(llval); + } + } + } + + // FIXME directly use the llvm intrinsic adjustment functions here + let llret = self.call(fn_ty, None, None, fn_ptr, &llargs, None, None); + if is_cleanup { + self.apply_attrs_to_cleanup_callsite(llret); + } + + llret + } + fn abort(&mut self) { let func = self.context.get_builtin_function("abort"); let func: RValue<'gcc> = unsafe { std::mem::transmute(func) }; diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index 14b3f3626efe9..1d1f29310fb90 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -1,7 +1,9 @@ use std::assert_matches::assert_matches; use std::cmp::Ordering; -use rustc_abi::{Align, BackendRepr, ExternAbi, Float, HasDataLayout, Primitive, Size}; +use rustc_abi::{ + Align, BackendRepr, ExternAbi, Float, HasDataLayout, Primitive, Size, WrappingRange, +}; use rustc_codegen_ssa::base::{compare_simd_types, wants_msvc_seh, wants_wasm_eh}; use rustc_codegen_ssa::codegen_attrs::autodiff_attrs; use rustc_codegen_ssa::common::{IntPredicate, TypeKind}; @@ -617,6 +619,56 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { Ok(()) } + fn codegen_llvm_intrinsic_call( + &mut self, + instance: ty::Instance<'tcx>, + args: &[OperandRef<'tcx, Self::Value>], + is_cleanup: bool, + ) -> Self::Value { + let fn_ptr = self.get_fn_addr(instance); + // FIXME remove usage of fn_abi + let fn_abi = self.fn_abi_of_instance(instance, ty::List::empty()); + assert!(!fn_abi.ret.is_indirect()); + let fn_ty = self.fn_decl_backend_type(fn_abi); + + let mut llargs = vec![]; + + for arg in args { + match arg.val { + OperandValue::ZeroSized => {} + OperandValue::Immediate(_) => llargs.push(arg.immediate()), + OperandValue::Pair(a, b) => { + llargs.push(a); + llargs.push(b); + } + OperandValue::Ref(op_place_val) => { + let mut llval = op_place_val.llval; + // We can't use `PlaceRef::load` here because the argument + // may have a type we don't treat as immediate, but the ABI + // used for this call is passing it by-value. In that case, + // the load would just produce `OperandValue::Ref` instead + // of the `OperandValue::Immediate` we need for the call. + llval = self.load(self.backend_type(arg.layout), llval, op_place_val.align); + if let BackendRepr::Scalar(scalar) = arg.layout.backend_repr { + if scalar.is_bool() { + self.range_metadata(llval, WrappingRange { start: 0, end: 1 }); + } + // We store bools as `i8` so we need to truncate to `i1`. + llval = self.to_immediate_scalar(llval, scalar); + } + llargs.push(llval); + } + } + } + + let llret = self.call(fn_ty, None, None, fn_ptr, &llargs, None, None); + if is_cleanup { + self.apply_attrs_to_cleanup_callsite(llret); + } + + llret + } + fn abort(&mut self) { self.call_intrinsic("llvm.trap", &[], &[]); } diff --git a/compiler/rustc_codegen_ssa/src/mir/block.rs b/compiler/rustc_codegen_ssa/src/mir/block.rs index 207f9ade18586..85d7c6e97a678 100644 --- a/compiler/rustc_codegen_ssa/src/mir/block.rs +++ b/compiler/rustc_codegen_ssa/src/mir/block.rs @@ -1038,8 +1038,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { && let Some(name) = bx.tcx().codegen_fn_attrs(instance.def_id()).symbol_name && name.as_str().starts_with("llvm.") { - let mut llargs = Vec::with_capacity(args.len()); - let dest_ty = destination.ty(&self.mir.local_decls, bx.tcx()).ty; let return_dest = if dest_ty.is_unit() { ReturnDest::Nothing @@ -1058,48 +1056,13 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { ReturnDest::Store(self.codegen_place(bx, destination.as_ref())) }; - for arg in args { - let op = self.codegen_operand(bx, &arg.node); - - match op.val { - ZeroSized => {} - Immediate(_) => llargs.push(op.immediate()), - Pair(a, b) => { - llargs.push(a); - llargs.push(b); - } - Ref(op_place_val) => { - let mut llval = op_place_val.llval; - // We can't use `PlaceRef::load` here because the argument - // may have a type we don't treat as immediate, but the ABI - // used for this call is passing it by-value. In that case, - // the load would just produce `OperandValue::Ref` instead - // of the `OperandValue::Immediate` we need for the call. - llval = bx.load(bx.backend_type(op.layout), llval, op_place_val.align); - if let BackendRepr::Scalar(scalar) = op.layout.backend_repr { - if scalar.is_bool() { - bx.range_metadata(llval, WrappingRange { start: 0, end: 1 }); - } - // We store bools as `i8` so we need to truncate to `i1`. - llval = bx.to_immediate_scalar(llval, scalar); - } - llargs.push(llval); - } - } - } + let args = + args.into_iter().map(|arg| self.codegen_operand(bx, &arg.node)).collect::>(); - let fn_ptr = bx.get_fn_addr(instance); self.set_debug_loc(bx, source_info); - // FIXME remove usage of fn_abi - let fn_abi = bx.fn_abi_of_instance(instance, ty::List::empty()); - assert!(!fn_abi.ret.is_indirect()); - let fn_ty = bx.fn_decl_backend_type(fn_abi); - - let llret = bx.call(fn_ty, None, None, fn_ptr, &llargs, None, None); - if self.mir[helper.bb].is_cleanup { - bx.apply_attrs_to_cleanup_callsite(llret); - } + let llret = + bx.codegen_llvm_intrinsic_call(instance, &args, self.mir[helper.bb].is_cleanup); if let Some(target) = target { self.store_return( diff --git a/compiler/rustc_codegen_ssa/src/traits/intrinsic.rs b/compiler/rustc_codegen_ssa/src/traits/intrinsic.rs index c5ecf43046c74..6761f6c512964 100644 --- a/compiler/rustc_codegen_ssa/src/traits/intrinsic.rs +++ b/compiler/rustc_codegen_ssa/src/traits/intrinsic.rs @@ -25,6 +25,13 @@ pub trait IntrinsicCallBuilderMethods<'tcx>: BackendTypes { span: Span, ) -> Result<(), ty::Instance<'tcx>>; + fn codegen_llvm_intrinsic_call( + &mut self, + instance: ty::Instance<'tcx>, + args: &[OperandRef<'tcx, Self::Value>], + is_cleanup: bool, + ) -> Self::Value; + fn abort(&mut self); fn assume(&mut self, val: Self::Value); fn expect(&mut self, cond: Self::Value, expected: bool) -> Self::Value; From 4f4812f9ccae74b4360a4bc3b38b83e8450a94e6 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Tue, 11 Nov 2025 11:03:25 +0100 Subject: [PATCH 3/6] Don't assume get_fn is only called from codegen_mir in cg_gcc --- compiler/rustc_codegen_gcc/src/builder.rs | 1 + compiler/rustc_codegen_gcc/src/context.rs | 4 +--- compiler/rustc_codegen_gcc/src/declare.rs | 8 ++------ 3 files changed, 4 insertions(+), 9 deletions(-) diff --git a/compiler/rustc_codegen_gcc/src/builder.rs b/compiler/rustc_codegen_gcc/src/builder.rs index 5657620879ca1..ddeeafd894cf0 100644 --- a/compiler/rustc_codegen_gcc/src/builder.rs +++ b/compiler/rustc_codegen_gcc/src/builder.rs @@ -514,6 +514,7 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> { type CodegenCx = CodegenCx<'gcc, 'tcx>; fn build(cx: &'a CodegenCx<'gcc, 'tcx>, block: Block<'gcc>) -> Builder<'a, 'gcc, 'tcx> { + *cx.current_func.borrow_mut() = Some(block.get_function()); Builder::with_cx(cx, block) } diff --git a/compiler/rustc_codegen_gcc/src/context.rs b/compiler/rustc_codegen_gcc/src/context.rs index c9ae96777de44..453214edf2e0f 100644 --- a/compiler/rustc_codegen_gcc/src/context.rs +++ b/compiler/rustc_codegen_gcc/src/context.rs @@ -391,9 +391,7 @@ impl<'gcc, 'tcx> MiscCodegenMethods<'tcx> for CodegenCx<'gcc, 'tcx> { } fn get_fn(&self, instance: Instance<'tcx>) -> Function<'gcc> { - let func = get_fn(self, instance); - *self.current_func.borrow_mut() = Some(func); - func + get_fn(self, instance) } fn get_fn_addr(&self, instance: Instance<'tcx>) -> RValue<'gcc> { diff --git a/compiler/rustc_codegen_gcc/src/declare.rs b/compiler/rustc_codegen_gcc/src/declare.rs index 691fd8729e39a..43225662762f3 100644 --- a/compiler/rustc_codegen_gcc/src/declare.rs +++ b/compiler/rustc_codegen_gcc/src/declare.rs @@ -100,18 +100,14 @@ impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> { let return_type = self.type_i32(); let variadic = false; self.linkage.set(FunctionType::Exported); - let func = declare_raw_fn( + declare_raw_fn( self, name, callconv, return_type, &[self.type_i32(), const_string], variadic, - ); - // NOTE: it is needed to set the current_func here as well, because get_fn() is not called - // for the main function. - *self.current_func.borrow_mut() = Some(func); - func + ) } pub fn declare_fn(&self, name: &str, fn_abi: &FnAbi<'tcx, Ty<'tcx>>) -> Function<'gcc> { From be36d2f6ff567128844d22c4006826d96a97db00 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Tue, 11 Nov 2025 11:14:03 +0100 Subject: [PATCH 4/6] Pass Function to Builder::function_call in cg_gcc --- compiler/rustc_codegen_gcc/src/builder.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/compiler/rustc_codegen_gcc/src/builder.rs b/compiler/rustc_codegen_gcc/src/builder.rs index ddeeafd894cf0..ea61afac03595 100644 --- a/compiler/rustc_codegen_gcc/src/builder.rs +++ b/compiler/rustc_codegen_gcc/src/builder.rs @@ -316,12 +316,10 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> { fn function_call( &mut self, - func: RValue<'gcc>, + func: Function<'gcc>, args: &[RValue<'gcc>], _funclet: Option<&Funclet>, ) -> RValue<'gcc> { - // TODO(antoyo): remove when the API supports a different type for functions. - let func: Function<'gcc> = self.cx.rvalue_as_function(func); let args = self.check_call("call", func, args); // gccjit requires to use the result of functions, even when it's not used. @@ -1753,6 +1751,8 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> { // FIXME(antoyo): remove when having a proper API. let gcc_func = unsafe { std::mem::transmute::, Function<'gcc>>(func) }; let call = if self.functions.borrow().values().any(|value| *value == gcc_func) { + // TODO(antoyo): remove when the API supports a different type for functions. + let func: Function<'gcc> = self.cx.rvalue_as_function(func); self.function_call(func, args, funclet) } else { // If it's a not function that was defined, it's a function pointer. From 915af4f6328351ce5e797e3e936c4249f99ce661 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Tue, 11 Nov 2025 11:52:43 +0100 Subject: [PATCH 5/6] Inline BuilderMethods::call for intrinsics Intrinsics only need a fraction of the functionality offered by BuilderMethods::call and in particular don't need the FnAbi to be computed other than (currently) as step towards computing the function value type. --- compiler/rustc_codegen_llvm/src/builder.rs | 2 +- compiler/rustc_codegen_llvm/src/intrinsic.rs | 17 ++++++++++++++++- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs index d441cd119a747..ff1e8ffc2deca 100644 --- a/compiler/rustc_codegen_llvm/src/builder.rs +++ b/compiler/rustc_codegen_llvm/src/builder.rs @@ -1622,7 +1622,7 @@ impl<'a, 'll, CX: Borrow>> GenericBuilder<'a, 'll, CX> { ret.expect("LLVM does not have support for catchret") } - fn check_call<'b>( + pub(crate) fn check_call<'b>( &mut self, typ: &str, fn_ty: &'ll Type, diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index 1d1f29310fb90..15350f5cf2337 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -1,5 +1,7 @@ use std::assert_matches::assert_matches; use std::cmp::Ordering; +use std::ffi::c_uint; +use std::ptr; use rustc_abi::{ Align, BackendRepr, ExternAbi, Float, HasDataLayout, Primitive, Size, WrappingRange, @@ -661,7 +663,20 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { } } - let llret = self.call(fn_ty, None, None, fn_ptr, &llargs, None, None); + debug!("call intrinsic {:?} with args ({:?})", instance, llargs); + let args = self.check_call("call", fn_ty, fn_ptr, &llargs); + let llret = unsafe { + llvm::LLVMBuildCallWithOperandBundles( + self.llbuilder, + fn_ty, + fn_ptr, + args.as_ptr() as *const &llvm::Value, + args.len() as c_uint, + ptr::dangling(), + 0, + c"".as_ptr(), + ) + }; if is_cleanup { self.apply_attrs_to_cleanup_callsite(llret); } From 6d4eb0e3f3137cabed500e89e6c7d50ff4a117d8 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Mon, 10 Nov 2025 13:42:32 +0000 Subject: [PATCH 6/6] Partially inline get_fn_addr/get_fn in codegen_llvm_intrinsic_call This moves all LLVM intrinsic handling out of the regular call path for cg_gcc and makes it easier to hook into this code for future cg_llvm changes. --- compiler/rustc_codegen_gcc/src/builder.rs | 2 +- compiler/rustc_codegen_gcc/src/context.rs | 7 ++-- compiler/rustc_codegen_gcc/src/declare.rs | 14 ------- .../rustc_codegen_gcc/src/intrinsic/mod.rs | 42 ++++++++++++++++++- compiler/rustc_codegen_llvm/src/context.rs | 3 ++ compiler/rustc_codegen_llvm/src/intrinsic.rs | 35 +++++++++++++++- compiler/rustc_codegen_ssa/src/mir/block.rs | 5 ++- 7 files changed, 86 insertions(+), 22 deletions(-) diff --git a/compiler/rustc_codegen_gcc/src/builder.rs b/compiler/rustc_codegen_gcc/src/builder.rs index ea61afac03595..703b1309b619e 100644 --- a/compiler/rustc_codegen_gcc/src/builder.rs +++ b/compiler/rustc_codegen_gcc/src/builder.rs @@ -314,7 +314,7 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> { self.block.get_function() } - fn function_call( + pub fn function_call( &mut self, func: Function<'gcc>, args: &[RValue<'gcc>], diff --git a/compiler/rustc_codegen_gcc/src/context.rs b/compiler/rustc_codegen_gcc/src/context.rs index 453214edf2e0f..f437d44e48b17 100644 --- a/compiler/rustc_codegen_gcc/src/context.rs +++ b/compiler/rustc_codegen_gcc/src/context.rs @@ -92,6 +92,8 @@ pub struct CodegenCx<'gcc, 'tcx> { pub instances: RefCell, LValue<'gcc>>>, /// Cache function instances of monomorphic and polymorphic items pub function_instances: RefCell, Function<'gcc>>>, + /// Cache function instances of intrinsics + pub intrinsic_instances: RefCell, Function<'gcc>>>, /// Cache generated vtables pub vtables: RefCell, Option>), RValue<'gcc>>>, @@ -280,6 +282,7 @@ impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> { linkage: Cell::new(FunctionType::Internal), instances: Default::default(), function_instances: Default::default(), + intrinsic_instances: Default::default(), on_stack_params: Default::default(), on_stack_function_params: Default::default(), vtables: Default::default(), @@ -397,9 +400,7 @@ impl<'gcc, 'tcx> MiscCodegenMethods<'tcx> for CodegenCx<'gcc, 'tcx> { fn get_fn_addr(&self, instance: Instance<'tcx>) -> RValue<'gcc> { let func_name = self.tcx.symbol_name(instance).name; - let func = if self.intrinsics.borrow().contains_key(func_name) { - self.intrinsics.borrow()[func_name] - } else if let Some(variable) = self.get_declared_value(func_name) { + let func = if let Some(variable) = self.get_declared_value(func_name) { return variable; } else { get_fn(self, instance) diff --git a/compiler/rustc_codegen_gcc/src/declare.rs b/compiler/rustc_codegen_gcc/src/declare.rs index 43225662762f3..706f9bd1f64f4 100644 --- a/compiler/rustc_codegen_gcc/src/declare.rs +++ b/compiler/rustc_codegen_gcc/src/declare.rs @@ -8,7 +8,6 @@ use rustc_target::callconv::FnAbi; use crate::abi::{FnAbiGcc, FnAbiGccExt}; use crate::context::CodegenCx; -use crate::intrinsic::llvm; impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> { pub fn get_or_insert_global( @@ -162,19 +161,6 @@ fn declare_raw_fn<'gcc>( param_types: &[Type<'gcc>], variadic: bool, ) -> Function<'gcc> { - if name.starts_with("llvm.") { - let intrinsic = match name { - "llvm.fma.f16" => { - // fma is not a target builtin, but a normal builtin, so we handle it differently - // here. - cx.context.get_builtin_function("fma") - } - _ => llvm::intrinsic(name, cx), - }; - - cx.intrinsics.borrow_mut().insert(name.to_string(), intrinsic); - return intrinsic; - } let func = if cx.functions.borrow().contains_key(name) { cx.functions.borrow()[name] } else { diff --git a/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs b/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs index c433fb42987bc..cd6a444ce7a1f 100644 --- a/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs +++ b/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs @@ -646,7 +646,47 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tc args: &[OperandRef<'tcx, Self::Value>], is_cleanup: bool, ) -> Self::Value { - let fn_ptr = self.get_fn_addr(instance); + let func = if let Some(&func) = self.intrinsic_instances.borrow().get(&instance) { + func + } else { + let sym = self.tcx.symbol_name(instance).name; + + let func = if let Some(func) = self.intrinsics.borrow().get(sym) { + *func + } else { + self.linkage.set(FunctionType::Extern); + let fn_abi = self.fn_abi_of_instance(instance, ty::List::empty()); + let fn_ty = fn_abi.gcc_type(self); + + let func = match sym { + "llvm.fma.f16" => { + // fma is not a target builtin, but a normal builtin, so we handle it differently + // here. + self.context.get_builtin_function("fma") + } + _ => llvm::intrinsic(sym, self), + }; + + self.intrinsics.borrow_mut().insert(sym.to_string(), func); + + self.on_stack_function_params + .borrow_mut() + .insert(func, fn_ty.on_stack_param_indices); + #[cfg(feature = "master")] + for fn_attr in fn_ty.fn_attributes { + func.add_attribute(fn_attr); + } + + crate::attributes::from_fn_attrs(self, func, instance); + + func + }; + + self.intrinsic_instances.borrow_mut().insert(instance, func); + + func + }; + let fn_ptr = func.get_address(None); let fn_ty = fn_ptr.get_type(); let mut llargs = vec![]; diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs index c01f163f2ee19..c4f75fee9ce8a 100644 --- a/compiler/rustc_codegen_llvm/src/context.rs +++ b/compiler/rustc_codegen_llvm/src/context.rs @@ -100,6 +100,8 @@ pub(crate) struct FullCx<'ll, 'tcx> { /// Cache instances of monomorphic and polymorphic items pub instances: RefCell, &'ll Value>>, + /// Cache instances of intrinsics + pub intrinsic_instances: RefCell, &'ll Value>>, /// Cache generated vtables pub vtables: RefCell, Option>), &'ll Value>>, /// Cache of constant strings, @@ -620,6 +622,7 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { tls_model, codegen_unit, instances: Default::default(), + intrinsic_instances: Default::default(), vtables: Default::default(), const_str_cache: Default::default(), const_globals: Default::default(), diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index 15350f5cf2337..31a62ad53d310 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -28,6 +28,7 @@ use crate::abi::FnAbiLlvmExt; use crate::builder::Builder; use crate::builder::autodiff::{adjust_activity_to_abi, generate_enzyme_call}; use crate::context::CodegenCx; +use crate::declare::declare_raw_fn; use crate::errors::AutoDiffWithoutEnable; use crate::llvm::{self, Metadata, Type, Value}; use crate::type_of::LayoutLlvmExt; @@ -627,11 +628,41 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { args: &[OperandRef<'tcx, Self::Value>], is_cleanup: bool, ) -> Self::Value { - let fn_ptr = self.get_fn_addr(instance); + let tcx = self.tcx(); + // FIXME remove usage of fn_abi let fn_abi = self.fn_abi_of_instance(instance, ty::List::empty()); assert!(!fn_abi.ret.is_indirect()); - let fn_ty = self.fn_decl_backend_type(fn_abi); + let fn_ty = fn_abi.llvm_type(self); + + let fn_ptr = if let Some(&llfn) = self.intrinsic_instances.borrow().get(&instance) { + llfn + } else { + let sym = tcx.symbol_name(instance).name; + + // FIXME use get_intrinsic + let llfn = if let Some(llfn) = self.get_declared_value(sym) { + llfn + } else { + // Function addresses in Rust are never significant, allowing functions to + // be merged. + let llfn = declare_raw_fn( + self, + sym, + fn_abi.llvm_cconv(self), + llvm::UnnamedAddr::Global, + llvm::Visibility::Default, + fn_ty, + ); + fn_abi.apply_attrs_llfn(self, llfn, Some(instance)); + + llfn + }; + + self.intrinsic_instances.borrow_mut().insert(instance, llfn); + + llfn + }; let mut llargs = vec![]; diff --git a/compiler/rustc_codegen_ssa/src/mir/block.rs b/compiler/rustc_codegen_ssa/src/mir/block.rs index 85d7c6e97a678..8df73abe9e049 100644 --- a/compiler/rustc_codegen_ssa/src/mir/block.rs +++ b/compiler/rustc_codegen_ssa/src/mir/block.rs @@ -9,7 +9,7 @@ use rustc_lint_defs::builtin::TAIL_CALL_TRACK_CALLER; use rustc_middle::mir::{self, AssertKind, InlineAsmMacro, SwitchTargets, UnwindTerminateReason}; use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf, ValidityRequirement}; use rustc_middle::ty::print::{with_no_trimmed_paths, with_no_visible_paths}; -use rustc_middle::ty::{self, Instance, Ty}; +use rustc_middle::ty::{self, Instance, Ty, TypeVisitableExt}; use rustc_middle::{bug, span_bug}; use rustc_session::config::OptLevel; use rustc_span::Span; @@ -1038,6 +1038,9 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { && let Some(name) = bx.tcx().codegen_fn_attrs(instance.def_id()).symbol_name && name.as_str().starts_with("llvm.") { + assert!(!instance.args.has_infer()); + assert!(!instance.args.has_escaping_bound_vars()); + let dest_ty = destination.ty(&self.mir.local_decls, bx.tcx()).ty; let return_dest = if dest_ty.is_unit() { ReturnDest::Nothing