Partially inline get_fn_addr/get_fn in codegen_llvm_intrinsic_call

This moves all LLVM intrinsic handling out of the regular call path for
cg_gcc and makes it easier to hook into this code for future cg_llvm
changes.

Author: bjorn3

compiler/rustc_codegen_gcc/src/builder.rs

@@ -314,7 +314,7 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {
         self.block.get_function()
     }
 
-    fn function_call(
+    pub fn function_call(
         &mut self,
         func: Function<'gcc>,
         args: &[RValue<'gcc>],

compiler/rustc_codegen_gcc/src/context.rs

@@ -92,6 +92,8 @@ pub struct CodegenCx<'gcc, 'tcx> {
     pub instances: RefCell<FxHashMap<Instance<'tcx>, LValue<'gcc>>>,
     /// Cache function instances of monomorphic and polymorphic items
     pub function_instances: RefCell<FxHashMap<Instance<'tcx>, Function<'gcc>>>,
+    /// Cache function instances of intrinsics
+    pub intrinsic_instances: RefCell<FxHashMap<Instance<'tcx>, Function<'gcc>>>,
     /// Cache generated vtables
     pub vtables:
         RefCell<FxHashMap<(Ty<'tcx>, Option<ty::ExistentialTraitRef<'tcx>>), RValue<'gcc>>>,
@@ -280,6 +282,7 @@ impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> {
             linkage: Cell::new(FunctionType::Internal),
             instances: Default::default(),
             function_instances: Default::default(),
+            intrinsic_instances: Default::default(),
             on_stack_params: Default::default(),
             on_stack_function_params: Default::default(),
             vtables: Default::default(),
@@ -397,9 +400,7 @@ impl<'gcc, 'tcx> MiscCodegenMethods<'tcx> for CodegenCx<'gcc, 'tcx> {
     fn get_fn_addr(&self, instance: Instance<'tcx>) -> RValue<'gcc> {
         let func_name = self.tcx.symbol_name(instance).name;
 
-        let func = if self.intrinsics.borrow().contains_key(func_name) {
-            self.intrinsics.borrow()[func_name]
-        } else if let Some(variable) = self.get_declared_value(func_name) {
+        let func = if let Some(variable) = self.get_declared_value(func_name) {
             return variable;
         } else {
             get_fn(self, instance)

compiler/rustc_codegen_gcc/src/declare.rs

@@ -8,7 +8,6 @@ use rustc_target::callconv::FnAbi;
 
 use crate::abi::{FnAbiGcc, FnAbiGccExt};
 use crate::context::CodegenCx;
-use crate::intrinsic::llvm;
 
 impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> {
     pub fn get_or_insert_global(
@@ -162,19 +161,6 @@ fn declare_raw_fn<'gcc>(
     param_types: &[Type<'gcc>],
     variadic: bool,
 ) -> Function<'gcc> {
-    if name.starts_with("llvm.") {
-        let intrinsic = match name {
-            "llvm.fma.f16" => {
-                // fma is not a target builtin, but a normal builtin, so we handle it differently
-                // here.
-                cx.context.get_builtin_function("fma")
-            }
-            _ => llvm::intrinsic(name, cx),
-        };
-
-        cx.intrinsics.borrow_mut().insert(name.to_string(), intrinsic);
-        return intrinsic;
-    }
     let func = if cx.functions.borrow().contains_key(name) {
         cx.functions.borrow()[name]
     } else {

compiler/rustc_codegen_gcc/src/intrinsic/mod.rs

@@ -646,7 +646,47 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tc
         args: &[OperandRef<'tcx, Self::Value>],
         is_cleanup: bool,
     ) -> Self::Value {
-        let fn_ptr = self.get_fn_addr(instance);
+        let func = if let Some(&func) = self.intrinsic_instances.borrow().get(&instance) {
+            func
+        } else {
+            let sym = self.tcx.symbol_name(instance).name;
+
+            let func = if let Some(func) = self.intrinsics.borrow().get(sym) {
+                *func
+            } else {
+                self.linkage.set(FunctionType::Extern);
+                let fn_abi = self.fn_abi_of_instance(instance, ty::List::empty());
+                let fn_ty = fn_abi.gcc_type(self);
+
+                let func = match sym {
+                    "llvm.fma.f16" => {
+                        // fma is not a target builtin, but a normal builtin, so we handle it differently
+                        // here.
+                        self.context.get_builtin_function("fma")
+                    }
+                    _ => llvm::intrinsic(sym, self),
+                };
+
+                self.intrinsics.borrow_mut().insert(sym.to_string(), func);
+
+                self.on_stack_function_params
+                    .borrow_mut()
+                    .insert(func, fn_ty.on_stack_param_indices);
+                #[cfg(feature = "master")]
+                for fn_attr in fn_ty.fn_attributes {
+                    func.add_attribute(fn_attr);
+                }
+
+                crate::attributes::from_fn_attrs(self, func, instance);
+
+                func
+            };
+
+            self.intrinsic_instances.borrow_mut().insert(instance, func);
+
+            func
+        };
+        let fn_ptr = func.get_address(None);
         let fn_ty = fn_ptr.get_type();
 
         let mut llargs = vec![];

compiler/rustc_codegen_llvm/src/context.rs

@@ -100,6 +100,8 @@ pub(crate) struct FullCx<'ll, 'tcx> {
 
     /// Cache instances of monomorphic and polymorphic items
     pub instances: RefCell<FxHashMap<Instance<'tcx>, &'ll Value>>,
+    /// Cache instances of intrinsics
+    pub intrinsic_instances: RefCell<FxHashMap<Instance<'tcx>, &'ll Value>>,
     /// Cache generated vtables
     pub vtables: RefCell<FxHashMap<(Ty<'tcx>, Option<ty::ExistentialTraitRef<'tcx>>), &'ll Value>>,
     /// Cache of constant strings,
@@ -620,6 +622,7 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> {
                 tls_model,
                 codegen_unit,
                 instances: Default::default(),
+                intrinsic_instances: Default::default(),
                 vtables: Default::default(),
                 const_str_cache: Default::default(),
                 const_globals: Default::default(),

compiler/rustc_codegen_llvm/src/intrinsic.rs

@@ -28,6 +28,7 @@ use crate::abi::FnAbiLlvmExt;
 use crate::builder::Builder;
 use crate::builder::autodiff::{adjust_activity_to_abi, generate_enzyme_call};
 use crate::context::CodegenCx;
+use crate::declare::declare_raw_fn;
 use crate::errors::AutoDiffWithoutEnable;
 use crate::llvm::{self, Metadata, Type, Value};
 use crate::type_of::LayoutLlvmExt;
@@ -627,11 +628,41 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
         args: &[OperandRef<'tcx, Self::Value>],
         is_cleanup: bool,
     ) -> Self::Value {
-        let fn_ptr = self.get_fn_addr(instance);
+        let tcx = self.tcx();
+
         // FIXME remove usage of fn_abi
         let fn_abi = self.fn_abi_of_instance(instance, ty::List::empty());
         assert!(!fn_abi.ret.is_indirect());
-        let fn_ty = self.fn_decl_backend_type(fn_abi);
+        let fn_ty = fn_abi.llvm_type(self);
+
+        let fn_ptr = if let Some(&llfn) = self.intrinsic_instances.borrow().get(&instance) {
+            llfn
+        } else {
+            let sym = tcx.symbol_name(instance).name;
+
+            // FIXME use get_intrinsic
+            let llfn = if let Some(llfn) = self.get_declared_value(sym) {
+                llfn
+            } else {
+                // Function addresses in Rust are never significant, allowing functions to
+                // be merged.
+                let llfn = declare_raw_fn(
+                    self,
+                    sym,
+                    fn_abi.llvm_cconv(self),
+                    llvm::UnnamedAddr::Global,
+                    llvm::Visibility::Default,
+                    fn_ty,
+                );
+                fn_abi.apply_attrs_llfn(self, llfn, Some(instance));
+
+                llfn
+            };
+
+            self.intrinsic_instances.borrow_mut().insert(instance, llfn);
+
+            llfn
+        };
 
         let mut llargs = vec![];
 

compiler/rustc_codegen_ssa/src/mir/block.rs

@@ -9,7 +9,7 @@ use rustc_lint_defs::builtin::TAIL_CALL_TRACK_CALLER;
 use rustc_middle::mir::{self, AssertKind, InlineAsmMacro, SwitchTargets, UnwindTerminateReason};
 use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf, ValidityRequirement};
 use rustc_middle::ty::print::{with_no_trimmed_paths, with_no_visible_paths};
-use rustc_middle::ty::{self, Instance, Ty};
+use rustc_middle::ty::{self, Instance, Ty, TypeVisitableExt};
 use rustc_middle::{bug, span_bug};
 use rustc_session::config::OptLevel;
 use rustc_span::Span;
@@ -1038,6 +1038,9 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
             && let Some(name) = bx.tcx().codegen_fn_attrs(instance.def_id()).symbol_name
             && name.as_str().starts_with("llvm.")
         {
+            assert!(!instance.args.has_infer());
+            assert!(!instance.args.has_escaping_bound_vars());
+
             let dest_ty = destination.ty(&self.mir.local_decls, bx.tcx()).ty;
             let return_dest = if dest_ty.is_unit() {
                 ReturnDest::Nothing