Index: llvm/include/llvm/Transforms/Utils/LowerIFunc.h =================================================================== --- /dev/null +++ llvm/include/llvm/Transforms/Utils/LowerIFunc.h @@ -0,0 +1,28 @@ +//===-- LowerIFunc.h --------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_TRANSFORMS_UTILS_LOWERIFUNC_H +#define LLVM_TRANSFORMS_UTILS_LOWERIFUNC_H + +#include "llvm/IR/PassManager.h" + +namespace llvm { + +/// Pass to replace calls to ifuncs with indirect calls. This could be used to +/// support ifunc on systems where the program loader does not natively support +/// it. Constant initializer uses of ifuncs are not handled. +class LowerIFuncPass : public PassInfoMixin { +public: + LowerIFuncPass() = default; + + PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM); +}; + +} // end namespace llvm + +#endif // LLVM_TRANSFORMS_UTILS_LOWERIFUNC_H Index: llvm/include/llvm/Transforms/Utils/ModuleUtils.h =================================================================== --- llvm/include/llvm/Transforms/Utils/ModuleUtils.h +++ llvm/include/llvm/Transforms/Utils/ModuleUtils.h @@ -15,6 +15,7 @@ #include "llvm/ADT/STLFunctionalExtras.h" #include "llvm/ADT/StringRef.h" +#include "llvm/IR/GlobalIFunc.h" #include "llvm/Support/Alignment.h" #include "llvm/Support/MemoryBufferRef.h" #include // for std::pair @@ -121,6 +122,21 @@ void embedBufferInModule(Module &M, MemoryBufferRef Buf, StringRef SectionName, Align Alignment = Align(1)); +/// Lower all calls to ifuncs by replacing uses with indirect calls loaded out +/// of a global table initialized in a global constructor. This will introduce +/// one constructor function and adds it to llvm.global_ctors. The constructor +/// will call the resolver function once for each ifunc. +/// +/// Leaves any unhandled constant initializer uses as-is. +/// +/// If \p IFuncsToLower is empty, all ifuncs in the module will be lowered. +/// If \p IFuncsToLower is non-empty, only the selected ifuncs will be lowered. +/// +/// The processed ifuncs without remaining users will be removed from the +/// module. +bool lowerGlobalIFuncUsersAsGlobalCtor( + Module &M, ArrayRef IFuncsToLower = {}); + class CallInst; namespace VFABI { /// Overwrite the Vector Function ABI variants attribute with the names provide Index: llvm/lib/Passes/PassBuilder.cpp =================================================================== --- llvm/lib/Passes/PassBuilder.cpp +++ llvm/lib/Passes/PassBuilder.cpp @@ -236,6 +236,7 @@ #include "llvm/Transforms/Utils/LoopSimplify.h" #include "llvm/Transforms/Utils/LoopVersioning.h" #include "llvm/Transforms/Utils/LowerGlobalDtors.h" +#include "llvm/Transforms/Utils/LowerIFunc.h" #include "llvm/Transforms/Utils/LowerInvoke.h" #include "llvm/Transforms/Utils/LowerSwitch.h" #include "llvm/Transforms/Utils/Mem2Reg.h" Index: llvm/lib/Passes/PassRegistry.def =================================================================== --- llvm/lib/Passes/PassRegistry.def +++ llvm/lib/Passes/PassRegistry.def @@ -79,6 +79,7 @@ MODULE_PASS("iroutliner", IROutlinerPass()) MODULE_PASS("print-ir-similarity", IRSimilarityAnalysisPrinterPass(dbgs())) MODULE_PASS("lower-global-dtors", LowerGlobalDtorsPass()) +MODULE_PASS("lower-ifunc", LowerIFuncPass()) MODULE_PASS("lowertypetests", LowerTypeTestsPass()) MODULE_PASS("metarenamer", MetaRenamerPass()) MODULE_PASS("mergefunc", MergeFunctionsPass()) Index: llvm/lib/Transforms/Utils/CMakeLists.txt =================================================================== --- llvm/lib/Transforms/Utils/CMakeLists.txt +++ llvm/lib/Transforms/Utils/CMakeLists.txt @@ -46,6 +46,7 @@ LoopVersioning.cpp LowerAtomic.cpp LowerGlobalDtors.cpp + LowerIFunc.cpp LowerInvoke.cpp LowerMemIntrinsics.cpp LowerSwitch.cpp Index: llvm/lib/Transforms/Utils/LowerIFunc.cpp =================================================================== --- /dev/null +++ llvm/lib/Transforms/Utils/LowerIFunc.cpp @@ -0,0 +1,27 @@ +//===- LowerIFunc.cpp -----------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements replacing calls to ifuncs by introducing indirect calls. +// +//===----------------------------------------------------------------------===// + +#include "llvm/Transforms/Utils/LowerIFunc.h" +#include "llvm/IR/Module.h" +#include "llvm/Pass.h" +#include "llvm/Transforms/Utils/ModuleUtils.h" + +using namespace llvm; + +/// Replace all call users of ifuncs in the module. +PreservedAnalyses LowerIFuncPass::run(Module &M, ModuleAnalysisManager &AM) { + if (M.ifunc_empty()) + return PreservedAnalyses::all(); + + lowerGlobalIFuncUsersAsGlobalCtor(M, {}); + return PreservedAnalyses::none(); +} Index: llvm/lib/Transforms/Utils/ModuleUtils.cpp =================================================================== --- llvm/lib/Transforms/Utils/ModuleUtils.cpp +++ llvm/lib/Transforms/Utils/ModuleUtils.cpp @@ -340,3 +340,102 @@ appendToCompilerUsed(M, GV); } + +bool llvm::lowerGlobalIFuncUsersAsGlobalCtor( + Module &M, ArrayRef FilteredIFuncsToLower) { + SmallVector AllIFuncs; + ArrayRef IFuncsToLower = FilteredIFuncsToLower; + if (FilteredIFuncsToLower.empty()) { // Default to lowering all ifuncs + for (GlobalIFunc &GI : M.ifuncs()) + AllIFuncs.push_back(&GI); + IFuncsToLower = AllIFuncs; + } + + bool UnhandledUsers = false; + LLVMContext &Ctx = M.getContext(); + const DataLayout &DL = M.getDataLayout(); + + PointerType *TableEntryTy = + Ctx.supportsTypedPointers() + ? PointerType::get(Type::getInt8Ty(Ctx), DL.getProgramAddressSpace()) + : PointerType::get(Ctx, DL.getProgramAddressSpace()); + + ArrayType *FuncPtrTableTy = + ArrayType::get(TableEntryTy, IFuncsToLower.size()); + + Align PtrAlign = DL.getABITypeAlign(TableEntryTy); + + // Create a global table of function pointers we'll initialize in a global + // constructor. + auto *FuncPtrTable = new GlobalVariable( + M, FuncPtrTableTy, false, GlobalValue::InternalLinkage, + PoisonValue::get(FuncPtrTableTy), "", nullptr, + GlobalVariable::NotThreadLocal, DL.getDefaultGlobalsAddressSpace()); + FuncPtrTable->setAlignment(PtrAlign); + + // Create a function to initialize the function pointer table. + Function *NewCtor = Function::Create( + FunctionType::get(Type::getVoidTy(Ctx), false), Function::InternalLinkage, + DL.getProgramAddressSpace(), "", &M); + + BasicBlock *BB = BasicBlock::Create(Ctx, "", NewCtor); + IRBuilder<> InitBuilder(BB); + + size_t TableIndex = 0; + for (GlobalIFunc *GI : IFuncsToLower) { + Function *ResolvedFunction = GI->getResolverFunction(); + + // We don't know what to pass to a resolver function taking arguments + // + // FIXME: Is this even valid? clang and gcc don't complain but this + // probably should be invalid IR. We could just pass through undef. + if (!std::empty(ResolvedFunction->getFunctionType()->params())) { + LLVM_DEBUG(dbgs() << "Not lowering ifunc resolver function " + << ResolvedFunction->getName() << " with parameters\n"); + UnhandledUsers = true; + continue; + } + + // Initialize the function pointer table. + CallInst *ResolvedFunc = InitBuilder.CreateCall(ResolvedFunction); + Value *Casted = InitBuilder.CreatePointerCast(ResolvedFunc, TableEntryTy); + Constant *GEP = cast(InitBuilder.CreateConstInBoundsGEP2_32( + FuncPtrTableTy, FuncPtrTable, 0, TableIndex++)); + InitBuilder.CreateAlignedStore(Casted, GEP, PtrAlign); + + // Update all users to load a pointer from the global table. + for (User *User : make_early_inc_range(GI->users())) { + Instruction *UserInst = dyn_cast(User); + if (!UserInst) { + // TODO: Should handle constantexpr casts in user instructions. Probably + // can't do much about constant initializers. + UnhandledUsers = true; + continue; + } + + IRBuilder<> UseBuilder(UserInst); + LoadInst *ResolvedTarget = + UseBuilder.CreateAlignedLoad(TableEntryTy, GEP, PtrAlign); + Value *ResolvedCast = + UseBuilder.CreatePointerCast(ResolvedTarget, GI->getType()); + UserInst->replaceUsesOfWith(GI, ResolvedCast); + } + + // If we handled all users, erase the ifunc. + if (GI->use_empty()) + GI->eraseFromParent(); + } + + InitBuilder.CreateRetVoid(); + + PointerType *ConstantDataTy = Ctx.supportsTypedPointers() + ? PointerType::get(Type::getInt8Ty(Ctx), 0) + : PointerType::get(Ctx, 0); + + // TODO: Is this the right priority? Probably should be before any other + // constructors? + const int Priority = 10; + appendToGlobalCtors(M, NewCtor, Priority, + ConstantPointerNull::get(ConstantDataTy)); + return UnhandledUsers; +} Index: llvm/test/Transforms/LowerIFunc/ifunc-alias.ll =================================================================== --- /dev/null +++ llvm/test/Transforms/LowerIFunc/ifunc-alias.ll @@ -0,0 +1,57 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals --include-generated-funcs +; RUN: opt -S -passes=lower-ifunc < %s | FileCheck %s + +define ptr @resolver() { + ret ptr inttoptr (i64 333 to ptr) +} + +@resolver_alias = alias ptr (), ptr @resolver +@ifunc_alias = alias ptr (), ptr @resolver_alias + +@ifunc0_kept = ifunc float (i64), ptr @resolver_alias +@ifunc1_removed = ifunc float (i64), ptr @resolver_alias + +@ifunc_def = ifunc float (i64), ptr @resolver +@alias_of_ifunc = alias float (i64), ptr @ifunc_def + +define float @call_ifunc_aliasee(i64 %arg) { + %call = call float @ifunc1_removed(i64 %arg) + ret float %call +} + +define float @call_alias_of_ifunc(i64 %arg) { + %call = call float @alias_of_ifunc(i64 %arg) + ret float %call +} +;. +; CHECK: @[[GLOB0:[0-9]+]] = internal global [3 x ptr] poison, align 8 +; CHECK: @[[LLVM_GLOBAL_CTORS:[a-zA-Z0-9_$"\\.-]+]] = appending global [1 x { i32, ptr, ptr }] [{ i32, ptr, ptr } { i32 10, ptr @[[GLOB1:[0-9]+]], ptr null }] +; CHECK: @[[RESOLVER_ALIAS:[a-zA-Z0-9_$"\\.-]+]] = alias ptr (), ptr @resolver +; CHECK: @[[IFUNC_ALIAS:[a-zA-Z0-9_$"\\.-]+]] = alias ptr (), ptr @resolver_alias +; CHECK: @[[ALIAS_OF_IFUNC:[a-zA-Z0-9_$"\\.-]+]] = alias float (i64), ptr @ifunc_def +; CHECK: @[[IFUNC_DEF:[a-zA-Z0-9_$"\\.-]+]] = ifunc float (i64), ptr @resolver +;. +; CHECK-LABEL: define {{[^@]+}}@resolver( +; CHECK-NEXT: ret ptr inttoptr (i64 333 to ptr) +; +; +; CHECK-LABEL: define {{[^@]+}}@call_ifunc_aliasee( +; CHECK-NEXT: [[TMP1:%.*]] = load ptr, ptr getelementptr inbounds ([3 x ptr], ptr @[[GLOB0]], i32 0, i32 1), align 8 +; CHECK-NEXT: [[CALL:%.*]] = call float [[TMP1]](i64 [[ARG:%.*]]) +; CHECK-NEXT: ret float [[CALL]] +; +; +; CHECK-LABEL: define {{[^@]+}}@call_alias_of_ifunc( +; CHECK-NEXT: [[CALL:%.*]] = call float @alias_of_ifunc(i64 [[ARG:%.*]]) +; CHECK-NEXT: ret float [[CALL]] +; +; +; CHECK-LABEL: define {{[^@]+}}@1( +; CHECK-NEXT: [[TMP1:%.*]] = call ptr @resolver() +; CHECK-NEXT: store ptr [[TMP1]], ptr @[[GLOB0]], align 8 +; CHECK-NEXT: [[TMP2:%.*]] = call ptr @resolver() +; CHECK-NEXT: store ptr [[TMP2]], ptr getelementptr inbounds ([3 x ptr], ptr @[[GLOB0]], i32 0, i32 1), align 8 +; CHECK-NEXT: [[TMP3:%.*]] = call ptr @resolver() +; CHECK-NEXT: store ptr [[TMP3]], ptr getelementptr inbounds ([3 x ptr], ptr @[[GLOB0]], i32 0, i32 2), align 8 +; CHECK-NEXT: ret void +; Index: llvm/test/Transforms/LowerIFunc/ifunc-nonsense-resolvers.ll =================================================================== --- /dev/null +++ llvm/test/Transforms/LowerIFunc/ifunc-nonsense-resolvers.ll @@ -0,0 +1,34 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals --include-generated-funcs +; RUN: opt -S -passes=lower-ifunc < %s | FileCheck %s + +@ifunc_with_arg = ifunc void (), ptr @resolver_with_arg + +; Test a resolver with an argument (which probably should not be legal +; IR). +define ptr @resolver_with_arg(i64 %arg) { + %cast = inttoptr i64 %arg to ptr + ret ptr %cast +} + +define void @call_with_arg() { + call void @ifunc_with_arg() + ret void +} +;. +; CHECK: @[[GLOB0:[0-9]+]] = internal global [1 x ptr] poison, align 8 +; CHECK: @[[LLVM_GLOBAL_CTORS:[a-zA-Z0-9_$"\\.-]+]] = appending global [1 x { i32, ptr, ptr }] [{ i32, ptr, ptr } { i32 10, ptr @[[GLOB1:[0-9]+]], ptr null }] +; CHECK: @[[IFUNC_WITH_ARG:[a-zA-Z0-9_$"\\.-]+]] = ifunc void (), ptr @resolver_with_arg +;. +; CHECK-LABEL: define {{[^@]+}}@resolver_with_arg( +; CHECK-NEXT: [[CAST:%.*]] = inttoptr i64 [[ARG:%.*]] to ptr +; CHECK-NEXT: ret ptr [[CAST]] +; +; +; CHECK-LABEL: define {{[^@]+}}@call_with_arg( +; CHECK-NEXT: call void @ifunc_with_arg() +; CHECK-NEXT: ret void +; +; +; CHECK-LABEL: define {{[^@]+}}@1( +; CHECK-NEXT: ret void +; Index: llvm/test/Transforms/LowerIFunc/ifunc-program-addrspace.ll =================================================================== --- /dev/null +++ llvm/test/Transforms/LowerIFunc/ifunc-program-addrspace.ll @@ -0,0 +1,101 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals --include-generated-funcs +; RUN: opt -S -passes=lower-ifunc < %s | FileCheck %s + +target datalayout = "P1" + +; Currently don't support expanding these uses. +@ifunc_used_in_constantinit_as1 = ifunc i32 (double), ptr addrspace(1) @resolver1_in_1 +@constant_init_user_addrspace1 = global ptr addrspace(1) @ifunc_used_in_constantinit_as1 + +@ifunc_used_in_constantinit_as1_cast = ifunc i32 (double), ptr @resolver1_in_0 +@constant_init_user_addrspace1_cast = global ptr addrspace(1) addrspacecast (ptr @ifunc_used_in_constantinit_as1_cast to ptr addrspace(1)) + +@ifunc_used_in_constantinit_as0_cast = ifunc i32 (double), ptr addrspace(1) @resolver0_in_1 +@constant_init_user_addrspace0_cast = global ptr addrspacecast (ptr addrspace(1) @ifunc_used_in_constantinit_as0_cast to ptr) + + +@ifunc_as1_resolver_in_0 = ifunc void (), ptr @resolver1_in_0 +@ifunc_as1_resolver_in_1 = ifunc void (), ptr addrspace(1) @resolver1_in_1 +@ifunc_as1_resolver_casted_in_1 = ifunc void (), ptr addrspace(1) addrspacecast (ptr @resolver1_in_0 to ptr addrspace(1)) + +define ptr addrspace(1) @resolver1_in_0() addrspace(0) { + ret ptr addrspace(1) inttoptr (i64 123 to ptr addrspace(1)) +} + +define ptr addrspace(1) @resolver1_in_1() addrspace(1) { + ret ptr addrspace(1) inttoptr (i64 456 to ptr addrspace(1)) +} + +define ptr addrspace(0) @resolver0_in_1() addrspace(1) { + ret ptr addrspace(0) inttoptr (i64 789 to ptr addrspace(0)) +} + +define void @call_ifuncs() addrspace(0) { + call addrspace(0) void @ifunc_as1_resolver_in_0() + call addrspace(1) void @ifunc_as1_resolver_in_1() + call addrspace(1) void @ifunc_as1_resolver_casted_in_1() + ret void +} + +define void @load_ifuncs() addrspace(0) { + %load0 = load volatile ptr addrspace(1), ptr @constant_init_user_addrspace1 + %load1 = load volatile ptr addrspace(1), ptr @constant_init_user_addrspace1_cast + %load2 = load volatile ptr, ptr @constant_init_user_addrspace0_cast + ret void +} +;. +; CHECK: @[[CONSTANT_INIT_USER_ADDRSPACE1:[a-zA-Z0-9_$"\\.-]+]] = global ptr addrspace(1) @ifunc_used_in_constantinit_as1 +; CHECK: @[[CONSTANT_INIT_USER_ADDRSPACE1_CAST:[a-zA-Z0-9_$"\\.-]+]] = global ptr addrspace(1) addrspacecast (ptr @ifunc_used_in_constantinit_as1_cast to ptr addrspace(1)) +; CHECK: @[[CONSTANT_INIT_USER_ADDRSPACE0_CAST:[a-zA-Z0-9_$"\\.-]+]] = global ptr addrspacecast (ptr addrspace(1) @ifunc_used_in_constantinit_as0_cast to ptr) +; CHECK: @[[GLOB0:[0-9]+]] = internal global [6 x ptr addrspace(1)] poison, align 8 +; CHECK: @[[LLVM_GLOBAL_CTORS:[a-zA-Z0-9_$"\\.-]+]] = appending global [1 x { i32, ptr addrspace(1), ptr }] [{ i32, ptr addrspace(1), ptr } { i32 10, ptr addrspace(1) @[[GLOB1:[0-9]+]], ptr null }] +; CHECK: @[[IFUNC_USED_IN_CONSTANTINIT_AS1:[a-zA-Z0-9_$"\\.-]+]] = ifunc i32 (double), ptr addrspace(1) @resolver1_in_1 +; CHECK: @[[IFUNC_USED_IN_CONSTANTINIT_AS1_CAST:[a-zA-Z0-9_$"\\.-]+]] = ifunc i32 (double), ptr @resolver1_in_0 +; CHECK: @[[IFUNC_USED_IN_CONSTANTINIT_AS0_CAST:[a-zA-Z0-9_$"\\.-]+]] = ifunc i32 (double), ptr addrspace(1) @resolver0_in_1 +;. +; CHECK-LABEL: define {{[^@]+}}@resolver1_in_0( +; CHECK-NEXT: ret ptr addrspace(1) inttoptr (i64 123 to ptr addrspace(1)) +; +; +; CHECK-LABEL: define {{[^@]+}}@resolver1_in_1( +; CHECK-NEXT: ret ptr addrspace(1) inttoptr (i64 456 to ptr addrspace(1)) +; +; +; CHECK-LABEL: define {{[^@]+}}@resolver0_in_1( +; CHECK-NEXT: ret ptr inttoptr (i64 789 to ptr) +; +; +; CHECK-LABEL: define {{[^@]+}}@call_ifuncs( +; CHECK-NEXT: [[TMP1:%.*]] = load ptr addrspace(1), ptr getelementptr inbounds ([6 x ptr addrspace(1)], ptr @[[GLOB0]], i32 0, i32 3), align 8 +; CHECK-NEXT: [[TMP2:%.*]] = addrspacecast ptr addrspace(1) [[TMP1]] to ptr +; CHECK-NEXT: call addrspace(0) void [[TMP2]]() +; CHECK-NEXT: [[TMP3:%.*]] = load ptr addrspace(1), ptr getelementptr inbounds ([6 x ptr addrspace(1)], ptr @[[GLOB0]], i32 0, i32 4), align 8 +; CHECK-NEXT: call addrspace(1) void [[TMP3]]() +; CHECK-NEXT: [[TMP4:%.*]] = load ptr addrspace(1), ptr getelementptr inbounds ([6 x ptr addrspace(1)], ptr @[[GLOB0]], i32 0, i32 5), align 8 +; CHECK-NEXT: call addrspace(1) void [[TMP4]]() +; CHECK-NEXT: ret void +; +; +; CHECK-LABEL: define {{[^@]+}}@load_ifuncs( +; CHECK-NEXT: [[LOAD0:%.*]] = load volatile ptr addrspace(1), ptr @constant_init_user_addrspace1, align 8 +; CHECK-NEXT: [[LOAD1:%.*]] = load volatile ptr addrspace(1), ptr @constant_init_user_addrspace1_cast, align 8 +; CHECK-NEXT: [[LOAD2:%.*]] = load volatile ptr, ptr @constant_init_user_addrspace0_cast, align 8 +; CHECK-NEXT: ret void +; +; +; CHECK-LABEL: define {{[^@]+}}@1( +; CHECK-NEXT: [[TMP1:%.*]] = call addrspace(1) ptr addrspace(1) @resolver1_in_1() +; CHECK-NEXT: store ptr addrspace(1) [[TMP1]], ptr @[[GLOB0]], align 8 +; CHECK-NEXT: [[TMP2:%.*]] = call addrspace(0) ptr addrspace(1) @resolver1_in_0() +; CHECK-NEXT: store ptr addrspace(1) [[TMP2]], ptr getelementptr inbounds ([6 x ptr addrspace(1)], ptr @[[GLOB0]], i32 0, i32 1), align 8 +; CHECK-NEXT: [[TMP3:%.*]] = call addrspace(1) ptr @resolver0_in_1() +; CHECK-NEXT: [[TMP4:%.*]] = addrspacecast ptr [[TMP3]] to ptr addrspace(1) +; CHECK-NEXT: store ptr addrspace(1) [[TMP4]], ptr getelementptr inbounds ([6 x ptr addrspace(1)], ptr @[[GLOB0]], i32 0, i32 2), align 8 +; CHECK-NEXT: [[TMP5:%.*]] = call addrspace(0) ptr addrspace(1) @resolver1_in_0() +; CHECK-NEXT: store ptr addrspace(1) [[TMP5]], ptr getelementptr inbounds ([6 x ptr addrspace(1)], ptr @[[GLOB0]], i32 0, i32 3), align 8 +; CHECK-NEXT: [[TMP6:%.*]] = call addrspace(1) ptr addrspace(1) @resolver1_in_1() +; CHECK-NEXT: store ptr addrspace(1) [[TMP6]], ptr getelementptr inbounds ([6 x ptr addrspace(1)], ptr @[[GLOB0]], i32 0, i32 4), align 8 +; CHECK-NEXT: [[TMP7:%.*]] = call addrspace(0) ptr addrspace(1) @resolver1_in_0() +; CHECK-NEXT: store ptr addrspace(1) [[TMP7]], ptr getelementptr inbounds ([6 x ptr addrspace(1)], ptr @[[GLOB0]], i32 0, i32 5), align 8 +; CHECK-NEXT: ret void +; Index: llvm/test/Transforms/LowerIFunc/ifunc-typed.ll =================================================================== --- /dev/null +++ llvm/test/Transforms/LowerIFunc/ifunc-typed.ll @@ -0,0 +1,54 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals --include-generated-funcs +; RUN: opt -S -passes=lower-ifunc < %s | FileCheck %s + +; Check that typed pointers still work + +@ifunc0 = ifunc i32(i32), i32(i32)* ()* @func0 + +define i32 (i32)* @func0() { + ret i32 (i32)* null +} + +define i32 (i64)* @func1() { + ret i32 (i64)* null +} + +@ifunc1 = ifunc float(double), float(double)* ()* bitcast (i32(i64)* ()* @func1 to float(double)* ()*) + +define i32 @call_ifuncs() { + %call0 = call i32 @ifunc0(i32 123) + %call1 = call i32 bitcast (float(double)* @ifunc1 to i32()*)() + %add = add i32 %call0, %call1 + ret i32 %add +} +;. +; CHECK: @[[GLOB0:[0-9]+]] = internal global [2 x i8*] poison, align 8 +; CHECK: @[[LLVM_GLOBAL_CTORS:[a-zA-Z0-9_$"\\.-]+]] = appending global [1 x { i32, void ()*, i8* }] [{ i32, void ()*, i8* } { i32 10, void ()* @[[GLOB1:[0-9]+]], i8* null }] +; CHECK: @[[IFUNC1:[a-zA-Z0-9_$"\\.-]+]] = ifunc float (double), bitcast (i32 (i64)* ()* @func1 to float (double)* ()*) +;. +; CHECK-LABEL: define {{[^@]+}}@func0( +; CHECK-NEXT: ret i32 (i32)* null +; +; +; CHECK-LABEL: define {{[^@]+}}@func1( +; CHECK-NEXT: ret i32 (i64)* null +; +; +; CHECK-LABEL: define {{[^@]+}}@call_ifuncs( +; CHECK-NEXT: [[TMP1:%.*]] = load i8*, i8** getelementptr inbounds ([2 x i8*], [2 x i8*]* @[[GLOB0]], i32 0, i32 0), align 8 +; CHECK-NEXT: [[TMP2:%.*]] = bitcast i8* [[TMP1]] to i32 (i32)* +; CHECK-NEXT: [[CALL0:%.*]] = call i32 [[TMP2]](i32 123) +; CHECK-NEXT: [[CALL1:%.*]] = call i32 bitcast (float (double)* @ifunc1 to i32 ()*)() +; CHECK-NEXT: [[ADD:%.*]] = add i32 [[CALL0]], [[CALL1]] +; CHECK-NEXT: ret i32 [[ADD]] +; +; +; CHECK-LABEL: define {{[^@]+}}@1( +; CHECK-NEXT: [[TMP1:%.*]] = call i32 (i32)* @func0() +; CHECK-NEXT: [[TMP2:%.*]] = bitcast i32 (i32)* [[TMP1]] to i8* +; CHECK-NEXT: store i8* [[TMP2]], i8** getelementptr inbounds ([2 x i8*], [2 x i8*]* @[[GLOB0]], i32 0, i32 0), align 8 +; CHECK-NEXT: [[TMP3:%.*]] = call i32 (i64)* @func1() +; CHECK-NEXT: [[TMP4:%.*]] = bitcast i32 (i64)* [[TMP3]] to i8* +; CHECK-NEXT: store i8* [[TMP4]], i8** getelementptr inbounds ([2 x i8*], [2 x i8*]* @[[GLOB0]], i32 0, i32 1), align 8 +; CHECK-NEXT: ret void +; Index: llvm/test/Transforms/LowerIFunc/lower-ifunc.ll =================================================================== --- /dev/null +++ llvm/test/Transforms/LowerIFunc/lower-ifunc.ll @@ -0,0 +1,191 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals --include-generated-funcs +; RUN: opt -S -passes=lower-ifunc < %s | FileCheck %s + +@initialized_with_ifunc = global ptr @ifunc_constant_initializer_user + +@ifunc_1 = ifunc void (), ptr @resolver1 +@ifunc_2 = ifunc void (), ptr @resolver1 + + +; Keep one with no users +@ifunc_no_users = ifunc float (i64), ptr @resolver2 + + +@ifunc7 = ifunc float (i64), ptr @resolver3 +@ifunc_ptr_arg = ifunc void (ptr), ptr @resolver4 + +@ifunc_nonvoid_0 = ifunc i32 (double), ptr @resolver5 +@ifunc_nonvoid_1 = ifunc i32 (double), ptr @resolver5 +@ifunc_constant_initializer_user = ifunc i32 (double), ptr @resolver5 + +define ptr @resolver1() { + ret ptr inttoptr (i64 123 to ptr) +} + +define ptr @resolver2() { + ret ptr inttoptr (i64 456 to ptr) +} + +define ptr @resolver3() { + ret ptr inttoptr (i64 789 to ptr) +} + +define ptr @resolver4() { + ret ptr inttoptr (i64 999 to ptr) +} + +define ptr @resolver5() { + ret ptr inttoptr (i64 420 to ptr) +} + + +; Test call to ifunc +define void @call_ifunc(ptr %ptr) { + call void @ifunc_1() + call void @ifunc_2() + ret void +} + +; Test value use of ifunc +define void @store_ifunc_2(ptr %ptr) { + store ptr @ifunc_2, ptr %ptr + store ptr %ptr, ptr @ifunc_2 + ret void +} + +declare void @other_func(ptr) + +; Check a call user, but not as the call operand +define void @call_ifunc_is_argument(ptr %ptr) { + call void @other_func(ptr @ifunc_2) + ret void +} + +; Check a call user calling the ifunc, and using the ifunc as an argument +define void @call_ifunc_both_call_argument(ptr %ptr) { + call void @ifunc_ptr_arg(ptr @ifunc_ptr_arg) + ret void +} + +define i32 @call_ifunc_nonvoid(double %arg) { + %ret = call i32 @ifunc_nonvoid_0(double %arg) + ret i32 %ret +} + +; Use site is different than ifunc function type +define float @call_different_type_ifunc_nonvoid(double %arg) { + %cast.arg = bitcast double %arg to i64 + %ret = call float(i64) @ifunc_nonvoid_0(i64 %cast.arg) + ret float %ret +} + +; FIXME: Should be able to expand this, but we miss the call +; instruction in the constexpr cast. +define i32 @call_addrspacecast_callee_type_ifunc_nonvoid(double %arg) { + %ret = call addrspace(1) i32 addrspacecast (ptr @ifunc_nonvoid_1 to ptr addrspace(1)) (double %arg) + ret i32 %ret +} + +define i32 @call_used_in_initializer(double %arg) { + %ret = call i32 @ifunc_constant_initializer_user(double %arg) + ret i32 %ret +} +;. +; CHECK: @[[INITIALIZED_WITH_IFUNC:[a-zA-Z0-9_$"\\.-]+]] = global ptr @ifunc_constant_initializer_user +; CHECK: @[[GLOB0:[0-9]+]] = internal global [8 x ptr] poison, align 8 +; CHECK: @[[LLVM_GLOBAL_CTORS:[a-zA-Z0-9_$"\\.-]+]] = appending global [1 x { i32, ptr, ptr }] [{ i32, ptr, ptr } { i32 10, ptr @[[GLOB1:[0-9]+]], ptr null }] +; CHECK: @[[IFUNC_NONVOID_1:[a-zA-Z0-9_$"\\.-]+]] = ifunc i32 (double), ptr @resolver5 +; CHECK: @[[IFUNC_CONSTANT_INITIALIZER_USER:[a-zA-Z0-9_$"\\.-]+]] = ifunc i32 (double), ptr @resolver5 +;. +; CHECK-LABEL: define {{[^@]+}}@resolver1( +; CHECK-NEXT: ret ptr inttoptr (i64 123 to ptr) +; +; +; CHECK-LABEL: define {{[^@]+}}@resolver2( +; CHECK-NEXT: ret ptr inttoptr (i64 456 to ptr) +; +; +; CHECK-LABEL: define {{[^@]+}}@resolver3( +; CHECK-NEXT: ret ptr inttoptr (i64 789 to ptr) +; +; +; CHECK-LABEL: define {{[^@]+}}@resolver4( +; CHECK-NEXT: ret ptr inttoptr (i64 999 to ptr) +; +; +; CHECK-LABEL: define {{[^@]+}}@resolver5( +; CHECK-NEXT: ret ptr inttoptr (i64 420 to ptr) +; +; +; CHECK-LABEL: define {{[^@]+}}@call_ifunc( +; CHECK-NEXT: [[TMP1:%.*]] = load ptr, ptr @[[GLOB0]], align 8 +; CHECK-NEXT: call void [[TMP1]]() +; CHECK-NEXT: [[TMP2:%.*]] = load ptr, ptr getelementptr inbounds ([8 x ptr], ptr @[[GLOB0]], i32 0, i32 1), align 8 +; CHECK-NEXT: call void [[TMP2]]() +; CHECK-NEXT: ret void +; +; +; CHECK-LABEL: define {{[^@]+}}@store_ifunc_2( +; CHECK-NEXT: [[TMP1:%.*]] = load ptr, ptr getelementptr inbounds ([8 x ptr], ptr @[[GLOB0]], i32 0, i32 1), align 8 +; CHECK-NEXT: store ptr [[TMP1]], ptr [[PTR:%.*]], align 8 +; CHECK-NEXT: [[TMP2:%.*]] = load ptr, ptr getelementptr inbounds ([8 x ptr], ptr @[[GLOB0]], i32 0, i32 1), align 8 +; CHECK-NEXT: store ptr [[PTR]], ptr [[TMP2]], align 8 +; CHECK-NEXT: ret void +; +; +; CHECK-LABEL: define {{[^@]+}}@call_ifunc_is_argument( +; CHECK-NEXT: [[TMP1:%.*]] = load ptr, ptr getelementptr inbounds ([8 x ptr], ptr @[[GLOB0]], i32 0, i32 1), align 8 +; CHECK-NEXT: call void @other_func(ptr [[TMP1]]) +; CHECK-NEXT: ret void +; +; +; CHECK-LABEL: define {{[^@]+}}@call_ifunc_both_call_argument( +; CHECK-NEXT: [[TMP1:%.*]] = load ptr, ptr getelementptr inbounds ([8 x ptr], ptr @[[GLOB0]], i32 0, i32 4), align 8 +; CHECK-NEXT: [[TMP2:%.*]] = load ptr, ptr getelementptr inbounds ([8 x ptr], ptr @[[GLOB0]], i32 0, i32 4), align 8 +; CHECK-NEXT: call void [[TMP1]](ptr [[TMP1]]) +; CHECK-NEXT: ret void +; +; +; CHECK-LABEL: define {{[^@]+}}@call_ifunc_nonvoid( +; CHECK-NEXT: [[TMP1:%.*]] = load ptr, ptr getelementptr inbounds ([8 x ptr], ptr @[[GLOB0]], i32 0, i32 5), align 8 +; CHECK-NEXT: [[RET:%.*]] = call i32 [[TMP1]](double [[ARG:%.*]]) +; CHECK-NEXT: ret i32 [[RET]] +; +; +; CHECK-LABEL: define {{[^@]+}}@call_different_type_ifunc_nonvoid( +; CHECK-NEXT: [[CAST_ARG:%.*]] = bitcast double [[ARG:%.*]] to i64 +; CHECK-NEXT: [[TMP1:%.*]] = load ptr, ptr getelementptr inbounds ([8 x ptr], ptr @[[GLOB0]], i32 0, i32 5), align 8 +; CHECK-NEXT: [[RET:%.*]] = call float [[TMP1]](i64 [[CAST_ARG]]) +; CHECK-NEXT: ret float [[RET]] +; +; +; CHECK-LABEL: define {{[^@]+}}@call_addrspacecast_callee_type_ifunc_nonvoid( +; CHECK-NEXT: [[RET:%.*]] = call addrspace(1) i32 addrspacecast (ptr @ifunc_nonvoid_1 to ptr addrspace(1))(double [[ARG:%.*]]) +; CHECK-NEXT: ret i32 [[RET]] +; +; +; CHECK-LABEL: define {{[^@]+}}@call_used_in_initializer( +; CHECK-NEXT: [[TMP1:%.*]] = load ptr, ptr getelementptr inbounds ([8 x ptr], ptr @[[GLOB0]], i32 0, i32 7), align 8 +; CHECK-NEXT: [[RET:%.*]] = call i32 [[TMP1]](double [[ARG:%.*]]) +; CHECK-NEXT: ret i32 [[RET]] +; +; +; CHECK-LABEL: define {{[^@]+}}@1( +; CHECK-NEXT: [[TMP1:%.*]] = call ptr @resolver1() +; CHECK-NEXT: store ptr [[TMP1]], ptr @[[GLOB0]], align 8 +; CHECK-NEXT: [[TMP2:%.*]] = call ptr @resolver1() +; CHECK-NEXT: store ptr [[TMP2]], ptr getelementptr inbounds ([8 x ptr], ptr @[[GLOB0]], i32 0, i32 1), align 8 +; CHECK-NEXT: [[TMP3:%.*]] = call ptr @resolver2() +; CHECK-NEXT: store ptr [[TMP3]], ptr getelementptr inbounds ([8 x ptr], ptr @[[GLOB0]], i32 0, i32 2), align 8 +; CHECK-NEXT: [[TMP4:%.*]] = call ptr @resolver3() +; CHECK-NEXT: store ptr [[TMP4]], ptr getelementptr inbounds ([8 x ptr], ptr @[[GLOB0]], i32 0, i32 3), align 8 +; CHECK-NEXT: [[TMP5:%.*]] = call ptr @resolver4() +; CHECK-NEXT: store ptr [[TMP5]], ptr getelementptr inbounds ([8 x ptr], ptr @[[GLOB0]], i32 0, i32 4), align 8 +; CHECK-NEXT: [[TMP6:%.*]] = call ptr @resolver5() +; CHECK-NEXT: store ptr [[TMP6]], ptr getelementptr inbounds ([8 x ptr], ptr @[[GLOB0]], i32 0, i32 5), align 8 +; CHECK-NEXT: [[TMP7:%.*]] = call ptr @resolver5() +; CHECK-NEXT: store ptr [[TMP7]], ptr getelementptr inbounds ([8 x ptr], ptr @[[GLOB0]], i32 0, i32 6), align 8 +; CHECK-NEXT: [[TMP8:%.*]] = call ptr @resolver5() +; CHECK-NEXT: store ptr [[TMP8]], ptr getelementptr inbounds ([8 x ptr], ptr @[[GLOB0]], i32 0, i32 7), align 8 +; CHECK-NEXT: ret void +;