diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp --- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp @@ -17,6 +17,7 @@ #include "AMDGPUTargetTransformInfo.h" #include "AMDGPUTargetMachine.h" #include "MCTargetDesc/AMDGPUMCTargetDesc.h" +#include "llvm/Analysis/InlineCost.h" #include "llvm/Analysis/LoopInfo.h" #include "llvm/Analysis/ValueTracking.h" #include "llvm/IR/IRBuilder.h" @@ -1176,10 +1177,67 @@ return true; } +static unsigned +adjustInliningThresholdUsingCallee(const Function *Callee, + const SITargetLowering *TLI, + const GCNTTIImpl *TTIImpl) { + const int NrOfSGPRUntilSpill = 26; + const int NrOfVGPRUntilSpill = 32; + + unsigned adjustThreshold = 0; + int SGPRsInUse = 0; + int VGPRsInUse = 0; + for (const Argument &A : Callee->args()) { + MVT ArgType = MVT::getVT(A.getType()); + if (A.getType()->isPointerTy()) { + ArgType = + TLI->getPointerTy(Callee->getParent()->getDataLayout(), + cast(A.getType())->getAddressSpace()); + ArgType = TLI->getRegisterTypeForCallingConv( + A.getContext(), Callee->getCallingConv(), ArgType); + } + unsigned CCRegNum = TLI->getNumRegistersForCallingConv( + A.getContext(), Callee->getCallingConv(), ArgType); + if (AMDGPU::isArgPassedInSGPR(&A)) + SGPRsInUse += CCRegNum; + else + VGPRsInUse += CCRegNum; + } + + // The cost of passing function arguments through the stack: + // 1 instruction to put a function argument on the stack in the caller. + // 1 instruction to take a function argument from the stack in callee. + // 1 instruction is explicitly take care of data dependencies in callee + // function. + InstructionCost ArgSpillCost(1); + ArgSpillCost += const_cast(TTIImpl)->getMemoryOpCost( + Instruction::Store, Type::getInt32Ty(Callee->getContext()), + TTIImpl->getDataLayout().getPointerPrefAlignment( + AMDGPUAS::PRIVATE_ADDRESS), + AMDGPUAS::PRIVATE_ADDRESS, TTI::TCK_SizeAndLatency); + ArgSpillCost += const_cast(TTIImpl)->getMemoryOpCost( + Instruction::Load, Type::getInt32Ty(Callee->getContext()), + TTIImpl->getDataLayout().getPointerPrefAlignment( + AMDGPUAS::PRIVATE_ADDRESS), + AMDGPUAS::PRIVATE_ADDRESS, TTI::TCK_SizeAndLatency); + + // The spill penalty is, currently, equal for either SGPR and VGPR; however, + // VGPR occupies more stack space in comparison which is not reflected in the + // penalty cost. + adjustThreshold += + std::max(0, SGPRsInUse - NrOfSGPRUntilSpill) * + *ArgSpillCost.getValue() * InlineConstants::getInstrCost(); + adjustThreshold += + std::max(0, VGPRsInUse - NrOfVGPRUntilSpill) * + *ArgSpillCost.getValue() * InlineConstants::getInstrCost(); + return adjustThreshold; +} + unsigned GCNTTIImpl::adjustInliningThreshold(const CallBase *CB) const { // If we have a pointer to private array passed into a function // it will not be optimized out, leaving scratch usage. // Increase the inline threshold to allow inlining in this case. + unsigned adjustThreshold = 0; uint64_t AllocaSize = 0; SmallPtrSet AIVisited; for (Value *PtrArg : CB->args()) { @@ -1201,9 +1259,10 @@ } } } - if (AllocaSize) - return ArgAllocaCost; - return 0; + adjustThreshold += + adjustInliningThresholdUsingCallee(CB->getCalledFunction(), TLI, this); + adjustThreshold += AllocaSize ? ArgAllocaCost : AllocaSize; + return adjustThreshold; } void GCNTTIImpl::getUnrollingPreferences(Loop *L, ScalarEvolution &SE, diff --git a/llvm/test/Transforms/Inline/AMDGPU/amdgpu-inline-stack-argument-i64.ll b/llvm/test/Transforms/Inline/AMDGPU/amdgpu-inline-stack-argument-i64.ll new file mode 100644 --- /dev/null +++ b/llvm/test/Transforms/Inline/AMDGPU/amdgpu-inline-stack-argument-i64.ll @@ -0,0 +1,96 @@ +; RUN: opt -mtriple=amdgcn-amd-amdhsa -S -passes=inline -inline-cost-full=true -inline-threshold=0 -inline-instr-cost=0 -inline-call-penalty=0 -debug-only=inline < %s 2>&1 | FileCheck %s +; REQUIRES: asserts + +; CHECK: NOT Inlining (cost={{[0-9]+}}, threshold={{[0-9]+}}), Call: %noinlinecall1 = call noundef i64 @non_inlining_call +; CHECK: NOT Inlining (cost={{[0-9]+}}, threshold={{[0-9]+}}), Call: %noinlinecall2 = call noundef i64 @non_inlining_call +; CHECK: Inlining (cost={{[0-9]+}}, threshold={{[0-9]+}}), Call: %inlinecall1 = call noundef i64 @inlining_call +; CHECK: Inlining (cost={{[0-9]+}}, threshold={{[0-9]+}}), Call: %inlinecall2 = call noundef i64 @inlining_call + +define noundef i64 @non_inlining_call(i64 noundef %a0, i64 noundef %b0, i64 noundef %c0, i64 noundef %d0, i64 noundef %e0, i64 noundef %f0, i64 noundef %g0, i64 noundef %h0, i64 noundef %i0, i64 noundef %j0, i64 noundef %k0, i64 noundef %l0, i64 noundef %m0, i64 noundef %n0, i64 noundef %o0, i64 noundef %p0) { +entry: + %xor = xor i64 %a0, %b0 + %xor1 = xor i64 %xor, %c0 + %xor2 = xor i64 %xor1, %d0 + %xor3 = xor i64 %xor2, %e0 + %xor4 = xor i64 %xor3, %f0 + %xor5 = xor i64 %xor4, %g0 + %xor6 = xor i64 %xor5, %h0 + %xor7 = xor i64 %xor6, %i0 + %xor8 = xor i64 %xor7, %j0 + %xor9 = xor i64 %xor8, %k0 + %xor10 = xor i64 %xor9, %l0 + %xor11 = xor i64 %xor10, %m0 + %xor12 = xor i64 %xor11, %n0 + %xor13 = xor i64 %xor12, %o0 + %xor14 = xor i64 %xor13, %p0 + ret i64 %xor14 +} + +define noundef i64 @inlining_call(i64 noundef %a0, i64 noundef %b0, i64 noundef %c0, i64 noundef %d0, i64 noundef %e0, i64 noundef %f0, i64 noundef %g0, i64 noundef %h0, i64 noundef %i0, i64 noundef %j0, i64 noundef %k0, i64 noundef %l0, i64 noundef %m0, i64 noundef %n0, i64 noundef %o0, i64 noundef %p0, i64 noundef %q0) { +entry: + %xor = xor i64 %a0, %b0 + %xor1 = xor i64 %xor, %c0 + %xor2 = xor i64 %xor1, %d0 + %xor3 = xor i64 %xor2, %e0 + %xor4 = xor i64 %xor3, %f0 + %xor5 = xor i64 %xor4, %g0 + %xor6 = xor i64 %xor5, %h0 + %xor7 = xor i64 %xor6, %i0 + %xor8 = xor i64 %xor7, %j0 + %xor9 = xor i64 %xor8, %k0 + %xor10 = xor i64 %xor9, %l0 + %xor11 = xor i64 %xor10, %m0 + %xor12 = xor i64 %xor11, %n0 + %xor13 = xor i64 %xor12, %o0 + %xor14 = xor i64 %xor13, %p0 + %xor15 = xor i64 %xor14, %q0 + ret i64 %xor15 +} + +; Calling each (non-)inlining function twice to make sure they won't get the sole call inlining cost bonus. +define i64 @Caller(ptr noundef %in) { +entry: + %arrayidx = getelementptr inbounds i64, ptr %in, i64 0 + %a0 = load i64, ptr %arrayidx, align 4 + %arrayidx1 = getelementptr inbounds i64, ptr %in, i64 1 + %b0 = load i64, ptr %arrayidx1, align 4 + %arrayidx2 = getelementptr inbounds i64, ptr %in, i64 2 + %c0 = load i64, ptr %arrayidx2, align 4 + %arrayidx3 = getelementptr inbounds i64, ptr %in, i64 3 + %d0 = load i64, ptr %arrayidx3, align 4 + %arrayidx4 = getelementptr inbounds i64, ptr %in, i64 4 + %e0 = load i64, ptr %arrayidx4, align 4 + %arrayidx5 = getelementptr inbounds i64, ptr %in, i64 5 + %f0 = load i64, ptr %arrayidx5, align 4 + %arrayidx6 = getelementptr inbounds i64, ptr %in, i64 6 + %g0 = load i64, ptr %arrayidx6, align 4 + %arrayidx7 = getelementptr inbounds i64, ptr %in, i64 7 + %h0 = load i64, ptr %arrayidx7, align 4 + %arrayidx8 = getelementptr inbounds i64, ptr %in, i64 8 + %i0 = load i64, ptr %arrayidx8, align 4 + %arrayidx9 = getelementptr inbounds i64, ptr %in, i64 9 + %j0 = load i64, ptr %arrayidx9, align 4 + %arrayidx10 = getelementptr inbounds i64, ptr %in, i64 10 + %k0 = load i64, ptr %arrayidx10, align 4 + %arrayidx11 = getelementptr inbounds i64, ptr %in, i64 11 + %l0 = load i64, ptr %arrayidx11, align 4 + %arrayidx12 = getelementptr inbounds i64, ptr %in, i64 12 + %m0 = load i64, ptr %arrayidx12, align 4 + %arrayidx13 = getelementptr inbounds i64, ptr %in, i64 13 + %n0 = load i64, ptr %arrayidx13, align 4 + %arrayidx14 = getelementptr inbounds i64, ptr %in, i64 14 + %o0 = load i64, ptr %arrayidx14, align 4 + %arrayidx15 = getelementptr inbounds i64, ptr %in, i64 15 + %p0 = load i64, ptr %arrayidx15, align 4 + %arrayidx16 = getelementptr inbounds i64, ptr %in, i64 16 + %q0 = load i64, ptr %arrayidx16, align 4 + %noinlinecall1 = call noundef i64 @non_inlining_call(i64 noundef %a0, i64 noundef %b0, i64 noundef %c0, i64 noundef %d0, i64 noundef %e0, i64 noundef %f0, i64 noundef %g0, i64 noundef %h0, i64 noundef %i0, i64 noundef %j0, i64 noundef %k0, i64 noundef %l0, i64 noundef %m0, i64 noundef %n0, i64 noundef %o0, i64 noundef %p0) + %add = add i64 0, %noinlinecall1 + %noinlinecall2 = call noundef i64 @non_inlining_call(i64 noundef %a0, i64 noundef %b0, i64 noundef %c0, i64 noundef %d0, i64 noundef %e0, i64 noundef %f0, i64 noundef %g0, i64 noundef %h0, i64 noundef %i0, i64 noundef %j0, i64 noundef %k0, i64 noundef %l0, i64 noundef %m0, i64 noundef %n0, i64 noundef %o0, i64 noundef %p0) + %add2 = add i64 %add, %noinlinecall2 + %inlinecall1 = call noundef i64 @inlining_call(i64 noundef %a0, i64 noundef %b0, i64 noundef %c0, i64 noundef %d0, i64 noundef %e0, i64 noundef %f0, i64 noundef %g0, i64 noundef %h0, i64 noundef %i0, i64 noundef %j0, i64 noundef %k0, i64 noundef %l0, i64 noundef %m0, i64 noundef %n0, i64 noundef %o0, i64 noundef %p0, i64 noundef %q0) + %add3 = add i64 %add2, %inlinecall1 + %inlinecall2 = call noundef i64 @inlining_call(i64 noundef %a0, i64 noundef %b0, i64 noundef %c0, i64 noundef %d0, i64 noundef %e0, i64 noundef %f0, i64 noundef %g0, i64 noundef %h0, i64 noundef %i0, i64 noundef %j0, i64 noundef %k0, i64 noundef %l0, i64 noundef %m0, i64 noundef %n0, i64 noundef %o0, i64 noundef %p0, i64 noundef %q0) + %add4 = add i64 %add3, %inlinecall2 + ret i64 %add4 +} diff --git a/llvm/test/Transforms/Inline/AMDGPU/amdgpu-inline-stack-argument.ll b/llvm/test/Transforms/Inline/AMDGPU/amdgpu-inline-stack-argument.ll new file mode 100644 --- /dev/null +++ b/llvm/test/Transforms/Inline/AMDGPU/amdgpu-inline-stack-argument.ll @@ -0,0 +1,160 @@ +; RUN: opt -mtriple=amdgcn-amd-amdhsa -S -passes=inline -inline-cost-full=true -inline-threshold=0 -inline-instr-cost=0 -inline-call-penalty=0 -debug-only=inline < %s 2>&1 | FileCheck %s +; REQUIRES: asserts + +; CHECK: NOT Inlining (cost={{[0-9]+}}, threshold={{[0-9]+}}), Call: %noinlinecall1 = call noundef i32 @non_inlining_call +; CHECK: NOT Inlining (cost={{[0-9]+}}, threshold={{[0-9]+}}), Call: %noinlinecall2 = call noundef i32 @non_inlining_call +; CHECK: Inlining (cost={{[0-9]+}}, threshold={{[0-9]+}}), Call: %inlinecall1 = call noundef i32 @inlining_call +; CHECK: Inlining (cost={{[0-9]+}}, threshold={{[0-9]+}}), Call: %inlinecall2 = call noundef i32 @inlining_call + +define noundef i32 @non_inlining_call(i32 noundef %a0, i32 noundef %b0, i32 noundef %c0, i32 noundef %d0, i32 noundef %e0, i32 noundef %f0, i32 noundef %g0, i32 noundef %h0, i32 noundef %i0, i32 noundef %j0, i32 noundef %k0, i32 noundef %l0, i32 noundef %m0, i32 noundef %n0, i32 noundef %o0, i32 noundef %p0, i32 noundef %q0, i32 noundef %r0, i32 noundef %s0, i32 noundef %t0, i32 noundef %u0, i32 noundef %v0, i32 noundef %w0, i32 noundef %x0, i32 noundef %y0, i32 noundef %z0, i32 noundef %a1, i32 noundef %b1, i32 noundef %c1, i32 noundef %d1, i32 noundef %e1, i32 noundef %f1) { +entry: + %xor = xor i32 %a0, %b0 + %xor1 = xor i32 %xor, %c0 + %xor2 = xor i32 %xor1, %d0 + %xor3 = xor i32 %xor2, %e0 + %xor4 = xor i32 %xor3, %f0 + %xor5 = xor i32 %xor4, %g0 + %xor6 = xor i32 %xor5, %h0 + %xor7 = xor i32 %xor6, %i0 + %xor8 = xor i32 %xor7, %j0 + %xor9 = xor i32 %xor8, %k0 + %xor10 = xor i32 %xor9, %l0 + %xor11 = xor i32 %xor10, %m0 + %xor12 = xor i32 %xor11, %n0 + %xor13 = xor i32 %xor12, %o0 + %xor14 = xor i32 %xor13, %p0 + %xor15 = xor i32 %xor14, %q0 + %xor16 = xor i32 %xor15, %r0 + %xor17 = xor i32 %xor16, %s0 + %xor18 = xor i32 %xor17, %t0 + %xor19 = xor i32 %xor18, %u0 + %xor20 = xor i32 %xor19, %v0 + %xor21 = xor i32 %xor20, %w0 + %xor22 = xor i32 %xor21, %x0 + %xor23 = xor i32 %xor22, %y0 + %xor24 = xor i32 %xor23, %z0 + %xor25 = xor i32 %xor24, %a1 + %xor26 = xor i32 %xor25, %b1 + %xor27 = xor i32 %xor26, %c1 + %xor28 = xor i32 %xor27, %d1 + %xor29 = xor i32 %xor28, %e1 + %xor30 = xor i32 %xor29, %f1 + ret i32 %xor30 +} + +define noundef i32 @inlining_call(i32 noundef %a0, i32 noundef %b0, i32 noundef %c0, i32 noundef %d0, i32 noundef %e0, i32 noundef %f0, i32 noundef %g0, i32 noundef %h0, i32 noundef %i0, i32 noundef %j0, i32 noundef %k0, i32 noundef %l0, i32 noundef %m0, i32 noundef %n0, i32 noundef %o0, i32 noundef %p0, i32 noundef %q0, i32 noundef %r0, i32 noundef %s0, i32 noundef %t0, i32 noundef %u0, i32 noundef %v0, i32 noundef %w0, i32 noundef %x0, i32 noundef %y0, i32 noundef %z0, i32 noundef %a1, i32 noundef %b1, i32 noundef %c1, i32 noundef %d1, i32 noundef %e1, i32 noundef %f1, i32 noundef %g1) { +entry: + %xor = xor i32 %a0, %b0 + %xor1 = xor i32 %xor, %c0 + %xor2 = xor i32 %xor1, %d0 + %xor3 = xor i32 %xor2, %e0 + %xor4 = xor i32 %xor3, %f0 + %xor5 = xor i32 %xor4, %g0 + %xor6 = xor i32 %xor5, %h0 + %xor7 = xor i32 %xor6, %i0 + %xor8 = xor i32 %xor7, %j0 + %xor9 = xor i32 %xor8, %k0 + %xor10 = xor i32 %xor9, %l0 + %xor11 = xor i32 %xor10, %m0 + %xor12 = xor i32 %xor11, %n0 + %xor13 = xor i32 %xor12, %o0 + %xor14 = xor i32 %xor13, %p0 + %xor15 = xor i32 %xor14, %q0 + %xor16 = xor i32 %xor15, %r0 + %xor17 = xor i32 %xor16, %s0 + %xor18 = xor i32 %xor17, %t0 + %xor19 = xor i32 %xor18, %u0 + %xor20 = xor i32 %xor19, %v0 + %xor21 = xor i32 %xor20, %w0 + %xor22 = xor i32 %xor21, %x0 + %xor23 = xor i32 %xor22, %y0 + %xor24 = xor i32 %xor23, %z0 + %xor25 = xor i32 %xor24, %a1 + %xor26 = xor i32 %xor25, %b1 + %xor27 = xor i32 %xor26, %c1 + %xor28 = xor i32 %xor27, %d1 + %xor29 = xor i32 %xor28, %e1 + %xor30 = xor i32 %xor29, %f1 + %xor31 = xor i32 %xor30, %g1 + ret i32 %xor31 +} + +; Calling each (non-)inlining function twice to make sure they won't get the sole call inlining cost bonus. +define i32 @Caller(ptr noundef %in) { +entry: + %arrayidx = getelementptr inbounds i32, ptr %in, i64 0 + %a0 = load i32, ptr %arrayidx, align 4 + %arrayidx1 = getelementptr inbounds i32, ptr %in, i64 1 + %b0 = load i32, ptr %arrayidx1, align 4 + %arrayidx2 = getelementptr inbounds i32, ptr %in, i64 2 + %c0 = load i32, ptr %arrayidx2, align 4 + %arrayidx3 = getelementptr inbounds i32, ptr %in, i64 3 + %d0 = load i32, ptr %arrayidx3, align 4 + %arrayidx4 = getelementptr inbounds i32, ptr %in, i64 4 + %e0 = load i32, ptr %arrayidx4, align 4 + %arrayidx5 = getelementptr inbounds i32, ptr %in, i64 5 + %f0 = load i32, ptr %arrayidx5, align 4 + %arrayidx6 = getelementptr inbounds i32, ptr %in, i64 6 + %g0 = load i32, ptr %arrayidx6, align 4 + %arrayidx7 = getelementptr inbounds i32, ptr %in, i64 7 + %h0 = load i32, ptr %arrayidx7, align 4 + %arrayidx8 = getelementptr inbounds i32, ptr %in, i64 8 + %i0 = load i32, ptr %arrayidx8, align 4 + %arrayidx9 = getelementptr inbounds i32, ptr %in, i64 9 + %j0 = load i32, ptr %arrayidx9, align 4 + %arrayidx10 = getelementptr inbounds i32, ptr %in, i64 10 + %k0 = load i32, ptr %arrayidx10, align 4 + %arrayidx11 = getelementptr inbounds i32, ptr %in, i64 11 + %l0 = load i32, ptr %arrayidx11, align 4 + %arrayidx12 = getelementptr inbounds i32, ptr %in, i64 12 + %m0 = load i32, ptr %arrayidx12, align 4 + %arrayidx13 = getelementptr inbounds i32, ptr %in, i64 13 + %n0 = load i32, ptr %arrayidx13, align 4 + %arrayidx14 = getelementptr inbounds i32, ptr %in, i64 14 + %o0 = load i32, ptr %arrayidx14, align 4 + %arrayidx15 = getelementptr inbounds i32, ptr %in, i64 15 + %p0 = load i32, ptr %arrayidx15, align 4 + %arrayidx16 = getelementptr inbounds i32, ptr %in, i64 16 + %q0 = load i32, ptr %arrayidx16, align 4 + %arrayidx17 = getelementptr inbounds i32, ptr %in, i64 17 + %r0 = load i32, ptr %arrayidx17, align 4 + %arrayidx18 = getelementptr inbounds i32, ptr %in, i64 18 + %s0 = load i32, ptr %arrayidx18, align 4 + %arrayidx19 = getelementptr inbounds i32, ptr %in, i64 19 + %t0 = load i32, ptr %arrayidx19, align 4 + %arrayidx20 = getelementptr inbounds i32, ptr %in, i64 20 + %u0 = load i32, ptr %arrayidx20, align 4 + %arrayidx21 = getelementptr inbounds i32, ptr %in, i64 21 + %v0 = load i32, ptr %arrayidx21, align 4 + %arrayidx22 = getelementptr inbounds i32, ptr %in, i64 22 + %w0 = load i32, ptr %arrayidx22, align 4 + %arrayidx23 = getelementptr inbounds i32, ptr %in, i64 23 + %x0 = load i32, ptr %arrayidx23, align 4 + %arrayidx24 = getelementptr inbounds i32, ptr %in, i64 24 + %y0 = load i32, ptr %arrayidx24, align 4 + %arrayidx25 = getelementptr inbounds i32, ptr %in, i64 25 + %z0 = load i32, ptr %arrayidx25, align 4 + %arrayidx26 = getelementptr inbounds i32, ptr %in, i64 26 + %a1 = load i32, ptr %arrayidx26, align 4 + %arrayidx27 = getelementptr inbounds i32, ptr %in, i64 27 + %b1 = load i32, ptr %arrayidx27, align 4 + %arrayidx28 = getelementptr inbounds i32, ptr %in, i64 28 + %c1 = load i32, ptr %arrayidx28, align 4 + %arrayidx29 = getelementptr inbounds i32, ptr %in, i64 29 + %d1 = load i32, ptr %arrayidx29, align 4 + %arrayidx30 = getelementptr inbounds i32, ptr %in, i64 30 + %e1 = load i32, ptr %arrayidx30, align 4 + %arrayidx31 = getelementptr inbounds i32, ptr %in, i64 31 + %f1 = load i32, ptr %arrayidx31, align 4 + %arrayidx32 = getelementptr inbounds i32, ptr %in, i64 32 + %g1 = load i32, ptr %arrayidx32, align 4 + %noinlinecall1 = call noundef i32 @non_inlining_call(i32 noundef %a0, i32 noundef %b0, i32 noundef %c0, i32 noundef %d0, i32 noundef %e0, i32 noundef %f0, i32 noundef %g0, i32 noundef %h0, i32 noundef %i0, i32 noundef %j0, i32 noundef %k0, i32 noundef %l0, i32 noundef %m0, i32 noundef %n0, i32 noundef %o0, i32 noundef %p0, i32 noundef %q0, i32 noundef %r0, i32 noundef %s0, i32 noundef %t0, i32 noundef %u0, i32 noundef %v0, i32 noundef %w0, i32 noundef %x0, i32 noundef %y0, i32 noundef %z0, i32 noundef %a1, i32 noundef %b1, i32 noundef %c1, i32 noundef %d1, i32 noundef %e1, i32 noundef %f1) + %add = add i32 0, %noinlinecall1 + %noinlinecall2 = call noundef i32 @non_inlining_call(i32 noundef %a0, i32 noundef %b0, i32 noundef %c0, i32 noundef %d0, i32 noundef %e0, i32 noundef %f0, i32 noundef %g0, i32 noundef %h0, i32 noundef %i0, i32 noundef %j0, i32 noundef %k0, i32 noundef %l0, i32 noundef %m0, i32 noundef %n0, i32 noundef %o0, i32 noundef %p0, i32 noundef %q0, i32 noundef %r0, i32 noundef %s0, i32 noundef %t0, i32 noundef %u0, i32 noundef %v0, i32 noundef %w0, i32 noundef %x0, i32 noundef %y0, i32 noundef %z0, i32 noundef %a1, i32 noundef %b1, i32 noundef %c1, i32 noundef %d1, i32 noundef %e1, i32 noundef %f1) + %add2 = add i32 %add, %noinlinecall2 + %inlinecall1 = call noundef i32 @inlining_call(i32 noundef %a0, i32 noundef %b0, i32 noundef %c0, i32 noundef %d0, i32 noundef %e0, i32 noundef %f0, i32 noundef %g0, i32 noundef %h0, i32 noundef %i0, i32 noundef %j0, i32 noundef %k0, i32 noundef %l0, i32 noundef %m0, i32 noundef %n0, i32 noundef %o0, i32 noundef %p0, i32 noundef %q0, i32 noundef %r0, i32 noundef %s0, i32 noundef %t0, i32 noundef %u0, i32 noundef %v0, i32 noundef %w0, i32 noundef %x0, i32 noundef %y0, i32 noundef %z0, i32 noundef %a1, i32 noundef %b1, i32 noundef %c1, i32 noundef %d1, i32 noundef %e1, i32 noundef %f1, i32 noundef %g1) + %add3 = add i32 %add2, %inlinecall1 + %inlinecall2 = call noundef i32 @inlining_call(i32 noundef %a0, i32 noundef %b0, i32 noundef %c0, i32 noundef %d0, i32 noundef %e0, i32 noundef %f0, i32 noundef %g0, i32 noundef %h0, i32 noundef %i0, i32 noundef %j0, i32 noundef %k0, i32 noundef %l0, i32 noundef %m0, i32 noundef %n0, i32 noundef %o0, i32 noundef %p0, i32 noundef %q0, i32 noundef %r0, i32 noundef %s0, i32 noundef %t0, i32 noundef %u0, i32 noundef %v0, i32 noundef %w0, i32 noundef %x0, i32 noundef %y0, i32 noundef %z0, i32 noundef %a1, i32 noundef %b1, i32 noundef %c1, i32 noundef %d1, i32 noundef %e1, i32 noundef %f1, i32 noundef %g1) + %add4 = add i32 %add3, %inlinecall2 + ret i32 %add4 +} diff --git a/llvm/test/Transforms/Inline/AMDGPU/amdgpu-inline-stack-ptr-argument.ll b/llvm/test/Transforms/Inline/AMDGPU/amdgpu-inline-stack-ptr-argument.ll new file mode 100644 --- /dev/null +++ b/llvm/test/Transforms/Inline/AMDGPU/amdgpu-inline-stack-ptr-argument.ll @@ -0,0 +1,192 @@ +; RUN: opt -mtriple=amdgcn-amd-amdhsa -S -passes=inline -inline-cost-full=true -inline-threshold=0 -inline-instr-cost=0 -inline-call-penalty=0 -debug-only=inline < %s 2>&1 | FileCheck %s +; REQUIRES: asserts + +; CHECK: NOT Inlining (cost={{[0-9]+}}, threshold={{[0-9]+}}), Call: %noinlinecall1 = call noundef i64 @non_inlining_call +; CHECK: NOT Inlining (cost={{[0-9]+}}, threshold={{[0-9]+}}), Call: %noinlinecall2 = call noundef i64 @non_inlining_call +; CHECK: Inlining (cost={{[0-9]+}}, threshold={{[0-9]+}}), Call: %inlinecall1 = call noundef i64 @inlining_call +; CHECK: Inlining (cost={{[0-9]+}}, threshold={{[0-9]+}}), Call: %inlinecall2 = call noundef i64 @inlining_call + +define noundef i64 @non_inlining_call(ptr noundef %ptra0, ptr noundef %ptrb0, ptr noundef %ptrc0, ptr noundef %ptrd0, ptr noundef %ptre0, ptr noundef %ptrf0, ptr noundef %ptrg0, ptr noundef %ptrh0, ptr noundef %ptri0, ptr noundef %ptrj0, ptr noundef %ptrk0, ptr noundef %ptrl0, ptr noundef %ptrm0, ptr noundef %ptrn0, ptr noundef %ptro0, ptr noundef %ptrp0, ptr noundef %ptrq0, ptr noundef %ptrr0, ptr noundef %ptrs0, ptr noundef %ptrt0, ptr noundef %ptru0, ptr noundef %ptrv0, ptr noundef %ptrw0, ptr noundef %ptrx0, ptr noundef %ptry0, ptr noundef %ptrz0, ptr noundef %ptra1, ptr noundef %ptrb1, ptr noundef %ptrc1, ptr noundef %ptrd1, ptr noundef %ptre1, ptr noundef %ptrf1) { +entry: + %a0 = load i64, ptr %ptra0, align 8 + %b0 = load i64, ptr %ptrb0, align 8 + %c0 = load i64, ptr %ptrc0, align 8 + %d0 = load i64, ptr %ptrd0, align 8 + %e0 = load i64, ptr %ptre0, align 8 + %f0 = load i64, ptr %ptrf0, align 8 + %g0 = load i64, ptr %ptrg0, align 8 + %h0 = load i64, ptr %ptrh0, align 8 + %i0 = load i64, ptr %ptri0, align 8 + %j0 = load i64, ptr %ptrj0, align 8 + %k0 = load i64, ptr %ptrk0, align 8 + %l0 = load i64, ptr %ptrl0, align 8 + %m0 = load i64, ptr %ptrm0, align 8 + %n0 = load i64, ptr %ptrn0, align 8 + %o0 = load i64, ptr %ptro0, align 8 + %p0 = load i64, ptr %ptrp0, align 8 + %q0 = load i64, ptr %ptrq0, align 8 + %r0 = load i64, ptr %ptrr0, align 8 + %s0 = load i64, ptr %ptrs0, align 8 + %t0 = load i64, ptr %ptrt0, align 8 + %u0 = load i64, ptr %ptru0, align 8 + %v0 = load i64, ptr %ptrv0, align 8 + %w0 = load i64, ptr %ptrw0, align 8 + %x0 = load i64, ptr %ptrx0, align 8 + %y0 = load i64, ptr %ptry0, align 8 + %z0 = load i64, ptr %ptrz0, align 8 + %a1 = load i64, ptr %ptra1, align 8 + %b1 = load i64, ptr %ptrb1, align 8 + %c1 = load i64, ptr %ptrc1, align 8 + %d1 = load i64, ptr %ptrd1, align 8 + %e1 = load i64, ptr %ptre1, align 8 + %f1 = load i64, ptr %ptrf1, align 8 + %xor = xor i64 %a0, %b0 + %xor1 = xor i64 %xor, %c0 + %xor2 = xor i64 %xor1, %d0 + %xor3 = xor i64 %xor2, %e0 + %xor4 = xor i64 %xor3, %f0 + %xor5 = xor i64 %xor4, %g0 + %xor6 = xor i64 %xor5, %h0 + %xor7 = xor i64 %xor6, %i0 + %xor8 = xor i64 %xor7, %j0 + %xor9 = xor i64 %xor8, %k0 + %xor10 = xor i64 %xor9, %l0 + %xor11 = xor i64 %xor10, %m0 + %xor12 = xor i64 %xor11, %n0 + %xor13 = xor i64 %xor12, %o0 + %xor14 = xor i64 %xor13, %p0 + %xor15 = xor i64 %xor14, %q0 + %xor16 = xor i64 %xor15, %r0 + %xor17 = xor i64 %xor16, %s0 + %xor18 = xor i64 %xor17, %t0 + %xor19 = xor i64 %xor18, %u0 + %xor20 = xor i64 %xor19, %v0 + %xor21 = xor i64 %xor20, %w0 + %xor22 = xor i64 %xor21, %x0 + %xor23 = xor i64 %xor22, %y0 + %xor24 = xor i64 %xor23, %z0 + %xor25 = xor i64 %xor24, %a1 + %xor26 = xor i64 %xor25, %b1 + %xor27 = xor i64 %xor26, %c1 + %xor28 = xor i64 %xor27, %d1 + %xor29 = xor i64 %xor28, %e1 + %xor30 = xor i64 %xor29, %f1 + ret i64 %xor30 +} + +define noundef i64 @inlining_call(ptr noundef %ptra0, ptr noundef %ptrb0, ptr noundef %ptrc0, ptr noundef %ptrd0, ptr noundef %ptre0, ptr noundef %ptrf0, ptr noundef %ptrg0, ptr noundef %ptrh0, ptr noundef %ptri0, ptr noundef %ptrj0, ptr noundef %ptrk0, ptr noundef %ptrl0, ptr noundef %ptrm0, ptr noundef %ptrn0, ptr noundef %ptro0, ptr noundef %ptrp0, ptr noundef %ptrq0, ptr noundef %ptrr0, ptr noundef %ptrs0, ptr noundef %ptrt0, ptr noundef %ptru0, ptr noundef %ptrv0, ptr noundef %ptrw0, ptr noundef %ptrx0, ptr noundef %ptry0, ptr noundef %ptrz0, ptr noundef %ptra1, ptr noundef %ptrb1, ptr noundef %ptrc1, ptr noundef %ptrd1, ptr noundef %ptre1, ptr noundef %ptrf1, ptr noundef %ptrg1) { +entry: + %a0 = load i64, ptr %ptra0, align 8 + %b0 = load i64, ptr %ptrb0, align 8 + %c0 = load i64, ptr %ptrc0, align 8 + %d0 = load i64, ptr %ptrd0, align 8 + %e0 = load i64, ptr %ptre0, align 8 + %f0 = load i64, ptr %ptrf0, align 8 + %g0 = load i64, ptr %ptrg0, align 8 + %h0 = load i64, ptr %ptrh0, align 8 + %i0 = load i64, ptr %ptri0, align 8 + %j0 = load i64, ptr %ptrj0, align 8 + %k0 = load i64, ptr %ptrk0, align 8 + %l0 = load i64, ptr %ptrl0, align 8 + %m0 = load i64, ptr %ptrm0, align 8 + %n0 = load i64, ptr %ptrn0, align 8 + %o0 = load i64, ptr %ptro0, align 8 + %p0 = load i64, ptr %ptrp0, align 8 + %q0 = load i64, ptr %ptrq0, align 8 + %r0 = load i64, ptr %ptrr0, align 8 + %s0 = load i64, ptr %ptrs0, align 8 + %t0 = load i64, ptr %ptrt0, align 8 + %u0 = load i64, ptr %ptru0, align 8 + %v0 = load i64, ptr %ptrv0, align 8 + %w0 = load i64, ptr %ptrw0, align 8 + %x0 = load i64, ptr %ptrx0, align 8 + %y0 = load i64, ptr %ptry0, align 8 + %z0 = load i64, ptr %ptrz0, align 8 + %a1 = load i64, ptr %ptra1, align 8 + %b1 = load i64, ptr %ptrb1, align 8 + %c1 = load i64, ptr %ptrc1, align 8 + %d1 = load i64, ptr %ptrd1, align 8 + %e1 = load i64, ptr %ptre1, align 8 + %f1 = load i64, ptr %ptrf1, align 8 + %g1 = load i64, ptr %ptrg1, align 8 + %xor = xor i64 %a0, %b0 + %xor1 = xor i64 %xor, %c0 + %xor2 = xor i64 %xor1, %d0 + %xor3 = xor i64 %xor2, %e0 + %xor4 = xor i64 %xor3, %f0 + %xor5 = xor i64 %xor4, %g0 + %xor6 = xor i64 %xor5, %h0 + %xor7 = xor i64 %xor6, %i0 + %xor8 = xor i64 %xor7, %j0 + %xor9 = xor i64 %xor8, %k0 + %xor10 = xor i64 %xor9, %l0 + %xor11 = xor i64 %xor10, %m0 + %xor12 = xor i64 %xor11, %n0 + %xor13 = xor i64 %xor12, %o0 + %xor14 = xor i64 %xor13, %p0 + %xor15 = xor i64 %xor14, %q0 + %xor16 = xor i64 %xor15, %r0 + %xor17 = xor i64 %xor16, %s0 + %xor18 = xor i64 %xor17, %t0 + %xor19 = xor i64 %xor18, %u0 + %xor20 = xor i64 %xor19, %v0 + %xor21 = xor i64 %xor20, %w0 + %xor22 = xor i64 %xor21, %x0 + %xor23 = xor i64 %xor22, %y0 + %xor24 = xor i64 %xor23, %z0 + %xor25 = xor i64 %xor24, %a1 + %xor26 = xor i64 %xor25, %b1 + %xor27 = xor i64 %xor26, %c1 + %xor28 = xor i64 %xor27, %d1 + %xor29 = xor i64 %xor28, %e1 + %xor30 = xor i64 %xor29, %f1 + %xor31 = xor i64 %xor30, %g1 + ret i64 %xor31 +} + +; Calling each (non-)inlining function twice to make sure they won't get the sole call inlining cost bonus. +define i64 @Caller(ptr noundef %in) { +entry: + %a0 = getelementptr inbounds i64, ptr %in, i64 0 + %b0 = getelementptr inbounds i64, ptr %in, i64 1 + %c0 = getelementptr inbounds i64, ptr %in, i64 2 + %d0 = getelementptr inbounds i64, ptr %in, i64 3 + %e0 = getelementptr inbounds i64, ptr %in, i64 4 + %f0 = getelementptr inbounds i64, ptr %in, i64 5 + %g0 = getelementptr inbounds i64, ptr %in, i64 6 + %h0 = getelementptr inbounds i64, ptr %in, i64 7 + %i0 = getelementptr inbounds i64, ptr %in, i64 8 + %j0 = getelementptr inbounds i64, ptr %in, i64 9 + %k0 = getelementptr inbounds i64, ptr %in, i64 10 + %l0 = getelementptr inbounds i64, ptr %in, i64 11 + %m0 = getelementptr inbounds i64, ptr %in, i64 12 + %n0 = getelementptr inbounds i64, ptr %in, i64 13 + %o0 = getelementptr inbounds i64, ptr %in, i64 14 + %p0 = getelementptr inbounds i64, ptr %in, i64 15 + %q0 = getelementptr inbounds i64, ptr %in, i64 16 + %r0 = getelementptr inbounds i64, ptr %in, i64 17 + %s0 = getelementptr inbounds i64, ptr %in, i64 18 + %t0 = getelementptr inbounds i64, ptr %in, i64 19 + %u0 = getelementptr inbounds i64, ptr %in, i64 20 + %v0 = getelementptr inbounds i64, ptr %in, i64 21 + %w0 = getelementptr inbounds i64, ptr %in, i64 22 + %x0 = getelementptr inbounds i64, ptr %in, i64 23 + %y0 = getelementptr inbounds i64, ptr %in, i64 24 + %z0 = getelementptr inbounds i64, ptr %in, i64 25 + %a1 = getelementptr inbounds i64, ptr %in, i64 26 + %b1 = getelementptr inbounds i64, ptr %in, i64 27 + %c1 = getelementptr inbounds i64, ptr %in, i64 28 + %d1 = getelementptr inbounds i64, ptr %in, i64 29 + %e1 = getelementptr inbounds i64, ptr %in, i64 30 + %f1 = getelementptr inbounds i64, ptr %in, i64 31 + %g1 = getelementptr inbounds i64, ptr %in, i64 32 + %noinlinecall1 = call noundef i64 @non_inlining_call(ptr noundef %a0, ptr noundef %b0, ptr noundef %c0, ptr noundef %d0, ptr noundef %e0, ptr noundef %f0, ptr noundef %g0, ptr noundef %h0, ptr noundef %i0, ptr noundef %j0, ptr noundef %k0, ptr noundef %l0, ptr noundef %m0, ptr noundef %n0, ptr noundef %o0, ptr noundef %p0, ptr noundef %q0, ptr noundef %r0, ptr noundef %s0, ptr noundef %t0, ptr noundef %u0, ptr noundef %v0, ptr noundef %w0, ptr noundef %x0, ptr noundef %y0, ptr noundef %z0, ptr noundef %a1, ptr noundef %b1, ptr noundef %c1, ptr noundef %d1, ptr noundef %e1, ptr noundef %f1) + %add = add i64 0, %noinlinecall1 + %noinlinecall2 = call noundef i64 @non_inlining_call(ptr noundef %a0, ptr noundef %b0, ptr noundef %c0, ptr noundef %d0, ptr noundef %e0, ptr noundef %f0, ptr noundef %g0, ptr noundef %h0, ptr noundef %i0, ptr noundef %j0, ptr noundef %k0, ptr noundef %l0, ptr noundef %m0, ptr noundef %n0, ptr noundef %o0, ptr noundef %p0, ptr noundef %q0, ptr noundef %r0, ptr noundef %s0, ptr noundef %t0, ptr noundef %u0, ptr noundef %v0, ptr noundef %w0, ptr noundef %x0, ptr noundef %y0, ptr noundef %z0, ptr noundef %a1, ptr noundef %b1, ptr noundef %c1, ptr noundef %d1, ptr noundef %e1, ptr noundef %f1) + %add2 = add i64 %add, %noinlinecall2 + %inlinecall1 = call noundef i64 @inlining_call(ptr noundef %a0, ptr noundef %b0, ptr noundef %c0, ptr noundef %d0, ptr noundef %e0, ptr noundef %f0, ptr noundef %g0, ptr noundef %h0, ptr noundef %i0, ptr noundef %j0, ptr noundef %k0, ptr noundef %l0, ptr noundef %m0, ptr noundef %n0, ptr noundef %o0, ptr noundef %p0, ptr noundef %q0, ptr noundef %r0, ptr noundef %s0, ptr noundef %t0, ptr noundef %u0, ptr noundef %v0, ptr noundef %w0, ptr noundef %x0, ptr noundef %y0, ptr noundef %z0, ptr noundef %a1, ptr noundef %b1, ptr noundef %c1, ptr noundef %d1, ptr noundef %e1, ptr noundef %f1, ptr noundef %g1) + %add3 = add i64 %add2, %inlinecall1 + %inlinecall2 = call noundef i64 @inlining_call(ptr noundef %a0, ptr noundef %b0, ptr noundef %c0, ptr noundef %d0, ptr noundef %e0, ptr noundef %f0, ptr noundef %g0, ptr noundef %h0, ptr noundef %i0, ptr noundef %j0, ptr noundef %k0, ptr noundef %l0, ptr noundef %m0, ptr noundef %n0, ptr noundef %o0, ptr noundef %p0, ptr noundef %q0, ptr noundef %r0, ptr noundef %s0, ptr noundef %t0, ptr noundef %u0, ptr noundef %v0, ptr noundef %w0, ptr noundef %x0, ptr noundef %y0, ptr noundef %z0, ptr noundef %a1, ptr noundef %b1, ptr noundef %c1, ptr noundef %d1, ptr noundef %e1, ptr noundef %f1, ptr noundef %g1) + %add4 = add i64 %add3, %inlinecall2 + ret i64 %add4 +}