diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.h b/llvm/lib/Target/AMDGPU/AMDGPU.h --- a/llvm/lib/Target/AMDGPU/AMDGPU.h +++ b/llvm/lib/Target/AMDGPU/AMDGPU.h @@ -123,9 +123,27 @@ void initializeAMDGPUPropagateAttributesEarlyPass(PassRegistry &); extern char &AMDGPUPropagateAttributesEarlyID; +struct AMDGPUPropagateAttributesEarlyPass + : PassInfoMixin { + AMDGPUPropagateAttributesEarlyPass(TargetMachine &TM) : TM(TM) {} + PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); + +private: + TargetMachine &TM; +}; + void initializeAMDGPUPropagateAttributesLatePass(PassRegistry &); extern char &AMDGPUPropagateAttributesLateID; +struct AMDGPUPropagateAttributesLatePass + : PassInfoMixin { + AMDGPUPropagateAttributesLatePass(TargetMachine &TM) : TM(TM) {} + PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM); + +private: + TargetMachine &TM; +}; + void initializeAMDGPURewriteOutArgumentsPass(PassRegistry &); extern char &AMDGPURewriteOutArgumentsID; diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPropagateAttributes.cpp b/llvm/lib/Target/AMDGPU/AMDGPUPropagateAttributes.cpp --- a/llvm/lib/Target/AMDGPU/AMDGPUPropagateAttributes.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUPropagateAttributes.cpp @@ -35,6 +35,7 @@ #include "llvm/CodeGen/TargetPassConfig.h" #include "llvm/IR/Function.h" #include "llvm/IR/Module.h" +#include "llvm/IR/PassManager.h" #include "llvm/Target/TargetMachine.h" #include "llvm/Transforms/Utils/Cloning.h" #include @@ -409,3 +410,21 @@ *llvm::createAMDGPUPropagateAttributesLatePass(const TargetMachine *TM) { return new AMDGPUPropagateAttributesLate(TM); } + +PreservedAnalyses +AMDGPUPropagateAttributesEarlyPass::run(Function &F, + FunctionAnalysisManager &AM) { + if (!AMDGPU::isEntryFunctionCC(F.getCallingConv())) + return PreservedAnalyses::all(); + + return AMDGPUPropagateAttributes(&TM, false).process(F) + ? PreservedAnalyses::none() + : PreservedAnalyses::all(); +} + +PreservedAnalyses +AMDGPUPropagateAttributesLatePass::run(Module &M, ModuleAnalysisManager &AM) { + return AMDGPUPropagateAttributes(&TM, true).process(M) + ? PreservedAnalyses::none() + : PreservedAnalyses::all(); +} diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp --- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp @@ -491,6 +491,15 @@ void AMDGPUTargetMachine::registerPassBuilderCallbacks(PassBuilder &PB, bool DebugPassManager) { + PB.registerPipelineParsingCallback( + [this](StringRef PassName, ModulePassManager &PM, + ArrayRef) { + if (PassName == "amdgpu-propagate-attributes-late") { + PM.addPass(AMDGPUPropagateAttributesLatePass(*this)); + return true; + } + return false; + }); PB.registerPipelineParsingCallback( [this](StringRef PassName, FunctionPassManager &PM, ArrayRef) { @@ -514,13 +523,19 @@ PM.addPass(AMDGPULowerKernelAttributesPass()); return true; } + if (PassName == "amdgpu-propagate-attributes-early") { + PM.addPass(AMDGPUPropagateAttributesEarlyPass(*this)); + return true; + } + return false; }); - PB.registerPipelineStartEPCallback([DebugPassManager]( + PB.registerPipelineStartEPCallback([this, DebugPassManager]( ModulePassManager &PM, PassBuilder::OptimizationLevel Level) { FunctionPassManager FPM(DebugPassManager); + FPM.addPass(AMDGPUPropagateAttributesEarlyPass(*this)); FPM.addPass(AMDGPUUseNativeCallsPass()); if (EnableLibCallSimplify && Level != PassBuilder::OptimizationLevel::O0) FPM.addPass(AMDGPUSimplifyLibCallsPass()); @@ -528,12 +543,15 @@ }); PB.registerPipelineEarlySimplificationEPCallback( - [](ModulePassManager &PM, PassBuilder::OptimizationLevel Level) { + [this](ModulePassManager &PM, PassBuilder::OptimizationLevel Level) { if (Level == PassBuilder::OptimizationLevel::O0) return; if (InternalizeSymbols) { PM.addPass(InternalizePass(mustPreserveGV)); + } + PM.addPass(AMDGPUPropagateAttributesLatePass(*this)); + if (InternalizeSymbols) { PM.addPass(GlobalDCEPass()); } }); diff --git a/llvm/test/CodeGen/AMDGPU/propagate-attributes-clone.ll b/llvm/test/CodeGen/AMDGPU/propagate-attributes-clone.ll --- a/llvm/test/CodeGen/AMDGPU/propagate-attributes-clone.ll +++ b/llvm/test/CodeGen/AMDGPU/propagate-attributes-clone.ll @@ -1,5 +1,7 @@ ; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -O1 < %s | FileCheck -check-prefixes=OPT,OPT-EXT %s +; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -passes='default' < %s | FileCheck -check-prefixes=OPT,OPT-EXT %s ; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -O1 --amdgpu-internalize-symbols < %s | FileCheck -check-prefixes=OPT,OPT-INT %s +; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -passes='default' --amdgpu-internalize-symbols < %s | FileCheck -check-prefixes=OPT,OPT-INT %s ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck -check-prefix=LLC %s ; OPT: declare void @foo4() local_unnamed_addr #0 diff --git a/llvm/test/CodeGen/AMDGPU/propagate-attributes-flat-work-group-size.ll b/llvm/test/CodeGen/AMDGPU/propagate-attributes-flat-work-group-size.ll --- a/llvm/test/CodeGen/AMDGPU/propagate-attributes-flat-work-group-size.ll +++ b/llvm/test/CodeGen/AMDGPU/propagate-attributes-flat-work-group-size.ll @@ -1,4 +1,5 @@ ; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -amdgpu-propagate-attributes-late %s | FileCheck %s +; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -passes=amdgpu-propagate-attributes-late %s | FileCheck %s ; CHECK: define internal void @max_flat_1_1024() #0 { define internal void @max_flat_1_1024() #0 { diff --git a/llvm/test/CodeGen/AMDGPU/propagate-attributes-single-set.ll b/llvm/test/CodeGen/AMDGPU/propagate-attributes-single-set.ll --- a/llvm/test/CodeGen/AMDGPU/propagate-attributes-single-set.ll +++ b/llvm/test/CodeGen/AMDGPU/propagate-attributes-single-set.ll @@ -1,4 +1,5 @@ ; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -O1 < %s | FileCheck -check-prefix=OPT %s +; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -passes='default' < %s | FileCheck -check-prefix=OPT %s ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck -check-prefix=LLC %s ; OPT: declare void @foo4() local_unnamed_addr #0 diff --git a/llvm/tools/opt/opt.cpp b/llvm/tools/opt/opt.cpp --- a/llvm/tools/opt/opt.cpp +++ b/llvm/tools/opt/opt.cpp @@ -463,8 +463,13 @@ // it exists. static bool shouldPinPassToLegacyPM(StringRef Pass) { std::vector PassNameExactToIgnore = { - "amdgpu-simplifylib", "amdgpu-usenative", "amdgpu-promote-alloca", - "amdgpu-promote-alloca-to-vector", "amdgpu-lower-kernel-attributes"}; + "amdgpu-simplifylib", + "amdgpu-usenative", + "amdgpu-promote-alloca", + "amdgpu-promote-alloca-to-vector", + "amdgpu-lower-kernel-attributes", + "amdgpu-propagate-attributes-early", + "amdgpu-propagate-attributes-late"}; for (const auto &P : PassNameExactToIgnore) if (Pass == P) return false;