[AMDGPU] Support for device scope shared variables

[hsmhsm]

Summary

WIP, this patch is uploaded to save the current state of the work,
and is not meant for review at the moment.

[hsmhsm]

Implemented few more necessary data structures

[hsmhsm]

Collect all associated LDS globals associated with a kernel.

[hsmhsm]

Fix bug and compute offset for each LDS

[hsmhsm]

Completed the handling of direct LDS globals within kernel

[hsmhsm]

Partial support for indirect LDS globals.

[hsmhsm]

Code refactoring as per the new logic of processing.

[hsmhsm]

Support for indirect LDS globals.

[hsmhsm]

First bug fix.

[hsmhsm]

Fixes previous incorrect commit.

[hsmhsm]

First cut of clode clean-up

[hsmhsm]

Support for replacing constant expressions [in progress].

[hsmhsm]

Successfully tested simple sum-reduce HIP application.

[hsmhsm]

Succesfully tested an HIP application which has more than one LDS globals.

[hsmhsm]

Successfully tested an HIP application which defines different shared
variables of different types within different device functions, and there
exist call graph path from kernel to all these device functions.

[hsmhsm]

Handle higher dimensional arrays.

[hsmhsm]

Updated introductory comment.

[hsmhsm]

Fixed minor bug in the updated code in the last commit.

[hsmhsm]

Make sure to run this pass as first AMDGPU IR pass.

[hsmhsm]

Retain original name for new cloned functions.

[hsmhsm]

Add lit test-cases.

[hsmhsm]

This revision was created just for tracking the progress of the work. Now that the code is ready for actual review, I created a new official revision - https://reviews.llvm.org/D91516, and hence closing this revision.

llvm/lib/Target/AMDGPU/AMDGPU.h

 extern char &AMDGPUOpenCLEnqueuedBlockLoweringID;
 
 void initializeGCNRegBankReassignPass(PassRegistry &);
 extern char &GCNRegBankReassignID;
 
 void initializeGCNNSAReassignPass(PassRegistry &);
 extern char &GCNNSAReassignID;
 
+ModulePass *createAMDGPUDeviceScopeSharedVariablePass();
+void initializeAMDGPUDeviceScopeSharedVariablePass(PassRegistry &);
+extern char &AMDGPUDeviceScopeSharedVariableID;
+
 namespace AMDGPU {
 enum TargetIndex {
   TI_CONSTDATA_START,
   TI_SCRATCH_RSRC_DWORD0,
   TI_SCRATCH_RSRC_DWORD1,
   TI_SCRATCH_RSRC_DWORD2,
   TI_SCRATCH_RSRC_DWORD3
 };

llvm/lib/Target/AMDGPU/AMDGPUAlwaysInlinePass.cpp

bool AMDGPUAlwaysInline::runOnModule(Module &M) {
   // objects defined in functions. LDS is always allocated by a kernel, and it
   // is difficult to manage LDS usage if a function may be used by multiple
   // kernels.
   //
   // OpenCL doesn't allow declaring LDS in non-kernels, so in practice this
   // should only appear when IPO passes manages to move LDs defined in a kernel
   // into a single user function.
 
+  if (!AMDGPUTargetMachine::EnableDeviceScopeSharedVariable) {
     for (GlobalVariable &GV : M.globals()) {
       // TODO: Region address
       unsigned AS = GV.getAddressSpace();
       if (AS != AMDGPUAS::LOCAL_ADDRESS && AS != AMDGPUAS::REGION_ADDRESS)
         continue;
 
       recursivelyVisitUsers(GV, FuncsToAlwaysInline);
     }
+  }
 
   if (!AMDGPUTargetMachine::EnableFunctionCalls || StressCalls) {
     auto IncompatAttr
       = StressCalls ? Attribute::AlwaysInline : Attribute::NoInline;
 
     for (Function &F : M) {
       if (!F.isDeclaration() && !F.use_empty() &&
           !F.hasFnAttribute(IncompatAttr)) {

llvm/lib/Target/AMDGPU/AMDGPUDeviceScopeSharedVariable.cpp

+//===-- AMDGPUDeviceScopeSharedVariables.cpp ------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// TODO:
+//
+//===----------------------------------------------------------------------===//
+
+#include "AMDGPU.h"
+#include "llvm/ADT/SetVector.h"
+#include "llvm/Analysis/CallGraph.h"
+#include "llvm/CodeGen/TargetPassConfig.h"
+#include "llvm/IR/Module.h"
+#include "llvm/IR/ValueMap.h"
+#include "llvm/InitializePasses.h"
+
+#define DEBUG_TYPE "amdgpu-device-scope-shared-variable"
+
+using namespace llvm;
+
+namespace {
+
+class AMDGPUDeviceScopeSharedVariable : public ModulePass {
+public:
+  static char ID;
+
+  AMDGPUDeviceScopeSharedVariable() : ModulePass(ID) {
+    initializeAMDGPUDeviceScopeSharedVariablePass(
+        *PassRegistry::getPassRegistry());
+  }
+
+  bool runOnModule(Module &M) override;
+};
+
+} // namespace
+
+char AMDGPUDeviceScopeSharedVariable::ID = 0;
+
+char &llvm::AMDGPUDeviceScopeSharedVariableID =
+    AMDGPUDeviceScopeSharedVariable::ID;
+
+ModulePass *llvm::createAMDGPUDeviceScopeSharedVariablePass() {
+  return new AMDGPUDeviceScopeSharedVariable();
+}
+
+INITIALIZE_PASS_BEGIN(AMDGPUDeviceScopeSharedVariable,
+                      "implement-amdgpu-device-scope-shared-variable",
+                      "Implement AMDPGU Device Scope Shared Variable",
+                      false /*only look at the cfg*/, false /*analysis pass*/)
+INITIALIZE_PASS_DEPENDENCY(AMDGPUAlwaysInline)
+INITIALIZE_PASS_DEPENDENCY(SimpleInliner)
+INITIALIZE_PASS_END(AMDGPUDeviceScopeSharedVariable,
+                    "handle-amdgpu-device-scope-shared-variable",
+                    "Handle AMDPGU Device Scope Shared Variable",
+                    false /*only look at the cfg*/, false /*analysis pass*/)
+
+static void createFunctionToLDSMap(
+    ValueMap<const GlobalVariable *, const Function *> &LDSToFunction,
+    ValueMap<const Function *, SetVector<const GlobalVariable *>>
+        &FunctionToLDS) {
+  for (auto it = LDSToFunction.begin(); it != LDSToFunction.end(); ++it) {
+    const GlobalVariable *LDSGlobal = it->first;
+    const Function *EnclosingFunction = it->second;
+    auto rit = FunctionToLDS.find(EnclosingFunction);
+    if (rit == FunctionToLDS.end()) {
+      SetVector<const GlobalVariable *> LDSSet;
+      LDSSet.insert(LDSGlobal);
+      FunctionToLDS[EnclosingFunction] = LDSSet;
+    } else
+      FunctionToLDS[EnclosingFunction].insert(LDSGlobal);
+  }
+}
+
+static void pairUpKernelWithLDSList(
+    const Function *K,
+    ValueMap<const Function *, SetVector<const Function *>> &KernelToCallie,
+    ValueMap<const Function *, SetVector<const GlobalVariable *>>
+        &FunctionToLDS,
+    ValueMap<const Function *, SetVector<const GlobalVariable *>>
+        &KernelToExtendedLDS) {
+  // Set which holds all the LDS globals which are defined either directly in
+  // kernel or indirectly in within callie(s)
+  SetVector<const GlobalVariable *> ExtendedLDSSet;
+
+  // Collect all the LDS globals defined within kernel
+  if (FunctionToLDS.find(K) != FunctionToLDS.end())
+    ExtendedLDSSet = FunctionToLDS[K];
+
+  // Collect all the LDS globals defined within the callie(s) of kernel
+  SetVector<const Function *> Callies = KernelToCallie[K];
+  for (const Function *Callie : Callies) {
+    if (FunctionToLDS.find(Callie) == FunctionToLDS.end())
+      continue;
+    SetVector<const GlobalVariable *> CallieLDSList = FunctionToLDS[Callie];
+    for (const GlobalVariable *CallieLDS : CallieLDSList)
+      ExtendedLDSSet.insert(CallieLDS);
+  }
+
+  KernelToExtendedLDS[K] = ExtendedLDSSet;
+}
+
+static void pairUpKernelWithCallieList(
+    Module &M, const Function *K,
+    ValueMap<const Function *, SetVector<const GlobalVariable *>>
+        &FunctionToLDS,
+    ValueMap<const Function *, SetVector<const Function *>> &KernelToCallie) {
+  // Get the call graph node associated with current kernel, traverse the call
+  // graph associated with the it in DFS manner and collect all the associated
+  // callies which define LDS global(s)
+  CallGraph CG = CallGraph(M);
+  const CallGraphNode *KernCGNode = CG[K];
+  SmallVector<const CallGraphNode *, 16> CGNodeStack;
+  SetVector<const CallGraphNode *> Visited;
+
+#ifndef NDEBUG
+  assert(KernCGNode && "Call graph node associated with kernel definition "
+                       "cannot be null\n");
+#endif
+
+  for (auto it = KernCGNode->begin(); it != KernCGNode->end(); ++it) {
+    const CallGraphNode *CGN = it->second;
+#ifndef NDEBUG
+    assert(CGN && "Call graph node associated with function definition cannot"
+                  " be null\n");
+#endif
+    CGNodeStack.push_back(CGN);
+  }
+
+  SetVector<const Function *> CallieSet;
+  while (!CGNodeStack.empty()) {
+    const CallGraphNode *CGNode = CGNodeStack.pop_back_val();
+    if (!Visited.insert(CGNode))
+      continue;
+
+    Function *F = CGNode->getFunction();
+    if (!F || F->isDeclaration()) {
+#ifndef NDEBUG
+      assert(CGNode->empty() && "Call graph node associated with function "
+                                "declaration should not have callie list\n");
+#endif
+      continue;
+    }
+
+    auto fit = FunctionToLDS.find(F);
+    if (fit != FunctionToLDS.end())
+      CallieSet.insert(F);
+
+    for (auto it = CGNode->begin(); it != CGNode->end(); ++it) {
+      const CallGraphNode *CGN = it->second;
+#ifndef NDEBUG
+      assert(CGN && "Call graph node associated with function definition cannot"
+                    " be null\n");
+#endif
+      CGNodeStack.push_back(CGN);
+    }
+  }
+
+  KernelToCallie[K] = CallieSet;
+}
+
+static void pairUpLDSGlobalWithEnclosingFunction(
+    const GlobalVariable *LDSGlobal,
+    ValueMap<const GlobalVariable *, const Function *> &LDSToFunction) {
+  // Recursively visit the user list of current LDS global, and find the
+  // enclosing function where the LDS global is defined, and the enclosing
+  // function should always be successfully found.
+  //
+  // TODO: Is there any other efficient way to find the enclosing functions of
+  // LDS globals?
+#ifndef NDEBUG
+  assert(!LDSGlobal->user_empty() &&
+         "LDS Global user list cannot be empty since it must have been defined "
+         "within either kernel or device function");
+#endif
+  SmallVector<const User *, 16> UserStack;
+  SetVector<const User *> Visited;
+
+  for (const User *U : LDSGlobal->users())
+    UserStack.push_back(U);
+
+  while (!UserStack.empty()) {
+    const User *U = UserStack.pop_back_val();
+    if (!Visited.insert(U))
+      continue;
+
+    if (const Instruction *I = dyn_cast<Instruction>(U)) {
+      const Function *F = I->getParent()->getParent();
+      if (F) {
+        LDSToFunction[LDSGlobal] = F;
+        return;
+      }
+      continue;
+    }
+
+    for (const User *UU : U->users())
+      UserStack.push_back(UU);
+  }
+#ifndef NDEBUG
+  assert(false && "Control is not expected to reach this point");
+#endif
+}
+
+static void
+getLDSGlobalSizeInBytes(Module &M, const GlobalVariable *LDSGlobal,
+                        ValueMap<const GlobalVariable *, uint64_t> &LDSToSize) {
+  Type *Ty = LDSGlobal->getValueType();
+  const DataLayout &DL = M.getDataLayout();
+  uint64_t SizeInBytes = DL.getTypeSizeInBits(Ty).getFixedSize() / 8;
+  LDSToSize[LDSGlobal] = SizeInBytes;
+}
+
+static bool handleDeviceScopeSharedVariablesForCurKernel(
+    Module &M, const Function *K,
+    ValueMap<const GlobalVariable *, const Function *> &LDSToFunction,
+    ValueMap<const Function *, SetVector<const GlobalVariable *>>
+        &FunctionToLDS,
+    ValueMap<const GlobalVariable *, uint64_t> &LDSToSize,
+    SetVector<const Function *> &CallieList,
+    SetVector<const GlobalVariable *> &ExtendedLDSList) {
+  // Compute the total size of all LDS globals
+  uint64_t TotalLDSSizeInBytes = 0;
+  for (auto it = LDSToSize.begin(); it != LDSToSize.end(); ++it)
+    TotalLDSSizeInBytes += it->second;
+
+  // Insert a new LDS global which is nothing but a single contigeous shared
+  // memory layout representing all the LDS globals associted with the kernel
+  // which includes those directly defined within the kernel and those
+  // indirectly defined within callies.
+  //
+  // The size of this new contigeous LDS global layout is equal to the sum of
+  // the sizes of all the associated LDS globals.
+  Type *NewLDSTy =
+      ArrayType::get(IntegerType::get(M.getContext(), 8), TotalLDSSizeInBytes);
+  GlobalVariable *NewLDS = new GlobalVariable(
+      M, NewLDSTy, false, GlobalValue::InternalLinkage,
+      UndefValue::get(NewLDSTy), Twine(K->getName()) + Twine("_LDSLayout"),
+      nullptr, GlobalVariable::NotThreadLocal, AMDGPUAS::LOCAL_ADDRESS);
+  NewLDS->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
+  NewLDS->setAlignment(MaybeAlign(M.getDataLayout().getPreferredAlign(NewLDS)));
+
+  return true;
+}
+
+static bool handleDeviceScopeSharedVariables(
+    Module &M, const SetVector<const Function *> &Kernels,
+    ValueMap<const GlobalVariable *, const Function *> &LDSToFunction,
+    ValueMap<const Function *, SetVector<const GlobalVariable *>>
+        &FunctionToLDS,
+    ValueMap<const Function *, SetVector<const Function *>> &KernelToCallie,
+    ValueMap<const Function *, SetVector<const GlobalVariable *>>
+        &KernelToExtendedLDS,
+    ValueMap<const GlobalVariable *, uint64_t> &LDSToSize) {
+  // Process LDS global list associated with each kernel
+  bool Changed = false;
+  for (const Function *K : Kernels) {
+    // No LDS globals to process? ignore the kernel, goto next one
+    if (KernelToExtendedLDS.find(K) == KernelToExtendedLDS.end())
+      continue;
+    Changed |= handleDeviceScopeSharedVariablesForCurKernel(
+        M, K, LDSToFunction, FunctionToLDS, LDSToSize, KernelToCallie[K],
+        KernelToExtendedLDS[K]);
+  }
+  return Changed;
+}
+
+static bool handleDeviceScopeSharedVariables(
+    Module &M, const SetVector<const GlobalVariable *> &LDSGlobals,
+    const SetVector<const Function *> &Kernels) {
+  // Pair up each LDS global with the enclosing function where the LDS global is
+  // defined
+  ValueMap<const GlobalVariable *, const Function *> LDSToFunction;
+  for (const GlobalVariable *LDSGlobal : LDSGlobals)
+    pairUpLDSGlobalWithEnclosingFunction(LDSGlobal, LDSToFunction);
+
+  // Create reverse map from enclosing function to LDS global list
+  ValueMap<const Function *, SetVector<const GlobalVariable *>> FunctionToLDS;
+  createFunctionToLDSMap(LDSToFunction, FunctionToLDS);
+
+  // Pair up kernels with callie list which define LDS globals
+  ValueMap<const Function *, SetVector<const Function *>> KernelToCallie;
+  for (const Function *K : Kernels)
+    pairUpKernelWithCallieList(M, K, FunctionToLDS, KernelToCallie);
+
+  // Pair up kernels with all the associated LDS globals including those
+  // directly defined within the kernel and those indirectly defined within
+  // the callies
+  ValueMap<const Function *, SetVector<const GlobalVariable *>>
+      KernelToExtendedLDS;
+  for (const Function *K : Kernels)
+    pairUpKernelWithLDSList(K, KernelToCallie, FunctionToLDS,
+                            KernelToExtendedLDS);
+
+  // Get the size of each LDS global in bytes
+  ValueMap<const GlobalVariable *, uint64_t> LDSToSize;
+  for (const GlobalVariable *LDSGlobal : LDSGlobals)
+    getLDSGlobalSizeInBytes(M, LDSGlobal, LDSToSize);
+
+  return handleDeviceScopeSharedVariables(M, Kernels, LDSToFunction,
+                                          FunctionToLDS, KernelToCallie,
+                                          KernelToExtendedLDS, LDSToSize);
+}
+
+static bool handleDeviceScopeSharedVariables(Module &M) {
+  // Collect all the (static) LDS globals defined within the current module
+  SetVector<const GlobalVariable *> LDSGlobals;
+  for (GlobalVariable &GV : M.globals())
+    if (GV.getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS &&
+        GV.hasInternalLinkage())
+      LDSGlobals.insert(&GV);
+
+  if (LDSGlobals.empty()) {
+    LLVM_DEBUG(dbgs() << "No LDS globals defined in the module " << M.getName()
+                      << ", skipping handling device of scope shared variables"
+                      << "\n");
+    return false;
+  }
+
+  // Collect all the amdgpu kernels defined within the current module
+  SetVector<const Function *> Kernels;
+  for (Function &F : M.functions()) {
+    if ((F.getCallingConv() == CallingConv::AMDGPU_KERNEL) &&
+        !F.isDeclaration())
+      Kernels.insert(&F);
+  }
+
+  if (Kernels.empty()) {
+    LLVM_DEBUG(dbgs() << "No kernels defined in the module " << M.getName()
+                      << ", skipping handling of device scope shared variables"
+                      << "\n");
+    return false;
+  }
+
+  return handleDeviceScopeSharedVariables(M, LDSGlobals, Kernels);
+}
+
+bool AMDGPUDeviceScopeSharedVariable::runOnModule(Module &M) {
+  LLVM_DEBUG(dbgs() << "===== Handling device scope shared variables in the "
+                       "module "
+                    << M.getName() << "\n");
+
+  // TODO: We only want to handle HIP kernels, and no kernels from from other
+  // programming languages, like OpenCL, OpenMP, etc. Do we need to add a
+  // condition here for it, and skip running the pass for non-HIP kernels?
+  if (skipModule(M)) {
+    LLVM_DEBUG(dbgs() << "Skipping handling of device scope shared variables "
+                         "in the module "
+                      << M.getName() << "\n");
+    return false;
+  }
+
+  bool Changed = handleDeviceScopeSharedVariables(M);
+
+  LLVM_DEBUG(dbgs() << "===== Done with hanlding device scope shared variables "
+                       "in the module "
+                    << M.getName() << "\n");
+
+  return Changed;
+}

llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.h

protected:
 
   StringRef getGPUName(const Function &F) const;
   StringRef getFeatureString(const Function &F) const;
 
 public:
   static bool EnableLateStructurizeCFG;
   static bool EnableFunctionCalls;
   static bool EnableFixedFunctionABI;
+  static bool EnableDeviceScopeSharedVariable;
 
   AMDGPUTargetMachine(const Target &T, const Triple &TT, StringRef CPU,
                       StringRef FS, TargetOptions Options,
                       Optional<Reloc::Model> RM, Optional<CodeModel::Model> CM,
                       CodeGenOpt::Level OL);
   ~AMDGPUTargetMachine() override;
 
   const TargetSubtargetInfo *getSubtargetImpl() const;

llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp

static cl::opt<bool> EnableScalarIRPasses(
   cl::init(true),
   cl::Hidden);
 
 static cl::opt<bool> EnableStructurizerWorkarounds(
     "amdgpu-enable-structurizer-workarounds",
     cl::desc("Enable workarounds for the StructurizeCFG pass"), cl::init(true),
     cl::Hidden);
 
+static cl::opt<bool, true> EnableDeviceScopeSharedVariable(
+    "amdgpu-enable-device-scope-shared-variable",
+    cl::desc("Support amdgpu device scope shared variables"),
+    cl::location(AMDGPUTargetMachine::EnableDeviceScopeSharedVariable),
+    cl::init(false), cl::Hidden);
+
 extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAMDGPUTarget() {
   // Register the target
   RegisterTargetMachine<R600TargetMachine> X(getTheAMDGPUTarget());
   RegisterTargetMachine<GCNTargetMachine> Y(getTheGCNTarget());
 
   PassRegistry *PR = PassRegistry::getPassRegistry();
   initializeR600ClauseMergePassPass(*PR);
   initializeR600ControlFlowFinalizerPass(*PR);
extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAMDGPUTarget() {
   initializeAMDGPUExternalAAWrapperPass(*PR);
   initializeAMDGPUUseNativeCallsPass(*PR);
   initializeAMDGPUSimplifyLibCallsPass(*PR);
   initializeAMDGPUInlinerPass(*PR);
   initializeAMDGPUPrintfRuntimeBindingPass(*PR);
   initializeGCNRegBankReassignPass(*PR);
   initializeGCNNSAReassignPass(*PR);
   initializeSIAddIMGInitPass(*PR);
+  initializeAMDGPUDeviceScopeSharedVariablePass(*PR);
 }
 
 static std::unique_ptr<TargetLoweringObjectFile> createTLOF(const Triple &TT) {
   return std::make_unique<AMDGPUTargetObjectFile>();
 }
 
 static ScheduleDAGInstrs *createR600MachineScheduler(MachineSchedContext *C) {
   return new ScheduleDAGMILive(C, std::make_unique<R600SchedStrategy>());
if (TT.getArch() == Triple::amdgcn) {
     else if (getMCSubtargetInfo()->checkFeatures("+wavefrontsize32"))
       MRI.reset(llvm::createGCNMCRegisterInfo(AMDGPUDwarfFlavour::Wave32));
   }
 }
 
 bool AMDGPUTargetMachine::EnableLateStructurizeCFG = false;
 bool AMDGPUTargetMachine::EnableFunctionCalls = false;
 bool AMDGPUTargetMachine::EnableFixedFunctionABI = false;
+bool AMDGPUTargetMachine::EnableDeviceScopeSharedVariable = false;
 
 AMDGPUTargetMachine::~AMDGPUTargetMachine() = default;
 
 StringRef AMDGPUTargetMachine::getGPUName(const Function &F) const {
   Attribute GPUAttr = F.getFnAttribute("target-cpu");
   return GPUAttr.isValid() ? GPUAttr.getValueAsString() : getTargetCPU();
 }
 
void AMDGPUPassConfig::addIRPasses() {
   // and
   //
   //   %0 = shl nsw %a, 2
   //   %1 = shl %a, 2
   //
   // but EarlyCSE can do neither of them.
   if (getOptLevel() != CodeGenOpt::None && EnableScalarIRPasses)
     addEarlyCSEOrGVNPass();
+
+  // We expect to run this pass as a last IR pass. Hence make sure that this
+  // pass is added as a last IR pass
+  if (EnableDeviceScopeSharedVariable)
+    addPass(createAMDGPUDeviceScopeSharedVariablePass());
 }
 
 void AMDGPUPassConfig::addCodeGenPrepare() {
   if (TM->getTargetTriple().getArch() == Triple::amdgcn)
     addPass(createAMDGPUAnnotateKernelFeaturesPass());
 
   if (TM->getTargetTriple().getArch() == Triple::amdgcn &&
       EnableLowerKernelArguments)

llvm/lib/Target/AMDGPU/CMakeLists.txt

add_llvm_target(AMDGPUCodeGen
   AMDGPUAlwaysInlinePass.cpp
   AMDGPUAnnotateKernelFeatures.cpp
   AMDGPUAnnotateUniformValues.cpp
   AMDGPUArgumentUsageInfo.cpp
   AMDGPUAsmPrinter.cpp
   AMDGPUAtomicOptimizer.cpp
   AMDGPUCallLowering.cpp
   AMDGPUCodeGenPrepare.cpp
+  AMDGPUDeviceScopeSharedVariable.cpp
   AMDGPUExportClustering.cpp
   AMDGPUFixFunctionBitcasts.cpp
   AMDGPUFrameLowering.cpp
   AMDGPUHSAMetadataStreamer.cpp
   AMDGPUInstCombineIntrinsic.cpp
   AMDGPUInstrInfo.cpp
   AMDGPUInstructionSelector.cpp
   AMDGPUISelDAGToDAG.cpp