[AMDGPU] Support for device scope shared variables

[hsmhsm]

Summary

WIP, this patch is uploaded to save the current state of the work,
and is not meant for review at the moment.

[hsmhsm]

Implemented few more necessary data structures

[hsmhsm]

Collect all associated LDS globals associated with a kernel.

[hsmhsm]

Fix bug and compute offset for each LDS

[hsmhsm]

Completed the handling of direct LDS globals within kernel

[hsmhsm]

Partial support for indirect LDS globals.

[hsmhsm]

Code refactoring as per the new logic of processing.

[hsmhsm]

Support for indirect LDS globals.

[hsmhsm]

First bug fix.

[hsmhsm]

Fixes previous incorrect commit.

[hsmhsm]

First cut of clode clean-up

[hsmhsm]

Support for replacing constant expressions [in progress].

[hsmhsm]

Successfully tested simple sum-reduce HIP application.

[hsmhsm]

Succesfully tested an HIP application which has more than one LDS globals.

[hsmhsm]

Successfully tested an HIP application which defines different shared
variables of different types within different device functions, and there
exist call graph path from kernel to all these device functions.

[hsmhsm]

Handle higher dimensional arrays.

[hsmhsm]

Updated introductory comment.

[hsmhsm]

Fixed minor bug in the updated code in the last commit.

[hsmhsm]

Make sure to run this pass as first AMDGPU IR pass.

[hsmhsm]

Retain original name for new cloned functions.

[hsmhsm]

Add lit test-cases.

[hsmhsm]

This revision was created just for tracking the progress of the work. Now that the code is ready for actual review, I created a new official revision - https://reviews.llvm.org/D91516, and hence closing this revision.

llvm/lib/Target/AMDGPU/AMDGPU.h

 extern char &AMDGPUOpenCLEnqueuedBlockLoweringID;
 
 void initializeGCNRegBankReassignPass(PassRegistry &);
 extern char &GCNRegBankReassignID;
 
 void initializeGCNNSAReassignPass(PassRegistry &);
 extern char &GCNNSAReassignID;
 
+ModulePass *createAMDGPUDeviceScopeSharedVariablePass();
+void initializeAMDGPUDeviceScopeSharedVariablePass(PassRegistry &);
+extern char &AMDGPUDeviceScopeSharedVariableID;
+
 namespace AMDGPU {
 enum TargetIndex {
   TI_CONSTDATA_START,
   TI_SCRATCH_RSRC_DWORD0,
   TI_SCRATCH_RSRC_DWORD1,
   TI_SCRATCH_RSRC_DWORD2,
   TI_SCRATCH_RSRC_DWORD3
 };

llvm/lib/Target/AMDGPU/AMDGPUAlwaysInlinePass.cpp

bool AMDGPUAlwaysInline::runOnModule(Module &M) {
   // objects defined in functions. LDS is always allocated by a kernel, and it
   // is difficult to manage LDS usage if a function may be used by multiple
   // kernels.
   //
   // OpenCL doesn't allow declaring LDS in non-kernels, so in practice this
   // should only appear when IPO passes manages to move LDs defined in a kernel
   // into a single user function.
 
+  if (!AMDGPUTargetMachine::EnableDeviceScopeSharedVariable) {
     for (GlobalVariable &GV : M.globals()) {
       // TODO: Region address
       unsigned AS = GV.getAddressSpace();
       if (AS != AMDGPUAS::LOCAL_ADDRESS && AS != AMDGPUAS::REGION_ADDRESS)
         continue;
 
       recursivelyVisitUsers(GV, FuncsToAlwaysInline);
     }
+  }
 
   if (!AMDGPUTargetMachine::EnableFunctionCalls || StressCalls) {
     auto IncompatAttr
       = StressCalls ? Attribute::AlwaysInline : Attribute::NoInline;
 
     for (Function &F : M) {
       if (!F.isDeclaration() && !F.use_empty() &&
           !F.hasFnAttribute(IncompatAttr)) {

llvm/lib/Target/AMDGPU/AMDGPUDeviceScopeSharedVariable.cpp

+//===-- AMDGPUDeviceScopeSharedVariables.cpp ------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// TODO:
+//
+//===----------------------------------------------------------------------===//
+
+#include "AMDGPU.h"
+#include "llvm/ADT/SetVector.h"
+#include "llvm/ADT/SmallPtrSet.h"
+#include "llvm/Analysis/CallGraph.h"
+#include "llvm/CodeGen/TargetPassConfig.h"
+#include "llvm/IR/IRBuilder.h"
+#include "llvm/IR/Module.h"
+#include "llvm/IR/ValueMap.h"
+#include "llvm/InitializePasses.h"
+#include "llvm/Transforms/Utils/Cloning.h"
+#include <llvm/ADT/SmallVector.h>
+#include <map>
+#include <set>
+
+#define DEBUG_TYPE "amdgpu-device-scope-shared-variable"
+
+using namespace llvm;
+
+namespace {
+
+class AMDGPUDeviceScopeSharedVariable : public ModulePass {
+public:
+  static char ID;
+
+  AMDGPUDeviceScopeSharedVariable() : ModulePass(ID) {
+    initializeAMDGPUDeviceScopeSharedVariablePass(
+        *PassRegistry::getPassRegistry());
+  }
+
+  bool runOnModule(Module &M) override;
+};
+
+} // namespace
+
+char AMDGPUDeviceScopeSharedVariable::ID = 0;
+
+char &llvm::AMDGPUDeviceScopeSharedVariableID =
+    AMDGPUDeviceScopeSharedVariable::ID;
+
+ModulePass *llvm::createAMDGPUDeviceScopeSharedVariablePass() {
+  return new AMDGPUDeviceScopeSharedVariable();
+}
+
+INITIALIZE_PASS_BEGIN(AMDGPUDeviceScopeSharedVariable,
+                      "implement-amdgpu-device-scope-shared-variable",
+                      "Implement AMDPGU Device Scope Shared Variable",
+                      false /*only look at the cfg*/, false /*analysis pass*/)
+INITIALIZE_PASS_DEPENDENCY(AMDGPUAlwaysInline)
+INITIALIZE_PASS_DEPENDENCY(SimpleInliner)
+INITIALIZE_PASS_END(AMDGPUDeviceScopeSharedVariable,
+                    "implement-amdgpu-device-scope-shared-variable",
+                    "Implement AMDPGU Device Scope Shared Variable",
+                    false /*only look at the cfg*/, false /*analysis pass*/)
+
+static void updateKernelToCallieList(
+    ValueMap<Function *, std::set<Function *>> &KernelToCallie,
+    ValueMap<Function *, Function *> &OldCallieToNewCallie) {
+  for (auto it = KernelToCallie.begin(); it != KernelToCallie.end(); ++it) {
+    Function *K = it->first;
+    std::set<Function *> OldCallieList = it->second;
+    std::set<Function *> NewCallieList;
+    for (Function *OldCallie : OldCallieList)
+      if (OldCallieToNewCallie.find(OldCallie) != OldCallieToNewCallie.end())
+        NewCallieList.insert(OldCallieToNewCallie[OldCallie]);
+    KernelToCallie[K] = NewCallieList;
+  }
+}
+
+static void
+updateLDSToFunctionMap(ValueMap<GlobalVariable *, Function *> &LDSToFunction,
+                       ValueMap<Function *, Function *> &OldCallieToNewCallie) {
+  for (auto it = LDSToFunction.begin(); it != LDSToFunction.end(); ++it) {
+    GlobalVariable *LDS = it->first;
+    Function *OldF = it->second;
+    if (OldCallieToNewCallie.find(OldF) != OldCallieToNewCallie.end())
+      LDSToFunction[LDS] = OldCallieToNewCallie[OldF];
+  }
+}
+
+static void createFunctionToLDSMap(
+    ValueMap<GlobalVariable *, Function *> &LDSToFunction,
+    ValueMap<Function *, std::set<GlobalVariable *>> &FunctionToLDS) {
+  for (auto it = LDSToFunction.begin(); it != LDSToFunction.end(); ++it) {
+    GlobalVariable *LDSGlobal = it->first;
+    Function *F = it->second;
+    auto rit = FunctionToLDS.find(F);
+    if (rit == FunctionToLDS.end()) {
+      std::set<GlobalVariable *> LDSSet;
+      LDSSet.insert(LDSGlobal);
+      FunctionToLDS[F] = LDSSet;
+    } else
+      FunctionToLDS[F].insert(LDSGlobal);
+  }
+}
+
+static void pairUpKernelWithLDSList(
+    Function *K, ValueMap<Function *, std::set<Function *>> &KernelToCallie,
+    ValueMap<Function *, std::set<GlobalVariable *>> &FunctionToLDS,
+    ValueMap<Function *, SetVector<GlobalVariable *>> &KernelToDirectLDS,
+    ValueMap<Function *, SetVector<GlobalVariable *>> &KernelToIndirectLDS) {
+  // If direct LDS globals exist within the kernel, collect it
+  if (FunctionToLDS.find(K) != FunctionToLDS.end()) {
+    std::set<GlobalVariable *> LDSList = FunctionToLDS[K];
+    SetVector<GlobalVariable *> DirectLDSList;
+    for (GlobalVariable *LDS : LDSList)
+      DirectLDSList.insert(LDS);
+    KernelToDirectLDS[K] = DirectLDSList;
+  }
+
+  // Collect all the indirect LDS globals defined within the callie(s) of the
+  // kernel
+  SetVector<GlobalVariable *> IndirectLDSSet;
+  std::set<Function *> Callies = KernelToCallie[K];
+  for (Function *Callie : Callies) {
+    if (FunctionToLDS.find(Callie) == FunctionToLDS.end())
+      continue;
+    std::set<GlobalVariable *> CallieLDSList = FunctionToLDS[Callie];
+    for (GlobalVariable *CallieLDS : CallieLDSList)
+      IndirectLDSSet.insert(CallieLDS);
+  }
+  if (!IndirectLDSSet.empty())
+    KernelToIndirectLDS[K] = IndirectLDSSet;
+}
+
+static void pairUpKernelWithCallieList(
+    Module &M, Function *K,
+    ValueMap<Function *, std::set<GlobalVariable *>> &FunctionToLDS,
+    ValueMap<Function *, std::set<Function *>> &KernelToCallie) {
+  // Get the call graph node associated with current kernel, traverse the call
+  // graph associated with it in DFS manner and collect all the associated
+  // callies which define LDS global(s)
+  CallGraph CG = CallGraph(M);
+  CallGraphNode *KernCGNode = CG[K];
+  SmallVector<CallGraphNode *, 16> CGNodeStack;
+  SetVector<CallGraphNode *> Visited;
+
+#ifndef NDEBUG
+  assert(KernCGNode && "Call graph node associated with kernel definition "
+                       "cannot be null\n");
+#endif
+
+  for (auto it = KernCGNode->begin(); it != KernCGNode->end(); ++it) {
+    CallGraphNode *CGN = it->second;
+#ifndef NDEBUG
+    assert(CGN && "Call graph node associated with function definition cannot"
+                  " be null\n");
+#endif
+    CGNodeStack.push_back(CGN);
+  }
+
+  std::set<Function *> CallieSet;
+  while (!CGNodeStack.empty()) {
+    CallGraphNode *CGNode = CGNodeStack.pop_back_val();
+    if (!Visited.insert(CGNode))
+      continue;
+
+    Function *F = CGNode->getFunction();
+    if (!F || F->isDeclaration()) {
+#ifndef NDEBUG
+      assert(CGNode->empty() && "Call graph node associated with function "
+                                "declaration should not have callie list\n");
+#endif
+      continue;
+    }
+
+    if (FunctionToLDS.find(F) != FunctionToLDS.end())
+      CallieSet.insert(F);
+
+    for (auto it = CGNode->begin(); it != CGNode->end(); ++it) {
+      CallGraphNode *CGN = it->second;
+#ifndef NDEBUG
+      assert(CGN && "Call graph node associated with function definition cannot"
+                    " be null\n");
+#endif
+      CGNodeStack.push_back(CGN);
+    }
+  }
+
+  KernelToCallie[K] = CallieSet;
+}
+
+static void pairUpLDSGlobalWithItsAssociatedFunction(
+    GlobalVariable *LDSGlobal,
+    ValueMap<GlobalVariable *, Function *> &LDSToFunction) {
+  // Recursively visit the user list of current LDS global, and find the
+  // enclosing function where the LDS global is defined, and the enclosing
+  // function should always be successfully found.
+  //
+  // TODO: Is there any other efficient way to find the enclosing functions of
+  // LDS globals?
+#ifndef NDEBUG
+  assert(!LDSGlobal->user_empty() &&
+         "LDS Global user list cannot be empty since it must have been defined "
+         "within either kernel or device function");
+#endif
+  SmallVector<User *, 16> UserStack;
+  SetVector<User *> Visited;
+
+  for (User *U : LDSGlobal->users())
+    UserStack.push_back(U);
+
+  while (!UserStack.empty()) {
+    User *U = UserStack.pop_back_val();
+    if (!Visited.insert(U))
+      continue;
+
+    if (Instruction *I = dyn_cast<Instruction>(U)) {
+      Function *F = I->getParent()->getParent();
+      if (F) {
+        LDSToFunction[LDSGlobal] = F;
+        return;
+      }
+      continue;
+    }
+
+    for (User *UU : U->users())
+      UserStack.push_back(UU);
+  }
+#ifndef NDEBUG
+  assert(false && "Control is not expected to reach this point");
+#endif
+}
+
+static void
+getLDSGlobalSizeInBytes(Module &M, GlobalVariable *LDSGlobal,
+                        ValueMap<GlobalVariable *, uint64_t> &LDSToSize) {
+  Type *Ty = LDSGlobal->getValueType();
+  const DataLayout &DL = M.getDataLayout();
+  uint64_t SizeInBytes = DL.getTypeSizeInBits(Ty).getFixedSize() / 8;
+  LDSToSize[LDSGlobal] = SizeInBytes;
+}
+
+static void
+eraseOldCallies(ValueMap<Function *, Function *> &OldCallieToNewCallie) {
+  // TODO: May be we can come-up with a more efficient implmentation erase old
+  // callies from the module. It depends on how many callies that we land-up
+  // erasing in a real world hip application. May be not many, hence, as of now,
+  // we have just employed a simplest method of repeatedly visiting the old
+  // callies and removing each of them, once their number of uses become 0.
+  bool Loopover = true;
+  while (Loopover) {
+    Loopover = false;
+    for (auto fit = OldCallieToNewCallie.begin();
+         fit != OldCallieToNewCallie.end(); ++fit) {
+      Function *OldCallie = fit->first;
+      if (OldCallie->hasNUsesOrMore(1))
+        continue;
+
+      OldCallieToNewCallie.erase(fit);
+      OldCallie->eraseFromParent();
+      Loopover = true;
+    }
+  }
+}
+
+static void updateFunctionAssociatedWithLDS(GlobalVariable *LDS, Value *BasePtr,
+                                            Function *F) {
+  // Suffix the names of the instructions with unique integer values
+  static int Suffix = 0;
+  ++Suffix;
+
+  // Keep track of all the erased to be instructions.
+  SetVector<Instruction *> ToBeErasedInsts;
+
+  // Traverse through each `use` of `LDS`, create a proper `ToBeReplacedInst`
+  // for each `use`, and accordingly replace it.
+  for (const User *U : LDS->users()) {
+    Instruction *UserInst = dyn_cast<Instruction>(const_cast<User *>(U));
+    if (!UserInst || UserInst->getParent()->getParent() != F)
+      continue;
+
+    Instruction *ToBeReplacedInst = nullptr;
+
+    if (GetElementPtrInst *GEPInst = dyn_cast<GetElementPtrInst>(UserInst)) {
+      // User instruction is GEP instruction, replace it as below
+      // 1. Extract the last operand of `GEPInst`, say, it is, `Offset`
+      // 2. Create pointer arithmetic instruction `BasePtr + Offset`
+#ifndef NDEBUG
+      assert(GEPInst->hasIndices() && "Expected one or more GEP indecies\n");
+#endif
+      Value *Offset = GEPInst->getOperand(GEPInst->getNumIndices());
+      ToBeReplacedInst = GetElementPtrInst::CreateInBounds(
+          GEPInst->getResultElementType(), BasePtr, Offset,
+          Twine(BasePtr->getName()) + Twine(".ptr.arith.") + Twine(Suffix),
+          UserInst);
+    } else if (LoadInst *LInst = dyn_cast<LoadInst>(UserInst)) {
+      // User instruction is LOAD instruction, replace pointer operand of
+      // LOAD instruction by `BasePtr`
+      ToBeReplacedInst = new LoadInst(LInst->getType(), BasePtr,
+                                      Twine(BasePtr->getName()) +
+                                          Twine(".load.") + Twine(Suffix),
+                                      UserInst);
+    } else if (StoreInst *SInst = dyn_cast<StoreInst>(UserInst)) {
+      // User instruction is STORE instruction, replace pointer operand of
+      // STORE instruction by `BasePtr`
+      ToBeReplacedInst =
+          new StoreInst(SInst->getValueOperand(), BasePtr, UserInst);
+    } else {
+      // TODO: Do we need to specially handle any other kind of instructions
+      // apart from GEP, LOAD, and STORE?
+#ifndef NDEBUG
+      assert(false && "Not implemented\n");
+#endif
+    }
+
+    // Replace `UserInst` by `ToBeReplacedInst` and erase `UserInst`.
+#ifndef NDEBUG
+    assert(ToBeReplacedInst && "To be replaced instruction cannot be null\n");
+#endif
+    ToBeReplacedInst->copyMetadata(*UserInst);
+    UserInst->replaceAllUsesWith(ToBeReplacedInst);
+  }
+
+  // Erase all the instructions which are got replaced by new ones.
+  for (Instruction *I : ToBeErasedInsts)
+    I->eraseFromParent();
+}
+
+static void getNewArgumentList(
+    GlobalVariable *LDS,
+    std::map<Function *, std::map<GlobalVariable *, Instruction *>>
+        &KernelToIndirectBasePtrInst,
+    Function *Caller, CallInst *CI, SmallVectorImpl<Value *> &NewArgs) {
+  Value *NewArg = nullptr;
+
+  if (Caller->getCallingConv() == CallingConv::AMDGPU_KERNEL) {
+    std::map<GlobalVariable *, Instruction *> LDSToIndirectBasePtrInst =
+        KernelToIndirectBasePtrInst[Caller];
+    NewArg = LDSToIndirectBasePtrInst[LDS];
+  } else
+    NewArg = Caller->getArg(Caller->arg_size() - 1);
+
+  for (auto it = CI->arg_begin(); it != CI->arg_end(); ++it)
+    NewArgs.push_back(*it);
+  NewArgs.push_back(NewArg);
+}
+
+static bool
+isNewClonedFunction(Function *F,
+                    ValueMap<Function *, Function *> &OldCallieToNewCallie) {
+  for (auto it = OldCallieToNewCallie.begin(); it != OldCallieToNewCallie.end();
+       ++it) {
+    Function *NF = it->second;
+    if (F == NF)
+      return true;
+  }
+  return false;
+}
+
+static void
+updateCallSites(GlobalVariable *LDS, Function *OldCallie, Function *NewCallie,
+                std::map<Function *, std::map<GlobalVariable *, Instruction *>>
+                    &KernelToIndirectBasePtrInst,
+                ValueMap<Function *, Function *> &OldCallieToNewCallie) {
+  // Update call sites for current callie.
+  for (User *U : OldCallie->users()) {
+    // Get call instruction.
+    CallInst *CI = dyn_cast<CallInst>(U);
+#ifndef NDEBUG
+    assert(CI && "Valid call instruction expected");
+#endif
+
+    // We are only interested in the call sites within kernel or within new
+    // cloned functions.
+    Function *Caller = CI->getParent()->getParent();
+    if (Caller->getCallingConv() != CallingConv::AMDGPU_KERNEL &&
+        !isNewClonedFunction(Caller, OldCallieToNewCallie))
+      continue;
+
+    // Get new argument list which can be used to insert new call instruction.
+    SmallVector<Value *, 8> NewArgs;
+    getNewArgumentList(LDS, KernelToIndirectBasePtrInst, Caller, CI, NewArgs);
+
+    // Insert new call instruction `NewCI` just before the existing call
+    // instruction `CI`.
+    CallInst *NewCI = CallInst::Create(NewCallie->getFunctionType(), NewCallie,
+                                       NewArgs, Twine(""), CI);
+    // TODO: Why copyMetadata() not copying meta data. I see metadat associated
+    // with CI, but it is not copied to NewCI. CI->hasMetadata() is false, why?
+    NewCI->copyMetadata(*CI);
+    NewCI->setTailCall(CI->isTailCall());
+    NewCI->setCallingConv(CI->getCallingConv());
+
+    // Now, since new updated call instruction is in place, delete old one.
+    CI->replaceAllUsesWith(NewCI);
+    CI->eraseFromParent();
+  }
+}
+
+static void
+updateCallSites(GlobalVariable *LDS,
+                std::map<Function *, std::map<GlobalVariable *, Instruction *>>
+                    &KernelToIndirectBasePtrInst,
+                ValueMap<Function *, Function *> &OldCallieToNewCallie) {
+  // Update call sites for all callies.
+  for (auto it = OldCallieToNewCallie.begin(); it != OldCallieToNewCallie.end();
+       ++it) {
+    Function *OldCallie = it->first;
+    Function *NewCallie = it->second;
+    updateCallSites(LDS, OldCallie, NewCallie, KernelToIndirectBasePtrInst,
+                    OldCallieToNewCallie);
+  }
+}
+
+static Function *cloneCallie(Module &M, Type *NewParamType, Function *Callie) {
+  // Create a new function type by adding `NewParamType` to the end of existing
+  // parameter list.
+  SmallVector<Type *, 8> NewParams;
+  FunctionType *FnTy = Callie->getFunctionType();
+  for (auto it = FnTy->param_begin(); it != FnTy->param_end(); ++it)
+    NewParams.push_back(*it);
+  NewParams.push_back(NewParamType);
+  FunctionType *NewFnTy =
+      FunctionType::get(FnTy->getReturnType(), NewParams, FnTy->isVarArg());
+
+  // Create a copy of the `Callie`, but with new function type
+  Function *NewCallie =
+      Function::Create(NewFnTy, Callie->getLinkage(), Callie->getAddressSpace(),
+                       Callie->getName() + Twine(".c"));
+
+  // TODO: what does this map required for?
+  ValueToValueMapTy VMap;
+  auto *NewCallieArgIt = NewCallie->arg_begin();
+  for (auto &Arg : Callie->args()) {
+    auto ArgName = Arg.getName();
+    NewCallieArgIt->setName(ArgName);
+    VMap[&Arg] = &(*NewCallieArgIt++);
+  }
+
+  // TODO: ModuleLevelChanges should be set to true or false?
+  SmallVector<ReturnInst *, 8> Returns;
+  CloneFunctionInto(NewCallie, Callie, VMap, /*ModuleLevelChanges=*/false,
+                    Returns);
+
+  // Copy all metadata
+  SmallVector<std::pair<unsigned, MDNode *>, 1> MDs;
+  Callie->getAllMetadata(MDs);
+  for (auto MDIt : MDs)
+    NewCallie->addMetadata(MDIt.first, *MDIt.second);
+
+  // Insert `NewCallie` just before `Callie` within the module.
+  M.getFunctionList().insert(Callie->getIterator(), NewCallie);
+
+  return NewCallie;
+}
+
+static void CollectCallGraphPathsBetweenKernelAndCallie(
+    Module &M, Function *K, Function *Callie,
+    SmallVectorImpl<SetVector<Function *>> &CGPaths) {
+  // Traverse the call graph associated with the kernel in DFS manner and
+  // collect all the paths from kernel to callie.
+  //
+  // TODO: Note that this algorithm will not work if there exist recursive
+  // calls, and the current assumption here is that the call graph is acyclic.
+  // We need to visit it back again to handle call graph which could contain
+  // cycles.
+  CallGraph CG = CallGraph(M);
+  CallGraphNode *KernCGNode = CG[K];
+#ifndef NDEBUG
+  assert(KernCGNode && "Call graph node associated with kernel definition "
+                       "cannot be null\n");
+#endif
+
+  SmallVector<SetVector<CallGraphNode *>, 8> Stack;
+  SetVector<CallGraphNode *> Path;
+  Path.insert(KernCGNode);
+  Stack.push_back(Path);
+
+  while (!Stack.empty()) {
+    SetVector<CallGraphNode *> TopPath = Stack.pop_back_val();
+    CallGraphNode *CGNode = TopPath.back();
+    Function *F = CGNode->getFunction();
+    if (F == Callie) {
+      SetVector<Function *> FPath;
+      for (CallGraphNode *CGN : TopPath)
+        FPath.insert(CGN->getFunction());
+      CGPaths.push_back(FPath);
+      continue;
+    }
+
+    for (auto it = CGNode->begin(); it != CGNode->end(); ++it) {
+      CallGraphNode *CGN = it->second;
+#ifndef NDEBUG
+      assert(CGN && "Call graph node associated with function definition cannot"
+                    " be null\n");
+#endif
+      SetVector<CallGraphNode *> ClonedPath(TopPath.begin(), TopPath.end());
+      ClonedPath.insert(CGN);
+      Stack.push_back(ClonedPath);
+    }
+  }
+}
+
+static void createCloneOfCalliesWithNewParameter(
+    Module &M, GlobalVariable *LDS, ValueMap<Function *, Function *> &KToC,
+    Type *BasePtrType, ValueMap<Function *, Function *> &OldCallieToNewCallie) {
+  // Update callies to accept the new parameter which is of type `BasePtrType`
+  // by creating their clones. Here is the brief sketch of the functionality of
+  // this function:
+  //
+  // 1. Collect all the call graph paths between the kernels and the callies
+  // 2. Traverse all the call graph paths from kernels to callies.
+  // 3. For each device function encoutered while traversing, create a clone of
+  //    it, by adding a new parameter of type BasePtr`s type to it's parameter
+  //    list, but also retain the original device function for a moment.
+  SmallVector<SetVector<Function *>, 8> CGPaths;
+  for (auto it = KToC.begin(); it != KToC.end(); ++it)
+    CollectCallGraphPathsBetweenKernelAndCallie(M, it->first, it->second,
+                                                CGPaths);
+
+  for (auto CGPath : CGPaths) {
+    // TODO: We can in-fact assert that length of `CGPath` is atleast two. But,
+    // we are okay for now.
+    if (CGPath.size() < 2)
+      continue;
+
+    // We are interested in cloning of only device functions in the call graph
+    // path, hence we start from second node in the call graph path.
+    auto it = CGPath.begin() + 1;
+    for (; it != CGPath.end(); ++it) {
+      Function *CurCallie = *it;
+
+      if (OldCallieToNewCallie.find(CurCallie) != OldCallieToNewCallie.end())
+        continue;
+
+      // The `CurCallie` is encountered first time for the LDS in question,
+      // Create a clone of it, and save it.
+      Function *NewCallie = cloneCallie(M, BasePtrType, CurCallie);
+      OldCallieToNewCallie[CurCallie] = NewCallie;
+    }
+  }
+}
+
+static Type *getBasePtrAccessInstType(
+    GlobalVariable *LDS,
+    std::map<Function *, std::map<GlobalVariable *, Instruction *>>
+        &KernelToIndirectBasePtrInst) {
+  for (auto it = KernelToIndirectBasePtrInst.begin();
+       it != KernelToIndirectBasePtrInst.end(); ++it)
+    for (auto bit = it->second.begin(); bit != it->second.end(); ++bit)
+      if (LDS == bit->first)
+        return bit->second->getType();
+
+#ifndef NDEBUG
+  assert(false && "Control is not expected to reach this point");
+#endif
+  return nullptr;
+}
+
+static void
+getKernelToCallieMap(GlobalVariable *LDS,
+                     ValueMap<GlobalVariable *, Function *> &LDSToFunction,
+                     ValueMap<Function *, std::set<Function *>> &KernelToCallie,
+                     ValueMap<Function *, Function *> &KToC) {
+  // Collect all call graph paths from kernels to end-callies which are
+  // associated with current LDS.
+  Function *Callie = LDSToFunction[LDS];
+  for (auto kit = KernelToCallie.begin(); kit != KernelToCallie.end(); ++kit) {
+    Function *K = kit->first;
+    std::set<Function *> Callies = kit->second;
+    if (Callies.find(Callie) != Callies.end())
+      KToC[K] = Callie;
+  }
+}
+
+static bool handleIndirectLDSGlobals(
+    Module &M, ValueMap<GlobalVariable *, Function *> &LDSToFunction,
+    ValueMap<Function *, std::set<GlobalVariable *>> &FunctionToLDS,
+    ValueMap<Function *, std::set<Function *>> &KernelToCallie,
+    ValueMap<Function *, SetVector<GlobalVariable *>> &KernelToIndirectLDS,
+    std::map<Function *, std::map<GlobalVariable *, Instruction *>>
+        &KernelToIndirectBasePtrInst) {
+  // Construct a grand list of all indirect LDS globals.
+  SetVector<GlobalVariable *> GrandLDSList;
+  for (auto it = KernelToIndirectLDS.begin(); it != KernelToIndirectLDS.end();
+       ++it)
+    for (GlobalVariable *LDS : it->second)
+      GrandLDSList.insert(LDS);
+
+  // Process each indirect LDS global one by one.
+  for (GlobalVariable *LDS : GrandLDSList) {
+    // For the current indirect LDS, pair-up kernel and end-callie.
+    ValueMap<Function *, Function *> KToC;
+    getKernelToCallieMap(LDS, LDSToFunction, KernelToCallie, KToC);
+
+    // For the current indirect LDS, get the base pointer access instruction
+    // type.
+    Type *BasePtrType =
+        getBasePtrAccessInstType(LDS, KernelToIndirectBasePtrInst);
+
+    // TODO
+    ValueMap<Function *, Function *> OldCallieToNewCallie;
+    createCloneOfCalliesWithNewParameter(M, LDS, KToC, BasePtrType,
+                                         OldCallieToNewCallie);
+
+    // TODO
+    updateLDSToFunctionMap(LDSToFunction, OldCallieToNewCallie);
+    updateKernelToCallieList(KernelToCallie, OldCallieToNewCallie);
+
+    // TODO
+    updateCallSites(LDS, KernelToIndirectBasePtrInst, OldCallieToNewCallie);
+
+    // TODO
+    eraseOldCallies(OldCallieToNewCallie);
+
+#ifndef NDEBUG
+    assert(OldCallieToNewCallie.empty() &&
+           "None of the old callies should alive by now\n");
+#endif
+
+    // TODO
+    Function *F = LDSToFunction[LDS];
+    updateFunctionAssociatedWithLDS(LDS, F->getArg(F->arg_size() - 1), F);
+  }
+
+  return true;
+}
+
+static bool handleDirectLDSGlobals(
+    std::map<Function *, std::map<GlobalVariable *, Instruction *>>
+        &KernelToDirectBasePtrInst) {
+  // Go through each kernel one by one, and handle all the direct globals
+  // defined within each of them.
+  for (auto it = KernelToDirectBasePtrInst.begin();
+       it != KernelToDirectBasePtrInst.end(); ++it) {
+    Function *K = it->first;
+    std::map<GlobalVariable *, Instruction *> DirectLDSToBasePtrInst =
+        it->second;
+
+    for (auto cit = DirectLDSToBasePtrInst.begin();
+         cit != DirectLDSToBasePtrInst.end(); ++cit) {
+      GlobalVariable *LDS = cit->first;
+      Instruction *BasePtr = cit->second;
+      updateFunctionAssociatedWithLDS(LDS, BasePtr, K);
+    }
+  }
+
+  return true;
+}
+
+static Instruction *insertBasePointerAccessInstructionsWithinKernel(
+    Module &M, Function *K, GlobalVariable *LDS, GlobalVariable *NewLDS,
+    uint64_t Offset) {
+  // Insert instructions as below at the begining of the entry basic block of
+  // the kernel
+  // 1. Insert GEP instruction which access the address `NewLDS + Offset`, say,
+  //    result is `GEPInst` which is of type `char*`.
+  // 2. Insert type cast instruction which type casts `GEPInst` from `char*` to
+  //    `basetype*` where `basetype` is base type of `LDS`, say the result is,
+  //    `CastInst`.
+  // 3. Return `CastInst`.
+
+  // Suffix the names of the instructions with unique integer values
+  static int Suffix = 0;
+  ++Suffix;
+
+  // Insert gep instruction
+  BasicBlock::const_iterator iit = K->getEntryBlock().getFirstInsertionPt();
+#ifndef NDEBUG
+  assert(iit != K->getEntryBlock().end() &&
+         "Entry basic block of the kernel cannot be empty, otherwise control "
+         "would not reach this point\n");
+#endif
+  const Instruction &EI = *iit;
+  Value *Indices[] = {Constant::getNullValue(Type::getInt32Ty(M.getContext())),
+                      Constant::getIntegerValue(
+                          Type::getInt64Ty(M.getContext()), APInt(64, Offset))};
+  Instruction *GEPInst = GetElementPtrInst::CreateInBounds(
+      NewLDS->getValueType(), const_cast<GlobalVariable *>(NewLDS), Indices,
+      Twine("dssv.gep.") + Twine(Suffix), const_cast<Instruction *>(&EI));
+
+  // Insert type-cast instruction
+  // TODO: Do we need to handle any other aggregate type apart from array type?
+  //       and, what about 2 and other higher dimensional arrays?
+  Type *LDSValueType = LDS->getValueType();
+  if (ArrayType *AT = dyn_cast<ArrayType>(LDSValueType))
+    LDSValueType = AT->getElementType();
+  PointerType *ToBeCastedType =
+      PointerType::get(LDSValueType, AMDGPUAS::LOCAL_ADDRESS);
+  Instruction *CastInst = new BitCastInst(GEPInst, ToBeCastedType,
+                                          Twine("dssv.cast.") + Twine(Suffix),
+                                          const_cast<Instruction *>(&EI));
+
+  // Return type-casted instruction
+  return CastInst;
+}
+
+static GlobalVariable *
+createSingleContiguousLayout(Module &M, Function *K,
+                             uint64_t &TotalLDSSizeInBytes) {
+  // Insert a new LDS global which is nothing but a single contigeous shared
+  // memory layout representing all the LDS globals associted with the kernel
+  // which includes those directly defined within the kernel and those
+  // indirectly defined within callies.
+  //
+  // The size of this new contigeous LDS global layout is equal to the sum of
+  // the sizes of all the associated LDS globals.
+  // TODO: what about the name of this new LDS global? is it fine or need to be
+  // changed?
+  Type *NewLDSTy =
+      ArrayType::get(IntegerType::get(M.getContext(), 8), TotalLDSSizeInBytes);
+  GlobalVariable *NewLDS = new GlobalVariable(
+      M, NewLDSTy, false, GlobalValue::InternalLinkage,
+      UndefValue::get(NewLDSTy), Twine(K->getName()) + Twine("_LDSLayout"),
+      nullptr, GlobalVariable::NotThreadLocal, AMDGPUAS::LOCAL_ADDRESS);
+  NewLDS->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
+  NewLDS->setAlignment(MaybeAlign(M.getDataLayout().getPreferredAlign(NewLDS)));
+
+  return NewLDS;
+}
+
+static void
+computeTotalLDSSizeInBytes(ValueMap<GlobalVariable *, uint64_t> &LDSToSize,
+                           SetVector<GlobalVariable *> &DirectLDSList,
+                           SetVector<GlobalVariable *> &IndirectLDSList,
+                           ValueMap<GlobalVariable *, uint64_t> &LDSToOffset,
+                           uint64_t &TotalLDSSizeInBytes) {
+  // Compute the total size of all LDS globals, and also offsets associated with
+  // them within in the new LDS global.
+  TotalLDSSizeInBytes = 0;
+  for (GlobalVariable *LDS : DirectLDSList) {
+    LDSToOffset[LDS] = TotalLDSSizeInBytes;
+    TotalLDSSizeInBytes += LDSToSize[LDS];
+  }
+  for (GlobalVariable *LDS : IndirectLDSList) {
+    LDSToOffset[LDS] = TotalLDSSizeInBytes;
+    TotalLDSSizeInBytes += LDSToSize[LDS];
+  }
+}
+
+static bool prepareKernelsForHandlingLDSGlobals(
+    Module &M, SetVector<Function *> &Kernels,
+    ValueMap<Function *, SetVector<GlobalVariable *>> &KernelToDirectLDS,
+    ValueMap<Function *, SetVector<GlobalVariable *>> &KernelToIndirectLDS,
+    ValueMap<GlobalVariable *, uint64_t> &LDSToSize,
+    std::map<Function *, std::map<GlobalVariable *, Instruction *>>
+        &KernelToDirectBasePtrInst,
+    std::map<Function *, std::map<GlobalVariable *, Instruction *>>
+        &KernelToIndirectBasePtrInst) {
+  bool Change = false;
+
+  // For each LDS global, insert base pointer access instruction within
+  // associated kernel(s).
+  for (Function *K : Kernels) {
+    // Copy both direct and indirect LDS list for current kernel.
+    SetVector<GlobalVariable *> DirectLDSList;
+    SetVector<GlobalVariable *> IndirectLDSList;
+    if (KernelToDirectLDS.find(K) != KernelToDirectLDS.end())
+      DirectLDSList = KernelToDirectLDS[K];
+    if (KernelToIndirectLDS.find(K) != KernelToIndirectLDS.end())
+      IndirectLDSList = KernelToIndirectLDS[K];
+
+    // No LDS globals to process? ignore the kernel, goto next kernel.
+    if (DirectLDSList.empty() && IndirectLDSList.empty())
+      continue;
+
+    // We are going process LDS globals atleast for one kernel, and hence, we
+    // are going to make module level changes.
+    Change = true;
+
+    // Create a single contigeous LDS latout for current kernel
+    uint64_t TotalLDSSizeInBytes;
+    ValueMap<GlobalVariable *, uint64_t> LDSToOffset;
+    computeTotalLDSSizeInBytes(LDSToSize, DirectLDSList, IndirectLDSList,
+                               LDSToOffset, TotalLDSSizeInBytes);
+    GlobalVariable *NewLDS =
+        createSingleContiguousLayout(M, K, TotalLDSSizeInBytes);
+
+    // For each LDS global (both direct and indirect ones), insert base pointer
+    // access instruction within kernel.
+    std::map<GlobalVariable *, Instruction *> DirectLDSToBasePtrInst;
+    std::map<GlobalVariable *, Instruction *> IndirectLDSToBasePtrInst;
+    for (GlobalVariable *LDS : DirectLDSList) {
+      Instruction *BasePtr = insertBasePointerAccessInstructionsWithinKernel(
+          M, K, LDS, NewLDS, LDSToOffset[LDS]);
+      DirectLDSToBasePtrInst[LDS] = BasePtr;
+    }
+    for (GlobalVariable *LDS : IndirectLDSList) {
+      Instruction *BasePtr = insertBasePointerAccessInstructionsWithinKernel(
+          M, K, LDS, NewLDS, LDSToOffset[LDS]);
+      IndirectLDSToBasePtrInst[LDS] = BasePtr;
+    }
+
+    KernelToDirectBasePtrInst[K] = DirectLDSToBasePtrInst;
+    KernelToIndirectBasePtrInst[K] = IndirectLDSToBasePtrInst;
+  }
+
+  return Change;
+}
+
+static bool handleDeviceScopeSharedVariables(
+    Module &M, SetVector<Function *> &Kernels,
+    ValueMap<GlobalVariable *, Function *> &LDSToFunction,
+    ValueMap<Function *, std::set<GlobalVariable *>> &FunctionToLDS,
+    ValueMap<Function *, std::set<Function *>> &KernelToCallie,
+    ValueMap<Function *, SetVector<GlobalVariable *>> &KernelToDirectLDS,
+    ValueMap<Function *, SetVector<GlobalVariable *>> &KernelToIndirectLDS,
+    ValueMap<GlobalVariable *, uint64_t> &LDSToSize) {
+  bool Change = false;
+  std::map<Function *, std::map<GlobalVariable *, Instruction *>>
+      KernelToDirectBasePtrInst;
+  std::map<Function *, std::map<GlobalVariable *, Instruction *>>
+      KernelToIndirectBasePtrInst;
+
+  // 1. Create a single contigeous LDS global layout for each kernel.
+  // 2. Compute base pointer offset for each LDS within the above single
+  //    contigeous LDS global layout, and insert it within the associated
+  //    kernel(s).
+  Change = prepareKernelsForHandlingLDSGlobals(
+      M, Kernels, KernelToDirectLDS, KernelToIndirectLDS, LDSToSize,
+      KernelToDirectBasePtrInst, KernelToIndirectBasePtrInst);
+
+  // None of the kernel has any LDS globals (direct and/or indirect ones)
+  // associated with them. Nothing to do, no changes being made to module.
+  if (!Change)
+    return false;
+
+  // Handle all indirect LDS globals defined within device functions.
+  if (!KernelToIndirectLDS.empty())
+    Change = handleIndirectLDSGlobals(M, LDSToFunction, FunctionToLDS,
+                                      KernelToCallie, KernelToIndirectLDS,
+                                      KernelToIndirectBasePtrInst);
+
+  // Handle all direct LDS globals defined within kernels.
+  if (!KernelToDirectLDS.empty())
+    Change = handleDirectLDSGlobals(KernelToDirectBasePtrInst);
+
+  return Change;
+}
+
+static bool
+handleDeviceScopeSharedVariables(Module &M,
+                                 SetVector<GlobalVariable *> &LDSGlobals,
+                                 SetVector<Function *> &Kernels) {
+  // Pair up each LDS global with the enclosing function where the LDS global is
+  // defined
+  ValueMap<GlobalVariable *, Function *> LDSToFunction;
+  for (GlobalVariable *LDSGlobal : LDSGlobals)
+    pairUpLDSGlobalWithItsAssociatedFunction(LDSGlobal, LDSToFunction);
+
+  // Create reverse map from enclosing function to LDS global list
+  ValueMap<Function *, std::set<GlobalVariable *>> FunctionToLDS;
+  createFunctionToLDSMap(LDSToFunction, FunctionToLDS);
+
+  // Pair up kernels with callie list which define LDS globals
+  ValueMap<Function *, std::set<Function *>> KernelToCallie;
+  for (Function *K : Kernels)
+    pairUpKernelWithCallieList(M, K, FunctionToLDS, KernelToCallie);
+
+  // Pair up kernels with all the LDS globals: both direct LDS globals (those
+  // directly defined within the kernels), and indirect LDS globals (those
+  // indirectly defined within the callies).
+  ValueMap<Function *, SetVector<GlobalVariable *>> KernelToDirectLDS;
+  ValueMap<Function *, SetVector<GlobalVariable *>> KernelToIndirectLDS;
+  for (Function *K : Kernels)
+    pairUpKernelWithLDSList(K, KernelToCallie, FunctionToLDS, KernelToDirectLDS,
+                            KernelToIndirectLDS);
+
+  // Get the size of each LDS global in bytes
+  ValueMap<GlobalVariable *, uint64_t> LDSToSize;
+  for (GlobalVariable *LDSGlobal : LDSGlobals)
+    getLDSGlobalSizeInBytes(M, LDSGlobal, LDSToSize);
+
+  return handleDeviceScopeSharedVariables(
+      M, Kernels, LDSToFunction, FunctionToLDS, KernelToCallie,
+      KernelToDirectLDS, KernelToIndirectLDS, LDSToSize);
+}
+
+static bool handleDeviceScopeSharedVariables(Module &M) {
+  // Collect all the (static) LDS globals defined within the current module
+  SetVector<GlobalVariable *> LDSGlobals;
+  for (GlobalVariable &GV : M.globals())
+    if (GV.getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS &&
+        GV.hasInternalLinkage())
+      LDSGlobals.insert(&GV);
+
+  if (LDSGlobals.empty()) {
+    LLVM_DEBUG(dbgs() << "No LDS globals defined in the module " << M.getName()
+                      << ", skipping handling device of scope shared variables"
+                      << "\n");
+    return false;
+  }
+
+  // Collect all the amdgpu kernels defined within the current module
+  SetVector<Function *> Kernels;
+  for (Function &F : M.functions()) {
+    if ((F.getCallingConv() == CallingConv::AMDGPU_KERNEL) &&
+        !F.isDeclaration())
+      Kernels.insert(&F);
+  }
+
+  if (Kernels.empty()) {
+    LLVM_DEBUG(dbgs() << "No kernels defined in the module " << M.getName()
+                      << ", skipping handling of device scope shared variables"
+                      << "\n");
+    return false;
+  }
+
+  return handleDeviceScopeSharedVariables(M, LDSGlobals, Kernels);
+}
+
+bool AMDGPUDeviceScopeSharedVariable::runOnModule(Module &M) {
+  LLVM_DEBUG(dbgs() << "===== Handling device scope shared variables in the "
+                       "module "
+                    << M.getName() << "\n");
+
+  // TODO: We only want to handle HIP kernels, and no kernels from from other
+  // programming languages, like OpenCL, OpenMP, etc. Do we need to add a
+  // condition here for it, and skip running the pass for non-HIP kernels?
+  if (skipModule(M)) {
+    LLVM_DEBUG(dbgs() << "Skipping handling of device scope shared variables "
+                         "in the module "
+                      << M.getName() << "\n");
+    return false;
+  }
+
+  bool Changed = handleDeviceScopeSharedVariables(M);
+
+  LLVM_DEBUG(dbgs() << "===== Done with hanlding device scope shared variables "
+                       "in the module "
+                    << M.getName() << "\n");
+
+  return Changed;
+}

llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.h

protected:
 
   StringRef getGPUName(const Function &F) const;
   StringRef getFeatureString(const Function &F) const;
 
 public:
   static bool EnableLateStructurizeCFG;
   static bool EnableFunctionCalls;
   static bool EnableFixedFunctionABI;
+  static bool EnableDeviceScopeSharedVariable;
 
   AMDGPUTargetMachine(const Target &T, const Triple &TT, StringRef CPU,
                       StringRef FS, TargetOptions Options,
                       Optional<Reloc::Model> RM, Optional<CodeModel::Model> CM,
                       CodeGenOpt::Level OL);
   ~AMDGPUTargetMachine() override;
 
   const TargetSubtargetInfo *getSubtargetImpl() const;

llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp

static cl::opt<bool> EnableScalarIRPasses(
   cl::init(true),
   cl::Hidden);
 
 static cl::opt<bool> EnableStructurizerWorkarounds(
     "amdgpu-enable-structurizer-workarounds",
     cl::desc("Enable workarounds for the StructurizeCFG pass"), cl::init(true),
     cl::Hidden);
 
+static cl::opt<bool, true> EnableDeviceScopeSharedVariable(
+    "amdgpu-enable-device-scope-shared-variable",
+    cl::desc("Support amdgpu device scope shared variables"),
+    cl::location(AMDGPUTargetMachine::EnableDeviceScopeSharedVariable),
+    cl::init(false), cl::Hidden);
+
 extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAMDGPUTarget() {
   // Register the target
   RegisterTargetMachine<R600TargetMachine> X(getTheAMDGPUTarget());
   RegisterTargetMachine<GCNTargetMachine> Y(getTheGCNTarget());
 
   PassRegistry *PR = PassRegistry::getPassRegistry();
   initializeR600ClauseMergePassPass(*PR);
   initializeR600ControlFlowFinalizerPass(*PR);
extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAMDGPUTarget() {
   initializeAMDGPUExternalAAWrapperPass(*PR);
   initializeAMDGPUUseNativeCallsPass(*PR);
   initializeAMDGPUSimplifyLibCallsPass(*PR);
   initializeAMDGPUInlinerPass(*PR);
   initializeAMDGPUPrintfRuntimeBindingPass(*PR);
   initializeGCNRegBankReassignPass(*PR);
   initializeGCNNSAReassignPass(*PR);
   initializeSIAddIMGInitPass(*PR);
+  initializeAMDGPUDeviceScopeSharedVariablePass(*PR);
 }
 
 static std::unique_ptr<TargetLoweringObjectFile> createTLOF(const Triple &TT) {
   return std::make_unique<AMDGPUTargetObjectFile>();
 }
 
 static ScheduleDAGInstrs *createR600MachineScheduler(MachineSchedContext *C) {
   return new ScheduleDAGMILive(C, std::make_unique<R600SchedStrategy>());
if (TT.getArch() == Triple::amdgcn) {
     else if (getMCSubtargetInfo()->checkFeatures("+wavefrontsize32"))
       MRI.reset(llvm::createGCNMCRegisterInfo(AMDGPUDwarfFlavour::Wave32));
   }
 }
 
 bool AMDGPUTargetMachine::EnableLateStructurizeCFG = false;
 bool AMDGPUTargetMachine::EnableFunctionCalls = false;
 bool AMDGPUTargetMachine::EnableFixedFunctionABI = false;
+bool AMDGPUTargetMachine::EnableDeviceScopeSharedVariable = false;
 
 AMDGPUTargetMachine::~AMDGPUTargetMachine() = default;
 
 StringRef AMDGPUTargetMachine::getGPUName(const Function &F) const {
   Attribute GPUAttr = F.getFnAttribute("target-cpu");
   return GPUAttr.isValid() ? GPUAttr.getValueAsString() : getTargetCPU();
 }
 
void AMDGPUPassConfig::addIRPasses() {
   // and
   //
   //   %0 = shl nsw %a, 2
   //   %1 = shl %a, 2
   //
   // but EarlyCSE can do neither of them.
   if (getOptLevel() != CodeGenOpt::None && EnableScalarIRPasses)
     addEarlyCSEOrGVNPass();
+
+  // We expect to run this pass as a last IR pass. Hence make sure that this
+  // pass is added as a last IR pass
+  if (EnableDeviceScopeSharedVariable)
+    addPass(createAMDGPUDeviceScopeSharedVariablePass());
 }
 
 void AMDGPUPassConfig::addCodeGenPrepare() {
   if (TM->getTargetTriple().getArch() == Triple::amdgcn)
     addPass(createAMDGPUAnnotateKernelFeaturesPass());
 
   if (TM->getTargetTriple().getArch() == Triple::amdgcn &&
       EnableLowerKernelArguments)

llvm/lib/Target/AMDGPU/CMakeLists.txt

add_llvm_target(AMDGPUCodeGen
   AMDGPUAlwaysInlinePass.cpp
   AMDGPUAnnotateKernelFeatures.cpp
   AMDGPUAnnotateUniformValues.cpp
   AMDGPUArgumentUsageInfo.cpp
   AMDGPUAsmPrinter.cpp
   AMDGPUAtomicOptimizer.cpp
   AMDGPUCallLowering.cpp
   AMDGPUCodeGenPrepare.cpp
+  AMDGPUDeviceScopeSharedVariable.cpp
   AMDGPUExportClustering.cpp
   AMDGPUFixFunctionBitcasts.cpp
   AMDGPUFrameLowering.cpp
   AMDGPUHSAMetadataStreamer.cpp
   AMDGPUInstCombineIntrinsic.cpp
   AMDGPUInstrInfo.cpp
   AMDGPUInstructionSelector.cpp
   AMDGPUISelDAGToDAG.cpp