[AMDGPU] Add address space based alias analysis pass

[rampitec]

Summary

This is direct port of HSAILAliasAnalysis pass, just cleaned for style and renamed.

[vpykhtin]

lib/Target/AMDGPU/AMDGPUAliasAnalysis.cpp

64

the check can be done using xor

T1->isVectorTy() ^ T2->isVectorTy()

[vpykhtin]

LGTM.

[arsenm]

lib/Target/AMDGPU/AMDGPUAliasAnalysis.cpp

2

Missing comment

14

doesn't match pass name

16–18

Should move this to the TargetMachine when adding it

45

flat and constant may alias, this isn't the OpenCL rules

59–60

Use cast<> if not checking it

72

formatting

79

Missin space

lib/Target/AMDGPU/AMDGPUAliasAnalysis.h

2–3

Missing comment

[arsenm]

lib/Target/AMDGPU/AMDGPUAliasAnalysis.cpp

76–82

We should probably be marking all constant loads in the frontend with invariant metadata.

84–102

I think BasicAA will handle all of this, we should only need to handle the address spaces

test/CodeGen/AMDGPU/vectorize-global-local.ll

16–23

Should run instnamer on this

[rampitec]

lib/Target/AMDGPU/AMDGPUAliasAnalysis.cpp

84–102

It was not handled really, that's why it got here...

[rampitec]

Addressed comments.

[arsenm]

lib/Target/AMDGPU/AMDGPUAliasAnalysis.cpp

84–102

I think this is only correct to do if F is a kernel

[rampitec]

lib/Target/AMDGPU/AMDGPUAliasAnalysis.cpp

84–102

You probably right, that is safer. I will change it. BTW, I have checked this method in BasicAA, it does not check argument attributes.

[rampitec]

Added check if function is a kernel before assuming constant argument memory.

[rampitec]

lib/Target/AMDGPU/AMDGPUAliasAnalysis.cpp

45

I think in the future as an extension module metadata can be checked to see if source was OpenCL and return NoAlias for flat and constant.

[arsenm]

LGTM

lib/Target/AMDGPU/AMDGPUAliasAnalysis.cpp

45

I think we really just need to mark constant accesses with invariant. Anything checking the raw language is broken

78

This is redundant with the assert getPointerAddressSpace will do

lib/Target/AMDGPU/AMDGPU.h

 extern char &SIAnnotateControlFlowPassID;
 
 void initializeSIDebuggerInsertNopsPass(PassRegistry&);
 extern char &SIDebuggerInsertNopsID;
 
 void initializeSIInsertWaitsPass(PassRegistry&);
 extern char &SIInsertWaitsID;
 
+ImmutablePass *createAMDGPUAAWrapperPass();
+void initializeAMDGPUAAWrapperPassPass(PassRegistry&);
+
 Target &getTheAMDGPUTarget();
 Target &getTheGCNTarget();
 
 namespace AMDGPU {
 enum TargetIndex {
   TI_CONSTDATA_START,
   TI_SCRATCH_RSRC_DWORD0,
   TI_SCRATCH_RSRC_DWORD1,

lib/Target/AMDGPU/AMDGPUAliasAnalysis.h

+//===- AMDGPUAliasAnalysis ---------------------------------------*- C++ -*-==//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+/// \file
+/// This is the AMGPU address space based alias analysis pass.
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_ANALYSIS_AMDGPUALIASANALYSIS_H
+#define LLVM_ANALYSIS_AMDGPUALIASANALYSIS_H
+
+#include "llvm/Analysis/AliasAnalysis.h"
+#include "llvm/IR/Function.h"
+#include "llvm/IR/Module.h"
+#include "llvm/Pass.h"
+
+namespace llvm {
+
+/// A simple AA result that uses TBAA metadata to answer queries.
+class AMDGPUAAResult : public AAResultBase<AMDGPUAAResult> {
+  friend AAResultBase<AMDGPUAAResult>;
+
+  const DataLayout &DL;
+
+public:
+  explicit AMDGPUAAResult(const DataLayout &DL) : AAResultBase(), DL(DL) {}
+  AMDGPUAAResult(AMDGPUAAResult &&Arg)
+      : AAResultBase(std::move(Arg)), DL(Arg.DL){}
+
+  /// Handle invalidation events from the new pass manager.
+  ///
+  /// By definition, this result is stateless and so remains valid.
+  bool invalidate(Function &, const PreservedAnalyses &) { return false; }
+
+  AliasResult alias(const MemoryLocation &LocA, const MemoryLocation &LocB);
+  bool pointsToConstantMemory(const MemoryLocation &Loc, bool OrLocal);
+
+private:
+  bool Aliases(const MDNode *A, const MDNode *B) const;
+  bool PathAliases(const MDNode *A, const MDNode *B) const;
+};
+
+/// Analysis pass providing a never-invalidated alias analysis result.
+class AMDGPUAA : public AnalysisInfoMixin<AMDGPUAA> {
+  friend AnalysisInfoMixin<AMDGPUAA>;
+  static char PassID;
+
+public:
+  typedef AMDGPUAAResult Result;
+
+  AMDGPUAAResult run(Function &F, AnalysisManager<Function> &AM) {
+    return AMDGPUAAResult(F.getParent()->getDataLayout());
+  }
+};
+
+/// Legacy wrapper pass to provide the AMDGPUAAResult object.
+class AMDGPUAAWrapperPass : public ImmutablePass {
+  std::unique_ptr<AMDGPUAAResult> Result;
+
+public:
+  static char ID;
+
+  AMDGPUAAWrapperPass() : ImmutablePass(ID) {
+    initializeAMDGPUAAWrapperPassPass(*PassRegistry::getPassRegistry());
+  }
+
+  AMDGPUAAResult &getResult() { return *Result; }
+  const AMDGPUAAResult &getResult() const { return *Result; }
+
+  bool doInitialization(Module &M) override {
+    Result.reset(new AMDGPUAAResult(M.getDataLayout()));
+    return false;
+  }
+  bool doFinalization(Module &M) override {
+    Result.reset();
+    return false;
+  }
+  void getAnalysisUsage(AnalysisUsage &AU) const override;
+};
+
+}
+#endif // LLVM_ANALYSIS_AMDGPUALIASANALYSIS_H

lib/Target/AMDGPU/AMDGPUAliasAnalysis.cpp

+//===- AMDGPUAliasAnalysis ---------------------------------------*- C++ -*-==//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+/// \file
+/// This is the AMGPU address space based alias analysis pass.
+//===----------------------------------------------------------------------===//
+
+#include "AMDGPU.h"
+#include "AMDGPUAliasAnalysis.h"
+#include "llvm/Analysis/AliasAnalysis.h"
+#include "llvm/Analysis/ValueTracking.h"
+#include "llvm/Analysis/Passes.h"
+#include "llvm/Support/raw_ostream.h"
+#include "llvm/IR/Function.h"
+#include "llvm/IR/Module.h"
+#include "llvm/Pass.h"
+
+using namespace llvm;
+
+#define DEBUG_TYPE "amdgpu-aa"
+
+// Register this pass...
+char AMDGPUAAWrapperPass::ID = 0;
+INITIALIZE_PASS(AMDGPUAAWrapperPass, "amdgpu-aa",
+                "AMDGPU Address space based Alias Analysis", false, true)
+
+ImmutablePass *llvm::createAMDGPUAAWrapperPass() {
+  return new AMDGPUAAWrapperPass();
+}
+
+void AMDGPUAAWrapperPass::getAnalysisUsage(AnalysisUsage &AU) const {
+  AU.setPreservesAll();
+}
+
+AliasResult AMDGPUAAResult::alias(const MemoryLocation &LocA,
+                                  const MemoryLocation &LocB) {
+  // This array is indexed by the AMDGPUAS::AddressSpaces
+  // enum elements PRIVATE_ADDRESS ... to FLAT_ADDRESS
+  // see "llvm/Transforms/AMDSPIRUtils.h"
+  static const AliasResult ASAliasRules[5][5] = {
+ /*             Private    Global    Constant  Group     Flat */
+ /* Private  */ {MayAlias, NoAlias , NoAlias , NoAlias , MayAlias},
+ /* Global   */ {NoAlias , MayAlias, NoAlias , NoAlias , MayAlias},
+ /* Constant */ {NoAlias , NoAlias , MayAlias, NoAlias , MayAlias},
+ /* Group    */ {NoAlias , NoAlias , NoAlias , MayAlias, MayAlias},
+ /* Flat     */ {MayAlias, MayAlias, MayAlias, MayAlias, MayAlias}
+  };
+  unsigned asA = LocA.Ptr->getType()->getPointerAddressSpace();
+  unsigned asB = LocB.Ptr->getType()->getPointerAddressSpace();
+  if (asA > AMDGPUAS::AddressSpaces::FLAT_ADDRESS ||
+      asB > AMDGPUAS::AddressSpaces::FLAT_ADDRESS)
+    report_fatal_error("Pointer address space out of range");
+
+  AliasResult Result = ASAliasRules[asA][asB];
+  if (Result == NoAlias) return Result;
+
+  if (isa<Argument>(LocA.Ptr) && isa<Argument>(LocB.Ptr)) {
+    Type *T1 = cast<PointerType>(LocA.Ptr->getType())->getElementType();
+    Type *T2 = cast<PointerType>(LocB.Ptr->getType())->getElementType();
+
+    if ((T1->isVectorTy() && !T2->isVectorTy()) ||
+        (T2->isVectorTy() && !T1->isVectorTy()))
+      return NoAlias;
+  }
+  // Forward the query to the next alias analysis.
+  return AAResultBase::alias(LocA, LocB);
+}
+
+bool AMDGPUAAResult::pointsToConstantMemory(const MemoryLocation &Loc,
+                                            bool OrLocal) {
+  const Value *Base = GetUnderlyingObject(Loc.Ptr, DL);
+  assert(Base->getType()->isPointerTy());
+
+  if (Base->getType()->getPointerAddressSpace() ==
+     AMDGPUAS::AddressSpaces::CONSTANT_ADDRESS) {
+    return true;
+  }
+
+  if (const GlobalVariable *GV = dyn_cast<GlobalVariable>(Base)) {
+    if (GV->isConstant())
+      return true;
+  } else if (const Argument *Arg = dyn_cast<Argument>(Base)) {
+    const Function *F = Arg->getParent();
+    unsigned ArgNo = Arg->getArgNo();
+    /* On an argument, ReadOnly attribute indicates that the function does
+       not write through this pointer argument, even though it may write
+       to the memory that the pointer points to.
+       On an argument, ReadNone attribute indicates that the function does
+       not dereference that pointer argument, even though it may read or write
+       the memory that the pointer points to if accessed through other pointers.
+     */
+    if (F->getAttributes().hasAttribute(ArgNo + 1, Attribute::NoAlias) &&
+          (F->getAttributes().hasAttribute(ArgNo + 1, Attribute::ReadNone) ||
+           F->getAttributes().hasAttribute(ArgNo + 1, Attribute::ReadOnly))) {
+      return true;
+    }
+  }
+  return AAResultBase::pointsToConstantMemory(Loc, OrLocal);
+}

lib/Target/AMDGPU/AMDGPUTargetMachine.cpp

 /// \file
 /// \brief The AMDGPU target machine contains all of the hardware specific
 /// information  needed to emit code for R600 and SI GPUs.
 //
 //===----------------------------------------------------------------------===//
 
 #include "AMDGPUTargetMachine.h"
 #include "AMDGPU.h"
+#include "AMDGPUAliasAnalysis.h"
 #include "AMDGPUCallLowering.h"
 #include "AMDGPUInstructionSelector.h"
 #include "AMDGPULegalizerInfo.h"
 #ifdef LLVM_BUILD_GLOBAL_ISEL
 #include "AMDGPURegisterBankInfo.h"
 #endif
 #include "AMDGPUTargetObjectFile.h"
 #include "AMDGPUTargetTransformInfo.h"

 
 // Option to run internalize pass.
 static cl::opt<bool> InternalizeSymbols(
   "amdgpu-internalize-symbols",
   cl::desc("Enable elimination of non-kernel functions and unused globals"),
   cl::init(false),
   cl::Hidden);
 
+// Enable address space based alias analysis
+static cl::opt<bool> EnableAMDGPUAliasAnalysis("enable-amdgpu-aa", cl::Hidden,
+  cl::desc("Enable AMDGPU Alias Analysis"),
+  cl::init(true));
+
 extern "C" void LLVMInitializeAMDGPUTarget() {
   // Register the target
   RegisterTargetMachine<R600TargetMachine> X(getTheAMDGPUTarget());
   RegisterTargetMachine<GCNTargetMachine> Y(getTheGCNTarget());
 
   PassRegistry *PR = PassRegistry::getPassRegistry();
   initializeSILowerI1CopiesPass(*PR);
   initializeSIFixSGPRCopiesPass(*PR);
extern "C" void LLVMInitializeAMDGPUTarget() {
   initializeAMDGPUUnifyMetadataPass(*PR);
   initializeSIAnnotateControlFlowPass(*PR);
   initializeSIInsertWaitsPass(*PR);
   initializeSIWholeQuadModePass(*PR);
   initializeSILowerControlFlowPass(*PR);
   initializeSIInsertSkipsPass(*PR);
   initializeSIDebuggerInsertNopsPass(*PR);
   initializeSIOptimizeExecMaskingPass(*PR);
+  initializeAMDGPUAAWrapperPassPass(*PR);
 }
 
 static std::unique_ptr<TargetLoweringObjectFile> createTLOF(const Triple &TT) {
   return llvm::make_unique<AMDGPUTargetObjectFile>();
 }
 
 static ScheduleDAGInstrs *createR600MachineScheduler(MachineSchedContext *C) {
   return new ScheduleDAGMILive(C, llvm::make_unique<R600SchedStrategy>());
void AMDGPUPassConfig::addIRPasses() {
   if (TM.getOptLevel() > CodeGenOpt::None) {
     addPass(createInferAddressSpacesPass());
     addPass(createAMDGPUPromoteAlloca(&TM));
 
     if (EnableSROA)
       addPass(createSROAPass());
 
     addStraightLineScalarOptimizationPasses();
+
+    if (EnableAMDGPUAliasAnalysis) {
+      addPass(createAMDGPUAAWrapperPass());
+      addPass(createExternalAAWrapperPass([](Pass &P, Function &,
+                                             AAResults &AAR) {
+        if (auto *WrapperPass = P.getAnalysisIfAvailable<AMDGPUAAWrapperPass>())
+          AAR.addAAResult(WrapperPass->getResult());
+        }));
+    }
   }
 
   TargetPassConfig::addIRPasses();
 
   // EarlyCSE is not always strong enough to clean up what LSR produces. For
   // example, GVN can combine
   //
   //   %0 = add %a, %b

lib/Target/AMDGPU/CMakeLists.txt

 else()
   set(GLOBAL_ISEL_BUILD_FILES"")
   set(LLVM_OPTIONAL_SOURCES LLVMGlobalISel ${GLOBAL_ISEL_FILES})
 endif()
 
 
 add_llvm_target(AMDGPUCodeGen
   AMDILCFGStructurizer.cpp
+  AMDGPUAliasAnalysis.cpp
   AMDGPUAlwaysInlinePass.cpp
   AMDGPUAnnotateKernelFeatures.cpp
   AMDGPUAnnotateUniformValues.cpp
   AMDGPUAsmPrinter.cpp
   AMDGPUCodeGenPrepare.cpp
   AMDGPUFrameLowering.cpp
   AMDGPUTargetObjectFile.cpp
   AMDGPUIntrinsicInfo.cpp

test/CodeGen/AMDGPU/amdgpu.private-memory.ll

-; RUN: llc -show-mc-encoding -mattr=+promote-alloca -amdgpu-load-store-vectorizer=0 -verify-machineinstrs -march=amdgcn < %s | FileCheck %s -check-prefix=SI-PROMOTE -check-prefix=SI -check-prefix=FUNC
-; RUN: llc -show-mc-encoding -mattr=+promote-alloca -amdgpu-load-store-vectorizer=0 -verify-machineinstrs -mtriple=amdgcn--amdhsa -mcpu=kaveri -mattr=-unaligned-buffer-access < %s | FileCheck %s -check-prefix=SI-PROMOTE -check-prefix=SI -check-prefix=FUNC -check-prefix=HSA-PROMOTE
-; RUN: llc -show-mc-encoding -mattr=-promote-alloca -amdgpu-load-store-vectorizer=0 -verify-machineinstrs -march=amdgcn < %s | FileCheck %s -check-prefix=SI-ALLOCA -check-prefix=SI -check-prefix=FUNC
-; RUN: llc -show-mc-encoding -mattr=-promote-alloca -amdgpu-load-store-vectorizer=0 -verify-machineinstrs -mtriple=amdgcn-amdhsa -mcpu=kaveri -mattr=-unaligned-buffer-access < %s | FileCheck %s -check-prefix=SI-ALLOCA -check-prefix=SI -check-prefix=FUNC -check-prefix=HSA-ALLOCA
-; RUN: llc -show-mc-encoding -mattr=+promote-alloca -amdgpu-load-store-vectorizer=0 -verify-machineinstrs -mtriple=amdgcn-amdhsa -march=amdgcn -mcpu=tonga -mattr=-unaligned-buffer-access < %s | FileCheck %s -check-prefix=SI-PROMOTE -check-prefix=SI -check-prefix=FUNC
-; RUN: llc -show-mc-encoding -mattr=-promote-alloca -amdgpu-load-store-vectorizer=0 -verify-machineinstrs -mtriple=amdgcn-amdhsa -march=amdgcn -mcpu=tonga -mattr=-unaligned-buffer-access < %s | FileCheck %s -check-prefix=SI-ALLOCA -check-prefix=SI -check-prefix=FUNC
+; RUN: llc -show-mc-encoding -mattr=+promote-alloca -amdgpu-load-store-vectorizer=0 -enable-amdgpu-aa=0 -verify-machineinstrs -march=amdgcn < %s | FileCheck %s -check-prefix=SI-PROMOTE -check-prefix=SI -check-prefix=FUNC
+; RUN: llc -show-mc-encoding -mattr=+promote-alloca -amdgpu-load-store-vectorizer=0 -enable-amdgpu-aa=0 -verify-machineinstrs -mtriple=amdgcn--amdhsa -mcpu=kaveri -mattr=-unaligned-buffer-access < %s | FileCheck %s -check-prefix=SI-PROMOTE -check-prefix=SI -check-prefix=FUNC -check-prefix=HSA-PROMOTE
+; RUN: llc -show-mc-encoding -mattr=-promote-alloca -amdgpu-load-store-vectorizer=0 -enable-amdgpu-aa=0 -verify-machineinstrs -march=amdgcn < %s | FileCheck %s -check-prefix=SI-ALLOCA -check-prefix=SI -check-prefix=FUNC
+; RUN: llc -show-mc-encoding -mattr=-promote-alloca -amdgpu-load-store-vectorizer=0 -enable-amdgpu-aa=0 -verify-machineinstrs -mtriple=amdgcn-amdhsa -mcpu=kaveri -mattr=-unaligned-buffer-access < %s | FileCheck %s -check-prefix=SI-ALLOCA -check-prefix=SI -check-prefix=FUNC -check-prefix=HSA-ALLOCA
+; RUN: llc -show-mc-encoding -mattr=+promote-alloca -amdgpu-load-store-vectorizer=0 -enable-amdgpu-aa=0 -verify-machineinstrs -mtriple=amdgcn-amdhsa -march=amdgcn -mcpu=tonga -mattr=-unaligned-buffer-access < %s | FileCheck %s -check-prefix=SI-PROMOTE -check-prefix=SI -check-prefix=FUNC
+; RUN: llc -show-mc-encoding -mattr=-promote-alloca -amdgpu-load-store-vectorizer=0 -enable-amdgpu-aa=0 -verify-machineinstrs -mtriple=amdgcn-amdhsa -march=amdgcn -mcpu=tonga -mattr=-unaligned-buffer-access < %s | FileCheck %s -check-prefix=SI-ALLOCA -check-prefix=SI -check-prefix=FUNC
 
 ; RUN: opt -S -mtriple=amdgcn-unknown-amdhsa -mcpu=kaveri -amdgpu-promote-alloca < %s | FileCheck -check-prefix=HSAOPT -check-prefix=OPT %s
 ; RUN: opt -S -mtriple=amdgcn-unknown-unknown -mcpu=kaveri -amdgpu-promote-alloca < %s | FileCheck -check-prefix=NOHSAOPT -check-prefix=OPT %s
 
 ; RUN: llc -march=r600 -mcpu=cypress < %s | FileCheck %s -check-prefix=R600 -check-prefix=FUNC
 
 
 ; HSAOPT: @mova_same_clause.stack = internal unnamed_addr addrspace(3) global [256 x [5 x i32]] undef, align 4

test/CodeGen/AMDGPU/insert_vector_elt.v2i16.ll

-; RUN: llc -verify-machineinstrs -march=amdgcn -mcpu=gfx901 -mattr=+flat-for-global,-fp64-fp16-denormals < %s | FileCheck -check-prefix=GCN -check-prefix=GFX9 -check-prefix=GFX89 %s
-; RUN: llc -verify-machineinstrs -march=amdgcn -mcpu=fiji -mattr=+flat-for-global < %s | FileCheck -check-prefix=GCN -check-prefix=CIVI -check-prefix=VI -check-prefix=GFX89 %s
-; RUN: llc -verify-machineinstrs -march=amdgcn -mcpu=hawaii -mattr=+flat-for-global < %s | FileCheck -check-prefix=GCN -check-prefix=CIVI -check-prefix=CI %s
+; RUN: llc -verify-machineinstrs -march=amdgcn -mcpu=gfx901 -enable-amdgpu-aa=0 -mattr=+flat-for-global,-fp64-fp16-denormals < %s | FileCheck -check-prefix=GCN -check-prefix=GFX9 -check-prefix=GFX89 %s
+; RUN: llc -verify-machineinstrs -march=amdgcn -mcpu=fiji -enable-amdgpu-aa=0 -mattr=+flat-for-global < %s | FileCheck -check-prefix=GCN -check-prefix=CIVI -check-prefix=VI -check-prefix=GFX89 %s
+; RUN: llc -verify-machineinstrs -march=amdgcn -mcpu=hawaii -enable-amdgpu-aa=0 -mattr=+flat-for-global < %s | FileCheck -check-prefix=GCN -check-prefix=CIVI -check-prefix=CI %s
 
 ; GCN-LABEL: {{^}}s_insertelement_v2i16_0:
 ; GCN: s_load_dword [[VEC:s[0-9]+]]
 
 ; CIVI: s_and_b32 [[ELT1:s[0-9]+]], [[VEC]], 0xffff0000{{$}}
 ; CIVI: s_or_b32 s{{[0-9]+}}, [[ELT1]], 0x3e7{{$}}
 
 ; GFX9-NOT: lshr

test/CodeGen/AMDGPU/si-triv-disjoint-mem-access.ll

-; RUN: llc -march=amdgcn -mcpu=bonaire -verify-machineinstrs -enable-misched -enable-aa-sched-mi < %s | FileCheck -check-prefix=FUNC -check-prefix=CI %s
+; RUN: llc -march=amdgcn -mcpu=bonaire -enable-amdgpu-aa=0 -verify-machineinstrs -enable-misched -enable-aa-sched-mi < %s | FileCheck -check-prefix=FUNC -check-prefix=CI %s
 
 declare void @llvm.SI.tbuffer.store.i32(<16 x i8>, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32)
 declare void @llvm.SI.tbuffer.store.v4i32(<16 x i8>, <4 x i32>, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32)
 declare void @llvm.amdgcn.s.barrier() #1
 declare i32 @llvm.amdgcn.workitem.id.x() #2
 
 
 @stored_lds_ptr = addrspace(3) global i32 addrspace(3)* undef, align 4

test/CodeGen/AMDGPU/split-vector-memoperand-offsets.ll

-; RUN: llc -march=amdgcn -mcpu=hawaii -verify-machineinstrs -mattr=-promote-alloca,-load-store-opt < %s | FileCheck -check-prefix=GCN %s
+; RUN: llc -march=amdgcn -mcpu=hawaii -enable-amdgpu-aa=0 -verify-machineinstrs -mattr=-promote-alloca,-load-store-opt < %s | FileCheck -check-prefix=GCN %s
 
 @sPrivateStorage = internal addrspace(3) global [256 x [8 x <4 x i64>]] undef
 
 ; GCN-LABEL: {{^}}ds_reorder_vector_split:
 
 ; Write zeroinitializer
 ; GCN-DAG: ds_write_b64 [[PTR:v[0-9]+]], [[VAL:v\[[0-9]+:[0-9]+\]]] offset:24
 ; GCN-DAG: ds_write_b64 [[PTR]], [[VAL]] offset:16

test/CodeGen/AMDGPU/vectorize-global-local.ll

+; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck %s
+; CHECK-DAG: flat_load_dwordx4
+; CHECK-DAG: flat_load_dwordx4
+; CHECK-DAG: flat_load_dwordx4
+; CHECK-DAG: flat_load_dwordx4
+; CHECK-DAG: ds_write2_b32
+; CHECK-DAG: ds_write2_b32
+; CHECK-DAG: ds_write2_b32
+; CHECK-DAG: ds_write2_b32
+; CHECK-DAG: ds_write2_b32
+; CHECK-DAG: ds_write2_b32
+; CHECK-DAG: ds_write2_b32
+; CHECK-DAG: ds_write2_b32
+
+define void @vectorize_global_local(i32 addrspace(1)* nocapture readonly %arg, i32 addrspace(3)* nocapture %arg1) {
+bb:
+  %tmp = load i32, i32 addrspace(1)* %arg, align 4
+  store i32 %tmp, i32 addrspace(3)* %arg1, align 4
+  %tmp2 = getelementptr inbounds i32, i32 addrspace(1)* %arg, i64 1
+  %tmp3 = load i32, i32 addrspace(1)* %tmp2, align 4
+  %tmp4 = getelementptr inbounds i32, i32 addrspace(3)* %arg1, i32 1
+  store i32 %tmp3, i32 addrspace(3)* %tmp4, align 4
+  %tmp5 = getelementptr inbounds i32, i32 addrspace(1)* %arg, i64 2
+  %tmp6 = load i32, i32 addrspace(1)* %tmp5, align 4
+  %tmp7 = getelementptr inbounds i32, i32 addrspace(3)* %arg1, i32 2
+  store i32 %tmp6, i32 addrspace(3)* %tmp7, align 4
+  %tmp8 = getelementptr inbounds i32, i32 addrspace(1)* %arg, i64 3
+  %tmp9 = load i32, i32 addrspace(1)* %tmp8, align 4
+  %tmp10 = getelementptr inbounds i32, i32 addrspace(3)* %arg1, i32 3
+  store i32 %tmp9, i32 addrspace(3)* %tmp10, align 4
+  %tmp11 = getelementptr inbounds i32, i32 addrspace(1)* %arg, i64 4
+  %tmp12 = load i32, i32 addrspace(1)* %tmp11, align 4
+  %tmp13 = getelementptr inbounds i32, i32 addrspace(3)* %arg1, i32 4
+  store i32 %tmp12, i32 addrspace(3)* %tmp13, align 4
+  %tmp14 = getelementptr inbounds i32, i32 addrspace(1)* %arg, i64 5
+  %tmp15 = load i32, i32 addrspace(1)* %tmp14, align 4
+  %tmp16 = getelementptr inbounds i32, i32 addrspace(3)* %arg1, i32 5
+  store i32 %tmp15, i32 addrspace(3)* %tmp16, align 4
+  %tmp17 = getelementptr inbounds i32, i32 addrspace(1)* %arg, i64 6
+  %tmp18 = load i32, i32 addrspace(1)* %tmp17, align 4
+  %tmp19 = getelementptr inbounds i32, i32 addrspace(3)* %arg1, i32 6
+  store i32 %tmp18, i32 addrspace(3)* %tmp19, align 4
+  %tmp20 = getelementptr inbounds i32, i32 addrspace(1)* %arg, i64 7
+  %tmp21 = load i32, i32 addrspace(1)* %tmp20, align 4
+  %tmp22 = getelementptr inbounds i32, i32 addrspace(3)* %arg1, i32 7
+  store i32 %tmp21, i32 addrspace(3)* %tmp22, align 4
+  %tmp23 = getelementptr inbounds i32, i32 addrspace(1)* %arg, i64 8
+  %tmp24 = load i32, i32 addrspace(1)* %tmp23, align 4
+  %tmp25 = getelementptr inbounds i32, i32 addrspace(3)* %arg1, i32 8
+  store i32 %tmp24, i32 addrspace(3)* %tmp25, align 4
+  %tmp26 = getelementptr inbounds i32, i32 addrspace(1)* %arg, i64 9
+  %tmp27 = load i32, i32 addrspace(1)* %tmp26, align 4
+  %tmp28 = getelementptr inbounds i32, i32 addrspace(3)* %arg1, i32 9
+  store i32 %tmp27, i32 addrspace(3)* %tmp28, align 4
+  %tmp29 = getelementptr inbounds i32, i32 addrspace(1)* %arg, i64 10
+  %tmp30 = load i32, i32 addrspace(1)* %tmp29, align 4
+  %tmp31 = getelementptr inbounds i32, i32 addrspace(3)* %arg1, i32 10
+  store i32 %tmp30, i32 addrspace(3)* %tmp31, align 4
+  %tmp32 = getelementptr inbounds i32, i32 addrspace(1)* %arg, i64 11
+  %tmp33 = load i32, i32 addrspace(1)* %tmp32, align 4
+  %tmp34 = getelementptr inbounds i32, i32 addrspace(3)* %arg1, i32 11
+  store i32 %tmp33, i32 addrspace(3)* %tmp34, align 4
+  %tmp35 = getelementptr inbounds i32, i32 addrspace(1)* %arg, i64 12
+  %tmp36 = load i32, i32 addrspace(1)* %tmp35, align 4
+  %tmp37 = getelementptr inbounds i32, i32 addrspace(3)* %arg1, i32 12
+  store i32 %tmp36, i32 addrspace(3)* %tmp37, align 4
+  %tmp38 = getelementptr inbounds i32, i32 addrspace(1)* %arg, i64 13
+  %tmp39 = load i32, i32 addrspace(1)* %tmp38, align 4
+  %tmp40 = getelementptr inbounds i32, i32 addrspace(3)* %arg1, i32 13
+  store i32 %tmp39, i32 addrspace(3)* %tmp40, align 4
+  %tmp41 = getelementptr inbounds i32, i32 addrspace(1)* %arg, i64 14
+  %tmp42 = load i32, i32 addrspace(1)* %tmp41, align 4
+  %tmp43 = getelementptr inbounds i32, i32 addrspace(3)* %arg1, i32 14
+  store i32 %tmp42, i32 addrspace(3)* %tmp43, align 4
+  %tmp44 = getelementptr inbounds i32, i32 addrspace(1)* %arg, i64 15
+  %tmp45 = load i32, i32 addrspace(1)* %tmp44, align 4
+  %tmp46 = getelementptr inbounds i32, i32 addrspace(3)* %arg1, i32 15
+  store i32 %tmp45, i32 addrspace(3)* %tmp46, align 4
+  ret void
+}