[amdgpu] Enable use of AA during codegen.

[hliao]

Register pressure tests have to disable AMDGPU AA to pass the test; otherwise, the register pressure is reduced after using AA.

[rampitec]

LGTM, but please run PSDB first.

[hliao]

Add an option to turn on/off the use of AA during codegen.

[hliao]

Fix more regression tests due to the enhanced AMDGPU AA.

llvm/lib/Target/AMDGPU/AMDGPUSubtarget.h

public:
   // the pointer. Set this to something above the minimum to avoid needing
   // dynamic realignment in common cases.
   Align getStackAlignment() const { return Align(16); }
 
   bool enableMachineScheduler() const override {
     return true;
   }
 
+  bool useAA() const override;
+
   bool enableSubRegLiveness() const override {
     return true;
   }
 
   void setScalarizeGlobalBehavior(bool b) { ScalarizeGlobal = b; }
   bool getScalarizeGlobalBehavior() const { return ScalarizeGlobal; }
 
   // static wrappers

llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp

static cl::opt<bool> EnableVGPRIndexMode(
   cl::desc("Use GPR indexing mode instead of movrel for vector indexing"),
   cl::init(false));
 
 static cl::opt<bool> EnableFlatScratch(
   "amdgpu-enable-flat-scratch",
   cl::desc("Use flat scratch instructions"),
   cl::init(false));
 
+static cl::opt<bool> UseAA("amdgpu-use-aa-in-codegen",
+                           cl::desc("Enable the use of AA during codegen."),
+                           cl::init(true));
+
 GCNSubtarget::~GCNSubtarget() = default;
 
 R600Subtarget &
 R600Subtarget::initializeSubtargetDependencies(const Triple &TT,
                                                StringRef GPU, StringRef FS) {
   SmallString<256> FullFS("+promote-alloca,");
   FullFS += FS;
   ParseSubtargetFeatures(GPU, /*TuneCPU*/ GPU, FullFS);

 bool GCNSubtarget::hasMadF16() const {
   return InstrInfo.pseudoToMCOpcode(AMDGPU::V_MAD_F16) != -1;
 }
 
 bool GCNSubtarget::useVGPRIndexMode() const {
   return !hasMovrel() || (EnableVGPRIndexMode && hasVGPRIndexMode());
 }
 
+bool GCNSubtarget::useAA() const { return UseAA; }
+
 unsigned GCNSubtarget::getOccupancyWithNumSGPRs(unsigned SGPRs) const {
   if (getGeneration() >= AMDGPUSubtarget::GFX10)
     return getMaxWavesPerEU();
 
   if (getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS) {
     if (SGPRs <= 80)
       return 10;
     if (SGPRs <= 88)

llvm/test/CodeGen/AMDGPU/ds-combine-with-dependence.ll

 ; RUN: llc -march=amdgcn -mcpu=fiji -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN %s
 
 
-; There is no dependence between the store and the two loads. So we can combine the loads
-; and the combined load is at the original place of the second load.
+; There is no dependence between the store and the two loads. So we can combine
+; the loads and schedule it freely.
 
 ; GCN-LABEL: {{^}}ds_combine_nodep
 
-; GCN: ds_write2_b32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} offset0:26 offset1:27
-; GCN-NEXT: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset0:7 offset1:8
+; GCN-DAG: ds_write2_b32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} offset0:26 offset1:27
+; GCN-DAG: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset0:7 offset1:8
+; GCN: s_waitcnt lgkmcnt({{[0-9]+}})
 define amdgpu_kernel void @ds_combine_nodep(float addrspace(1)* %out, float addrspace(3)* %inptr) {
 
   %base = bitcast float addrspace(3)* %inptr to i8 addrspace(3)*
   %addr0 = getelementptr i8, i8 addrspace(3)* %base, i32 24
   %tmp0 = bitcast i8 addrspace(3)* %addr0 to float addrspace(3)*
   %vaddr0 = bitcast float addrspace(3)* %tmp0 to <3 x float> addrspace(3)*
   %load0 = load <3 x float>, <3 x float> addrspace(3)* %vaddr0, align 4
   %v0 = extractelement <3 x float> %load0, i32 2

llvm/test/CodeGen/AMDGPU/global_smrd.ll

bb:
   %tmp17 = getelementptr inbounds float, float addrspace(1)* %arg1, i32 %tmp
   store float %tmp16, float addrspace(1)* %tmp17, align 4, !tbaa !8
   ret void
 }
 
 
 ; uniform load dominated by no-alias store - scalarize
 ; CHECK-LABEL: @no_memdep_alias_arg
-; CHECK: flat_store_dword
-; CHECK: s_load_dword [[SVAL:s[0-9]+]]
+; CHECK: s_load_dwordx2 s{{\[}}[[IN_LO:[0-9]+]]:[[IN_HI:[0-9]+]]], s[4:5], 0x0
+; CHECK: s_load_dword [[SVAL:s[0-9]+]], s{{\[}}[[IN_LO]]:[[IN_HI]]], 0x0
 ; CHECK: v_mov_b32_e32 [[VVAL:v[0-9]+]], [[SVAL]]
 ; CHECK: flat_store_dword v[{{[0-9]+:[0-9]+}}], [[VVAL]]
 
 define amdgpu_kernel void @no_memdep_alias_arg(i32 addrspace(1)* noalias %in, [8 x i32], i32 addrspace(1)* %out0, [8 x i32], i32 addrspace(1)* %out1) {
   store i32 0, i32 addrspace(1)* %out0
   %val = load i32, i32 addrspace(1)* %in
   store i32 %val, i32 addrspace(1)* %out1
   ret void

llvm/test/CodeGen/AMDGPU/insert_vector_elt.v2i16.ll

 ; CI-NEXT:    s_load_dword s4, s[4:5], 0x4
 ; CI-NEXT:    v_lshlrev_b32_e32 v2, 3, v0
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    v_mov_b32_e32 v1, s3
 ; CI-NEXT:    v_add_i32_e32 v0, vcc, s2, v2
 ; CI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; CI-NEXT:    flat_load_dword v4, v[0:1]
 ; CI-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
-; CI-NEXT:    s_mov_b32 s2, 0xffff
 ; CI-NEXT:    s_mov_b32 s3, 0
+; CI-NEXT:    s_mov_b32 s2, 0xffff
 ; CI-NEXT:    v_mov_b32_e32 v3, s1
 ; CI-NEXT:    s_lshl_b32 s1, s4, 16
 ; CI-NEXT:    s_and_b32 s4, s4, s2
 ; CI-NEXT:    v_add_i32_e32 v2, vcc, s0, v2
 ; CI-NEXT:    s_or_b32 s0, s4, s1
 ; CI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
 ; CI-NEXT:    s_waitcnt vmcnt(1)
 ; CI-NEXT:    v_lshlrev_b32_e32 v4, 4, v4

llvm/test/CodeGen/AMDGPU/memcpy-fixed-align.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 < %s | FileCheck %s -check-prefix=MUBUF
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -amdgpu-enable-flat-scratch < %s | FileCheck %s -check-prefix=FLATSCR
 
 ; Make sure there's no assertion from passing a 0 alignment value
 define void @memcpy_fixed_align(i8 addrspace(5)*  %dst, i8 addrspace(1)* %src) {
 ; MUBUF-LABEL: memcpy_fixed_align:
 ; MUBUF:       ; %bb.0:
 ; MUBUF-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; MUBUF-NEXT:    global_load_dword v0, v[1:2], off offset:36
-; MUBUF-NEXT:    s_waitcnt vmcnt(0)
-; MUBUF-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:36
-; MUBUF-NEXT:    global_load_dword v0, v[1:2], off offset:32
-; MUBUF-NEXT:    s_waitcnt vmcnt(0)
-; MUBUF-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:32
+; MUBUF-NEXT:    global_load_dword v11, v[1:2], off offset:32
 ; MUBUF-NEXT:    global_load_dwordx4 v[3:6], v[1:2], off offset:16
-; MUBUF-NEXT:    s_waitcnt vmcnt(0)
+; MUBUF-NEXT:    global_load_dwordx4 v[7:10], v[1:2], off
+; MUBUF-NEXT:    s_waitcnt vmcnt(3)
+; MUBUF-NEXT:    buffer_store_dword v0, off, s[0:3], s32 offset:36
+; MUBUF-NEXT:    s_waitcnt vmcnt(3)
+; MUBUF-NEXT:    buffer_store_dword v11, off, s[0:3], s32 offset:32
+; MUBUF-NEXT:    s_waitcnt vmcnt(3)
 ; MUBUF-NEXT:    buffer_store_dword v6, off, s[0:3], s32 offset:28
 ; MUBUF-NEXT:    buffer_store_dword v5, off, s[0:3], s32 offset:24
 ; MUBUF-NEXT:    buffer_store_dword v4, off, s[0:3], s32 offset:20
 ; MUBUF-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:16
-; MUBUF-NEXT:    global_load_dwordx4 v[0:3], v[1:2], off
-; MUBUF-NEXT:    s_waitcnt vmcnt(0)
-; MUBUF-NEXT:    buffer_store_dword v3, off, s[0:3], s32 offset:12
-; MUBUF-NEXT:    buffer_store_dword v2, off, s[0:3], s32 offset:8
-; MUBUF-NEXT:    buffer_store_dword v1, off, s[0:3], s32 offset:4
-; MUBUF-NEXT:    buffer_store_dword v0, off, s[0:3], s32
+; MUBUF-NEXT:    s_waitcnt vmcnt(6)
+; MUBUF-NEXT:    buffer_store_dword v10, off, s[0:3], s32 offset:12
+; MUBUF-NEXT:    buffer_store_dword v9, off, s[0:3], s32 offset:8
+; MUBUF-NEXT:    buffer_store_dword v8, off, s[0:3], s32 offset:4
+; MUBUF-NEXT:    buffer_store_dword v7, off, s[0:3], s32
 ; MUBUF-NEXT:    s_waitcnt vmcnt(0)
 ; MUBUF-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; FLATSCR-LABEL: memcpy_fixed_align:
 ; FLATSCR:       ; %bb.0:
 ; FLATSCR-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; FLATSCR-NEXT:    global_load_dword v0, v[1:2], off offset:36
-; FLATSCR-NEXT:    s_waitcnt vmcnt(0)
-; FLATSCR-NEXT:    scratch_store_dword off, v0, s32 offset:36
-; FLATSCR-NEXT:    global_load_dword v0, v[1:2], off offset:32
-; FLATSCR-NEXT:    s_waitcnt vmcnt(0)
-; FLATSCR-NEXT:    scratch_store_dword off, v0, s32 offset:32
+; FLATSCR-NEXT:    global_load_dword v11, v[1:2], off offset:32
 ; FLATSCR-NEXT:    global_load_dwordx4 v[3:6], v[1:2], off offset:16
-; FLATSCR-NEXT:    s_waitcnt vmcnt(0)
+; FLATSCR-NEXT:    global_load_dwordx4 v[7:10], v[1:2], off
+; FLATSCR-NEXT:    s_waitcnt vmcnt(3)
+; FLATSCR-NEXT:    scratch_store_dword off, v0, s32 offset:36
+; FLATSCR-NEXT:    s_waitcnt vmcnt(3)
+; FLATSCR-NEXT:    scratch_store_dword off, v11, s32 offset:32
+; FLATSCR-NEXT:    s_waitcnt vmcnt(3)
 ; FLATSCR-NEXT:    scratch_store_dword off, v6, s32 offset:28
 ; FLATSCR-NEXT:    scratch_store_dword off, v5, s32 offset:24
 ; FLATSCR-NEXT:    scratch_store_dword off, v4, s32 offset:20
 ; FLATSCR-NEXT:    scratch_store_dword off, v3, s32 offset:16
-; FLATSCR-NEXT:    global_load_dwordx4 v[0:3], v[1:2], off
-; FLATSCR-NEXT:    s_waitcnt vmcnt(0)
-; FLATSCR-NEXT:    scratch_store_dword off, v3, s32 offset:12
-; FLATSCR-NEXT:    scratch_store_dword off, v2, s32 offset:8
-; FLATSCR-NEXT:    scratch_store_dword off, v1, s32 offset:4
-; FLATSCR-NEXT:    scratch_store_dword off, v0, s32
+; FLATSCR-NEXT:    s_waitcnt vmcnt(6)
+; FLATSCR-NEXT:    scratch_store_dword off, v10, s32 offset:12
+; FLATSCR-NEXT:    scratch_store_dword off, v9, s32 offset:8
+; FLATSCR-NEXT:    scratch_store_dword off, v8, s32 offset:4
+; FLATSCR-NEXT:    scratch_store_dword off, v7, s32
 ; FLATSCR-NEXT:    s_waitcnt vmcnt(0)
 ; FLATSCR-NEXT:    s_setpc_b64 s[30:31]
   %alloca = alloca [40 x i8], addrspace(5)
   %cast = bitcast [40 x i8] addrspace(5)* %alloca to i8 addrspace(5)*
   call void @llvm.memcpy.p5i8.p1i8.i64(i8 addrspace(5)* align 4 dereferenceable(40) %cast, i8 addrspace(1)* align 4 dereferenceable(40) %src, i64 40, i1 false)
   ret void
 }
 
 declare void @llvm.memcpy.p5i8.p1i8.i64(i8 addrspace(5)* noalias nocapture writeonly, i8 addrspace(1)* noalias nocapture readonly, i64, i1 immarg) #0
 
 attributes #0 = { argmemonly nounwind willreturn }

llvm/test/CodeGen/AMDGPU/memory_clause.ll

 ; GCN-NEXT:    buffer_load_dword v12, v0, s[0:3], 0 offen offset:36
 ; GCN-NEXT:    buffer_load_dword v13, v0, s[0:3], 0 offen offset:40
 ; GCN-NEXT:    buffer_load_dword v14, v0, s[0:3], 0 offen offset:44
 ; GCN-NEXT:    buffer_load_dword v15, v0, s[0:3], 0 offen offset:48
 ; GCN-NEXT:    buffer_load_dword v16, v0, s[0:3], 0 offen offset:52
 ; GCN-NEXT:    buffer_load_dword v17, v0, s[0:3], 0 offen offset:56
 ; GCN-NEXT:    v_add_u32_e32 v1, v1, v2
 ; GCN-NEXT:    buffer_load_dword v0, v0, s[0:3], 0 offen offset:60
-; GCN-NEXT:    s_waitcnt vmcnt(15)
-; GCN-NEXT:    buffer_store_dword v3, v1, s[0:3], 0 offen
-; GCN-NEXT:    s_waitcnt vmcnt(15)
-; GCN-NEXT:    buffer_store_dword v4, v1, s[0:3], 0 offen offset:4
-; GCN-NEXT:    s_waitcnt vmcnt(15)
-; GCN-NEXT:    buffer_store_dword v5, v1, s[0:3], 0 offen offset:8
-; GCN-NEXT:    s_waitcnt vmcnt(15)
+; GCN-NEXT:    s_waitcnt vmcnt(12)
 ; GCN-NEXT:    buffer_store_dword v6, v1, s[0:3], 0 offen offset:12
-; GCN-NEXT:    s_waitcnt vmcnt(15)
-; GCN-NEXT:    buffer_store_dword v7, v1, s[0:3], 0 offen offset:16
-; GCN-NEXT:    s_waitcnt vmcnt(15)
-; GCN-NEXT:    buffer_store_dword v8, v1, s[0:3], 0 offen offset:20
-; GCN-NEXT:    s_waitcnt vmcnt(15)
-; GCN-NEXT:    buffer_store_dword v9, v1, s[0:3], 0 offen offset:24
-; GCN-NEXT:    s_waitcnt vmcnt(15)
+; GCN-NEXT:    buffer_store_dword v5, v1, s[0:3], 0 offen offset:8
+; GCN-NEXT:    buffer_store_dword v4, v1, s[0:3], 0 offen offset:4
+; GCN-NEXT:    buffer_store_dword v3, v1, s[0:3], 0 offen
+; GCN-NEXT:    s_waitcnt vmcnt(12)
 ; GCN-NEXT:    buffer_store_dword v10, v1, s[0:3], 0 offen offset:28
-; GCN-NEXT:    s_waitcnt vmcnt(15)
-; GCN-NEXT:    buffer_store_dword v11, v1, s[0:3], 0 offen offset:32
-; GCN-NEXT:    s_waitcnt vmcnt(15)
-; GCN-NEXT:    buffer_store_dword v12, v1, s[0:3], 0 offen offset:36
-; GCN-NEXT:    s_waitcnt vmcnt(15)
-; GCN-NEXT:    buffer_store_dword v13, v1, s[0:3], 0 offen offset:40
-; GCN-NEXT:    s_waitcnt vmcnt(15)
+; GCN-NEXT:    buffer_store_dword v9, v1, s[0:3], 0 offen offset:24
+; GCN-NEXT:    buffer_store_dword v8, v1, s[0:3], 0 offen offset:20
+; GCN-NEXT:    buffer_store_dword v7, v1, s[0:3], 0 offen offset:16
+; GCN-NEXT:    s_waitcnt vmcnt(12)
 ; GCN-NEXT:    buffer_store_dword v14, v1, s[0:3], 0 offen offset:44
-; GCN-NEXT:    s_waitcnt vmcnt(15)
-; GCN-NEXT:    buffer_store_dword v15, v1, s[0:3], 0 offen offset:48
-; GCN-NEXT:    s_waitcnt vmcnt(15)
-; GCN-NEXT:    buffer_store_dword v16, v1, s[0:3], 0 offen offset:52
-; GCN-NEXT:    s_waitcnt vmcnt(15)
-; GCN-NEXT:    buffer_store_dword v17, v1, s[0:3], 0 offen offset:56
-; GCN-NEXT:    s_waitcnt vmcnt(15)
+; GCN-NEXT:    buffer_store_dword v13, v1, s[0:3], 0 offen offset:40
+; GCN-NEXT:    buffer_store_dword v12, v1, s[0:3], 0 offen offset:36
+; GCN-NEXT:    buffer_store_dword v11, v1, s[0:3], 0 offen offset:32
+; GCN-NEXT:    s_waitcnt vmcnt(12)
 ; GCN-NEXT:    buffer_store_dword v0, v1, s[0:3], 0 offen offset:60
+; GCN-NEXT:    buffer_store_dword v17, v1, s[0:3], 0 offen offset:56
+; GCN-NEXT:    buffer_store_dword v16, v1, s[0:3], 0 offen offset:52
+; GCN-NEXT:    buffer_store_dword v15, v1, s[0:3], 0 offen offset:48
 ; GCN-NEXT:    s_waitcnt vmcnt(0)
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
 bb:
   %tmp = tail call i32 @llvm.amdgcn.workitem.id.x()
   %tmp2 = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(5)* %arg, i32 %tmp
   %tmp3 = load <4 x i32>, <4 x i32> addrspace(5)* %tmp2, align 16
   %tmp4 = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(5)* %arg1, i32 %tmp
   %tmp5 = add nuw nsw i32 %tmp, 1

llvm/test/CodeGen/AMDGPU/private-element-size.ll

 
 ; ALL-LABEL: {{^}}private_elt_size_i64:
 ; HSA-ELT16: private_element_size = 3
 ; HSA-ELT8: private_element_size = 2
 ; HSA-ELT4: private_element_size = 1
 
 ; HSA-ELTGE8-DAG: buffer_store_dwordx2 {{v\[[0-9]+:[0-9]+\]}}, {{off|v[0-9]}}, s[0:3], 0 offset:1
 ; HSA-ELTGE8-DAG: buffer_store_dwordx2 {{v\[[0-9]+:[0-9]+\]}}, {{off|v[0-9]}}, s[0:3], 0 offset:2
-
-; HSA-ELTGE8: buffer_load_dwordx2 {{v\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}, s[0:3], 0 offen
+; HSA-ELTGE8-DAG: buffer_load_dwordx2 [[VAL:v\[[0-9]+:[0-9]+\]]], v{{[0-9]+}}, s[0:3], 0 offen
+; HSA-ELTGE8: flat_store_dwordx2 {{v\[[0-9]+:[0-9]+\]}}, [[VAL]]
 
 
 ; HSA-ELT4-DAG: buffer_store_dword {{v[0-9]+}}, off, s[0:3], 0 offset:16{{$}}
 ; HSA-ELT4-DAG: buffer_store_dword {{v[0-9]+}}, off, s[0:3], 0 offset:20{{$}}
 ; HSA-ELT4-DAG: buffer_store_dword {{v[0-9]+}}, off, s[0:3], 0 offset:24{{$}}
 ; HSA-ELT4-DAG: buffer_store_dword {{v[0-9]+}}, off, s[0:3], 0 offset:28{{$}}
 
-; HSA-ELT4: buffer_load_dword {{v[0-9]+}}, v{{[0-9]+}}, s[0:3], 0 offen offset:4{{$}}
-; HSA-ELT4: buffer_load_dword {{v[0-9]+}}, v{{[0-9]+}}, s[0:3], 0 offen{{$}}
+; HSA-ELT4-DAG: buffer_load_dword v[[HI:[0-9]+]], v{{[0-9]+}}, s[0:3], 0 offen offset:4{{$}}
+; HSA-ELT4-DAG: buffer_load_dword v[[LO:[0-9]+]], v{{[0-9]+}}, s[0:3], 0 offen{{$}}
+; HSA-ELT4: flat_store_dwordx2 {{v\[[0-9]+:[0-9]+\]}}, v{{\[}}[[LO]]:[[HI]]]
 define amdgpu_kernel void @private_elt_size_i64(i64 addrspace(1)* %out, i32 addrspace(1)* %index.array) #0 {
 entry:
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %idxprom = sext i32 %tid to i64
   %gep.index = getelementptr inbounds i32, i32 addrspace(1)* %index.array, i64 %idxprom
   %index.load = load i32, i32 addrspace(1)* %gep.index
   %index = and i32 %index.load, 2
   %alloca = alloca [2 x i64], align 16, addrspace(5)

 
 ; ALL-LABEL: {{^}}private_elt_size_f64:
 ; HSA-ELT16: private_element_size = 3
 ; HSA-ELT8: private_element_size = 2
 ; HSA-ELT4: private_element_size = 1
 
 ; HSA-ELTGE8-DAG: buffer_store_dwordx2 {{v\[[0-9]+:[0-9]+\]}}, off, s[0:3], 0 offset:16
 ; HSA-ELTGE8-DAG: buffer_store_dwordx2 {{v\[[0-9]+:[0-9]+\]}}, off, s[0:3], 0 offset:24
-
-; HSA-ELTGE8: buffer_load_dwordx2 {{v\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}, s[0:3], 0 offen
+; HSA-ELTGE8-DAG: buffer_load_dwordx2 [[VAL:v\[[0-9]+:[0-9]+\]]], v{{[0-9]+}}, s[0:3], 0 offen
+; HSA-ELTGE8: flat_store_dwordx2 {{v\[[0-9]+:[0-9]+\]}}, [[VAL]]
 
 
 ; HSA-ELT4-DAG: buffer_store_dword {{v[0-9]+}}, off, s[0:3], 0 offset:16{{$}}
 ; HSA-ELT4-DAG: buffer_store_dword {{v[0-9]+}}, off, s[0:3], 0 offset:20{{$}}
 ; HSA-ELT4-DAG: buffer_store_dword {{v[0-9]+}}, off, s[0:3], 0 offset:24{{$}}
 ; HSA-ELT4-DAG: buffer_store_dword {{v[0-9]+}}, off, s[0:3], 0 offset:28{{$}}
 
-; HSA-ELT4: buffer_load_dword {{v[0-9]+}}, v{{[0-9]+}}, s[0:3], 0 offen offset:4{{$}}
-; HSA-ELT4: buffer_load_dword {{v[0-9]+}}, v{{[0-9]+}}, s[0:3], 0 offen{{$}}
+; HSA-ELT4-DAG: buffer_load_dword v[[HI:[0-9]+]], v{{[0-9]+}}, s[0:3], 0 offen offset:4{{$}}
+; HSA-ELT4-DAG: buffer_load_dword v[[LO:[0-9]+]], v{{[0-9]+}}, s[0:3], 0 offen{{$}}
+; HSA-ELT4: flat_store_dwordx2 {{v\[[0-9]+:[0-9]+\]}}, v{{\[}}[[LO]]:[[HI]]]
 define amdgpu_kernel void @private_elt_size_f64(double addrspace(1)* %out, i32 addrspace(1)* %index.array) #0 {
 entry:
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %idxprom = sext i32 %tid to i64
   %gep.index = getelementptr inbounds i32, i32 addrspace(1)* %index.array, i64 %idxprom
   %index.load = load i32, i32 addrspace(1)* %gep.index
   %index = and i32 %index.load, 2
   %alloca = alloca [2 x double], align 16, addrspace(5)

llvm/test/CodeGen/AMDGPU/schedule-regpressure-limit.ll

-; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -march=amdgcn -mcpu=tonga -misched=gcn-minreg -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -march=amdgcn -mcpu=tonga -misched=gcn-max-occupancy-experimental -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -enable-amdgpu-aa=0 -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -enable-amdgpu-aa=0 -march=amdgcn -mcpu=tonga -misched=gcn-minreg -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -enable-amdgpu-aa=0 -march=amdgcn -mcpu=tonga -misched=gcn-max-occupancy-experimental -verify-machineinstrs < %s | FileCheck %s
 
 ; We expect a two digit VGPR usage here, not a three digit.
 ; CHECK: NumVgprs: {{[0-9][0-9]$}}
 
 define amdgpu_kernel void @load_fma_store(float addrspace(3)* nocapture readonly %arg, float addrspace(1)* nocapture %arg1) {
 bb:
   %tmp = getelementptr inbounds float, float addrspace(3)* %arg, i32 1
   %tmp2 = load float, float addrspace(3)* %tmp, align 4

llvm/test/CodeGen/AMDGPU/schedule-regpressure-limit2.ll

-; RUN: llc -march=amdgcn -mcpu=tahiti -misched=gcn-minreg -verify-machineinstrs < %s | FileCheck -check-prefixes=SI,SI-MINREG %s
-; RUN: llc -march=amdgcn -mcpu=tahiti -misched=gcn-max-occupancy-experimental -verify-machineinstrs < %s | FileCheck -check-prefixes=SI,SI-MAXOCC %s
-; RUN: llc -march=amdgcn -mcpu=fiji -misched=gcn-minreg -verify-machineinstrs < %s | FileCheck -check-prefixes=VI,VI-MINREG %s
-; RUN: llc -march=amdgcn -mcpu=fiji -misched=gcn-max-occupancy-experimental -verify-machineinstrs < %s | FileCheck -check-prefixes=VI,VI-MAXOCC %s
+; RUN: llc -march=amdgcn -mcpu=tahiti -enable-amdgpu-aa=0 -misched=gcn-minreg -verify-machineinstrs < %s | FileCheck -check-prefixes=SI,SI-MINREG %s
+; RUN: llc -march=amdgcn -mcpu=tahiti -enable-amdgpu-aa=0 -misched=gcn-max-occupancy-experimental -verify-machineinstrs < %s | FileCheck -check-prefixes=SI,SI-MAXOCC %s
+; RUN: llc -march=amdgcn -mcpu=fiji -enable-amdgpu-aa=0 -misched=gcn-minreg -verify-machineinstrs < %s | FileCheck -check-prefixes=VI,VI-MINREG %s
+; RUN: llc -march=amdgcn -mcpu=fiji -enable-amdgpu-aa=0 -misched=gcn-max-occupancy-experimental -verify-machineinstrs < %s | FileCheck -check-prefixes=VI,VI-MAXOCC %s
 
 ; SI-MINREG: NumSgprs: {{[1-9]$}}
 ; SI-MINREG: NumVgprs: {{[1-9]$}}
 
 ; SI-MAXOCC: NumSgprs: {{[0-4][0-9]$}}
 ; SI-MAXOCC: NumVgprs: {{[0-4][0-9]$}}
 
 ; stores may alias loads

llvm/test/CodeGen/AMDGPU/schedule-regpressure-limit3.ll

-; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -march=amdgcn -mcpu=tonga -misched=gcn-ilp -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -march=amdgcn -mcpu=tonga -enable-amdgpu-aa=0 -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -march=amdgcn -mcpu=tonga -enable-amdgpu-aa=0 -misched=gcn-ilp -verify-machineinstrs < %s | FileCheck %s
 
 ; We expect a three digit VGPR usage here since only one wave requested.
 ; CHECK: NumVgprs: {{[0-9][0-9][0-9]$}}
 
 define amdgpu_kernel void @load_fma_store(float addrspace(3)* nocapture readonly %arg, float addrspace(1)* nocapture %arg1) #1 {
 bb:
   %tmp = getelementptr inbounds float, float addrspace(3)* %arg, i32 1
   %tmp2 = load float, float addrspace(3)* %tmp, align 4

llvm/test/CodeGen/AMDGPU/undefined-subreg-liverange.ll

 define amdgpu_ps float @valley_partially_undef_copy() #0 {
 ; CHECK-LABEL: valley_partially_undef_copy:
 ; CHECK:       ; %bb.0: ; %bb
 ; CHECK-NEXT:    s_mov_b32 s3, 0xf000
 ; CHECK-NEXT:    s_mov_b32 s2, -1
 ; CHECK-NEXT:    buffer_load_dword v1, off, s[0:3], 0
 ; CHECK-NEXT:    buffer_load_dword v0, off, s[0:3], 0
 ; CHECK-NEXT:    v_mov_b32_e32 v2, 0x7fc00000
-; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
 ; CHECK-NEXT:    buffer_store_dword v2, off, s[0:3], 0
+; CHECK-NEXT:    s_waitcnt vmcnt(1)
+; CHECK-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
 ; CHECK-NEXT:    v_cmp_ne_u32_e64 s[0:1], 0, v1
 ; CHECK-NEXT:  BB1_1: ; %bb9
 ; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    s_andn2_b64 vcc, exec, s[0:1]
 ; CHECK-NEXT:    s_cbranch_vccnz BB1_1
 ; CHECK-NEXT:  ; %bb.2: ; %bb11
 ; CHECK-NEXT:    s_mov_b32 s3, 0xf000
 ; CHECK-NEXT:    s_mov_b32 s2, -1