Index: llvm/trunk/lib/Target/AMDGPU/SIISelLowering.cpp =================================================================== --- llvm/trunk/lib/Target/AMDGPU/SIISelLowering.cpp +++ llvm/trunk/lib/Target/AMDGPU/SIISelLowering.cpp @@ -156,12 +156,14 @@ setOperationAction(ISD::LOAD, MVT::v8i32, Custom); setOperationAction(ISD::LOAD, MVT::v16i32, Custom); setOperationAction(ISD::LOAD, MVT::i1, Custom); + setOperationAction(ISD::LOAD, MVT::v32i32, Custom); setOperationAction(ISD::STORE, MVT::v2i32, Custom); setOperationAction(ISD::STORE, MVT::v4i32, Custom); setOperationAction(ISD::STORE, MVT::v8i32, Custom); setOperationAction(ISD::STORE, MVT::v16i32, Custom); setOperationAction(ISD::STORE, MVT::i1, Custom); + setOperationAction(ISD::STORE, MVT::v32i32, Custom); setTruncStoreAction(MVT::v2i32, MVT::v2i16, Expand); setTruncStoreAction(MVT::v4i32, MVT::v4i16, Expand); @@ -246,7 +248,7 @@ // We only support LOAD/STORE and vector manipulation ops for vectors // with > 4 elements. for (MVT VT : {MVT::v8i32, MVT::v8f32, MVT::v16i32, MVT::v16f32, - MVT::v2i64, MVT::v2f64, MVT::v4i16, MVT::v4f16 }) { + MVT::v2i64, MVT::v2f64, MVT::v4i16, MVT::v4f16, MVT::v32i32 }) { for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; ++Op) { switch (Op) { case ISD::LOAD: @@ -6196,7 +6198,7 @@ if (AS == AMDGPUAS::CONSTANT_ADDRESS || AS == AMDGPUAS::CONSTANT_ADDRESS_32BIT) { - if (!Op->isDivergent() && Alignment >= 4) + if (!Op->isDivergent() && Alignment >= 4 && NumElements < 32) return SDValue(); // Non-uniform loads will be selected to MUBUF instructions, so they // have the same legalization requirements as global and private @@ -6209,7 +6211,7 @@ AS == AMDGPUAS::GLOBAL_ADDRESS) { if (Subtarget->getScalarizeGlobalBehavior() && !Op->isDivergent() && !Load->isVolatile() && isMemOpHasNoClobberedMemOperand(Load) && - Alignment >= 4) + Alignment >= 4 && NumElements < 32) return SDValue(); // Non-uniform loads will be selected to MUBUF instructions, so they // have the same legalization requirements as global and private Index: llvm/trunk/test/CodeGen/AMDGPU/load-constant-i32.ll =================================================================== --- llvm/trunk/test/CodeGen/AMDGPU/load-constant-i32.ll +++ llvm/trunk/test/CodeGen/AMDGPU/load-constant-i32.ll @@ -2,6 +2,7 @@ ; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=kaveri -verify-machineinstrs < %s | FileCheck -allow-deprecated-dag-overlap -check-prefix=GCN -check-prefix=GCN-HSA -check-prefix=FUNC %s ; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -allow-deprecated-dag-overlap -check-prefix=GCN -check-prefix=GCN-NOHSA -check-prefix=FUNC %s ; RUN: llc -march=r600 -mcpu=redwood < %s | FileCheck -allow-deprecated-dag-overlap -check-prefix=EG -check-prefix=FUNC %s +; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -allow-deprecated-dag-overlap -check-prefix=GCN -check-prefix=GCN-HSA -check-prefix=FUNC %s ; FUNC-LABEL: {{^}}constant_load_i32: ; GCN: s_load_dword s{{[0-9]+}} @@ -187,10 +188,10 @@ ; GCN-NOHSA-DAG: buffer_store_dwordx4 ; GCN-NOHSA-DAG: buffer_store_dwordx4 -; GCN-HSA-DAG: flat_store_dwordx4 -; GCN-HSA-DAG: flat_store_dwordx4 -; GCN-SA-DAG: flat_store_dwordx4 -; GCN-HSA-DAG: flat_store_dwordx4 +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 +; GCN-SA-DAG: {{flat|global}}_store_dwordx4 +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 define amdgpu_kernel void @constant_zextload_v8i32_to_v8i64(<8 x i64> addrspace(1)* %out, <8 x i32> addrspace(4)* %in) #0 { %ld = load <8 x i32>, <8 x i32> addrspace(4)* %in %ext = zext <8 x i32> %ld to <8 x i64> @@ -215,10 +216,10 @@ ; GCN-NOHSA-DAG: buffer_store_dwordx4 ; GCN-NOHSA-DAG: buffer_store_dwordx4 -; GCN-HSA-DAG: flat_store_dwordx4 -; GCN-HSA-DAG: flat_store_dwordx4 -; GCN-HSA-DAG: flat_store_dwordx4 -; GCN-HSA-DAG: flat_store_dwordx4 +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 define amdgpu_kernel void @constant_sextload_v8i32_to_v8i64(<8 x i64> addrspace(1)* %out, <8 x i32> addrspace(4)* %in) #0 { %ld = load <8 x i32>, <8 x i32> addrspace(4)* %in %ext = sext <8 x i32> %ld to <8 x i64> @@ -259,14 +260,14 @@ ; GCN-NOHSA: buffer_store_dwordx4 ; GCN-NOHSA: buffer_store_dwordx4 -; GCN-HSA: flat_store_dwordx4 -; GCN-HSA: flat_store_dwordx4 -; GCN-HSA: flat_store_dwordx4 -; GCN-HSA: flat_store_dwordx4 -; GCN-HSA: flat_store_dwordx4 -; GCN-HSA: flat_store_dwordx4 -; GCN-HSA: flat_store_dwordx4 -; GCN-HSA: flat_store_dwordx4 +; GCN-HSA: {{flat|global}}_store_dwordx4 +; GCN-HSA: {{flat|global}}_store_dwordx4 +; GCN-HSA: {{flat|global}}_store_dwordx4 +; GCN-HSA: {{flat|global}}_store_dwordx4 +; GCN-HSA: {{flat|global}}_store_dwordx4 +; GCN-HSA: {{flat|global}}_store_dwordx4 +; GCN-HSA: {{flat|global}}_store_dwordx4 +; GCN-HSA: {{flat|global}}_store_dwordx4 define amdgpu_kernel void @constant_zextload_v16i32_to_v16i64(<16 x i64> addrspace(1)* %out, <16 x i32> addrspace(4)* %in) #0 { %ld = load <16 x i32>, <16 x i32> addrspace(4)* %in %ext = zext <16 x i32> %ld to <16 x i64> @@ -299,25 +300,25 @@ ; GCN-NOHSA-DAG: buffer_store_dwordx4 ; GCN-NOHSA-DAG: buffer_store_dwordx4 -; GCN-HSA-DAG: flat_store_dwordx4 -; GCN-HSA-DAG: flat_store_dwordx4 -; GCN-HSA-DAG: flat_store_dwordx4 -; GCN-HSA-DAG: flat_store_dwordx4 - -; GCN-HSA-DAG: flat_store_dwordx4 -; GCN-HSA-DAG: flat_store_dwordx4 -; GCN-HSA-DAG: flat_store_dwordx4 -; GCN-HSA-DAG: flat_store_dwordx4 - -; GCN-HSA-DAG: flat_store_dwordx4 -; GCN-HSA-DAG: flat_store_dwordx4 -; GCN-HSA-DAG: flat_store_dwordx4 -; GCN-HSA-DAG: flat_store_dwordx4 - -; GCN-HSA-DAG: flat_store_dwordx4 -; GCN-HSA-DAG: flat_store_dwordx4 -; GCN-HSA-DAG: flat_store_dwordx4 -; GCN-HSA-DAG: flat_store_dwordx4 +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 + +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 + +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 + +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 define amdgpu_kernel void @constant_sextload_v32i32_to_v32i64(<32 x i64> addrspace(1)* %out, <32 x i32> addrspace(4)* %in) #0 { %ld = load <32 x i32>, <32 x i32> addrspace(4)* %in @@ -351,25 +352,25 @@ ; GCN-NOHSA-DAG: buffer_store_dwordx4 -; GCN-HSA-DAG: flat_store_dwordx4 -; GCN-HSA-DAG: flat_store_dwordx4 -; GCN-HSA-DAG: flat_store_dwordx4 -; GCN-HSA-DAG: flat_store_dwordx4 - -; GCN-HSA-DAG: flat_store_dwordx4 -; GCN-HSA-DAG: flat_store_dwordx4 -; GCN-HSA-DAG: flat_store_dwordx4 -; GCN-HSA-DAG: flat_store_dwordx4 - -; GCN-HSA-DAG: flat_store_dwordx4 -; GCN-HSA-DAG: flat_store_dwordx4 -; GCN-HSA-DAG: flat_store_dwordx4 -; GCN-HSA-DAG: flat_store_dwordx4 - -; GCN-HSA-DAG: flat_store_dwordx4 -; GCN-HSA-DAG: flat_store_dwordx4 -; GCN-HSA-DAG: flat_store_dwordx4 -; GCN-HSA-DAG: flat_store_dwordx4 +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 + +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 + +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 + +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 define amdgpu_kernel void @constant_zextload_v32i32_to_v32i64(<32 x i64> addrspace(1)* %out, <32 x i32> addrspace(4)* %in) #0 { %ld = load <32 x i32>, <32 x i32> addrspace(4)* %in %ext = zext <32 x i32> %ld to <32 x i64> @@ -377,4 +378,53 @@ ret void } +; FUNC-LABEL: {{^}}constant_load_v32i32: +; GCN: s_load_dwordx16 +; GCN: s_load_dwordx16 + +; GCN-NOHSA-DAG: buffer_store_dwordx4 +; GCN-NOHSA-DAG: buffer_store_dwordx4 +; GCN-NOHSA-DAG: buffer_store_dwordx4 +; GCN-NOHSA-DAG: buffer_store_dwordx4 + +; GCN-NOHSA-DAG: buffer_store_dwordx4 +; GCN-NOHSA-DAG: buffer_store_dwordx4 +; GCN-NOHSA-DAG: buffer_store_dwordx4 +; GCN-NOHSA-DAG: buffer_store_dwordx4 + +; GCN-NOHSA-DAG: buffer_store_dwordx4 +; GCN-NOHSA-DAG: buffer_store_dwordx4 +; GCN-NOHSA-DAG: buffer_store_dwordx4 +; GCN-NOHSA-DAG: buffer_store_dwordx4 + +; GCN-NOHSA-DAG: buffer_store_dwordx4 +; GCN-NOHSA-DAG: buffer_store_dwordx4 +; GCN-NOHSA-DAG: buffer_store_dwordx4 +; GCN-NOHSA-DAG: buffer_store_dwordx4 + +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 + +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 + +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 + +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 +define amdgpu_kernel void @constant_load_v32i32(<32 x i32> addrspace(1)* %out, <32 x i32> addrspace(4)* %in) #0 { + %ld = load <32 x i32>, <32 x i32> addrspace(4)* %in + store <32 x i32> %ld, <32 x i32> addrspace(1)* %out + ret void +} + attributes #0 = { nounwind } Index: llvm/trunk/test/CodeGen/AMDGPU/load-global-i32.ll =================================================================== --- llvm/trunk/test/CodeGen/AMDGPU/load-global-i32.ll +++ llvm/trunk/test/CodeGen/AMDGPU/load-global-i32.ll @@ -2,11 +2,12 @@ ; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgcn--amdhsa -mcpu=kaveri -verify-machineinstrs < %s | FileCheck -allow-deprecated-dag-overlap -check-prefix=GCN -check-prefix=GCN-HSA -check-prefix=FUNC %s ; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -allow-deprecated-dag-overlap -check-prefix=GCN -check-prefix=GCN-NOHSA -check-prefix=FUNC %s ; RUN: llc -amdgpu-scalarize-global-loads=false -march=r600 -mcpu=redwood < %s | FileCheck -allow-deprecated-dag-overlap -check-prefix=EG -check-prefix=FUNC %s +; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgcn--amdhsa -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -allow-deprecated-dag-overlap -check-prefix=GCN -check-prefix=GCN-HSA -check-prefix=FUNC %s ; FUNC-LABEL: {{^}}global_load_i32: ; GCN-NOHSA: buffer_load_dword v{{[0-9]+}} -; GCN-HSA: flat_load_dword +; GCN-HSA: {{flat|global}}_load_dword ; EG: VTX_READ_32 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0 define amdgpu_kernel void @global_load_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %in) #0 { @@ -18,7 +19,7 @@ ; FUNC-LABEL: {{^}}global_load_v2i32: ; GCN-NOHSA: buffer_load_dwordx2 -; GCN-HSA: flat_load_dwordx2 +; GCN-HSA: {{flat|global}}_load_dwordx2 ; EG: VTX_READ_64 define amdgpu_kernel void @global_load_v2i32(<2 x i32> addrspace(1)* %out, <2 x i32> addrspace(1)* %in) #0 { @@ -30,7 +31,7 @@ ; FUNC-LABEL: {{^}}global_load_v3i32: ; GCN-NOHSA: buffer_load_dwordx4 -; GCN-HSA: flat_load_dwordx4 +; GCN-HSA: {{flat|global}}_load_dwordx4 ; EG: VTX_READ_128 define amdgpu_kernel void @global_load_v3i32(<3 x i32> addrspace(1)* %out, <3 x i32> addrspace(1)* %in) #0 { @@ -42,7 +43,7 @@ ; FUNC-LABEL: {{^}}global_load_v4i32: ; GCN-NOHSA: buffer_load_dwordx4 -; GCN-HSA: flat_load_dwordx4 +; GCN-HSA: {{flat|global}}_load_dwordx4 ; EG: VTX_READ_128 define amdgpu_kernel void @global_load_v4i32(<4 x i32> addrspace(1)* %out, <4 x i32> addrspace(1)* %in) #0 { @@ -55,8 +56,8 @@ ; FUNC-LABEL: {{^}}global_load_v8i32: ; GCN-NOHSA: buffer_load_dwordx4 ; GCN-NOHSA: buffer_load_dwordx4 -; GCN-HSA: flat_load_dwordx4 -; GCN-HSA: flat_load_dwordx4 +; GCN-HSA: {{flat|global}}_load_dwordx4 +; GCN-HSA: {{flat|global}}_load_dwordx4 ; EG: VTX_READ_128 ; EG: VTX_READ_128 @@ -73,10 +74,10 @@ ; GCN-NOHSA: buffer_load_dwordx4 ; GCN-NOHSA: buffer_load_dwordx4 -; GCN-HSA: flat_load_dwordx4 -; GCN-HSA: flat_load_dwordx4 -; GCN-HSA: flat_load_dwordx4 -; GCN-HSA: flat_load_dwordx4 +; GCN-HSA: {{flat|global}}_load_dwordx4 +; GCN-HSA: {{flat|global}}_load_dwordx4 +; GCN-HSA: {{flat|global}}_load_dwordx4 +; GCN-HSA: {{flat|global}}_load_dwordx4 ; EG: VTX_READ_128 ; EG: VTX_READ_128 @@ -91,11 +92,11 @@ ; FUNC-LABEL: {{^}}global_zextload_i32_to_i64: ; GCN-NOHSA-DAG: buffer_load_dword v[[LO:[0-9]+]], -; GCN-HSA-DAG: flat_load_dword v[[LO:[0-9]+]], +; GCN-HSA-DAG: {{flat|global}}_load_dword v[[LO:[0-9]+]], ; GCN-DAG: v_mov_b32_e32 v[[HI:[0-9]+]], 0{{$}} ; GCN-NOHSA: buffer_store_dwordx2 v{{\[}}[[LO]]:[[HI]]] -; GCN-HSA: flat_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[LO]]:[[HI]]] +; GCN-HSA: {{flat|global}}_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[LO]]:[[HI]]] ; EG: MEM_RAT_CACHELESS STORE_RAW T{{[0-9]+}}.XY define amdgpu_kernel void @global_zextload_i32_to_i64(i64 addrspace(1)* %out, i32 addrspace(1)* %in) #0 { @@ -107,10 +108,10 @@ ; FUNC-LABEL: {{^}}global_sextload_i32_to_i64: ; GCN-NOHSA: buffer_load_dword v[[LO:[0-9]+]] -; GCN-HSA: flat_load_dword v[[LO:[0-9]+]] +; GCN-HSA: {{flat|global}}_load_dword v[[LO:[0-9]+]] ; GCN: v_ashrrev_i32_e32 v[[HI:[0-9]+]], 31, v[[LO]] ; GCN-NOHSA: buffer_store_dwordx2 v{{\[}}[[LO]]:[[HI]]{{\]}} -; GCN-HSA: flat_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[LO]]:[[HI]]{{\]}} +; GCN-HSA: {{flat|global}}_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[LO]]:[[HI]]{{\]}} ; EG: MEM_RAT @@ -128,8 +129,8 @@ ; GCN-NOHSA: buffer_load_dword ; GCN-NOHSA: buffer_store_dwordx2 -; GCN-HSA: flat_load_dword -; GCN-HSA: flat_store_dwordx2 +; GCN-HSA: {{flat|global}}_load_dword +; GCN-HSA: {{flat|global}}_store_dwordx2 define amdgpu_kernel void @global_zextload_v1i32_to_v1i64(<1 x i64> addrspace(1)* %out, <1 x i32> addrspace(1)* %in) #0 { %ld = load <1 x i32>, <1 x i32> addrspace(1)* %in %ext = zext <1 x i32> %ld to <1 x i64> @@ -139,10 +140,10 @@ ; FUNC-LABEL: {{^}}global_sextload_v1i32_to_v1i64: ; GCN-NOHSA: buffer_load_dword v[[LO:[0-9]+]] -; GCN-HSA: flat_load_dword v[[LO:[0-9]+]] +; GCN-HSA: {{flat|global}}_load_dword v[[LO:[0-9]+]] ; GCN: v_ashrrev_i32_e32 v[[HI:[0-9]+]], 31, v[[LO]] ; GCN-NOHSA: buffer_store_dwordx2 v{{\[}}[[LO]]:[[HI]]{{\]}} -; GCN-HSA: flat_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[LO]]:[[HI]]{{\]}} +; GCN-HSA: {{flat|global}}_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[LO]]:[[HI]]{{\]}} define amdgpu_kernel void @global_sextload_v1i32_to_v1i64(<1 x i64> addrspace(1)* %out, <1 x i32> addrspace(1)* %in) #0 { %ld = load <1 x i32>, <1 x i32> addrspace(1)* %in %ext = sext <1 x i32> %ld to <1 x i64> @@ -154,8 +155,8 @@ ; GCN-NOHSA: buffer_load_dwordx2 ; GCN-NOHSA: buffer_store_dwordx4 -; GCN-HSA: flat_load_dwordx2 -; GCN-HSA: flat_store_dwordx4 +; GCN-HSA: {{flat|global}}_load_dwordx2 +; GCN-HSA: {{flat|global}}_store_dwordx4 define amdgpu_kernel void @global_zextload_v2i32_to_v2i64(<2 x i64> addrspace(1)* %out, <2 x i32> addrspace(1)* %in) #0 { %ld = load <2 x i32>, <2 x i32> addrspace(1)* %in %ext = zext <2 x i32> %ld to <2 x i64> @@ -165,13 +166,13 @@ ; FUNC-LABEL: {{^}}global_sextload_v2i32_to_v2i64: ; GCN-NOHSA: buffer_load_dwordx2 -; GCN-HSA: flat_load_dwordx2 +; GCN-HSA: {{flat|global}}_load_dwordx2 ; GCN-DAG: v_ashrrev_i32 ; GCN-DAG: v_ashrrev_i32 ; GCN-NOHSA-DAG: buffer_store_dwordx4 -; GCN-HSA-DAG: flat_store_dwordx4 +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 define amdgpu_kernel void @global_sextload_v2i32_to_v2i64(<2 x i64> addrspace(1)* %out, <2 x i32> addrspace(1)* %in) #0 { %ld = load <2 x i32>, <2 x i32> addrspace(1)* %in %ext = sext <2 x i32> %ld to <2 x i64> @@ -184,9 +185,9 @@ ; GCN-NOHSA: buffer_store_dwordx4 ; GCN-NOHSA: buffer_store_dwordx4 -; GCN-HSA: flat_load_dwordx4 -; GCN-HSA: flat_store_dwordx4 -; GCN-HSA: flat_store_dwordx4 +; GCN-HSA: {{flat|global}}_load_dwordx4 +; GCN-HSA: {{flat|global}}_store_dwordx4 +; GCN-HSA: {{flat|global}}_store_dwordx4 define amdgpu_kernel void @global_zextload_v4i32_to_v4i64(<4 x i64> addrspace(1)* %out, <4 x i32> addrspace(1)* %in) #0 { %ld = load <4 x i32>, <4 x i32> addrspace(1)* %in %ext = zext <4 x i32> %ld to <4 x i64> @@ -196,7 +197,7 @@ ; FUNC-LABEL: {{^}}global_sextload_v4i32_to_v4i64: ; GCN-NOHSA: buffer_load_dwordx4 -; GCN-HSA: flat_load_dwordx4 +; GCN-HSA: {{flat|global}}_load_dwordx4 ; GCN-DAG: v_ashrrev_i32 ; GCN-DAG: v_ashrrev_i32 @@ -206,8 +207,8 @@ ; GCN-NOHSA-DAG: buffer_store_dwordx4 ; GCN-NOHSA-DAG: buffer_store_dwordx4 -; GCN-HSA-DAG: flat_store_dwordx4 -; GCN-HSA-DAG: flat_store_dwordx4 +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 define amdgpu_kernel void @global_sextload_v4i32_to_v4i64(<4 x i64> addrspace(1)* %out, <4 x i32> addrspace(1)* %in) #0 { %ld = load <4 x i32>, <4 x i32> addrspace(1)* %in %ext = sext <4 x i32> %ld to <4 x i64> @@ -219,18 +220,18 @@ ; GCN-NOHSA: buffer_load_dwordx4 ; GCN-NOHSA: buffer_load_dwordx4 -; GCN-HSA: flat_load_dwordx4 -; GCN-HSA: flat_load_dwordx4 +; GCN-HSA: {{flat|global}}_load_dwordx4 +; GCN-HSA: {{flat|global}}_load_dwordx4 ; GCN-NOHSA-DAG: buffer_store_dwordx4 ; GCN-NOHSA-DAG: buffer_store_dwordx4 ; GCN-NOHSA-DAG: buffer_store_dwordx4 ; GCN-NOHSA-DAG: buffer_store_dwordx4 -; GCN-HSA-DAG: flat_store_dwordx4 -; GCN-HSA-DAG: flat_store_dwordx4 -; GCN-SA-DAG: flat_store_dwordx4 -; GCN-HSA-DAG: flat_store_dwordx4 +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 define amdgpu_kernel void @global_zextload_v8i32_to_v8i64(<8 x i64> addrspace(1)* %out, <8 x i32> addrspace(1)* %in) #0 { %ld = load <8 x i32>, <8 x i32> addrspace(1)* %in %ext = zext <8 x i32> %ld to <8 x i64> @@ -242,8 +243,8 @@ ; GCN-NOHSA: buffer_load_dwordx4 ; GCN-NOHSA: buffer_load_dwordx4 -; GCN-HSA: flat_load_dwordx4 -; GCN-HSA: flat_load_dwordx4 +; GCN-HSA: {{flat|global}}_load_dwordx4 +; GCN-HSA: {{flat|global}}_load_dwordx4 ; GCN-DAG: v_ashrrev_i32 ; GCN-DAG: v_ashrrev_i32 @@ -259,10 +260,10 @@ ; GCN-NOHSA-DAG: buffer_store_dwordx4 ; GCN-NOHSA-DAG: buffer_store_dwordx4 -; GCN-HSA-DAG: flat_store_dwordx4 -; GCN-HSA-DAG: flat_store_dwordx4 -; GCN-HSA-DAG: flat_store_dwordx4 -; GCN-HSA-DAG: flat_store_dwordx4 +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 define amdgpu_kernel void @global_sextload_v8i32_to_v8i64(<8 x i64> addrspace(1)* %out, <8 x i32> addrspace(1)* %in) #0 { %ld = load <8 x i32>, <8 x i32> addrspace(1)* %in %ext = sext <8 x i32> %ld to <8 x i64> @@ -276,10 +277,10 @@ ; GCN-NOHSA: buffer_load_dwordx4 ; GCN-NOHSA: buffer_load_dwordx4 -; GCN-HSA: flat_load_dwordx4 -; GCN-HSA: flat_load_dwordx4 -; GCN-HSA: flat_load_dwordx4 -; GCN-HSA: flat_load_dwordx4 +; GCN-HSA: {{flat|global}}_load_dwordx4 +; GCN-HSA: {{flat|global}}_load_dwordx4 +; GCN-HSA: {{flat|global}}_load_dwordx4 +; GCN-HSA: {{flat|global}}_load_dwordx4 ; GCN-DAG: v_ashrrev_i32 @@ -287,28 +288,28 @@ ; GCN-DAG: v_ashrrev_i32 ; GCN-DAG: v_ashrrev_i32 ; GCN-NOHSA-DAG: buffer_store_dwordx4 -; GCN-HSA-DAG: flat_store_dwordx4 +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 ; GCN-DAG: v_ashrrev_i32 ; GCN-DAG: v_ashrrev_i32 ; GCN-DAG: v_ashrrev_i32 ; GCN-DAG: v_ashrrev_i32 ; GCN-NOHSA-DAG: buffer_store_dwordx4 -; GCN-HSA-DAG: flat_store_dwordx4 +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 ; GCN-DAG: v_ashrrev_i32 ; GCN-DAG: v_ashrrev_i32 ; GCN-DAG: v_ashrrev_i32 ; GCN-DAG: v_ashrrev_i32 ; GCN-NOHSA-DAG: buffer_store_dwordx4 -; GCN-HSA-DAG: flat_store_dwordx4 +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 ; GCN-DAG: v_ashrrev_i32 ; GCN-DAG: v_ashrrev_i32 ; GCN-DAG: v_ashrrev_i32 ; GCN-DAG: v_ashrrev_i32 ; GCN-NOHSA-DAG: buffer_store_dwordx4 -; GCN-HSA-DAG: flat_store_dwordx4 +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 define amdgpu_kernel void @global_sextload_v16i32_to_v16i64(<16 x i64> addrspace(1)* %out, <16 x i32> addrspace(1)* %in) #0 { %ld = load <16 x i32>, <16 x i32> addrspace(1)* %in %ext = sext <16 x i32> %ld to <16 x i64> @@ -322,10 +323,10 @@ ; GCN-NOHSA: buffer_load_dwordx4 ; GCN-NOHSA: buffer_load_dwordx4 -; GCN-HSA: flat_load_dwordx4 -; GCN-HSA: flat_load_dwordx4 -; GCN-HSA: flat_load_dwordx4 -; GCN-HSA: flat_load_dwordx4 +; GCN-HSA: {{flat|global}}_load_dwordx4 +; GCN-HSA: {{flat|global}}_load_dwordx4 +; GCN-HSA: {{flat|global}}_load_dwordx4 +; GCN-HSA: {{flat|global}}_load_dwordx4 ; GCN-NOHSA: buffer_store_dwordx4 ; GCN-NOHSA: buffer_store_dwordx4 @@ -336,14 +337,14 @@ ; GCN-NOHSA: buffer_store_dwordx4 ; GCN-NOHSA: buffer_store_dwordx4 -; GCN-HSA: flat_store_dwordx4 -; GCN-HSA: flat_store_dwordx4 -; GCN-HSA: flat_store_dwordx4 -; GCN-HSA: flat_store_dwordx4 -; GCN-HSA: flat_store_dwordx4 -; GCN-HSA: flat_store_dwordx4 -; GCN-HSA: flat_store_dwordx4 -; GCN-HSA: flat_store_dwordx4 +; GCN-HSA: {{flat|global}}_store_dwordx4 +; GCN-HSA: {{flat|global}}_store_dwordx4 +; GCN-HSA: {{flat|global}}_store_dwordx4 +; GCN-HSA: {{flat|global}}_store_dwordx4 +; GCN-HSA: {{flat|global}}_store_dwordx4 +; GCN-HSA: {{flat|global}}_store_dwordx4 +; GCN-HSA: {{flat|global}}_store_dwordx4 +; GCN-HSA: {{flat|global}}_store_dwordx4 define amdgpu_kernel void @global_zextload_v16i32_to_v16i64(<16 x i64> addrspace(1)* %out, <16 x i32> addrspace(1)* %in) #0 { %ld = load <16 x i32>, <16 x i32> addrspace(1)* %in %ext = zext <16 x i32> %ld to <16 x i64> @@ -362,14 +363,14 @@ ; GCN-NOHSA: buffer_load_dwordx4 ; GCN-NOHSA-DAG: buffer_load_dwordx4 -; GCN-HSA: flat_load_dwordx4 -; GCN-HSA: flat_load_dwordx4 -; GCN-HSA: flat_load_dwordx4 -; GCN-HSA: flat_load_dwordx4 -; GCN-HSA: flat_load_dwordx4 -; GCN-HSA: flat_load_dwordx4 -; GCN-HSA: flat_load_dwordx4 -; GCN-HSA: flat_load_dwordx4 +; GCN-HSA: {{flat|global}}_load_dwordx4 +; GCN-HSA: {{flat|global}}_load_dwordx4 +; GCN-HSA: {{flat|global}}_load_dwordx4 +; GCN-HSA: {{flat|global}}_load_dwordx4 +; GCN-HSA: {{flat|global}}_load_dwordx4 +; GCN-HSA: {{flat|global}}_load_dwordx4 +; GCN-HSA: {{flat|global}}_load_dwordx4 +; GCN-HSA: {{flat|global}}_load_dwordx4 ; GCN-DAG: v_ashrrev_i32 ; GCN-DAG: v_ashrrev_i32 @@ -424,25 +425,25 @@ ; GCN-NOHSA: buffer_store_dwordx4 ; GCN-NOHSA: buffer_store_dwordx4 -; GCN-HSA-DAG: flat_store_dwordx4 -; GCN-HSA-DAG: flat_store_dwordx4 -; GCN-HSA-DAG: flat_store_dwordx4 -; GCN-HSA-DAG: flat_store_dwordx4 +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 -; GCN-HSA-DAG: flat_store_dwordx4 -; GCN-HSA-DAG: flat_store_dwordx4 -; GCN-HSA-DAG: flat_store_dwordx4 -; GCN-HSA-DAG: flat_store_dwordx4 +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 -; GCN-HSA-DAG: flat_store_dwordx4 -; GCN-HSA-DAG: flat_store_dwordx4 -; GCN-HSA-DAG: flat_store_dwordx4 -; GCN-HSA-DAG: flat_store_dwordx4 +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 -; GCN-HSA-DAG: flat_store_dwordx4 -; GCN-HSA-DAG: flat_store_dwordx4 -; GCN-HSA-DAG: flat_store_dwordx4 -; GCN-HSA-DAG: flat_store_dwordx4 +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 define amdgpu_kernel void @global_sextload_v32i32_to_v32i64(<32 x i64> addrspace(1)* %out, <32 x i32> addrspace(1)* %in) #0 { %ld = load <32 x i32>, <32 x i32> addrspace(1)* %in @@ -461,14 +462,14 @@ ; GCN-NOHSA: buffer_load_dwordx4 ; GCN-NOHSA: buffer_load_dwordx4 -; GCN-HSA: flat_load_dwordx4 -; GCN-HSA: flat_load_dwordx4 -; GCN-HSA: flat_load_dwordx4 -; GCN-HSA: flat_load_dwordx4 -; GCN-HSA: flat_load_dwordx4 -; GCN-HSA: flat_load_dwordx4 -; GCN-HSA: flat_load_dwordx4 -; GCN-HSA: flat_load_dwordx4 +; GCN-HSA: {{flat|global}}_load_dwordx4 +; GCN-HSA: {{flat|global}}_load_dwordx4 +; GCN-HSA: {{flat|global}}_load_dwordx4 +; GCN-HSA: {{flat|global}}_load_dwordx4 +; GCN-HSA: {{flat|global}}_load_dwordx4 +; GCN-HSA: {{flat|global}}_load_dwordx4 +; GCN-HSA: {{flat|global}}_load_dwordx4 +; GCN-HSA: {{flat|global}}_load_dwordx4 ; GCN-NOHSA-DAG: buffer_store_dwordx4 @@ -492,25 +493,25 @@ ; GCN-NOHSA-DAG: buffer_store_dwordx4 -; GCN-HSA-DAG: flat_store_dwordx4 -; GCN-HSA-DAG: flat_store_dwordx4 -; GCN-HSA-DAG: flat_store_dwordx4 -; GCN-HSA-DAG: flat_store_dwordx4 +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 -; GCN-HSA-DAG: flat_store_dwordx4 -; GCN-HSA-DAG: flat_store_dwordx4 -; GCN-HSA-DAG: flat_store_dwordx4 -; GCN-HSA-DAG: flat_store_dwordx4 +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 -; GCN-HSA-DAG: flat_store_dwordx4 -; GCN-HSA-DAG: flat_store_dwordx4 -; GCN-HSA-DAG: flat_store_dwordx4 -; GCN-HSA-DAG: flat_store_dwordx4 +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 -; GCN-HSA-DAG: flat_store_dwordx4 -; GCN-HSA-DAG: flat_store_dwordx4 -; GCN-HSA-DAG: flat_store_dwordx4 -; GCN-HSA-DAG: flat_store_dwordx4 +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 define amdgpu_kernel void @global_zextload_v32i32_to_v32i64(<32 x i64> addrspace(1)* %out, <32 x i32> addrspace(1)* %in) #0 { %ld = load <32 x i32>, <32 x i32> addrspace(1)* %in %ext = zext <32 x i32> %ld to <32 x i64> @@ -518,4 +519,69 @@ ret void } +; FUNC-LABEL: {{^}}global_load_v32i32: +; GCN-NOHSA: buffer_load_dwordx4 +; GCN-NOHSA: buffer_load_dwordx4 +; GCN-NOHSA: buffer_load_dwordx4 +; GCN-NOHSA: buffer_load_dwordx4 +; GCN-NOHSA: buffer_load_dwordx4 +; GCN-NOHSA: buffer_load_dwordx4 +; GCN-NOHSA: buffer_load_dwordx4 +; GCN-NOHSA: buffer_load_dwordx4 + +; GCN-HSA: {{flat|global}}_load_dwordx4 +; GCN-HSA: {{flat|global}}_load_dwordx4 +; GCN-HSA: {{flat|global}}_load_dwordx4 +; GCN-HSA: {{flat|global}}_load_dwordx4 +; GCN-HSA: {{flat|global}}_load_dwordx4 +; GCN-HSA: {{flat|global}}_load_dwordx4 +; GCN-HSA: {{flat|global}}_load_dwordx4 +; GCN-HSA: {{flat|global}}_load_dwordx4 + + +; GCN-NOHSA-DAG: buffer_store_dwordx4 +; GCN-NOHSA-DAG: buffer_store_dwordx4 +; GCN-NOHSA-DAG: buffer_store_dwordx4 +; GCN-NOHSA-DAG: buffer_store_dwordx4 + +; GCN-NOHSA-DAG: buffer_store_dwordx4 +; GCN-NOHSA-DAG: buffer_store_dwordx4 +; GCN-NOHSA-DAG: buffer_store_dwordx4 +; GCN-NOHSA-DAG: buffer_store_dwordx4 + +; GCN-NOHSA-DAG: buffer_store_dwordx4 +; GCN-NOHSA-DAG: buffer_store_dwordx4 +; GCN-NOHSA-DAG: buffer_store_dwordx4 +; GCN-NOHSA-DAG: buffer_store_dwordx4 + +; GCN-NOHSA-DAG: buffer_store_dwordx4 +; GCN-NOHSA-DAG: buffer_store_dwordx4 +; GCN-NOHSA-DAG: buffer_store_dwordx4 +; GCN-NOHSA-DAG: buffer_store_dwordx4 + +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 + +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 + +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 + +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 +; GCN-HSA-DAG: {{flat|global}}_store_dwordx4 +define amdgpu_kernel void @global_load_v32i32(<32 x i32> addrspace(1)* %out, <32 x i32> addrspace(1)* %in) #0 { + %ld = load <32 x i32>, <32 x i32> addrspace(1)* %in + store <32 x i32> %ld, <32 x i32> addrspace(1)* %out + ret void +} + attributes #0 = { nounwind } Index: llvm/trunk/test/CodeGen/AMDGPU/load-local-i32.ll =================================================================== --- llvm/trunk/test/CodeGen/AMDGPU/load-local-i32.ll +++ llvm/trunk/test/CodeGen/AMDGPU/load-local-i32.ll @@ -265,4 +265,14 @@ ret void } +; FUNC-LABEL: {{^}}local_load_v32i32: +; SICIVI: s_mov_b32 m0, -1 +; GFX9-NOT: m0 + +define amdgpu_kernel void @local_load_v32i32(<32 x i32> addrspace(3)* %out, <32 x i32> addrspace(3)* %in) #0 { + %ld = load <32 x i32>, <32 x i32> addrspace(3)* %in + store <32 x i32> %ld, <32 x i32> addrspace(3)* %out + ret void +} + attributes #0 = { nounwind }