Index: lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp
===================================================================
--- lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp
+++ lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp
@@ -18,6 +18,7 @@
 #include "AMDGPUTargetMachine.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Analysis/DivergenceAnalysis.h"
+#include "llvm/Analysis/Loads.h"
 #include "llvm/CodeGen/Passes.h"
 #include "llvm/CodeGen/TargetPassConfig.h"
 #include "llvm/IR/Attributes.h"
@@ -38,6 +39,7 @@
 #include "llvm/IR/Value.h"
 #include "llvm/Pass.h"
 #include "llvm/Support/Casting.h"
+#include "AMDGPU.h"
 #include <cassert>
 #include <iterator>
 
@@ -53,6 +55,7 @@
   DivergenceAnalysis *DA = nullptr;
   Module *Mod = nullptr;
   bool HasUnsafeFPMath = false;
+  AMDGPUAS AMDGPUASI;
 
   /// \brief Copies exact/nsw/nuw flags (if any) from binary operation \p I to
   /// binary operation \p V.
@@ -133,6 +136,7 @@
 
   bool visitInstruction(Instruction &I) { return false; }
   bool visitBinaryOperator(BinaryOperator &I);
+  bool visitLoadInst(LoadInst &I);
   bool visitICmpInst(ICmpInst &I);
   bool visitSelectInst(SelectInst &I);
 
@@ -441,6 +445,28 @@
   return Changed;
 }
 
+bool AMDGPUCodeGenPrepare::visitLoadInst(LoadInst &I) {
+  Type * Ty = I.getType();
+  VectorType *VT = dyn_cast<VectorType>(Ty);
+  if (I.getPointerAddressSpace() == AMDGPUASI.CONSTANT_ADDRESS &&
+      !I.isVolatile() && (!VT || (VT && VT->getBitWidth() < 32)) &&
+      needsPromotionToI32(I.getType()) && DA->isUniform(&I)) {
+    IRBuilder<> Builder(&I);
+    Builder.SetCurrentDebugLocation(I.getDebugLoc());
+    Type *I32Ty = getI32Ty(Builder, I.getType());
+    Type *PT = PointerType::get(I32Ty, I.getPointerAddressSpace());
+    Value *BitCast= Builder.CreateBitCast(I.getPointerOperand(), PT);
+    Value *WidenLoad = Builder.CreateLoad(BitCast);
+    Value *TruncRes = Builder.CreateTrunc(WidenLoad, I.getType());
+
+    I.replaceAllUsesWith(TruncRes);
+    I.eraseFromParent();
+    return true;
+  }
+
+  return false;
+}
+
 bool AMDGPUCodeGenPrepare::visitICmpInst(ICmpInst &I) {
   bool Changed = false;
 
Index: test/CodeGen/AMDGPU/load-constant-i16.ll
===================================================================
--- test/CodeGen/AMDGPU/load-constant-i16.ll
+++ test/CodeGen/AMDGPU/load-constant-i16.ll
@@ -4,8 +4,8 @@
 ; RUN: llc -march=r600 -mcpu=redwood -verify-machineinstrs < %s | FileCheck -check-prefix=EG -check-prefix=FUNC %s
 
 ; FUNC-LABEL: {{^}}constant_load_i16:
-; GCN-NOHSA: buffer_load_ushort v{{[0-9]+}}
-; GCN-HSA: flat_load_ushort
+; GCN-NOHSA: s_load_dword s{{[0-9]+}}
+; GCN-HSA: s_load_dword
 
 ; EG: VTX_READ_16 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0, #1
 define amdgpu_kernel void @constant_load_i16(i16 addrspace(1)* %out, i16 addrspace(2)* %in) {
@@ -73,10 +73,10 @@
 }
 
 ; FUNC-LABEL: {{^}}constant_zextload_i16_to_i32:
-; GCN-NOHSA: buffer_load_ushort
+; GCN-NOHSA: s_load_dword
 ; GCN-NOHSA: buffer_store_dword
 
-; GCN-HSA: flat_load_ushort
+; GCN-HSA: s_load_dword
 ; GCN-HSA: flat_store_dword
 
 ; EG: VTX_READ_16 T{{[0-9]+\.X, T[0-9]+\.X}}, 0, #1
@@ -88,10 +88,10 @@
 }
 
 ; FUNC-LABEL: {{^}}constant_sextload_i16_to_i32:
-; GCN-NOHSA: buffer_load_sshort
+; GCN-NOHSA: s_load_dword
 ; GCN-NOHSA: buffer_store_dword
 
-; GCN-HSA: flat_load_sshort
+; GCN-HSA: s_load_dword
 ; GCN-HSA: flat_store_dword
 
 ; EG: VTX_READ_16 [[DST:T[0-9]\.[XYZW]]], [[DST]], 0, #1
@@ -105,8 +105,8 @@
 }
 
 ; FUNC-LABEL: {{^}}constant_zextload_v1i16_to_v1i32:
-; GCN-NOHSA: buffer_load_ushort
-; GCN-HSA: flat_load_ushort
+; GCN-NOHSA: s_load_dword
+; GCN-HSA: s_load_dword
 
 ; EG: VTX_READ_16 T{{[0-9]+\.X, T[0-9]+\.X}}, 0, #1
 define amdgpu_kernel void @constant_zextload_v1i16_to_v1i32(<1 x i32> addrspace(1)* %out, <1 x i16> addrspace(2)* %in) #0 {
@@ -117,8 +117,8 @@
 }
 
 ; FUNC-LABEL: {{^}}constant_sextload_v1i16_to_v1i32:
-; GCN-NOHSA: buffer_load_sshort
-; GCN-HSA: flat_load_sshort
+; GCN-NOHSA: s_load_dword
+; GCN-HSA: s_load_dword
 
 ; EG: VTX_READ_16 [[DST:T[0-9]\.[XYZW]]], [[DST]], 0, #1
 ; EG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, [[DST]], 0.0, literal
@@ -429,12 +429,12 @@
 }
 
 ; FUNC-LABEL: {{^}}constant_zextload_i16_to_i64:
-; GCN-NOHSA-DAG: buffer_load_ushort v[[LO:[0-9]+]],
-; GCN-HSA-DAG: flat_load_ushort v[[LO:[0-9]+]],
+; GCN-NOHSA-DAG: s_load_dword s[[LO:[0-9]+]],
+; GCN-HSA-DAG: s_load_dword s[[LO:[0-9]+]],
 ; GCN-DAG: v_mov_b32_e32 v[[HI:[0-9]+]], 0{{$}}
 
 ; GCN-NOHSA: buffer_store_dwordx2 v{{\[}}[[LO]]:[[HI]]]
-; GCN-HSA: flat_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[LO]]:[[HI]]{{\]}}
+; GCN-HSA: flat_store_dwordx2 v{{\[}}[[LO]]:{{[0-9]+}}{{\]}}, v{{\[}}{{[0-9]+}}:[[HI]]{{\]}}
 
 ; EG: VTX_READ_16 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0, #1
 ; EG: MOV {{.*}}, 0.0
@@ -451,14 +451,13 @@
 ;          t31: i64 = any_extend t28 
 ;        t33: i64 = sign_extend_inreg t31, ValueType:ch:i16
 
-; GCN-NOHSA-SI-DAG: buffer_load_sshort v[[LO:[0-9]+]],
-; GCN-HSA-DAG: flat_load_sshort v[[LO:[0-9]+]],
-; GCN-NOHSA-VI-DAG: buffer_load_ushort v[[ULO:[0-9]+]],
-; GCN-NOHSA-VI-DAG: v_bfe_i32 v[[LO:[0-9]+]], v[[ULO]], 0, 16
-; GCN-DAG: v_ashrrev_i32_e32 v[[HI:[0-9]+]], 31, v[[LO]]
+; GCN-NOHSA-SI-DAG: s_load_dword s[[LO:[0-9]+]],
+; GCN-HSA-DAG: s_load_dword s[[LO:[0-9]+]],
+; GCN-NOHSA-VI-DAG: s_load_dword s[[ULO:[0-9]+]],
+; GCN-NOHSA-VI-DAG: s_bfe_i64 s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0x100000
 
 ; GCN-NOHSA: buffer_store_dwordx2 v{{\[}}[[LO]]:[[HI]]]
-; GCN-HSA: flat_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[LO]]:[[HI]]{{\]}}
+; GCN-HSA: flat_store_dwordx2 v{{\[}}{{[0-9]+}}:[[HI]]{{\]}}, v{{\[}}[[LO]]:{{[0-9]+}}{{\]}}
 
 ; EG: VTX_READ_16 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0, #1
 ; EG: ASHR {{\**}} {{T[0-9]\.[XYZW]}}, {{.*}}, literal
Index: test/CodeGen/AMDGPU/load-constant-i8.ll
===================================================================
--- test/CodeGen/AMDGPU/load-constant-i8.ll
+++ test/CodeGen/AMDGPU/load-constant-i8.ll
@@ -5,8 +5,8 @@
 
 
 ; FUNC-LABEL: {{^}}constant_load_i8:
-; GCN-NOHSA: buffer_load_ubyte v{{[0-9]+}}
-; GCN-HSA: flat_load_ubyte
+; GCN-NOHSA: s_load_dword s{{[0-9]+}}
+; GCN-HSA: s_load_dword
 
 ; EG: VTX_READ_8 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0, #1
 ; TODO: NOT AND
@@ -18,8 +18,8 @@
 }
 
 ; FUNC-LABEL: {{^}}constant_load_v2i8:
-; GCN-NOHSA: buffer_load_ushort v
-; GCN-HSA: flat_load_ushort v
+; GCN-NOHSA: s_load_dwordx2 s
+; GCN-HSA: s_load_dwordx2 s
 
 ; EG: VTX_READ_16 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0, #1
 define amdgpu_kernel void @constant_load_v2i8(<2 x i8> addrspace(1)* %out, <2 x i8> addrspace(2)* %in) #0 {
@@ -30,7 +30,7 @@
 }
 
 ; FUNC-LABEL: {{^}}constant_load_v3i8:
-; GCN: s_load_dword s
+; GCN: s_load_dwordx4 s
 
 ; EG: VTX_READ_32 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0, #1
 define amdgpu_kernel void @constant_load_v3i8(<3 x i8> addrspace(1)* %out, <3 x i8> addrspace(2)* %in) #0 {
@@ -74,8 +74,8 @@
 }
 
 ; FUNC-LABEL: {{^}}constant_zextload_i8_to_i32:
-; GCN-NOHSA: buffer_load_ubyte v{{[0-9]+}},
-; GCN-HSA: flat_load_ubyte
+; GCN-NOHSA: s_load_dword s{{[0-9]+}},
+; GCN-HSA: s_load_dword
 
 ; EG: VTX_READ_8 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0, #1
 define amdgpu_kernel void @constant_zextload_i8_to_i32(i32 addrspace(1)* %out, i8 addrspace(2)* %in) #0 {
@@ -86,8 +86,8 @@
 }
 
 ; FUNC-LABEL: {{^}}constant_sextload_i8_to_i32:
-; GCN-NOHSA: buffer_load_sbyte
-; GCN-HSA: flat_load_sbyte
+; GCN-NOHSA: s_load_dword
+; GCN-HSA: s_load_dword
 
 ; EG: VTX_READ_8 [[DST:T[0-9]+\.X]], T{{[0-9]+}}.X, 0, #1
 ; EG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, [[DST]], 0.0, literal
@@ -122,8 +122,8 @@
 }
 
 ; FUNC-LABEL: {{^}}constant_zextload_v2i8_to_v2i32:
-; GCN-NOHSA: buffer_load_ushort
-; GCN-HSA: flat_load_ushort
+; GCN-NOHSA: s_load_dword
+; GCN-HSA: s_load_dword
 
 ; EG: VTX_READ_16 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0, #1
 ; TODO: This should use DST, but for some there are redundant MOVs
@@ -137,12 +137,12 @@
 }
 
 ; FUNC-LABEL: {{^}}constant_sextload_v2i8_to_v2i32:
-; GCN-NOHSA: buffer_load_ushort
+; GCN-NOHSA: s_load_dwordx2
 
-; GCN-HSA: flat_load_ushort
+; GCN-HSA: s_load_dword
 
-; GCN: v_bfe_i32
-; GCN: v_bfe_i32
+; GCN: s_sext_i32
+; GCN: s_sext_i32
 
 ; EG: VTX_READ_16 [[DST:T[0-9]+\.X]], T{{[0-9]+}}.X, 0, #1
 ; TODO: These should use DST, but for some there are redundant MOVs
@@ -158,10 +158,10 @@
 }
 
 ; FUNC-LABEL: {{^}}constant_zextload_v3i8_to_v3i32:
-; GCN: s_load_dword s
+; GCN: s_load_dwordx4 s
 
-; GCN-DAG: s_bfe_u32
-; GCN-DAG: s_bfe_u32
+; GCN-DAG: s_and_b32
+; GCN-DAG: s_and_b32
 ; GCN-DAG: s_and_b32
 
 ; EG: VTX_READ_32 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0, #1
@@ -179,11 +179,11 @@
 }
 
 ; FUNC-LABEL: {{^}}constant_sextload_v3i8_to_v3i32:
-; GCN: s_load_dword s
+; GCN: s_load_dwordx4 s
 
-; GCN-DAG: s_bfe_i32
-; GCN-DAG: s_bfe_i32
-; GCN-DAG: s_bfe_i32
+; GCN-DAG: s_sext_i32
+; GCN-DAG: s_sext_i32
+; GCN-DAG: s_sext_i32
 
 ; EG: VTX_READ_32 [[DST:T[0-9]+\.X]], T{{[0-9]+}}.X, 0, #1
 ; TODO: These should use DST, but for some there are redundant MOVs
@@ -562,11 +562,11 @@
 ; FUNC-LABEL: {{^}}constant_zextload_i8_to_i64:
 ; GCN-DAG: v_mov_b32_e32 v[[HI:[0-9]+]], 0{{$}}
 
-; GCN-NOHSA-DAG: buffer_load_ubyte v[[LO:[0-9]+]],
+; GCN-NOHSA-DAG: s_load_dword s[[LO:[0-9]+]],
 ; GCN-NOHSA: buffer_store_dwordx2 v{{\[}}[[LO]]:[[HI]]]
 
-; GCN-HSA-DAG: flat_load_ubyte v[[LO:[0-9]+]],
-; GCN-HSA: flat_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[LO]]:[[HI]]]
+; GCN-HSA-DAG: s_load_dword s[[LO:[0-9]+]],
+; GCN-HSA: flat_store_dwordx2 v{{\[}}[[LO]]:{{[0-9]+}}{{\]}}, v{{\[}}{{[0-9]+}}:[[HI]]{{\]}}
 
 ; EG: VTX_READ_8 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0, #1
 ; EG: MOV {{.*}}, 0.0
@@ -578,12 +578,12 @@
 }
 
 ; FUNC-LABEL: {{^}}constant_sextload_i8_to_i64:
-; GCN-NOHSA: buffer_load_sbyte v[[LO:[0-9]+]],
-; GCN-HSA: flat_load_sbyte v[[LO:[0-9]+]],
-; GCN: v_ashrrev_i32_e32 v[[HI:[0-9]+]], 31, v[[LO]]
+; GCN-NOHSA: s_load_dword s[[LO:[0-9]+]],
+; GCN-HSA: s_load_dword s[[LO:[0-9]+]],
+; GCN: s_bfe_i64 s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0x80000
 
 ; GCN-NOHSA: buffer_store_dwordx2 v{{\[}}[[LO]]:[[HI]]{{\]}}
-; GCN-HSA: flat_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[LO]]:[[HI]]{{\]}}
+; GCN-HSA: flat_store_dwordx2 v{{\[}}{{[0-9]+}}:[[HI]]{{\]}}, v{{\[}}[[LO]]:{{[0-9]+}}{{\]}}
 
 ; EG: VTX_READ_8 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0, #1
 ; EG: ASHR {{\**}} {{T[0-9]\.[XYZW]}}, {{.*}}, literal
@@ -739,10 +739,10 @@
 ; }
 
 ; FUNC-LABEL: {{^}}constant_zextload_i8_to_i16:
-; GCN-NOHSA: buffer_load_ubyte v[[VAL:[0-9]+]],
+; GCN-NOHSA: s_load_dword s[[VAL:[0-9]+]],
 ; GCN-NOHSA: buffer_store_short v[[VAL]]
 
-; GCN-HSA: flat_load_ubyte v[[VAL:[0-9]+]],
+; GCN-HSA: s_load_dword s[[VAL:[0-9]+]],
 ; GCN-HSA: flat_store_short v{{\[[0-9]+:[0-9]+\]}}, v[[VAL]]
 define amdgpu_kernel void @constant_zextload_i8_to_i16(i16 addrspace(1)* %out, i8 addrspace(2)* %in) #0 {
   %a = load i8, i8 addrspace(2)* %in
@@ -752,8 +752,8 @@
 }
 
 ; FUNC-LABEL: {{^}}constant_sextload_i8_to_i16:
-; GCN-NOHSA: buffer_load_sbyte v[[VAL:[0-9]+]],
-; GCN-HSA: flat_load_sbyte v[[VAL:[0-9]+]],
+; GCN-NOHSA: s_load_dword s[[VAL:[0-9]+]],
+; GCN-HSA: s_load_dword s[[VAL:[0-9]+]],
 
 ; GCN-NOHSA: buffer_store_short v[[VAL]]
 ; GCN-HSA: flat_store_short v{{\[[0-9]+:[0-9]+\]}}, v[[VAL]]
Index: test/CodeGen/AMDGPU/unaligned-load-store.ll
===================================================================
--- test/CodeGen/AMDGPU/unaligned-load-store.ll
+++ test/CodeGen/AMDGPU/unaligned-load-store.ll
@@ -519,7 +519,7 @@
 }
 
 ; SI-LABEL: {{^}}constant_align4_load_i8:
-; SI: buffer_load_ubyte
+; SI: s_load_dword
 ; SI: buffer_store_byte
 define amdgpu_kernel void @constant_align4_load_i8(i8 addrspace(2)* %p, i8 addrspace(1)* %r) #0 {
   %v = load i8, i8 addrspace(2)* %p, align 4
@@ -528,7 +528,7 @@
 }
 
 ; SI-LABEL: {{^}}constant_align2_load_i8:
-; SI: buffer_load_ubyte
+; SI: s_load_dword
 ; SI: buffer_store_byte
 define amdgpu_kernel void @constant_align2_load_i8(i8 addrspace(2)* %p, i8 addrspace(1)* %r) #0 {
   %v = load i8, i8 addrspace(2)* %p, align 2