DAG: Avoid stack lowering if bitcast has an illegal vector result type

[arsenm]

Summary

A bitcast of <10 x i32> to <5 x i64> was ending up on the
stack. Instead of doing that, handle the case where the new type
doesn't evenly divide but the elements do. Extract the individual
elements and pad with undef.

Avoids stack usage for bitcasts involving <5 x i64>. In some of these
cases, later optimizations actually eliminated the stack objects but
left behind the unused temporary stack object to final emission.

Fixes: SWDEV-377548

[RKSimon]

llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp

4888

replace with Ops.append() ?

[arsenm]

Use append

[RKSimon]

LGTM

[arsenm]

ab6b48b7116d118131e7ef459e4d234b4366564f

llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp

case TargetLowering::TypeWidenVector:
     if (WidenVT.bitsEq(InVT))
       // The input widens to the same size. Convert to the widen value.
       return DAG.getNode(ISD::BITCAST, dl, WidenVT, InOp);
     break;
   }
 
   unsigned WidenSize = WidenVT.getSizeInBits();
   unsigned InSize = InVT.getSizeInBits();
+  unsigned InScalarSize = InVT.getScalarSizeInBits();
   // x86mmx is not an acceptable vector element type, so don't try.
-  if (WidenSize % InSize == 0 && InVT != MVT::x86mmx) {
+  if (WidenSize % InScalarSize == 0 && InVT != MVT::x86mmx) {
     // Determine new input vector type.  The new input vector type will use
     // the same element type (if its a vector) or use the input type as a
     // vector.  It is the same size as the type to widen to.
     EVT NewInVT;
-    unsigned NewNumElts = WidenSize / InSize;
+    unsigned NewNumParts = WidenSize / InSize;
     if (InVT.isVector()) {
       EVT InEltVT = InVT.getVectorElementType();
       NewInVT = EVT::getVectorVT(*DAG.getContext(), InEltVT,
                                  WidenSize / InEltVT.getSizeInBits());
     } else {
-      NewInVT = EVT::getVectorVT(*DAG.getContext(), InVT, NewNumElts);
+      NewInVT = EVT::getVectorVT(*DAG.getContext(), InVT, NewNumParts);
     }
 
     if (TLI.isTypeLegal(NewInVT)) {
       SDValue NewVec;
       if (InVT.isVector()) {
         // Because the result and the input are different vector types, widening
-        // the result could create a legal type but widening the input might make
-        // it an illegal type that might lead to repeatedly splitting the input
-        // and then widening it. To avoid this, we widen the input only if
+        // the result could create a legal type but widening the input might
+        // make it an illegal type that might lead to repeatedly splitting the
+        // input and then widening it. To avoid this, we widen the input only if
         // it results in a legal type.
-        SmallVector<SDValue, 16> Ops(NewNumElts, DAG.getUNDEF(InVT));
+        if (WidenSize % InSize == 0) {
+          SmallVector<SDValue, 16> Ops(NewNumParts, DAG.getUNDEF(InVT));
           Ops[0] = InOp;
 
           NewVec = DAG.getNode(ISD::CONCAT_VECTORS, dl, NewInVT, Ops);
         } else {
+          SmallVector<SDValue, 16> Ops;
+          DAG.ExtractVectorElements(InOp, Ops);
+          Ops.append(WidenSize / InScalarSize - Ops.size(),
+                     DAG.getUNDEF(InVT.getVectorElementType()));
+
+          NewVec = DAG.getNode(ISD::BUILD_VECTOR, dl, NewInVT, Ops);
+        }
+      } else {
         NewVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NewInVT, InOp);
       }
       return DAG.getNode(ISD::BITCAST, dl, WidenVT, NewVec);
     }
   }
 
   return CreateStackStoreLoad(InOp, WidenVT);
 }

llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.ll

if:
   %cmp1 = icmp eq i32 %cond, 1
   br i1 %cmp1, label %if, label %end
 
 end:
   %phi_cast = phi <4 x double> [zeroinitializer, %entry], [%cast, %if]
   store <4 x double> %phi_cast, ptr addrspace(1) %out
   ret void
 }
+
+; FUNC-LABEL: {{^}}bitcast_v20f16_to_v5f64:
+; SI: ScratchSize: 0
+define amdgpu_kernel void @bitcast_v20f16_to_v5f64(i32 %cond, ptr addrspace(1) %out, <20 x half> %value) {
+entry:
+  %cmp0 = icmp eq i32 %cond, 0
+  br i1 %cmp0, label %if, label %end
+
+if:
+  %phi_value = phi <20 x half> [zeroinitializer, %entry], [%value, %if]
+  %cast = bitcast <20 x half> %phi_value to <5 x double>
+  %cmp1 = icmp eq i32 %cond, 1
+  br i1 %cmp1, label %if, label %end
+
+end:
+  %phi_cast = phi <5 x double> [zeroinitializer, %entry], [%cast, %if]
+  store <5 x double> %phi_cast, ptr addrspace(1) %out
+  ret void
+}
+
+; FUNC-LABEL: {{^}}bitcast_v10f32_to_v5f64:
+; SI: ScratchSize: 0
+define amdgpu_kernel void @bitcast_v10f32_to_v5f64(i32 %cond, ptr addrspace(1) %out, <10 x float> %value) {
+entry:
+  %cmp0 = icmp eq i32 %cond, 0
+  br i1 %cmp0, label %if, label %end
+
+if:
+  %phi_value = phi <10 x float> [zeroinitializer, %entry], [%value, %if]
+  %cast = bitcast <10 x float> %phi_value to <5 x double>
+  %cmp1 = icmp eq i32 %cond, 1
+  br i1 %cmp1, label %if, label %end
+
+end:
+  %phi_cast = phi <5 x double> [zeroinitializer, %entry], [%cast, %if]
+  store <5 x double> %phi_cast, ptr addrspace(1) %out
+  ret void
+}
+
+; FUNC-LABEL: {{^}}bitcast_v10i32_to_v5f64:
+; SI: ScratchSize: 0
+define amdgpu_kernel void @bitcast_v10i32_to_v5f64(i32 %cond, ptr addrspace(1) %out, <10 x i32> %value) {
+entry:
+  %cmp0 = icmp eq i32 %cond, 0
+  br i1 %cmp0, label %if, label %end
+
+if:
+  %phi_value = phi <10 x i32> [zeroinitializer, %entry], [%value, %if]
+  %cast = bitcast <10 x i32> %phi_value to <5 x double>
+  %cmp1 = icmp eq i32 %cond, 1
+  br i1 %cmp1, label %if, label %end
+
+end:
+  %phi_cast = phi <5 x double> [zeroinitializer, %entry], [%cast, %if]
+  store <5 x double> %phi_cast, ptr addrspace(1) %out
+  ret void
+}
+
+; FUNC-LABEL: {{^}}bitcast_v10f32_to_v5i64:
+; SI: ScratchSize: 0
+define amdgpu_kernel void @bitcast_v10f32_to_v5i64(i32 %cond, ptr addrspace(1) %out, <10 x float> %value) {
+entry:
+  %cmp0 = icmp eq i32 %cond, 0
+  br i1 %cmp0, label %if, label %end
+
+if:
+  %phi_value = phi <10 x float> [zeroinitializer, %entry], [%value, %if]
+  %cast = bitcast <10 x float> %phi_value to <5 x i64>
+  %cmp1 = icmp eq i32 %cond, 1
+  br i1 %cmp1, label %if, label %end
+
+end:
+  %phi_cast = phi <5 x i64> [zeroinitializer, %entry], [%cast, %if]
+  store <5 x i64> %phi_cast, ptr addrspace(1) %out
+  ret void
+}
+
+; FUNC-LABEL: {{^}}bitcast_v10i32_to_v5i64:
+; SI: ScratchSize: 0
+define amdgpu_kernel void @bitcast_v10i32_to_v5i64(i32 %cond, ptr addrspace(1) %out, <10 x i32> %value) {
+entry:
+  %cmp0 = icmp eq i32 %cond, 0
+  br i1 %cmp0, label %if, label %end
+
+if:
+  %phi_value = phi <10 x i32> [zeroinitializer, %entry], [%value, %if]
+  %cast = bitcast <10 x i32> %phi_value to <5 x i64>
+  %cmp1 = icmp eq i32 %cond, 1
+  br i1 %cmp1, label %if, label %end
+
+end:
+  %phi_cast = phi <5 x i64> [zeroinitializer, %entry], [%cast, %if]
+  store <5 x i64> %phi_cast, ptr addrspace(1) %out
+  ret void
+}
+
+; FUNC-LABEL: {{^}}bitcast_v40i8_to_v5f64:
+; SI: ScratchSize: 0
+define amdgpu_kernel void @bitcast_v40i8_to_v5f64(i32 %cond, ptr addrspace(1) %out, <40 x i8> %value) {
+entry:
+  %cmp0 = icmp eq i32 %cond, 0
+  br i1 %cmp0, label %if, label %end
+
+if:
+  %phi_value = phi <40 x i8> [zeroinitializer, %entry], [%value, %if]
+  %cast = bitcast <40 x i8> %phi_value to <5 x double>
+  %cmp1 = icmp eq i32 %cond, 1
+  br i1 %cmp1, label %if, label %end
+
+end:
+  %phi_cast = phi <5 x double> [zeroinitializer, %entry], [%cast, %if]
+  store <5 x double> %phi_cast, ptr addrspace(1) %out
+  ret void
+}
+
+; FUNC-LABEL: {{^}}bitcast_v40i8_to_v5i64:
+; SI: ScratchSize: 0
+define amdgpu_kernel void @bitcast_v40i8_to_v5i64(i32 %cond, ptr addrspace(1) %out, <40 x i8> %value) {
+entry:
+  %cmp0 = icmp eq i32 %cond, 0
+  br i1 %cmp0, label %if, label %end
+
+if:
+  %phi_value = phi <40 x i8> [zeroinitializer, %entry], [%value, %if]
+  %cast = bitcast <40 x i8> %phi_value to <5 x i64>
+  %cmp1 = icmp eq i32 %cond, 1
+  br i1 %cmp1, label %if, label %end
+
+end:
+  %phi_cast = phi <5 x i64> [zeroinitializer, %entry], [%cast, %if]
+  store <5 x i64> %phi_cast, ptr addrspace(1) %out
+  ret void
+}
+
+; FUNC-LABEL: {{^}}bitcast_v5f64_to_v10f32:
+; SI: ScratchSize: 0
+define amdgpu_kernel void @bitcast_v5f64_to_v10f32(i32 %cond, ptr addrspace(1) %out, <5 x double> %value) {
+entry:
+  %cmp0 = icmp eq i32 %cond, 0
+  br i1 %cmp0, label %if, label %end
+
+if:
+  %phi_value = phi <5 x double> [zeroinitializer, %entry], [%value, %if]
+  %cast = bitcast <5 x double> %phi_value to <10 x float>
+  %cmp1 = icmp eq i32 %cond, 1
+  br i1 %cmp1, label %if, label %end
+
+end:
+  %phi_cast = phi <10 x float> [zeroinitializer, %entry], [%cast, %if]
+  store <10 x float> %phi_cast, ptr addrspace(1) %out
+  ret void
+}
+
+; FUNC-LABEL: {{^}}bitcast_v5f64_to_v10i32:
+; SI: ScratchSize: 0
+define amdgpu_kernel void @bitcast_v5f64_to_v10i32(i32 %cond, ptr addrspace(1) %out, <5 x double> %value) {
+entry:
+  %cmp0 = icmp eq i32 %cond, 0
+  br i1 %cmp0, label %if, label %end
+
+if:
+  %phi_value = phi <5 x double> [zeroinitializer, %entry], [%value, %if]
+  %cast = bitcast <5 x double> %phi_value to <10 x i32>
+  %cmp1 = icmp eq i32 %cond, 1
+  br i1 %cmp1, label %if, label %end
+
+end:
+  %phi_cast = phi <10 x i32> [zeroinitializer, %entry], [%cast, %if]
+  store <10 x i32> %phi_cast, ptr addrspace(1) %out
+  ret void
+}
+
+; FUNC-LABEL: {{^}}bitcast_v5i64_to_v10f32:
+; SI: ScratchSize: 0
+define amdgpu_kernel void @bitcast_v5i64_to_v10f32(i32 %cond, ptr addrspace(1) %out, <5 x i64> %value) {
+entry:
+  %cmp0 = icmp eq i32 %cond, 0
+  br i1 %cmp0, label %if, label %end
+
+if:
+  %phi_value = phi <5 x i64> [zeroinitializer, %entry], [%value, %if]
+  %cast = bitcast <5 x i64> %phi_value to <10 x float>
+  %cmp1 = icmp eq i32 %cond, 1
+  br i1 %cmp1, label %if, label %end
+
+end:
+  %phi_cast = phi <10 x float> [zeroinitializer, %entry], [%cast, %if]
+  store <10 x float> %phi_cast, ptr addrspace(1) %out
+  ret void
+}
+
+; FUNC-LABEL: {{^}}bitcast_v5i64_to_v10i32:
+; SI: ScratchSize: 0
+define amdgpu_kernel void @bitcast_v5i64_to_v10i32(i32 %cond, ptr addrspace(1) %out, <5 x i64> %value) {
+entry:
+  %cmp0 = icmp eq i32 %cond, 0
+  br i1 %cmp0, label %if, label %end
+
+if:
+  %phi_value = phi <5 x i64> [zeroinitializer, %entry], [%value, %if]
+  %cast = bitcast <5 x i64> %phi_value to <10 x i32>
+  %cmp1 = icmp eq i32 %cond, 1
+  br i1 %cmp1, label %if, label %end
+
+end:
+  %phi_cast = phi <10 x i32> [zeroinitializer, %entry], [%cast, %if]
+  store <10 x i32> %phi_cast, ptr addrspace(1) %out
+  ret void
+}
+
+; FUNC-LABEL: {{^}}bitcast_v6f64_to_v12i32:
+; SI: ScratchSize: 0
+define amdgpu_kernel void @bitcast_v6f64_to_v12i32(i32 %cond, ptr addrspace(1) %out, <6 x double> %value) {
+entry:
+  %cmp0 = icmp eq i32 %cond, 0
+  br i1 %cmp0, label %if, label %end
+
+if:
+  %phi_value = phi <6 x double> [zeroinitializer, %entry], [%value, %if]
+  %cast = bitcast <6 x double> %phi_value to <12 x i32>
+  %cmp1 = icmp eq i32 %cond, 1
+  br i1 %cmp1, label %if, label %end
+
+end:
+  %phi_cast = phi <12 x i32> [zeroinitializer, %entry], [%cast, %if]
+  store <12 x i32> %phi_cast, ptr addrspace(1) %out
+  ret void
+}
+
+; FUNC-LABEL: {{^}}bitcast_v6f64_to_v12f32:
+; SI: ScratchSize: 0
+define amdgpu_kernel void @bitcast_v6f64_to_v12f32(i32 %cond, ptr addrspace(1) %out, <6 x double> %value) {
+entry:
+  %cmp0 = icmp eq i32 %cond, 0
+  br i1 %cmp0, label %if, label %end
+
+if:
+  %phi_value = phi <6 x double> [zeroinitializer, %entry], [%value, %if]
+  %cast = bitcast <6 x double> %phi_value to <12 x float>
+  %cmp1 = icmp eq i32 %cond, 1
+  br i1 %cmp1, label %if, label %end
+
+end:
+  %phi_cast = phi <12 x float> [zeroinitializer, %entry], [%cast, %if]
+  store <12 x float> %phi_cast, ptr addrspace(1) %out
+  ret void
+}
+
+; FUNC-LABEL: {{^}}bitcast_v12i32_to_v6i64:
+; SI: ScratchSize: 0
+define amdgpu_kernel void @bitcast_v12i32_to_v6i64(i32 %cond, ptr addrspace(1) %out, <12 x i32> %value) {
+entry:
+  %cmp0 = icmp eq i32 %cond, 0
+  br i1 %cmp0, label %if, label %end
+
+if:
+  %phi_value = phi <12 x i32> [zeroinitializer, %entry], [%value, %if]
+  %cast = bitcast <12 x i32> %phi_value to <6 x i64>
+  %cmp1 = icmp eq i32 %cond, 1
+  br i1 %cmp1, label %if, label %end
+
+end:
+  %phi_cast = phi <6 x i64> [zeroinitializer, %entry], [%cast, %if]
+  store <6 x i64> %phi_cast, ptr addrspace(1) %out
+  ret void
+}
+
+; FUNC-LABEL: {{^}}bitcast_v12i32_to_v6f64:
+; SI: ScratchSize: 0
+define amdgpu_kernel void @bitcast_v12i32_to_v6f64(i32 %cond, ptr addrspace(1) %out, <12 x i32> %value) {
+entry:
+  %cmp0 = icmp eq i32 %cond, 0
+  br i1 %cmp0, label %if, label %end
+
+if:
+  %phi_value = phi <12 x i32> [zeroinitializer, %entry], [%value, %if]
+  %cast = bitcast <12 x i32> %phi_value to <6 x double>
+  %cmp1 = icmp eq i32 %cond, 1
+  br i1 %cmp1, label %if, label %end
+
+end:
+  %phi_cast = phi <6 x double> [zeroinitializer, %entry], [%cast, %if]
+  store <6 x double> %phi_cast, ptr addrspace(1) %out
+  ret void
+}
+
+; FUNC-LABEL: {{^}}bitcast_v6i64_to_v12i32:
+; SI: ScratchSize: 0
+define amdgpu_kernel void @bitcast_v6i64_to_v12i32(i32 %cond, ptr addrspace(1) %out, <6 x i64> %value) {
+entry:
+  %cmp0 = icmp eq i32 %cond, 0
+  br i1 %cmp0, label %if, label %end
+
+if:
+  %phi_value = phi <6 x i64> [zeroinitializer, %entry], [%value, %if]
+  %cast = bitcast <6 x i64> %phi_value to <12 x i32>
+  %cmp1 = icmp eq i32 %cond, 1
+  br i1 %cmp1, label %if, label %end
+
+end:
+  %phi_cast = phi <12 x i32> [zeroinitializer, %entry], [%cast, %if]
+  store <12 x i32> %phi_cast, ptr addrspace(1) %out
+  ret void
+}
+
+; FUNC-LABEL: {{^}}bitcast_v7i64_to_v14i32:
+; SI: ScratchSize: 0
+define amdgpu_kernel void @bitcast_v7i64_to_v14i32(i32 %cond, ptr addrspace(1) %out, <7 x i64> %value) {
+entry:
+  %cmp0 = icmp eq i32 %cond, 0
+  br i1 %cmp0, label %if, label %end
+
+if:
+  %phi_value = phi <7 x i64> [zeroinitializer, %entry], [%value, %if]
+  %cast = bitcast <7 x i64> %phi_value to <14 x i32>
+  %cmp1 = icmp eq i32 %cond, 1
+  br i1 %cmp1, label %if, label %end
+
+end:
+  %phi_cast = phi <14 x i32> [zeroinitializer, %entry], [%cast, %if]
+  store <14 x i32> %phi_cast, ptr addrspace(1) %out
+  ret void
+}
+
+; FUNC-LABEL: {{^}}bitcast_v7f64_to_v14i32:
+; SI: ScratchSize: 0
+define amdgpu_kernel void @bitcast_v7f64_to_v14i32(i32 %cond, ptr addrspace(1) %out, <7 x double> %value) {
+entry:
+  %cmp0 = icmp eq i32 %cond, 0
+  br i1 %cmp0, label %if, label %end
+
+if:
+  %phi_value = phi <7 x double> [zeroinitializer, %entry], [%value, %if]
+  %cast = bitcast <7 x double> %phi_value to <14 x i32>
+  %cmp1 = icmp eq i32 %cond, 1
+  br i1 %cmp1, label %if, label %end
+
+end:
+  %phi_cast = phi <14 x i32> [zeroinitializer, %entry], [%cast, %if]
+  store <14 x i32> %phi_cast, ptr addrspace(1) %out
+  ret void
+}
+
+; FUNC-LABEL: {{^}}bitcast_v9i64_to_v18i32:
+; SI: ScratchSize: 0
+define amdgpu_kernel void @bitcast_v9i64_to_v18i32(i32 %cond, ptr addrspace(1) %out, <9 x i64> %value) {
+entry:
+  %cmp0 = icmp eq i32 %cond, 0
+  br i1 %cmp0, label %if, label %end
+
+if:
+  %phi_value = phi <9 x i64> [zeroinitializer, %entry], [%value, %if]
+  %cast = bitcast <9 x i64> %phi_value to <18 x i32>
+  %cmp1 = icmp eq i32 %cond, 1
+  br i1 %cmp1, label %if, label %end
+
+end:
+  %phi_cast = phi <18 x i32> [zeroinitializer, %entry], [%cast, %if]
+  store <18 x i32> %phi_cast, ptr addrspace(1) %out
+  ret void
+}
+
+; FUNC-LABEL: {{^}}bitcast_v10i64_to_v20i32:
+; SI: ScratchSize: 0
+define amdgpu_kernel void @bitcast_v10i64_to_v20i32(i32 %cond, ptr addrspace(1) %out, <10 x i64> %value) {
+entry:
+  %cmp0 = icmp eq i32 %cond, 0
+  br i1 %cmp0, label %if, label %end
+
+if:
+  %phi_value = phi <10 x i64> [zeroinitializer, %entry], [%value, %if]
+  %cast = bitcast <10 x i64> %phi_value to <20 x i32>
+  %cmp1 = icmp eq i32 %cond, 1
+  br i1 %cmp1, label %if, label %end
+
+end:
+  %phi_cast = phi <20 x i32> [zeroinitializer, %entry], [%cast, %if]
+  store <20 x i32> %phi_cast, ptr addrspace(1) %out
+  ret void
+}
+
+; FUNC-LABEL: {{^}}bitcast_v11i64_to_v20i32:
+; SI: ScratchSize: 0
+define amdgpu_kernel void @bitcast_v11i64_to_v20i32(i32 %cond, ptr addrspace(1) %out, <11 x i64> %value) {
+entry:
+  %cmp0 = icmp eq i32 %cond, 0
+  br i1 %cmp0, label %if, label %end
+
+if:
+  %phi_value = phi <11 x i64> [zeroinitializer, %entry], [%value, %if]
+  %cast = bitcast <11 x i64> %phi_value to <22 x i32>
+  %cmp1 = icmp eq i32 %cond, 1
+  br i1 %cmp1, label %if, label %end
+
+end:
+  %phi_cast = phi <22 x i32> [zeroinitializer, %entry], [%cast, %if]
+  store <22 x i32> %phi_cast, ptr addrspace(1) %out
+  ret void
+}
+
+; FUNC-LABEL: {{^}}bitcast_v12i64_to_v22i32:
+; SI: ScratchSize: 0
+define amdgpu_kernel void @bitcast_v12i64_to_v22i32(i32 %cond, ptr addrspace(1) %out, <12 x i64> %value) {
+entry:
+  %cmp0 = icmp eq i32 %cond, 0
+  br i1 %cmp0, label %if, label %end
+
+if:
+  %phi_value = phi <12 x i64> [zeroinitializer, %entry], [%value, %if]
+  %cast = bitcast <12 x i64> %phi_value to <24 x i32>
+  %cmp1 = icmp eq i32 %cond, 1
+  br i1 %cmp1, label %if, label %end
+
+end:
+  %phi_cast = phi <24 x i32> [zeroinitializer, %entry], [%cast, %if]
+  store <24 x i32> %phi_cast, ptr addrspace(1) %out
+  ret void
+}
+
+; FUNC-LABEL: {{^}}bitcast_v13i64_to_v24i32:
+; SI: ScratchSize: 0
+define amdgpu_kernel void @bitcast_v13i64_to_v24i32(i32 %cond, ptr addrspace(1) %out, <13 x i64> %value) {
+entry:
+  %cmp0 = icmp eq i32 %cond, 0
+  br i1 %cmp0, label %if, label %end
+
+if:
+  %phi_value = phi <13 x i64> [zeroinitializer, %entry], [%value, %if]
+  %cast = bitcast <13 x i64> %phi_value to <26 x i32>
+  %cmp1 = icmp eq i32 %cond, 1
+  br i1 %cmp1, label %if, label %end
+
+end:
+  %phi_cast = phi <26 x i32> [zeroinitializer, %entry], [%cast, %if]
+  store <26 x i32> %phi_cast, ptr addrspace(1) %out
+  ret void
+}
+
+; FUNC-LABEL: {{^}}bitcast_v14i64_to_v26i32:
+; SI: ScratchSize: 0
+define amdgpu_kernel void @bitcast_v14i64_to_v26i32(i32 %cond, ptr addrspace(1) %out, <14 x i64> %value) {
+entry:
+  %cmp0 = icmp eq i32 %cond, 0
+  br i1 %cmp0, label %if, label %end
+
+if:
+  %phi_value = phi <14 x i64> [zeroinitializer, %entry], [%value, %if]
+  %cast = bitcast <14 x i64> %phi_value to <28 x i32>
+  %cmp1 = icmp eq i32 %cond, 1
+  br i1 %cmp1, label %if, label %end
+
+end:
+  %phi_cast = phi <28 x i32> [zeroinitializer, %entry], [%cast, %if]
+  store <28 x i32> %phi_cast, ptr addrspace(1) %out
+  ret void
+}
+
+; FUNC-LABEL: {{^}}bitcast_v15i64_to_v26i32:
+; SI: ScratchSize: 0
+define amdgpu_kernel void @bitcast_v15i64_to_v26i32(i32 %cond, ptr addrspace(1) %out, <15 x i64> %value) {
+entry:
+  %cmp0 = icmp eq i32 %cond, 0
+  br i1 %cmp0, label %if, label %end
+
+if:
+  %phi_value = phi <15 x i64> [zeroinitializer, %entry], [%value, %if]
+  %cast = bitcast <15 x i64> %phi_value to <30 x i32>
+  %cmp1 = icmp eq i32 %cond, 1
+  br i1 %cmp1, label %if, label %end
+
+end:
+  %phi_cast = phi <30 x i32> [zeroinitializer, %entry], [%cast, %if]
+  store <30 x i32> %phi_cast, ptr addrspace(1) %out
+  ret void
+}

llvm/test/CodeGen/AMDGPU/copy-illegal-type.ll

 ; SI-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x11
 ; SI-NEXT:    s_mov_b32 s11, 0xf000
 ; SI-NEXT:    s_mov_b32 s6, 0
 ; SI-NEXT:    s_mov_b32 s7, s11
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
 ; SI-NEXT:    v_mov_b32_e32 v1, 0
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-NEXT:    buffer_load_dword v0, v[0:1], s[4:7], 0 addr64
-; SI-NEXT:    s_mov_b32 s24, SCRATCH_RSRC_DWORD0
-; SI-NEXT:    s_mov_b32 s25, SCRATCH_RSRC_DWORD1
-; SI-NEXT:    s_mov_b32 s26, -1
-; SI-NEXT:    s_mov_b32 s27, 0xe8f000
-; SI-NEXT:    s_add_u32 s24, s24, s3
 ; SI-NEXT:    s_load_dwordx8 s[0:7], s[0:1], 0x9
 ; SI-NEXT:    s_mov_b32 s10, -1
-; SI-NEXT:    s_addc_u32 s25, s25, 0
 ; SI-NEXT:    s_mov_b32 s14, s10
 ; SI-NEXT:    s_mov_b32 s15, s11
+; SI-NEXT:    s_mov_b32 s18, s10
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-NEXT:    s_mov_b32 s8, s0
 ; SI-NEXT:    s_mov_b32 s9, s1
-; SI-NEXT:    s_mov_b32 s18, s10
 ; SI-NEXT:    s_mov_b32 s19, s11
 ; SI-NEXT:    s_mov_b32 s22, s10
 ; SI-NEXT:    s_mov_b32 s23, s11
 ; SI-NEXT:    s_mov_b32 s12, s2
 ; SI-NEXT:    s_mov_b32 s13, s3
 ; SI-NEXT:    s_mov_b32 s16, s4
 ; SI-NEXT:    s_mov_b32 s17, s5
 ; SI-NEXT:    s_mov_b32 s20, s6
 ; SI-NEXT:    s_mov_b32 s21, s7
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_store_dword v0, off, s[8:11], 0
 ; SI-NEXT:    buffer_store_dword v0, off, s[12:15], 0
 ; SI-NEXT:    buffer_store_dword v0, off, s[16:19], 0
 ; SI-NEXT:    buffer_store_dword v0, off, s[20:23], 0
 ; SI-NEXT:    s_endpgm
 ;
 ; VI-LABEL: test_copy_v4i8_x4:
 ; VI:       ; %bb.0:
-; VI-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
-; VI-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
-; VI-NEXT:    s_mov_b32 s90, -1
-; VI-NEXT:    s_mov_b32 s91, 0xe80000
-; VI-NEXT:    s_add_u32 s88, s88, s3
 ; VI-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x44
 ; VI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
 ; VI-NEXT:    s_mov_b32 s11, 0xf000
 ; VI-NEXT:    s_mov_b32 s10, -1
-; VI-NEXT:    s_addc_u32 s89, s89, 0
+; VI-NEXT:    s_mov_b32 s14, s10
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v1, s3
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v0
 ; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; VI-NEXT:    flat_load_dword v0, v[0:1]
 ; VI-NEXT:    s_load_dwordx8 s[0:7], s[0:1], 0x24
-; VI-NEXT:    s_mov_b32 s14, s10
 ; VI-NEXT:    s_mov_b32 s15, s11
 ; VI-NEXT:    s_mov_b32 s18, s10
 ; VI-NEXT:    s_mov_b32 s19, s11
+; VI-NEXT:    s_mov_b32 s22, s10
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_mov_b32 s8, s0
 ; VI-NEXT:    s_mov_b32 s9, s1
-; VI-NEXT:    s_mov_b32 s22, s10
 ; VI-NEXT:    s_mov_b32 s23, s11
 ; VI-NEXT:    s_mov_b32 s12, s2
 ; VI-NEXT:    s_mov_b32 s13, s3
 ; VI-NEXT:    s_mov_b32 s16, s4
 ; VI-NEXT:    s_mov_b32 s17, s5
 ; VI-NEXT:    s_mov_b32 s20, s6
 ; VI-NEXT:    s_mov_b32 s21, s7
 ; VI-NEXT:    s_waitcnt vmcnt(0)

llvm/test/CodeGen/AMDGPU/copy-to-reg-scc-clobber.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -march=amdgcn -mcpu=gfx90a -O3 -pre-RA-sched=source < %s | FileCheck -check-prefix=RRLIST %s
 ; RUN: llc -march=amdgcn -mcpu=gfx90a -O3 -pre-RA-sched=fast < %s | FileCheck -check-prefix=FAST %s
 
 
 define protected amdgpu_kernel void @sccClobber(ptr addrspace(1) %a, ptr addrspace(1) %b, ptr addrspace(1) %e, ptr addrspace(1) %f, ptr addrspace(1) %pout.coerce) {
 ; RRLIST-LABEL: sccClobber:
 ; RRLIST:       ; %bb.0: ; %entry
 ; RRLIST-NEXT:    s_load_dwordx8 s[4:11], s[0:1], 0x24
-; RRLIST-NEXT:    s_mov_b32 s20, SCRATCH_RSRC_DWORD0
-; RRLIST-NEXT:    s_mov_b32 s21, SCRATCH_RSRC_DWORD1
-; RRLIST-NEXT:    s_mov_b32 s22, -1
-; RRLIST-NEXT:    s_mov_b32 s23, 0xe00000
-; RRLIST-NEXT:    s_add_u32 s20, s20, s3
+; RRLIST-NEXT:    v_mov_b32_e32 v2, 0
 ; RRLIST-NEXT:    s_waitcnt lgkmcnt(0)
 ; RRLIST-NEXT:    s_load_dword s16, s[8:9], 0x0
 ; RRLIST-NEXT:    s_load_dwordx2 s[2:3], s[6:7], 0x0
 ; RRLIST-NEXT:    s_load_dwordx2 s[12:13], s[4:5], 0x0
 ; RRLIST-NEXT:    s_load_dwordx2 s[14:15], s[0:1], 0x44
 ; RRLIST-NEXT:    s_load_dword s17, s[10:11], 0x0
-; RRLIST-NEXT:    s_addc_u32 s21, s21, 0
 ; RRLIST-NEXT:    s_waitcnt lgkmcnt(0)
-; RRLIST-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
 ; RRLIST-NEXT:    s_min_i32 s4, s16, 0
+; RRLIST-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
 ; RRLIST-NEXT:    v_cmp_lt_i64_e32 vcc, s[12:13], v[0:1]
 ; RRLIST-NEXT:    s_and_b64 s[0:1], vcc, exec
 ; RRLIST-NEXT:    s_cselect_b32 s0, s16, s17
 ; RRLIST-NEXT:    s_cmp_eq_u64 s[12:13], s[2:3]
 ; RRLIST-NEXT:    s_cselect_b32 s0, s4, s0
-; RRLIST-NEXT:    v_mov_b32_e32 v2, 0
 ; RRLIST-NEXT:    v_mov_b32_e32 v0, s0
 ; RRLIST-NEXT:    global_store_dword v2, v0, s[14:15]
 ; RRLIST-NEXT:    s_endpgm
 ;
 ; FAST-LABEL: sccClobber:
 ; FAST:       ; %bb.0: ; %entry
 ; FAST-NEXT:    s_load_dwordx8 s[4:11], s[0:1], 0x24
-; FAST-NEXT:    s_mov_b32 s20, SCRATCH_RSRC_DWORD0
-; FAST-NEXT:    s_mov_b32 s21, SCRATCH_RSRC_DWORD1
-; FAST-NEXT:    s_mov_b32 s22, -1
-; FAST-NEXT:    s_mov_b32 s23, 0xe00000
-; FAST-NEXT:    s_add_u32 s20, s20, s3
+; FAST-NEXT:    v_mov_b32_e32 v2, 0
 ; FAST-NEXT:    s_waitcnt lgkmcnt(0)
 ; FAST-NEXT:    s_load_dword s16, s[8:9], 0x0
 ; FAST-NEXT:    s_load_dwordx2 s[2:3], s[6:7], 0x0
 ; FAST-NEXT:    s_load_dwordx2 s[12:13], s[4:5], 0x0
 ; FAST-NEXT:    s_load_dwordx2 s[14:15], s[0:1], 0x44
 ; FAST-NEXT:    s_load_dword s17, s[10:11], 0x0
-; FAST-NEXT:    s_addc_u32 s21, s21, 0
 ; FAST-NEXT:    s_waitcnt lgkmcnt(0)
-; FAST-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
 ; FAST-NEXT:    s_min_i32 s4, s16, 0
+; FAST-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
 ; FAST-NEXT:    v_cmp_lt_i64_e32 vcc, s[12:13], v[0:1]
 ; FAST-NEXT:    s_and_b64 s[0:1], vcc, exec
 ; FAST-NEXT:    s_cselect_b32 s0, s16, s17
 ; FAST-NEXT:    s_cmp_eq_u64 s[12:13], s[2:3]
 ; FAST-NEXT:    s_cselect_b32 s0, s4, s0
-; FAST-NEXT:    v_mov_b32_e32 v2, 0
 ; FAST-NEXT:    v_mov_b32_e32 v0, s0
 ; FAST-NEXT:    global_store_dword v2, v0, s[14:15]
 ; FAST-NEXT:    s_endpgm
 entry:
   %i = load i64, ptr addrspace(1) %a, align 8
   %i.1 = load i64, ptr addrspace(1) %b, align 8
   %i.2 = load i32, ptr addrspace(1) %e, align 4
   %i.3 = load i32, ptr addrspace(1) %f, align 4

llvm/test/CodeGen/AMDGPU/flat_atomics_i64.ll

entry:
   %gep = getelementptr i64, ptr %ptr, i64 4
   %val = cmpxchg volatile ptr %gep, i64 %old, i64 %in seq_cst seq_cst
   ret void
 }
 
 define amdgpu_kernel void @atomic_cmpxchg_i64_ret_addr64_offset(ptr %out, ptr %out2, i64 %in, i64 %index, i64 %old) {
 ; GCN1-LABEL: atomic_cmpxchg_i64_ret_addr64_offset:
 ; GCN1:       ; %bb.0: ; %entry
-; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
 ; GCN1-NEXT:    s_load_dwordx8 s[4:11], s[0:1], 0x9
 ; GCN1-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x11
-; GCN1-NEXT:    s_mov_b32 s14, -1
-; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
-; GCN1-NEXT:    s_add_u32 s12, s12, s3
-; GCN1-NEXT:    s_addc_u32 s13, s13, 0
 ; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN1-NEXT:    s_lshl_b64 s[2:3], s[10:11], 3
 ; GCN1-NEXT:    v_mov_b32_e32 v2, s0
 ; GCN1-NEXT:    s_add_u32 s0, s4, s2
 ; GCN1-NEXT:    s_addc_u32 s3, s5, s3
 ; GCN1-NEXT:    s_add_u32 s2, s0, 32
 ; GCN1-NEXT:    s_addc_u32 s3, s3, 0
 ; GCN1-NEXT:    v_mov_b32_e32 v5, s3
 ; GCN1-NEXT:    v_mov_b32_e32 v0, s8
 ; GCN1-NEXT:    v_mov_b32_e32 v1, s9
 ; GCN1-NEXT:    v_mov_b32_e32 v3, s1
 ; GCN1-NEXT:    v_mov_b32_e32 v4, s2
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    buffer_wbinvl1_vol
 ; GCN1-NEXT:    v_mov_b32_e32 v2, s6
 ; GCN1-NEXT:    v_mov_b32_e32 v3, s7
 ; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; GCN1-NEXT:    s_endpgm
 ;
 ; GCN2-LABEL: atomic_cmpxchg_i64_ret_addr64_offset:
 ; GCN2:       ; %bb.0: ; %entry
-; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
-; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
 ; GCN2-NEXT:    s_load_dwordx8 s[4:11], s[0:1], 0x24
 ; GCN2-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x44
-; GCN2-NEXT:    s_mov_b32 s90, -1
-; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
-; GCN2-NEXT:    s_add_u32 s88, s88, s3
-; GCN2-NEXT:    s_addc_u32 s89, s89, 0
 ; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN2-NEXT:    s_lshl_b64 s[2:3], s[10:11], 3
 ; GCN2-NEXT:    v_mov_b32_e32 v2, s0
 ; GCN2-NEXT:    s_add_u32 s0, s4, s2
 ; GCN2-NEXT:    s_addc_u32 s3, s5, s3
 ; GCN2-NEXT:    s_add_u32 s2, s0, 32
 ; GCN2-NEXT:    s_addc_u32 s3, s3, 0
 ; GCN2-NEXT:    v_mov_b32_e32 v5, s3
entry:
   %ptr = getelementptr i64, ptr %out, i64 %index
   %val = cmpxchg volatile ptr %ptr, i64 %old, i64 %in seq_cst seq_cst
   ret void
 }
 
 define amdgpu_kernel void @atomic_cmpxchg_i64_ret_addr64(ptr %out, ptr %out2, i64 %in, i64 %index, i64 %old) {
 ; GCN1-LABEL: atomic_cmpxchg_i64_ret_addr64:
 ; GCN1:       ; %bb.0: ; %entry
-; GCN1-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GCN1-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
 ; GCN1-NEXT:    s_load_dwordx8 s[4:11], s[0:1], 0x9
 ; GCN1-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x11
-; GCN1-NEXT:    s_mov_b32 s14, -1
-; GCN1-NEXT:    s_mov_b32 s15, 0xe8f000
-; GCN1-NEXT:    s_add_u32 s12, s12, s3
-; GCN1-NEXT:    s_addc_u32 s13, s13, 0
 ; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN1-NEXT:    s_lshl_b64 s[2:3], s[10:11], 3
 ; GCN1-NEXT:    s_add_u32 s2, s4, s2
 ; GCN1-NEXT:    s_addc_u32 s3, s5, s3
 ; GCN1-NEXT:    v_mov_b32_e32 v5, s3
 ; GCN1-NEXT:    v_mov_b32_e32 v0, s8
 ; GCN1-NEXT:    v_mov_b32_e32 v1, s9
 ; GCN1-NEXT:    v_mov_b32_e32 v2, s0
 ; GCN1-NEXT:    v_mov_b32_e32 v3, s1
 ; GCN1-NEXT:    v_mov_b32_e32 v4, s2
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    buffer_wbinvl1_vol
 ; GCN1-NEXT:    v_mov_b32_e32 v2, s6
 ; GCN1-NEXT:    v_mov_b32_e32 v3, s7
 ; GCN1-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; GCN1-NEXT:    s_endpgm
 ;
 ; GCN2-LABEL: atomic_cmpxchg_i64_ret_addr64:
 ; GCN2:       ; %bb.0: ; %entry
-; GCN2-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
-; GCN2-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
 ; GCN2-NEXT:    s_load_dwordx8 s[4:11], s[0:1], 0x24
 ; GCN2-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x44
-; GCN2-NEXT:    s_mov_b32 s90, -1
-; GCN2-NEXT:    s_mov_b32 s91, 0xe80000
-; GCN2-NEXT:    s_add_u32 s88, s88, s3
-; GCN2-NEXT:    s_addc_u32 s89, s89, 0
 ; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN2-NEXT:    s_lshl_b64 s[2:3], s[10:11], 3
 ; GCN2-NEXT:    s_add_u32 s2, s4, s2
 ; GCN2-NEXT:    s_addc_u32 s3, s5, s3
 ; GCN2-NEXT:    v_mov_b32_e32 v5, s3
 ; GCN2-NEXT:    v_mov_b32_e32 v0, s8
 ; GCN2-NEXT:    v_mov_b32_e32 v1, s9
 ; GCN2-NEXT:    v_mov_b32_e32 v2, s0

llvm/test/CodeGen/AMDGPU/global_atomics_i64.ll

entry:
   %gep = getelementptr i64, ptr addrspace(1) %ptr, i64 4
   %val = cmpxchg volatile ptr addrspace(1) %gep, i64 %old, i64 %in seq_cst seq_cst
   ret void
 }
 
 define amdgpu_kernel void @atomic_cmpxchg_i64_ret_addr64_offset(ptr addrspace(1) %out, ptr addrspace(1) %out2, i64 %in, i64 %index, i64 %old) {
 ; CI-LABEL: atomic_cmpxchg_i64_ret_addr64_offset:
 ; CI:       ; %bb.0: ; %entry
-; CI-NEXT:    s_mov_b32 s16, SCRATCH_RSRC_DWORD0
 ; CI-NEXT:    s_load_dwordx8 s[4:11], s[0:1], 0x9
-; CI-NEXT:    s_mov_b32 s17, SCRATCH_RSRC_DWORD1
-; CI-NEXT:    s_mov_b32 s18, -1
 ; CI-NEXT:    s_load_dwordx2 s[12:13], s[0:1], 0x11
-; CI-NEXT:    s_mov_b32 s19, 0xe8f000
-; CI-NEXT:    s_add_u32 s16, s16, s3
-; CI-NEXT:    s_addc_u32 s17, s17, 0
+; CI-NEXT:    s_mov_b32 s3, 0xf000
+; CI-NEXT:    s_mov_b32 s2, -1
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    s_lshl_b64 s[10:11], s[10:11], 3
-; CI-NEXT:    s_mov_b32 s3, 0xf000
 ; CI-NEXT:    v_mov_b32_e32 v4, s10
 ; CI-NEXT:    s_mov_b32 s0, s6
 ; CI-NEXT:    s_mov_b32 s1, s7
 ; CI-NEXT:    s_mov_b32 s6, 0
 ; CI-NEXT:    s_mov_b32 s7, s3
 ; CI-NEXT:    v_mov_b32_e32 v0, s8
 ; CI-NEXT:    v_mov_b32_e32 v1, s9
 ; CI-NEXT:    v_mov_b32_e32 v2, s12
 ; CI-NEXT:    v_mov_b32_e32 v3, s13
 ; CI-NEXT:    v_mov_b32_e32 v5, s11
 ; CI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; CI-NEXT:    buffer_atomic_cmpswap_x2 v[0:3], v[4:5], s[4:7], 0 addr64 offset:32 glc
 ; CI-NEXT:    s_waitcnt vmcnt(0)
 ; CI-NEXT:    buffer_wbinvl1_vol
-; CI-NEXT:    s_mov_b32 s2, -1
 ; CI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; CI-NEXT:    s_endpgm
 ;
 ; VI-LABEL: atomic_cmpxchg_i64_ret_addr64_offset:
 ; VI:       ; %bb.0: ; %entry
-; VI-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
-; VI-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
 ; VI-NEXT:    s_load_dwordx8 s[4:11], s[0:1], 0x24
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x44
-; VI-NEXT:    s_mov_b32 s90, -1
-; VI-NEXT:    s_mov_b32 s91, 0xe80000
-; VI-NEXT:    s_add_u32 s88, s88, s3
-; VI-NEXT:    s_addc_u32 s89, s89, 0
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_lshl_b64 s[2:3], s[10:11], 3
 ; VI-NEXT:    v_mov_b32_e32 v2, s0
 ; VI-NEXT:    s_add_u32 s0, s4, s2
 ; VI-NEXT:    s_addc_u32 s3, s5, s3
 ; VI-NEXT:    s_add_u32 s2, s0, 32
 ; VI-NEXT:    s_addc_u32 s3, s3, 0
 ; VI-NEXT:    v_mov_b32_e32 v5, s3

 ; VI-NEXT:    s_mov_b32 s2, -1
 ; VI-NEXT:    s_mov_b32 s0, s6
 ; VI-NEXT:    s_mov_b32 s1, s7
 ; VI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; VI-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: atomic_cmpxchg_i64_ret_addr64_offset:
 ; GFX9:       ; %bb.0: ; %entry
-; GFX9-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
 ; GFX9-NEXT:    s_load_dwordx8 s[4:11], s[0:1], 0x24
-; GFX9-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX9-NEXT:    s_mov_b32 s14, -1
+; GFX9-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x44
-; GFX9-NEXT:    s_mov_b32 s15, 0xe00000
-; GFX9-NEXT:    s_add_u32 s12, s12, s3
-; GFX9-NEXT:    s_addc_u32 s13, s13, 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    s_lshl_b64 s[2:3], s[10:11], 3
 ; GFX9-NEXT:    s_add_u32 s2, s4, s2
-; GFX9-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s8
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s9
 ; GFX9-NEXT:    s_addc_u32 s3, s5, s3
 ; GFX9-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX9-NEXT:    v_mov_b32_e32 v3, s1
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] offset:32 glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
entry:
   %ptr = getelementptr i64, ptr addrspace(1) %out, i64 %index
   %val = cmpxchg volatile ptr addrspace(1) %ptr, i64 %old, i64 %in seq_cst seq_cst
   ret void
 }
 
 define amdgpu_kernel void @atomic_cmpxchg_i64_ret_addr64(ptr addrspace(1) %out, ptr addrspace(1) %out2, i64 %in, i64 %index, i64 %old) {
 ; CI-LABEL: atomic_cmpxchg_i64_ret_addr64:
 ; CI:       ; %bb.0: ; %entry
-; CI-NEXT:    s_mov_b32 s16, SCRATCH_RSRC_DWORD0
 ; CI-NEXT:    s_load_dwordx8 s[4:11], s[0:1], 0x9
-; CI-NEXT:    s_mov_b32 s17, SCRATCH_RSRC_DWORD1
-; CI-NEXT:    s_mov_b32 s18, -1
 ; CI-NEXT:    s_load_dwordx2 s[12:13], s[0:1], 0x11
-; CI-NEXT:    s_mov_b32 s19, 0xe8f000
-; CI-NEXT:    s_add_u32 s16, s16, s3
-; CI-NEXT:    s_addc_u32 s17, s17, 0
+; CI-NEXT:    s_mov_b32 s3, 0xf000
+; CI-NEXT:    s_mov_b32 s2, -1
 ; CI-NEXT:    s_waitcnt lgkmcnt(0)
 ; CI-NEXT:    s_lshl_b64 s[10:11], s[10:11], 3
-; CI-NEXT:    s_mov_b32 s3, 0xf000
 ; CI-NEXT:    v_mov_b32_e32 v4, s10
 ; CI-NEXT:    s_mov_b32 s0, s6
 ; CI-NEXT:    s_mov_b32 s1, s7
 ; CI-NEXT:    s_mov_b32 s6, 0
 ; CI-NEXT:    s_mov_b32 s7, s3
 ; CI-NEXT:    v_mov_b32_e32 v0, s8
 ; CI-NEXT:    v_mov_b32_e32 v1, s9
 ; CI-NEXT:    v_mov_b32_e32 v2, s12
 ; CI-NEXT:    v_mov_b32_e32 v3, s13
 ; CI-NEXT:    v_mov_b32_e32 v5, s11
 ; CI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; CI-NEXT:    buffer_atomic_cmpswap_x2 v[0:3], v[4:5], s[4:7], 0 addr64 glc
 ; CI-NEXT:    s_waitcnt vmcnt(0)
 ; CI-NEXT:    buffer_wbinvl1_vol
-; CI-NEXT:    s_mov_b32 s2, -1
 ; CI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; CI-NEXT:    s_endpgm
 ;
 ; VI-LABEL: atomic_cmpxchg_i64_ret_addr64:
 ; VI:       ; %bb.0: ; %entry
-; VI-NEXT:    s_mov_b32 s88, SCRATCH_RSRC_DWORD0
-; VI-NEXT:    s_mov_b32 s89, SCRATCH_RSRC_DWORD1
 ; VI-NEXT:    s_load_dwordx8 s[4:11], s[0:1], 0x24
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x44
-; VI-NEXT:    s_mov_b32 s90, -1
-; VI-NEXT:    s_mov_b32 s91, 0xe80000
-; VI-NEXT:    s_add_u32 s88, s88, s3
-; VI-NEXT:    s_addc_u32 s89, s89, 0
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_lshl_b64 s[2:3], s[10:11], 3
 ; VI-NEXT:    s_add_u32 s2, s4, s2
 ; VI-NEXT:    s_addc_u32 s3, s5, s3
 ; VI-NEXT:    v_mov_b32_e32 v5, s3
 ; VI-NEXT:    v_mov_b32_e32 v0, s8
 ; VI-NEXT:    v_mov_b32_e32 v1, s9
 ; VI-NEXT:    v_mov_b32_e32 v2, s0
 ; VI-NEXT:    v_mov_b32_e32 v3, s1
 ; VI-NEXT:    v_mov_b32_e32 v4, s2
 ; VI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; VI-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
 ; VI-NEXT:    s_mov_b32 s3, 0xf000
 ; VI-NEXT:    s_mov_b32 s2, -1
 ; VI-NEXT:    s_mov_b32 s0, s6
 ; VI-NEXT:    s_mov_b32 s1, s7
 ; VI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; VI-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: atomic_cmpxchg_i64_ret_addr64:
 ; GFX9:       ; %bb.0: ; %entry
-; GFX9-NEXT:    s_mov_b32 s12, SCRATCH_RSRC_DWORD0
 ; GFX9-NEXT:    s_load_dwordx8 s[4:11], s[0:1], 0x24
-; GFX9-NEXT:    s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX9-NEXT:    s_mov_b32 s14, -1
+; GFX9-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x44
-; GFX9-NEXT:    s_mov_b32 s15, 0xe00000
-; GFX9-NEXT:    s_add_u32 s12, s12, s3
-; GFX9-NEXT:    s_addc_u32 s13, s13, 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    s_lshl_b64 s[2:3], s[10:11], 3
 ; GFX9-NEXT:    s_add_u32 s2, s4, s2
-; GFX9-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s8
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s9
 ; GFX9-NEXT:    s_addc_u32 s3, s5, s3
 ; GFX9-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX9-NEXT:    v_mov_b32_e32 v3, s1
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)

llvm/test/CodeGen/AMDGPU/insert_vector_dynelt.ll

entry:
   %v = insertelement <2 x double> %vec, double 1.000000e+00, i32 %sel
   store <2 x double> %v, ptr addrspace(1) %out
   ret void
 }
 
 define amdgpu_kernel void @double5_inselt(ptr addrspace(1) %out, <5 x double> %vec, i32 %sel) {
 ; GCN-LABEL: double5_inselt:
 ; GCN:       ; %bb.0: ; %entry
-; GCN-NEXT:    s_mov_b32 s16, SCRATCH_RSRC_DWORD0
-; GCN-NEXT:    s_mov_b32 s17, SCRATCH_RSRC_DWORD1
-; GCN-NEXT:    s_mov_b32 s18, -1
-; GCN-NEXT:    s_mov_b32 s19, 0xe80000
-; GCN-NEXT:    s_add_u32 s16, s16, s3
 ; GCN-NEXT:    s_load_dword s12, s[0:1], 0xa4
 ; GCN-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0x84
 ; GCN-NEXT:    s_load_dwordx2 s[10:11], s[0:1], 0x24
 ; GCN-NEXT:    s_load_dwordx8 s[0:7], s[0:1], 0x64
-; GCN-NEXT:    s_addc_u32 s17, s17, 0
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    s_cmp_eq_u32 s12, 4
 ; GCN-NEXT:    s_cselect_b32 s9, 0x3ff00000, s9
 ; GCN-NEXT:    s_cselect_b32 s8, 0, s8
 ; GCN-NEXT:    s_cmp_eq_u32 s12, 1
 ; GCN-NEXT:    s_cselect_b32 s3, 0x3ff00000, s3
 ; GCN-NEXT:    s_cselect_b32 s2, 0, s2
 ; GCN-NEXT:    s_cmp_eq_u32 s12, 0

llvm/test/CodeGen/AMDGPU/select.f16.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -amdgpu-scalarize-global-loads=false -mtriple=amdgcn-- -mcpu=tahiti -verify-machineinstrs  | FileCheck %s --check-prefix=SI
 ; RUN: llc < %s -amdgpu-scalarize-global-loads=false -mtriple=amdgcn-- -mcpu=fiji -mattr=-flat-for-global -verify-machineinstrs | FileCheck %s --check-prefix=VI
 
 define amdgpu_kernel void @select_f16(
 ; SI-LABEL: select_f16:
 ; SI:       ; %bb.0: ; %entry
-; SI-NEXT:    s_mov_b32 s24, SCRATCH_RSRC_DWORD0
 ; SI-NEXT:    s_load_dwordx8 s[4:11], s[0:1], 0x9
-; SI-NEXT:    s_mov_b32 s25, SCRATCH_RSRC_DWORD1
-; SI-NEXT:    s_mov_b32 s26, -1
-; SI-NEXT:    s_mov_b32 s27, 0xe8f000
-; SI-NEXT:    s_add_u32 s24, s24, s3
 ; SI-NEXT:    s_mov_b32 s3, 0xf000
 ; SI-NEXT:    s_mov_b32 s2, -1
 ; SI-NEXT:    s_load_dwordx2 s[12:13], s[0:1], 0x11
+; SI-NEXT:    s_mov_b32 s18, s2
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-NEXT:    s_mov_b32 s16, s6
 ; SI-NEXT:    s_mov_b32 s17, s7
-; SI-NEXT:    s_mov_b32 s18, s2
 ; SI-NEXT:    s_mov_b32 s19, s3
 ; SI-NEXT:    s_mov_b32 s20, s8
 ; SI-NEXT:    s_mov_b32 s21, s9
 ; SI-NEXT:    s_mov_b32 s22, s2
 ; SI-NEXT:    s_mov_b32 s23, s3
 ; SI-NEXT:    s_mov_b32 s8, s10
 ; SI-NEXT:    s_mov_b32 s9, s11
 ; SI-NEXT:    s_mov_b32 s10, s2
 ; SI-NEXT:    s_mov_b32 s11, s3
 ; SI-NEXT:    s_mov_b32 s14, s2
 ; SI-NEXT:    s_mov_b32 s15, s3
 ; SI-NEXT:    buffer_load_ushort v0, off, s[16:19], 0 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_load_ushort v1, off, s[20:23], 0 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_load_ushort v2, off, s[8:11], 0 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_load_ushort v3, off, s[12:15], 0 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    s_mov_b32 s0, s4
 ; SI-NEXT:    s_mov_b32 s1, s5
-; SI-NEXT:    s_addc_u32 s25, s25, 0
 ; SI-NEXT:    v_cvt_f32_f16_e32 v0, v0
 ; SI-NEXT:    v_cvt_f32_f16_e32 v1, v1
 ; SI-NEXT:    v_cvt_f32_f16_e32 v2, v2
 ; SI-NEXT:    v_cvt_f32_f16_e32 v3, v3
 ; SI-NEXT:    v_cmp_lt_f32_e32 vcc, v0, v1
 ; SI-NEXT:    v_cndmask_b32_e32 v0, v3, v2, vcc
 ; SI-NEXT:    v_cvt_f16_f32_e32 v0, v0
 ; SI-NEXT:    buffer_store_short v0, off, s[0:3], 0
 ; SI-NEXT:    s_endpgm
 ;
 ; VI-LABEL: select_f16:
 ; VI:       ; %bb.0: ; %entry
-; VI-NEXT:    s_mov_b32 s24, SCRATCH_RSRC_DWORD0
 ; VI-NEXT:    s_load_dwordx8 s[4:11], s[0:1], 0x24
-; VI-NEXT:    s_mov_b32 s25, SCRATCH_RSRC_DWORD1
-; VI-NEXT:    s_mov_b32 s26, -1
-; VI-NEXT:    s_mov_b32 s27, 0xe80000
-; VI-NEXT:    s_add_u32 s24, s24, s3
 ; VI-NEXT:    s_mov_b32 s3, 0xf000
 ; VI-NEXT:    s_mov_b32 s2, -1
 ; VI-NEXT:    s_load_dwordx2 s[12:13], s[0:1], 0x44
+; VI-NEXT:    s_mov_b32 s18, s2
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_mov_b32 s16, s6
 ; VI-NEXT:    s_mov_b32 s17, s7
-; VI-NEXT:    s_mov_b32 s18, s2
 ; VI-NEXT:    s_mov_b32 s19, s3
 ; VI-NEXT:    s_mov_b32 s20, s8
 ; VI-NEXT:    s_mov_b32 s21, s9
 ; VI-NEXT:    s_mov_b32 s22, s2
 ; VI-NEXT:    s_mov_b32 s23, s3
 ; VI-NEXT:    s_mov_b32 s8, s10
 ; VI-NEXT:    s_mov_b32 s9, s11
 ; VI-NEXT:    s_mov_b32 s10, s2
 ; VI-NEXT:    s_mov_b32 s11, s3
 ; VI-NEXT:    s_mov_b32 s14, s2
 ; VI-NEXT:    s_mov_b32 s15, s3
 ; VI-NEXT:    buffer_load_ushort v0, off, s[16:19], 0 glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_load_ushort v1, off, s[20:23], 0 glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_load_ushort v2, off, s[8:11], 0 glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_load_ushort v3, off, s[12:15], 0 glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    s_mov_b32 s0, s4
 ; VI-NEXT:    s_mov_b32 s1, s5
-; VI-NEXT:    s_addc_u32 s25, s25, 0
 ; VI-NEXT:    v_cmp_lt_f16_e32 vcc, v0, v1
 ; VI-NEXT:    v_cndmask_b32_e32 v0, v3, v2, vcc
 ; VI-NEXT:    buffer_store_short v0, off, s[0:3], 0
 ; VI-NEXT:    s_endpgm
     ptr addrspace(1) %r,
     ptr addrspace(1) %a,
     ptr addrspace(1) %b,
     ptr addrspace(1) %c,
entry:
   %r.val = select i1 %fcmp, half %c.val, half 0xH3800
   store half %r.val, ptr addrspace(1) %r
   ret void
 }
 
 define amdgpu_kernel void @select_v2f16(
 ; SI-LABEL: select_v2f16:
 ; SI:       ; %bb.0: ; %entry
-; SI-NEXT:    s_mov_b32 s24, SCRATCH_RSRC_DWORD0
 ; SI-NEXT:    s_load_dwordx8 s[4:11], s[0:1], 0x9
-; SI-NEXT:    s_mov_b32 s25, SCRATCH_RSRC_DWORD1
-; SI-NEXT:    s_mov_b32 s26, -1
 ; SI-NEXT:    s_load_dwordx2 s[12:13], s[0:1], 0x11
-; SI-NEXT:    s_mov_b32 s27, 0xe8f000
-; SI-NEXT:    s_add_u32 s24, s24, s3
 ; SI-NEXT:    s_mov_b32 s3, 0xf000
 ; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:    s_mov_b32 s18, s2
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-NEXT:    s_mov_b32 s16, s6
 ; SI-NEXT:    s_mov_b32 s17, s7
-; SI-NEXT:    s_mov_b32 s18, s2
 ; SI-NEXT:    s_mov_b32 s19, s3
 ; SI-NEXT:    s_mov_b32 s20, s8
 ; SI-NEXT:    s_mov_b32 s21, s9
 ; SI-NEXT:    s_mov_b32 s22, s2
 ; SI-NEXT:    s_mov_b32 s23, s3
 ; SI-NEXT:    s_mov_b32 s14, s2
 ; SI-NEXT:    s_mov_b32 s15, s3
 ; SI-NEXT:    buffer_load_dword v1, off, s[16:19], 0
 ; SI-NEXT:    s_mov_b32 s8, s10
 ; SI-NEXT:    s_mov_b32 s9, s11
 ; SI-NEXT:    buffer_load_dword v0, off, s[12:15], 0
 ; SI-NEXT:    s_mov_b32 s10, s2
 ; SI-NEXT:    s_mov_b32 s11, s3
 ; SI-NEXT:    buffer_load_dword v2, off, s[20:23], 0
 ; SI-NEXT:    buffer_load_dword v3, off, s[8:11], 0
 ; SI-NEXT:    s_mov_b32 s0, s4
 ; SI-NEXT:    s_mov_b32 s1, s5
-; SI-NEXT:    s_addc_u32 s25, s25, 0
 ; SI-NEXT:    s_waitcnt vmcnt(3)
 ; SI-NEXT:    v_lshrrev_b32_e32 v5, 16, v1
 ; SI-NEXT:    v_cvt_f32_f16_e32 v5, v5
 ; SI-NEXT:    v_cvt_f32_f16_e32 v1, v1
 ; SI-NEXT:    s_waitcnt vmcnt(2)
 ; SI-NEXT:    v_cvt_f32_f16_e32 v4, v0
 ; SI-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
 ; SI-NEXT:    s_waitcnt vmcnt(1)

 ; SI-NEXT:    v_cvt_f16_f32_e32 v1, v1
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
 ; SI-NEXT:    v_or_b32_e32 v0, v1, v0
 ; SI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; SI-NEXT:    s_endpgm
 ;
 ; VI-LABEL: select_v2f16:
 ; VI:       ; %bb.0: ; %entry
-; VI-NEXT:    s_mov_b32 s24, SCRATCH_RSRC_DWORD0
-; VI-NEXT:    s_mov_b32 s25, SCRATCH_RSRC_DWORD1
 ; VI-NEXT:    s_load_dwordx8 s[4:11], s[0:1], 0x24
 ; VI-NEXT:    s_load_dwordx2 s[12:13], s[0:1], 0x44
-; VI-NEXT:    s_mov_b32 s26, -1
-; VI-NEXT:    s_mov_b32 s27, 0xe80000
-; VI-NEXT:    s_add_u32 s24, s24, s3
 ; VI-NEXT:    s_mov_b32 s3, 0xf000
 ; VI-NEXT:    s_mov_b32 s2, -1
 ; VI-NEXT:    s_mov_b32 s14, s2
 ; VI-NEXT:    s_mov_b32 s15, s3
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_mov_b32 s16, s6
 ; VI-NEXT:    s_mov_b32 s17, s7
 ; VI-NEXT:    s_mov_b32 s18, s2
 ; VI-NEXT:    s_mov_b32 s19, s3
 ; VI-NEXT:    s_mov_b32 s20, s8
 ; VI-NEXT:    s_mov_b32 s21, s9
 ; VI-NEXT:    s_mov_b32 s22, s2
 ; VI-NEXT:    s_mov_b32 s23, s3
 ; VI-NEXT:    s_mov_b32 s8, s10
 ; VI-NEXT:    s_mov_b32 s9, s11
 ; VI-NEXT:    s_mov_b32 s10, s2
 ; VI-NEXT:    buffer_load_dword v0, off, s[12:15], 0
 ; VI-NEXT:    buffer_load_dword v1, off, s[20:23], 0
 ; VI-NEXT:    buffer_load_dword v2, off, s[16:19], 0
 ; VI-NEXT:    s_mov_b32 s11, s3
 ; VI-NEXT:    buffer_load_dword v3, off, s[8:11], 0
 ; VI-NEXT:    s_mov_b32 s0, s4
 ; VI-NEXT:    s_mov_b32 s1, s5
-; VI-NEXT:    s_addc_u32 s25, s25, 0
 ; VI-NEXT:    s_waitcnt vmcnt(3)
 ; VI-NEXT:    v_lshrrev_b32_e32 v4, 16, v0
 ; VI-NEXT:    s_waitcnt vmcnt(2)
 ; VI-NEXT:    v_lshrrev_b32_e32 v5, 16, v1
 ; VI-NEXT:    s_waitcnt vmcnt(1)
 ; VI-NEXT:    v_lshrrev_b32_e32 v6, 16, v2
 ; VI-NEXT:    v_cmp_lt_f16_e32 vcc, v2, v1
 ; VI-NEXT:    s_waitcnt vmcnt(0)

llvm/test/CodeGen/AMDGPU/v_madak_f16.ll

entry:
 
   store half %r.val, ptr addrspace(1) %r
   ret void
 }
 
 define amdgpu_kernel void @madak_f16_use_2(
 ; SI-LABEL: madak_f16_use_2:
 ; SI:       ; %bb.0: ; %entry
-; SI-NEXT:    s_mov_b32 s20, SCRATCH_RSRC_DWORD0
 ; SI-NEXT:    s_load_dwordx8 s[4:11], s[0:1], 0x9
-; SI-NEXT:    s_mov_b32 s21, SCRATCH_RSRC_DWORD1
-; SI-NEXT:    s_mov_b32 s22, -1
-; SI-NEXT:    s_mov_b32 s23, 0xe8f000
 ; SI-NEXT:    s_load_dwordx2 s[12:13], s[0:1], 0x11
-; SI-NEXT:    s_add_u32 s20, s20, s3
 ; SI-NEXT:    s_mov_b32 s3, 0xf000
 ; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:    s_mov_b32 s18, s2
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-NEXT:    s_mov_b32 s16, s8
 ; SI-NEXT:    s_mov_b32 s17, s9
-; SI-NEXT:    s_mov_b32 s18, s2
 ; SI-NEXT:    s_mov_b32 s19, s3
 ; SI-NEXT:    s_mov_b32 s8, s10
 ; SI-NEXT:    s_mov_b32 s9, s11
 ; SI-NEXT:    s_mov_b32 s10, s2
 ; SI-NEXT:    s_mov_b32 s11, s3
 ; SI-NEXT:    s_mov_b32 s14, s2
 ; SI-NEXT:    s_mov_b32 s15, s3
 ; SI-NEXT:    buffer_load_ushort v0, off, s[16:19], 0 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_load_ushort v1, off, s[8:11], 0 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_load_ushort v2, off, s[12:15], 0 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_mov_b32_e32 v3, 0x41200000
 ; SI-NEXT:    s_mov_b32 s0, s4
 ; SI-NEXT:    s_mov_b32 s1, s5
-; SI-NEXT:    s_addc_u32 s21, s21, 0
 ; SI-NEXT:    s_mov_b32 s8, s6
 ; SI-NEXT:    s_mov_b32 s9, s7
 ; SI-NEXT:    v_cvt_f32_f16_e32 v0, v0
 ; SI-NEXT:    v_cvt_f32_f16_e32 v1, v1
 ; SI-NEXT:    v_cvt_f32_f16_e32 v2, v2
 ; SI-NEXT:    v_madak_f32 v1, v0, v1, 0x41200000
 ; SI-NEXT:    v_mac_f32_e32 v3, v0, v2
 ; SI-NEXT:    v_cvt_f16_f32_e32 v0, v1
 ; SI-NEXT:    v_cvt_f16_f32_e32 v1, v3
 ; SI-NEXT:    buffer_store_short v0, off, s[0:3], 0
 ; SI-NEXT:    buffer_store_short v1, off, s[8:11], 0
 ; SI-NEXT:    s_endpgm
 ;
 ; VI-LABEL: madak_f16_use_2:
 ; VI:       ; %bb.0: ; %entry
-; VI-NEXT:    s_mov_b32 s20, SCRATCH_RSRC_DWORD0
 ; VI-NEXT:    s_load_dwordx8 s[4:11], s[0:1], 0x24
-; VI-NEXT:    s_mov_b32 s21, SCRATCH_RSRC_DWORD1
-; VI-NEXT:    s_mov_b32 s22, -1
-; VI-NEXT:    s_mov_b32 s23, 0xe80000
 ; VI-NEXT:    s_load_dwordx2 s[12:13], s[0:1], 0x44
-; VI-NEXT:    s_add_u32 s20, s20, s3
 ; VI-NEXT:    s_mov_b32 s3, 0xf000
 ; VI-NEXT:    s_mov_b32 s2, -1
+; VI-NEXT:    s_mov_b32 s18, s2
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    s_mov_b32 s16, s8
 ; VI-NEXT:    s_mov_b32 s17, s9
-; VI-NEXT:    s_mov_b32 s18, s2
 ; VI-NEXT:    s_mov_b32 s19, s3
 ; VI-NEXT:    s_mov_b32 s8, s10
 ; VI-NEXT:    s_mov_b32 s9, s11
 ; VI-NEXT:    s_mov_b32 s10, s2
 ; VI-NEXT:    s_mov_b32 s11, s3
 ; VI-NEXT:    s_mov_b32 s14, s2
 ; VI-NEXT:    s_mov_b32 s15, s3
 ; VI-NEXT:    buffer_load_ushort v0, off, s[16:19], 0 glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_load_ushort v1, off, s[8:11], 0 glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_load_ushort v2, off, s[12:15], 0 glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v3, 0x4900
 ; VI-NEXT:    s_mov_b32 s0, s4
 ; VI-NEXT:    s_mov_b32 s1, s5
-; VI-NEXT:    s_addc_u32 s21, s21, 0
 ; VI-NEXT:    s_mov_b32 s8, s6
 ; VI-NEXT:    s_mov_b32 s9, s7
 ; VI-NEXT:    v_madak_f16 v1, v0, v1, 0x4900
 ; VI-NEXT:    v_mac_f16_e32 v3, v0, v2
 ; VI-NEXT:    buffer_store_short v1, off, s[0:3], 0
 ; VI-NEXT:    buffer_store_short v3, off, s[8:11], 0
 ; VI-NEXT:    s_endpgm
     ptr addrspace(1) %r0,

llvm/test/CodeGen/AMDGPU/vector_shuffle.packed.ll

 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    v_mov_b32_e32 v2, v7
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: shuffle_v6f16_452367:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT:    global_load_b96 v[4:6], v[0:1], off
-; GFX11-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-NEXT:    scratch_store_b96 off, v[4:6], s32
-; GFX11-NEXT:    global_load_b96 v[4:6], v[2:3], off
-; GFX11-NEXT:    scratch_load_b128 v[0:3], off, s32
-; GFX11-NEXT:    s_waitcnt vmcnt(1)
-; GFX11-NEXT:    scratch_store_b96 off, v[4:6], s32 offset:16
-; GFX11-NEXT:    scratch_load_b32 v3, off, s32 offset:16
+; GFX11-NEXT:    v_dual_mov_b32 v4, v3 :: v_dual_mov_b32 v3, v2
+; GFX11-NEXT:    global_load_b96 v[0:2], v[0:1], off
+; GFX11-NEXT:    global_load_b32 v3, v[3:4], off
 ; GFX11-NEXT:    s_waitcnt vmcnt(1)
 ; GFX11-NEXT:    v_mov_b32_e32 v0, v2
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    v_mov_b32_e32 v2, v3
-; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %val0 = load <6 x half>, ptr addrspace(1) %arg0
   %val1 = load <6 x half>, ptr addrspace(1) %arg1
   %shuffle = shufflevector <6 x half> %val0, <6 x half> %val1, <6 x i32> <i32 4, i32 5, i32 2, i32 3, i32 6, i32 7>
   ret <6 x half> %shuffle
 }
 
 define amdgpu_kernel void @fma_shuffle(ptr addrspace(1) nocapture readonly %A, ptr addrspace(1) nocapture readonly %B, ptr addrspace(1) nocapture %C)  {