[X86] Support v2i32 gather/scatter indices with -x86-experimental-vector-widening-legalization

[craig.topper]

Summary

This is split out from D41062 to cover the code in LegalVectorTypes.cpp

[efriedma]

Widening everything when the index needs to be widened is awkward. It's not clear what's supposed to happen if widening the index to make it legal makes the value illegal. The way it's implemented, you'll end up in an infinite loop of splitting and widening. (Not sure the infinite loop can happen for legal gather/scatter operations on x86, but it could show up in other cases.) And for test_scatter_v2i32_index, we don't generate the expected instruction in either mode; it should be vscatterdpd %xmm0, (%rdi,%xmm1,8) {%k1}.

Can we implement some rule that allows the index of an MGATHER to have more elements than the data, and the upper elements are ignored? Sort of like an implicit SIGN_EXTEND_VECTOR_INREG.

[craig.topper]

Take Eli's suggestion and only widen the index. This fixes the scatter/gather index only widening cases to use the better instruction. The default 'promote' case is still promoting to a v2i64 index type since that's what type legalization will naturally do.

I've also added KNL and AVX2 command lines so we can see those too. On KNL we prefer to use the 512-bit gather/scatter with a vXi1 mask. For AVX2 we don't have scatter and the gather mask is a vector register.

[craig.topper]

I think I accidentally merged by commits on my tree and picked up some extra files.

[efriedma]

LGTM.

Please update ISDOpcodes.h at some point with more info on what exactly the operands to MGATHER/MSCATTER mean; I had to dig a bit to figure it out, and it's only getting more complicated with the new legalization rules.

lib/CodeGen/SelectionDAG/LegalizeTypes.h

private:
   bool WidenVectorOperand(SDNode *N, unsigned OpNo);
   SDValue WidenVecOp_BITCAST(SDNode *N);
   SDValue WidenVecOp_CONCAT_VECTORS(SDNode *N);
   SDValue WidenVecOp_EXTEND(SDNode *N);
   SDValue WidenVecOp_EXTRACT_VECTOR_ELT(SDNode *N);
   SDValue WidenVecOp_EXTRACT_SUBVECTOR(SDNode *N);
   SDValue WidenVecOp_STORE(SDNode* N);
   SDValue WidenVecOp_MSTORE(SDNode* N, unsigned OpNo);
+  SDValue WidenVecOp_MGATHER(SDNode* N, unsigned OpNo);
   SDValue WidenVecOp_MSCATTER(SDNode* N, unsigned OpNo);
   SDValue WidenVecOp_SETCC(SDNode* N);
 
   SDValue WidenVecOp_Convert(SDNode *N);
   SDValue WidenVecOp_FCOPYSIGN(SDNode *N);
 
   //===--------------------------------------------------------------------===//
   // Vector Widening Utilities Support: LegalizeVectorTypes.cpp

lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp

#endif
     llvm_unreachable("Do not know how to widen this operator's operand!");
 
   case ISD::BITCAST:            Res = WidenVecOp_BITCAST(N); break;
   case ISD::CONCAT_VECTORS:     Res = WidenVecOp_CONCAT_VECTORS(N); break;
   case ISD::EXTRACT_SUBVECTOR:  Res = WidenVecOp_EXTRACT_SUBVECTOR(N); break;
   case ISD::EXTRACT_VECTOR_ELT: Res = WidenVecOp_EXTRACT_VECTOR_ELT(N); break;
   case ISD::STORE:              Res = WidenVecOp_STORE(N); break;
   case ISD::MSTORE:             Res = WidenVecOp_MSTORE(N, OpNo); break;
+  case ISD::MGATHER:            Res = WidenVecOp_MGATHER(N, OpNo); break;
   case ISD::MSCATTER:           Res = WidenVecOp_MSCATTER(N, OpNo); break;
   case ISD::SETCC:              Res = WidenVecOp_SETCC(N); break;
   case ISD::FCOPYSIGN:          Res = WidenVecOp_FCOPYSIGN(N); break;
 
   case ISD::ANY_EXTEND:
   case ISD::SIGN_EXTEND:
   case ISD::ZERO_EXTEND:
     Res = WidenVecOp_EXTEND(N);
SDValue DAGTypeLegalizer::WidenVecOp_MSTORE(SDNode *N, unsigned OpNo) {
   assert(Mask.getValueType().getVectorNumElements() ==
          StVal.getValueType().getVectorNumElements() &&
          "Mask and data vectors should have the same number of elements");
   return DAG.getMaskedStore(MST->getChain(), dl, StVal, MST->getBasePtr(),
                             Mask, MST->getMemoryVT(), MST->getMemOperand(),
                             false, MST->isCompressingStore());
 }
 
+SDValue DAGTypeLegalizer::WidenVecOp_MGATHER(SDNode *N, unsigned OpNo) {
+  assert(OpNo == 4 && "Can widen only the index of mgather");
+  auto *MG = cast<MaskedGatherSDNode>(N);
+  SDValue DataOp = MG->getPassThru();
+  SDValue Mask = MG->getMask();
+  SDValue Scale = MG->getScale();
+
+  // Just widen the index. It's allowed to have extra elements.
+  SDValue Index = GetWidenedVector(MG->getIndex());
+
+  SDLoc dl(N);
+  SDValue Ops[] = {MG->getChain(), DataOp, Mask, MG->getBasePtr(), Index,
+                   Scale};
+  SDValue Res = DAG.getMaskedGather(MG->getVTList(), MG->getMemoryVT(), dl, Ops,
+                                    MG->getMemOperand());
+  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
+  ReplaceValueWith(SDValue(N, 0), Res.getValue(0));
+  return SDValue();
+}
+
 SDValue DAGTypeLegalizer::WidenVecOp_MSCATTER(SDNode *N, unsigned OpNo) {
-  assert(OpNo == 1 && "Can widen only data operand of mscatter");
   MaskedScatterSDNode *MSC = cast<MaskedScatterSDNode>(N);
   SDValue DataOp = MSC->getValue();
   SDValue Mask = MSC->getMask();
-  EVT MaskVT = Mask.getValueType();
+  SDValue Index = MSC->getIndex();
   SDValue Scale = MSC->getScale();
 
-  // Widen the value.
-  SDValue WideVal = GetWidenedVector(DataOp);
-  EVT WideVT = WideVal.getValueType();
-  unsigned NumElts = WideVT.getVectorNumElements();
-  SDLoc dl(N);
+  unsigned NumElts;
+  if (OpNo == 1) {
+    DataOp = GetWidenedVector(DataOp);
+    NumElts = DataOp.getValueType().getVectorNumElements();
+
+    // Widen index.
+    EVT IndexVT = Index.getValueType();
+    EVT WideIndexVT = EVT::getVectorVT(*DAG.getContext(),
+                                       IndexVT.getVectorElementType(), NumElts);
+    Index = ModifyToType(Index, WideIndexVT);
 
     // The mask should be widened as well.
+    EVT MaskVT = Mask.getValueType();
     EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(),
                                       MaskVT.getVectorElementType(), NumElts);
     Mask = ModifyToType(Mask, WideMaskVT, true);
+  } else if (OpNo == 4) {
+    // Just widen the index. It's allowed to have extra elements.
+    Index = GetWidenedVector(Index);
+  } else
+    llvm_unreachable("Can't widen this operand of mscatter");
 
-  // Widen index.
-  SDValue Index = MSC->getIndex();
-  EVT WideIndexVT = EVT::getVectorVT(*DAG.getContext(),
-                                     Index.getValueType().getScalarType(),
-                                     NumElts);
-  Index = ModifyToType(Index, WideIndexVT);
-
-  SDValue Ops[] = {MSC->getChain(), WideVal, Mask, MSC->getBasePtr(), Index,
+  SDValue Ops[] = {MSC->getChain(), DataOp, Mask, MSC->getBasePtr(), Index,
                    Scale};
   return DAG.getMaskedScatter(DAG.getVTList(MVT::Other),
-                              MSC->getMemoryVT(), dl, Ops,
+                              MSC->getMemoryVT(), SDLoc(N), Ops,
                               MSC->getMemOperand());
 }
 
 SDValue DAGTypeLegalizer::WidenVecOp_SETCC(SDNode *N) {
   SDValue InOp0 = GetWidenedVector(N->getOperand(0));
   SDValue InOp1 = GetWidenedVector(N->getOperand(1));
   SDLoc dl(N);
   EVT VT = N->getValueType(0);

lib/CodeGen/SelectionDAG/SelectionDAG.cpp

auto *N = newSDNode<MaskedGatherSDNode>(dl.getIROrder(), dl.getDebugLoc(),
                                           VTs, VT, MMO);
   createOperands(N, Ops);
 
   assert(N->getPassThru().getValueType() == N->getValueType(0) &&
          "Incompatible type of the PassThru value in MaskedGatherSDNode");
   assert(N->getMask().getValueType().getVectorNumElements() ==
              N->getValueType(0).getVectorNumElements() &&
          "Vector width mismatch between mask and data");
-  assert(N->getIndex().getValueType().getVectorNumElements() ==
+  assert(N->getIndex().getValueType().getVectorNumElements() >=
              N->getValueType(0).getVectorNumElements() &&
          "Vector width mismatch between index and data");
   assert(isa<ConstantSDNode>(N->getScale()) &&
          cast<ConstantSDNode>(N->getScale())->getAPIntValue().isPowerOf2() &&
          "Scale should be a constant power of 2");
 
   CSEMap.InsertNode(N, IP);
   InsertNode(N);
SDValue SelectionDAG::getMaskedScatter(SDVTList VTs, EVT VT, const SDLoc &dl,
   }
   auto *N = newSDNode<MaskedScatterSDNode>(dl.getIROrder(), dl.getDebugLoc(),
                                            VTs, VT, MMO);
   createOperands(N, Ops);
 
   assert(N->getMask().getValueType().getVectorNumElements() ==
              N->getValue().getValueType().getVectorNumElements() &&
          "Vector width mismatch between mask and data");
-  assert(N->getIndex().getValueType().getVectorNumElements() ==
+  assert(N->getIndex().getValueType().getVectorNumElements() >=
              N->getValue().getValueType().getVectorNumElements() &&
          "Vector width mismatch between index and data");
   assert(isa<ConstantSDNode>(N->getScale()) &&
          cast<ConstantSDNode>(N->getScale())->getAPIntValue().isPowerOf2() &&
          "Scale should be a constant power of 2");
 
   CSEMap.InsertNode(N, IP);
   InsertNode(N);

test/CodeGen/X86/masked_gather_scatter_widen.ll

+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=x86_64-unknown-linux-gnu -mattr=+avx512vl -mattr=+avx512dq -x86-experimental-vector-widening-legalization < %s | FileCheck %s --check-prefix=CHECK --check-prefix=WIDEN --check-prefix=WIDEN_SKX
+; RUN: llc -mtriple=x86_64-unknown-linux-gnu -mattr=+avx512f -x86-experimental-vector-widening-legalization < %s | FileCheck %s --check-prefix=CHECK --check-prefix=WIDEN --check-prefix=WIDEN_KNL
+; RUN: llc -mtriple=x86_64-unknown-linux-gnu -mattr=+avx512vl -mattr=+avx512dq < %s | FileCheck %s --check-prefix=CHECK --check-prefix=PROMOTE --check-prefix=PROMOTE_SKX
+; RUN: llc -mtriple=x86_64-unknown-linux-gnu -mattr=+avx512f < %s | FileCheck %s --check-prefix=CHECK --check-prefix=PROMOTE --check-prefix=PROMOTE_KNL
+; RUN: llc -mtriple=x86_64-unknown-linux-gnu -mcpu=skylake -x86-experimental-vector-widening-legalization < %s | FileCheck %s --check-prefix=WIDEN_AVX2
+; RUN: llc -mtriple=x86_64-unknown-linux-gnu -mcpu=skylake < %s | FileCheck %s --check-prefix=PROMOTE_AVX2
+
+define <2 x double> @test_gather_v2i32_index(double* %base, <2 x i32> %ind, <2 x i1> %mask, <2 x double> %src0) {
+; WIDEN_SKX-LABEL: test_gather_v2i32_index:
+; WIDEN_SKX:       # %bb.0:
+; WIDEN_SKX-NEXT:    vpsllq $63, %xmm1, %xmm1
+; WIDEN_SKX-NEXT:    vpmovq2m %xmm1, %k1
+; WIDEN_SKX-NEXT:    vgatherdpd (%rdi,%xmm0,8), %xmm2 {%k1}
+; WIDEN_SKX-NEXT:    vmovapd %xmm2, %xmm0
+; WIDEN_SKX-NEXT:    retq
+;
+; WIDEN_KNL-LABEL: test_gather_v2i32_index:
+; WIDEN_KNL:       # %bb.0:
+; WIDEN_KNL-NEXT:    # kill: def $xmm2 killed $xmm2 def $zmm2
+; WIDEN_KNL-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
+; WIDEN_KNL-NEXT:    vpsllq $63, %xmm1, %xmm1
+; WIDEN_KNL-NEXT:    vptestmq %zmm1, %zmm1, %k0
+; WIDEN_KNL-NEXT:    kshiftlw $14, %k0, %k0
+; WIDEN_KNL-NEXT:    kshiftrw $14, %k0, %k1
+; WIDEN_KNL-NEXT:    vgatherdpd (%rdi,%ymm0,8), %zmm2 {%k1}
+; WIDEN_KNL-NEXT:    vmovapd %xmm2, %xmm0
+; WIDEN_KNL-NEXT:    vzeroupper
+; WIDEN_KNL-NEXT:    retq
+;
+; PROMOTE_SKX-LABEL: test_gather_v2i32_index:
+; PROMOTE_SKX:       # %bb.0:
+; PROMOTE_SKX-NEXT:    vpsllq $32, %xmm0, %xmm0
+; PROMOTE_SKX-NEXT:    vpsraq $32, %xmm0, %xmm0
+; PROMOTE_SKX-NEXT:    vpsllq $63, %xmm1, %xmm1
+; PROMOTE_SKX-NEXT:    vpmovq2m %xmm1, %k1
+; PROMOTE_SKX-NEXT:    vgatherqpd (%rdi,%xmm0,8), %xmm2 {%k1}
+; PROMOTE_SKX-NEXT:    vmovapd %xmm2, %xmm0
+; PROMOTE_SKX-NEXT:    retq
+;
+; PROMOTE_KNL-LABEL: test_gather_v2i32_index:
+; PROMOTE_KNL:       # %bb.0:
+; PROMOTE_KNL-NEXT:    # kill: def $xmm2 killed $xmm2 def $zmm2
+; PROMOTE_KNL-NEXT:    vpsllq $32, %xmm0, %xmm0
+; PROMOTE_KNL-NEXT:    vpsraq $32, %zmm0, %zmm0
+; PROMOTE_KNL-NEXT:    vpsllq $63, %xmm1, %xmm1
+; PROMOTE_KNL-NEXT:    vptestmq %zmm1, %zmm1, %k0
+; PROMOTE_KNL-NEXT:    kshiftlw $14, %k0, %k0
+; PROMOTE_KNL-NEXT:    kshiftrw $14, %k0, %k1
+; PROMOTE_KNL-NEXT:    vgatherqpd (%rdi,%zmm0,8), %zmm2 {%k1}
+; PROMOTE_KNL-NEXT:    vmovapd %xmm2, %xmm0
+; PROMOTE_KNL-NEXT:    vzeroupper
+; PROMOTE_KNL-NEXT:    retq
+;
+; WIDEN_AVX2-LABEL: test_gather_v2i32_index:
+; WIDEN_AVX2:       # %bb.0:
+; WIDEN_AVX2-NEXT:    vpsllq $63, %xmm1, %xmm1
+; WIDEN_AVX2-NEXT:    vgatherdpd %xmm1, (%rdi,%xmm0,8), %xmm2
+; WIDEN_AVX2-NEXT:    vmovapd %xmm2, %xmm0
+; WIDEN_AVX2-NEXT:    retq
+;
+; PROMOTE_AVX2-LABEL: test_gather_v2i32_index:
+; PROMOTE_AVX2:       # %bb.0:
+; PROMOTE_AVX2-NEXT:    vpsllq $32, %xmm0, %xmm3
+; PROMOTE_AVX2-NEXT:    vpsrad $31, %xmm3, %xmm3
+; PROMOTE_AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm3[1],xmm0[2],xmm3[3]
+; PROMOTE_AVX2-NEXT:    vpsllq $63, %xmm1, %xmm1
+; PROMOTE_AVX2-NEXT:    vgatherqpd %xmm1, (%rdi,%xmm0,8), %xmm2
+; PROMOTE_AVX2-NEXT:    vmovapd %xmm2, %xmm0
+; PROMOTE_AVX2-NEXT:    retq
+  %gep.random = getelementptr double, double* %base, <2 x i32> %ind
+  %res = call <2 x double> @llvm.masked.gather.v2f64.v2p0f64(<2 x double*> %gep.random, i32 4, <2 x i1> %mask, <2 x double> %src0)
+  ret <2 x double> %res
+}
+
+define void @test_scatter_v2i32_index(<2 x double> %a1, double* %base, <2 x i32> %ind, <2 x i1> %mask) {
+; WIDEN_SKX-LABEL: test_scatter_v2i32_index:
+; WIDEN_SKX:       # %bb.0:
+; WIDEN_SKX-NEXT:    vpsllq $63, %xmm2, %xmm2
+; WIDEN_SKX-NEXT:    vpmovq2m %xmm2, %k1
+; WIDEN_SKX-NEXT:    vscatterdpd %xmm0, (%rdi,%xmm1,8) {%k1}
+; WIDEN_SKX-NEXT:    retq
+;
+; WIDEN_KNL-LABEL: test_scatter_v2i32_index:
+; WIDEN_KNL:       # %bb.0:
+; WIDEN_KNL-NEXT:    # kill: def $xmm1 killed $xmm1 def $ymm1
+; WIDEN_KNL-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; WIDEN_KNL-NEXT:    vpsllq $63, %xmm2, %xmm2
+; WIDEN_KNL-NEXT:    vptestmq %zmm2, %zmm2, %k0
+; WIDEN_KNL-NEXT:    kshiftlw $14, %k0, %k0
+; WIDEN_KNL-NEXT:    kshiftrw $14, %k0, %k1
+; WIDEN_KNL-NEXT:    vscatterdpd %zmm0, (%rdi,%ymm1,8) {%k1}
+; WIDEN_KNL-NEXT:    vzeroupper
+; WIDEN_KNL-NEXT:    retq
+;
+; PROMOTE_SKX-LABEL: test_scatter_v2i32_index:
+; PROMOTE_SKX:       # %bb.0:
+; PROMOTE_SKX-NEXT:    vpsllq $63, %xmm2, %xmm2
+; PROMOTE_SKX-NEXT:    vpmovq2m %xmm2, %k1
+; PROMOTE_SKX-NEXT:    vpsllq $32, %xmm1, %xmm1
+; PROMOTE_SKX-NEXT:    vpsraq $32, %xmm1, %xmm1
+; PROMOTE_SKX-NEXT:    vscatterqpd %xmm0, (%rdi,%xmm1,8) {%k1}
+; PROMOTE_SKX-NEXT:    retq
+;
+; PROMOTE_KNL-LABEL: test_scatter_v2i32_index:
+; PROMOTE_KNL:       # %bb.0:
+; PROMOTE_KNL-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; PROMOTE_KNL-NEXT:    vpsllq $32, %xmm1, %xmm1
+; PROMOTE_KNL-NEXT:    vpsraq $32, %zmm1, %zmm1
+; PROMOTE_KNL-NEXT:    vpsllq $63, %xmm2, %xmm2
+; PROMOTE_KNL-NEXT:    vptestmq %zmm2, %zmm2, %k0
+; PROMOTE_KNL-NEXT:    kshiftlw $14, %k0, %k0
+; PROMOTE_KNL-NEXT:    kshiftrw $14, %k0, %k1
+; PROMOTE_KNL-NEXT:    vscatterqpd %zmm0, (%rdi,%zmm1,8) {%k1}
+; PROMOTE_KNL-NEXT:    vzeroupper
+; PROMOTE_KNL-NEXT:    retq
+;
+; WIDEN_AVX2-LABEL: test_scatter_v2i32_index:
+; WIDEN_AVX2:       # %bb.0:
+; WIDEN_AVX2-NEXT:    vpmovsxdq %xmm1, %xmm1
+; WIDEN_AVX2-NEXT:    vpsllq $3, %xmm1, %xmm1
+; WIDEN_AVX2-NEXT:    vmovq %rdi, %xmm3
+; WIDEN_AVX2-NEXT:    vpbroadcastq %xmm3, %xmm3
+; WIDEN_AVX2-NEXT:    vpaddq %xmm1, %xmm3, %xmm1
+; WIDEN_AVX2-NEXT:    vpextrb $0, %xmm2, %eax
+; WIDEN_AVX2-NEXT:    testb $1, %al
+; WIDEN_AVX2-NEXT:    je .LBB1_2
+; WIDEN_AVX2-NEXT:  # %bb.1: # %cond.store
+; WIDEN_AVX2-NEXT:    vmovq %xmm1, %rax
+; WIDEN_AVX2-NEXT:    vmovlpd %xmm0, (%rax)
+; WIDEN_AVX2-NEXT:  .LBB1_2: # %else
+; WIDEN_AVX2-NEXT:    vpextrb $8, %xmm2, %eax
+; WIDEN_AVX2-NEXT:    testb $1, %al
+; WIDEN_AVX2-NEXT:    jne .LBB1_3
+; WIDEN_AVX2-NEXT:  # %bb.4: # %else2
+; WIDEN_AVX2-NEXT:    retq
+; WIDEN_AVX2-NEXT:  .LBB1_3: # %cond.store1
+; WIDEN_AVX2-NEXT:    vpextrq $1, %xmm1, %rax
+; WIDEN_AVX2-NEXT:    vmovhpd %xmm0, (%rax)
+; WIDEN_AVX2-NEXT:    retq
+;
+; PROMOTE_AVX2-LABEL: test_scatter_v2i32_index:
+; PROMOTE_AVX2:       # %bb.0:
+; PROMOTE_AVX2-NEXT:    vpsllq $32, %xmm1, %xmm3
+; PROMOTE_AVX2-NEXT:    vpsrad $31, %xmm3, %xmm3
+; PROMOTE_AVX2-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0],xmm3[1],xmm1[2],xmm3[3]
+; PROMOTE_AVX2-NEXT:    vpsllq $3, %xmm1, %xmm1
+; PROMOTE_AVX2-NEXT:    vmovq %rdi, %xmm3
+; PROMOTE_AVX2-NEXT:    vpbroadcastq %xmm3, %xmm3
+; PROMOTE_AVX2-NEXT:    vpaddq %xmm1, %xmm3, %xmm1
+; PROMOTE_AVX2-NEXT:    vpextrb $0, %xmm2, %eax
+; PROMOTE_AVX2-NEXT:    testb $1, %al
+; PROMOTE_AVX2-NEXT:    je .LBB1_2
+; PROMOTE_AVX2-NEXT:  # %bb.1: # %cond.store
+; PROMOTE_AVX2-NEXT:    vmovq %xmm1, %rax
+; PROMOTE_AVX2-NEXT:    vmovlpd %xmm0, (%rax)
+; PROMOTE_AVX2-NEXT:  .LBB1_2: # %else
+; PROMOTE_AVX2-NEXT:    vpextrb $8, %xmm2, %eax
+; PROMOTE_AVX2-NEXT:    testb $1, %al
+; PROMOTE_AVX2-NEXT:    jne .LBB1_3
+; PROMOTE_AVX2-NEXT:  # %bb.4: # %else2
+; PROMOTE_AVX2-NEXT:    retq
+; PROMOTE_AVX2-NEXT:  .LBB1_3: # %cond.store1
+; PROMOTE_AVX2-NEXT:    vpextrq $1, %xmm1, %rax
+; PROMOTE_AVX2-NEXT:    vmovhpd %xmm0, (%rax)
+; PROMOTE_AVX2-NEXT:    retq
+  %gep = getelementptr double, double *%base, <2 x i32> %ind
+  call void @llvm.masked.scatter.v2f64.v2p0f64(<2 x double> %a1, <2 x double*> %gep, i32 4, <2 x i1> %mask)
+  ret void
+}
+
+define <2 x i32> @test_gather_v2i32_data(<2 x i32*> %ptr, <2 x i1> %mask, <2 x i32> %src0) {
+; WIDEN_SKX-LABEL: test_gather_v2i32_data:
+; WIDEN_SKX:       # %bb.0:
+; WIDEN_SKX-NEXT:    vpsllq $63, %xmm1, %xmm1
+; WIDEN_SKX-NEXT:    vpmovq2m %xmm1, %k1
+; WIDEN_SKX-NEXT:    vpgatherqd (,%xmm0), %xmm2 {%k1}
+; WIDEN_SKX-NEXT:    vmovdqa %xmm2, %xmm0
+; WIDEN_SKX-NEXT:    retq
+;
+; WIDEN_KNL-LABEL: test_gather_v2i32_data:
+; WIDEN_KNL:       # %bb.0:
+; WIDEN_KNL-NEXT:    # kill: def $xmm2 killed $xmm2 def $ymm2
+; WIDEN_KNL-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; WIDEN_KNL-NEXT:    vpsllq $63, %xmm1, %xmm1
+; WIDEN_KNL-NEXT:    vptestmq %zmm1, %zmm1, %k0
+; WIDEN_KNL-NEXT:    kshiftlw $14, %k0, %k0
+; WIDEN_KNL-NEXT:    kshiftrw $14, %k0, %k1
+; WIDEN_KNL-NEXT:    vpgatherqd (,%zmm0), %ymm2 {%k1}
+; WIDEN_KNL-NEXT:    vmovdqa %xmm2, %xmm0
+; WIDEN_KNL-NEXT:    vzeroupper
+; WIDEN_KNL-NEXT:    retq
+;
+; PROMOTE_SKX-LABEL: test_gather_v2i32_data:
+; PROMOTE_SKX:       # %bb.0:
+; PROMOTE_SKX-NEXT:    vpsllq $63, %xmm1, %xmm1
+; PROMOTE_SKX-NEXT:    vpmovq2m %xmm1, %k1
+; PROMOTE_SKX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm2[0,2,2,3]
+; PROMOTE_SKX-NEXT:    vpgatherqd (,%xmm0), %xmm1 {%k1}
+; PROMOTE_SKX-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm1[0],zero,xmm1[1],zero
+; PROMOTE_SKX-NEXT:    retq
+;
+; PROMOTE_KNL-LABEL: test_gather_v2i32_data:
+; PROMOTE_KNL:       # %bb.0:
+; PROMOTE_KNL-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; PROMOTE_KNL-NEXT:    vpsllq $63, %xmm1, %xmm1
+; PROMOTE_KNL-NEXT:    vptestmq %zmm1, %zmm1, %k0
+; PROMOTE_KNL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm2[0,2,2,3]
+; PROMOTE_KNL-NEXT:    kshiftlw $14, %k0, %k0
+; PROMOTE_KNL-NEXT:    kshiftrw $14, %k0, %k1
+; PROMOTE_KNL-NEXT:    vpgatherqd (,%zmm0), %ymm1 {%k1}
+; PROMOTE_KNL-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm1[0],zero,xmm1[1],zero
+; PROMOTE_KNL-NEXT:    vzeroupper
+; PROMOTE_KNL-NEXT:    retq
+;
+; WIDEN_AVX2-LABEL: test_gather_v2i32_data:
+; WIDEN_AVX2:       # %bb.0:
+; WIDEN_AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; WIDEN_AVX2-NEXT:    vpslld $31, %xmm1, %xmm1
+; WIDEN_AVX2-NEXT:    vpgatherqd %xmm1, (,%xmm0), %xmm2
+; WIDEN_AVX2-NEXT:    vmovdqa %xmm2, %xmm0
+; WIDEN_AVX2-NEXT:    retq
+;
+; PROMOTE_AVX2-LABEL: test_gather_v2i32_data:
+; PROMOTE_AVX2:       # %bb.0:
+; PROMOTE_AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
+; PROMOTE_AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; PROMOTE_AVX2-NEXT:    vpslld $31, %xmm1, %xmm1
+; PROMOTE_AVX2-NEXT:    vpgatherqd %xmm1, (,%xmm0), %xmm2
+; PROMOTE_AVX2-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm2[0],zero,xmm2[1],zero
+; PROMOTE_AVX2-NEXT:    retq
+  %res = call <2 x i32> @llvm.masked.gather.v2i32.v2p0i32(<2 x i32*> %ptr, i32 4, <2 x i1> %mask, <2 x i32> %src0)
+  ret <2 x i32>%res
+}
+
+define void @test_scatter_v2i32_data(<2 x i32>%a1, <2 x i32*> %ptr, <2 x i1>%mask) {
+; WIDEN_SKX-LABEL: test_scatter_v2i32_data:
+; WIDEN_SKX:       # %bb.0:
+; WIDEN_SKX-NEXT:    vpsllq $63, %xmm2, %xmm2
+; WIDEN_SKX-NEXT:    vpmovq2m %xmm2, %k1
+; WIDEN_SKX-NEXT:    vpscatterqd %xmm0, (,%xmm1) {%k1}
+; WIDEN_SKX-NEXT:    retq
+;
+; WIDEN_KNL-LABEL: test_scatter_v2i32_data:
+; WIDEN_KNL:       # %bb.0:
+; WIDEN_KNL-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; WIDEN_KNL-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
+; WIDEN_KNL-NEXT:    vpsllq $63, %xmm2, %xmm2
+; WIDEN_KNL-NEXT:    vptestmq %zmm2, %zmm2, %k0
+; WIDEN_KNL-NEXT:    kshiftlw $14, %k0, %k0
+; WIDEN_KNL-NEXT:    kshiftrw $14, %k0, %k1
+; WIDEN_KNL-NEXT:    vpscatterqd %ymm0, (,%zmm1) {%k1}
+; WIDEN_KNL-NEXT:    vzeroupper
+; WIDEN_KNL-NEXT:    retq
+;
+; PROMOTE_SKX-LABEL: test_scatter_v2i32_data:
+; PROMOTE_SKX:       # %bb.0:
+; PROMOTE_SKX-NEXT:    vpsllq $63, %xmm2, %xmm2
+; PROMOTE_SKX-NEXT:    vpmovq2m %xmm2, %k1
+; PROMOTE_SKX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; PROMOTE_SKX-NEXT:    vpscatterqd %xmm0, (,%xmm1) {%k1}
+; PROMOTE_SKX-NEXT:    retq
+;
+; PROMOTE_KNL-LABEL: test_scatter_v2i32_data:
+; PROMOTE_KNL:       # %bb.0:
+; PROMOTE_KNL-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; PROMOTE_KNL-NEXT:    vpsllq $63, %xmm2, %xmm2
+; PROMOTE_KNL-NEXT:    vptestmq %zmm2, %zmm2, %k0
+; PROMOTE_KNL-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; PROMOTE_KNL-NEXT:    kshiftlw $14, %k0, %k0
+; PROMOTE_KNL-NEXT:    kshiftrw $14, %k0, %k1
+; PROMOTE_KNL-NEXT:    vpscatterqd %ymm0, (,%zmm1) {%k1}
+; PROMOTE_KNL-NEXT:    vzeroupper
+; PROMOTE_KNL-NEXT:    retq
+;
+; WIDEN_AVX2-LABEL: test_scatter_v2i32_data:
+; WIDEN_AVX2:       # %bb.0:
+; WIDEN_AVX2-NEXT:    vpextrb $0, %xmm2, %eax
+; WIDEN_AVX2-NEXT:    testb $1, %al
+; WIDEN_AVX2-NEXT:    je .LBB3_2
+; WIDEN_AVX2-NEXT:  # %bb.1: # %cond.store
+; WIDEN_AVX2-NEXT:    vmovq %xmm1, %rax
+; WIDEN_AVX2-NEXT:    vmovss %xmm0, (%rax)
+; WIDEN_AVX2-NEXT:  .LBB3_2: # %else
+; WIDEN_AVX2-NEXT:    vpextrb $8, %xmm2, %eax
+; WIDEN_AVX2-NEXT:    testb $1, %al
+; WIDEN_AVX2-NEXT:    jne .LBB3_3
+; WIDEN_AVX2-NEXT:  # %bb.4: # %else2
+; WIDEN_AVX2-NEXT:    retq
+; WIDEN_AVX2-NEXT:  .LBB3_3: # %cond.store1
+; WIDEN_AVX2-NEXT:    vpextrq $1, %xmm1, %rax
+; WIDEN_AVX2-NEXT:    vextractps $1, %xmm0, (%rax)
+; WIDEN_AVX2-NEXT:    retq
+;
+; PROMOTE_AVX2-LABEL: test_scatter_v2i32_data:
+; PROMOTE_AVX2:       # %bb.0:
+; PROMOTE_AVX2-NEXT:    vpextrb $0, %xmm2, %eax
+; PROMOTE_AVX2-NEXT:    testb $1, %al
+; PROMOTE_AVX2-NEXT:    je .LBB3_2
+; PROMOTE_AVX2-NEXT:  # %bb.1: # %cond.store
+; PROMOTE_AVX2-NEXT:    vmovq %xmm1, %rax
+; PROMOTE_AVX2-NEXT:    vmovss %xmm0, (%rax)
+; PROMOTE_AVX2-NEXT:  .LBB3_2: # %else
+; PROMOTE_AVX2-NEXT:    vpextrb $8, %xmm2, %eax
+; PROMOTE_AVX2-NEXT:    testb $1, %al
+; PROMOTE_AVX2-NEXT:    jne .LBB3_3
+; PROMOTE_AVX2-NEXT:  # %bb.4: # %else2
+; PROMOTE_AVX2-NEXT:    retq
+; PROMOTE_AVX2-NEXT:  .LBB3_3: # %cond.store1
+; PROMOTE_AVX2-NEXT:    vpextrq $1, %xmm1, %rax
+; PROMOTE_AVX2-NEXT:    vextractps $2, %xmm0, (%rax)
+; PROMOTE_AVX2-NEXT:    retq
+  call void @llvm.masked.scatter.v2i32.v2p0i32(<2 x i32> %a1, <2 x i32*> %ptr, i32 4, <2 x i1> %mask)
+  ret void
+}
+
+define <2 x i32> @test_gather_v2i32_data_index(i32* %base, <2 x i32> %ind, <2 x i1> %mask, <2 x i32> %src0) {
+; WIDEN_SKX-LABEL: test_gather_v2i32_data_index:
+; WIDEN_SKX:       # %bb.0:
+; WIDEN_SKX-NEXT:    vpsllq $63, %xmm1, %xmm1
+; WIDEN_SKX-NEXT:    vpmovq2m %xmm1, %k1
+; WIDEN_SKX-NEXT:    vpgatherdd (%rdi,%xmm0,4), %xmm2 {%k1}
+; WIDEN_SKX-NEXT:    vmovdqa %xmm2, %xmm0
+; WIDEN_SKX-NEXT:    retq
+;
+; WIDEN_KNL-LABEL: test_gather_v2i32_data_index:
+; WIDEN_KNL:       # %bb.0:
+; WIDEN_KNL-NEXT:    # kill: def $xmm2 killed $xmm2 def $zmm2
+; WIDEN_KNL-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; WIDEN_KNL-NEXT:    vpsllq $63, %xmm1, %xmm1
+; WIDEN_KNL-NEXT:    vptestmq %zmm1, %zmm1, %k0
+; WIDEN_KNL-NEXT:    kshiftlw $14, %k0, %k0
+; WIDEN_KNL-NEXT:    kshiftrw $14, %k0, %k1
+; WIDEN_KNL-NEXT:    vpgatherdd (%rdi,%zmm0,4), %zmm2 {%k1}
+; WIDEN_KNL-NEXT:    vmovdqa %xmm2, %xmm0
+; WIDEN_KNL-NEXT:    vzeroupper
+; WIDEN_KNL-NEXT:    retq
+;
+; PROMOTE_SKX-LABEL: test_gather_v2i32_data_index:
+; PROMOTE_SKX:       # %bb.0:
+; PROMOTE_SKX-NEXT:    vpsllq $63, %xmm1, %xmm1
+; PROMOTE_SKX-NEXT:    vpmovq2m %xmm1, %k1
+; PROMOTE_SKX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; PROMOTE_SKX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm2[0,2,2,3]
+; PROMOTE_SKX-NEXT:    vpgatherdd (%rdi,%xmm0,4), %xmm1 {%k1}
+; PROMOTE_SKX-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm1[0],zero,xmm1[1],zero
+; PROMOTE_SKX-NEXT:    retq
+;
+; PROMOTE_KNL-LABEL: test_gather_v2i32_data_index:
+; PROMOTE_KNL:       # %bb.0:
+; PROMOTE_KNL-NEXT:    vpsllq $63, %xmm1, %xmm1
+; PROMOTE_KNL-NEXT:    vptestmq %zmm1, %zmm1, %k0
+; PROMOTE_KNL-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; PROMOTE_KNL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm2[0,2,2,3]
+; PROMOTE_KNL-NEXT:    kshiftlw $14, %k0, %k0
+; PROMOTE_KNL-NEXT:    kshiftrw $14, %k0, %k1
+; PROMOTE_KNL-NEXT:    vpgatherdd (%rdi,%zmm0,4), %zmm1 {%k1}
+; PROMOTE_KNL-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm1[0],zero,xmm1[1],zero
+; PROMOTE_KNL-NEXT:    vzeroupper
+; PROMOTE_KNL-NEXT:    retq
+;
+; WIDEN_AVX2-LABEL: test_gather_v2i32_data_index:
+; WIDEN_AVX2:       # %bb.0:
+; WIDEN_AVX2-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,2],zero,zero
+; WIDEN_AVX2-NEXT:    vpslld $31, %xmm1, %xmm1
+; WIDEN_AVX2-NEXT:    vpgatherdd %xmm1, (%rdi,%xmm0,4), %xmm2
+; WIDEN_AVX2-NEXT:    vmovdqa %xmm2, %xmm0
+; WIDEN_AVX2-NEXT:    retq
+;
+; PROMOTE_AVX2-LABEL: test_gather_v2i32_data_index:
+; PROMOTE_AVX2:       # %bb.0:
+; PROMOTE_AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; PROMOTE_AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
+; PROMOTE_AVX2-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,2],zero,zero
+; PROMOTE_AVX2-NEXT:    vpslld $31, %xmm1, %xmm1
+; PROMOTE_AVX2-NEXT:    vpgatherdd %xmm1, (%rdi,%xmm0,4), %xmm2
+; PROMOTE_AVX2-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm2[0],zero,xmm2[1],zero
+; PROMOTE_AVX2-NEXT:    retq
+  %gep.random = getelementptr i32, i32* %base, <2 x i32> %ind
+  %res = call <2 x i32> @llvm.masked.gather.v2i32.v2p0i32(<2 x i32*> %gep.random, i32 4, <2 x i1> %mask, <2 x i32> %src0)
+  ret <2 x i32> %res
+}
+
+define void @test_scatter_v2i32_data_index(<2 x i32> %a1, i32* %base, <2 x i32> %ind, <2 x i1> %mask) {
+; WIDEN_SKX-LABEL: test_scatter_v2i32_data_index:
+; WIDEN_SKX:       # %bb.0:
+; WIDEN_SKX-NEXT:    vpsllq $63, %xmm2, %xmm2
+; WIDEN_SKX-NEXT:    vpmovq2m %xmm2, %k1
+; WIDEN_SKX-NEXT:    vpscatterdd %xmm0, (%rdi,%xmm1,4) {%k1}
+; WIDEN_SKX-NEXT:    retq
+;
+; WIDEN_KNL-LABEL: test_scatter_v2i32_data_index:
+; WIDEN_KNL:       # %bb.0:
+; WIDEN_KNL-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; WIDEN_KNL-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; WIDEN_KNL-NEXT:    vpsllq $63, %xmm2, %xmm2
+; WIDEN_KNL-NEXT:    vptestmq %zmm2, %zmm2, %k0
+; WIDEN_KNL-NEXT:    kshiftlw $14, %k0, %k0
+; WIDEN_KNL-NEXT:    kshiftrw $14, %k0, %k1
+; WIDEN_KNL-NEXT:    vpscatterdd %zmm0, (%rdi,%zmm1,4) {%k1}
+; WIDEN_KNL-NEXT:    vzeroupper
+; WIDEN_KNL-NEXT:    retq
+;
+; PROMOTE_SKX-LABEL: test_scatter_v2i32_data_index:
+; PROMOTE_SKX:       # %bb.0:
+; PROMOTE_SKX-NEXT:    vpsllq $63, %xmm2, %xmm2
+; PROMOTE_SKX-NEXT:    vpmovq2m %xmm2, %k1
+; PROMOTE_SKX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; PROMOTE_SKX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; PROMOTE_SKX-NEXT:    vpscatterdd %xmm0, (%rdi,%xmm1,4) {%k1}
+; PROMOTE_SKX-NEXT:    retq
+;
+; PROMOTE_KNL-LABEL: test_scatter_v2i32_data_index:
+; PROMOTE_KNL:       # %bb.0:
+; PROMOTE_KNL-NEXT:    vpsllq $63, %xmm2, %xmm2
+; PROMOTE_KNL-NEXT:    vptestmq %zmm2, %zmm2, %k0
+; PROMOTE_KNL-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; PROMOTE_KNL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; PROMOTE_KNL-NEXT:    kshiftlw $14, %k0, %k0
+; PROMOTE_KNL-NEXT:    kshiftrw $14, %k0, %k1
+; PROMOTE_KNL-NEXT:    vpscatterdd %zmm0, (%rdi,%zmm1,4) {%k1}
+; PROMOTE_KNL-NEXT:    vzeroupper
+; PROMOTE_KNL-NEXT:    retq
+;
+; WIDEN_AVX2-LABEL: test_scatter_v2i32_data_index:
+; WIDEN_AVX2:       # %bb.0:
+; WIDEN_AVX2-NEXT:    vpmovsxdq %xmm1, %xmm1
+; WIDEN_AVX2-NEXT:    vpsllq $2, %xmm1, %xmm1
+; WIDEN_AVX2-NEXT:    vmovq %rdi, %xmm3
+; WIDEN_AVX2-NEXT:    vpbroadcastq %xmm3, %xmm3
+; WIDEN_AVX2-NEXT:    vpaddq %xmm1, %xmm3, %xmm1
+; WIDEN_AVX2-NEXT:    vpextrb $0, %xmm2, %eax
+; WIDEN_AVX2-NEXT:    testb $1, %al
+; WIDEN_AVX2-NEXT:    je .LBB5_2
+; WIDEN_AVX2-NEXT:  # %bb.1: # %cond.store
+; WIDEN_AVX2-NEXT:    vmovq %xmm1, %rax
+; WIDEN_AVX2-NEXT:    vmovss %xmm0, (%rax)
+; WIDEN_AVX2-NEXT:  .LBB5_2: # %else
+; WIDEN_AVX2-NEXT:    vpextrb $8, %xmm2, %eax
+; WIDEN_AVX2-NEXT:    testb $1, %al
+; WIDEN_AVX2-NEXT:    jne .LBB5_3
+; WIDEN_AVX2-NEXT:  # %bb.4: # %else2
+; WIDEN_AVX2-NEXT:    retq
+; WIDEN_AVX2-NEXT:  .LBB5_3: # %cond.store1
+; WIDEN_AVX2-NEXT:    vpextrq $1, %xmm1, %rax
+; WIDEN_AVX2-NEXT:    vextractps $1, %xmm0, (%rax)
+; WIDEN_AVX2-NEXT:    retq
+;
+; PROMOTE_AVX2-LABEL: test_scatter_v2i32_data_index:
+; PROMOTE_AVX2:       # %bb.0:
+; PROMOTE_AVX2-NEXT:    vpsllq $32, %xmm1, %xmm3
+; PROMOTE_AVX2-NEXT:    vpsrad $31, %xmm3, %xmm3
+; PROMOTE_AVX2-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0],xmm3[1],xmm1[2],xmm3[3]
+; PROMOTE_AVX2-NEXT:    vpsllq $2, %xmm1, %xmm1
+; PROMOTE_AVX2-NEXT:    vmovq %rdi, %xmm3
+; PROMOTE_AVX2-NEXT:    vpbroadcastq %xmm3, %xmm3
+; PROMOTE_AVX2-NEXT:    vpaddq %xmm1, %xmm3, %xmm1
+; PROMOTE_AVX2-NEXT:    vpextrb $0, %xmm2, %eax
+; PROMOTE_AVX2-NEXT:    testb $1, %al
+; PROMOTE_AVX2-NEXT:    je .LBB5_2
+; PROMOTE_AVX2-NEXT:  # %bb.1: # %cond.store
+; PROMOTE_AVX2-NEXT:    vmovq %xmm1, %rax
+; PROMOTE_AVX2-NEXT:    vmovss %xmm0, (%rax)
+; PROMOTE_AVX2-NEXT:  .LBB5_2: # %else
+; PROMOTE_AVX2-NEXT:    vpextrb $8, %xmm2, %eax
+; PROMOTE_AVX2-NEXT:    testb $1, %al
+; PROMOTE_AVX2-NEXT:    jne .LBB5_3
+; PROMOTE_AVX2-NEXT:  # %bb.4: # %else2
+; PROMOTE_AVX2-NEXT:    retq
+; PROMOTE_AVX2-NEXT:  .LBB5_3: # %cond.store1
+; PROMOTE_AVX2-NEXT:    vpextrq $1, %xmm1, %rax
+; PROMOTE_AVX2-NEXT:    vextractps $2, %xmm0, (%rax)
+; PROMOTE_AVX2-NEXT:    retq
+  %gep = getelementptr i32, i32 *%base, <2 x i32> %ind
+  call void @llvm.masked.scatter.v2i32.v2p0i32(<2 x i32> %a1, <2 x i32*> %gep, i32 4, <2 x i1> %mask)
+  ret void
+}
+
+declare <2 x double> @llvm.masked.gather.v2f64.v2p0f64(<2 x double*>, i32, <2 x i1>, <2 x double>)
+declare void @llvm.masked.scatter.v2f64.v2p0f64(<2 x double>, <2 x double*>, i32, <2 x i1>)
+declare <2 x i32> @llvm.masked.gather.v2i32.v2p0i32(<2 x i32*>, i32, <2 x i1>, <2 x i32>)
+declare void @llvm.masked.scatter.v2i32.v2p0i32(<2 x i32> , <2 x i32*> , i32 , <2 x i1>)