[SelectionDAGBuilder] Don't create MGATHER/MSCATTER with Scale != ElemSize

[nikic]

Summary

This is an alternative to D124530. In getUniformBase() only create scales that match the gather/scatter element size. If targets also support other scales, then they can produce those scales in DAG combines. This is what X86 already does (as long as the resulting scale would be 1, 2, 4 or 8).

This essentially restores the pre-opaque-pointer state of things.

Fixes https://github.com/llvm/llvm-project/issues/55021.

[paulwalker-arm]

In principle I prefer this route as it means a target doesn't have to support something they never want to use and it'll also allow DAGCombine more time to optimise the parts that cannot be done by the target's gather/scatter instructions themselves.

llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp

4410

Shouldn't we also always allow unit scaling (i.e. ScaleVal==1)? I know there's the IndexType = ISD::SIGNED_SCALED oddity, but that's the unnecessary duplication I mentioned before that I have inflight patches to remove (see D123381).

[nikic]

Also allow "scaling" by 1.

[pengfei]

LGTM, thanks!

llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp

 // %res = call <8 x i32> @llvm.masked.gather.v8i32(<8 x i32*> %gep.ptr, ..
 //
 // When the first GEP operand is a single pointer - it is the uniform base we
 // are looking for. If first operand of the GEP is a splat vector - we
 // extract the splat value and use it as a uniform base.
 // In all other cases the function returns 'false'.
 static bool getUniformBase(const Value *Ptr, SDValue &Base, SDValue &Index,
                            ISD::MemIndexType &IndexType, SDValue &Scale,
-                           SelectionDAGBuilder *SDB, const BasicBlock *CurBB) {
+                           SelectionDAGBuilder *SDB, const BasicBlock *CurBB,
+                           uint64_t ElemSize) {
   SelectionDAG& DAG = SDB->DAG;
   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
   const DataLayout &DL = DAG.getDataLayout();
 
   assert(Ptr->getType()->isVectorTy() && "Unexpected pointer type");
 
   // Handle splat constant pointer.
   if (auto *C = dyn_cast<Constant>(Ptr)) {
static bool getUniformBase(const Value *Ptr, SDValue &Base, SDValue &Index,
   // Make sure the base is scalar and the index is a vector.
   if (BasePtr->getType()->isVectorTy() || !IndexVal->getType()->isVectorTy())
     return false;
 
   Base = SDB->getValue(BasePtr);
   Index = SDB->getValue(IndexVal);
   IndexType = ISD::SIGNED_SCALED;
 
-  // MGATHER/MSCATTER only support scaling by a power-of-two.
+  // MGATHER/MSCATTER are only required to support scaling by one or by the
+  // element size. Other scales may be produced using target-specific DAG
+  // combines.
   uint64_t ScaleVal = DL.getTypeAllocSize(GEP->getResultElementType());
-  if (!isPowerOf2_64(ScaleVal))
+  if (ScaleVal != ElemSize && ScaleVal != 1)
     return false;
 
   Scale =
       DAG.getTargetConstant(ScaleVal, SDB->getCurSDLoc(), TLI.getPointerTy(DL));
   return true;
 }
 
 void SelectionDAGBuilder::visitMaskedScatter(const CallInst &I) {
Align Alignment = cast<ConstantInt>(I.getArgOperand(2))
                         .getValueOr(DAG.getEVTAlign(VT.getScalarType()));
   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
 
   SDValue Base;
   SDValue Index;
   ISD::MemIndexType IndexType;
   SDValue Scale;
   bool UniformBase = getUniformBase(Ptr, Base, Index, IndexType, Scale, this,
-                                    I.getParent());
+                                    I.getParent(), VT.getScalarStoreSize());
 
   unsigned AS = Ptr->getType()->getScalarType()->getPointerAddressSpace();
   MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
       MachinePointerInfo(AS), MachineMemOperand::MOStore,
       // TODO: Make MachineMemOperands aware of scalable
       // vectors.
       MemoryLocation::UnknownSize, Alignment, I.getAAMetadata());
   if (!UniformBase) {
void SelectionDAGBuilder::visitMaskedGather(const CallInst &I) {
   const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range);
 
   SDValue Root = DAG.getRoot();
   SDValue Base;
   SDValue Index;
   ISD::MemIndexType IndexType;
   SDValue Scale;
   bool UniformBase = getUniformBase(Ptr, Base, Index, IndexType, Scale, this,
-                                    I.getParent());
+                                    I.getParent(), VT.getScalarStoreSize());
   unsigned AS = Ptr->getType()->getScalarType()->getPointerAddressSpace();
   MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
       MachinePointerInfo(AS), MachineMemOperand::MOLoad,
       // TODO: Make MachineMemOperands aware of scalable
       // vectors.
       MemoryLocation::UnknownSize, Alignment, I.getAAMetadata(), Ranges);
 
   if (!UniformBase) {
if (!IsGather) {
     unsigned AS =
         PtrOperand->getType()->getScalarType()->getPointerAddressSpace();
     MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
         MachinePointerInfo(AS), MachineMemOperand::MOLoad,
         MemoryLocation::UnknownSize, *Alignment, AAInfo, Ranges);
     SDValue Base, Index, Scale;
     ISD::MemIndexType IndexType;
     bool UniformBase = getUniformBase(PtrOperand, Base, Index, IndexType, Scale,
-                                      this, VPIntrin.getParent());
+                                      this, VPIntrin.getParent(),
+                                      VT.getScalarStoreSize());
     if (!UniformBase) {
       Base = DAG.getConstant(0, DL, TLI.getPointerTy(DAG.getDataLayout()));
       Index = getValue(PtrOperand);
       IndexType = ISD::SIGNED_UNSCALED;
       Scale =
           DAG.getTargetConstant(1, DL, TLI.getPointerTy(DAG.getDataLayout()));
     }
     EVT IdxVT = Index.getValueType();
if (!IsScatter) {
     unsigned AS =
         PtrOperand->getType()->getScalarType()->getPointerAddressSpace();
     MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
         MachinePointerInfo(AS), MachineMemOperand::MOStore,
         MemoryLocation::UnknownSize, *Alignment, AAInfo);
     SDValue Base, Index, Scale;
     ISD::MemIndexType IndexType;
     bool UniformBase = getUniformBase(PtrOperand, Base, Index, IndexType, Scale,
-                                      this, VPIntrin.getParent());
+                                      this, VPIntrin.getParent(),
+                                      VT.getScalarStoreSize());
     if (!UniformBase) {
       Base = DAG.getConstant(0, DL, TLI.getPointerTy(DAG.getDataLayout()));
       Index = getValue(PtrOperand);
       IndexType = ISD::SIGNED_UNSCALED;
       Scale =
           DAG.getTargetConstant(1, DL, TLI.getPointerTy(DAG.getDataLayout()));
     }
     EVT IdxVT = Index.getValueType();

llvm/test/CodeGen/X86/gather-scatter-opaque-ptr-2.ll

+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=x86_64-linux -mcpu=skylake -mattr=+avx2 < %s | FileCheck %s
+
+define <2 x float> @gather_v2f32_scale_512(ptr %result, <2 x i64> %idx, <2 x i1> %mask) {
+; CHECK-LABEL: gather_v2f32_scale_512:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpsllq $9, %xmm0, %xmm2
+; CHECK-NEXT:    vpshufd {{.*#+}} xmm0 = xmm1[0,2,2,3]
+; CHECK-NEXT:    vpslld $31, %xmm0, %xmm1
+; CHECK-NEXT:    vpxor %xmm0, %xmm0, %xmm0
+; CHECK-NEXT:    vgatherqps %xmm1, (%rdi,%xmm2), %xmm0
+; CHECK-NEXT:    retq
+  %gep = getelementptr inbounds [512 x i8], ptr %result, <2 x i64> %idx
+  %res = call <2 x float> @llvm.masked.gather.v2f32.v2p0(<2 x ptr> %gep, i32 0, <2 x i1> %mask, <2 x float> zeroinitializer)
+  ret <2 x float> %res
+}
+
+define <2 x float> @gather_v2f32_scale_16(ptr %result, <2 x i64> %idx, <2 x i1> %mask) {
+; CHECK-LABEL: gather_v2f32_scale_16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpsllq $4, %xmm0, %xmm2
+; CHECK-NEXT:    vpshufd {{.*#+}} xmm0 = xmm1[0,2,2,3]
+; CHECK-NEXT:    vpslld $31, %xmm0, %xmm1
+; CHECK-NEXT:    vpxor %xmm0, %xmm0, %xmm0
+; CHECK-NEXT:    vgatherqps %xmm1, (%rdi,%xmm2), %xmm0
+; CHECK-NEXT:    retq
+  %gep = getelementptr inbounds [16 x i8], ptr %result, <2 x i64> %idx
+  %res = call <2 x float> @llvm.masked.gather.v2f32.v2p0(<2 x ptr> %gep, i32 0, <2 x i1> %mask, <2 x float> zeroinitializer)
+  ret <2 x float> %res
+}
+
+define <2 x float> @gather_v2f32_scale_8(ptr %result, <2 x i64> %idx, <2 x i1> %mask) {
+; CHECK-LABEL: gather_v2f32_scale_8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; CHECK-NEXT:    vpslld $31, %xmm1, %xmm2
+; CHECK-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:    vgatherqps %xmm2, (%rdi,%xmm0,8), %xmm1
+; CHECK-NEXT:    vmovaps %xmm1, %xmm0
+; CHECK-NEXT:    retq
+  %gep = getelementptr inbounds [8 x i8], ptr %result, <2 x i64> %idx
+  %res = call <2 x float> @llvm.masked.gather.v2f32.v2p0(<2 x ptr> %gep, i32 0, <2 x i1> %mask, <2 x float> zeroinitializer)
+  ret <2 x float> %res
+}
+
+define <2 x float> @gather_v2f32_scale_4(ptr %result, <2 x i64> %idx, <2 x i1> %mask) {
+; CHECK-LABEL: gather_v2f32_scale_4:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; CHECK-NEXT:    vpslld $31, %xmm1, %xmm2
+; CHECK-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:    vgatherqps %xmm2, (%rdi,%xmm0,4), %xmm1
+; CHECK-NEXT:    vmovaps %xmm1, %xmm0
+; CHECK-NEXT:    retq
+  %gep = getelementptr inbounds [4 x i8], ptr %result, <2 x i64> %idx
+  %res = call <2 x float> @llvm.masked.gather.v2f32.v2p0(<2 x ptr> %gep, i32 0, <2 x i1> %mask, <2 x float> zeroinitializer)
+  ret <2 x float> %res
+}
+
+define <2 x float> @gather_v2f32_scale_3(ptr %result, <2 x i64> %idx, <2 x i1> %mask) {
+; CHECK-LABEL: gather_v2f32_scale_3:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpaddq %xmm0, %xmm0, %xmm2
+; CHECK-NEXT:    vpaddq %xmm0, %xmm2, %xmm2
+; CHECK-NEXT:    vpshufd {{.*#+}} xmm0 = xmm1[0,2,2,3]
+; CHECK-NEXT:    vpslld $31, %xmm0, %xmm1
+; CHECK-NEXT:    vpxor %xmm0, %xmm0, %xmm0
+; CHECK-NEXT:    vgatherqps %xmm1, (%rdi,%xmm2), %xmm0
+; CHECK-NEXT:    retq
+  %gep = getelementptr inbounds [3 x i8], ptr %result, <2 x i64> %idx
+  %res = call <2 x float> @llvm.masked.gather.v2f32.v2p0(<2 x ptr> %gep, i32 0, <2 x i1> %mask, <2 x float> zeroinitializer)
+  ret <2 x float> %res
+}
+
+define <2 x float> @gather_v2f32_scale_1(ptr %result, <2 x i64> %idx, <2 x i1> %mask) {
+; CHECK-LABEL: gather_v2f32_scale_1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; CHECK-NEXT:    vpslld $31, %xmm1, %xmm2
+; CHECK-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:    vgatherqps %xmm2, (%rdi,%xmm0), %xmm1
+; CHECK-NEXT:    vmovaps %xmm1, %xmm0
+; CHECK-NEXT:    retq
+  %gep = getelementptr inbounds [1 x i8], ptr %result, <2 x i64> %idx
+  %res = call <2 x float> @llvm.masked.gather.v2f32.v2p0(<2 x ptr> %gep, i32 0, <2 x i1> %mask, <2 x float> zeroinitializer)
+  ret <2 x float> %res
+}
+
+declare <2 x float> @llvm.masked.gather.v2f32.v2p0(<2 x ptr>, i32 immarg, <2 x i1>, <2 x float>)

llvm/test/CodeGen/X86/gather-scatter-opaque-ptr.ll

+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=x86_64-linux -mcpu=skylake-avx512 < %s | FileCheck %s
+
+define void @scatter_scale_512(ptr %result, <4 x i64> %idx, <4 x i1> %mask) {
+; CHECK-LABEL: scatter_scale_512:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpslld $31, %xmm1, %xmm1
+; CHECK-NEXT:    vpmovd2m %xmm1, %k1
+; CHECK-NEXT:    vpsllq $9, %ymm0, %ymm0
+; CHECK-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:    vscatterqpd %ymm1, (%rdi,%ymm0) {%k1}
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+  %gep = getelementptr inbounds [512 x i8], ptr %result, <4 x i64> %idx
+  call void @llvm.masked.scatter.v4f64.v4p0(<4 x double> zeroinitializer, <4 x ptr> %gep, i32 0, <4 x i1> %mask)
+  ret void
+}
+
+define void @scatter_scale_16(ptr %result, <4 x i64> %idx, <4 x i1> %mask) {
+; CHECK-LABEL: scatter_scale_16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpslld $31, %xmm1, %xmm1
+; CHECK-NEXT:    vpmovd2m %xmm1, %k1
+; CHECK-NEXT:    vpsllq $4, %ymm0, %ymm0
+; CHECK-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:    vscatterqpd %ymm1, (%rdi,%ymm0) {%k1}
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+  %gep = getelementptr inbounds [16 x i8], ptr %result, <4 x i64> %idx
+  call void @llvm.masked.scatter.v4f64.v4p0(<4 x double> zeroinitializer, <4 x ptr> %gep, i32 0, <4 x i1> %mask)
+  ret void
+}
+
+define void @scatter_scale_8(ptr %result, <4 x i64> %idx, <4 x i1> %mask) {
+; CHECK-LABEL: scatter_scale_8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpslld $31, %xmm1, %xmm1
+; CHECK-NEXT:    vpmovd2m %xmm1, %k1
+; CHECK-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:    vscatterqpd %ymm1, (%rdi,%ymm0,8) {%k1}
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+  %gep = getelementptr inbounds [8 x i8], ptr %result, <4 x i64> %idx
+  call void @llvm.masked.scatter.v4f64.v4p0(<4 x double> zeroinitializer, <4 x ptr> %gep, i32 0, <4 x i1> %mask)
+  ret void
+}
+
+define void @scatter_scale_4(ptr %result, <4 x i64> %idx, <4 x i1> %mask) {
+; CHECK-LABEL: scatter_scale_4:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpslld $31, %xmm1, %xmm1
+; CHECK-NEXT:    vpmovd2m %xmm1, %k1
+; CHECK-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:    vscatterqpd %ymm1, (%rdi,%ymm0,4) {%k1}
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+  %gep = getelementptr inbounds [4 x i8], ptr %result, <4 x i64> %idx
+  call void @llvm.masked.scatter.v4f64.v4p0(<4 x double> zeroinitializer, <4 x ptr> %gep, i32 0, <4 x i1> %mask)
+  ret void
+}
+
+define void @scatter_scale_3(ptr %result, <4 x i64> %idx, <4 x i1> %mask) {
+; CHECK-LABEL: scatter_scale_3:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpslld $31, %xmm1, %xmm1
+; CHECK-NEXT:    vpmovd2m %xmm1, %k1
+; CHECK-NEXT:    vpaddq %ymm0, %ymm0, %ymm1
+; CHECK-NEXT:    vpaddq %ymm0, %ymm1, %ymm0
+; CHECK-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:    vscatterqpd %ymm1, (%rdi,%ymm0) {%k1}
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+  %gep = getelementptr inbounds [3 x i8], ptr %result, <4 x i64> %idx
+  call void @llvm.masked.scatter.v4f64.v4p0(<4 x double> zeroinitializer, <4 x ptr> %gep, i32 0, <4 x i1> %mask)
+  ret void
+}
+
+define void @scatter_scale_1(ptr %result, <4 x i64> %idx, <4 x i1> %mask) {
+; CHECK-LABEL: scatter_scale_1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpslld $31, %xmm1, %xmm1
+; CHECK-NEXT:    vpmovd2m %xmm1, %k1
+; CHECK-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:    vscatterqpd %ymm1, (%rdi,%ymm0) {%k1}
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+  %gep = getelementptr inbounds [1 x i8], ptr %result, <4 x i64> %idx
+  call void @llvm.masked.scatter.v4f64.v4p0(<4 x double> zeroinitializer, <4 x ptr> %gep, i32 0, <4 x i1> %mask)
+  ret void
+}
+
+define <4 x double> @gather_scale_512(ptr %result, <4 x i64> %idx, <4 x i1> %mask) {
+; CHECK-LABEL: gather_scale_512:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpslld $31, %xmm1, %xmm1
+; CHECK-NEXT:    vpmovd2m %xmm1, %k1
+; CHECK-NEXT:    vpsllq $9, %ymm0, %ymm1
+; CHECK-NEXT:    vpxor %xmm0, %xmm0, %xmm0
+; CHECK-NEXT:    vgatherqpd (%rdi,%ymm1), %ymm0 {%k1}
+; CHECK-NEXT:    retq
+  %gep = getelementptr inbounds [512 x i8], ptr %result, <4 x i64> %idx
+  %res = call <4 x double> @llvm.masked.gather.v4f64.v4p0(<4 x ptr> %gep, i32 0, <4 x i1> %mask, <4 x double> zeroinitializer)
+  ret <4 x double> %res
+}
+
+define <4 x double> @gather_scale_16(ptr %result, <4 x i64> %idx, <4 x i1> %mask) {
+; CHECK-LABEL: gather_scale_16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpslld $31, %xmm1, %xmm1
+; CHECK-NEXT:    vpmovd2m %xmm1, %k1
+; CHECK-NEXT:    vpsllq $4, %ymm0, %ymm1
+; CHECK-NEXT:    vpxor %xmm0, %xmm0, %xmm0
+; CHECK-NEXT:    vgatherqpd (%rdi,%ymm1), %ymm0 {%k1}
+; CHECK-NEXT:    retq
+  %gep = getelementptr inbounds [16 x i8], ptr %result, <4 x i64> %idx
+  %res = call <4 x double> @llvm.masked.gather.v4f64.v4p0(<4 x ptr> %gep, i32 0, <4 x i1> %mask, <4 x double> zeroinitializer)
+  ret <4 x double> %res
+}
+
+define <4 x double> @gather_scale_8(ptr %result, <4 x i64> %idx, <4 x i1> %mask) {
+; CHECK-LABEL: gather_scale_8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpslld $31, %xmm1, %xmm1
+; CHECK-NEXT:    vpmovd2m %xmm1, %k1
+; CHECK-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:    vgatherqpd (%rdi,%ymm0,8), %ymm1 {%k1}
+; CHECK-NEXT:    vmovapd %ymm1, %ymm0
+; CHECK-NEXT:    retq
+  %gep = getelementptr inbounds [8 x i8], ptr %result, <4 x i64> %idx
+  %res = call <4 x double> @llvm.masked.gather.v4f64.v4p0(<4 x ptr> %gep, i32 0, <4 x i1> %mask, <4 x double> zeroinitializer)
+  ret <4 x double> %res
+}
+
+define <4 x double> @gather_scale_4(ptr %result, <4 x i64> %idx, <4 x i1> %mask) {
+; CHECK-LABEL: gather_scale_4:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpslld $31, %xmm1, %xmm1
+; CHECK-NEXT:    vpmovd2m %xmm1, %k1
+; CHECK-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:    vgatherqpd (%rdi,%ymm0,4), %ymm1 {%k1}
+; CHECK-NEXT:    vmovapd %ymm1, %ymm0
+; CHECK-NEXT:    retq
+  %gep = getelementptr inbounds [4 x i8], ptr %result, <4 x i64> %idx
+  %res = call <4 x double> @llvm.masked.gather.v4f64.v4p0(<4 x ptr> %gep, i32 0, <4 x i1> %mask, <4 x double> zeroinitializer)
+  ret <4 x double> %res
+}
+
+define <4 x double> @gather_scale_3(ptr %result, <4 x i64> %idx, <4 x i1> %mask) {
+; CHECK-LABEL: gather_scale_3:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpslld $31, %xmm1, %xmm1
+; CHECK-NEXT:    vpmovd2m %xmm1, %k1
+; CHECK-NEXT:    vpaddq %ymm0, %ymm0, %ymm1
+; CHECK-NEXT:    vpaddq %ymm0, %ymm1, %ymm1
+; CHECK-NEXT:    vpxor %xmm0, %xmm0, %xmm0
+; CHECK-NEXT:    vgatherqpd (%rdi,%ymm1), %ymm0 {%k1}
+; CHECK-NEXT:    retq
+  %gep = getelementptr inbounds [3 x i8], ptr %result, <4 x i64> %idx
+  %res = call <4 x double> @llvm.masked.gather.v4f64.v4p0(<4 x ptr> %gep, i32 0, <4 x i1> %mask, <4 x double> zeroinitializer)
+  ret <4 x double> %res
+}
+
+define <4 x double> @gather_scale_1(ptr %result, <4 x i64> %idx, <4 x i1> %mask) {
+; CHECK-LABEL: gather_scale_1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpslld $31, %xmm1, %xmm1
+; CHECK-NEXT:    vpmovd2m %xmm1, %k1
+; CHECK-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:    vgatherqpd (%rdi,%ymm0), %ymm1 {%k1}
+; CHECK-NEXT:    vmovapd %ymm1, %ymm0
+; CHECK-NEXT:    retq
+  %gep = getelementptr inbounds [1 x i8], ptr %result, <4 x i64> %idx
+  %res = call <4 x double> @llvm.masked.gather.v4f64.v4p0(<4 x ptr> %gep, i32 0, <4 x i1> %mask, <4 x double> zeroinitializer)
+  ret <4 x double> %res
+}
+
+declare void @llvm.masked.scatter.v4f64.v4p0(<4 x double>, <4 x ptr>, i32 immarg, <4 x i1>)
+declare <4 x double> @llvm.masked.gather.v4f64.v4p0(<4 x ptr>, i32 immarg, <4 x i1>, <4 x double>)