Index: docs/LangRef.rst =================================================================== --- docs/LangRef.rst +++ docs/LangRef.rst @@ -6096,7 +6096,8 @@ The '``getelementptr``' instruction is used to get the address of a subelement of an :ref:`aggregate ` data structure. It performs -address calculation only and does not access memory. +address calculation only and does not access memory. The instruction can also +be used to calculate a vector of such addresses. Arguments: """""""""" @@ -6222,12 +6223,61 @@ ; yields i32*:iptr %iptr = getelementptr [10 x i32], [10 x i32]* @arr, i16 0, i16 0 -In cases where the pointer argument is a vector of pointers, each index -must be a vector with the same number of elements. For example: +Vector of pointers: +""""""""""""""""""" + +The ``getelementptr`` returns a vector of pointers, instead of a single address, +when one or more of its arguments is a vector. In such cases, all vector +arguments should have the same number of elements, and every scalar argument +will be effectively broadcasted into a vector during address calculation. + +.. code-block:: llvm + + ; All arguments are vectors: + ; A[i] = ptrs[i] + offsets[i]*sizeof(i8) + %A = getelementptr i8, <4 x i8*> %ptrs, <4 x i64> %offsets + + ; Add the same scalar offset to each pointer of a vector: + ; A[i] = ptrs[i] + offset*sizeof(i8) + %A = getelementptr i8, <4 x i8*> %ptrs, i64 %offset + + ; Add distinct offsets to the same pointer: + ; A[i] = ptr + offsets[i]*sizeof(i8) + %A = getelementptr i8, i8* %ptr, <4 x i64> %offsets + + ; In all cases described above the type of the result is <4 x i8*> + +The two following instructions are equivalent: + +.. code-block:: llvm + + getelementptr %struct.ST, <4 x %struct.ST*> %s, <4 x i64> %ind1, + <4 x i32> , + <4 x i32> , + <4 x i32> %ind4, + <4 x i64> + + getelementptr %struct.ST, <4 x %struct.ST*> %s, <4 x i64> %ind1, + i32 2, i32 1, <4 x i32> %ind4, i64 13 + +Let's look at the C code, where the vector version of ``getelementptr`` +makes sense: + +.. code-block:: c + + // Let's assume that we vectorize the following loop: + double *A, B; int *C; + for (int i = 0; i < size; ++i) { + A[i] = B[C[i]]; + } .. code-block:: llvm - %A = getelementptr i8, <4 x i8*> %ptrs, <4 x i64> %offsets, + ; get pointers for 8 elements from array B + %ptrs = getelementptr double, double* %B, <8 x i32> %C + ; load 8 elements from array B into A + %A = call <8 x double> @llvm.masked.gather.v8f64(<8 x double*> %ptrs, + i32 8, <8 x i1> %mask, <8 x double> %passthru) Conversion Operations --------------------- Index: include/llvm/IR/Instructions.h =================================================================== --- include/llvm/IR/Instructions.h +++ include/llvm/IR/Instructions.h @@ -837,10 +837,14 @@ if (!PointeeType) PointeeType = cast(Ptr->getType()->getScalarType())->getElementType(); + else if (PointeeType->isVectorTy() == Ptr->getType()->isVectorTy()) + assert( + PointeeType == cast + (Ptr->getType()->getScalarType())->getElementType()->getScalarType()); else assert( - PointeeType == - cast(Ptr->getType()->getScalarType())->getElementType()); + PointeeType->getScalarType() == cast + (Ptr->getType()->getScalarType())->getElementType()->getScalarType()); return new (Values) GetElementPtrInst(PointeeType, Ptr, IdxList, Values, NameStr, InsertBefore); } @@ -852,10 +856,12 @@ if (!PointeeType) PointeeType = cast(Ptr->getType()->getScalarType())->getElementType(); + else if (PointeeType->isVectorTy() == Ptr->getType()->isVectorTy()) + assert(PointeeType == cast + (Ptr->getType()->getScalarType())->getElementType()->getScalarType()); else - assert( - PointeeType == - cast(Ptr->getType()->getScalarType())->getElementType()); + assert(PointeeType->getScalarType() == cast + (Ptr->getType()->getScalarType())->getElementType()->getScalarType()); return new (Values) GetElementPtrInst(PointeeType, Ptr, IdxList, Values, NameStr, InsertAtEnd); } @@ -971,7 +977,11 @@ unsigned NumElem = cast(Ptr->getType())->getNumElements(); return VectorType::get(PtrTy, NumElem); } - + for (Value *Index : IdxList) + if (Index->getType()->isVectorTy()) { + unsigned NumElts = cast(Index->getType())->getNumElements(); + return VectorType::get(PtrTy, NumElts); + } // Scalar GEP return PtrTy; } Index: lib/AsmParser/LLParser.cpp =================================================================== --- lib/AsmParser/LLParser.cpp +++ lib/AsmParser/LLParser.cpp @@ -5541,6 +5541,11 @@ SmallVector Indices; bool AteExtraComma = false; + // GEP returns a vector of pointers if at least one of parameters is a vector. + // All vector parameters should have the same vector width. + unsigned GepWidth = BaseType->isVectorTy() ? + cast(BaseType)->getNumElements() : 0; + while (EatIfPresent(lltok::comma)) { if (Lex.getKind() == lltok::MetadataVar) { AteExtraComma = true; @@ -5549,14 +5554,13 @@ if (ParseTypeAndValue(Val, EltLoc, PFS)) return true; if (!Val->getType()->getScalarType()->isIntegerTy()) return Error(EltLoc, "getelementptr index must be an integer"); - if (Val->getType()->isVectorTy() != Ptr->getType()->isVectorTy()) - return Error(EltLoc, "getelementptr index type missmatch"); + if (Val->getType()->isVectorTy()) { unsigned ValNumEl = cast(Val->getType())->getNumElements(); - unsigned PtrNumEl = cast(Ptr->getType())->getNumElements(); - if (ValNumEl != PtrNumEl) + if (GepWidth && GepWidth != ValNumEl) return Error(EltLoc, "getelementptr vector index has a wrong number of elements"); + GepWidth = ValNumEl; } Indices.push_back(Val); } Index: lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp =================================================================== --- lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -2760,6 +2760,16 @@ SDValue N = getValue(Op0); SDLoc dl = getCurSDLoc(); + // Normalize Vector GEP - all scalar operands should be converted to the + // splat vector + unsigned VectorWidth = + I.getType()->isVectorTy()? cast(I.getType())->getVectorNumElements() : 0; + + if (VectorWidth && !N.getValueType().isVector()) { + MVT VT = MVT::getVectorVT(N.getValueType().getSimpleVT(), VectorWidth); + SmallVector Ops(VectorWidth, N); + N = DAG.getNode(ISD::BUILD_VECTOR, dl, VT, Ops); + } for (GetElementPtrInst::const_op_iterator OI = I.op_begin()+1, E = I.op_end(); OI != E; ++OI) { const Value *Idx = *OI; @@ -2779,12 +2789,20 @@ unsigned PtrSize = PtrTy.getSizeInBits(); APInt ElementSize(PtrSize, DL->getTypeAllocSize(Ty)); - // If this is a constant subscript, handle it quickly. - if (const auto *CI = dyn_cast(Idx)) { + // If this is a scalar constant or a splat vector of constants, + // handle it quickly. + const auto *CI = dyn_cast(Idx); + if (!CI && isa(Idx) && + cast(Idx)->getSplatValue()) + CI = cast(cast(Idx)->getSplatValue()); + + if (CI) { if (CI->isZero()) continue; APInt Offs = ElementSize * CI->getValue().sextOrTrunc(PtrSize); - SDValue OffsVal = DAG.getConstant(Offs, dl, PtrTy); + SDValue OffsVal = VectorWidth ? + DAG.getConstant(Offs, dl, MVT::getVectorVT(PtrTy, VectorWidth)) : + DAG.getConstant(Offs, dl, PtrTy); N = DAG.getNode(ISD::ADD, dl, N.getValueType(), N, OffsVal); continue; } @@ -2792,6 +2810,11 @@ // N = N + Idx * ElementSize; SDValue IdxN = getValue(Idx); + if (!IdxN.getValueType().isVector() && VectorWidth) { + MVT VT = MVT::getVectorVT(IdxN.getValueType().getSimpleVT(), VectorWidth); + SmallVector Ops(VectorWidth, IdxN); + IdxN = DAG.getNode(ISD::BUILD_VECTOR, dl, VT, Ops); + } // If the index is smaller or larger than intptr_t, truncate or extend // it. IdxN = DAG.getSExtOrTrunc(IdxN, dl, N.getValueType()); Index: lib/IR/Verifier.cpp =================================================================== --- lib/IR/Verifier.cpp +++ lib/IR/Verifier.cpp @@ -2540,10 +2540,6 @@ Assert(isa(TargetTy), "GEP base pointer is not a vector or a vector of pointers", &GEP); Assert(GEP.getSourceElementType()->isSized(), "GEP into unsized type!", &GEP); - Assert(GEP.getPointerOperandType()->isVectorTy() == - GEP.getType()->isVectorTy(), - "Vector GEP must return a vector value", &GEP); - SmallVector Idxs(GEP.idx_begin(), GEP.idx_end()); Type *ElTy = GetElementPtrInst::getIndexedType(GEP.getSourceElementType(), Idxs); @@ -2553,17 +2549,20 @@ GEP.getResultElementType() == ElTy, "GEP is not of right type for indices!", &GEP, ElTy); - if (GEP.getPointerOperandType()->isVectorTy()) { + if (GEP.getType()->isVectorTy()) { // Additional checks for vector GEPs. - unsigned GepWidth = GEP.getPointerOperandType()->getVectorNumElements(); - Assert(GepWidth == GEP.getType()->getVectorNumElements(), - "Vector GEP result width doesn't match operand's", &GEP); + unsigned GepWidth = cast(GEP.getType())->getVectorNumElements(); + if (GEP.getPointerOperandType()->isVectorTy()) + Assert(GepWidth == GEP.getPointerOperandType()->getVectorNumElements(), + "Vector GEP result width doesn't match operand's", &GEP); for (unsigned i = 0, e = Idxs.size(); i != e; ++i) { Type *IndexTy = Idxs[i]->getType(); - Assert(IndexTy->isVectorTy(), "Vector GEP must have vector indices!", - &GEP); - unsigned IndexWidth = IndexTy->getVectorNumElements(); - Assert(IndexWidth == GepWidth, "Invalid GEP index vector width", &GEP); + if (IndexTy->isVectorTy()) { + unsigned IndexWidth = cast(IndexTy)->getVectorNumElements(); + Assert(IndexWidth == GepWidth, "Invalid GEP index vector width", &GEP); + } + Assert(IndexTy->getScalarType()->isIntegerTy(), + "All GEP indices should be of integer type"); } } visitInstruction(GEP); Index: test/Assembler/getelementptr_vec_idx1.ll =================================================================== --- test/Assembler/getelementptr_vec_idx1.ll +++ test/Assembler/getelementptr_vec_idx1.ll @@ -1,8 +1,8 @@ ; RUN: not llvm-as < %s >/dev/null 2> %t ; RUN: FileCheck %s < %t -; Test that a vector index is only used with a vector pointer. +; Test that a vector GEP may be used with a scalar base, the result is a vector of pointers -; CHECK: getelementptr index type missmatch +; CHECK: '%w' defined with type '<2 x i32*> define i32 @test(i32* %a) { %w = getelementptr i32, i32* %a, <2 x i32> Index: test/Assembler/getelementptr_vec_idx2.ll =================================================================== --- test/Assembler/getelementptr_vec_idx2.ll +++ test/Assembler/getelementptr_vec_idx2.ll @@ -1,10 +1,24 @@ ; RUN: not llvm-as < %s >/dev/null 2> %t ; RUN: FileCheck %s < %t -; Test that a vector pointer is only used with a vector index. +; Test that a vector pointer may be used with a scalar index. +; Test that a vector pointer and vector index should have the same vector width -; CHECK: getelementptr index type missmatch - -define <2 x i32> @test(<2 x i32*> %a) { +; This code is correct +define <2 x i32*> @test2(<2 x i32*> %a) { %w = getelementptr i32, <2 x i32*> %a, i32 2 + ret <2 x i32*> %w +} + +; This code is correct +define <2 x i32*> @test3(i32* %a) { + %w = getelementptr i32, i32* %a, <2 x i32> + ret <2 x i32*> %w +} + +; CHECK: getelementptr vector index has a wrong number of elements + +define <2 x i32> @test1(<2 x i32*> %a) { + %w = getelementptr i32, <2 x i32*> %a, <4 x i32> ret <2 x i32> %w } + Index: test/Assembler/getelementptr_vec_idx3.ll =================================================================== --- test/Assembler/getelementptr_vec_idx3.ll +++ test/Assembler/getelementptr_vec_idx3.ll @@ -1,8 +1,8 @@ ; RUN: not llvm-as < %s >/dev/null 2> %t ; RUN: FileCheck %s < %t -; Test that vector indices have the same number of elements as the pointer. +; Test that a vector GEP may be used with a scalar base, the result is a vector of pointers -; CHECK: getelementptr index type missmatch +; CHECK: '%w' defined with type '<2 x <4 x i32>*>' define <4 x i32> @test(<4 x i32>* %a) { %w = getelementptr <4 x i32>, <4 x i32>* %a, <2 x i32> Index: test/CodeGen/X86/vector-gep.ll =================================================================== --- test/CodeGen/X86/vector-gep.ll +++ test/CodeGen/X86/vector-gep.ll @@ -92,3 +92,36 @@ ;CHECK: ret } +;CHECK-LABEL: AGEP7: +define <4 x i8*> @AGEP7(<4 x i8*> %param, i32 %off) nounwind { +entry: +;CHECK-LABEL: AGEP7 +;CHECK-NOT: pslld + %A = getelementptr i8, <4 x i8*> %param, i32 %off + ret <4 x i8*> %A +;CHECK: ret +} + +;CHECK-LABEL: AGEP8: +define <4 x i8*> @AGEP8(<4 x i8*> %param, i8 %off) nounwind { +entry: +;CHECK-LABEL: AGEP8 +;CHECK: vpaddd + %A = getelementptr i8, <4 x i8*> %param, i8 %off + ret <4 x i8*> %A +;CHECK: ret +} + +;CHECK-LABEL: AGEP9: +define <4 x i16*> @AGEP9(i16* %param, <4 x i32> %off) nounwind { +entry: +;CHECK-LABEL: AGEP9 +; Multiply offset by two (add it to itself). +;CHECK: vpadd +; add the base to the offset +;CHECK: vbroadcastss +;CHECK-NEXT: vpadd + %A = getelementptr i16, i16* %param, <4 x i32> %off + ret <4 x i16*> %A +;CHECK: ret +}