Changeset View
Changeset View
Standalone View
Standalone View
lib/Target/X86/X86InterleavedAccess.cpp
| Show First 20 Lines • Show All 66 Lines • ▼ Show 20 Lines | class X86InterleavedAccessGroup { | ||||
| /// Out-V1 = p2, q2, r2, s2 | /// Out-V1 = p2, q2, r2, s2 | ||||
| /// Out-V2 = p3, q3, r3, s3 | /// Out-V2 = p3, q3, r3, s3 | ||||
| /// Out-V3 = P4, q4, r4, s4 | /// Out-V3 = P4, q4, r4, s4 | ||||
| void transpose_4x4(ArrayRef<Instruction *> InputVectors, | void transpose_4x4(ArrayRef<Instruction *> InputVectors, | ||||
| SmallVectorImpl<Value *> &TransposedMatrix); | SmallVectorImpl<Value *> &TransposedMatrix); | ||||
| void interleave8bitStride4(ArrayRef<Instruction *> InputVectors, | void interleave8bitStride4(ArrayRef<Instruction *> InputVectors, | ||||
| SmallVectorImpl<Value *> &TransposedMatrix, | SmallVectorImpl<Value *> &TransposedMatrix, | ||||
| unsigned NumSubVecElems); | unsigned NumSubVecElems); | ||||
| void interleave8bitStride4VF8(ArrayRef<Instruction *> InputVectors, | |||||
| SmallVectorImpl<Value *> &TransposedMatrix); | |||||
| void deinterleave8bitStride3(ArrayRef<Instruction *> InputVectors, | void deinterleave8bitStride3(ArrayRef<Instruction *> InputVectors, | ||||
| SmallVectorImpl<Value *> &TransposedMatrix, | SmallVectorImpl<Value *> &TransposedMatrix, | ||||
| unsigned NumSubVecElems); | unsigned NumSubVecElems); | ||||
| public: | public: | ||||
| /// In order to form an interleaved access group X86InterleavedAccessGroup | /// In order to form an interleaved access group X86InterleavedAccessGroup | ||||
| /// requires a wide-load instruction \p 'I', a group of interleaved-vectors | /// requires a wide-load instruction \p 'I', a group of interleaved-vectors | ||||
| /// \p Shuffs, reference to the first indices of each interleaved-vector | /// \p Shuffs, reference to the first indices of each interleaved-vector | ||||
| Show All 39 Lines | if (isa<LoadInst>(Inst)) { | ||||
| WideInstSize = DL.getTypeSizeInBits(Shuffles[0]->getType()); | WideInstSize = DL.getTypeSizeInBits(Shuffles[0]->getType()); | ||||
| // We support shuffle represents stride 4 for byte type with size of | // We support shuffle represents stride 4 for byte type with size of | ||||
| // WideInstSize. | // WideInstSize. | ||||
| if (ShuffleElemSize == 64 && WideInstSize == 1024 && Factor == 4) | if (ShuffleElemSize == 64 && WideInstSize == 1024 && Factor == 4) | ||||
| return true; | return true; | ||||
| if (ShuffleElemSize == 8 && isa<StoreInst>(Inst) && Factor == 4 && | if (ShuffleElemSize == 8 && isa<StoreInst>(Inst) && Factor == 4 && | ||||
| (WideInstSize == 512 || WideInstSize == 1024)) | (WideInstSize == 256 || WideInstSize == 512 || WideInstSize == 1024)) | ||||
| return true; | return true; | ||||
| if (ShuffleElemSize == 8 && isa<LoadInst>(Inst) && Factor == 3 && | if (ShuffleElemSize == 8 && isa<LoadInst>(Inst) && Factor == 3 && | ||||
| (WideInstSize == 384 || WideInstSize == 768)) | (WideInstSize == 384 || WideInstSize == 768)) | ||||
Farhana: Please remove the extra space after "&&".
| |||||
Not Done ReplyInline ActionsDo we also need to check DL.getTypeSizeInBits(ShuffleVecTy) == 256 here? Farhana: Do we also need to check DL.getTypeSizeInBits(ShuffleVecTy) == 256 here? | |||||
Not Done ReplyInline ActionsYes since in default mode, the compose function creates 4 xmm where only half of the wide is been used. I want that high bits will be left undef so I am checking 4*xmm = 256 in the next step will fill the undef part. m_zuckerman: Yes since in default mode, the compose function creates 4 xmm where only half of the wide is… | |||||
| return true; | return true; | ||||
| return false; | return false; | ||||
| } | } | ||||
| void X86InterleavedAccessGroup::decompose( | void X86InterleavedAccessGroup::decompose( | ||||
| Instruction *VecInst, unsigned NumSubVectors, VectorType *SubVecTy, | Instruction *VecInst, unsigned NumSubVectors, VectorType *SubVecTy, | ||||
| SmallVectorImpl<Instruction *> &DecomposedVectors) { | SmallVectorImpl<Instruction *> &DecomposedVectors) { | ||||
| ▲ Show 20 Lines • Show All 62 Lines • ▼ Show 20 Lines | static void createConcatShuffleMask(int NumElements, | ||||
| int NumHalfElements = NumElements / 2; | int NumHalfElements = NumElements / 2; | ||||
| int Offset = Low ? 0 : NumHalfElements; | int Offset = Low ? 0 : NumHalfElements; | ||||
| for (int i = 0; i < NumHalfElements; ++i) | for (int i = 0; i < NumHalfElements; ++i) | ||||
| Mask.push_back(i + Offset); | Mask.push_back(i + Offset); | ||||
| for (int i = 0; i < NumHalfElements; ++i) | for (int i = 0; i < NumHalfElements; ++i) | ||||
| Mask.push_back(i + Offset + NumElements); | Mask.push_back(i + Offset + NumElements); | ||||
| } | } | ||||
| // Changing the scale of the vector type by reducing the number of elements and | // Changing the scale of the vector type by reducing the number of elements and | ||||
createLowUnpackMask RKSimon: createLowUnpackMask | |||||
Can you please add some comments about what this function does? Farhana: Can you please add some comments about what this function does? | |||||
| // doubling the scalar size. | // doubling the scalar size. | ||||
| static MVT scaleVectorType(MVT VT) { | static MVT scaleVectorType(MVT VT) { | ||||
Please remove the unknown character. Farhana: Please remove the unknown character. | |||||
| unsigned ScalarSize = VT.getVectorElementType().getScalarSizeInBits() * 2; | unsigned ScalarSize = VT.getVectorElementType().getScalarSizeInBits() * 2; | ||||
| return MVT::getVectorVT(MVT::getIntegerVT(ScalarSize), | return MVT::getVectorVT(MVT::getIntegerVT(ScalarSize), | ||||
| VT.getVectorNumElements() / 2); | VT.getVectorNumElements() / 2); | ||||
'unsigned' looks like a more suitable type, That will also allow 'i' in the for loop below to be of 'unsigned' type, and that is better because it is being pushed into Mask, a container of type 'uint32_t'. zvi: 'unsigned' looks like a more suitable type, That will also allow 'i' in the for loop below to… | |||||
| } | } | ||||
Not Done ReplyInline ActionsIs there anything special about this unpackMask? Why do we need a separate unpackmask generator? Why aren't you extending the createUnpackShuffleMask()? If createLowUnpackMask is unpacking in a different way than createUnpackShuffleMask() that distinction should be documented and clearly noted why we need a separate unpackMask-creator. Farhana: Is there anything special about this unpackMask? Why do we need a separate unpackmask generator? | |||||
Not Done ReplyInline ActionsThe regular unpack function doesn't create * 2XN * as we need in this case and creates only half from one vector and another half from the second vector. In mine function, we use all the vector and the not only half. For the backend, the result is the same but the way they create is different. m_zuckerman: The regular unpack function doesn't create * 2XN * as we need in this case and creates only… | |||||
| void X86InterleavedAccessGroup::interleave8bitStride4VF8( | |||||
| ArrayRef<Instruction *> Matrix, | |||||
| SmallVectorImpl<Value *> &TransposedMatrix) { | |||||
| // Assuming we start from the following vectors: | |||||
| // Matrix[0]= c0 c1 c2 c3 c4 ... c7 | |||||
| // Matrix[1]= m0 m1 m2 m3 m4 ... m7 | |||||
| // Matrix[2]= y0 y1 y2 y3 y4 ... y7 | |||||
| // Matrix[3]= k0 k1 k2 k3 k4 ... k7 | |||||
The comment doesn't address the special case v8i8. Can you please explain why the exception is needed? zvi: The comment doesn't address the special case v8i8. Can you please explain why the exception is… | |||||
| MVT VT = MVT::v8i16; | |||||
*scaleVectorType zvi: *scaleVectorType | |||||
| TransposedMatrix.resize(2); | |||||
| SmallVector<uint32_t, 16> MaskLow; | |||||
| SmallVector<uint32_t, 32> MaskLowTemp1, MaskLowWord; | |||||
| SmallVector<uint32_t, 32> MaskHighTemp1, MaskHighWord; | |||||
| for (unsigned i = 0; i < 8; ++i) { | |||||
| MaskLow.push_back(i); | |||||
| MaskLow.push_back(i + 8); | |||||
| } | |||||
| createUnpackShuffleMask<uint32_t>(VT, MaskLowTemp1, true, false); | |||||
| createUnpackShuffleMask<uint32_t>(VT, MaskHighTemp1, false, false); | |||||
| scaleShuffleMask<uint32_t>(2, MaskHighTemp1, MaskHighWord); | |||||
| scaleShuffleMask<uint32_t>(2, MaskLowTemp1, MaskLowWord); | |||||
| // IntrVec1Low = c0 m0 c1 m1 c2 m2 c3 m3 c4 m4 c5 m5 c6 m6 c7 m7 | |||||
| // IntrVec2Low = y0 k0 y1 k1 y2 k2 y3 k3 y4 k4 y5 k5 y6 k6 y7 k7 | |||||
| Value *IntrVec1Low = | |||||
| Builder.CreateShuffleVector(Matrix[0], Matrix[1], MaskLow); | |||||
| Value *IntrVec2Low = | |||||
| Builder.CreateShuffleVector(Matrix[2], Matrix[3], MaskLow); | |||||
| // TransposedMatrix[0] = c0 m0 y0 k0 c1 m1 y1 k1 c2 m2 y2 k2 c3 m3 y3 k3 | |||||
| // TransposedMatrix[1] = c4 m4 y4 k4 c5 m5 y5 k5 c6 m6 y6 k6 c7 m7 y7 k7 | |||||
| TransposedMatrix[0] = | |||||
| Builder.CreateShuffleVector(IntrVec1Low, IntrVec2Low, MaskLowWord); | |||||
| TransposedMatrix[1] = | |||||
| Builder.CreateShuffleVector(IntrVec1Low, IntrVec2Low, MaskHighWord); | |||||
| } | |||||
| void X86InterleavedAccessGroup::interleave8bitStride4( | void X86InterleavedAccessGroup::interleave8bitStride4( | ||||
| ArrayRef<Instruction *> Matrix, SmallVectorImpl<Value *> &TransposedMatrix, | ArrayRef<Instruction *> Matrix, SmallVectorImpl<Value *> &TransposedMatrix, | ||||
| unsigned numberOfElement) { | unsigned numberOfElement) { | ||||
| // Example: Assuming we start from the following vectors: | // Example: Assuming we start from the following vectors: | ||||
| // Matrix[0]= c0 c1 c2 c3 c4 ... c31 | // Matrix[0]= c0 c1 c2 c3 c4 ... c31 | ||||
| // Matrix[1]= m0 m1 m2 m3 m4 ... m31 | // Matrix[1]= m0 m1 m2 m3 m4 ... m31 | ||||
| // Matrix[2]= y0 y1 y2 y3 y4 ... y31 | // Matrix[2]= y0 y1 y2 y3 y4 ... y31 | ||||
| // Matrix[3]= k0 k1 k2 k3 k4 ... k31 | // Matrix[3]= k0 k1 k2 k3 k4 ... k31 | ||||
| MVT VT = MVT::getVectorVT(MVT::i8, numberOfElement); | MVT VT = MVT::getVectorVT(MVT::i8, numberOfElement); | ||||
| MVT HalfVT = scaleVectorType(VT); | MVT HalfVT = scaleVectorType(VT); | ||||
Not Done ReplyInline ActionsCan you please commit these minor changes: leaner computation of VT and Half, variable renaming in a couple of lines below as an NFC commit and rebase this patch? zvi: Can you please commit these minor changes: leaner computation of VT and Half, variable renaming… | |||||
| TransposedMatrix.resize(4); | TransposedMatrix.resize(4); | ||||
| SmallVector<uint32_t, 32> MaskHigh; | SmallVector<uint32_t, 32> MaskHigh; | ||||
| SmallVector<uint32_t, 32> MaskLow; | SmallVector<uint32_t, 32> MaskLow; | ||||
| SmallVector<uint32_t, 32> MaskHighTemp1; | SmallVector<uint32_t, 32> MaskHighTemp1; | ||||
| SmallVector<uint32_t, 32> MaskLowTemp1; | SmallVector<uint32_t, 32> MaskLowTemp1; | ||||
| SmallVector<uint32_t, 32> MaskHighWord; | SmallVector<uint32_t, 32> MaskHighWord; | ||||
| SmallVector<uint32_t, 32> MaskLowWord; | SmallVector<uint32_t, 32> MaskLowWord; | ||||
| Show All 22 Lines | void X86InterleavedAccessGroup::interleave8bitStride4( | ||||
| Value *IntrVec1Low = | Value *IntrVec1Low = | ||||
| Builder.CreateShuffleVector(Matrix[0], Matrix[1], MaskLow); | Builder.CreateShuffleVector(Matrix[0], Matrix[1], MaskLow); | ||||
| Value *IntrVec1High = | Value *IntrVec1High = | ||||
| Builder.CreateShuffleVector(Matrix[0], Matrix[1], MaskHigh); | Builder.CreateShuffleVector(Matrix[0], Matrix[1], MaskHigh); | ||||
| Value *IntrVec2Low = | Value *IntrVec2Low = | ||||
| Builder.CreateShuffleVector(Matrix[2], Matrix[3], MaskLow); | Builder.CreateShuffleVector(Matrix[2], Matrix[3], MaskLow); | ||||
| Value *IntrVec2High = | Value *IntrVec2High = | ||||
| Builder.CreateShuffleVector(Matrix[2], Matrix[3], MaskHigh); | Builder.CreateShuffleVector(Matrix[2], Matrix[3], MaskHigh); | ||||
Not Done ReplyInline ActionsI think it would be better to create a function that handles the v8i8 case instead of having these if's + early exit. It will also allow reomving the if from scaleVectorType to the point where it may not be needed. zvi: I think it would be better to create a function that handles the v8i8 case instead of having… | |||||
| // cmyk4 cmyk5 cmyk6 cmyk7 | cmyk20 cmyk21 cmyk22 cmyk23 | // cmyk4 cmyk5 cmyk6 cmyk7 | cmyk20 cmyk21 cmyk22 cmyk23 | ||||
| // cmyk12 cmyk13 cmyk14 cmyk15 | cmyk28 cmyk29 cmyk30 cmyk31 | // cmyk12 cmyk13 cmyk14 cmyk15 | cmyk28 cmyk29 cmyk30 cmyk31 | ||||
| // cmyk0 cmyk1 cmyk2 cmyk3 | cmyk16 cmyk17 cmyk18 cmyk19 | // cmyk0 cmyk1 cmyk2 cmyk3 | cmyk16 cmyk17 cmyk18 cmyk19 | ||||
| // cmyk8 cmyk9 cmyk10 cmyk11 | cmyk24 cmyk25 cmyk26 cmyk27 | // cmyk8 cmyk9 cmyk10 cmyk11 | cmyk24 cmyk25 cmyk26 cmyk27 | ||||
| Value *High = | Value *High = | ||||
| Builder.CreateShuffleVector(IntrVec1Low, IntrVec2Low, MaskHighWord); | Builder.CreateShuffleVector(IntrVec1Low, IntrVec2Low, MaskHighWord); | ||||
| Value *High1 = | Value *High1 = | ||||
| ▲ Show 20 Lines • Show All 253 Lines • ▼ Show 20 Lines | decompose(Shuffles[0], Factor, VectorType::get(ShuffleEltTy, NumSubVecElems), | ||||
| DecomposedVectors); | DecomposedVectors); | ||||
| // 2. Transpose the interleaved-vectors into vectors of contiguous | // 2. Transpose the interleaved-vectors into vectors of contiguous | ||||
| // elements. | // elements. | ||||
| switch (NumSubVecElems) { | switch (NumSubVecElems) { | ||||
| case 4: | case 4: | ||||
| transpose_4x4(DecomposedVectors, TransposedVectors); | transpose_4x4(DecomposedVectors, TransposedVectors); | ||||
| break; | break; | ||||
| case 8: | |||||
| interleave8bitStride4VF8(DecomposedVectors, TransposedVectors); | |||||
| break; | |||||
| case 16: | case 16: | ||||
| case 32: | case 32: | ||||
| interleave8bitStride4(DecomposedVectors, TransposedVectors, NumSubVecElems); | interleave8bitStride4(DecomposedVectors, TransposedVectors, NumSubVecElems); | ||||
| break; | break; | ||||
| default: | default: | ||||
| return false; | return false; | ||||
| } | } | ||||
| ▲ Show 20 Lines • Show All 58 Lines • Show Last 20 Lines | |||||
Please remove the extra space after "&&".