Index: lib/CodeGen/InterleavedAccessPass.cpp =================================================================== --- lib/CodeGen/InterleavedAccessPass.cpp +++ lib/CodeGen/InterleavedAccessPass.cpp @@ -156,12 +156,17 @@ return false; } -/// \brief Check if the mask is RE-interleave mask for an interleaved store. +/// \brief Check if the mask can be used in an interleaved store. +// +/// It checks for a more general pattern than the RE-interleave mask. +/// I.e. +/// E.g. For a Factor of 2 (NumSubElts=4): <4, 32, 5, 33, 6, 34, 7, 35> +/// E.g. For a Factor of 3 (NumSubElts=4): <4, 32, 16, 5, 33, 17, 6, 34, 18, 7, 35, 19> +/// E.g. For a Factor of 4 (NumSubElts=2): <8, 2, 12, 4, 9, 3, 13, 5> /// +/// The particular case of an RE-interleave mask is: /// I.e. <0, NumSubElts, ... , NumSubElts*(Factor - 1), 1, NumSubElts + 1, ...> -/// -/// E.g. The RE-interleave mask (Factor = 2) could be: -/// <0, 4, 1, 5, 2, 6, 3, 7> +/// E.g. For a Factor of 2 (NumSubElts=4): <0, 4, 1, 5, 2, 6, 3, 7> static bool isReInterleaveMask(ArrayRef Mask, unsigned &Factor) { unsigned NumElts = Mask.size(); if (NumElts < 4) @@ -176,17 +181,56 @@ if (!isPowerOf2_32(NumSubElts)) continue; - // Check whether each element matchs the RE-interleaved rule. Ignore undef - // elements. - unsigned i = 0; - for (; i < NumElts; i++) - if (Mask[i] >= 0 && - static_cast(Mask[i]) != - (i % Factor) * NumSubElts + i / Factor) + // Check whether each element matches the general interleaved rule. + // Ignore undef elements, as long as the defined elements match the rule. + // Outer loop processes all factors (x, y, z in the above example) + unsigned i = 0, j; + for (; i < Factor; i++) { + int PreviousMask = -1; + int PreviousPos = -1; + //Inner loop processes all consecutive accesses (x, x+1... in the example) + for (j = 0; j < NumSubElts-1; j++) { + unsigned ij = j*Factor + i; + if (Mask[ij] >= 0 && Mask[ij + Factor] >= 0 && + static_cast(Mask[ij]) + 1 != + static_cast(Mask[ij + Factor])) + break; + + // Undefs are allowed, but the defined elements must still be consecutive: + // i.e.: x,..., undef,..., x + 2,..., undef,..., undef,..., x + 5, .... + // Verify this by storing the last non-undef followed by an undef + // Check that following non-undef masks are incremented with the + // corresponding distance. + if (PreviousMask > 0 && Mask[ij] < 0 && Mask[ij + Factor] >= 0 && + static_cast(PreviousMask) + (j - PreviousPos) + 1 != + static_cast(Mask[ij + Factor])) + break; + if (Mask[ij] >= 0 && Mask[ij + Factor] < 0) { + PreviousMask = Mask[ij]; + PreviousPos = j; + } + } + if (j < NumSubElts-1) break; - // Find a RE-interleaved mask of current factor. - if (i == NumElts) + //Check the start of the i range (j=0) is >0 + int StartMask = 0; + if (Mask[i] > 0) + StartMask = Mask[i]; + else { + for (unsigned j = 1; j < NumSubElts; j++) { + if (Mask[j*Factor + i] >= 0) { + StartMask = Mask[j*Factor + i] - j; + break; + } + } + if (StartMask < 0) + break; + } + } + + // Find a interleaved mask of current factor. + if (i == Factor) return true; } Index: lib/Target/AArch64/AArch64ISelLowering.cpp =================================================================== --- lib/Target/AArch64/AArch64ISelLowering.cpp +++ lib/Target/AArch64/AArch64ISelLowering.cpp @@ -7198,7 +7198,7 @@ /// /// E.g. Lower an interleaved store (Factor = 3): /// %i.vec = shuffle <8 x i32> %v0, <8 x i32> %v1, -/// <0, 4, 8, 1, 5, 9, 2, 6, 10, 3, 7, 11> +/// <0, 4, 8, 1, 5, 9, 2, 6, 10, 3, 7, 11> /// store <12 x i32> %i.vec, <12 x i32>* %ptr /// /// Into: @@ -7209,6 +7209,17 @@ /// /// Note that the new shufflevectors will be removed and we'll only generate one /// st3 instruction in CodeGen. +/// +/// Example for a more general valid mask (Factor 3). Lower: +/// %i.vec = shuffle <32 x i32> %v0, <32 x i32> %v1, +/// <4, 32, 16, 5, 33, 17, 6, 34, 18, 7, 35, 19> +/// store <12 x i32> %i.vec, <12 x i32>* %ptr +/// +/// Into: +/// %sub.v0 = shuffle <32 x i32> %v0, <32 x i32> v1, <4, 5, 6, 7> +/// %sub.v1 = shuffle <32 x i32> %v0, <32 x i32> v1, <32, 33, 34, 35> +/// %sub.v2 = shuffle <32 x i32> %v0, <32 x i32> v1, <16, 17, 18, 19> +/// call void llvm.aarch64.neon.st3(%sub.v0, %sub.v1, %sub.v2, %ptr) bool AArch64TargetLowering::lowerInterleavedStore(StoreInst *SI, ShuffleVectorInst *SVI, unsigned Factor) const { @@ -7260,9 +7271,28 @@ SmallVector Ops; // Split the shufflevector operands into sub vectors for the new stN call. - for (unsigned i = 0; i < Factor; i++) - Ops.push_back(Builder.CreateShuffleVector( - Op0, Op1, getSequentialMask(Builder, NumSubElts * i, NumSubElts))); + ArrayRef Mask = SVI->getShuffleMask(); + for (unsigned i = 0; i < Factor; i++) { + if (Mask[i] >= 0) + Ops.push_back(Builder.CreateShuffleVector( + Op0, Op1, getSequentialMask(Builder, Mask[i], NumSubElts))); + else { + unsigned StartMask = 0; + for (unsigned j = 1; j < NumSubElts; j++) { + if (Mask[j*Factor + i] >= 0) { + StartMask = Mask[j*Factor + i] - j; + break; + } + } + // Note: If all elements in a chunk are undefs, StartMask=0! + // Note: Filling undef gaps with random elements is ok, since + // those elements were being written anyway (with undefs). + // In the case of all undefs we're defaulting to using elems from 0 + // Note: StartMask cannot be negative, it's checked in isReInterleaveMask + Ops.push_back(Builder.CreateShuffleVector( + Op0, Op1, getSequentialMask(Builder, StartMask, NumSubElts))); + } + } Ops.push_back(Builder.CreateBitCast(SI->getPointerOperand(), PtrTy)); Builder.CreateCall(StNFunc, Ops); Index: lib/Target/ARM/ARMISelLowering.cpp =================================================================== --- lib/Target/ARM/ARMISelLowering.cpp +++ lib/Target/ARM/ARMISelLowering.cpp @@ -12986,6 +12986,17 @@ /// /// Note that the new shufflevectors will be removed and we'll only generate one /// vst3 instruction in CodeGen. +/// +/// Example for a more general valid mask (Factor 3). Lower: +/// %i.vec = shuffle <32 x i32> %v0, <32 x i32> %v1, +/// <4, 32, 16, 5, 33, 17, 6, 34, 18, 7, 35, 19> +/// store <12 x i32> %i.vec, <12 x i32>* %ptr +/// +/// Into: +/// %sub.v0 = shuffle <32 x i32> %v0, <32 x i32> v1, <4, 5, 6, 7> +/// %sub.v1 = shuffle <32 x i32> %v0, <32 x i32> v1, <32, 33, 34, 35> +/// %sub.v2 = shuffle <32 x i32> %v0, <32 x i32> v1, <16, 17, 18, 19> +/// call void llvm.arm.neon.vst3(%ptr, %sub.v0, %sub.v1, %sub.v2, 4) bool ARMTargetLowering::lowerInterleavedStore(StoreInst *SI, ShuffleVectorInst *SVI, unsigned Factor) const { @@ -13041,9 +13052,24 @@ SI->getModule(), StoreInts[Factor - 2], Tys); // Split the shufflevector operands into sub vectors for the new vstN call. - for (unsigned i = 0; i < Factor; i++) - Ops.push_back(Builder.CreateShuffleVector( - Op0, Op1, getSequentialMask(Builder, NumSubElts * i, NumSubElts))); + ArrayRef Mask = SVI->getShuffleMask(); + for (unsigned i = 0; i < Factor; i++) { + if (Mask[i] >= 0) + Ops.push_back(Builder.CreateShuffleVector( + Op0, Op1, getSequentialMask(Builder, Mask[i], NumSubElts))); + else { + unsigned StartMask = 0; + for (unsigned j = 1; j < NumSubElts; j++) { + if (Mask[j*Factor + i] >= 0) { + StartMask = Mask[j*Factor + i] - j; + break; + } + } + // Correctness notes in AArch64ISelLowering apply here as well. + Ops.push_back(Builder.CreateShuffleVector( + Op0, Op1, getSequentialMask(Builder, StartMask, NumSubElts))); + } + } Ops.push_back(Builder.getInt32(SI->getAlignment())); Builder.CreateCall(VstNFunc, Ops); Index: test/CodeGen/AArch64/aarch64-interleaved-accesses.ll =================================================================== --- test/CodeGen/AArch64/aarch64-interleaved-accesses.ll +++ test/CodeGen/AArch64/aarch64-interleaved-accesses.ll @@ -280,3 +280,14 @@ %3 = extractelement <8 x i32> %1, i32 2 ret i32 %3 } + +; NEON-LABEL: store_general_mask_factor4: +; NEON: st4 { v3.2s, v4.2s, v5.2s, v6.2s }, [x0] +; NONEON-LABEL: store_general_mask_factor4: +; NONEON-NOT: st4 +define void @store_general_mask_factor4(i32* %ptr, <32 x i32> %v0, <32 x i32> %v1) { + %base = bitcast i32* %ptr to <8 x i32>* + %i.vec = shufflevector <32 x i32> %v0, <32 x i32> %v1, <8 x i32> + store <8 x i32> %i.vec, <8 x i32>* %base, align 4 + ret void +} Index: test/CodeGen/ARM/arm-interleaved-accesses.ll =================================================================== --- test/CodeGen/ARM/arm-interleaved-accesses.ll +++ test/CodeGen/ARM/arm-interleaved-accesses.ll @@ -316,3 +316,14 @@ %3 = extractelement <8 x i32> %1, i32 2 ret i32 %3 } + +; NEON-LABEL: store_general_mask_factor4: +; NEON: vst4.32 {d18, d19, d20, d21}, [r0] +; NONEON-LABEL: store_general_mask_factor4: +; NONEON-NOT: vst4.32 +define void @store_general_mask_factor4(i32* %ptr, <32 x i32> %v0, <32 x i32> %v1) { + %base = bitcast i32* %ptr to <8 x i32>* + %i.vec = shufflevector <32 x i32> %v0, <32 x i32> %v1, <8 x i32> + store <8 x i32> %i.vec, <8 x i32>* %base, align 4 + ret void +}