Index: lib/Target/X86/X86ISelLowering.h =================================================================== --- lib/Target/X86/X86ISelLowering.h +++ lib/Target/X86/X86ISelLowering.h @@ -1066,6 +1066,12 @@ ArrayRef Shuffles, ArrayRef Indices, unsigned Factor) const override; + + /// \brief Lower interleaved store(s) into target specific + /// instructions/intrinsics. + bool lowerInterleavedStore(StoreInst *SI, ShuffleVectorInst *SVI, + unsigned Factor) const override; + protected: std::pair findRepresentativeClass(const TargetRegisterInfo *TRI, Index: lib/Target/X86/X86InterleavedAccess.cpp =================================================================== --- lib/Target/X86/X86InterleavedAccess.cpp +++ lib/Target/X86/X86InterleavedAccess.cpp @@ -16,6 +16,7 @@ #include "X86ISelLowering.h" #include "X86TargetMachine.h" +#include "llvm/Analysis/VectorUtils.h" using namespace llvm; @@ -50,9 +51,8 @@ IRBuilder<> &Builder; /// \brief Breaks down a vector \p 'Inst' of N elements into \p NumSubVectors - /// sub vectors of type \p T. Returns true and the sub-vectors in - /// \p DecomposedVectors if it decomposes the Inst, returns false otherwise. - bool decompose(Instruction *Inst, unsigned NumSubVectors, VectorType *T, + /// sub vectors of type \p T. Returns the sub-vectors in \p DecomposedVectors. + void decompose(Instruction *Inst, unsigned NumSubVectors, VectorType *T, SmallVectorImpl &DecomposedVectors); /// \brief Performs matrix transposition on a 4x4 matrix \p InputVectors and @@ -80,8 +80,7 @@ /// target information \p STarget. explicit X86InterleavedAccessGroup(Instruction *I, ArrayRef Shuffs, - ArrayRef Ind, - const unsigned F, + ArrayRef Ind, const unsigned F, const X86Subtarget &STarget, IRBuilder<> &B) : Inst(I), Shuffles(Shuffs), Indices(Ind), Factor(F), Subtarget(STarget), @@ -102,35 +101,50 @@ uint64_t ShuffleVecSize = DL.getTypeSizeInBits(ShuffleVecTy); Type *ShuffleEltTy = ShuffleVecTy->getVectorElementType(); - if (DL.getTypeSizeInBits(Inst->getType()) < Factor * ShuffleVecSize) - return false; + // Currently, lowering is supported for 4-element vectors of 64 bits on AVX. + uint64_t ExpectedShuffleVecSize; + if (isa(Inst)) + ExpectedShuffleVecSize = 256; + else + ExpectedShuffleVecSize = 1024; - // Currently, lowering is supported for 64 bits on AVX. - if (!Subtarget.hasAVX() || ShuffleVecSize != 256 || + if (!Subtarget.hasAVX() || ShuffleVecSize != ExpectedShuffleVecSize || DL.getTypeSizeInBits(ShuffleEltTy) != 64 || Factor != 4) return false; return true; } -bool X86InterleavedAccessGroup::decompose( +void X86InterleavedAccessGroup::decompose( Instruction *VecInst, unsigned NumSubVectors, VectorType *SubVecTy, SmallVectorImpl &DecomposedVectors) { + + assert((isa(VecInst) || isa(VecInst)) && + "Expected Load or Shuffle"); + Type *VecTy = VecInst->getType(); (void)VecTy; assert(VecTy->isVectorTy() && DL.getTypeSizeInBits(VecTy) >= DL.getTypeSizeInBits(SubVecTy) * NumSubVectors && "Invalid Inst-size!!!"); - assert(VecTy->getVectorElementType() == SubVecTy->getVectorElementType() && - "Element type mismatched!!!"); - if (!isa(VecInst)) - return false; + if (auto *SVI = dyn_cast(VecInst)) { + Value *Op0 = SVI->getOperand(0); + Value *Op1 = SVI->getOperand(1); + // Generate N shuffles of T type + for (unsigned i = 0; i < NumSubVectors; i++) + DecomposedVectors.push_back( + cast(Builder.CreateShuffleVector( + Op0, Op1, + createSequentialMask(Builder, i * Factor, NumSubVectors, 0)))); + return; + } + + // Decompose the load instruction LoadInst *LI = cast(VecInst); Type *VecBasePtrTy = SubVecTy->getPointerTo(LI->getPointerAddressSpace()); - Value *VecBasePtr = Builder.CreateBitCast(LI->getPointerOperand(), VecBasePtrTy); @@ -142,8 +156,6 @@ Builder.CreateAlignedLoad(NewBasePtr, LI->getAlignment()); DecomposedVectors.push_back(NewLoad); } - - return true; } void X86InterleavedAccessGroup::transpose_4x4( @@ -181,22 +193,43 @@ // instructions/intrinsics. bool X86InterleavedAccessGroup::lowerIntoOptimizedSequence() { SmallVector DecomposedVectors; - VectorType *VecTy = Shuffles[0]->getType(); - // Try to generate target-sized register(/instruction). - if (!decompose(Inst, Factor, VecTy, DecomposedVectors)) - return false; + VectorType *ShuffleTy = Shuffles[0]->getType(); + Type *ShuffleEltTy = ShuffleTy->getVectorElementType(); + VectorType *SubVecTy = VectorType::get(ShuffleEltTy, Factor); SmallVector TransposedVectors; - // Perform matrix-transposition in order to compute interleaved - // results by generating some sort of (optimized) target-specific - // instructions. + if (isa(Inst)) { + // Try to generate target-sized register(/instruction). + decompose(Inst, Factor, SubVecTy, DecomposedVectors); + + // Perform matrix-transposition in order to compute interleaved + // results by generating some sort of (optimized) target-specific + // instructions. + transpose_4x4(DecomposedVectors, TransposedVectors); + + // Now replace the unoptimized-interleaved-vectors with the + // transposed-interleaved vectors. + for (unsigned i = 0; i < Shuffles.size(); i++) + Shuffles[i]->replaceAllUsesWith(TransposedVectors[Indices[i]]); + + return true; + } + + // Lower a group of interleaved stores + // Decompose interleaved wide shuffle + decompose(Shuffles[0], Factor, SubVecTy, DecomposedVectors); + + // Transpose to generate a set of contiguous vectors transpose_4x4(DecomposedVectors, TransposedVectors); - // Now replace the unoptimized-interleaved-vectors with the - // transposed-interleaved vectors. - for (unsigned i = 0; i < Shuffles.size(); i++) - Shuffles[i]->replaceAllUsesWith(TransposedVectors[Indices[i]]); + // Concatenate the contiguous vectors into a wide vector. + Value *WideVec = concatenateVectors(Builder, TransposedVectors); + // Generate a store instruction for wide-vec + StoreInst *SI = cast(Inst); + Builder.CreateAlignedStore(WideVec, SI->getPointerOperand(), + SI->getAlignment()); + return true; } @@ -220,3 +253,28 @@ return Grp.isSupported() && Grp.lowerIntoOptimizedSequence(); } + +bool X86TargetLowering::lowerInterleavedStore(StoreInst *SI, + ShuffleVectorInst *SVI, + unsigned Factor) const { + assert(Factor >= 2 && Factor <= getMaxSupportedInterleaveFactor() && + "Invalid interleave factor"); + + VectorType *VecTy = SVI->getType(); + assert(VecTy->getVectorNumElements() % Factor == 0 && + "Invalid interleaved store"); + + SmallVector Indices; + auto Mask = SVI->getShuffleMask(); + for (unsigned i = 0; i < Factor; i++) + Indices.push_back(i); + + ArrayRef Shuffles = makeArrayRef(SVI); + + // Create an interleaved access group. + IRBuilder<> Builder(SI); + X86InterleavedAccessGroup Grp(SI, Shuffles, Indices, Factor, Subtarget, + Builder); + + return Grp.isSupported() && Grp.lowerIntoOptimizedSequence(); +} Index: test/CodeGen/X86/x86-interleaved-access.ll =================================================================== --- test/CodeGen/X86/x86-interleaved-access.ll +++ test/CodeGen/X86/x86-interleaved-access.ll @@ -139,3 +139,68 @@ %add3 = add <4 x i64> %add2, %strided.v3 ret <4 x i64> %add3 } + +define void @store_factorf64_4(<16 x double>* %ptr, <4 x double> %v0, <4 x double> %v1, <4 x double> %v2, <4 x double> %v3) { +; AVX-LABEL: store_factorf64_4: +; AVX: # BB#0: +; AVX-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm4 +; AVX-NEXT: vinsertf128 $1, %xmm3, %ymm1, %ymm5 +; AVX-NEXT: vperm2f128 {{.*#+}} ymm0 = ymm0[2,3],ymm2[2,3] +; AVX-NEXT: vperm2f128 {{.*#+}} ymm1 = ymm1[2,3],ymm3[2,3] +; AVX-NEXT: vunpcklpd {{.*#+}} ymm2 = ymm4[0],ymm5[0],ymm4[2],ymm5[2] +; AVX-NEXT: vunpcklpd {{.*#+}} ymm3 = ymm0[0],ymm1[0],ymm0[2],ymm1[2] +; AVX-NEXT: vunpckhpd {{.*#+}} ymm4 = ymm4[1],ymm5[1],ymm4[3],ymm5[3] +; AVX-NEXT: vunpckhpd {{.*#+}} ymm0 = ymm0[1],ymm1[1],ymm0[3],ymm1[3] +; AVX-NEXT: vmovupd %ymm0, 96(%rdi) +; AVX-NEXT: vmovupd %ymm3, 64(%rdi) +; AVX-NEXT: vmovupd %ymm4, 32(%rdi) +; AVX-NEXT: vmovupd %ymm2, (%rdi) +; AVX-NEXT: vzeroupper +; AVX-NEXT: retq + %s0 = shufflevector <4 x double> %v0, <4 x double> %v1, <8 x i32> + %s1 = shufflevector <4 x double> %v2, <4 x double> %v3, <8 x i32> + %interleaved.vec = shufflevector <8 x double> %s0, <8 x double> %s1, <16 x i32> + store <16 x double> %interleaved.vec, <16 x double>* %ptr, align 16 + ret void +} + +define void @store_factori64_4(<16 x i64>* %ptr, <4 x i64> %v0, <4 x i64> %v1, <4 x i64> %v2, <4 x i64> %v3) { +; AVX1-LABEL: store_factori64_4: +; AVX1: # BB#0: +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm4 +; AVX1-NEXT: vinsertf128 $1, %xmm3, %ymm1, %ymm5 +; AVX1-NEXT: vperm2f128 {{.*#+}} ymm0 = ymm0[2,3],ymm2[2,3] +; AVX1-NEXT: vperm2f128 {{.*#+}} ymm1 = ymm1[2,3],ymm3[2,3] +; AVX1-NEXT: vunpcklpd {{.*#+}} ymm2 = ymm4[0],ymm5[0],ymm4[2],ymm5[2] +; AVX1-NEXT: vunpcklpd {{.*#+}} ymm3 = ymm0[0],ymm1[0],ymm0[2],ymm1[2] +; AVX1-NEXT: vunpckhpd {{.*#+}} ymm4 = ymm4[1],ymm5[1],ymm4[3],ymm5[3] +; AVX1-NEXT: vunpckhpd {{.*#+}} ymm0 = ymm0[1],ymm1[1],ymm0[3],ymm1[3] +; AVX1-NEXT: vmovupd %ymm0, 96(%rdi) +; AVX1-NEXT: vmovupd %ymm3, 64(%rdi) +; AVX1-NEXT: vmovupd %ymm4, 32(%rdi) +; AVX1-NEXT: vmovupd %ymm2, (%rdi) +; AVX1-NEXT: vzeroupper +; AVX1-NEXT: retq +; +; AVX2-LABEL: store_factori64_4: +; AVX2: # BB#0: +; AVX2-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm4 +; AVX2-NEXT: vinserti128 $1, %xmm3, %ymm1, %ymm5 +; AVX2-NEXT: vperm2i128 {{.*#+}} ymm0 = ymm0[2,3],ymm2[2,3] +; AVX2-NEXT: vperm2i128 {{.*#+}} ymm1 = ymm1[2,3],ymm3[2,3] +; AVX2-NEXT: vpunpcklqdq {{.*#+}} ymm2 = ymm4[0],ymm5[0],ymm4[2],ymm5[2] +; AVX2-NEXT: vpunpcklqdq {{.*#+}} ymm3 = ymm0[0],ymm1[0],ymm0[2],ymm1[2] +; AVX2-NEXT: vpunpckhqdq {{.*#+}} ymm4 = ymm4[1],ymm5[1],ymm4[3],ymm5[3] +; AVX2-NEXT: vpunpckhqdq {{.*#+}} ymm0 = ymm0[1],ymm1[1],ymm0[3],ymm1[3] +; AVX2-NEXT: vmovdqu %ymm0, 96(%rdi) +; AVX2-NEXT: vmovdqu %ymm3, 64(%rdi) +; AVX2-NEXT: vmovdqu %ymm4, 32(%rdi) +; AVX2-NEXT: vmovdqu %ymm2, (%rdi) +; AVX2-NEXT: vzeroupper +; AVX2-NEXT: retq + %s0 = shufflevector <4 x i64> %v0, <4 x i64> %v1, <8 x i32> + %s1 = shufflevector <4 x i64> %v2, <4 x i64> %v3, <8 x i32> + %interleaved.vec = shufflevector <8 x i64> %s0, <8 x i64> %s1, <16 x i32> + store <16 x i64> %interleaved.vec, <16 x i64>* %ptr, align 16 + ret void +} Index: test/Transforms/InterleavedAccess/X86/interleaved-accesses-64bits-avx.ll =================================================================== --- test/Transforms/InterleavedAccess/X86/interleaved-accesses-64bits-avx.ll +++ test/Transforms/InterleavedAccess/X86/interleaved-accesses-64bits-avx.ll @@ -1,7 +1,7 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py ; RUN: opt < %s -mtriple=x86_64-pc-linux -mattr=+avx -interleaved-access -S | FileCheck %s -; This file tests the function `llvm::lowerInterleavedLoad`. +; This file tests the function `llvm::lowerInterleavedLoad/store()`. define <4 x double> @load_factorf64_4(<16 x double>* %ptr) { ; CHECK-LABEL: @load_factorf64_4( @@ -102,4 +102,60 @@ ret <4 x double> %mul } +define void @store_factorf64_4(<16 x double>* %ptr, <4 x double> %v0, <4 x double> %v1, <4 x double> %v2, <4 x double> %v3) { +; CHECK-LABEL: @store_factorf64_4( +; CHECK-NEXT: [[S0:%.*]] = shufflevector <4 x double> [[V0:%.*]], <4 x double> [[V1:%.*]], <8 x i32> +; CHECK-NEXT: [[S1:%.*]] = shufflevector <4 x double> [[V2:%.*]], <4 x double> [[V3:%.*]], <8 x i32> +; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <8 x double> [[S0]], <8 x double> [[S1]], <4 x i32> +; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <8 x double> [[S0]], <8 x double> [[S1]], <4 x i32> +; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <8 x double> [[S0]], <8 x double> [[S1]], <4 x i32> +; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <8 x double> [[S0]], <8 x double> [[S1]], <4 x i32> +; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <4 x double> [[TMP1]], <4 x double> [[TMP3]], <4 x i32> +; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <4 x double> [[TMP2]], <4 x double> [[TMP4]], <4 x i32> +; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <4 x double> [[TMP1]], <4 x double> [[TMP3]], <4 x i32> +; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <4 x double> [[TMP2]], <4 x double> [[TMP4]], <4 x i32> +; CHECK-NEXT: [[TMP9:%.*]] = shufflevector <4 x double> [[TMP5]], <4 x double> [[TMP6]], <4 x i32> +; CHECK-NEXT: [[TMP10:%.*]] = shufflevector <4 x double> [[TMP7]], <4 x double> [[TMP8]], <4 x i32> +; CHECK-NEXT: [[TMP11:%.*]] = shufflevector <4 x double> [[TMP5]], <4 x double> [[TMP6]], <4 x i32> +; CHECK-NEXT: [[TMP12:%.*]] = shufflevector <4 x double> [[TMP7]], <4 x double> [[TMP8]], <4 x i32> +; CHECK-NEXT: [[TMP13:%.*]] = shufflevector <4 x double> [[TMP9]], <4 x double> [[TMP11]], <8 x i32> +; CHECK-NEXT: [[TMP14:%.*]] = shufflevector <4 x double> [[TMP10]], <4 x double> [[TMP12]], <8 x i32> +; CHECK-NEXT: [[TMP15:%.*]] = shufflevector <8 x double> [[TMP13]], <8 x double> [[TMP14]], <16 x i32> +; CHECK-NEXT: store <16 x double> [[TMP15]], <16 x double>* [[PTR:%.*]], align 16 +; CHECK-NEXT: ret void +; + %s0 = shufflevector <4 x double> %v0, <4 x double> %v1, <8 x i32> + %s1 = shufflevector <4 x double> %v2, <4 x double> %v3, <8 x i32> + %interleaved.vec = shufflevector <8 x double> %s0, <8 x double> %s1, <16 x i32> + store <16 x double> %interleaved.vec, <16 x double>* %ptr, align 16 + ret void +} +define void @store_factori64_4(<16 x i64>* %ptr, <4 x i64> %v0, <4 x i64> %v1, <4 x i64> %v2, <4 x i64> %v3) { +; CHECK-LABEL: @store_factori64_4( +; CHECK-NEXT: [[S0:%.*]] = shufflevector <4 x i64> [[V0:%.*]], <4 x i64> [[V1:%.*]], <8 x i32> +; CHECK-NEXT: [[S1:%.*]] = shufflevector <4 x i64> [[V2:%.*]], <4 x i64> [[V3:%.*]], <8 x i32> +; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <8 x i64> [[S0]], <8 x i64> [[S1]], <4 x i32> +; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <8 x i64> [[S0]], <8 x i64> [[S1]], <4 x i32> +; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <8 x i64> [[S0]], <8 x i64> [[S1]], <4 x i32> +; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <8 x i64> [[S0]], <8 x i64> [[S1]], <4 x i32> +; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <4 x i64> [[TMP1]], <4 x i64> [[TMP3]], <4 x i32> +; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <4 x i64> [[TMP2]], <4 x i64> [[TMP4]], <4 x i32> +; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <4 x i64> [[TMP1]], <4 x i64> [[TMP3]], <4 x i32> +; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <4 x i64> [[TMP2]], <4 x i64> [[TMP4]], <4 x i32> +; CHECK-NEXT: [[TMP9:%.*]] = shufflevector <4 x i64> [[TMP5]], <4 x i64> [[TMP6]], <4 x i32> +; CHECK-NEXT: [[TMP10:%.*]] = shufflevector <4 x i64> [[TMP7]], <4 x i64> [[TMP8]], <4 x i32> +; CHECK-NEXT: [[TMP11:%.*]] = shufflevector <4 x i64> [[TMP5]], <4 x i64> [[TMP6]], <4 x i32> +; CHECK-NEXT: [[TMP12:%.*]] = shufflevector <4 x i64> [[TMP7]], <4 x i64> [[TMP8]], <4 x i32> +; CHECK-NEXT: [[TMP13:%.*]] = shufflevector <4 x i64> [[TMP9]], <4 x i64> [[TMP11]], <8 x i32> +; CHECK-NEXT: [[TMP14:%.*]] = shufflevector <4 x i64> [[TMP10]], <4 x i64> [[TMP12]], <8 x i32> +; CHECK-NEXT: [[TMP15:%.*]] = shufflevector <8 x i64> [[TMP13]], <8 x i64> [[TMP14]], <16 x i32> +; CHECK-NEXT: store <16 x i64> [[TMP15]], <16 x i64>* [[PTR:%.*]], align 16 +; CHECK-NEXT: ret void +; + %s0 = shufflevector <4 x i64> %v0, <4 x i64> %v1, <8 x i32> + %s1 = shufflevector <4 x i64> %v2, <4 x i64> %v3, <8 x i32> + %interleaved.vec = shufflevector <8 x i64> %s0, <8 x i64> %s1, <16 x i32> + store <16 x i64> %interleaved.vec, <16 x i64>* %ptr, align 16 + ret void +}