Index: lib/CodeGen/SelectionDAG/DAGCombiner.cpp =================================================================== --- lib/CodeGen/SelectionDAG/DAGCombiner.cpp +++ lib/CodeGen/SelectionDAG/DAGCombiner.cpp @@ -387,6 +387,8 @@ SDValue MatchBSwapHWordLow(SDNode *N, SDValue N0, SDValue N1, bool DemandHighBits = true); SDValue MatchBSwapHWord(SDNode *N, SDValue N0, SDValue N1); + bool sumMatchConstant(SDValue Left, SDValue Right, unsigned Sum, + const SDLoc &DL); SDNode *MatchRotatePosNeg(SDValue Shifted, SDValue Pos, SDValue Neg, SDValue InnerPos, SDValue InnerNeg, unsigned PosOpcode, unsigned NegOpcode, @@ -4585,6 +4587,20 @@ return nullptr; } +// if Left + Right == Sum (constant or constant splat vector) +bool DAGCombiner::sumMatchConstant(SDValue Left, SDValue Right, unsigned Sum, + const SDLoc &DL) { + EVT ShiftVT = Left.getValueType(); + if (ShiftVT != Right.getValueType()) return false; + + SDValue ShiftSum = DAG.FoldConstantArithmetic(ISD::ADD, DL, ShiftVT, + Left.getNode(), Right.getNode()); + if (!ShiftSum) return false; + + ConstantSDNode *CSum = isConstOrConstSplat(ShiftSum); + return CSum && CSum->getZExtValue() == Sum; +} + // MatchRotate - Handle an 'or' of two operands. If this is one of the many // idioms for rotate, and if the target supports rotation instructions, generate // a rot[lr]. @@ -4630,30 +4646,26 @@ // fold (or (shl x, C1), (srl x, C2)) -> (rotl x, C1) // fold (or (shl x, C1), (srl x, C2)) -> (rotr x, C2) - if (isConstOrConstSplat(LHSShiftAmt) && isConstOrConstSplat(RHSShiftAmt)) { - uint64_t LShVal = isConstOrConstSplat(LHSShiftAmt)->getZExtValue(); - uint64_t RShVal = isConstOrConstSplat(RHSShiftAmt)->getZExtValue(); - if ((LShVal + RShVal) != EltSizeInBits) - return nullptr; - + if (sumMatchConstant(LHSShiftAmt, RHSShiftAmt, EltSizeInBits, DL)) { SDValue Rot = DAG.getNode(HasROTL ? ISD::ROTL : ISD::ROTR, DL, VT, LHSShiftArg, HasROTL ? LHSShiftAmt : RHSShiftAmt); // If there is an AND of either shifted operand, apply it to the result. if (LHSMask.getNode() || RHSMask.getNode()) { - SDValue Mask = DAG.getAllOnesConstant(DL, VT); + SDValue One = DAG.getAllOnesConstant(DL, VT); + SDValue Mask = One; if (LHSMask.getNode()) { - APInt RHSBits = APInt::getLowBitsSet(EltSizeInBits, LShVal); + SDValue RHSBits = DAG.FoldConstantArithmetic( + ISD::SRL, DL, VT, One.getNode(), RHSShiftAmt.getNode()); Mask = DAG.getNode(ISD::AND, DL, VT, Mask, - DAG.getNode(ISD::OR, DL, VT, LHSMask, - DAG.getConstant(RHSBits, DL, VT))); + DAG.getNode(ISD::OR, DL, VT, LHSMask, RHSBits)); } if (RHSMask.getNode()) { - APInt LHSBits = APInt::getHighBitsSet(EltSizeInBits, RShVal); + SDValue LHSBits = DAG.FoldConstantArithmetic( + ISD::SHL, DL, VT, One.getNode(), LHSShiftAmt.getNode()); Mask = DAG.getNode(ISD::AND, DL, VT, Mask, - DAG.getNode(ISD::OR, DL, VT, RHSMask, - DAG.getConstant(LHSBits, DL, VT))); + DAG.getNode(ISD::OR, DL, VT, RHSMask, LHSBits)); } Rot = DAG.getNode(ISD::AND, DL, VT, Rot, Mask); Index: test/CodeGen/X86/combine-rotates.ll =================================================================== --- test/CodeGen/X86/combine-rotates.ll +++ test/CodeGen/X86/combine-rotates.ll @@ -6,12 +6,7 @@ define <4 x i32> @combine_vec_rot_rot(<4 x i32> %x) { ; XOP-LABEL: combine_vec_rot_rot: ; XOP: # BB#0: -; XOP-NEXT: vpshld {{.*}}(%rip), %xmm0, %xmm1 -; XOP-NEXT: vpshld {{.*}}(%rip), %xmm0, %xmm0 -; XOP-NEXT: vpor %xmm0, %xmm1, %xmm0 -; XOP-NEXT: vpshld {{.*}}(%rip), %xmm0, %xmm1 -; XOP-NEXT: vpshld {{.*}}(%rip), %xmm0, %xmm0 -; XOP-NEXT: vpor %xmm0, %xmm1, %xmm0 +; XOP-NEXT: vprotd {{.*}}(%rip), %xmm0, %xmm0 ; XOP-NEXT: retq ; ; AVX512-LABEL: combine_vec_rot_rot: Index: test/CodeGen/X86/rotate_vec.ll =================================================================== --- test/CodeGen/X86/rotate_vec.ll +++ test/CodeGen/X86/rotate_vec.ll @@ -15,9 +15,7 @@ define <4 x i32> @rot_v4i32_non_splat(<4 x i32> %x) { ; CHECK-LABEL: rot_v4i32_non_splat: ; CHECK: # BB#0: -; CHECK-NEXT: vpsrlvd {{.*}}(%rip), %xmm0, %xmm1 -; CHECK-NEXT: vpsllvd {{.*}}(%rip), %xmm0, %xmm0 -; CHECK-NEXT: vpor %xmm0, %xmm1, %xmm0 +; CHECK-NEXT: vprotd {{.*}}(%rip), %xmm0, %xmm0 ; CHECK-NEXT: retq %1 = lshr <4 x i32> %x, %2 = shl <4 x i32> %x, @@ -43,12 +41,8 @@ define <4 x i32> @rot_v4i32_non_splat_2masks(<4 x i32> %x) { ; CHECK-LABEL: rot_v4i32_non_splat_2masks: ; CHECK: # BB#0: -; CHECK-NEXT: vpsrlvd {{.*}}(%rip), %xmm0, %xmm1 -; CHECK-NEXT: vpxor %xmm2, %xmm2, %xmm2 -; CHECK-NEXT: vpblendw {{.*#+}} xmm1 = xmm2[0],xmm1[1],xmm2[2],xmm1[3],xmm2[4],xmm1[5],xmm2[6],xmm1[7] -; CHECK-NEXT: vpsllvd {{.*}}(%rip), %xmm0, %xmm0 -; CHECK-NEXT: vpblendw {{.*#+}} xmm0 = xmm2[0,1,2],xmm0[3],xmm2[4,5,6],xmm0[7] -; CHECK-NEXT: vpor %xmm0, %xmm1, %xmm0 +; CHECK-NEXT: vprotd {{.*}}(%rip), %xmm0, %xmm0 +; CHECK-NEXT: vpand {{.*}}(%rip), %xmm0, %xmm0 ; CHECK-NEXT: retq %1 = lshr <4 x i32> %x, %2 = and <4 x i32> %1, Index: test/CodeGen/X86/vector-rotate-128.ll =================================================================== --- test/CodeGen/X86/vector-rotate-128.ll +++ test/CodeGen/X86/vector-rotate-128.ll @@ -207,7 +207,7 @@ ; ; AVX2-LABEL: var_rotate_v4i32: ; AVX2: # BB#0: -; AVX2-NEXT: vpbroadcastd {{.*}}(%rip), %xmm2 +; AVX2-NEXT: vpbroadcastd {{.*#+}} xmm2 = [32,32,32,32] ; AVX2-NEXT: vpsubd %xmm1, %xmm2, %xmm2 ; AVX2-NEXT: vpsllvd %xmm1, %xmm0, %xmm1 ; AVX2-NEXT: vpsrlvd %xmm2, %xmm0, %xmm0 @@ -216,7 +216,7 @@ ; ; AVX512-LABEL: var_rotate_v4i32: ; AVX512: # BB#0: -; AVX512-NEXT: vpbroadcastd {{.*}}(%rip), %xmm2 +; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm2 = [32,32,32,32] ; AVX512-NEXT: vpsubd %xmm1, %xmm2, %xmm2 ; AVX512-NEXT: vpsllvd %xmm1, %xmm0, %xmm1 ; AVX512-NEXT: vpsrlvd %xmm2, %xmm0, %xmm0 @@ -851,21 +851,10 @@ ; AVX512-NEXT: vpor %xmm0, %xmm1, %xmm0 ; AVX512-NEXT: retq ; -; XOPAVX1-LABEL: constant_rotate_v2i64: -; XOPAVX1: # BB#0: -; XOPAVX1-NEXT: vpshlq {{.*}}(%rip), %xmm0, %xmm1 -; XOPAVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2 -; XOPAVX1-NEXT: vpsubq {{.*}}(%rip), %xmm2, %xmm2 -; XOPAVX1-NEXT: vpshlq %xmm2, %xmm0, %xmm0 -; XOPAVX1-NEXT: vpor %xmm0, %xmm1, %xmm0 -; XOPAVX1-NEXT: retq -; -; XOPAVX2-LABEL: constant_rotate_v2i64: -; XOPAVX2: # BB#0: -; XOPAVX2-NEXT: vpsllvq {{.*}}(%rip), %xmm0, %xmm1 -; XOPAVX2-NEXT: vpsrlvq {{.*}}(%rip), %xmm0, %xmm0 -; XOPAVX2-NEXT: vpor %xmm0, %xmm1, %xmm0 -; XOPAVX2-NEXT: retq +; XOP-LABEL: constant_rotate_v2i64: +; XOP: # BB#0: +; XOP-NEXT: vprotq {{.*}}(%rip), %xmm0, %xmm0 +; XOP-NEXT: retq ; ; X32-SSE-LABEL: constant_rotate_v2i64: ; X32-SSE: # BB#0: @@ -958,19 +947,10 @@ ; AVX512-NEXT: vpor %xmm0, %xmm1, %xmm0 ; AVX512-NEXT: retq ; -; XOPAVX1-LABEL: constant_rotate_v4i32: -; XOPAVX1: # BB#0: -; XOPAVX1-NEXT: vpshld {{.*}}(%rip), %xmm0, %xmm1 -; XOPAVX1-NEXT: vpshld {{.*}}(%rip), %xmm0, %xmm0 -; XOPAVX1-NEXT: vpor %xmm0, %xmm1, %xmm0 -; XOPAVX1-NEXT: retq -; -; XOPAVX2-LABEL: constant_rotate_v4i32: -; XOPAVX2: # BB#0: -; XOPAVX2-NEXT: vpsllvd {{.*}}(%rip), %xmm0, %xmm1 -; XOPAVX2-NEXT: vpsrlvd {{.*}}(%rip), %xmm0, %xmm0 -; XOPAVX2-NEXT: vpor %xmm0, %xmm1, %xmm0 -; XOPAVX2-NEXT: retq +; XOP-LABEL: constant_rotate_v4i32: +; XOP: # BB#0: +; XOP-NEXT: vprotd {{.*}}(%rip), %xmm0, %xmm0 +; XOP-NEXT: retq ; ; X32-SSE-LABEL: constant_rotate_v4i32: ; X32-SSE: # BB#0: @@ -1100,11 +1080,7 @@ ; ; XOP-LABEL: constant_rotate_v8i16: ; XOP: # BB#0: -; XOP-NEXT: vpshlw {{.*}}(%rip), %xmm0, %xmm1 -; XOP-NEXT: vpxor %xmm2, %xmm2, %xmm2 -; XOP-NEXT: vpsubw {{.*}}(%rip), %xmm2, %xmm2 -; XOP-NEXT: vpshlw %xmm2, %xmm0, %xmm0 -; XOP-NEXT: vpor %xmm0, %xmm1, %xmm0 +; XOP-NEXT: vprotw {{.*}}(%rip), %xmm0, %xmm0 ; XOP-NEXT: retq ; ; X32-SSE-LABEL: constant_rotate_v8i16: @@ -1281,11 +1257,7 @@ ; ; XOP-LABEL: constant_rotate_v16i8: ; XOP: # BB#0: -; XOP-NEXT: vpshlb {{.*}}(%rip), %xmm0, %xmm1 -; XOP-NEXT: vpxor %xmm2, %xmm2, %xmm2 -; XOP-NEXT: vpsubb {{.*}}(%rip), %xmm2, %xmm2 -; XOP-NEXT: vpshlb %xmm2, %xmm0, %xmm0 -; XOP-NEXT: vpor %xmm0, %xmm1, %xmm0 +; XOP-NEXT: vprotb {{.*}}(%rip), %xmm0, %xmm0 ; XOP-NEXT: retq ; ; X32-SSE-LABEL: constant_rotate_v16i8: Index: test/CodeGen/X86/vector-rotate-256.ll =================================================================== --- test/CodeGen/X86/vector-rotate-256.ll +++ test/CodeGen/X86/vector-rotate-256.ll @@ -41,7 +41,7 @@ ; ; AVX2-LABEL: var_rotate_v4i64: ; AVX2: # BB#0: -; AVX2-NEXT: vpbroadcastq {{.*}}(%rip), %ymm2 +; AVX2-NEXT: vpbroadcastq {{.*#+}} ymm2 = [64,64,64,64] ; AVX2-NEXT: vpsubq %ymm1, %ymm2, %ymm2 ; AVX2-NEXT: vpsllvq %ymm1, %ymm0, %ymm1 ; AVX2-NEXT: vpsrlvq %ymm2, %ymm0, %ymm0 @@ -50,7 +50,7 @@ ; ; AVX512-LABEL: var_rotate_v4i64: ; AVX512: # BB#0: -; AVX512-NEXT: vpbroadcastq {{.*}}(%rip), %ymm2 +; AVX512-NEXT: vpbroadcastq {{.*#+}} ymm2 = [64,64,64,64] ; AVX512-NEXT: vpsubq %ymm1, %ymm2, %ymm2 ; AVX512-NEXT: vpsllvq %ymm1, %ymm0, %ymm1 ; AVX512-NEXT: vpsrlvq %ymm2, %ymm0, %ymm0 @@ -128,7 +128,7 @@ ; ; AVX2-LABEL: var_rotate_v8i32: ; AVX2: # BB#0: -; AVX2-NEXT: vpbroadcastd {{.*}}(%rip), %ymm2 +; AVX2-NEXT: vpbroadcastd {{.*#+}} ymm2 = [32,32,32,32,32,32,32,32] ; AVX2-NEXT: vpsubd %ymm1, %ymm2, %ymm2 ; AVX2-NEXT: vpsllvd %ymm1, %ymm0, %ymm1 ; AVX2-NEXT: vpsrlvd %ymm2, %ymm0, %ymm0 @@ -137,7 +137,7 @@ ; ; AVX512-LABEL: var_rotate_v8i32: ; AVX512: # BB#0: -; AVX512-NEXT: vpbroadcastd {{.*}}(%rip), %ymm2 +; AVX512-NEXT: vpbroadcastd {{.*#+}} ymm2 = [32,32,32,32,32,32,32,32] ; AVX512-NEXT: vpsubd %ymm1, %ymm2, %ymm2 ; AVX512-NEXT: vpsllvd %ymm1, %ymm0, %ymm1 ; AVX512-NEXT: vpsrlvd %ymm2, %ymm0, %ymm0 @@ -558,21 +558,18 @@ ; ; XOPAVX1-LABEL: constant_rotate_v8i32: ; XOPAVX1: # BB#0: -; XOPAVX1-NEXT: vpshld {{.*}}(%rip), %xmm0, %xmm1 -; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm2 -; XOPAVX1-NEXT: vpshld {{.*}}(%rip), %xmm2, %xmm3 -; XOPAVX1-NEXT: vinsertf128 $1, %xmm3, %ymm1, %ymm1 -; XOPAVX1-NEXT: vpshld {{.*}}(%rip), %xmm0, %xmm0 -; XOPAVX1-NEXT: vpshld {{.*}}(%rip), %xmm2, %xmm2 -; XOPAVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 -; XOPAVX1-NEXT: vorps %ymm0, %ymm1, %ymm0 +; XOPAVX1-NEXT: vprotd {{.*}}(%rip), %xmm0, %xmm1 +; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm0 +; XOPAVX1-NEXT: vprotd {{.*}}(%rip), %xmm0, %xmm0 +; XOPAVX1-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0 ; XOPAVX1-NEXT: retq ; ; XOPAVX2-LABEL: constant_rotate_v8i32: ; XOPAVX2: # BB#0: -; XOPAVX2-NEXT: vpsllvd {{.*}}(%rip), %ymm0, %ymm1 -; XOPAVX2-NEXT: vpsrlvd {{.*}}(%rip), %ymm0, %ymm0 -; XOPAVX2-NEXT: vpor %ymm0, %ymm1, %ymm0 +; XOPAVX2-NEXT: vprotd {{.*}}(%rip), %xmm0, %xmm1 +; XOPAVX2-NEXT: vextracti128 $1, %ymm0, %xmm0 +; XOPAVX2-NEXT: vprotd {{.*}}(%rip), %xmm0, %xmm0 +; XOPAVX2-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0 ; XOPAVX2-NEXT: retq %shl = shl <8 x i32> %a, %lshr = lshr <8 x i32> %a, @@ -643,30 +640,18 @@ ; ; XOPAVX1-LABEL: constant_rotate_v16i16: ; XOPAVX1: # BB#0: -; XOPAVX1-NEXT: vpshlw {{.*}}(%rip), %xmm0, %xmm1 -; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm2 -; XOPAVX1-NEXT: vpshlw {{.*}}(%rip), %xmm2, %xmm3 -; XOPAVX1-NEXT: vinsertf128 $1, %xmm3, %ymm1, %ymm1 -; XOPAVX1-NEXT: vpxor %xmm3, %xmm3, %xmm3 -; XOPAVX1-NEXT: vpsubw {{.*}}(%rip), %xmm3, %xmm4 -; XOPAVX1-NEXT: vpshlw %xmm4, %xmm2, %xmm2 -; XOPAVX1-NEXT: vpsubw {{.*}}(%rip), %xmm3, %xmm3 -; XOPAVX1-NEXT: vpshlw %xmm3, %xmm0, %xmm0 -; XOPAVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 -; XOPAVX1-NEXT: vorps %ymm0, %ymm1, %ymm0 +; XOPAVX1-NEXT: vprotw {{.*}}(%rip), %xmm0, %xmm1 +; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm0 +; XOPAVX1-NEXT: vprotw {{.*}}(%rip), %xmm0, %xmm0 +; XOPAVX1-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0 ; XOPAVX1-NEXT: retq ; ; XOPAVX2-LABEL: constant_rotate_v16i16: ; XOPAVX2: # BB#0: -; XOPAVX2-NEXT: vpmullw {{.*}}(%rip), %ymm0, %ymm1 -; XOPAVX2-NEXT: vpxor %xmm2, %xmm2, %xmm2 -; XOPAVX2-NEXT: vpsubw {{.*}}(%rip), %xmm2, %xmm3 -; XOPAVX2-NEXT: vextracti128 $1, %ymm0, %xmm4 -; XOPAVX2-NEXT: vpshlw %xmm3, %xmm4, %xmm3 -; XOPAVX2-NEXT: vpsubw {{.*}}(%rip), %xmm2, %xmm2 -; XOPAVX2-NEXT: vpshlw %xmm2, %xmm0, %xmm0 -; XOPAVX2-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm0 -; XOPAVX2-NEXT: vpor %ymm0, %ymm1, %ymm0 +; XOPAVX2-NEXT: vprotw {{.*}}(%rip), %xmm0, %xmm1 +; XOPAVX2-NEXT: vextracti128 $1, %ymm0, %xmm0 +; XOPAVX2-NEXT: vprotw {{.*}}(%rip), %xmm0, %xmm0 +; XOPAVX2-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0 ; XOPAVX2-NEXT: retq %shl = shl <16 x i16> %a, %lshr = lshr <16 x i16> %a, @@ -768,32 +753,20 @@ ; ; XOPAVX1-LABEL: constant_rotate_v32i8: ; XOPAVX1: # BB#0: -; XOPAVX1-NEXT: vmovdqa {{.*#+}} xmm1 = [0,1,2,3,4,5,6,7,8,7,6,5,4,3,2,1] -; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm2 -; XOPAVX1-NEXT: vpshlb %xmm1, %xmm2, %xmm3 -; XOPAVX1-NEXT: vpshlb %xmm1, %xmm0, %xmm1 -; XOPAVX1-NEXT: vinsertf128 $1, %xmm3, %ymm1, %ymm1 -; XOPAVX1-NEXT: vpxor %xmm3, %xmm3, %xmm3 -; XOPAVX1-NEXT: vpsubb {{.*}}(%rip), %xmm3, %xmm3 -; XOPAVX1-NEXT: vpshlb %xmm3, %xmm2, %xmm2 -; XOPAVX1-NEXT: vpshlb %xmm3, %xmm0, %xmm0 -; XOPAVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 -; XOPAVX1-NEXT: vorps %ymm0, %ymm1, %ymm0 +; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm1 +; XOPAVX1-NEXT: vmovdqa {{.*#+}} xmm2 = [0,1,2,3,4,5,6,7,8,7,6,5,4,3,2,1] +; XOPAVX1-NEXT: vprotb %xmm2, %xmm1, %xmm1 +; XOPAVX1-NEXT: vprotb %xmm2, %xmm0, %xmm0 +; XOPAVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 ; XOPAVX1-NEXT: retq ; ; XOPAVX2-LABEL: constant_rotate_v32i8: ; XOPAVX2: # BB#0: -; XOPAVX2-NEXT: vmovdqa {{.*#+}} xmm1 = [0,1,2,3,4,5,6,7,8,7,6,5,4,3,2,1] -; XOPAVX2-NEXT: vextracti128 $1, %ymm0, %xmm2 -; XOPAVX2-NEXT: vpshlb %xmm1, %xmm2, %xmm3 -; XOPAVX2-NEXT: vpshlb %xmm1, %xmm0, %xmm1 -; XOPAVX2-NEXT: vinserti128 $1, %xmm3, %ymm1, %ymm1 -; XOPAVX2-NEXT: vpxor %xmm3, %xmm3, %xmm3 -; XOPAVX2-NEXT: vpsubb {{.*}}(%rip), %xmm3, %xmm3 -; XOPAVX2-NEXT: vpshlb %xmm3, %xmm2, %xmm2 -; XOPAVX2-NEXT: vpshlb %xmm3, %xmm0, %xmm0 -; XOPAVX2-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0 -; XOPAVX2-NEXT: vpor %ymm0, %ymm1, %ymm0 +; XOPAVX2-NEXT: vextracti128 $1, %ymm0, %xmm1 +; XOPAVX2-NEXT: vmovdqa {{.*#+}} xmm2 = [0,1,2,3,4,5,6,7,8,7,6,5,4,3,2,1] +; XOPAVX2-NEXT: vprotb %xmm2, %xmm1, %xmm1 +; XOPAVX2-NEXT: vprotb %xmm2, %xmm0, %xmm0 +; XOPAVX2-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0 ; XOPAVX2-NEXT: retq %shl = shl <32 x i8> %a, %lshr = lshr <32 x i8> %a,