[LV] Consider ExtractValue as uniform.

[sdesmalen]

Summary

Since all operands to ExtractValue must be loop-invariant when we deem
the loop vectorizable, we can consider ExtractValue to be uniform.

[dmgreen]

llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

4963

This doesn't appear to be used.

[david-arm]

llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

4966

I think we can probably hoist this out of the for loop and just create it once.

llvm/test/Transforms/LoopVectorize/AArch64/extractvalue-no-scalarization-required.ll

4

Could you add at least one test for an interleave > 1 either here or in the SVE test, just to prove we've got the parts right?

[sdesmalen]

• Removed unused variable.
• Hoisted ExtractValue and Splat out of the loop.
• Updated test.

[sdesmalen]

llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

4963

I somehow didn't notice that before posting the patch. Thanks for pointing out!

4966

Good point, I've made the change.

[fhahn]

since it's operands must always be scalar or loop-invariant.

At the moment, it must always be loop-invariant, right? Because instructions with struct-typed return values are not supported. Can't we just mark it as uniform?

[sdesmalen]

Mark ExtractValue as uniform.

[sdesmalen]

In D107286#2924610, @fhahn wrote:

since it's operands must always be scalar or loop-invariant.

At the moment, it must always be loop-invariant, right? Because instructions with struct-typed return values are not supported. Can't we just mark it as uniform?

That's a good point, that works indeed.

[david-arm]

llvm/test/Transforms/LoopVectorize/AArch64/extractvalue-no-scalarization-required.ll

16

Is it worth also adding a CHECK for the debug output showing extractvalue as a uniform instruction?

[sdesmalen]

Added check for debug-output showing extractvalue as uniform.

[david-arm]

LGTM!

[fhahn]

LGTM, thanks

[sdesmalen]

Cheers for the review comments!

llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

for (unsigned Part = 0; Part < UF; ++Part) {
       addMetadata(Cast, &I);
     }
     break;
   }
   default:
     // This instruction is not vectorized by simple widening.
     LLVM_DEBUG(dbgs() << "LV: Found an unhandled instruction: " << I);
     llvm_unreachable("Unhandled instruction!");
   } // end of switch.
 }
 
 void InnerLoopVectorizer::widenCallInstruction(CallInst &I, VPValue *Def,
                                                VPUser &ArgOperands,
                                                VPTransformState &State) {
   assert(!isa<DbgInfoIntrinsic>(I) &&
          "DbgInfoIntrinsic should have been dropped during VPlan construction");
   setDebugLocFromInst(&I);
 
   Module *M = I.getParent()->getParent()->getParent();
   auto *CI = cast<CallInst>(&I);
for (auto &I : *BB) {
         case Intrinsic::lifetime_end:
           if (TheLoop->hasLoopInvariantOperands(&I))
             addToWorklistIfAllowed(&I);
         default:
           break;
         }
       }
 
+      // ExtractValue instructions must be uniform, because the operands are
+      // known to be loop-invariant.
+      if (auto *EVI = dyn_cast<ExtractValueInst>(&I)) {
+        assert(isOutOfScope(EVI->getAggregateOperand()) &&
+               "Expected aggregate value to be loop invariant");
+        addToWorklistIfAllowed(EVI);
+        continue;
+      }
+
       // If there's no pointer operand, there's nothing to do.
       auto *Ptr = getLoadStorePointerOperand(&I);
       if (!Ptr)
         continue;
 
       // A uniform memory op is itself uniform.  We exclude uniform stores
       // here as they demand the last lane, not the first one.
       if (isa<LoadInst>(I) && Legal->isUniformMemOp(I))

llvm/test/Transforms/LoopVectorize/AArch64/extractvalue-no-scalarization-required.ll

 ; REQUIRES: asserts
 
 ; RUN: opt -loop-vectorize -mtriple=arm64-apple-ios %s -S -debug -disable-output 2>&1 | FileCheck --check-prefix=CM %s
 ; RUN: opt -loop-vectorize -force-vector-width=2 -force-vector-interleave=1 %s -S | FileCheck --check-prefix=FORCED %s
 
 ; Test case from PR41294.
 
 ; Check scalar cost for extractvalue. The constant and loop invariant operands are free,
 ; leaving cost 3 for scalarizing the result + 2 for executing the op with VF 2.
 
+; CM: LV: Found uniform instruction:   %a = extractvalue { i64, i64 } %sv, 0
+; CM: LV: Found uniform instruction:   %b = extractvalue { i64, i64 } %sv, 1
+
 ; CM: LV: Scalar loop costs: 5.
 ; CM: LV: Found an estimated cost of 0 for VF 2 For instruction:   %a = extractvalue { i64, i64 } %sv, 0
 ; CM-NEXT: LV: Found an estimated cost of 0 for VF 2 For instruction:   %b = extractvalue { i64, i64 } %sv, 1
 
 ; Check that the extractvalue operands are actually free in vector code.
 
 ; FORCED-LABEL: vector.body:                                      ; preds = %vector.body, %vector.ph
 ; FORCED-NEXT:    %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
 ; FORCED-NEXT:    %0 = add i32 %index, 0
 ; FORCED-NEXT:    %1 = extractvalue { i64, i64 } %sv, 0
-; FORCED-NEXT:    %2 = extractvalue { i64, i64 } %sv, 0
-; FORCED-NEXT:    %3 = insertelement <2 x i64> poison, i64 %1, i32 0
-; FORCED-NEXT:    %4 = insertelement <2 x i64> %3, i64 %2, i32 1
-; FORCED-NEXT:    %5 = extractvalue { i64, i64 } %sv, 1
-; FORCED-NEXT:    %6 = extractvalue { i64, i64 } %sv, 1
-; FORCED-NEXT:    %7 = insertelement <2 x i64> poison, i64 %5, i32 0
-; FORCED-NEXT:    %8 = insertelement <2 x i64> %7, i64 %6, i32 1
-; FORCED-NEXT:    %9 = getelementptr i64, i64* %dst, i32 %0
-; FORCED-NEXT:    %10 = add <2 x i64> %4, %8
-; FORCED-NEXT:    %11 = getelementptr i64, i64* %9, i32 0
-; FORCED-NEXT:    %12 = bitcast i64* %11 to <2 x i64>*
-; FORCED-NEXT:    store <2 x i64> %10, <2 x i64>* %12, align 4
+; FORCED-NEXT:    %broadcast.splatinsert = insertelement <2 x i64> poison, i64 %1, i32 0
+; FORCED-NEXT:    %broadcast.splat = shufflevector <2 x i64> %broadcast.splatinsert, <2 x i64> poison, <2 x i32> zeroinitializer
+; FORCED-NEXT:    %2 = extractvalue { i64, i64 } %sv, 1
+; FORCED-NEXT:    %broadcast.splatinsert1 = insertelement <2 x i64> poison, i64 %2, i32 0
+; FORCED-NEXT:    %broadcast.splat2 = shufflevector <2 x i64> %broadcast.splatinsert1, <2 x i64> poison, <2 x i32> zeroinitializer
+; FORCED-NEXT:    %3 = getelementptr i64, i64* %dst, i32 %0
+; FORCED-NEXT:    %4 = add <2 x i64> %broadcast.splat, %broadcast.splat2
+; FORCED-NEXT:    %5 = getelementptr i64, i64* %3, i32 0
+; FORCED-NEXT:    %6 = bitcast i64* %5 to <2 x i64>*
+; FORCED-NEXT:    store <2 x i64> %4, <2 x i64>* %6, align 4
 ; FORCED-NEXT:    %index.next = add nuw i32 %index, 2
-; FORCED-NEXT:    %13 = icmp eq i32 %index.next, 0
-; FORCED-NEXT:    br i1 %13, label %middle.block, label %vector.body, !llvm.loop !0
+; FORCED-NEXT:    %7 = icmp eq i32 %index.next, 0
+; FORCED-NEXT:    br i1 %7, label %middle.block, label %vector.body, !llvm.loop !0
 
 define void @test1(i64* %dst, {i64, i64} %sv) {
 entry:
   br label %loop.body
 
 loop.body:
   %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop.body ]
   %a = extractvalue { i64, i64 } %sv, 0
   %b = extractvalue { i64, i64 } %sv, 1
   %addr = getelementptr i64, i64* %dst, i32 %iv
   %add = add i64 %a, %b
   store i64 %add, i64* %addr
   %iv.next = add nsw i32 %iv, 1
   %cond = icmp ne i32 %iv.next, 0
   br i1 %cond, label %loop.body, label %exit
 
 exit:
   ret void
 }
 
 
 ; Similar to the test case above, but checks getVectorCallCost as well.
 declare float @pow(float, float) readnone nounwind
 
+; CM: LV: Found uniform instruction:   %a = extractvalue { float, float } %sv, 0
+; CM: LV: Found uniform instruction:   %b = extractvalue { float, float } %sv, 1
+
 ; CM: LV: Scalar loop costs: 14.
 ; CM: LV: Found an estimated cost of 0 for VF 2 For instruction:   %a = extractvalue { float, float } %sv, 0
 ; CM-NEXT: LV: Found an estimated cost of 0 for VF 2 For instruction:   %b = extractvalue { float, float } %sv, 1
 
 ; FORCED-LABEL: define void @test_getVectorCallCost
 
 ; FORCED-LABEL: vector.body:                                      ; preds = %vector.body, %vector.ph
 ; FORCED-NEXT:    %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
 ; FORCED-NEXT:    %0 = add i32 %index, 0
 ; FORCED-NEXT:    %1 = extractvalue { float, float } %sv, 0
-; FORCED-NEXT:    %2 = extractvalue { float, float } %sv, 0
-; FORCED-NEXT:    %3 = insertelement <2 x float> poison, float %1, i32 0
-; FORCED-NEXT:    %4 = insertelement <2 x float> %3, float %2, i32 1
-; FORCED-NEXT:    %5 = extractvalue { float, float } %sv, 1
-; FORCED-NEXT:    %6 = extractvalue { float, float } %sv, 1
-; FORCED-NEXT:    %7 = insertelement <2 x float> poison, float %5, i32 0
-; FORCED-NEXT:    %8 = insertelement <2 x float> %7, float %6, i32 1
-; FORCED-NEXT:    %9 = getelementptr float, float* %dst, i32 %0
-; FORCED-NEXT:    %10 = call <2 x float> @llvm.pow.v2f32(<2 x float> %4, <2 x float> %8)
-; FORCED-NEXT:    %11 = getelementptr float, float* %9, i32 0
-; FORCED-NEXT:    %12 = bitcast float* %11 to <2 x float>*
-; FORCED-NEXT:    store <2 x float> %10, <2 x float>* %12, align 4
+; FORCED-NEXT:    %broadcast.splatinsert = insertelement <2 x float> poison, float %1, i32 0
+; FORCED-NEXT:    %broadcast.splat = shufflevector <2 x float> %broadcast.splatinsert, <2 x float> poison, <2 x i32> zeroinitializer
+; FORCED-NEXT:    %2 = extractvalue { float, float } %sv, 1
+; FORCED-NEXT:    %broadcast.splatinsert1 = insertelement <2 x float> poison, float %2, i32 0
+; FORCED-NEXT:    %broadcast.splat2 = shufflevector <2 x float> %broadcast.splatinsert1, <2 x float> poison, <2 x i32> zeroinitializer
+; FORCED-NEXT:    %3 = getelementptr float, float* %dst, i32 %0
+; FORCED-NEXT:    %4 = call <2 x float> @llvm.pow.v2f32(<2 x float> %broadcast.splat, <2 x float> %broadcast.splat2)
+; FORCED-NEXT:    %5 = getelementptr float, float* %3, i32 0
+; FORCED-NEXT:    %6 = bitcast float* %5 to <2 x float>*
+; FORCED-NEXT:    store <2 x float> %4, <2 x float>* %6, align 4
 ; FORCED-NEXT:    %index.next = add nuw i32 %index, 2
-; FORCED-NEXT:    %13 = icmp eq i32 %index.next, 0
-; FORCED-NEXT:    br i1 %13, label %middle.block, label %vector.body, !llvm.loop !4
+; FORCED-NEXT:    %7 = icmp eq i32 %index.next, 0
+; FORCED-NEXT:    br i1 %7, label %middle.block, label %vector.body, !llvm.loop !4
 
 define void @test_getVectorCallCost(float* %dst, {float, float} %sv) {
 entry:
   br label %loop.body
 
 loop.body:
   %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop.body ]
   %a = extractvalue { float, float } %sv, 0

llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-extractvalue.ll

+; RUN: opt -S -loop-vectorize -scalable-vectorization=on < %s | FileCheck %s
+
+target triple = "aarch64-unknown-linux-gnu"
+
+define void @widen_extractvalue(i64* %dst, {i64, i64} %sv) #0 {
+; CHECK-LABEL: @widen_extractvalue(
+; CHECK: vector.body:
+; CHECK:        [[EXTRACT0:%.*]] = extractvalue { i64, i64 } [[SV:%.*]], 0
+; CHECK-NEXT:   [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[EXTRACT0]], i32 0
+; CHECK-NEXT:   [[DOTSPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[DOTSPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
+; CHECK-NEXT:   [[EXTRACT1:%.*]] = extractvalue { i64, i64 } [[SV]], 1
+; CHECK-NEXT:   [[DOTSPLATINSERT1:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[EXTRACT1]], i32 0
+; CHECK-NEXT:   [[DOTSPLAT2:%.*]] = shufflevector <vscale x 2 x i64> [[DOTSPLATINSERT1]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
+; CHECK:        [[ADD:%.*]] = add <vscale x 2 x i64> [[DOTSPLAT]], [[DOTSPLAT2]]
+entry:
+  br label %loop.body
+
+loop.body:
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop.body ]
+  %a = extractvalue { i64, i64 } %sv, 0
+  %b = extractvalue { i64, i64 } %sv, 1
+  %addr = getelementptr i64, i64* %dst, i32 %iv
+  %add = add i64 %a, %b
+  store i64 %add, i64* %addr
+  %iv.next = add nsw i32 %iv, 1
+  %cond = icmp ne i32 %iv.next, 0
+  br i1 %cond, label %loop.body, label %exit, !llvm.loop !0
+
+exit:
+  ret void
+}
+
+attributes #0 = { "target-features"="+sve" }
+
+!0 = distinct !{!0, !1, !2, !3, !4, !5}
+!1 = !{!"llvm.loop.mustprogress"}
+!2 = !{!"llvm.loop.vectorize.width", i32 2}
+!3 = !{!"llvm.loop.vectorize.scalable.enable", i1 true}
+!4 = !{!"llvm.loop.interleave.count", i32 1}
+!5 = !{!"llvm.loop.vectorize.enable", i1 true}
+