[LV] Consider ExtractValue as uniform.

[sdesmalen]

Summary

Since all operands to ExtractValue must be loop-invariant when we deem
the loop vectorizable, we can consider ExtractValue to be uniform.

[dmgreen]

llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

4963

This doesn't appear to be used.

[david-arm]

llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

4966

I think we can probably hoist this out of the for loop and just create it once.

llvm/test/Transforms/LoopVectorize/AArch64/extractvalue-no-scalarization-required.ll

4

Could you add at least one test for an interleave > 1 either here or in the SVE test, just to prove we've got the parts right?

[sdesmalen]

• Removed unused variable.
• Hoisted ExtractValue and Splat out of the loop.
• Updated test.

[sdesmalen]

llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

4963

I somehow didn't notice that before posting the patch. Thanks for pointing out!

4966

Good point, I've made the change.

[fhahn]

since it's operands must always be scalar or loop-invariant.

At the moment, it must always be loop-invariant, right? Because instructions with struct-typed return values are not supported. Can't we just mark it as uniform?

[sdesmalen]

Mark ExtractValue as uniform.

[sdesmalen]

In D107286#2924610, @fhahn wrote:

since it's operands must always be scalar or loop-invariant.

At the moment, it must always be loop-invariant, right? Because instructions with struct-typed return values are not supported. Can't we just mark it as uniform?

That's a good point, that works indeed.

[david-arm]

llvm/test/Transforms/LoopVectorize/AArch64/extractvalue-no-scalarization-required.ll

13

Is it worth also adding a CHECK for the debug output showing extractvalue as a uniform instruction?

[sdesmalen]

Added check for debug-output showing extractvalue as uniform.

[david-arm]

LGTM!

[fhahn]

LGTM, thanks

[sdesmalen]

Cheers for the review comments!

llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

case Instruction::BitCast: {
     for (unsigned Part = 0; Part < UF; ++Part) {
       Value *A = State.get(User.getOperand(0), Part);
       Value *Cast = Builder.CreateCast(CI->getOpcode(), A, DestTy);
       State.set(Def, Cast, Part);
       addMetadata(Cast, &I);
     }
     break;
   }
+  case Instruction::ExtractValue: {
+    auto *EVI = cast<ExtractValueInst>(&I);
+    setDebugLocFromInst(EVI);
+
+    Value *Val = Builder.CreateExtractValue(EVI->getAggregateOperand(),
+                                            EVI->getIndices());
+    Value *VecVal = Builder.CreateVectorSplat(VF, Val);
+    for (unsigned Part = 0; Part < UF; ++Part) {
+      State.set(Def, VecVal, Part);
+      addMetadata(VecVal, &I);
+    }
+    break;
+  }
   default:
     // This instruction is not vectorized by simple widening.
     LLVM_DEBUG(dbgs() << "LV: Found an unhandled instruction: " << I);
     llvm_unreachable("Unhandled instruction!");
   } // end of switch.
 }
 
 void InnerLoopVectorizer::widenCallInstruction(CallInst &I, VPValue *Def,

 VPWidenRecipe *VPRecipeBuilder::tryToWiden(Instruction *I,
                                            ArrayRef<VPValue *> Operands) const {
   auto IsVectorizableOpcode = [](unsigned Opcode) {
     switch (Opcode) {
     case Instruction::Add:
     case Instruction::And:
     case Instruction::AShr:
     case Instruction::BitCast:
+    case Instruction::ExtractValue:
     case Instruction::FAdd:
     case Instruction::FCmp:
     case Instruction::FDiv:
     case Instruction::FMul:
     case Instruction::FNeg:
     case Instruction::FPExt:
     case Instruction::FPToSI:
     case Instruction::FPToUI:

llvm/test/Transforms/LoopVectorize/AArch64/extractvalue-no-scalarization-required.ll

 ; REQUIRES: asserts
 
 ; RUN: opt -loop-vectorize -mtriple=arm64-apple-ios %s -S -debug -disable-output 2>&1 | FileCheck --check-prefix=CM %s
 ; RUN: opt -loop-vectorize -force-vector-width=2 -force-vector-interleave=1 %s -S | FileCheck --check-prefix=FORCED %s
 
 ; Test case from PR41294.
 
 ; Check scalar cost for extractvalue. The constant and loop invariant operands are free,
 ; leaving cost 3 for scalarizing the result + 2 for executing the op with VF 2.
 
 ; CM: LV: Scalar loop costs: 5.
 ; CM: LV: Found an estimated cost of 0 for VF 2 For instruction:   %a = extractvalue { i64, i64 } %sv, 0
 ; CM-NEXT: LV: Found an estimated cost of 0 for VF 2 For instruction:   %b = extractvalue { i64, i64 } %sv, 1
 
 ; Check that the extractvalue operands are actually free in vector code.
 
 ; FORCED-LABEL: vector.body:                                      ; preds = %vector.body, %vector.ph
 ; FORCED-NEXT:    %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
 ; FORCED-NEXT:    %0 = add i32 %index, 0
 ; FORCED-NEXT:    %1 = extractvalue { i64, i64 } %sv, 0
-; FORCED-NEXT:    %2 = extractvalue { i64, i64 } %sv, 0
-; FORCED-NEXT:    %3 = insertelement <2 x i64> poison, i64 %1, i32 0
-; FORCED-NEXT:    %4 = insertelement <2 x i64> %3, i64 %2, i32 1
-; FORCED-NEXT:    %5 = extractvalue { i64, i64 } %sv, 1
-; FORCED-NEXT:    %6 = extractvalue { i64, i64 } %sv, 1
-; FORCED-NEXT:    %7 = insertelement <2 x i64> poison, i64 %5, i32 0
-; FORCED-NEXT:    %8 = insertelement <2 x i64> %7, i64 %6, i32 1
-; FORCED-NEXT:    %9 = getelementptr i64, i64* %dst, i32 %0
-; FORCED-NEXT:    %10 = add <2 x i64> %4, %8
-; FORCED-NEXT:    %11 = getelementptr i64, i64* %9, i32 0
-; FORCED-NEXT:    %12 = bitcast i64* %11 to <2 x i64>*
-; FORCED-NEXT:    store <2 x i64> %10, <2 x i64>* %12, align 4
+; FORCED-NEXT:    %.splatinsert = insertelement <2 x i64> poison, i64 %1, i32 0
+; FORCED-NEXT:    %.splat = shufflevector <2 x i64> %.splatinsert, <2 x i64> poison, <2 x i32> zeroinitializer
+; FORCED-NEXT:    %2 = extractvalue { i64, i64 } %sv, 1
+; FORCED-NEXT:    %.splatinsert1 = insertelement <2 x i64> poison, i64 %2, i32 0
+; FORCED-NEXT:    %.splat2 = shufflevector <2 x i64> %.splatinsert1, <2 x i64> poison, <2 x i32> zeroinitializer
+; FORCED-NEXT:    %3 = getelementptr i64, i64* %dst, i32 %0
+; FORCED-NEXT:    %4 = add <2 x i64> %.splat, %.splat2
+; FORCED-NEXT:    %5 = getelementptr i64, i64* %3, i32 0
+; FORCED-NEXT:    %6 = bitcast i64* %5 to <2 x i64>*
+; FORCED-NEXT:    store <2 x i64> %4, <2 x i64>* %6, align 4
 ; FORCED-NEXT:    %index.next = add nuw i32 %index, 2
-; FORCED-NEXT:    %13 = icmp eq i32 %index.next, 0
-; FORCED-NEXT:    br i1 %13, label %middle.block, label %vector.body, !llvm.loop !0
+; FORCED-NEXT:    %7 = icmp eq i32 %index.next, 0
+; FORCED-NEXT:    br i1 %7, label %middle.block, label %vector.body, !llvm.loop !0
 
 define void @test1(i64* %dst, {i64, i64} %sv) {
 entry:
   br label %loop.body
 
 loop.body:
   %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop.body ]
   %a = extractvalue { i64, i64 } %sv, 0

 ; CM-NEXT: LV: Found an estimated cost of 0 for VF 2 For instruction:   %b = extractvalue { float, float } %sv, 1
 
 ; FORCED-LABEL: define void @test_getVectorCallCost
 
 ; FORCED-LABEL: vector.body:                                      ; preds = %vector.body, %vector.ph
 ; FORCED-NEXT:    %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
 ; FORCED-NEXT:    %0 = add i32 %index, 0
 ; FORCED-NEXT:    %1 = extractvalue { float, float } %sv, 0
-; FORCED-NEXT:    %2 = extractvalue { float, float } %sv, 0
-; FORCED-NEXT:    %3 = insertelement <2 x float> poison, float %1, i32 0
-; FORCED-NEXT:    %4 = insertelement <2 x float> %3, float %2, i32 1
-; FORCED-NEXT:    %5 = extractvalue { float, float } %sv, 1
-; FORCED-NEXT:    %6 = extractvalue { float, float } %sv, 1
-; FORCED-NEXT:    %7 = insertelement <2 x float> poison, float %5, i32 0
-; FORCED-NEXT:    %8 = insertelement <2 x float> %7, float %6, i32 1
-; FORCED-NEXT:    %9 = getelementptr float, float* %dst, i32 %0
-; FORCED-NEXT:    %10 = call <2 x float> @llvm.pow.v2f32(<2 x float> %4, <2 x float> %8)
-; FORCED-NEXT:    %11 = getelementptr float, float* %9, i32 0
-; FORCED-NEXT:    %12 = bitcast float* %11 to <2 x float>*
-; FORCED-NEXT:    store <2 x float> %10, <2 x float>* %12, align 4
+; FORCED-NEXT:    %.splatinsert = insertelement <2 x float> poison, float %1, i32 0
+; FORCED-NEXT:    %.splat = shufflevector <2 x float> %.splatinsert, <2 x float> poison, <2 x i32> zeroinitializer
+; FORCED-NEXT:    %2 = extractvalue { float, float } %sv, 1
+; FORCED-NEXT:    %.splatinsert1 = insertelement <2 x float> poison, float %2, i32 0
+; FORCED-NEXT:    %.splat2 = shufflevector <2 x float> %.splatinsert1, <2 x float> poison, <2 x i32> zeroinitializer
+; FORCED-NEXT:    %3 = getelementptr float, float* %dst, i32 %0
+; FORCED-NEXT:    %4 = call <2 x float> @llvm.pow.v2f32(<2 x float> %.splat, <2 x float> %.splat2)
+; FORCED-NEXT:    %5 = getelementptr float, float* %3, i32 0
+; FORCED-NEXT:    %6 = bitcast float* %5 to <2 x float>*
+; FORCED-NEXT:    store <2 x float> %4, <2 x float>* %6, align 4
 ; FORCED-NEXT:    %index.next = add nuw i32 %index, 2
-; FORCED-NEXT:    %13 = icmp eq i32 %index.next, 0
-; FORCED-NEXT:    br i1 %13, label %middle.block, label %vector.body, !llvm.loop !4
+; FORCED-NEXT:    %7 = icmp eq i32 %index.next, 0
+; FORCED-NEXT:    br i1 %7, label %middle.block, label %vector.body, !llvm.loop !4
 
 define void @test_getVectorCallCost(float* %dst, {float, float} %sv) {
 entry:
   br label %loop.body
 
 loop.body:
   %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop.body ]
   %a = extractvalue { float, float } %sv, 0

llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-extractvalue.ll

+; RUN: opt -S -loop-vectorize -scalable-vectorization=on < %s | FileCheck %s
+
+target triple = "aarch64-unknown-linux-gnu"
+
+define void @widen_extractvalue(i64* %dst, {i64, i64} %sv) #0 {
+; CHECK-LABEL: @widen_extractvalue(
+; CHECK: vector.body:
+; CHECK:        [[EXTRACT0:%.*]] = extractvalue { i64, i64 } [[SV:%.*]], 0
+; CHECK-NEXT:   [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[EXTRACT0]], i32 0
+; CHECK-NEXT:   [[DOTSPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[DOTSPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
+; CHECK-NEXT:   [[EXTRACT1:%.*]] = extractvalue { i64, i64 } [[SV]], 1
+; CHECK-NEXT:   [[DOTSPLATINSERT1:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[EXTRACT1]], i32 0
+; CHECK-NEXT:   [[DOTSPLAT2:%.*]] = shufflevector <vscale x 2 x i64> [[DOTSPLATINSERT1]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
+; CHECK:        [[ADD:%.*]] = add <vscale x 2 x i64> [[DOTSPLAT]], [[DOTSPLAT2]]
+entry:
+  br label %loop.body
+
+loop.body:
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop.body ]
+  %a = extractvalue { i64, i64 } %sv, 0
+  %b = extractvalue { i64, i64 } %sv, 1
+  %addr = getelementptr i64, i64* %dst, i32 %iv
+  %add = add i64 %a, %b
+  store i64 %add, i64* %addr
+  %iv.next = add nsw i32 %iv, 1
+  %cond = icmp ne i32 %iv.next, 0
+  br i1 %cond, label %loop.body, label %exit, !llvm.loop !0
+
+exit:
+  ret void
+}
+
+attributes #0 = { "target-features"="+sve" }
+
+!0 = distinct !{!0, !1, !2, !3, !4, !5}
+!1 = !{!"llvm.loop.mustprogress"}
+!2 = !{!"llvm.loop.vectorize.width", i32 2}
+!3 = !{!"llvm.loop.vectorize.scalable.enable", i1 true}
+!4 = !{!"llvm.loop.interleave.count", i32 2}
+!5 = !{!"llvm.loop.vectorize.enable", i1 true}
+