[PowerPC] vector cost model add cost to extract i1

[RolandF]

Summary

Try to avoid some unprofitable predication on PPC. Recognize in the cost model that computing on i1 values will require extra mask or compare operation.

[RolandF]

Oops C++ fail and missed some tests.

[shchenz]

llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp

738

I must miss something. I compile following cases:

define i1 @ext_ext_or_reduction_v4i1(<4 x i1> %z) {
  %z1 = extractelement <4 x i1> %z, i32 1
  ret i1 %z1
}

define i32 @ext_ext_or_reduction_v4i32(<4 x i32> %z) {
  %z1 = extractelement <4 x i32> %z, i32 1
  ret i32 %z1
}

Seems llc generates exactly same instructions for them at both pwr9 and pwr8. So don't understand why here we need extra cost for i1 types, the i1 type must have some kinds of users?

[RolandF]

llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp

738

You will see the extra overhead if you compile the provided test case. But looking at your example, notice that the value produced has 64 or 32 live bits - there are garbage bits in the results. The return performs no calculation, it just functions like a copy, but the user would still have to get rid of those bits. The uses will be scalar code - if we try to charge those with the overhead we will make the scalar loop cost more too, plus scalar code will already have the masking code there.

[shchenz]

I think the extra cost considering for i1 type looks reasonable.

llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp

738

I see. And pwr8 seems uses more rotate/clear instructions than pwr9 does as pwr9 has vextubrx. I guess we don't need to model that detail for scalar operations?

llvm/test/Transforms/LoopVectorize/PowerPC/predcost.ll

2

nit: I run the bugpoint and get following narrow down input:

target datalayout = "e-m:e-Fn32-i64:64-n32:64-S128-v256:256:256-v512:512:512"
target triple = "powerpc64le-unknown-linux-gnu"

define dso_local void @_tc(ptr nocapture noundef %aaa, i64 noundef %bbb) local_unnamed_addr {
entry:
  br label %for.body

for.cond.cleanup.loopexit:                        ; preds = %for.inc
  ret void

for.body:                                         ; preds = %for.inc, %entry
  %i.08 = phi i64 [ %inc, %for.inc ], [ 0, %entry ]
  %arrayidx = getelementptr inbounds i8, ptr %aaa, i64 %i.08
  %0 = load i8, ptr %arrayidx, align 1
  %cmp1 = icmp eq i8 %0, 0
  br i1 %cmp1, label %if.then, label %for.inc

if.then:                                          ; preds = %for.body
  store i8 32, ptr %arrayidx, align 1
  br label %for.inc

for.inc:                                          ; preds = %if.then, %for.body
  %inc = add nuw nsw i64 %i.08, 1
  %exitcond.not = icmp eq i64 %inc, %bbb
  br i1 %exitcond.not, label %for.cond.cleanup.loopexit, label %for.body
}

[RolandF]

llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp

738

For pwr8 there might be 1 added instruction or there might be 2. Uses in arithmetic/logical operations should only need 1. So I could use either 1 or 2 and 1 is enough to prevent the unprofitable case.

llvm/test/Transforms/LoopVectorize/PowerPC/predcost.ll

2

thanks, will update on commit

llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp

 #include "llvm/Transforms/InstCombine/InstCombiner.h"
 #include "llvm/Transforms/Utils/Local.h"
 #include <optional>
 
 using namespace llvm;
 
 #define DEBUG_TYPE "ppctti"
 
+static cl::opt<bool> VecMaskCost("ppc-vec-mask-cost",
+cl::desc("add masking cost for i1 vectors"), cl::init(true), cl::Hidden);
+
 static cl::opt<bool> DisablePPCConstHoist("disable-ppc-constant-hoisting",
 cl::desc("disable constant hoisting on PPC"), cl::init(false), cl::Hidden);
 
 static cl::opt<bool>
 EnablePPCColdCC("ppc-enable-coldcc", cl::Hidden, cl::init(false),
                 cl::desc("Enable using coldcc calling conv for cold "
                          "internal functions"));
 
if (ST->hasVSX() && Val->getScalarType()->isDoubleTy()) {
     // Double-precision scalars are already located in index #0 (or #1 if LE).
     if (ISD == ISD::EXTRACT_VECTOR_ELT &&
         Index == (ST->isLittleEndian() ? 1 : 0))
       return 0;
 
     return Cost;
 
   } else if (Val->getScalarType()->isIntegerTy() && Index != -1U) {
+    unsigned EltSize = Val->getScalarSizeInBits();
+    // Computing on 1 bit values requires extra mask or compare operations.
+    unsigned MaskCost = VecMaskCost && EltSize == 1 ? 1 : 0;
     if (ST->hasP9Altivec()) {
       if (ISD == ISD::INSERT_VECTOR_ELT)
         // A move-to VSR and a permute/insert.  Assume vector operation cost
         // for both (cost will be 2x on P9).
         return 2 * CostFactor;
 
       // It's an extract.  Maybe we can do a cheap move-from VSR.
       unsigned EltSize = Val->getScalarSizeInBits();
       if (EltSize == 64) {
         unsigned MfvsrdIndex = ST->isLittleEndian() ? 1 : 0;
         if (Index == MfvsrdIndex)
           return 1;
       } else if (EltSize == 32) {
         unsigned MfvsrwzIndex = ST->isLittleEndian() ? 2 : 1;
         if (Index == MfvsrwzIndex)
           return 1;
       }
 
       // We need a vector extract (or mfvsrld).  Assume vector operation cost.
       // The cost of the load constant for a vector extract is disregarded
       // (invariant, easily schedulable).
-      return CostFactor;
+      return CostFactor + MaskCost;
 
-    } else if (ST->hasDirectMove())
+    } else if (ST->hasDirectMove()) {
       // Assume permute has standard cost.
       // Assume move-to/move-from VSR have 2x standard cost.
+      if (ISD == ISD::INSERT_VECTOR_ELT)
         return 3;
+      return 3 + MaskCost;
+    }
   }
 
   // Estimated cost of a load-hit-store delay.  This was obtained
   // experimentally as a minimum needed to prevent unprofitable
   // vectorization for the paq8p benchmark.  It may need to be
   // raised further if other unprofitable cases remain.
   unsigned LHSPenalty = 2;
   if (ISD == ISD::INSERT_VECTOR_ELT)

llvm/test/Analysis/CostModel/PowerPC/reduce-and.ll

 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
 ; RUN: opt < %s -mtriple=powerpc64-unknown-linux-gnu -mcpu=pwr8 -passes="print<cost-model>" -cost-kind=throughput 2>&1 -disable-output | FileCheck %s
 
 define i32 @reduce_i1(i32 %arg) {
 ; CHECK-LABEL: 'reduce_i1'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1 = call i1 @llvm.vector.reduce.and.v1i1(<1 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V1 = call i1 @llvm.vector.reduce.and.v1i1(<1 x i1> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i1 @llvm.vector.reduce.and.v2i1(<2 x i1> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4 = call i1 @llvm.vector.reduce.and.v4i1(<4 x i1> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8 = call i1 @llvm.vector.reduce.and.v8i1(<8 x i1> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16 = call i1 @llvm.vector.reduce.and.v16i1(<16 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 97 for instruction: %V32 = call i1 @llvm.vector.reduce.and.v32i1(<32 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 193 for instruction: %V64 = call i1 @llvm.vector.reduce.and.v64i1(<64 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 386 for instruction: %V128 = call i1 @llvm.vector.reduce.and.v128i1(<128 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 129 for instruction: %V32 = call i1 @llvm.vector.reduce.and.v32i1(<32 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 257 for instruction: %V64 = call i1 @llvm.vector.reduce.and.v64i1(<64 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 514 for instruction: %V128 = call i1 @llvm.vector.reduce.and.v128i1(<128 x i1> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
   %V1   = call i1 @llvm.vector.reduce.and.v1i1(<1 x i1> undef)
   %V2   = call i1 @llvm.vector.reduce.and.v2i1(<2 x i1> undef)
   %V4   = call i1 @llvm.vector.reduce.and.v4i1(<4 x i1> undef)
   %V8   = call i1 @llvm.vector.reduce.and.v8i1(<8 x i1> undef)
   %V16  = call i1 @llvm.vector.reduce.and.v16i1(<16 x i1> undef)
   %V32  = call i1 @llvm.vector.reduce.and.v32i1(<32 x i1> undef)

llvm/test/Analysis/CostModel/PowerPC/reduce-or.ll

 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
 ; RUN: opt < %s -mtriple=powerpc64-unknown-linux-gnu -mcpu=pwr8 -passes="print<cost-model>" -cost-kind=throughput 2>&1 -disable-output | FileCheck %s
 
 define i32 @reduce_i1(i32 %arg) {
 ; CHECK-LABEL: 'reduce_i1'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V1 = call i1 @llvm.vector.reduce.or.v1i1(<1 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V1 = call i1 @llvm.vector.reduce.or.v1i1(<1 x i1> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2 = call i1 @llvm.vector.reduce.or.v2i1(<2 x i1> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4 = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8 = call i1 @llvm.vector.reduce.or.v8i1(<8 x i1> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16 = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 97 for instruction: %V32 = call i1 @llvm.vector.reduce.or.v32i1(<32 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 193 for instruction: %V64 = call i1 @llvm.vector.reduce.or.v64i1(<64 x i1> undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 386 for instruction: %V128 = call i1 @llvm.vector.reduce.or.v128i1(<128 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 129 for instruction: %V32 = call i1 @llvm.vector.reduce.or.v32i1(<32 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 257 for instruction: %V64 = call i1 @llvm.vector.reduce.or.v64i1(<64 x i1> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 514 for instruction: %V128 = call i1 @llvm.vector.reduce.or.v128i1(<128 x i1> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
   %V1   = call i1 @llvm.vector.reduce.or.v1i1(<1 x i1> undef)
   %V2   = call i1 @llvm.vector.reduce.or.v2i1(<2 x i1> undef)
   %V4   = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> undef)
   %V8   = call i1 @llvm.vector.reduce.or.v8i1(<8 x i1> undef)
   %V16  = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> undef)
   %V32  = call i1 @llvm.vector.reduce.or.v32i1(<32 x i1> undef)

llvm/test/Transforms/LoopVectorize/PowerPC/predcost.ll

+; RUN: opt -ppc-vec-mask-cost=true -aa-pipeline=basic-aa -mcpu=pwr8 -S -passes=loop-vectorize < %s | FileCheck %s
+
+target datalayout = "e-m:e-Fn32-i64:64-n32:64-S128-v256:256:256-v512:512:512"
+target triple = "powerpc64le-unknown-linux-gnu"
+
+define dso_local void @_tc(ptr nocapture noundef %aaa, i64 noundef %bbb) local_unnamed_addr {
+; CHECK-NOT: extractelement <16 x i1>
+entry:
+  br label %for.body
+
+for.cond.cleanup.loopexit:                        ; preds = %for.inc
+  ret void
+
+for.body:                                         ; preds = %for.inc, %entry
+  %i.08 = phi i64 [ %inc, %for.inc ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds i8, ptr %aaa, i64 %i.08
+  %0 = load i8, ptr %arrayidx, align 1
+  %cmp1 = icmp eq i8 %0, 0
+  br i1 %cmp1, label %if.then, label %for.inc
+
+if.then:                                          ; preds = %for.body
+  store i8 32, ptr %arrayidx, align 1
+  br label %for.inc
+
+for.inc:                                          ; preds = %if.then, %for.body
+  %inc = add nuw nsw i64 %i.08, 1
+  %exitcond.not = icmp eq i64 %inc, %bbb
+  br i1 %exitcond.not, label %for.cond.cleanup.loopexit, label %for.body
+}