[CostModel] Unify Shuffle and InsertElement Costs

[samparker]

Summary

Extract the existing code from getInstructionThroughput into TTImpl::getUserCost. The duplicated code in the AMDGPU backend has also been removed.

[dfukalov]

llvm/test/CodeGen/SystemZ/splitMove_undefReg_mverifier.ll

213

If the change is refactoring, why it affects codegen test?

[samparker]

llvm/test/CodeGen/SystemZ/splitMove_undefReg_mverifier.ll

213

I think this instruction was just broken, the ShuffleVectorInst class hit an assert on it.

[dfukalov]

llvm/test/CodeGen/SystemZ/splitMove_undefReg_mverifier.ll

213

But how it was not asserted before the change? Why change in cost model can change ShuffleVectorInst behavior in codegen test?

[samparker]

llvm/test/CodeGen/SystemZ/splitMove_undefReg_mverifier.ll

213

Well the program flow has now completely changed. I would expect a pass run by llc has queried getUserCost which now queries the ShuffleCost where it wouldn't have before. It was only accessible when called directly or when a client queried getInstructionThroughput, which are really only things that the vectorizer does, and the vectorizer wouldn't have been running on this test.

llvm/include/llvm/Analysis/TargetTransformInfoImpl.h

case Instruction::Select: {
                                            CostKind, I);
     }
     case Instruction::ICmp:
     case Instruction::FCmp: {
       Type *ValTy = U->getOperand(0)->getType();
       return TargetTTI->getCmpSelInstrCost(Opcode, ValTy, U->getType(),
                                            CostKind, I);
     }
+    case Instruction::InsertElement: {
+      auto *IE = cast<InsertElementInst>(U);
+      auto *CI = dyn_cast<ConstantInt>(IE->getOperand(2));
+      unsigned Idx = CI ? CI->getZExtValue() : -1;
+      return TargetTTI->getVectorInstrCost(Opcode, Ty, Idx);
+    }
+    case Instruction::ShuffleVector: {
+      auto *Shuffle = cast<ShuffleVectorInst>(U);
+      auto *VecTy = cast<VectorType>(U->getType());
+      auto *VecSrcTy = cast<VectorType>(U->getOperand(0)->getType());
+
+      // TODO: Identify and add costs for insert subvector, etc.
+      int SubIndex;
+      if (Shuffle->isExtractSubvectorMask(SubIndex))
+        return TargetTTI->getShuffleCost(TTI::SK_ExtractSubvector, VecSrcTy,
+                                         SubIndex, VecTy);
+      else if (Shuffle->changesLength())
+        return CostKind == TTI::TCK_RecipThroughput ? -1 : 1;
+      else if (Shuffle->isIdentity())
+        return 0;
+      else if (Shuffle->isReverse())
+        return TargetTTI->getShuffleCost(TTI::SK_Reverse, VecTy, 0, nullptr);
+      else if (Shuffle->isSelect())
+        return TargetTTI->getShuffleCost(TTI::SK_Select, VecTy, 0, nullptr);
+      else if (Shuffle->isTranspose())
+        return TargetTTI->getShuffleCost(TTI::SK_Transpose, VecTy, 0, nullptr);
+      else if (Shuffle->isZeroEltSplat())
+        return TargetTTI->getShuffleCost(TTI::SK_Broadcast, VecTy, 0, nullptr);
+      else if (Shuffle->isSingleSource())
+        return TargetTTI->getShuffleCost(TTI::SK_PermuteSingleSrc, VecTy, 0,
+                                         nullptr);
+
+      return TargetTTI->getShuffleCost(TTI::SK_PermuteTwoSrc, VecTy, 0,
+                                       nullptr);
+    }
     }
     // By default, just classify everything as 'basic'.
     return TTI::TCC_Basic;
   }
 
   int getInstructionLatency(const Instruction *I) {
     SmallVector<const Value *, 4> Operands(I->value_op_begin(),
                                            I->value_op_end());

llvm/lib/Analysis/TargetTransformInfo.cpp

case RK_UnsignedMinMax:
           /*IsPairwiseForm=*/true, /*IsUnsigned=*/true);
     case RK_None:
       break;
     }
 
     return getVectorInstrCost(I->getOpcode(), EEI->getOperand(0)->getType(),
                               Idx);
   }
-  case Instruction::InsertElement: {
-    const InsertElementInst *IE = cast<InsertElementInst>(I);
-    ConstantInt *CI = dyn_cast<ConstantInt>(IE->getOperand(2));
-    unsigned Idx = -1;
-    if (CI)
-      Idx = CI->getZExtValue();
-    return getVectorInstrCost(I->getOpcode(), IE->getType(), Idx);
-  }
+  case Instruction::InsertElement:
   case Instruction::ExtractValue:
-    return 0; // Model all ExtractValue nodes as free.
-  case Instruction::ShuffleVector: {
-    const ShuffleVectorInst *Shuffle = cast<ShuffleVectorInst>(I);
-    auto *Ty = cast<VectorType>(Shuffle->getType());
-    auto *SrcTy = cast<VectorType>(Shuffle->getOperand(0)->getType());
-
-    // TODO: Identify and add costs for insert subvector, etc.
-    int SubIndex;
-    if (Shuffle->isExtractSubvectorMask(SubIndex))
-      return TTIImpl->getShuffleCost(SK_ExtractSubvector, SrcTy, SubIndex, Ty);
-
-    if (Shuffle->changesLength())
-      return -1;
-
-    if (Shuffle->isIdentity())
-      return 0;
-
-    if (Shuffle->isReverse())
-      return TTIImpl->getShuffleCost(SK_Reverse, Ty, 0, nullptr);
-
-    if (Shuffle->isSelect())
-      return TTIImpl->getShuffleCost(SK_Select, Ty, 0, nullptr);
-
-    if (Shuffle->isTranspose())
-      return TTIImpl->getShuffleCost(SK_Transpose, Ty, 0, nullptr);
-
-    if (Shuffle->isZeroEltSplat())
-      return TTIImpl->getShuffleCost(SK_Broadcast, Ty, 0, nullptr);
-
-    if (Shuffle->isSingleSource())
-      return TTIImpl->getShuffleCost(SK_PermuteSingleSrc, Ty, 0, nullptr);
-
-    return TTIImpl->getShuffleCost(SK_PermuteTwoSrc, Ty, 0, nullptr);
-  }
+  case Instruction::ShuffleVector:
+    return getUserCost(I, CostKind);
   case Instruction::Call:
     return getUserCost(I, CostKind);
   default:
     // We don't have any information on this instruction.
     return -1;
   }
 }
 

llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp

GCNTTIImpl::getUserCost(const User *U, ArrayRef<const Value *> Operands,
   switch (I->getOpcode()) {
   case Instruction::ExtractElement: {
     ConstantInt *CI = dyn_cast<ConstantInt>(I->getOperand(1));
     unsigned Idx = -1;
     if (CI)
       Idx = CI->getZExtValue();
     return getVectorInstrCost(I->getOpcode(), I->getOperand(0)->getType(), Idx);
   }
-  case Instruction::InsertElement: {
-    ConstantInt *CI = dyn_cast<ConstantInt>(I->getOperand(2));
-    unsigned Idx = -1;
-    if (CI)
-      Idx = CI->getZExtValue();
-    return getVectorInstrCost(I->getOpcode(), I->getType(), Idx);
-  }
-  case Instruction::ShuffleVector: {
-    const ShuffleVectorInst *Shuffle = cast<ShuffleVectorInst>(I);
-    auto *Ty = cast<VectorType>(Shuffle->getType());
-    auto *SrcTy = cast<VectorType>(Shuffle->getOperand(0)->getType());
-
-    // TODO: Identify and add costs for insert subvector, etc.
-    int SubIndex;
-    if (Shuffle->isExtractSubvectorMask(SubIndex))
-      return getShuffleCost(TTI::SK_ExtractSubvector, SrcTy, SubIndex, Ty);
-
-    if (Shuffle->changesLength())
-      return BaseT::getUserCost(U, Operands, CostKind);
-
-    if (Shuffle->isIdentity())
-      return 0;
-
-    if (Shuffle->isReverse())
-      return getShuffleCost(TTI::SK_Reverse, Ty, 0, nullptr);
-
-    if (Shuffle->isSelect())
-      return getShuffleCost(TTI::SK_Select, Ty, 0, nullptr);
-
-    if (Shuffle->isTranspose())
-      return getShuffleCost(TTI::SK_Transpose, Ty, 0, nullptr);
-
-    if (Shuffle->isZeroEltSplat())
-      return getShuffleCost(TTI::SK_Broadcast, Ty, 0, nullptr);
-
-    if (Shuffle->isSingleSource())
-      return getShuffleCost(TTI::SK_PermuteSingleSrc, Ty, 0, nullptr);
-
-    return getShuffleCost(TTI::SK_PermuteTwoSrc, Ty, 0, nullptr);
-  }
   case Instruction::FNeg:
     return getArithmeticInstrCost(I->getOpcode(), I->getType(), CostKind,
                                   TTI::OK_AnyValue, TTI::OK_AnyValue,
                                   TTI::OP_None, TTI::OP_None, Operands, I);
   default:
     break;
   }
 

llvm/test/CodeGen/SystemZ/splitMove_undefReg_mverifier.ll

CF240:                                            ; preds = %CF240, %CF260, %CF
   %Sl109 = select i1 %Cmp, i16 27357, i16 %Sl102
   %Cmp110 = icmp sge <4 x i16> %B9, zeroinitializer
   %L111 = load i64, i64* %Sl
   store i8 %L57, i8* %0
   %E112 = extractelement <2 x i1> %Shuff98, i32 0
   br i1 %E112, label %CF240, label %CF254
 
 CF254:                                            ; preds = %CF254, %CF267, %CF264, %CF240
-  %Shuff113 = shufflevector <2 x i32> %I68, <2 x i32> zeroinitializer, <2 x i32> undef
+  %Shuff113 = shufflevector <2 x i32> %I68, <2 x i32> zeroinitializer, <2 x i32><i32 undef, i32 0>
   %I114 = insertelement <4 x i16> zeroinitializer, i16 27357, i32 3
   %B115 = and i16 %Sl102, %Sl11
   %FC116 = uitofp i16 %B115 to double
   %Sl117 = select i1 %L81, i32* %1, i32* %1
   %Cmp118 = icmp ne i64 %Sl94, %L50
   br i1 %Cmp118, label %CF254, label %CF267
 
 CF267:                                            ; preds = %CF254