[AArch64] Add getCFInstrCost, treat branches as free for throughput.

[fhahn]

Summary

D79164/2596da31740f changed getCFInstrCost to return 1 per default.
AArch64 did not have its own implementation, hence the throughput cost
of CFI instructions is overestimated. On most cores, most branches should
be predicated and essentially free throughput wise.

This restores a 9% performance regression on a SPEC2006 benchmark on
AArch64 with -O3 LTO & PGO.

This patch effectively restores pre 2596da31740f behavior for AArch64
and undoes the AArch64 test changes of the patch.

[samparker]

LGTM - and sorry for causing trouble.

llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h

public:
 
   int getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src,
                        TTI::TargetCostKind CostKind,
                        const Instruction *I = nullptr);
 
   int getExtractWithExtendCost(unsigned Opcode, Type *Dst, VectorType *VecTy,
                                unsigned Index);
 
+  unsigned getCFInstrCost(unsigned Opcode, TTI::TargetCostKind CostKind);
+
   int getVectorInstrCost(unsigned Opcode, Type *Val, unsigned Index);
 
   int getArithmeticInstrCost(
       unsigned Opcode, Type *Ty,
       TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput,
       TTI::OperandValueKind Opd1Info = TTI::OK_AnyValue,
       TTI::OperandValueKind Opd2Info = TTI::OK_AnyValue,
       TTI::OperandValueProperties Opd1PropInfo = TTI::OP_None,

llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp

case Instruction::ZExt:
     if (DstVT.getSizeInBits() != 64u || SrcVT.getSizeInBits() == 32u)
       return Cost;
   }
 
   // If we are unable to perform the extend for free, get the default cost.
   return Cost + getCastInstrCost(Opcode, Dst, Src, CostKind);
 }
 
+unsigned AArch64TTIImpl::getCFInstrCost(unsigned Opcode,
+                                        TTI::TargetCostKind CostKind) {
+  if (CostKind != TTI::TCK_RecipThroughput)
+    return Opcode == Instruction::PHI ? 0 : 1;
+  // Branches are assumed to be predicted.
+  return CostKind == TTI::TCK_RecipThroughput ? 0 : 1;
+}
+
 int AArch64TTIImpl::getVectorInstrCost(unsigned Opcode, Type *Val,
                                        unsigned Index) {
   assert(Val->isVectorTy() && "This must be a vector type");
 
   if (Index != -1U) {
     // Legalize the type.
     std::pair<int, MVT> LT = TLI->getTypeLegalizationCost(DL, Val);
 

llvm/test/Analysis/CostModel/AArch64/aggregates.ll

 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
 ; RUN: opt < %s -mtriple=aarch64-unknown-linux-gnu -cost-model -cost-kind=throughput -analyze | FileCheck %s --check-prefixes=ALL,THROUGHPUT
 ; RUN: opt < %s -mtriple=aarch64-unknown-linux-gnu -cost-model -cost-kind=latency -analyze | FileCheck %s --check-prefixes=ALL,LATENCY
 ; RUN: opt < %s -mtriple=aarch64-unknown-linux-gnu -cost-model -cost-kind=code-size -analyze | FileCheck %s --check-prefixes=ALL,CODESIZE
 
 define i32 @extract_first_i32({i32, i32} %agg) {
-; ALL-LABEL: 'extract_first_i32'
-; ALL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractvalue { i32, i32 } %agg, 0
-; ALL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 %r
+; THROUGHPUT-LABEL: 'extract_first_i32'
+; THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractvalue { i32, i32 } %agg, 0
+; THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %r
+;
+; LATENCY-LABEL: 'extract_first_i32'
+; LATENCY-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractvalue { i32, i32 } %agg, 0
+; LATENCY-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 %r
+;
+; CODESIZE-LABEL: 'extract_first_i32'
+; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractvalue { i32, i32 } %agg, 0
+; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 %r
 ;
   %r = extractvalue {i32, i32} %agg, 0
   ret i32 %r
 }
 
 define i32 @extract_second_i32({i32, i32} %agg) {
-; ALL-LABEL: 'extract_second_i32'
-; ALL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractvalue { i32, i32 } %agg, 1
-; ALL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 %r
+; THROUGHPUT-LABEL: 'extract_second_i32'
+; THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractvalue { i32, i32 } %agg, 1
+; THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %r
+;
+; LATENCY-LABEL: 'extract_second_i32'
+; LATENCY-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractvalue { i32, i32 } %agg, 1
+; LATENCY-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 %r
+;
+; CODESIZE-LABEL: 'extract_second_i32'
+; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractvalue { i32, i32 } %agg, 1
+; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 %r
 ;
   %r = extractvalue {i32, i32} %agg, 1
   ret i32 %r
 }
 
 define i32 @extract_i32({i32, i1} %agg) {
-; ALL-LABEL: 'extract_i32'
-; ALL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractvalue { i32, i1 } %agg, 0
-; ALL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 %r
+; THROUGHPUT-LABEL: 'extract_i32'
+; THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractvalue { i32, i1 } %agg, 0
+; THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %r
+;
+; LATENCY-LABEL: 'extract_i32'
+; LATENCY-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractvalue { i32, i1 } %agg, 0
+; LATENCY-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 %r
+;
+; CODESIZE-LABEL: 'extract_i32'
+; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractvalue { i32, i1 } %agg, 0
+; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 %r
 ;
   %r = extractvalue {i32, i1} %agg, 0
   ret i32 %r
 }
 
 define i1 @extract_i1({i32, i1} %agg) {
-; ALL-LABEL: 'extract_i1'
-; ALL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractvalue { i32, i1 } %agg, 1
-; ALL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i1 %r
+; THROUGHPUT-LABEL: 'extract_i1'
+; THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractvalue { i32, i1 } %agg, 1
+; THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i1 %r
+;
+; LATENCY-LABEL: 'extract_i1'
+; LATENCY-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractvalue { i32, i1 } %agg, 1
+; LATENCY-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i1 %r
+;
+; CODESIZE-LABEL: 'extract_i1'
+; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractvalue { i32, i1 } %agg, 1
+; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i1 %r
 ;
   %r = extractvalue {i32, i1} %agg, 1
   ret i1 %r
 }
 
 define float @extract_float({i32, float} %agg) {
-; ALL-LABEL: 'extract_float'
-; ALL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractvalue { i32, float } %agg, 1
-; ALL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret float %r
+; THROUGHPUT-LABEL: 'extract_float'
+; THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractvalue { i32, float } %agg, 1
+; THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret float %r
+;
+; LATENCY-LABEL: 'extract_float'
+; LATENCY-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractvalue { i32, float } %agg, 1
+; LATENCY-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret float %r
+;
+; CODESIZE-LABEL: 'extract_float'
+; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractvalue { i32, float } %agg, 1
+; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret float %r
 ;
   %r = extractvalue {i32, float} %agg, 1
   ret float %r
 }
 
 define [42 x i42] @extract_array({i32, [42 x i42]} %agg) {
-; ALL-LABEL: 'extract_array'
-; ALL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractvalue { i32, [42 x i42] } %agg, 1
-; ALL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret [42 x i42] %r
+; THROUGHPUT-LABEL: 'extract_array'
+; THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractvalue { i32, [42 x i42] } %agg, 1
+; THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret [42 x i42] %r
+;
+; LATENCY-LABEL: 'extract_array'
+; LATENCY-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractvalue { i32, [42 x i42] } %agg, 1
+; LATENCY-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret [42 x i42] %r
+;
+; CODESIZE-LABEL: 'extract_array'
+; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractvalue { i32, [42 x i42] } %agg, 1
+; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret [42 x i42] %r
 ;
   %r = extractvalue {i32, [42 x i42]} %agg, 1
   ret [42 x i42] %r
 }
 
 define <42 x i42> @extract_vector({i32, <42 x i42>} %agg) {
-; ALL-LABEL: 'extract_vector'
-; ALL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractvalue { i32, <42 x i42> } %agg, 1
-; ALL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <42 x i42> %r
+; THROUGHPUT-LABEL: 'extract_vector'
+; THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractvalue { i32, <42 x i42> } %agg, 1
+; THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <42 x i42> %r
+;
+; LATENCY-LABEL: 'extract_vector'
+; LATENCY-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractvalue { i32, <42 x i42> } %agg, 1
+; LATENCY-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <42 x i42> %r
+;
+; CODESIZE-LABEL: 'extract_vector'
+; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractvalue { i32, <42 x i42> } %agg, 1
+; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <42 x i42> %r
 ;
   %r = extractvalue {i32, <42 x i42>} %agg, 1
   ret <42 x i42> %r
 }
 
 %T1 = type { i32, float, <4 x i1> }
 
 define %T1 @extract_struct({i32, %T1} %agg) {
-; ALL-LABEL: 'extract_struct'
-; ALL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractvalue { i32, %T1 } %agg, 1
-; ALL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret %T1 %r
+; THROUGHPUT-LABEL: 'extract_struct'
+; THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractvalue { i32, %T1 } %agg, 1
+; THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret %T1 %r
+;
+; LATENCY-LABEL: 'extract_struct'
+; LATENCY-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractvalue { i32, %T1 } %agg, 1
+; LATENCY-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret %T1 %r
+;
+; CODESIZE-LABEL: 'extract_struct'
+; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r = extractvalue { i32, %T1 } %agg, 1
+; CODESIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret %T1 %r
 ;
   %r = extractvalue {i32, %T1} %agg, 1
   ret %T1 %r
 }

llvm/test/Analysis/CostModel/AArch64/cast.ll

 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 39 for instruction: %r242 = uitofp <16 x i8> undef to <16 x double>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 39 for instruction: %r243 = sitofp <16 x i8> undef to <16 x double>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 38 for instruction: %r244 = uitofp <16 x i16> undef to <16 x double>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 38 for instruction: %r245 = sitofp <16 x i16> undef to <16 x double>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 38 for instruction: %r246 = uitofp <16 x i16> undef to <16 x double>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 38 for instruction: %r247 = sitofp <16 x i16> undef to <16 x double>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r248 = uitofp <16 x i64> undef to <16 x double>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %r249 = sitofp <16 x i64> undef to <16 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
   %r0 = sext i1 undef to i8
   %r1 = zext i1 undef to i8
   %r2 = sext i1 undef to i16
   %r3 = zext i1 undef to i16
   %r4 = sext i1 undef to i32
   %r5 = zext i1 undef to i32
   %r6 = sext i1 undef to i64

 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i8 %r18, i8* undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i16 %r19, i16* undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i32 %r20, i32* undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i64 %r21, i64* undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i8 %r22, i8* undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i16 %r23, i16* undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i32 %r24, i32* undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i64 %r25, i64* undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 %r12
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %r12
 ;
   %r0 = sext i8 %a to i16
   %r1 = sext i8 %a to i32
   %r2 = sext i8 %a to i64
   %r3 = sext i16 %b to i32
   %r4 = sext i16 %b to i64
   %r5 = sext i32 %c to i64
   store i16 %r0, i16* undef

 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %a = bitcast i32 undef to i32
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %b = bitcast float undef to float
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %c = bitcast i32 undef to float
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %d = bitcast float undef to i32
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %e = bitcast i64 undef to double
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f = bitcast double undef to i64
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %g = bitcast half undef to i16
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %h = bitcast i16 undef to half
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
   %a = bitcast i32 undef to i32
   %b = bitcast float undef to float
   %c = bitcast i32 undef to float
   %d = bitcast float undef to i32
   %e = bitcast i64 undef to double
   %f = bitcast double undef to i64
   %g = bitcast half undef to i16

 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4 = sext <2 x i8> %loadv2i8 to <2 x i64>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v5 = zext <2 x i8> %loadv2i8 to <2 x i64>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v6 = sext <4 x i16> %loadv4i16 to <4 x i32>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v7 = zext <4 x i16> %loadv4i16 to <4 x i32>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8 = sext <2 x i16> %loadv2i16 to <2 x i64>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v9 = zext <2 x i16> %loadv2i16 to <2 x i64>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v10 = sext <2 x i32> %loadv2i32 to <2 x i64>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v11 = zext <2 x i32> %loadv2i32 to <2 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
   %loadi8 = load i8, i8* undef
   %loadi16 = load i16, i16* undef
   %loadi32 = load i32, i32* undef
   %loadv2i8 = load <2 x i8>, <2 x i8>* undef
   %loadv4i8 = load <4 x i8>, <4 x i8>* undef
   %loadv8i8 = load <8 x i8>, <8 x i8>* undef
   %loadv2i16 = load <2 x i16>, <2 x i16>* undef

 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r2 = trunc i64 undef to i32
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i32 %r2, i32* undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r3 = trunc i32 undef to i8
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i8 %r3, i8* undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r4 = trunc i32 undef to i16
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i16 %r4, i16* undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %r5 = trunc i16 undef to i8
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i8 %r5, i8* undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
   %r0 = trunc i64 undef to i8
   store i8 %r0, i8* undef
   %r1 = trunc i64 undef to i16
   store i16 %r1, i16* undef
   %r2 = trunc i64 undef to i32
   store i32 %r2, i32* undef
   %r3 = trunc i32 undef to i8
   store i8 %r3, i8* undef
   %r4 = trunc i32 undef to i16
   store i16 %r4, i16* undef
   %r5 = trunc i16 undef to i8
   store i8 %r5, i8* undef
   ret i32 undef
 }

llvm/test/Analysis/CostModel/AArch64/cmp.ll

 ; CHECK-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a5 = icmp ult <8 x i16> undef, undef
 ; CHECK-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a6 = icmp sge <4 x i32> undef, undef
 ; CHECK-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a7 = fcmp oge half undef, undef
 ; CHECK-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a8 = fcmp ogt float undef, undef
 ; CHECK-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a9 = fcmp ogt double undef, undef
 ; CHECK-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 29 for instruction: %a10 = fcmp olt <8 x half> undef, undef
 ; CHECK-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a11 = fcmp oge <4 x float> undef, undef
 ; CHECK-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a12 = fcmp oge <2 x double> undef, undef
-; CHECK-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+; CHECK-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; CHECK-SIZE-LABEL: 'cmps'
 ; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a0 = icmp slt i8 undef, undef
 ; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a1 = icmp ult i16 undef, undef
 ; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a2 = icmp sge i32 undef, undef
 ; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a3 = icmp ne i64 undef, undef
 ; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a4 = icmp slt <16 x i8> undef, undef
 ; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a5 = icmp ult <8 x i16> undef, undef

llvm/test/Analysis/CostModel/AArch64/select.ll

 ; CHECK-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v5 = select i1 undef, float undef, float undef
 ; CHECK-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v6 = select i1 undef, double undef, double undef
 ; CHECK-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v13b = select <16 x i1> undef, <16 x i16> undef, <16 x i16> undef
 ; CHECK-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v15b = select <8 x i1> undef, <8 x i32> undef, <8 x i32> undef
 ; CHECK-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v15c = select <16 x i1> undef, <16 x i32> undef, <16 x i32> undef
 ; CHECK-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v16a = select <4 x i1> undef, <4 x i64> undef, <4 x i64> undef
 ; CHECK-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %v16b = select <8 x i1> undef, <8 x i64> undef, <8 x i64> undef
 ; CHECK-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %v16c = select <16 x i1> undef, <16 x i64> undef, <16 x i64> undef
-; CHECK-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+; CHECK-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; CHECK-SIZE-LABEL: 'select'
 ; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v1 = select i1 undef, i8 undef, i8 undef
 ; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2 = select i1 undef, i16 undef, i16 undef
 ; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v3 = select i1 undef, i32 undef, i32 undef
 ; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4 = select i1 undef, i64 undef, i64 undef
 ; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v5 = select i1 undef, float undef, float undef
 ; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v6 = select i1 undef, double undef, double undef

llvm/test/Analysis/CostModel/AArch64/shuffle-broadcast.ll

 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
 ; RUN: opt < %s -mtriple=aarch64--linux-gnu -cost-model -analyze | FileCheck %s
 
 define void @broadcast() {
 ; CHECK-LABEL: 'broadcast'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v7 = shufflevector <2 x i8> undef, <2 x i8> undef, <2 x i32> zeroinitializer
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8 = shufflevector <4 x i8> undef, <4 x i8> undef, <4 x i32> zeroinitializer
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v9 = shufflevector <8 x i8> undef, <8 x i8> undef, <8 x i32> zeroinitializer
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v10 = shufflevector <16 x i8> undef, <16 x i8> undef, <16 x i32> zeroinitializer
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v11 = shufflevector <2 x i16> undef, <2 x i16> undef, <2 x i32> zeroinitializer
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v12 = shufflevector <4 x i16> undef, <4 x i16> undef, <4 x i32> zeroinitializer
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v13 = shufflevector <8 x i16> undef, <8 x i16> undef, <8 x i32> zeroinitializer
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v14 = shufflevector <2 x i32> undef, <2 x i32> undef, <2 x i32> zeroinitializer
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v15 = shufflevector <4 x i32> undef, <4 x i32> undef, <4 x i32> zeroinitializer
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16 = shufflevector <2 x float> undef, <2 x float> undef, <2 x i32> zeroinitializer
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v17 = shufflevector <4 x float> undef, <4 x float> undef, <4 x i32> zeroinitializer
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %v7 = shufflevector <2 x i8> undef, <2 x i8>undef, <2 x i32> zeroinitializer
   %v8 = shufflevector <4 x i8> undef, <4 x i8>undef, <4 x i32> zeroinitializer
   %v9 = shufflevector <8 x i8> undef, <8 x i8>undef, <8 x i32> zeroinitializer
   %v10 = shufflevector <16 x i8> undef, <16 x i8>undef, <16 x i32> zeroinitializer
 
   %v11 = shufflevector <2 x i16> undef, <2 x i16>undef, <2 x i32> zeroinitializer
   %v12 = shufflevector <4 x i16> undef, <4 x i16>undef, <4 x i32> zeroinitializer

llvm/test/Analysis/CostModel/AArch64/store.ll

 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <16 x i8> undef, <16 x i8>* undef, align 4
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <2 x double> undef, <2 x double>* undef, align 4
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x float> undef, <4 x float>* undef, align 4
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <8 x half> undef, <8 x half>* undef, align 4
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: store <2 x i8> undef, <2 x i8>* undef, align 2
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x i8> undef, <4 x i8>* undef, align 4
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %1 = load <2 x i8>, <2 x i8>* undef, align 2
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %2 = load <4 x i8>, <4 x i8>* undef, align 4
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; SIZE-LABEL: 'getMemoryOpCost'
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x i64> undef, <4 x i64>* undef, align 4
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <8 x i32> undef, <8 x i32>* undef, align 4
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <16 x i16> undef, <16 x i16>* undef, align 4
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <32 x i8> undef, <32 x i8>* undef, align 4
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x double> undef, <4 x double>* undef, align 4
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <8 x float> undef, <8 x float>* undef, align 4

 ; SLOW_MISALIGNED_128_STORE-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: store <16 x i8> undef, <16 x i8>* undef, align 4
 ; SLOW_MISALIGNED_128_STORE-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: store <2 x double> undef, <2 x double>* undef, align 4
 ; SLOW_MISALIGNED_128_STORE-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: store <4 x float> undef, <4 x float>* undef, align 4
 ; SLOW_MISALIGNED_128_STORE-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: store <8 x half> undef, <8 x half>* undef, align 4
 ; SLOW_MISALIGNED_128_STORE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: store <2 x i8> undef, <2 x i8>* undef, align 2
 ; SLOW_MISALIGNED_128_STORE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store <4 x i8> undef, <4 x i8>* undef, align 4
 ; SLOW_MISALIGNED_128_STORE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %1 = load <2 x i8>, <2 x i8>* undef, align 2
 ; SLOW_MISALIGNED_128_STORE-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %2 = load <4 x i8>, <4 x i8>* undef, align 4
-; SLOW_MISALIGNED_128_STORE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+; SLOW_MISALIGNED_128_STORE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   store <4 x i64> undef, <4 x i64> * undef
   store <8 x i32> undef, <8 x i32> * undef
   store <16 x i16> undef, <16 x i16> * undef
   store <32 x i8> undef, <32 x i8> * undef
 
   store <4 x double> undef, <4 x double> * undef
   store <8 x float> undef, <8 x float> * undef

llvm/test/Transforms/LoopVectorize/AArch64/aarch64-predication.ll

 ; REQUIRES: asserts
 ; RUN: opt < %s -loop-vectorize -disable-output -debug-only=loop-vectorize 2>&1 | FileCheck %s --check-prefix=COST
 ; RUN: opt < %s -loop-vectorize -force-vector-width=2 -instcombine -simplifycfg -S | FileCheck %s
 
 target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
 target triple = "aarch64--linux-gnu"
 
 ; This test checks that we correctly compute the scalarized operands for a
 ; user-specified vectorization factor when interleaving is disabled. We use the
-; "optsize" attribute to disable all interleaving calculations.  A cost of 5
+; "optsize" attribute to disable all interleaving calculations.  A cost of 4
 ; for %tmp4 indicates that we would scalarize it's operand (%tmp3), giving
 ; %tmp4 a lower scalarization overhead.
 ;
 ; COST-LABEL:  predicated_udiv_scalarized_operand
-; COST:        LV: Found an estimated cost of 5 for VF 2 For instruction: %tmp4 = udiv i64 %tmp2, %tmp3
+; COST:        LV: Found an estimated cost of 4 for VF 2 For instruction: %tmp4 = udiv i64 %tmp2, %tmp3
 ;
 ; CHECK-LABEL: @predicated_udiv_scalarized_operand(
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %entry ], [ [[INDEX_NEXT:%.*]], %[[PRED_UDIV_CONTINUE2:.*]] ]
 ; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <2 x i64> [ zeroinitializer, %entry ], [ [[TMP17:%.*]], %[[PRED_UDIV_CONTINUE2]] ]
 ; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i64, i64* %a, i64 [[INDEX]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i64* [[TMP0]] to <2 x i64>*
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x i64>, <2 x i64>* [[TMP1]], align 4

llvm/test/Transforms/LoopVectorize/AArch64/extractvalue-no-scalarization-required.ll

 ; REQUIRES: asserts
 
 ; RUN: opt -loop-vectorize -mtriple=arm64-apple-ios %s -S -debug -disable-output 2>&1 | FileCheck --check-prefix=CM %s
 ; RUN: opt -loop-vectorize -force-vector-width=2 -force-vector-interleave=1 %s -S | FileCheck --check-prefix=FORCED %s
 
 ; Test case from PR41294.
 
 ; Check scalar cost for extractvalue. The constant and loop invariant operands are free,
 ; leaving cost 3 for scalarizing the result + 2 for executing the op with VF 2.
 
-; CM: LV: Scalar loop costs: 9.
+; CM: LV: Scalar loop costs: 7.
 ; CM: LV: Found an estimated cost of 5 for VF 2 For instruction:   %a = extractvalue { i64, i64 } %sv, 0
 ; CM-NEXT: LV: Found an estimated cost of 5 for VF 2 For instruction:   %b = extractvalue { i64, i64 } %sv, 1
 
 ; Check that the extractvalue operands are actually free in vector code.
 
 ; FORCED-LABEL: vector.body:                                      ; preds = %vector.body, %vector.ph
 ; FORCED-NEXT:    %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
 ; FORCED-NEXT:    %0 = add i32 %index, 0

 exit:
   ret void
 }
 
 
 ; Similar to the test case above, but checks getVectorCallCost as well.
 declare float @pow(float, float) readnone nounwind
 
-; CM: LV: Scalar loop costs: 18.
+; CM: LV: Scalar loop costs: 16.
 ; CM: LV: Found an estimated cost of 5 for VF 2 For instruction:   %a = extractvalue { float, float } %sv, 0
 ; CM-NEXT: LV: Found an estimated cost of 5 for VF 2 For instruction:   %b = extractvalue { float, float } %sv, 1
 
 ; FORCED-LABEL: define void @test_getVectorCallCost
 
 ; FORCED-LABEL: vector.body:                                      ; preds = %vector.body, %vector.ph
 ; FORCED-NEXT:    %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
 ; FORCED-NEXT:    %0 = add i32 %index, 0

llvm/test/Transforms/LoopVectorize/AArch64/predication_costs.ll

 ; This test checks that we correctly compute the cost of the predicated udiv
 ; instruction. If we assume the block probability is 50%, we compute the cost
 ; as:
 ;
 ; Cost of udiv:
 ;   (udiv(2) + extractelement(6) + insertelement(3)) / 2 = 5
 ;
 ; CHECK: Scalarizing and predicating: %tmp4 = udiv i32 %tmp2, %tmp3
-; CHECK: Found an estimated cost of 6 for VF 2 For instruction: %tmp4 = udiv i32 %tmp2, %tmp3
+; CHECK: Found an estimated cost of 5 for VF 2 For instruction: %tmp4 = udiv i32 %tmp2, %tmp3
 ;
 define i32 @predicated_udiv(i32* %a, i32* %b, i1 %c, i64 %n) {
 entry:
   br label %for.body
 
 for.body:
   %i = phi i64 [ 0, %entry ], [ %i.next, %for.inc ]
   %r = phi i32 [ 0, %entry ], [ %tmp6, %for.inc ]

 ; Cost of add:
 ;   (add(2) + extractelement(3)) / 2 = 2
 ; Cost of udiv:
 ;   (udiv(2) + extractelement(3) + insertelement(3)) / 2 = 4
 ;
 ; CHECK: Scalarizing: %tmp3 = add nsw i32 %tmp2, %x
 ; CHECK: Scalarizing and predicating: %tmp4 = udiv i32 %tmp2, %tmp3
 ; CHECK: Found an estimated cost of 2 for VF 2 For instruction: %tmp3 = add nsw i32 %tmp2, %x
-; CHECK: Found an estimated cost of 5 for VF 2 For instruction: %tmp4 = udiv i32 %tmp2, %tmp3
+; CHECK: Found an estimated cost of 4 for VF 2 For instruction: %tmp4 = udiv i32 %tmp2, %tmp3
 ;
 define i32 @predicated_udiv_scalarized_operand(i32* %a, i1 %c, i32 %x, i64 %n) {
 entry:
   br label %for.body
 
 for.body:
   %i = phi i64 [ 0, %entry ], [ %i.next, %for.inc ]
   %r = phi i32 [ 0, %entry ], [ %tmp6, %for.inc ]

 ;   store(4) / 2 = 2
 ;
 ; CHECK-NOT: Scalarizing: %tmp2 = add i32 %tmp1, %x
 ; CHECK:     Scalarizing and predicating: %tmp3 = sdiv i32 %tmp1, %tmp2
 ; CHECK:     Scalarizing and predicating: %tmp4 = udiv i32 %tmp3, %tmp2
 ; CHECK:     Scalarizing: %tmp5 = sub i32 %tmp4, %x
 ; CHECK:     Scalarizing and predicating: store i32 %tmp5, i32* %tmp0, align 4
 ; CHECK:     Found an estimated cost of 1 for VF 2 For instruction: %tmp2 = add i32 %tmp1, %x
-; CHECK:     Found an estimated cost of 6 for VF 2 For instruction: %tmp3 = sdiv i32 %tmp1, %tmp2
-; CHECK:     Found an estimated cost of 6 for VF 2 For instruction: %tmp4 = udiv i32 %tmp3, %tmp2
+; CHECK:     Found an estimated cost of 5 for VF 2 For instruction: %tmp3 = sdiv i32 %tmp1, %tmp2
+; CHECK:     Found an estimated cost of 5 for VF 2 For instruction: %tmp4 = udiv i32 %tmp3, %tmp2
 ; CHECK:     Found an estimated cost of 2 for VF 2 For instruction: %tmp5 = sub i32 %tmp4, %x
 ; CHECK:     Found an estimated cost of 2 for VF 2 For instruction: store i32 %tmp5, i32* %tmp0, align 4
 ;
 define void @predication_multi_context(i32* %a, i1 %c, i32 %x, i64 %n) {
 entry:
   br label %for.body
 
 for.body: