diff --git a/compiler-rt/include/profile/InstrProfData.inc b/compiler-rt/include/profile/InstrProfData.inc --- a/compiler-rt/include/profile/InstrProfData.inc +++ b/compiler-rt/include/profile/InstrProfData.inc @@ -857,6 +857,12 @@ #endif /* defined(_MSC_VER) && !defined(__clang__) */ +/* Return the number of the value range buckets */ +INSTR_PROF_VISIBILITY INSTR_PROF_INLINE uint64_t +InstrProfGetNumBuckets() { + return 22; +} + /* Map an (observed) memop size value to the representative value of its range. * For example, 5 -> 5, 22 -> 17, 99 -> 65, 256 -> 256, 1001 -> 513. */ INSTR_PROF_VISIBILITY INSTR_PROF_INLINE uint64_t diff --git a/llvm/include/llvm/ProfileData/InstrProfData.inc b/llvm/include/llvm/ProfileData/InstrProfData.inc --- a/llvm/include/llvm/ProfileData/InstrProfData.inc +++ b/llvm/include/llvm/ProfileData/InstrProfData.inc @@ -857,6 +857,12 @@ #endif /* defined(_MSC_VER) && !defined(__clang__) */ +/* Return the number of the value range buckets */ +INSTR_PROF_VISIBILITY INSTR_PROF_INLINE uint64_t +InstrProfGetNumBuckets() { + return 22; +} + /* Map an (observed) memop size value to the representative value of its range. * For example, 5 -> 5, 22 -> 17, 99 -> 65, 256 -> 256, 1001 -> 513. */ INSTR_PROF_VISIBILITY INSTR_PROF_INLINE uint64_t diff --git a/llvm/lib/Transforms/Instrumentation/PGOMemOPSizeOpt.cpp b/llvm/lib/Transforms/Instrumentation/PGOMemOPSizeOpt.cpp --- a/llvm/lib/Transforms/Instrumentation/PGOMemOPSizeOpt.cpp +++ b/llvm/lib/Transforms/Instrumentation/PGOMemOPSizeOpt.cpp @@ -225,7 +225,7 @@ TargetLibraryInfo &TLI) : Func(Func), BFI(BFI), ORE(ORE), DT(DT), TLI(TLI), Changed(false) { ValueDataArray = - std::make_unique(MemOPMaxVersion + 2); + std::make_unique(InstrProfGetNumBuckets()); } bool isChanged() const { return Changed; } void perform() { @@ -298,9 +298,9 @@ if (!MemOPOptMemcmpBcmp && (MO.isMemcmp(TLI) || MO.isBcmp(TLI))) return false; - uint32_t NumVals, MaxNumPromotions = MemOPMaxVersion + 2; + uint32_t NumVals, MaxNumVals = InstrProfGetNumBuckets(); uint64_t TotalCount; - if (!getValueProfDataFromInst(*MO.I, IPVK_MemOPSize, MaxNumPromotions, + if (!getValueProfDataFromInst(*MO.I, IPVK_MemOPSize, MaxNumVals, ValueDataArray.get(), NumVals, TotalCount)) return false; @@ -342,19 +342,25 @@ int64_t LastV = -1; // Default case is in the front -- save the slot here. CaseCounts.push_back(0); - for (auto &VD : VDs) { + SmallVector RemainingVDs; + for (auto I = VDs.begin(), E = VDs.end(); I != E; ++I) { + auto &VD = *I; int64_t V = VD.Value; uint64_t C = VD.Count; if (MemOPScaleCount) C = getScaledCount(C, ActualCount, SavedTotalCount); - if (!InstrProfIsSingleValRange(V) || V > MemOpMaxOptSize) + if (!InstrProfIsSingleValRange(V) || V > MemOpMaxOptSize) { + RemainingVDs.push_back(VD); continue; + } // ValueCounts are sorted on the count. Break at the first un-profitable // value. - if (!isProfitable(C, RemainCount)) + if (!isProfitable(C, RemainCount)) { + RemainingVDs.insert(RemainingVDs.end(), I, E); break; + } if (V == LastV) { LLVM_DEBUG(dbgs() << "Invalid Profile Data in Function " << Func.getName() @@ -375,8 +381,10 @@ assert(SavedRemainCount >= VD.Count); SavedRemainCount -= VD.Count; - if (++Version > MemOPMaxVersion && MemOPMaxVersion != 0) + if (++Version > MemOPMaxVersion && MemOPMaxVersion != 0) { + RemainingVDs.insert(RemainingVDs.end(), I + 1, E); break; + } } if (Version == 0) @@ -441,10 +449,12 @@ // Clear the value profile data. MO.I->setMetadata(LLVMContext::MD_prof, nullptr); // If all promoted, we don't need the MD.prof metadata. - if (SavedRemainCount > 0 || Version != NumVals) + if (SavedRemainCount > 0 || Version != NumVals) { // Otherwise we need update with the un-promoted records back. - annotateValueSite(*Func.getParent(), *MO.I, VDs.slice(Version), - SavedRemainCount, IPVK_MemOPSize, NumVals); + ArrayRef RemVDs(RemainingVDs); + annotateValueSite(*Func.getParent(), *MO.I, RemVDs, SavedRemainCount, + IPVK_MemOPSize, NumVals); + } LLVM_DEBUG(dbgs() << "\n\n== Basic Block After==\n"); diff --git a/llvm/test/Transforms/PGOProfile/memop_size_opt.ll b/llvm/test/Transforms/PGOProfile/memop_size_opt.ll --- a/llvm/test/Transforms/PGOProfile/memop_size_opt.ll +++ b/llvm/test/Transforms/PGOProfile/memop_size_opt.ll @@ -38,22 +38,22 @@ ; MEMOP_OPT: switch i64 %conv, label %[[DEFAULT_LABEL:.*]] [ ; MEMOP_OPT: i64 0, label %[[CASE_1_LABEL:.*]] -; MEMOP_OPT: ], !prof [[SWITCH_BW:![0-9]+]] +; MEMOP_OPT: ], !prof [[SWITCH_BW1:![0-9]+]] ; MEMOP_OPT: [[CASE_1_LABEL]]: ; MEMOP_OPT: call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst, i8* %src, i64 0, i1 false) ; MEMOP_OPT: br label %[[MERGE_LABEL:.*]] ; MEMOP_OPT: [[DEFAULT_LABEL]]: -; MEMOP_OPT: call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst, i8* %src, i64 %conv, i1 false), !prof [[NEWVP:![0-9]+]] +; MEMOP_OPT: call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst, i8* %src, i64 %conv, i1 false), !prof [[NEWVP1:![0-9]+]] ; MEMOP_OPT: br label %[[MERGE_LABEL]] ; MEMOP_OPT: [[MERGE_LABEL]]: ; MEMOP_OPT: switch i64 %conv, label %[[DEFAULT_LABEL2:.*]] [ ; MEMOP_OPT: i64 0, label %[[CASE_1_LABEL2:.*]] -; MEMOP_OPT: ], !prof [[SWITCH_BW:![0-9]+]] +; MEMOP_OPT: ], !prof [[SWITCH_BW2:![0-9]+]] ; MEMOP_OPT: [[CASE_1_LABEL2]]: ; MEMOP_OPT: call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst2, i8* %src2, i64 0, i1 false) ; MEMOP_OPT: br label %[[MERGE_LABEL2:.*]] ; MEMOP_OPT: [[DEFAULT_LABEL2]]: -; MEMOP_OPT: call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst2, i8* %src2, i64 %conv, i1 false), !prof [[NEWVP]] +; MEMOP_OPT: call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst2, i8* %src2, i64 %conv, i1 false), !prof [[NEWVP2:![0-9]+]] ; MEMOP_OPT: br label %[[MERGE_LABEL2]] ; MEMOP_OPT: [[MERGE_LABEL2]]: ; MEMOP_OPT: br label %for.inc @@ -105,23 +105,23 @@ ; MEMOP_OPT: switch i64 %conv, label %[[DEFAULT_LABEL:.*]] [ ; MEMOP_OPT: i64 0, label %[[CASE_1_LABEL:.*]] -; MEMOP_OPT: ], !prof [[SWITCH_BW:![0-9]+]] +; MEMOP_OPT: ], !prof [[SWITCH_BW1]] ; MEMOP_OPT: [[CASE_1_LABEL]]: ; MEMOP_OPT: %[[RV:.*]] = call i32 @memcmp(i8* %dst, i8* %src, i64 0) ; MEMOP_OPT: br label %[[MERGE_LABEL:.*]] ; MEMOP_OPT: [[DEFAULT_LABEL]]: -; MEMOP_OPT: %[[RVD:.*]] = call i32 @memcmp(i8* %dst, i8* %src, i64 %conv), !prof [[NEWVP:![0-9]+]] +; MEMOP_OPT: %[[RVD:.*]] = call i32 @memcmp(i8* %dst, i8* %src, i64 %conv), !prof [[NEWVP1]] ; MEMOP_OPT: br label %[[MERGE_LABEL]] ; MEMOP_OPT: [[MERGE_LABEL]]: ; MEMOP_OPT: %[[PHI:.*]] = phi i32 [ %[[RVD]], %[[DEFAULT_LABEL]] ], [ %[[RV]], %[[CASE_1_LABEL]] ] ; MEMOP_OPT: switch i64 %conv, label %[[DEFAULT_LABEL2:.*]] [ ; MEMOP_OPT: i64 0, label %[[CASE_1_LABEL2:.*]] -; MEMOP_OPT: ], !prof [[SWITCH_BW:![0-9]+]] +; MEMOP_OPT: ], !prof [[SWITCH_BW2]] ; MEMOP_OPT: [[CASE_1_LABEL2]]: ; MEMOP_OPT: %[[RV2:.*]] = call i32 @bcmp(i8* %dst2, i8* %src2, i64 0) ; MEMOP_OPT: br label %[[MERGE_LABEL2:.*]] ; MEMOP_OPT: [[DEFAULT_LABEL2]]: -; MEMOP_OPT: %[[RVD2:.*]] = call i32 @bcmp(i8* %dst2, i8* %src2, i64 %conv), !prof [[NEWVP]] +; MEMOP_OPT: %[[RVD2:.*]] = call i32 @bcmp(i8* %dst2, i8* %src2, i64 %conv), !prof [[NEWVP2]] ; MEMOP_OPT: br label %[[MERGE_LABEL2]] ; MEMOP_OPT: [[MERGE_LABEL2]]: ; MEMOP_OPT: %[[PHI2:.*]] = phi i32 [ %[[RVD2]], %[[DEFAULT_LABEL2]] ], [ %[[RV2]], %[[CASE_1_LABEL2]] ] @@ -143,12 +143,14 @@ ret void } -; MEMOP_OPT: [[SWITCH_BW]] = !{!"branch_weights", i32 457, i32 99} +; MEMOP_OPT: [[SWITCH_BW1]] = !{!"branch_weights", i32 457, i32 99} ; Should be 457 total left (original total count 556, minus 99 from specialized -; value 1, which is removed from VP array. Also, we only end up with 5 total -; values, since the default max number of promotions is 5 and therefore -; the rest of the values are ignored when extracting the VP metadata. -; MEMOP_OPT: [[NEWVP]] = !{!"VP", i32 1, i64 457, i64 2, i64 88, i64 3, i64 77, i64 9, i64 72, i64 4, i64 66} +; value 0, which is removed from VP array. This should preserve all unpromoted values. +; MEMOP_OPT: [[NEWVP1]] = !{!"VP", i32 1, i64 457, i64 2, i64 88, i64 3, i64 77, i64 9, i64 72, i64 4, i64 66, i64 5, i64 55, i64 6, i64 44, i64 7, i64 33, i64 8, i64 22} +; MEMOP_OPT: [[SWITCH_BW2]] = !{!"branch_weights", i32 465, i32 91} +; Should be 465 total left (original total count 556, minus 91 from specialized +; value 0, which is removed from VP array. This should preserve all unpromoted values. Range values 9, 17, 33, 65, 129 should be skipped and preserved. +; MEMOP_OPT: [[NEWVP2]] = !{!"VP", i32 1, i64 465, i64 9, i64 94, i64 17, i64 93, i64 33, i64 93, i64 65, i64 92, i64 129, i64 92, i64 2, i64 1} !llvm.module.flags = !{!0} @@ -183,7 +185,7 @@ !28 = !{!"branch_weights", i32 20, i32 1} !29 = !{!"branch_weights", i32 556, i32 20} !30 = !{!"VP", i32 1, i64 556, i64 0, i64 99, i64 2, i64 88, i64 3, i64 77, i64 9, i64 72, i64 4, i64 66, i64 5, i64 55, i64 6, i64 44, i64 7, i64 33, i64 8, i64 22} -!31 = !{!"VP", i32 1, i64 556, i64 0, i64 99, i64 2, i64 88, i64 3, i64 77, i64 9, i64 72, i64 4, i64 66, i64 5, i64 55, i64 6, i64 44, i64 7, i64 33, i64 8, i64 22} +!31 = !{!"VP", i32 1, i64 556, i64 9, i64 94, i64 17, i64 93, i64 33, i64 93, i64 65, i64 92, i64 129, i64 92, i64 0, i64 91, i64 2, i64 1} declare void @llvm.lifetime.start(i64, i8* nocapture) @@ -219,7 +221,7 @@ ; YAML-NEXT: - String: 'optimized ' ; YAML-NEXT: - Memop: memcpy ; YAML-NEXT: - String: ' with count ' -; YAML-NEXT: - Count: '99' +; YAML-NEXT: - Count: '91' ; YAML-NEXT: - String: ' out of ' ; YAML-NEXT: - Total: '556' ; YAML-NEXT: - String: ' for ' @@ -251,7 +253,7 @@ ; YAML-NEXT: - String: 'optimized ' ; YAML-NEXT: - Memop: bcmp ; YAML-NEXT: - String: ' with count ' -; YAML-NEXT: - Count: '99' +; YAML-NEXT: - Count: '91' ; YAML-NEXT: - String: ' out of ' ; YAML-NEXT: - Total: '556' ; YAML-NEXT: - String: ' for '