Index: llvm/trunk/include/llvm/ProfileData/SampleProf.h =================================================================== --- llvm/trunk/include/llvm/ProfileData/SampleProf.h +++ llvm/trunk/include/llvm/ProfileData/SampleProf.h @@ -0,0 +1,206 @@ +//=-- SampleProf.h - Sampling profiling format support --------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file contains common definitions used in the reading and writing of +// sample profile data. +// +//===----------------------------------------------------------------------===// +#ifndef LLVM_PROFILEDATA_SAMPLEPROF_H_ +#define LLVM_PROFILEDATA_SAMPLEPROF_H_ + +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/Support/raw_ostream.h" + +#include + +namespace llvm { + +const std::error_category &sampleprof_category(); + +enum class sampleprof_error { + success = 0, + bad_magic, + unsupported_version, + too_large, + truncated, + malformed +}; + +inline std::error_code make_error_code(sampleprof_error E) { + return std::error_code(static_cast(E), sampleprof_category()); +} + +} // end namespace llvm + +namespace std { +template <> +struct is_error_code_enum : std::true_type {}; +} + +namespace llvm { + +namespace sampleprof { + +static inline uint64_t SPMagic() { + return uint64_t('S') << (64 - 8) | uint64_t('P') << (64 - 16) | + uint64_t('R') << (64 - 24) | uint64_t('O') << (64 - 32) | + uint64_t('F') << (64 - 40) | uint64_t('4') << (64 - 48) | + uint64_t('2') << (64 - 56) | uint64_t(0xff); +} + +static inline uint64_t SPVersion() { return 100; } + +/// \brief Represents the relative location of an instruction. +/// +/// Instruction locations are specified by the line offset from the +/// beginning of the function (marked by the line where the function +/// header is) and the discriminator value within that line. +/// +/// The discriminator value is useful to distinguish instructions +/// that are on the same line but belong to different basic blocks +/// (e.g., the two post-increment instructions in "if (p) x++; else y++;"). +struct LineLocation { + LineLocation(int L, unsigned D) : LineOffset(L), Discriminator(D) {} + int LineOffset; + unsigned Discriminator; +}; + +} // End namespace sampleprof + +template <> struct DenseMapInfo { + typedef DenseMapInfo OffsetInfo; + typedef DenseMapInfo DiscriminatorInfo; + static inline sampleprof::LineLocation getEmptyKey() { + return sampleprof::LineLocation(OffsetInfo::getEmptyKey(), + DiscriminatorInfo::getEmptyKey()); + } + static inline sampleprof::LineLocation getTombstoneKey() { + return sampleprof::LineLocation(OffsetInfo::getTombstoneKey(), + DiscriminatorInfo::getTombstoneKey()); + } + static inline unsigned getHashValue(sampleprof::LineLocation Val) { + return DenseMapInfo>::getHashValue( + std::pair(Val.LineOffset, Val.Discriminator)); + } + static inline bool isEqual(sampleprof::LineLocation LHS, + sampleprof::LineLocation RHS) { + return LHS.LineOffset == RHS.LineOffset && + LHS.Discriminator == RHS.Discriminator; + } +}; + +namespace sampleprof { + +/// \brief Representation of a single sample record. +/// +/// A sample record is represented by a positive integer value, which +/// indicates how frequently was the associated line location executed. +/// +/// Additionally, if the associated location contains a function call, +/// the record will hold a list of all the possible called targets. For +/// direct calls, this will be the exact function being invoked. For +/// indirect calls (function pointers, virtual table dispatch), this +/// will be a list of one or more functions. +class SampleRecord { +public: + typedef SmallVector, 8> CallTargetList; + + SampleRecord() : NumSamples(0), CallTargets() {} + + /// \brief Increment the number of samples for this record by \p S. + void addSamples(unsigned S) { NumSamples += S; } + + /// \brief Add called function \p F with samples \p S. + void addCalledTarget(std::string F, unsigned S) { + CallTargets.push_back(std::make_pair(F, S)); + } + + /// \brief Return true if this sample record contains function calls. + bool hasCalls() const { return CallTargets.size() > 0; } + + unsigned getSamples() const { return NumSamples; } + const CallTargetList &getCallTargets() const { return CallTargets; } + +private: + unsigned NumSamples; + CallTargetList CallTargets; +}; + +typedef DenseMap BodySampleMap; + +/// \brief Representation of the samples collected for a function. +/// +/// This data structure contains all the collected samples for the body +/// of a function. Each sample corresponds to a LineLocation instance +/// within the body of the function. +class FunctionSamples { +public: + FunctionSamples() : TotalSamples(0), TotalHeadSamples(0) {} + void print(raw_ostream &OS); + void addTotalSamples(unsigned Num) { TotalSamples += Num; } + void addHeadSamples(unsigned Num) { TotalHeadSamples += Num; } + void addBodySamples(int LineOffset, unsigned Discriminator, unsigned Num) { + assert(LineOffset >= 0); + // When dealing with instruction weights, we use the value + // zero to indicate the absence of a sample. If we read an + // actual zero from the profile file, use the value 1 to + // avoid the confusion later on. + if (Num == 0) + Num = 1; + BodySamples[LineLocation(LineOffset, Discriminator)].addSamples(Num); + } + void addCalledTargetSamples(int LineOffset, unsigned Discriminator, + std::string FName, unsigned Num) { + assert(LineOffset >= 0); + BodySamples[LineLocation(LineOffset, Discriminator)].addCalledTarget(FName, + Num); + } + + /// \brief Return the number of samples collected at the given location. + /// Each location is specified by \p LineOffset and \p Discriminator. + unsigned samplesAt(int LineOffset, unsigned Discriminator) { + return BodySamples[LineLocation(LineOffset, Discriminator)].getSamples(); + } + + bool empty() const { return BodySamples.empty(); } + + /// \brief Return the total number of samples collected inside the function. + unsigned getTotalSamples() const { return TotalSamples; } + + /// \brief Return the total number of samples collected at the head of the + /// function. + unsigned getHeadSamples() const { return TotalHeadSamples; } + + /// \brief Return all the samples collected in the body of the function. + const BodySampleMap &getBodySamples() const { return BodySamples; } + +private: + /// \brief Total number of samples collected inside this function. + /// + /// Samples are cumulative, they include all the samples collected + /// inside this function and all its inlined callees. + unsigned TotalSamples; + + /// \brief Total number of samples collected at the head of the function. + unsigned TotalHeadSamples; + + /// \brief Map instruction locations to collected samples. + /// + /// Each entry in this map contains the number of samples + /// collected at the corresponding line offset. All line locations + /// are an offset from the start of the function. + BodySampleMap BodySamples; +}; + +} // End namespace sampleprof + +} // End namespace llvm + +#endif // LLVM_PROFILEDATA_SAMPLEPROF_H_ Index: llvm/trunk/include/llvm/ProfileData/SampleProfReader.h =================================================================== --- llvm/trunk/include/llvm/ProfileData/SampleProfReader.h +++ llvm/trunk/include/llvm/ProfileData/SampleProfReader.h @@ -17,104 +17,19 @@ #include "llvm/IR/DiagnosticInfo.h" #include "llvm/IR/Function.h" #include "llvm/IR/LLVMContext.h" -#include "llvm/IR/Module.h" #include "llvm/ADT/StringMap.h" #include "llvm/ADT/StringRef.h" #include "llvm/ADT/Twine.h" +#include "llvm/ProfileData/SampleProf.h" #include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/ErrorOr.h" +#include "llvm/Support/MemoryBuffer.h" #include "llvm/Support/raw_ostream.h" -using namespace llvm; - -namespace sampleprof { - -/// \brief Represents the relative location of an instruction. -/// -/// Instruction locations are specified by the line offset from the -/// beginning of the function (marked by the line where the function -/// header is) and the discriminator value within that line. -/// -/// The discriminator value is useful to distinguish instructions -/// that are on the same line but belong to different basic blocks -/// (e.g., the two post-increment instructions in "if (p) x++; else y++;"). -struct LineLocation { - LineLocation(int L, unsigned D) : LineOffset(L), Discriminator(D) {} - int LineOffset; - unsigned Discriminator; -}; -} // End namespace sampleprof - namespace llvm { -template <> struct DenseMapInfo { - typedef DenseMapInfo OffsetInfo; - typedef DenseMapInfo DiscriminatorInfo; - static inline sampleprof::LineLocation getEmptyKey() { - return sampleprof::LineLocation(OffsetInfo::getEmptyKey(), - DiscriminatorInfo::getEmptyKey()); - } - static inline sampleprof::LineLocation getTombstoneKey() { - return sampleprof::LineLocation(OffsetInfo::getTombstoneKey(), - DiscriminatorInfo::getTombstoneKey()); - } - static inline unsigned getHashValue(sampleprof::LineLocation Val) { - return DenseMapInfo>::getHashValue( - std::pair(Val.LineOffset, Val.Discriminator)); - } - static inline bool isEqual(sampleprof::LineLocation LHS, - sampleprof::LineLocation RHS) { - return LHS.LineOffset == RHS.LineOffset && - LHS.Discriminator == RHS.Discriminator; - } -}; -} namespace sampleprof { -typedef DenseMap BodySampleMap; - -/// \brief Representation of the samples collected for a function. -/// -/// This data structure contains all the collected samples for the body -/// of a function. Each sample corresponds to a LineLocation instance -/// within the body of the function. -class FunctionSamples { -public: - FunctionSamples() - : TotalSamples(0), TotalHeadSamples(0) {} - void print(raw_ostream & OS); - void addTotalSamples(unsigned Num) { TotalSamples += Num; } - void addHeadSamples(unsigned Num) { TotalHeadSamples += Num; } - void addBodySamples(int LineOffset, unsigned Discriminator, unsigned Num) { - assert(LineOffset >= 0); - BodySamples[LineLocation(LineOffset, Discriminator)] += Num; - } - - /// \brief Return the number of samples collected at the given location. - /// Each location is specified by \p LineOffset and \p Discriminator. - unsigned samplesAt(int LineOffset, unsigned Discriminator) { - return BodySamples.lookup(LineLocation(LineOffset, Discriminator)); - } - - bool empty() { return BodySamples.empty(); } - -private: - /// \brief Total number of samples collected inside this function. - /// - /// Samples are cumulative, they include all the samples collected - /// inside this function and all its inlined callees. - unsigned TotalSamples; - - /// \brief Total number of samples collected at the head of the function. - unsigned TotalHeadSamples; - - /// \brief Map instruction locations to collected samples. - /// - /// Each entry in this map contains the number of samples - /// collected at the corresponding line offset. All line locations - /// are an offset from the start of the function. - BodySampleMap BodySamples; -}; - /// \brief Sample-based profile reader. /// /// Each profile contains sample counts for all the functions @@ -139,19 +54,24 @@ /// protection against source code shuffling, line numbers should /// be relative to the start of the function. /// -/// The reader supports two file formats: text and bitcode. The text format -/// is useful for debugging and testing, while the bitcode format is more +/// The reader supports two file formats: text and binary. The text format +/// is useful for debugging and testing, while the binary format is more /// compact. They can both be used interchangeably. class SampleProfileReader { public: - SampleProfileReader(const Module &M, StringRef F) - : Profiles(0), Filename(F), M(M) {} + SampleProfileReader(std::unique_ptr B, LLVMContext &C) + : Profiles(0), Ctx(C), Buffer(std::move(B)) {} + + virtual ~SampleProfileReader() {} /// \brief Print all the profiles to dbgs(). void dump(); - /// \brief Load sample profiles from the associated file. - bool load(); + /// \brief Read and validate the file header. + virtual std::error_code readHeader() = 0; + + /// \brief Read sample profiles from the associated file. + virtual std::error_code read() = 0; /// \brief Print the profile for \p FName on stream \p OS. void printFunctionProfile(raw_ostream &OS, StringRef FName); @@ -166,14 +86,17 @@ /// \brief Report a parse error message. void reportParseError(int64_t LineNumber, Twine Msg) const { - DiagnosticInfoSampleProfile Diag(Filename.data(), LineNumber, Msg); - M.getContext().diagnose(Diag); + DiagnosticInfoSampleProfile Diag(Buffer->getBufferIdentifier(), LineNumber, + Msg); + Ctx.diagnose(Diag); } -protected: - bool loadText(); - bool loadBitcode() { llvm_unreachable("not implemented"); } + /// \brief Create a sample profile reader appropriate to the file format. + static std::error_code create(std::string Filename, + std::unique_ptr &Reader, + LLVMContext &C); +protected: /// \brief Map every function to its associated profile. /// /// The profile of every function executed at runtime is collected @@ -181,14 +104,68 @@ /// to their corresponding profiles. StringMap Profiles; - /// \brief Path name to the file holding the profile data. - StringRef Filename; + /// \brief LLVM context used to emit diagnostics. + LLVMContext &Ctx; + + /// \brief Memory buffer holding the profile file. + std::unique_ptr Buffer; +}; + +class SampleProfileReaderText : public SampleProfileReader { +public: + SampleProfileReaderText(std::unique_ptr B, LLVMContext &C) + : SampleProfileReader(std::move(B), C) {} + + /// \brief Read and validate the file header. + std::error_code readHeader() override { return sampleprof_error::success; } + + /// \brief Read sample profiles from the associated file. + std::error_code read() override; +}; + +class SampleProfileReaderBinary : public SampleProfileReader { +public: + SampleProfileReaderBinary(std::unique_ptr B, LLVMContext &C) + : SampleProfileReader(std::move(B), C), Data(nullptr), End(nullptr) {} + + /// \brief Read and validate the file header. + std::error_code readHeader() override; + + /// \brief Read sample profiles from the associated file. + std::error_code read() override; + + /// \brief Return true if \p Buffer is in the format supported by this class. + static bool hasFormat(const MemoryBuffer &Buffer); + +protected: + /// \brief Read a numeric value of type T from the profile. + /// + /// If an error occurs during decoding, a diagnostic message is emitted and + /// EC is set. + /// + /// \returns the read value. + template ErrorOr readNumber(); - /// \brief Module being compiled. Used to access the current - /// LLVM context for diagnostics. - const Module &M; + /// \brief Read a string from the profile. + /// + /// If an error occurs during decoding, a diagnostic message is emitted and + /// EC is set. + /// + /// \returns the read value. + ErrorOr readString(); + + /// \brief Return true if we've reached the end of file. + bool at_eof() const { return Data >= End; } + + /// \brief Points to the current location in the buffer. + const uint8_t *Data; + + /// \brief Points to the end of the buffer. + const uint8_t *End; }; } // End namespace sampleprof +} // End namespace llvm + #endif // LLVM_PROFILEDATA_SAMPLEPROFREADER_H Index: llvm/trunk/include/llvm/ProfileData/SampleProfWriter.h =================================================================== --- llvm/trunk/include/llvm/ProfileData/SampleProfWriter.h +++ llvm/trunk/include/llvm/ProfileData/SampleProfWriter.h @@ -0,0 +1,82 @@ +//===- SampleProfWriter.h - Write LLVM sample profile data ----------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file contains definitions needed for writing sample profiles. +// +//===----------------------------------------------------------------------===// +#ifndef LLVM_PROFILEDATA_SAMPLEPROFWRITER_H +#define LLVM_PROFILEDATA_SAMPLEPROFWRITER_H + +#include "llvm/ADT/StringRef.h" +#include "llvm/IR/Function.h" +#include "llvm/IR/Module.h" +#include "llvm/ProfileData/SampleProf.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/raw_ostream.h" + +namespace llvm { + +namespace sampleprof { + +/// \brief Sample-based profile writer. Base class. +class SampleProfileWriter { +public: + SampleProfileWriter(StringRef Filename, std::error_code &EC, + sys::fs::OpenFlags Flags) + : OS(Filename, EC, Flags) {} + virtual ~SampleProfileWriter() {} + + /// \brief Write sample profiles in \p S for function \p F. + /// + /// \returns true if the file was updated successfully. False, otherwise. + virtual bool write(const Function &F, const FunctionSamples &S) = 0; + + /// \brief Write all the sample profiles for all the functions in \p M. + /// + /// \returns true if the file was updated successfully. False, otherwise. + bool write(const Module &M, StringMap &P) { + for (Module::const_iterator I = M.begin(), E = M.end(); I != E; ++I) + if (!write((*I), P[I->getName()])) + return false; + return true; + } + +protected: + /// \brief Output stream where to emit the profile to. + raw_fd_ostream OS; +}; + +/// \brief Sample-based profile writer (text format). +class SampleProfileWriterText : public SampleProfileWriter { +public: + SampleProfileWriterText(StringRef F, std::error_code &EC) + : SampleProfileWriter(F, EC, sys::fs::F_Text) {} + + bool write(const Function &F, const FunctionSamples &S) override; + bool write(const Module &M, StringMap &P) { + return SampleProfileWriter::write(M, P); + } +}; + +/// \brief Sample-based profile writer (binary format). +class SampleProfileWriterBinary : public SampleProfileWriter { +public: + SampleProfileWriterBinary(StringRef F, std::error_code &EC); + + bool write(const Function &F, const FunctionSamples &S) override; + bool write(const Module &M, StringMap &P) { + return SampleProfileWriter::write(M, P); + } +}; + +} // End namespace sampleprof + +} // End namespace sampleprof + +#endif // LLVM_PROFILEDATA_SAMPLEPROFWRITER_H Index: llvm/trunk/lib/ProfileData/CMakeLists.txt =================================================================== --- llvm/trunk/lib/ProfileData/CMakeLists.txt +++ llvm/trunk/lib/ProfileData/CMakeLists.txt @@ -5,5 +5,7 @@ CoverageMapping.cpp CoverageMappingWriter.cpp CoverageMappingReader.cpp + SampleProf.cpp SampleProfReader.cpp + SampleProfWriter.cpp ) Index: llvm/trunk/lib/ProfileData/SampleProf.cpp =================================================================== --- llvm/trunk/lib/ProfileData/SampleProf.cpp +++ llvm/trunk/lib/ProfileData/SampleProf.cpp @@ -0,0 +1,49 @@ +//=-- SampleProf.cpp - Sample profiling format support --------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file contains common definitions used in the reading and writing of +// sample profile data. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ProfileData/SampleProf.h" +#include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/ManagedStatic.h" + +using namespace llvm; + +namespace { +class SampleProfErrorCategoryType : public std::error_category { + const char *name() const LLVM_NOEXCEPT override { return "llvm.sampleprof"; } + std::string message(int IE) const override { + sampleprof_error E = static_cast(IE); + switch (E) { + case sampleprof_error::success: + return "Success"; + case sampleprof_error::bad_magic: + return "Invalid file format (bad magic)"; + case sampleprof_error::unsupported_version: + return "Unsupported format version"; + case sampleprof_error::too_large: + return "Too much profile data"; + case sampleprof_error::truncated: + return "Truncated profile data"; + case sampleprof_error::malformed: + return "Malformed profile data"; + } + llvm_unreachable("A value of sampleprof_error has no message."); + } +}; +} + +static ManagedStatic ErrorCategory; + +const std::error_category &llvm::sampleprof_category() { + return *ErrorCategory; +} Index: llvm/trunk/lib/ProfileData/SampleProfReader.cpp =================================================================== --- llvm/trunk/lib/ProfileData/SampleProfReader.cpp +++ llvm/trunk/lib/ProfileData/SampleProfReader.cpp @@ -8,8 +8,8 @@ //===----------------------------------------------------------------------===// // // This file implements the class that reads LLVM sample profiles. It -// supports two file formats: text and bitcode. The textual representation -// is useful for debugging and testing purposes. The bitcode representation +// supports two file formats: text and binary. The textual representation +// is useful for debugging and testing purposes. The binary representation // is more compact, resulting in smaller file sizes. However, they can // both be used interchangeably. // @@ -95,13 +95,15 @@ //===----------------------------------------------------------------------===// #include "llvm/ProfileData/SampleProfReader.h" +#include "llvm/ProfileData/SampleProfWriter.h" // REMOVE #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorOr.h" -#include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/LEB128.h" #include "llvm/Support/LineIterator.h" +#include "llvm/Support/MemoryBuffer.h" #include "llvm/Support/Regex.h" -using namespace sampleprof; +using namespace llvm::sampleprof; using namespace llvm; /// \brief Print the samples collected for a function on stream \p OS. @@ -112,10 +114,22 @@ << " sampled lines\n"; for (BodySampleMap::const_iterator SI = BodySamples.begin(), SE = BodySamples.end(); - SI != SE; ++SI) - OS << "\tline offset: " << SI->first.LineOffset - << ", discriminator: " << SI->first.Discriminator - << ", number of samples: " << SI->second << "\n"; + SI != SE; ++SI) { + LineLocation Loc = SI->first; + SampleRecord Sample = SI->second; + OS << "\tline offset: " << Loc.LineOffset + << ", discriminator: " << Loc.Discriminator + << ", number of samples: " << Sample.getSamples(); + if (Sample.hasCalls()) { + OS << ", calls:"; + for (SampleRecord::CallTargetList::const_iterator + I = Sample.getCallTargets().begin(), + E = Sample.getCallTargets().end(); + I != E; ++I) + OS << " " << (*I).first << ":" << (*I).second; + } + OS << "\n"; + } OS << "\n"; } @@ -125,7 +139,7 @@ /// \param FName Name of the function to print. void SampleProfileReader::printFunctionProfile(raw_ostream &OS, StringRef FName) { - OS << "Function: " << FName << ":\n"; + OS << "Function: " << FName << ": "; Profiles[FName].print(OS); } @@ -150,22 +164,15 @@ /// the expected format. /// /// \returns true if the file was loaded successfully, false otherwise. -bool SampleProfileReader::loadText() { - ErrorOr> BufferOrErr = - MemoryBuffer::getFile(Filename); - if (std::error_code EC = BufferOrErr.getError()) { - std::string Msg(EC.message()); - M.getContext().diagnose(DiagnosticInfoSampleProfile(Filename.data(), Msg)); - return false; - } - MemoryBuffer &Buffer = *BufferOrErr.get(); - line_iterator LineIt(Buffer, /*SkipBlanks=*/true, '#'); +std::error_code SampleProfileReaderText::read() { + line_iterator LineIt(*Buffer, /*SkipBlanks=*/true, '#'); // Read the profile of each function. Since each function may be // mentioned more than once, and we are collecting flat profiles, // accumulate samples as we parse them. Regex HeadRE("^([^0-9].*):([0-9]+):([0-9]+)$"); - Regex LineSample("^([0-9]+)\\.?([0-9]+)?: ([0-9]+)(.*)$"); + Regex LineSampleRE("^([0-9]+)\\.?([0-9]+)?: ([0-9]+)(.*)$"); + Regex CallSampleRE(" +([^0-9 ][^ ]*):([0-9]+)"); while (!LineIt.is_at_eof()) { // Read the header of each function. // @@ -179,11 +186,11 @@ // // The only requirement we place on the identifier, then, is that it // should not begin with a number. - SmallVector Matches; + SmallVector Matches; if (!HeadRE.match(*LineIt, &Matches)) { reportParseError(LineIt.line_number(), "Expected 'mangled_name:NUM:NUM', found " + *LineIt); - return false; + return sampleprof_error::malformed; } assert(Matches.size() == 4); StringRef FName = Matches[1]; @@ -199,11 +206,11 @@ // Now read the body. The body of the function ends when we reach // EOF or when we see the start of the next function. while (!LineIt.is_at_eof() && isdigit((*LineIt)[0])) { - if (!LineSample.match(*LineIt, &Matches)) { + if (!LineSampleRE.match(*LineIt, &Matches)) { reportParseError( LineIt.line_number(), "Expected 'NUM[.NUM]: NUM[ mangled_name:NUM]*', found " + *LineIt); - return false; + return sampleprof_error::malformed; } assert(Matches.size() == 5); unsigned LineOffset, NumSamples, Discriminator = 0; @@ -212,27 +219,194 @@ Matches[2].getAsInteger(10, Discriminator); Matches[3].getAsInteger(10, NumSamples); - // FIXME: Handle called targets (in Matches[4]). + // If there are function calls in this line, generate a call sample + // entry for each call. + std::string CallsLine(Matches[4]); + while (CallsLine != "") { + SmallVector CallSample; + if (!CallSampleRE.match(CallsLine, &CallSample)) { + reportParseError(LineIt.line_number(), + "Expected 'mangled_name:NUM', found " + CallsLine); + return sampleprof_error::malformed; + } + StringRef CalledFunction = CallSample[1]; + unsigned CalledFunctionSamples; + CallSample[2].getAsInteger(10, CalledFunctionSamples); + FProfile.addCalledTargetSamples(LineOffset, Discriminator, + CalledFunction, CalledFunctionSamples); + CallsLine = CallSampleRE.sub("", CallsLine); + } - // When dealing with instruction weights, we use the value - // zero to indicate the absence of a sample. If we read an - // actual zero from the profile file, return it as 1 to - // avoid the confusion later on. - if (NumSamples == 0) - NumSamples = 1; FProfile.addBodySamples(LineOffset, Discriminator, NumSamples); ++LineIt; } } - return true; + return sampleprof_error::success; +} + +template +ErrorOr SampleProfileReaderBinary::readNumber() { + unsigned NumBytesRead = 0; + std::error_code EC; + uint64_t Val = decodeULEB128(Data, &NumBytesRead); + + if (Val > std::numeric_limits::max()) + EC = sampleprof_error::malformed; + else if (Data + NumBytesRead > End) + EC = sampleprof_error::truncated; + else + EC = sampleprof_error::success; + + if (EC) { + reportParseError(0, EC.message()); + return EC; + } + + Data += NumBytesRead; + return static_cast(Val); +} + +ErrorOr SampleProfileReaderBinary::readString() { + std::error_code EC; + StringRef Str(reinterpret_cast(Data)); + if (Data + Str.size() + 1 > End) { + EC = sampleprof_error::truncated; + reportParseError(0, EC.message()); + return EC; + } + + Data += Str.size() + 1; + return Str; +} + +std::error_code SampleProfileReaderBinary::read() { + while (!at_eof()) { + auto FName(readString()); + if (std::error_code EC = FName.getError()) + return EC; + + Profiles[*FName] = FunctionSamples(); + FunctionSamples &FProfile = Profiles[*FName]; + + auto Val = readNumber(); + if (std::error_code EC = Val.getError()) + return EC; + FProfile.addTotalSamples(*Val); + + Val = readNumber(); + if (std::error_code EC = Val.getError()) + return EC; + FProfile.addHeadSamples(*Val); + + // Read the samples in the body. + auto NumRecords = readNumber(); + if (std::error_code EC = NumRecords.getError()) + return EC; + for (unsigned I = 0; I < *NumRecords; ++I) { + auto LineOffset = readNumber(); + if (std::error_code EC = LineOffset.getError()) + return EC; + + auto Discriminator = readNumber(); + if (std::error_code EC = Discriminator.getError()) + return EC; + + auto NumSamples = readNumber(); + if (std::error_code EC = NumSamples.getError()) + return EC; + + auto NumCalls = readNumber(); + if (std::error_code EC = NumCalls.getError()) + return EC; + + for (unsigned J = 0; J < *NumCalls; ++J) { + auto CalledFunction(readString()); + if (std::error_code EC = CalledFunction.getError()) + return EC; + + auto CalledFunctionSamples = readNumber(); + if (std::error_code EC = CalledFunctionSamples.getError()) + return EC; + + FProfile.addCalledTargetSamples(*LineOffset, *Discriminator, + *CalledFunction, + *CalledFunctionSamples); + } + + FProfile.addBodySamples(*LineOffset, *Discriminator, *NumSamples); + } + } + + return sampleprof_error::success; +} + +std::error_code SampleProfileReaderBinary::readHeader() { + Data = reinterpret_cast(Buffer->getBufferStart()); + End = Data + Buffer->getBufferSize(); + + // Read and check the magic identifier. + auto Magic = readNumber(); + if (std::error_code EC = Magic.getError()) + return EC; + else if (*Magic != SPMagic()) + return sampleprof_error::bad_magic; + + // Read the version number. + auto Version = readNumber(); + if (std::error_code EC = Version.getError()) + return EC; + else if (*Version != SPVersion()) + return sampleprof_error::unsupported_version; + + return sampleprof_error::success; +} + +bool SampleProfileReaderBinary::hasFormat(const MemoryBuffer &Buffer) { + const uint8_t *Data = + reinterpret_cast(Buffer.getBufferStart()); + uint64_t Magic = decodeULEB128(Data); + return Magic == SPMagic(); } -/// \brief Load execution samples from a file. +/// \brief Prepare a memory buffer for the contents of \p Filename. /// -/// This function examines the header of the given file to determine -/// whether to use the text or the bitcode loader. -bool SampleProfileReader::load() { - // TODO Actually detect the file format. - return loadText(); +/// \returns an error code indicating the status of the buffer. +static std::error_code +setupMemoryBuffer(std::string Filename, std::unique_ptr &Buffer) { + auto BufferOrErr = MemoryBuffer::getFileOrSTDIN(Filename); + if (std::error_code EC = BufferOrErr.getError()) + return EC; + Buffer = std::move(BufferOrErr.get()); + + // Sanity check the file. + if (Buffer->getBufferSize() > std::numeric_limits::max()) + return sampleprof_error::too_large; + + return sampleprof_error::success; +} + +/// \brief Create a sample profile reader based on the format of the input file. +/// +/// \param Filename The file to open. +/// +/// \param Reader The reader to instantiate according to \p Filename's format. +/// +/// \param C The LLVM context to use to emit diagnostics. +/// +/// \returns an error code indicating the status of the created reader. +std::error_code +SampleProfileReader::create(std::string Filename, + std::unique_ptr &Reader, + LLVMContext &C) { + std::unique_ptr Buffer; + if (std::error_code EC = setupMemoryBuffer(Filename, Buffer)) + return EC; + + if (SampleProfileReaderBinary::hasFormat(*Buffer)) + Reader.reset(new SampleProfileReaderBinary(std::move(Buffer), C)); + else + Reader.reset(new SampleProfileReaderText(std::move(Buffer), C)); + + return Reader->readHeader(); } Index: llvm/trunk/lib/ProfileData/SampleProfWriter.cpp =================================================================== --- llvm/trunk/lib/ProfileData/SampleProfWriter.cpp +++ llvm/trunk/lib/ProfileData/SampleProfWriter.cpp @@ -0,0 +1,110 @@ +//===- SampleProfWriter.cpp - Write LLVM sample profile data --------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file implements the class that writes LLVM sample profiles. It +// supports two file formats: text and binary. The textual representation +// is useful for debugging and testing purposes. The binary representation +// is more compact, resulting in smaller file sizes. However, they can +// both be used interchangeably. +// +// See lib/ProfileData/SampleProfReader.cpp for documentation on each of the +// supported formats. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ProfileData/SampleProfWriter.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/ErrorOr.h" +#include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/LEB128.h" +#include "llvm/Support/LineIterator.h" +#include "llvm/Support/Regex.h" + +using namespace llvm::sampleprof; +using namespace llvm; + +/// \brief Write samples to a text file. +bool SampleProfileWriterText::write(const Function &F, + const FunctionSamples &S) { + if (S.empty()) + return true; + + OS << F.getName() << ":" << S.getTotalSamples() << ":" << S.getHeadSamples() + << "\n"; + + for (BodySampleMap::const_iterator I = S.getBodySamples().begin(), + E = S.getBodySamples().end(); + I != E; ++I) { + LineLocation Loc = I->first; + SampleRecord Sample = I->second; + if (Loc.Discriminator == 0) + OS << Loc.LineOffset << ": "; + else + OS << Loc.LineOffset << "." << Loc.Discriminator << ": "; + + OS << Sample.getSamples(); + + for (SampleRecord::CallTargetList::const_iterator + I = Sample.getCallTargets().begin(), + E = Sample.getCallTargets().end(); + I != E; ++I) + OS << " " << (*I).first << ":" << (*I).second; + OS << "\n"; + } + + return true; +} + +SampleProfileWriterBinary::SampleProfileWriterBinary(StringRef F, + std::error_code &EC) + : SampleProfileWriter(F, EC, sys::fs::F_None) { + if (EC) + return; + + // Write the file header. + encodeULEB128(SPMagic(), OS); + encodeULEB128(SPVersion(), OS); +} + +/// \brief Write samples to a binary file. +/// +/// \returns true if the samples were written successfully, false otherwise. +bool SampleProfileWriterBinary::write(const Function &F, + const FunctionSamples &S) { + if (S.empty()) + return true; + + OS << F.getName(); + encodeULEB128(0, OS); + encodeULEB128(S.getTotalSamples(), OS); + encodeULEB128(S.getHeadSamples(), OS); + encodeULEB128(S.getBodySamples().size(), OS); + for (BodySampleMap::const_iterator I = S.getBodySamples().begin(), + E = S.getBodySamples().end(); + I != E; ++I) { + LineLocation Loc = I->first; + SampleRecord Sample = I->second; + encodeULEB128(Loc.LineOffset, OS); + encodeULEB128(Loc.Discriminator, OS); + encodeULEB128(Sample.getSamples(), OS); + encodeULEB128(Sample.getCallTargets().size(), OS); + for (SampleRecord::CallTargetList::const_iterator + I = Sample.getCallTargets().begin(), + E = Sample.getCallTargets().end(); + I != E; ++I) { + std::string Callee = (*I).first; + unsigned CalleeSamples = (*I).second; + OS << Callee; + encodeULEB128(0, OS); + encodeULEB128(CalleeSamples, OS); + } + } + + return true; +} Index: llvm/trunk/lib/Transforms/Scalar/SampleProfile.cpp =================================================================== --- llvm/trunk/lib/Transforms/Scalar/SampleProfile.cpp +++ llvm/trunk/lib/Transforms/Scalar/SampleProfile.cpp @@ -737,8 +737,14 @@ "Sample Profile loader", false, false) bool SampleProfileLoader::doInitialization(Module &M) { - Reader.reset(new SampleProfileReader(M, Filename)); - ProfileIsValid = Reader->load(); + if (std::error_code EC = + SampleProfileReader::create(Filename, Reader, M.getContext())) { + std::string Msg = "Could not open profile: " + EC.message(); + DiagnosticInfoSampleProfile Diag(Filename.data(), Msg); + M.getContext().diagnose(Diag); + return false; + } + ProfileIsValid = (Reader->read() == sampleprof_error::success); return true; } Index: llvm/trunk/test/Transforms/SampleProfile/Inputs/fnptr.prof =================================================================== --- llvm/trunk/test/Transforms/SampleProfile/Inputs/fnptr.prof +++ llvm/trunk/test/Transforms/SampleProfile/Inputs/fnptr.prof @@ -0,0 +1,12 @@ +_Z3fooi:7711:610 +1: 610 +_Z3bari:20301:1437 +1: 1437 +main:184019:0 +4: 534 +6: 2080 +9: 2064 _Z3bari:1471 _Z3fooi:631 +5.1: 1075 +5: 1075 +7: 534 +4.2: 534 Index: llvm/trunk/test/Transforms/SampleProfile/fnptr.ll =================================================================== --- llvm/trunk/test/Transforms/SampleProfile/fnptr.ll +++ llvm/trunk/test/Transforms/SampleProfile/fnptr.ll @@ -0,0 +1,155 @@ +; The two profiles used in this test are the same but encoded in different +; formats. This checks that we produce the same profile annotations regardless +; of the profile format. +; +; RUN: opt < %s -sample-profile -sample-profile-file=%S/Inputs/fnptr.prof | opt -analyze -branch-prob | FileCheck %s +; RUN: opt < %s -sample-profile -sample-profile-file=%S/Inputs/fnptr.binprof | opt -analyze -branch-prob | FileCheck %s + +; CHECK: edge for.body3 -> if.then probability is 534 / 2598 = 20.5543% +; CHECK: edge for.body3 -> if.else probability is 2064 / 2598 = 79.4457% +; CHECK: edge for.inc -> for.inc12 probability is 1052 / 2598 = 40.4927% +; CHECK: edge for.inc -> for.body3 probability is 1546 / 2598 = 59.5073% +; CHECK: edge for.inc12 -> for.end14 probability is 518 / 1052 = 49.2395% +; CHECK: edge for.inc12 -> for.cond1.preheader probability is 534 / 1052 = 50.7605% + +; Original C++ test case. +; +; #include +; #include +; #include +; +; #define N 10000 +; #define M 6000 +; +; double foo(int x) { +; return x * sin((double)x); +; } +; +; double bar(int x) { +; return x - cos((double)x); +; } +; +; int main() { +; double (*fptr)(int); +; double S = 0; +; for (int i = 0; i < N; i++) +; for (int j = 0; j < M; j++) { +; fptr = (rand() % 100 < 30) ? foo : bar; +; if (rand() % 100 < 10) +; S += (*fptr)(i + j * 300); +; else +; S += (*fptr)(i - j / 840); +; } +; printf("S = %lf\n", S); +; return 0; +; } + +@.str = private unnamed_addr constant [9 x i8] c"S = %lf\0A\00", align 1 + +define double @_Z3fooi(i32 %x) #0 { +entry: + %conv = sitofp i32 %x to double, !dbg !2 + %call = tail call double @sin(double %conv) #3, !dbg !8 + %mul = fmul double %conv, %call, !dbg !8 + ret double %mul, !dbg !8 +} + +declare double @sin(double) #1 + +define double @_Z3bari(i32 %x) #0 { +entry: + %conv = sitofp i32 %x to double, !dbg !9 + %call = tail call double @cos(double %conv) #3, !dbg !11 + %sub = fsub double %conv, %call, !dbg !11 + ret double %sub, !dbg !11 +} + +declare double @cos(double) #1 + +define i32 @main() #2 { +entry: + br label %for.cond1.preheader, !dbg !12 + +for.cond1.preheader: ; preds = %for.inc12, %entry + %i.025 = phi i32 [ 0, %entry ], [ %inc13, %for.inc12 ] + %S.024 = phi double [ 0.000000e+00, %entry ], [ %S.2.lcssa, %for.inc12 ] + br label %for.body3, !dbg !14 + +for.body3: ; preds = %for.inc, %for.cond1.preheader + %j.023 = phi i32 [ 0, %for.cond1.preheader ], [ %inc, %for.inc ] + %S.122 = phi double [ %S.024, %for.cond1.preheader ], [ %S.2, %for.inc ] + %call = tail call i32 @rand() #3, !dbg !15 + %rem = srem i32 %call, 100, !dbg !15 + %cmp4 = icmp slt i32 %rem, 30, !dbg !15 + %_Z3fooi._Z3bari = select i1 %cmp4, double (i32)* @_Z3fooi, double (i32)* @_Z3bari, !dbg !15 + %call5 = tail call i32 @rand() #3, !dbg !16 + %rem6 = srem i32 %call5, 100, !dbg !16 + %cmp7 = icmp slt i32 %rem6, 10, !dbg !16 + br i1 %cmp7, label %if.then, label %if.else, !dbg !16, !prof !17 + +if.then: ; preds = %for.body3 + %mul = mul nsw i32 %j.023, 300, !dbg !18 + %add = add nsw i32 %mul, %i.025, !dbg !18 + %call8 = tail call double %_Z3fooi._Z3bari(i32 %add), !dbg !18 + br label %for.inc, !dbg !18 + +if.else: ; preds = %for.body3 + %div = sdiv i32 %j.023, 840, !dbg !19 + %sub = sub nsw i32 %i.025, %div, !dbg !19 + %call10 = tail call double %_Z3fooi._Z3bari(i32 %sub), !dbg !19 + br label %for.inc + +for.inc: ; preds = %if.then, %if.else + %call8.pn = phi double [ %call8, %if.then ], [ %call10, %if.else ] + %S.2 = fadd double %S.122, %call8.pn, !dbg !18 + %inc = add nsw i32 %j.023, 1, !dbg !20 + %exitcond = icmp eq i32 %j.023, 5999, !dbg !14 + br i1 %exitcond, label %for.inc12, label %for.body3, !dbg !14, !prof !21 + +for.inc12: ; preds = %for.inc + %S.2.lcssa = phi double [ %S.2, %for.inc ] + %inc13 = add nsw i32 %i.025, 1, !dbg !22 + %exitcond26 = icmp eq i32 %i.025, 9999, !dbg !12 + br i1 %exitcond26, label %for.end14, label %for.cond1.preheader, !dbg !12, !prof !23 + +for.end14: ; preds = %for.inc12 + %S.2.lcssa.lcssa = phi double [ %S.2.lcssa, %for.inc12 ] + %call15 = tail call i32 (i8*, ...)* @printf(i8* getelementptr inbounds ([9 x i8]* @.str, i64 0, i64 0), double %S.2.lcssa.lcssa), !dbg !24 + ret i32 0, !dbg !25 +} + +; Function Attrs: nounwind +declare i32 @rand() #1 + +; Function Attrs: nounwind +declare i32 @printf(i8* nocapture readonly, ...) #1 + +!llvm.module.flags = !{!0} +!llvm.ident = !{!1} + +!0 = metadata !{i32 2, metadata !"Debug Info Version", i32 2} +!1 = metadata !{metadata !"clang version 3.6.0 "} +!2 = metadata !{i32 9, i32 3, metadata !3, null} +!3 = metadata !{metadata !"0x2e\00foo\00foo\00\008\000\001\000\000\00256\001\008", metadata !4, metadata !5, metadata !6, null, double (i32)* @_Z3fooi, null, null, metadata !7} ; [ DW_TAG_subprogram ] [line 8] [def] [foo] +!4 = metadata !{metadata !"fnptr.cc", metadata !"."} +!5 = metadata !{metadata !"0x29", metadata !4} ; [ DW_TAG_file_type ] [./fnptr.cc] +!6 = metadata !{metadata !"0x15\00\000\000\000\000\000\000", null, null, null, metadata !7, null, null, null} ; [ DW_TAG_subroutine_type ] [line 0, size 0, align 0, offset 0] [from ] +!7 = metadata !{} +!8 = metadata !{i32 9, i32 14, metadata !3, null} +!9 = metadata !{i32 13, i32 3, metadata !10, null} +!10 = metadata !{metadata !"0x2e\00bar\00bar\00\0012\000\001\000\000\00256\001\0012", metadata !4, metadata !5, metadata !6, null, double (i32)* @_Z3bari, null, null, metadata !7} ; [ DW_TAG_subprogram ] [line 12] [def] [bar] +!11 = metadata !{i32 13, i32 14, metadata !10, null} +!12 = metadata !{i32 19, i32 3, metadata !13, null} +!13 = metadata !{metadata !"0x2e\00main\00main\00\0016\000\001\000\000\00256\001\0016", metadata !4, metadata !5, metadata !6, null, i32 ()* @main, null, null, metadata !7} ; [ DW_TAG_subprogram ] [line 16] [def] [main] +!14 = metadata !{i32 20, i32 5, metadata !13, null} +!15 = metadata !{i32 21, i32 15, metadata !13, null} +!16 = metadata !{i32 22, i32 11, metadata !13, null} +!17 = metadata !{metadata !"branch_weights", i32 534, i32 2064} +!18 = metadata !{i32 23, i32 14, metadata !13, null} +!19 = metadata !{i32 25, i32 14, metadata !13, null} +!20 = metadata !{i32 20, i32 28, metadata !13, null} +!21 = metadata !{metadata !"branch_weights", i32 0, i32 1075} +!22 = metadata !{i32 19, i32 26, metadata !13, null} +!23 = metadata !{metadata !"branch_weights", i32 0, i32 534} +!24 = metadata !{i32 27, i32 3, metadata !13, null} +!25 = metadata !{i32 28, i32 3, metadata !13, null} Index: llvm/trunk/test/Transforms/SampleProfile/syntax.ll =================================================================== --- llvm/trunk/test/Transforms/SampleProfile/syntax.ll +++ llvm/trunk/test/Transforms/SampleProfile/syntax.ll @@ -12,7 +12,7 @@ ret void } ; NO-DEBUG: warning: No debug information found in function empty: Function profile not used -; MISSING-FILE: error: missing.prof: +; MISSING-FILE: missing.prof: Could not open profile: No such file or directory ; BAD-FN-HEADER: error: {{.*}}bad_fn_header.prof:1: Expected 'mangled_name:NUM:NUM', found 3empty:100:BAD ; BAD-SAMPLE-LINE: error: {{.*}}bad_sample_line.prof:3: Expected 'NUM[.NUM]: NUM[ mangled_name:NUM]*', found 1: BAD ; BAD-LINE-VALUES: error: {{.*}}bad_line_values.prof:2: Expected 'mangled_name:NUM:NUM', found -1: 10