[Remarks][NFC] Move the serialization to lib/Remarks

[thegameg]

Summary

Separate the remark serialization to YAML from the LLVM Diagnostics.

This adds a new serialization abstraction: remarks::Serializer. It's completely independent from lib/IR and it provides an easy way to replace YAML by providing a new remarks::Serializer.

[JDevlieghere]

LGTM. This seems straightforward. I skipped over the YAML stuff though, as it appears to have been moved.

[sbc100]

This seems to have broken the BUILD_SHARED_LIBS=ON build:

FAILED: lib/libLLVMLTO.so.9svn 
: && /usr/local/google/home/sbc/dev/chromium/src/third_party/llvm-build/Release+Asserts/bin/clang++ -fPIC -fPIC -fvisibility-inlines-hidden -Werror=date-time -Werror=unguarded-availability-new -std=c++11 -Wall -Wextra -Wno-unused-parameter -Wwrite-strings -Wcast-qual -Wmissing-field-initializers -pedantic -Wno-long-long -Wimplicit-fallthrough -Wcovered-switch-default -Wno-noexcept-type -Wnon-virtual-dtor -Wdelete-non-virtual-dtor -Wstring-conversion -fdiagnostics-color -ffunction-sections -fdata-sections -O3  -fuse-ld=gold -Wl,-z,defs -Wl,-z,nodelete   -Wl,-rpath-link,/usr/local/google/home/sbc/dev/wasm/llvm-build/./lib  -Wl,-O3 -Wl,--gc-sections -shared -Wl,-soname,libLLVMLTO.so.9svn -o lib/libLLVMLTO.so.9svn lib/LTO/CMakeFiles/LLVMLTO.dir/Caching.cpp.o lib/LTO/CMakeFiles/LLVMLTO.dir/LTO.cpp.o lib/LTO/CMakeFiles/LLVMLTO.dir/LTOBackend.cpp.o lib/LTO/CMakeFiles/LLVMLTO.dir/LTOModule.cpp.o lib/LTO/CMakeFiles/LLVMLTO.dir/LTOCodeGenerator.cpp.o lib/LTO/CMakeFiles/LLVMLTO.dir/SummaryBasedOptimizations.cpp.o lib/LTO/CMakeFiles/LLVMLTO.dir/UpdateCompilerUsed.cpp.o lib/LTO/CMakeFiles/LLVMLTO.dir/ThinLTOCodeGenerator.cpp.o  -Wl,-rpath,"\$ORIGIN/../lib" lib/libLLVMAggressiveInstCombine.so.9svn lib/libLLVMAnalysis.so.9svn lib/libLLVMBitReader.so.9svn lib/libLLVMBitWriter.so.9svn lib/libLLVMCodeGen.so.9svn lib/libLLVMCore.so.9svn lib/libLLVMInstCombine.so.9svn lib/libLLVMLinker.so.9svn lib/libLLVMMC.so.9svn lib/libLLVMObjCARCOpts.so.9svn lib/libLLVMObject.so.9svn lib/libLLVMPasses.so.9svn lib/libLLVMScalarOpts.so.9svn lib/libLLVMSupport.so.9svn lib/libLLVMTarget.so.9svn lib/libLLVMTransformUtils.so.9svn lib/libLLVMipo.so.9svn && :
lib/LTO/CMakeFiles/LLVMLTO.dir/LTO.cpp.o:LTO.cpp:function llvm::lto::setupOptimizationRemarks(llvm::LLVMContext&, llvm::StringRef, llvm::StringRef, bool, int): error: undefined reference to 'llvm::remarks::YAMLSerializer::YAMLSerializer(llvm::raw_ostream&, llvm::remarks::UseStringTable)'

[sbc100]

Proposed fix: https://reviews.llvm.org/D62709

[thegameg]

Sorry about that. Thanks @sbc100!

cfe/trunk/lib/CodeGen/CodeGenAction.cpp

void HandleTranslationUnit(ASTContext &C) override {
             CodeGenOpts.OptRecordFile, EC, sys::fs::F_None);
         if (EC) {
           Diags.Report(diag::err_cannot_open_file) <<
             CodeGenOpts.OptRecordFile << EC.message();
           return;
         }
 
         Ctx.setRemarkStreamer(llvm::make_unique<RemarkStreamer>(
-            CodeGenOpts.OptRecordFile, OptRecordFile->os()));
+            CodeGenOpts.OptRecordFile,
+            llvm::make_unique<remarks::YAMLSerializer>(OptRecordFile->os())));
 
         if (!CodeGenOpts.OptRecordPasses.empty())
           if (Error E = Ctx.getRemarkStreamer()->setFilter(
                   CodeGenOpts.OptRecordPasses))
             Diags.Report(diag::err_drv_optimization_remark_pattern)
                 << toString(std::move(E)) << CodeGenOpts.OptRecordPasses;
 
         if (CodeGenOpts.getProfileUse() != CodeGenOptions::ProfileNone)

llvm/trunk/include/llvm/IR/DiagnosticInfo.h

public:
   /// Return true if this optimization remark is enabled by one of
   /// of the LLVM command line flags (-pass-remarks, -pass-remarks-missed,
   /// or -pass-remarks-analysis). Note that this only handles the LLVM
   /// flags. We cannot access Clang flags from here (they are handled
   /// in BackendConsumer::OptimizationRemarkHandler).
   virtual bool isEnabled() const = 0;
 
   StringRef getPassName() const { return PassName; }
+  StringRef getRemarkName() const { return RemarkName; }
   std::string getMsg() const;
   Optional<uint64_t> getHotness() const { return Hotness; }
   void setHotness(Optional<uint64_t> H) { Hotness = H; }
 
   bool isVerbose() const { return IsVerbose; }
 
+  ArrayRef<Argument> getArgs() const { return Args; }
+
   static bool classof(const DiagnosticInfo *DI) {
     return (DI->getKind() >= DK_FirstRemark &&
             DI->getKind() <= DK_LastRemark) ||
            (DI->getKind() >= DK_FirstMachineRemark &&
             DI->getKind() <= DK_LastMachineRemark);
   }
 
   bool isPassed() const {

 
 protected:
   /// Name of the pass that triggers this report. If this matches the
   /// regular expression given in -Rpass=regexp, then the remark will
   /// be emitted.
   const char *PassName;
 
   /// Textual identifier for the remark (single-word, camel-case). Can be used
-  /// by external tools reading the YAML output file for optimization remarks to
+  /// by external tools reading the output file for optimization remarks to
   /// identify the remark.
   StringRef RemarkName;
 
   /// If profile information is available, this is the number of times the
   /// corresponding code was executed in a profile instrumentation run.
   Optional<uint64_t> Hotness;
 
   /// Arguments collected via the streaming interface.
   SmallVector<Argument, 4> Args;
 
   /// The remark is expected to be noisy.
   bool IsVerbose = false;
 
   /// If positive, the index of the first argument that only appear in
   /// the optimization records and not in the remark printed in the compiler
   /// output.
   int FirstExtraArgIndex = -1;
-
-  friend struct yaml::MappingTraits<DiagnosticInfoOptimizationBase *>;
 };
 
 /// Allow the insertion operator to return the actual remark type rather than a
 /// common base class.  This allows returning the result of the insertion
 /// directly by value, e.g. return OptimizationRemarkAnalysis(...) << "blah".
 template <class RemarkT>
 RemarkT &
 operator<<(RemarkT &R,
static bool classof(const DiagnosticInfo *DI) {
     return DI->getKind() == DK_Unsupported;
   }
 
   const Twine &getMessage() const { return Msg; }
 
   void print(DiagnosticPrinter &DP) const override;
 };
 
-namespace yaml {
-template <> struct MappingTraits<DiagnosticInfoOptimizationBase *> {
-  static void mapping(IO &io, DiagnosticInfoOptimizationBase *&OptDiag);
-};
-} // namespace yaml
-
 } // end namespace llvm
 
 #endif // LLVM_IR_DIAGNOSTICINFO_H

llvm/trunk/include/llvm/IR/RemarkStreamer.h

 //===- llvm/IR/RemarkStreamer.h - Remark Streamer ---------------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 // This file declares the main interface for outputting remarks.
 //
 //===----------------------------------------------------------------------===//
 
 #ifndef LLVM_IR_REMARKSTREAMER_H
 #define LLVM_IR_REMARKSTREAMER_H
 
 #include "llvm/IR/DiagnosticInfo.h"
-#include "llvm/Remarks/RemarkStringTable.h"
+#include "llvm/Remarks/RemarkSerializer.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/Regex.h"
-#include "llvm/Support/YAMLTraits.h"
 #include "llvm/Support/raw_ostream.h"
 #include <string>
 #include <vector>
 
 namespace llvm {
 /// Streamer for remarks.
 class RemarkStreamer {
   /// The filename that the remark diagnostics are emitted to.
   const std::string Filename;
-  /// The open raw_ostream that the remark diagnostics are emitted to.
-  raw_ostream &OS;
   /// The regex used to filter remarks based on the passes that emit them.
   Optional<Regex> PassFilter;
+  /// The object used to serialize the remarks to a specific format.
+  std::unique_ptr<remarks::Serializer> Serializer;
 
-  /// The YAML streamer.
-  yaml::Output YAMLOutput;
-
-  /// The string table containing all the unique strings used in the output.
-  /// The table will be serialized in a section to be consumed after the
-  /// compilation.
-  remarks::StringTable StrTab;
+  /// Temporary buffer for converting diagnostics into remark objects. This is
+  /// used for the remark arguments that are converted from a vector of
+  /// diagnostic arguments to a vector of remark arguments.
+  SmallVector<remarks::Argument, 8> TmpArgs;
+  /// Convert diagnostics into remark objects. The result uses \p TmpArgs as a
+  /// temporary buffer for the remark arguments, and relies on all the strings
+  /// to be kept in memory until the next call to `toRemark`.
+  /// The lifetime of the members of the result is bound to the lifetime of both
+  /// the remark streamer and the LLVM diagnostics.
+  remarks::Remark toRemark(const DiagnosticInfoOptimizationBase &Diag);
 
 public:
-  RemarkStreamer(StringRef Filename, raw_ostream& OS);
+  RemarkStreamer(StringRef Filename,
+                 std::unique_ptr<remarks::Serializer> Serializer);
   /// Return the filename that the remark diagnostics are emitted to.
   StringRef getFilename() const { return Filename; }
   /// Return stream that the remark diagnostics are emitted to.
-  raw_ostream &getStream() { return OS; }
+  raw_ostream &getStream() { return Serializer->OS; }
+  /// Return the serializer used for this stream.
+  remarks::Serializer &getSerializer() { return *Serializer; }
   /// Set a pass filter based on a regex \p Filter.
   /// Returns an error if the regex is invalid.
   Error setFilter(StringRef Filter);
   /// Emit a diagnostic through the streamer.
   void emit(const DiagnosticInfoOptimizationBase &Diag);
-  /// The string table used during emission.
-  remarks::StringTable &getStringTable() { return StrTab; }
-  const remarks::StringTable &getStringTable() const { return StrTab; }
 };
 } // end namespace llvm
 
 #endif // LLVM_IR_REMARKSTREAMER_H

llvm/trunk/include/llvm/Remarks/RemarkSerializer.h

+//===-- RemarkSerializer.h - Remark serialization interface -----*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file provides an interface for serializing remarks to different formats.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_REMARKS_REMARK_SERIALIZER_H
+#define LLVM_REMARKS_REMARK_SERIALIZER_H
+
+#include "llvm/Remarks/Remark.h"
+#include "llvm/Remarks/RemarkStringTable.h"
+#include "llvm/Support/YAMLTraits.h"
+#include "llvm/Support/raw_ostream.h"
+
+namespace llvm {
+namespace remarks {
+
+/// This is the base class for a remark serializer.
+/// It includes support for using a string table while emitting.
+struct Serializer {
+  /// The open raw_ostream that the remark diagnostics are emitted to.
+  raw_ostream &OS;
+  /// The string table containing all the unique strings used in the output.
+  /// The table can be serialized to be consumed after the compilation.
+  Optional<StringTable> StrTab;
+
+  Serializer(raw_ostream &OS) : OS(OS), StrTab() {}
+
+  /// This is just an interface.
+  virtual ~Serializer() = default;
+  virtual void emit(const Remark &Remark) = 0;
+};
+
+/// Wether the serializer should use a string table while emitting.
+enum class UseStringTable { No, Yes };
+
+/// Serialize the remarks to YAML. One remark entry looks like this:
+/// --- !<TYPE>
+/// Pass:            <PASSNAME>
+/// Name:            <REMARKNAME>
+/// DebugLoc:        { File: <SOURCEFILENAME>, Line: <SOURCELINE>,
+///                    Column: <SOURCECOLUMN> }
+/// Function:        <FUNCTIONNAME>
+/// Args:
+///   - <KEY>: <VALUE>
+///     DebugLoc:        { File: <FILE>, Line: <LINE>, Column: <COL> }
+/// ...
+struct YAMLSerializer : public Serializer {
+  /// The YAML streamer.
+  yaml::Output YAMLOutput;
+
+  YAMLSerializer(raw_ostream &OS,
+                 UseStringTable UseStringTable = UseStringTable::No);
+
+  /// Emit a remark to the stream.
+  void emit(const Remark &Remark) override;
+};
+
+} // end namespace remarks
+} // end namespace llvm
+
+#endif /* LLVM_REMARKS_REMARK_SERIALIZER_H */

llvm/trunk/lib/CodeGen/AsmPrinter/AsmPrinter.cpp

 #include "llvm/MC/MCSubtargetInfo.h"
 #include "llvm/MC/MCSymbol.h"
 #include "llvm/MC/MCSymbolELF.h"
 #include "llvm/MC/MCTargetOptions.h"
 #include "llvm/MC/MCValue.h"
 #include "llvm/MC/SectionKind.h"
 #include "llvm/Pass.h"
 #include "llvm/Remarks/Remark.h"
+#include "llvm/Remarks/RemarkStringTable.h"
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/Format.h"
 #include "llvm/Support/MathExtras.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/TargetRegistry.h"
if (auto *GA = dyn_cast<GlobalAlias>(&GIS)) {
     }
   }
 }
 
 void AsmPrinter::emitRemarksSection(Module &M) {
   RemarkStreamer *RS = M.getContext().getRemarkStreamer();
   if (!RS)
     return;
+  const remarks::Serializer &Serializer = RS->getSerializer();
 
   // Switch to the right section: .remarks/__remarks.
   MCSection *RemarksSection =
       OutContext.getObjectFileInfo()->getRemarksSection();
   OutStreamer->SwitchSection(RemarksSection);
 
   // Emit the magic number.
   OutStreamer->EmitBytes(remarks::Magic);
   // Explicitly emit a '\0'.
   OutStreamer->EmitIntValue(/*Value=*/0, /*Size=*/1);
 
   // Emit the version number: little-endian uint64_t.
   // The version number is located at the offset 0x0 in the section.
   std::array<char, 8> Version;
   support::endian::write64le(Version.data(), remarks::Version);
   OutStreamer->EmitBinaryData(StringRef(Version.data(), Version.size()));
 
   // Emit the string table in the section.
   // Note: we need to use the streamer here to emit it in the section. We can't
   // just use the serialize function with a raw_ostream because of the way
   // MCStreamers work.
-  const remarks::StringTable &StrTab = RS->getStringTable();
-  std::vector<StringRef> StrTabStrings = StrTab.serialize();
-  uint64_t StrTabSize = StrTab.SerializedSize;
+  uint64_t StrTabSize =
+      Serializer.StrTab ? Serializer.StrTab->SerializedSize : 0;
   // Emit the total size of the string table (the size itself excluded):
   // little-endian uint64_t.
   // The total size is located after the version number.
+  // Note: even if no string table is used, emit 0.
   std::array<char, 8> StrTabSizeBuf;
   support::endian::write64le(StrTabSizeBuf.data(), StrTabSize);
   OutStreamer->EmitBinaryData(
       StringRef(StrTabSizeBuf.data(), StrTabSizeBuf.size()));
+
+  if (const Optional<remarks::StringTable> &StrTab = Serializer.StrTab) {
+    std::vector<StringRef> StrTabStrings = StrTab->serialize();
     // Emit a list of null-terminated strings.
     // Note: the order is important here: the ID used in the remarks corresponds
     // to the position of the string in the section.
     for (StringRef Str : StrTabStrings) {
       OutStreamer->EmitBytes(Str);
       // Explicitly emit a '\0'.
       OutStreamer->EmitIntValue(/*Value=*/0, /*Size=*/1);
     }
+  }
 
   // Emit the null-terminated absolute path to the remark file.
   // The path is located at the offset 0x4 in the section.
   StringRef FilenameRef = RS->getFilename();
   SmallString<128> Filename = FilenameRef;
   sys::fs::make_absolute(Filename);
   assert(!Filename.empty() && "The filename can't be empty.");
   OutStreamer->EmitBytes(Filename);

llvm/trunk/lib/IR/DiagnosticInfo.cpp

 #include "llvm/Support/raw_ostream.h"
 #include <atomic>
 #include <cassert>
 #include <memory>
 #include <string>
 
 using namespace llvm;
 
-cl::opt<bool> UseStringTable("remarks-yaml-string-table", cl::init(false));
-
 int llvm::getNextAvailablePluginDiagnosticKind() {
   static std::atomic<int> PluginKindID(DK_FirstPluginKind);
   return ++PluginKindID;
 }
 
 const char *OptimizationRemarkAnalysis::AlwaysPrint = "";
 
 DiagnosticInfoInlineAsm::DiagnosticInfoInlineAsm(const Instruction &I,
for (const DiagnosticInfoOptimizationBase::Argument &Arg :
                                     ? Args.end()
                                     : Args.begin() + FirstExtraArgIndex))
     OS << Arg.Val;
   return OS.str();
 }
 
 void OptimizationRemarkAnalysisFPCommute::anchor() {}
 void OptimizationRemarkAnalysisAliasing::anchor() {}
-
-template <typename T>
-static void mapRemarkHeader(
-    yaml::IO &io, T PassName, T RemarkName, DiagnosticLocation DL,
-    T FunctionName, Optional<uint64_t> Hotness,
-    SmallVectorImpl<DiagnosticInfoOptimizationBase::Argument> &Args) {
-  io.mapRequired("Pass", PassName);
-  io.mapRequired("Name", RemarkName);
-  if (!io.outputting() || DL.isValid())
-    io.mapOptional("DebugLoc", DL);
-  io.mapRequired("Function", FunctionName);
-  io.mapOptional("Hotness", Hotness);
-  io.mapOptional("Args", Args);
-}
-
-namespace llvm {
-namespace yaml {
-
-void MappingTraits<DiagnosticInfoOptimizationBase *>::mapping(
-    IO &io, DiagnosticInfoOptimizationBase *&OptDiag) {
-  assert(io.outputting() && "input not yet implemented");
-
-  if (io.mapTag("!Passed",
-                (OptDiag->getKind() == DK_OptimizationRemark ||
-                 OptDiag->getKind() == DK_MachineOptimizationRemark)))
-    ;
-  else if (io.mapTag(
-               "!Missed",
-               (OptDiag->getKind() == DK_OptimizationRemarkMissed ||
-                OptDiag->getKind() == DK_MachineOptimizationRemarkMissed)))
-    ;
-  else if (io.mapTag(
-               "!Analysis",
-               (OptDiag->getKind() == DK_OptimizationRemarkAnalysis ||
-                OptDiag->getKind() == DK_MachineOptimizationRemarkAnalysis)))
-    ;
-  else if (io.mapTag("!AnalysisFPCommute",
-                     OptDiag->getKind() ==
-                         DK_OptimizationRemarkAnalysisFPCommute))
-    ;
-  else if (io.mapTag("!AnalysisAliasing",
-                     OptDiag->getKind() ==
-                         DK_OptimizationRemarkAnalysisAliasing))
-    ;
-  else if (io.mapTag("!Failure", OptDiag->getKind() == DK_OptimizationFailure))
-    ;
-  else
-    llvm_unreachable("Unknown remark type");
-
-  // These are read-only for now.
-  DiagnosticLocation DL = OptDiag->getLocation();
-  StringRef FN =
-      GlobalValue::dropLLVMManglingEscape(OptDiag->getFunction().getName());
-
-  StringRef PassName(OptDiag->PassName);
-  if (UseStringTable) {
-    remarks::StringTable &StrTab =
-        reinterpret_cast<RemarkStreamer *>(io.getContext())->getStringTable();
-    unsigned PassID = StrTab.add(PassName).first;
-    unsigned NameID = StrTab.add(OptDiag->RemarkName).first;
-    unsigned FunctionID = StrTab.add(FN).first;
-    mapRemarkHeader(io, PassID, NameID, DL, FunctionID, OptDiag->Hotness,
-                    OptDiag->Args);
-  } else {
-    mapRemarkHeader(io, PassName, OptDiag->RemarkName, DL, FN, OptDiag->Hotness,
-                    OptDiag->Args);
-  }
-}
-
-template <> struct MappingTraits<DiagnosticLocation> {
-  static void mapping(IO &io, DiagnosticLocation &DL) {
-    assert(io.outputting() && "input not yet implemented");
-
-    StringRef File = DL.getRelativePath();
-    unsigned Line = DL.getLine();
-    unsigned Col = DL.getColumn();
-
-    if (UseStringTable) {
-      remarks::StringTable &StrTab =
-          reinterpret_cast<RemarkStreamer *>(io.getContext())->getStringTable();
-      unsigned FileID = StrTab.add(File).first;
-      io.mapRequired("File", FileID);
-    } else {
-      io.mapRequired("File", File);
-    }
-
-    io.mapRequired("Line", Line);
-    io.mapRequired("Column", Col);
-  }
-
-  static const bool flow = true;
-};
-
-/// Helper struct for multiline string block literals. Use this type to preserve
-/// newlines in strings.
-struct StringBlockVal {
-  StringRef Value;
-  StringBlockVal(const std::string &Value) : Value(Value) {}
-};
-
-template <> struct BlockScalarTraits<StringBlockVal> {
-  static void output(const StringBlockVal &S, void *Ctx, raw_ostream &OS) {
-    return ScalarTraits<StringRef>::output(S.Value, Ctx, OS);
-  }
-
-  static StringRef input(StringRef Scalar, void *Ctx, StringBlockVal &S) {
-    return ScalarTraits<StringRef>::input(Scalar, Ctx, S.Value);
-  }
-};
-
-// Implement this as a mapping for now to get proper quotation for the value.
-template <> struct MappingTraits<DiagnosticInfoOptimizationBase::Argument> {
-  static void mapping(IO &io, DiagnosticInfoOptimizationBase::Argument &A) {
-    assert(io.outputting() && "input not yet implemented");
-
-    if (UseStringTable) {
-      remarks::StringTable &StrTab =
-          reinterpret_cast<RemarkStreamer *>(io.getContext())->getStringTable();
-      auto ValueID = StrTab.add(A.Val).first;
-      io.mapRequired(A.Key.data(), ValueID);
-    } else if (StringRef(A.Val).count('\n') > 1) {
-      StringBlockVal S(A.Val);
-      io.mapRequired(A.Key.data(), S);
-    } else {
-      io.mapRequired(A.Key.data(), A.Val);
-    }
-    if (A.Loc.isValid())
-      io.mapOptional("DebugLoc", A.Loc);
-  }
-};
-
-} // end namespace yaml
-} // end namespace llvm
-
-LLVM_YAML_IS_SEQUENCE_VECTOR(DiagnosticInfoOptimizationBase::Argument)

llvm/trunk/lib/IR/RemarkStreamer.cpp

 //===- llvm/IR/RemarkStreamer.cpp - Remark Streamer -*- C++ -------------*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 // This file contains the implementation of the remark outputting as part of
 // LLVMContext.
 //
 //===----------------------------------------------------------------------===//
 
 #include "llvm/IR/RemarkStreamer.h"
+#include "llvm/IR/DiagnosticInfo.h"
+#include "llvm/IR/Function.h"
+#include "llvm/IR/GlobalValue.h"
 
 using namespace llvm;
 
-RemarkStreamer::RemarkStreamer(StringRef Filename, raw_ostream &OS)
-    : Filename(Filename), OS(OS),
-      YAMLOutput(OS, reinterpret_cast<void *>(this)), StrTab() {
+RemarkStreamer::RemarkStreamer(StringRef Filename,
+                               std::unique_ptr<remarks::Serializer> Serializer)
+    : Filename(Filename), PassFilter(), Serializer(std::move(Serializer)) {
   assert(!Filename.empty() && "This needs to be a real filename.");
 }
 
 Error RemarkStreamer::setFilter(StringRef Filter) {
   Regex R = Regex(Filter);
   std::string RegexError;
   if (!R.isValid(RegexError))
     return createStringError(std::make_error_code(std::errc::invalid_argument),
                              RegexError.data());
   PassFilter = std::move(R);
   return Error::success();
 }
 
+/// DiagnosticKind -> remarks::Type
+static remarks::Type toRemarkType(enum DiagnosticKind Kind) {
+  switch (Kind) {
+  default:
+    return remarks::Type::Unknown;
+  case DK_OptimizationRemark:
+  case DK_MachineOptimizationRemark:
+    return remarks::Type::Passed;
+  case DK_OptimizationRemarkMissed:
+  case DK_MachineOptimizationRemarkMissed:
+    return remarks::Type::Missed;
+  case DK_OptimizationRemarkAnalysis:
+  case DK_MachineOptimizationRemarkAnalysis:
+    return remarks::Type::Analysis;
+  case DK_OptimizationRemarkAnalysisFPCommute:
+    return remarks::Type::AnalysisFPCommute;
+  case DK_OptimizationRemarkAnalysisAliasing:
+    return remarks::Type::AnalysisAliasing;
+  case DK_OptimizationFailure:
+    return remarks::Type::Failure;
+  }
+}
+
+/// DiagnosticLocation -> remarks::RemarkLocation.
+static Optional<remarks::RemarkLocation>
+toRemarkLocation(const DiagnosticLocation &DL) {
+  if (!DL.isValid())
+    return None;
+  StringRef File = DL.getRelativePath();
+  unsigned Line = DL.getLine();
+  unsigned Col = DL.getColumn();
+  return remarks::RemarkLocation{File, Line, Col};
+}
+
+/// LLVM Diagnostic -> Remark
+remarks::Remark
+RemarkStreamer::toRemark(const DiagnosticInfoOptimizationBase &Diag) {
+  // Re-use the buffer.
+  TmpArgs.clear();
+
+  remarks::Remark R; // The result.
+  R.RemarkType = toRemarkType(static_cast<DiagnosticKind>(Diag.getKind()));
+  R.PassName = Diag.getPassName();
+  R.RemarkName = Diag.getRemarkName();
+  R.FunctionName =
+      GlobalValue::dropLLVMManglingEscape(Diag.getFunction().getName());
+  R.Loc = toRemarkLocation(Diag.getLocation());
+  R.Hotness = Diag.getHotness();
+
+  // Use TmpArgs to build the list of arguments and re-use the memory allocated
+  // from previous remark conversions.
+  for (const DiagnosticInfoOptimizationBase::Argument &Arg : Diag.getArgs()) {
+    TmpArgs.emplace_back();
+    TmpArgs.back().Key = Arg.Key;
+    TmpArgs.back().Val = Arg.Val;
+    TmpArgs.back().Loc = toRemarkLocation(Arg.Loc);
+  }
+  R.Args = TmpArgs; // This is valid until the next call to this function.
+
+  return R;
+}
+
 void RemarkStreamer::emit(const DiagnosticInfoOptimizationBase &Diag) {
   if (Optional<Regex> &Filter = PassFilter)
     if (!Filter->match(Diag.getPassName()))
       return;
 
-  DiagnosticInfoOptimizationBase *DiagPtr =
-      const_cast<DiagnosticInfoOptimizationBase *>(&Diag);
-  YAMLOutput << DiagPtr;
+  // First, convert the diagnostic to a remark.
+  remarks::Remark R = toRemark(Diag);
+  // Then, emit the remark through the serializer.
+  Serializer->emit(R);
 }

llvm/trunk/lib/LTO/LTO.cpp

lto::setupOptimizationRemarks(LLVMContext &Context,
   if (Count != -1)
     Filename += ".thin." + llvm::utostr(Count) + ".yaml";
 
   std::error_code EC;
   auto DiagnosticFile =
       llvm::make_unique<ToolOutputFile>(Filename, EC, sys::fs::F_None);
   if (EC)
     return errorCodeToError(EC);
-  Context.setRemarkStreamer(
-      llvm::make_unique<RemarkStreamer>(Filename, DiagnosticFile->os()));
+  Context.setRemarkStreamer(llvm::make_unique<RemarkStreamer>(
+      Filename,
+      llvm::make_unique<remarks::YAMLSerializer>(DiagnosticFile->os())));
 
   if (!LTORemarksPasses.empty())
     if (Error E = Context.getRemarkStreamer()->setFilter(LTORemarksPasses))
       return std::move(E);
 
   DiagnosticFile->keep();
   return std::move(DiagnosticFile);
 }

llvm/trunk/lib/Remarks/CMakeLists.txt

 add_llvm_library(LLVMRemarks
   Remark.cpp
   RemarkParser.cpp
   RemarkStringTable.cpp
   YAMLRemarkParser.cpp
+  YAMLRemarkSerializer.cpp
 )

llvm/trunk/lib/Remarks/YAMLRemarkSerializer.cpp

+//===- YAMLRemarkSerializer.cpp -------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file provides the implementation of the YAML remark serializer using
+// LLVM's YAMLTraits.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Remarks/RemarkSerializer.h"
+#include "llvm/Support/CommandLine.h"
+
+using namespace llvm;
+using namespace llvm::remarks;
+
+cl::opt<bool> RemarksYAMLStringTable("remarks-yaml-string-table",
+                                     cl::init(false));
+
+// Use the same keys whether we use a string table or not (respectively, T is an
+// unsigned or a StringRef).
+template <typename T>
+static void mapRemarkHeader(yaml::IO &io, T PassName, T RemarkName,
+                            Optional<RemarkLocation> RL, T FunctionName,
+                            Optional<uint64_t> Hotness,
+                            ArrayRef<Argument> Args) {
+  io.mapRequired("Pass", PassName);
+  io.mapRequired("Name", RemarkName);
+  io.mapOptional("DebugLoc", RL);
+  io.mapRequired("Function", FunctionName);
+  io.mapOptional("Hotness", Hotness);
+  io.mapOptional("Args", Args);
+}
+
+namespace llvm {
+namespace yaml {
+
+template <> struct MappingTraits<remarks::Remark *> {
+  static void mapping(IO &io, remarks::Remark *&Remark) {
+    assert(io.outputting() && "input not yet implemented");
+
+    if (io.mapTag("!Passed", (Remark->RemarkType == Type::Passed)))
+      ;
+    else if (io.mapTag("!Missed", (Remark->RemarkType == Type::Missed)))
+      ;
+    else if (io.mapTag("!Analysis", (Remark->RemarkType == Type::Analysis)))
+      ;
+    else if (io.mapTag("!AnalysisFPCommute",
+                       (Remark->RemarkType == Type::AnalysisFPCommute)))
+      ;
+    else if (io.mapTag("!AnalysisAliasing",
+                       (Remark->RemarkType == Type::AnalysisAliasing)))
+      ;
+    else if (io.mapTag("!Failure", (Remark->RemarkType == Type::Failure)))
+      ;
+    else
+      llvm_unreachable("Unknown remark type");
+
+    if (Optional<StringTable> &StrTab =
+            reinterpret_cast<YAMLSerializer *>(io.getContext())->StrTab) {
+      unsigned PassID = StrTab->add(Remark->PassName).first;
+      unsigned NameID = StrTab->add(Remark->RemarkName).first;
+      unsigned FunctionID = StrTab->add(Remark->FunctionName).first;
+      mapRemarkHeader(io, PassID, NameID, Remark->Loc, FunctionID,
+                      Remark->Hotness, Remark->Args);
+    } else {
+      mapRemarkHeader(io, Remark->PassName, Remark->RemarkName, Remark->Loc,
+                      Remark->FunctionName, Remark->Hotness, Remark->Args);
+    }
+  }
+};
+
+template <> struct MappingTraits<RemarkLocation> {
+  static void mapping(IO &io, RemarkLocation &RL) {
+    assert(io.outputting() && "input not yet implemented");
+
+    StringRef File = RL.SourceFilePath;
+    unsigned Line = RL.SourceLine;
+    unsigned Col = RL.SourceColumn;
+
+    if (Optional<StringTable> &StrTab =
+            reinterpret_cast<YAMLSerializer *>(io.getContext())->StrTab) {
+      unsigned FileID = StrTab->add(File).first;
+      io.mapRequired("File", FileID);
+    } else {
+      io.mapRequired("File", File);
+    }
+
+    io.mapRequired("Line", Line);
+    io.mapRequired("Column", Col);
+  }
+
+  static const bool flow = true;
+};
+
+/// Helper struct for multiline string block literals. Use this type to preserve
+/// newlines in strings.
+struct StringBlockVal {
+  StringRef Value;
+  StringBlockVal(const std::string &Value) : Value(Value) {}
+};
+
+template <> struct BlockScalarTraits<StringBlockVal> {
+  static void output(const StringBlockVal &S, void *Ctx, raw_ostream &OS) {
+    return ScalarTraits<StringRef>::output(S.Value, Ctx, OS);
+  }
+
+  static StringRef input(StringRef Scalar, void *Ctx, StringBlockVal &S) {
+    return ScalarTraits<StringRef>::input(Scalar, Ctx, S.Value);
+  }
+};
+
+/// ArrayRef is not really compatible with the YAMLTraits. Everything should be
+/// immutable in an ArrayRef, while the SequenceTraits expect a mutable version
+/// for inputting, but we're only using the outputting capabilities here.
+/// This is a hack, but still nicer than having to manually call the YAMLIO
+/// internal methods.
+/// Keep this in this file so that it doesn't get misused from YAMLTraits.h.
+template <typename T> struct SequenceTraits<ArrayRef<T>> {
+  static size_t size(IO &io, ArrayRef<T> &seq) { return seq.size(); }
+  static Argument &element(IO &io, ArrayRef<T> &seq, size_t index) {
+    assert(io.outputting() && "input not yet implemented");
+    // The assert above should make this "safer" to satisfy the YAMLTraits.
+    return const_cast<T &>(seq[index]);
+  }
+};
+
+/// Implement this as a mapping for now to get proper quotation for the value.
+template <> struct MappingTraits<Argument> {
+  static void mapping(IO &io, Argument &A) {
+    assert(io.outputting() && "input not yet implemented");
+
+    if (Optional<StringTable> &StrTab =
+            reinterpret_cast<YAMLSerializer *>(io.getContext())->StrTab) {
+      auto ValueID = StrTab->add(A.Val).first;
+      io.mapRequired(A.Key.data(), ValueID);
+    } else if (StringRef(A.Val).count('\n') > 1) {
+      StringBlockVal S(A.Val);
+      io.mapRequired(A.Key.data(), S);
+    } else {
+      io.mapRequired(A.Key.data(), A.Val);
+    }
+    io.mapOptional("DebugLoc", A.Loc);
+  }
+};
+
+} // end namespace yaml
+} // end namespace llvm
+
+LLVM_YAML_IS_SEQUENCE_VECTOR(Argument)
+
+YAMLSerializer::YAMLSerializer(raw_ostream &OS, UseStringTable UseStringTable)
+    : Serializer(OS), YAMLOutput(OS, reinterpret_cast<void *>(this)) {
+  if (UseStringTable == UseStringTable::Yes || RemarksYAMLStringTable)
+    StrTab.emplace();
+}
+
+void YAMLSerializer::emit(const Remark &Remark) {
+  // Again, YAMLTraits expect a non-const object for inputting, but we're not
+  // using that here.
+  auto R = const_cast<remarks::Remark *>(&Remark);
+  YAMLOutput << R;
+}

llvm/trunk/tools/llc/llc.cpp

int main(int argc, char **argv) {
   if (RemarksFilename != "") {
     std::error_code EC;
     YamlFile =
         llvm::make_unique<ToolOutputFile>(RemarksFilename, EC, sys::fs::F_None);
     if (EC) {
       WithColor::error(errs(), argv[0]) << EC.message() << '\n';
       return 1;
     }
-    Context.setRemarkStreamer(
-        llvm::make_unique<RemarkStreamer>(RemarksFilename, YamlFile->os()));
+    Context.setRemarkStreamer(llvm::make_unique<RemarkStreamer>(
+        RemarksFilename,
+        llvm::make_unique<remarks::YAMLSerializer>(YamlFile->os())));
 
     if (!RemarksPasses.empty())
       if (Error E = Context.getRemarkStreamer()->setFilter(RemarksPasses)) {
         WithColor::error(errs(), argv[0]) << E << '\n';
         return 1;
       }
   }
 

llvm/trunk/tools/opt/opt.cpp

if (RemarksFilename != "") {
     std::error_code EC;
     OptRemarkFile =
         llvm::make_unique<ToolOutputFile>(RemarksFilename, EC, sys::fs::F_None);
     if (EC) {
       errs() << EC.message() << '\n';
       return 1;
     }
     Context.setRemarkStreamer(llvm::make_unique<RemarkStreamer>(
-        RemarksFilename, OptRemarkFile->os()));
+        RemarksFilename,
+        llvm::make_unique<remarks::YAMLSerializer>(OptRemarkFile->os())));
 
     if (!RemarksPasses.empty())
       if (Error E = Context.getRemarkStreamer()->setFilter(RemarksPasses)) {
         errs() << E << '\n';
         return 1;
       }
   }