YAML: Add support for literal block scalar I/O.

[arphaman]

Summary

This patch builds on top of the patch that implements parsing of YAML block scalars (http://reviews.llvm.org/D9503).
This patch gives the users an ability to serialize scalars using the YAML literal block scalar notation by allowing them
to implement a specialization of the yaml::BlockScalarTraits class for their custom types.

llvm/trunk/docs/YamlIO.rst

struct ScalarTraits<MyCustomType> {
       static StringRef input(StringRef scalar, void*, T &value) {
         // do custom parsing here.  Return the empty string on success,
         // or an error message on failure.
         return StringRef();
       }
       // Determine if this scalar needs quotes.
       static bool mustQuote(StringRef) { return true; }
     };
-    
+Block Scalars
+-------------
+
+YAML block scalars are string literals that are represented in YAML using the
+literal block notation, just like the example shown below:
+
+.. code-block:: yaml
+
+    text: |
+      First line
+      Second line
+
+The YAML I/O library provides support for translating between YAML block scalars
+and specific C++ types by allowing you to specialize BlockScalarTraits<> on
+your data type. The library doesn't provide any built-in support for block
+scalar I/O for types like std::string and llvm::StringRef as they are already
+supported by YAML I/O and use the ordinary scalar notation by default.
+
+BlockScalarTraits specializations are very similar to the
+ScalarTraits specialization - YAML I/O will provide the native type and your
+specialization must create a temporary llvm::StringRef when writing, and
+it will also provide an llvm::StringRef that has the value of that block scalar
+and your specialization must convert that to your native data type when reading.
+An example of a custom type with an appropriate specialization of
+BlockScalarTraits is shown below:
+
+.. code-block:: c++
+
+    using llvm::yaml::BlockScalarTraits;
+    using llvm::yaml::IO;
+
+    struct MyStringType {
+      std::string Str;
+    };
+
+    template <>
+    struct BlockScalarTraits<MyStringType> {
+      static void output(const MyStringType &Value, void *Ctxt,
+                         llvm::raw_ostream &OS) {
+        OS << Value.Str;
+      }
+
+      static StringRef input(StringRef Scalar, void *Ctxt,
+                             MyStringType &Value) {
+        Value.Str = Scalar.str();
+        return StringRef();
+      }
+    };
+
+    
 
 Mappings
 ========
 
 To be translated to or from a YAML mapping for your type T you must specialize  
 llvm::yaml::MappingTraits on T and implement the "void mapping(IO &io, T&)" 
 method. If your native data structures use pointers to a class everywhere,
 you can specialize on the class pointer.  Examples:

llvm/trunk/include/llvm/Support/YAMLTraits.h

struct ScalarTraits {
   // StringRef on success or an error string if string is malformed:
   //static StringRef input(StringRef scalar, void *ctxt, T &value);
   //
   // Function to determine if the value should be quoted.
   //static bool mustQuote(StringRef);
 };
 
 
+/// This class should be specialized by type that requires custom conversion
+/// to/from a YAML literal block scalar. For example:
+///
+///    template <>
+///    struct BlockScalarTraits<MyType> {
+///      static void output(const MyType &Value, void*, llvm::raw_ostream &Out)
+///      {
+///        // stream out custom formatting
+///        Out << Val;
+///      }
+///      static StringRef input(StringRef Scalar, void*, MyType &Value) {
+///        // parse scalar and set `value`
+///        // return empty string on success, or error string
+///        return StringRef();
+///      }
+///    };
+template <typename T>
+struct BlockScalarTraits {
+  // Must provide:
+  //
+  // Function to write the value as a string:
+  // static void output(const T &Value, void *ctx, llvm::raw_ostream &Out);
+  //
+  // Function to convert a string to a value.  Returns the empty
+  // StringRef on success or an error string if string is malformed:
+  // static StringRef input(StringRef Scalar, void *ctxt, T &Value);
+};
+
+
 /// This class should be specialized by any type that needs to be converted
 /// to/from a YAML sequence.  For example:
 ///
 ///    template<>
 ///    struct SequenceTraits< std::vector<MyType> > {
 ///      static size_t size(IO &io, std::vector<MyType> &seq) {
 ///        return seq.size();
 ///      }
struct has_ScalarTraits
   static double test(...);
 
 public:
   static bool const value =
       (sizeof(test<ScalarTraits<T>>(nullptr, nullptr, nullptr)) == 1);
 };
 
 
+// Test if BlockScalarTraits<T> is defined on type T.
+template <class T>
+struct has_BlockScalarTraits
+{
+  typedef StringRef (*Signature_input)(StringRef, void *, T &);
+  typedef void (*Signature_output)(const T &, void *, llvm::raw_ostream &);
+
+  template <typename U>
+  static char test(SameType<Signature_input, &U::input> *,
+                   SameType<Signature_output, &U::output> *);
+
+  template <typename U>
+  static double test(...);
+
+public:
+  static bool const value =
+      (sizeof(test<BlockScalarTraits<T>>(nullptr, nullptr)) == 1);
+};
+
+
 // Test if MappingTraits<T> is defined on type T.
 template <class T>
 struct has_MappingTraits
 {
   typedef void (*Signature_mapping)(class IO&, T&);
 
   template <typename U>
   static char test(SameType<Signature_mapping, &U::mapping>*);

 }
 
 
 template<typename T>
 struct missingTraits : public std::integral_constant<bool,
                                          !has_ScalarEnumerationTraits<T>::value
                                       && !has_ScalarBitSetTraits<T>::value
                                       && !has_ScalarTraits<T>::value
+                                      && !has_BlockScalarTraits<T>::value
                                       && !has_MappingTraits<T>::value
                                       && !has_SequenceTraits<T>::value
                                       && !has_DocumentListTraits<T>::value >  {};
 
 template<typename T>
 struct validatedMappingTraits : public std::integral_constant<bool,
                                        has_MappingTraits<T>::value
                                     && has_MappingValidateTraits<T>::value> {};
public:
   virtual bool matchEnumFallback() = 0;
   virtual void endEnumScalar() = 0;
 
   virtual bool beginBitSetScalar(bool &) = 0;
   virtual bool bitSetMatch(const char*, bool) = 0;
   virtual void endBitSetScalar() = 0;
 
   virtual void scalarString(StringRef &, bool) = 0;
+  virtual void blockScalarString(StringRef &) = 0;
 
   virtual void setError(const Twine &) = 0;
 
   template <typename T>
   void enumCase(T &Val, const char* Str, const T ConstVal) {
     if ( matchEnumScalar(Str, outputting() && Val == ConstVal) ) {
       Val = ConstVal;
     }
else {
     io.scalarString(Str, ScalarTraits<T>::mustQuote(Str));
     StringRef Result = ScalarTraits<T>::input(Str, io.getContext(), Val);
     if ( !Result.empty() ) {
       io.setError(llvm::Twine(Result));
     }
   }
 }
 
+template <typename T>
+typename std::enable_if<has_BlockScalarTraits<T>::value, void>::type
+yamlize(IO &YamlIO, T &Val, bool) {
+  if (YamlIO.outputting()) {
+    std::string Storage;
+    llvm::raw_string_ostream Buffer(Storage);
+    BlockScalarTraits<T>::output(Val, YamlIO.getContext(), Buffer);
+    StringRef Str = Buffer.str();
+    YamlIO.blockScalarString(Str);
+  } else {
+    StringRef Str;
+    YamlIO.blockScalarString(Str);
+    StringRef Result =
+        BlockScalarTraits<T>::input(Str, YamlIO.getContext(), Val);
+    if (!Result.empty())
+      YamlIO.setError(llvm::Twine(Result));
+  }
+}
 
 template<typename T>
 typename std::enable_if<validatedMappingTraits<T>::value, void>::type
 yamlize(IO &io, T &Val, bool) {
   if (has_FlowTraits<MappingTraits<T>>::value)
     io.beginFlowMapping();
   else
     io.beginMapping();
private:
   void beginEnumScalar() override;
   bool matchEnumScalar(const char*, bool) override;
   bool matchEnumFallback() override;
   void endEnumScalar() override;
   bool beginBitSetScalar(bool &) override;
   bool bitSetMatch(const char *, bool ) override;
   void endBitSetScalar() override;
   void scalarString(StringRef &, bool) override;
+  void blockScalarString(StringRef &) override;
   void setError(const Twine &message) override;
   bool canElideEmptySequence() override;
 
   class HNode {
     virtual void anchor();
   public:
     HNode(Node *n) : _node(n) { }
     virtual ~HNode() { }
private:
   class ScalarHNode : public HNode {
     void anchor() override;
   public:
     ScalarHNode(Node *n, StringRef s) : HNode(n), _value(s) { }
 
     StringRef value() const { return _value; }
 
     static inline bool classof(const HNode *n) {
-      return ScalarNode::classof(n->_node);
+      return ScalarNode::classof(n->_node) ||
+             BlockScalarNode::classof(n->_node);
     }
     static inline bool classof(const ScalarHNode *) { return true; }
   protected:
     StringRef _value;
   };
 
   class MapHNode : public HNode {
     void anchor() override;
public:
   void beginEnumScalar() override;
   bool matchEnumScalar(const char*, bool) override;
   bool matchEnumFallback() override;
   void endEnumScalar() override;
   bool beginBitSetScalar(bool &) override;
   bool bitSetMatch(const char *, bool ) override;
   void endBitSetScalar() override;
   void scalarString(StringRef &, bool) override;
+  void blockScalarString(StringRef &) override;
   void setError(const Twine &message) override;
   bool canElideEmptySequence() override;
 public:
   // These are only used by operator<<. They could be private
   // if that templated operator could be made a friend.
   void beginDocuments();
   bool preflightDocument(unsigned);
   void postflightDocument();

 inline
 typename std::enable_if<has_SequenceTraits<T>::value, Input &>::type
 operator>>(Input &yin, T &docSeq) {
   if (yin.setCurrentDocument())
     yamlize(yin, docSeq, true);
   return yin;
 }
 
+// Define non-member operator>> so that Input can stream in a block scalar.
+template <typename T>
+inline
+typename std::enable_if<has_BlockScalarTraits<T>::value, Input &>::type
+operator>>(Input &In, T &Val) {
+  if (In.setCurrentDocument())
+    yamlize(In, Val, true);
+  return In;
+}
+
 // Provide better error message about types missing a trait specialization
 template <typename T>
 inline
 typename std::enable_if<missingTraits<T>::value, Input &>::type
 operator>>(Input &yin, T &docSeq) {
   char missing_yaml_trait_for_type[sizeof(MissingTrait<T>)];
   return yin;
 }
operator<<(Output &yout, T &seq) {
   if ( yout.preflightDocument(0) ) {
     yamlize(yout, seq, true);
     yout.postflightDocument();
   }
   yout.endDocuments();
   return yout;
 }
 
+// Define non-member operator<< so that Output can stream out a block scalar.
+template <typename T>
+inline
+typename std::enable_if<has_BlockScalarTraits<T>::value, Output &>::type
+operator<<(Output &Out, T &Val) {
+  Out.beginDocuments();
+  if (Out.preflightDocument(0)) {
+    yamlize(Out, Val, true);
+    Out.postflightDocument();
+  }
+  Out.endDocuments();
+  return Out;
+}
+
 // Provide better error message about types missing a trait specialization
 template <typename T>
 inline
 typename std::enable_if<missingTraits<T>::value, Output &>::type
 operator<<(Output &yout, T &seq) {
   char missing_yaml_trait_for_type[sizeof(MissingTrait<T>)];
   return yout;
 }

llvm/trunk/lib/Support/YAMLTraits.cpp

 //===- lib/Support/YAMLTraits.cpp -----------------------------------------===//
 //
 //                             The LLVM Linker
 //
 // This file is distributed under the University of Illinois Open Source
 // License. See LICENSE.TXT for details.
 //
 //===----------------------------------------------------------------------===//
 
 #include "llvm/Support/YAMLTraits.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/Twine.h"
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/Errc.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/Format.h"
+#include "llvm/Support/LineIterator.h"
 #include "llvm/Support/YAMLParser.h"
 #include "llvm/Support/raw_ostream.h"
 #include <cctype>
 #include <cstring>
 using namespace llvm;
 using namespace yaml;
 
 //===----------------------------------------------------------------------===//

 void Input::scalarString(StringRef &S, bool) {
   if (ScalarHNode *SN = dyn_cast<ScalarHNode>(CurrentNode)) {
     S = SN->value();
   } else {
     setError(CurrentNode, "unexpected scalar");
   }
 }
 
+void Input::blockScalarString(StringRef &S) { scalarString(S, false); }
+
 void Input::setError(HNode *hnode, const Twine &message) {
   assert(hnode && "HNode must not be NULL");
   this->setError(hnode->_node, message);
 }
 
 void Input::setError(Node *node, const Twine &message) {
   Strm->printError(node, message);
   EC = make_error_code(errc::invalid_argument);
 }
 
 std::unique_ptr<Input::HNode> Input::createHNodes(Node *N) {
   SmallString<128> StringStorage;
   if (ScalarNode *SN = dyn_cast<ScalarNode>(N)) {
     StringRef KeyStr = SN->getValue(StringStorage);
     if (!StringStorage.empty()) {
       // Copy string to permanent storage
       unsigned Len = StringStorage.size();
       char *Buf = StringAllocator.Allocate<char>(Len);
       memcpy(Buf, &StringStorage[0], Len);
       KeyStr = StringRef(Buf, Len);
     }
     return llvm::make_unique<ScalarHNode>(N, KeyStr);
+  } else if (BlockScalarNode *BSN = dyn_cast<BlockScalarNode>(N)) {
+    StringRef Value = BSN->getValue();
+    char *Buf = StringAllocator.Allocate<char>(Value.size());
+    memcpy(Buf, Value.data(), Value.size());
+    return llvm::make_unique<ScalarHNode>(N, StringRef(Buf, Value.size()));
   } else if (SequenceNode *SQ = dyn_cast<SequenceNode>(N)) {
     auto SQHNode = llvm::make_unique<SequenceHNode>(N);
     for (Node &SN : *SQ) {
       auto Entry = this->createHNodes(&SN);
       if (EC)
         break;
       SQHNode->Entries.push_back(std::move(Entry));
     }
if (S[j] == '\'') {
       i = j + 1;
     }
     ++j;
   }
   output(StringRef(&Base[i], j - i));
   this->outputUpToEndOfLine("'"); // Ending single quote.
 }
 
+void Output::blockScalarString(StringRef &S) {
+  if (!StateStack.empty())
+    newLineCheck();
+  output(" |");
+  outputNewLine();
+
+  unsigned Indent = StateStack.empty() ? 1 : StateStack.size();
+
+  auto Buffer = MemoryBuffer::getMemBuffer(S, "", false);
+  for (line_iterator Lines(*Buffer, false); !Lines.is_at_end(); ++Lines) {
+    for (unsigned I = 0; I < Indent; ++I) {
+      output("  ");
+    }
+    output(*Lines);
+    outputNewLine();
+  }
+}
+
 void Output::setError(const Twine &message) {
 }
 
 bool Output::canElideEmptySequence() {
   // Normally, with an optional key/value where the value is an empty sequence,
   // the whole key/value can be not written.  But, that produces wrong yaml
   // if the key/value is the only thing in the map and the map is used in
   // a sequence.  This detects if the this sequence is the first key/value

llvm/trunk/unittests/Support/YAMLIOTest.cpp

std::string intermediate;
     EXPECT_EQ(100,    map2.f2.length);
     EXPECT_EQ(400,    map2.f2.width);
     EXPECT_EQ(10,     map2.f3);
   }
 }
 
 
 //===----------------------------------------------------------------------===//
+//  Test BlockScalarTraits
+//===----------------------------------------------------------------------===//
+
+struct MultilineStringType {
+  std::string str;
+};
+
+struct MultilineStringTypeMap {
+  MultilineStringType name;
+  MultilineStringType description;
+  MultilineStringType ingredients;
+  MultilineStringType recipes;
+  MultilineStringType warningLabels;
+  MultilineStringType documentation;
+  int price;
+};
+
+namespace llvm {
+namespace yaml {
+  template <>
+  struct MappingTraits<MultilineStringTypeMap> {
+    static void mapping(IO &io, MultilineStringTypeMap& s) {
+      io.mapRequired("name", s.name);
+      io.mapRequired("description", s.description);
+      io.mapRequired("ingredients", s.ingredients);
+      io.mapRequired("recipes", s.recipes);
+      io.mapRequired("warningLabels", s.warningLabels);
+      io.mapRequired("documentation", s.documentation);
+      io.mapRequired("price", s.price);
+     }
+  };
+
+  // MultilineStringType is formatted as a yaml block literal scalar. A value of
+  // "Hello\nWorld" would be represented in yaml as
+  //  |
+  //    Hello
+  //    World
+  template <>
+  struct BlockScalarTraits<MultilineStringType> {
+    static void output(const MultilineStringType &value, void *ctxt,
+                       llvm::raw_ostream &out) {
+      out << value.str;
+    }
+    static StringRef input(StringRef scalar, void *ctxt,
+                           MultilineStringType &value) {
+      value.str = scalar.str();
+      return StringRef();
+    }
+  };
+}
+}
+
+LLVM_YAML_IS_DOCUMENT_LIST_VECTOR(MultilineStringType)
+
+//
+// Test writing then reading back custom values
+//
+TEST(YAMLIO, TestReadWriteMultilineStringType) {
+  std::string intermediate;
+  {
+    MultilineStringTypeMap map;
+    map.name.str = "An Item";
+    map.description.str = "Hello\nWorld";
+    map.ingredients.str = "SubItem 1\nSub Item 2\n\nSub Item 3\n";
+    map.recipes.str = "\n\nTest 1\n\n\n";
+    map.warningLabels.str = "";
+    map.documentation.str = "\n\n";
+    map.price = 350;
+
+    llvm::raw_string_ostream ostr(intermediate);
+    Output yout(ostr);
+    yout << map;
+  }
+  {
+    Input yin(intermediate);
+    MultilineStringTypeMap map2;
+    yin >> map2;
+
+    EXPECT_FALSE(yin.error());
+    EXPECT_EQ(map2.name.str, "An Item\n");
+    EXPECT_EQ(map2.description.str, "Hello\nWorld\n");
+    EXPECT_EQ(map2.ingredients.str, "SubItem 1\nSub Item 2\n\nSub Item 3\n");
+    EXPECT_EQ(map2.recipes.str, "\n\nTest 1\n");
+    EXPECT_TRUE(map2.warningLabels.str.empty());
+    EXPECT_TRUE(map2.documentation.str.empty());
+    EXPECT_EQ(map2.price, 350);
+  }
+}
+
+//
+// Test writing then reading back custom values
+//
+TEST(YAMLIO, TestReadWriteBlockScalarDocuments) {
+  std::string intermediate;
+  {
+    std::vector<MultilineStringType> documents;
+    MultilineStringType doc;
+    doc.str = "Hello\nWorld";
+    documents.push_back(doc);
+
+    llvm::raw_string_ostream ostr(intermediate);
+    Output yout(ostr);
+    yout << documents;
+
+    // Verify that the block scalar header was written out on the same line
+    // as the document marker.
+    EXPECT_NE(llvm::StringRef::npos, llvm::StringRef(ostr.str()).find("--- |"));
+  }
+  {
+    Input yin(intermediate);
+    std::vector<MultilineStringType> documents2;
+    yin >> documents2;
+
+    EXPECT_FALSE(yin.error());
+    EXPECT_EQ(documents2.size(), size_t(1));
+    EXPECT_EQ(documents2[0].str, "Hello\nWorld\n");
+  }
+}
+
+TEST(YAMLIO, TestReadWriteBlockScalarValue) {
+  std::string intermediate;
+  {
+    MultilineStringType doc;
+    doc.str = "Just a block\nscalar doc";
+
+    llvm::raw_string_ostream ostr(intermediate);
+    Output yout(ostr);
+    yout << doc;
+  }
+  {
+    Input yin(intermediate);
+    MultilineStringType doc;
+    yin >> doc;
+
+    EXPECT_FALSE(yin.error());
+    EXPECT_EQ(doc.str, "Just a block\nscalar doc\n");
+  }
+}
+
+//===----------------------------------------------------------------------===//
 //  Test flow sequences
 //===----------------------------------------------------------------------===//
 
 LLVM_YAML_STRONG_TYPEDEF(int, MyNumber)
 LLVM_YAML_IS_FLOW_SEQUENCE_VECTOR(MyNumber)
 LLVM_YAML_IS_FLOW_SEQUENCE_VECTOR(llvm::StringRef)
 
 namespace llvm {