[libFuzzer] Use octal instead of hex escape sequences in PrintASCII

[hans]

Summary

Previously, PrintASCII would print the string "\ta" as "\x09a". However, in C/C++ those strings are not the same: the trailing 'a' is part of the escape sequence, which means it's equivalent to "\x9a". This is an annoying quirk of the standard. (See https://eel.is/c++draft/lex.ccon#nt:hexadecimal-escape-sequence)

To fix this, output three-digit octal escape sequences instead. Since octal escapes are limited to max three digits, this avoids the problem of subsequent characters unintentionally becoming part of the escape sequence.

Dictionary files still use the non-C-compatible hex escapes, but I believe we can't change the format since it comes from AFL, and libfuzzer never writes such files, it only has to read them, so they're not affected by this change.

(One alternative, which might be nice since hex sequences are more readable, would be to use hex sequences when the following character is not a valid hex digit. It would add some complexity though, and we'd still need the octals as a fallback, so not sure if it would be worth it.)

[morehouse]

LGTM

compiler-rt/lib/fuzzer/FuzzerIO.h

 std::string TempPath(const char *Prefix, const char *Extension);
 
 bool IsInterestingCoverageFile(const std::string &FileName);
 
 void DupAndCloseStderr();
 
 void CloseStdout();
 
+// For testing.
+FILE *GetOutputFile();
+void SetOutputFile(FILE *NewOutputFile);
+
 void Printf(const char *Fmt, ...);
 void VPrintf(bool Verbose, const char *Fmt, ...);
 
 // Print using raw syscalls, useful when printing at early init stages.
 void RawPrint(const char *Str);
 
 // Platform specific functions:
 bool IsFile(const std::string &Path);

compiler-rt/lib/fuzzer/FuzzerIO.cpp

 #include <iterator>
 #include <sys/stat.h>
 #include <sys/types.h>
 
 namespace fuzzer {
 
 static FILE *OutputFile = stderr;
 
+FILE *GetOutputFile() {
+  return OutputFile;
+}
+
+void SetOutputFile(FILE *NewOutputFile) {
+  OutputFile = NewOutputFile;
+}
+
 long GetEpoch(const std::string &Path) {
   struct stat St;
   if (stat(Path.c_str(), &St))
     return 0;  // Can't stat, be conservative.
   return St.st_mtime;
 }
 
 Unit FileToVector(const std::string &Path, size_t MaxSize, bool ExitOnError) {

compiler-rt/lib/fuzzer/FuzzerUtil.cpp

 void PrintASCIIByte(uint8_t Byte) {
   if (Byte == '\\')
     Printf("\\\\");
   else if (Byte == '"')
     Printf("\\\"");
   else if (Byte >= 32 && Byte < 127)
     Printf("%c", Byte);
   else
-    Printf("\\x%02x", Byte);
+    Printf("\\%03o", Byte);
 }
 
 void PrintASCII(const uint8_t *Data, size_t Size, const char *PrintAfter) {
   for (size_t i = 0; i < Size; i++)
     PrintASCIIByte(Data[i]);
   Printf("%s", PrintAfter);
 }
 

compiler-rt/lib/fuzzer/tests/FuzzerUnittest.cpp

TEST(FuzzerUtil, Base64) {
   EXPECT_EQ("eHk=", Base64({'x', 'y'}));
   EXPECT_EQ("YWJj", Base64({'a', 'b', 'c'}));
   EXPECT_EQ("eHl6", Base64({'x', 'y', 'z'}));
   EXPECT_EQ("YWJjeA==", Base64({'a', 'b', 'c', 'x'}));
   EXPECT_EQ("YWJjeHk=", Base64({'a', 'b', 'c', 'x', 'y'}));
   EXPECT_EQ("YWJjeHl6", Base64({'a', 'b', 'c', 'x', 'y', 'z'}));
 }
 
+#ifdef __GLIBC__
+class PrintfCapture {
+ public:
+  PrintfCapture() {
+    OldOutputFile = GetOutputFile();
+    SetOutputFile(open_memstream(&Buffer, &Size));
+  }
+  ~PrintfCapture() {
+    fclose(GetOutputFile());
+    SetOutputFile(OldOutputFile);
+    free(Buffer);
+  }
+  std::string str() { return std::string(Buffer, Size); }
+
+ private:
+  char *Buffer;
+  size_t Size;
+  FILE *OldOutputFile;
+};
+
+TEST(FuzzerUtil, PrintASCII) {
+  auto f = [](const char *Str, const char *PrintAfter = "") {
+    PrintfCapture Capture;
+    PrintASCII(reinterpret_cast<const uint8_t*>(Str), strlen(Str), PrintAfter);
+    return Capture.str();
+  };
+  EXPECT_EQ("hello", f("hello"));
+  EXPECT_EQ("c:\\\\", f("c:\\"));
+  EXPECT_EQ("\\\"hi\\\"", f("\"hi\""));
+  EXPECT_EQ("\\011a", f("\ta"));
+  EXPECT_EQ("\\0111", f("\t1"));
+  EXPECT_EQ("hello\\012", f("hello\n"));
+  EXPECT_EQ("hello\n", f("hello", "\n"));
+}
+#endif
+
 TEST(Corpus, Distribution) {
   DataFlowTrace DFT;
   Random Rand(0);
   struct EntropicOptions Entropic = {false, 0xFF, 100, false};
   std::unique_ptr<InputCorpus> C(new InputCorpus("", Entropic));
   size_t N = 10;
   size_t TriesPerUnit = 1<<16;
   for (size_t i = 0; i < N; i++)