Support: Add llvm::format_memory_size to convert number of bytes to human readable string.

[marsupial]

Summary

Adds ability to write suffixed values instead of number of bytes

[marsupial]

Fix typo in documentation.

[ruiu]

You can use SI prefixes (i.e. K/M/G/T/etc.) with other units than M, so combining the two and call it FormattedMemory doesn't sound like a good idea. In addition to that, SI prefixes are not correct as they represents powers of 10 instead of powers of 2.

How about adding FormattedBinaryPrefix class which prints out Ki/Mi/Gi/... for 2^10/2^20/2^30/.... ? For Ki/Mi/Gi, see https://en.wikipedia.org/wiki/Binary_prefix.

[marsupial]

I appreciate the differences, but this is specifically to output an amount of memory in a human readable form.
Using KiB, MiB, etc may be pedantically correct, but it limits the usefulness of this to format text for a user who may have little knowledge of the differences between SI and IEC suffixes.
Every operating system I'm aware of does it this way and having output correlate to how the OS reports memory is more important.

[marsupial]

@ruiu
What about an optional argument to specific IEC suffixes.
Using IEC by default is fine by me, just that there is a way to not do so.

I agree the name is probably too general, but format_binary_prefix is a bit obtuse.
What about format_memory_size instead?

[marsupial]

Added support for IEC, SI, and Customary output.

[ruiu]

I'd like to listen to other people as to Mi/Ki/Gi or M/K/G.

[marsupial]

Ping.

include/llvm/Support/Format.h

 format_bytes_with_ascii(ArrayRef<uint8_t> Bytes,
                         Optional<uint64_t> FirstByteOffset = None,
                         uint32_t NumPerLine = 16, uint8_t ByteGroupSize = 4,
                         uint32_t IndentLevel = 0, bool Upper = false) {
   return FormattedBytes(Bytes, IndentLevel, FirstByteOffset, NumPerLine,
                         ByteGroupSize, Upper, true);
 }
 
+/// This is a helper class used for format_memory.
+class FormattedMemorySize {
+public:
+  enum Format { Binary, Decimal, Customary = -1 };
+  FormattedMemorySize(double NB, unsigned W = 2, Format S = Binary)
+      : NumBytes(NB), Width(W), Suffix(S) {}
+
+private:
+  friend class raw_ostream;
+  const double NumBytes;
+  const unsigned Width;
+  const Format Suffix;
+};
+
+/// format_memory_size - Output \p NB as human readable amount of bytes with
+/// IEC suffixes. When the amount is greater than half the amount for the next
+/// suffix that next suffix is used.  If no suffix exists for the amount of
+/// bytes: the number is printed as bytes.
+///
+///   OS << format_memory_size(1024, 2)         => "1.00 KiB"
+///   OS << format_memory_size(524288, 2)       => "0.50 MiB"
+///   OS << format_memory_size(1048576, 4)      => "1.0000 MiB"
+///   OS << format_memory_size(pow(1024, 5), 3) => "1.000 PiB"
+inline FormattedMemorySize format_memory_size(double NB, unsigned W = 2) {
+  return FormattedMemorySize(NB, W);
+}
+
+/// format_memory_size_si - Output \p NB as human readable amount of bytes with
+/// SI suffixes. When the amount is greater than half the amount for the next
+/// suffix that next suffix is used.  If no suffix exists for the amount of
+/// bytes: the number is printed as bytes.
+///
+///   OS << format_memory_size(1000, 2)         => "1.00 kB"
+///   OS << format_memory_size(500000, 2)       => "0.50 MB"
+///   OS << format_memory_size(1000000, 4)      => "1.0000 MB"
+///   OS << format_memory_size(pow(1000, 5), 3) => "1.000 PB"
+inline FormattedMemorySize format_memory_size_si(double NB, unsigned W = 2) {
+  return FormattedMemorySize(NB, W, FormattedMemorySize::Decimal);
+}
+
+/// format_memory_size_customary - Output \p NB as human readable amount of
+/// bytes with SI suffixes, but still as power of 2. Techincally incorrect but
+/// still often used for RAM, older unix tools, and operating systems.
+///
+///   OS << format_memory_size_customary(1024, 2)   => "1.00 kB"
+///   OS << format_memory_size_customary(524288, 2) => "0.50 MB"
+inline FormattedMemorySize format_memory_size_customary(double NB,
+                                                        unsigned W = 2) {
+  return FormattedMemorySize(NB, W, FormattedMemorySize::Customary);
+}
+
 } // end namespace llvm
 
 #endif

include/llvm/Support/raw_ostream.h

 
 namespace llvm {
 
 class formatv_object_base;
 class format_object_base;
 class FormattedString;
 class FormattedNumber;
 class FormattedBytes;
+class FormattedMemorySize;
 
 namespace sys {
 namespace fs {
 enum OpenFlags : unsigned;
 } // end namespace fs
 } // end namespace sys
 
 /// This class implements an extremely fast bulk output stream that can *only*
public:
   raw_ostream &operator<<(const FormattedNumber &);
 
   // Formatted output, see the formatv() function in Support/FormatVariadic.h.
   raw_ostream &operator<<(const formatv_object_base &);
 
   // Formatted output, see the format_bytes() function in Support/Format.h.
   raw_ostream &operator<<(const FormattedBytes &);
 
+  // Formatted output, see the format_memory() function in Support/Format.h.
+  raw_ostream &operator<<(const FormattedMemorySize &);
+
   /// indent - Insert 'NumSpaces' spaces.
   raw_ostream &indent(unsigned NumSpaces);
 
   /// Changes the foreground color of text that will be output from this point
   /// forward.
   /// @param Color ANSI color to use, the special SAVEDCOLOR can be used to
   /// change only the bold attribute, and keep colors untouched
   /// @param Bold bold/brighter text, default false

lib/Support/raw_ostream.cpp

while (!Bytes.empty()) {
     Bytes = Bytes.drop_front(Line.size());
     LineIndex += Line.size();
     if (LineIndex < Size)
       *this << '\n';
   }
   return *this;
 }
 
+raw_ostream &raw_ostream::operator<<(const FormattedMemorySize &FM) {
+  constexpr size_t NumSuffixes = 8;
+  struct ByteTable {
+    ByteTable(float Base) {
+      for (size_t I = 0; I <= NumSuffixes; ++I)
+        Table[I] = std::pow(Base, I + 1);
+    }
+    double operator[](int I) const { return Table[I]; }
+    double Table[NumSuffixes + 1];
+  };
+  static const ByteTable Base2(1024), Base10(1000);
+  const ByteTable &BT =
+      FM.Suffix != FormattedMemorySize::Decimal ? Base2 : Base10;
+
+  if (FM.NumBytes >= BT[0]) {
+    const bool Binary = FM.Suffix == FormattedMemorySize::Binary;
+    const char Suffix[NumSuffixes] = {
+        Binary ? 'K' : 'k', 'M', 'G', 'T', 'P', 'E', 'Z', 'Y'};
+    for (size_t I = 1; I <= NumSuffixes; ++I) {
+      if (FM.NumBytes >= BT[I] * 0.5)
+        continue;
+      this->operator<<(llvm::format("%.*f ", FM.Width, FM.NumBytes / BT[I-1]));
+      this->operator<<(Suffix[I-1]);
+      this->operator<<(Binary ? "iB" : "B");
+      return *this;
+    }
+  }
+  this->operator<<(llvm::format("%.0f B", FM.NumBytes));
+  return *this;
+}
+
 /// indent - Insert 'NumSpaces' spaces.
 raw_ostream &raw_ostream::indent(unsigned NumSpaces) {
   static const char Spaces[] = "                                "
                                "                                "
                                "                ";
 
   // Usually the indentation is small, handle it with a fastpath.
   if (NumSpaces < array_lengthof(Spaces))

unittests/Support/raw_ostream_test.cpp

 //===- llvm/unittest/Support/raw_ostream_test.cpp - raw_ostream tests -----===//
 //
 //                     The LLVM Compiler Infrastructure
 //
 // This file is distributed under the University of Illinois Open Source
 // License. See LICENSE.TXT for details.
 //
 //===----------------------------------------------------------------------===//
 
 #include "llvm/ADT/SmallString.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/Format.h"
 #include "llvm/Support/raw_ostream.h"
 #include "gtest/gtest.h"
 
+#include <cmath>
+
 using namespace llvm;
 
 namespace {
 
 template<typename T> std::string printToString(const T &Value) {
   std::string res;
   llvm::raw_string_ostream(res) << Value;
   return res;    
TEST(raw_ostreamTest, Justify) {
   EXPECT_EQ("big",    printToString(right_justify("big", 1), 3));
   EXPECT_EQ("   on    ",    printToString(center_justify("on", 9), 9));
   EXPECT_EQ("   off    ",    printToString(center_justify("off", 10), 10));
   EXPECT_EQ("single ",    printToString(center_justify("single", 7), 7));
   EXPECT_EQ("none",    printToString(center_justify("none", 1), 4));
   EXPECT_EQ("none",    printToString(center_justify("none", 1), 1));
 }
 
+static std::string format_memory_pow2(double Pow, double Mult = 1) {
+  const double Baseline = std::pow(1024, Pow);
+  return printToString(llvm::format_memory_size(Baseline * 0.4)) + " " +
+         printToString(llvm::format_memory_size(Baseline * 0.5)) + " " +
+         printToString(llvm::format_memory_size(Baseline));
+}
+
+static std::string format_memory_pow10(double Pow, double Mult = 1) {
+  const double Baseline = std::pow(1000, Pow);
+  return printToString(llvm::format_memory_size_si(Baseline * 0.4)) + " " +
+         printToString(llvm::format_memory_size_si(Baseline * 0.5)) + " " +
+         printToString(llvm::format_memory_size_si(Baseline));
+}
+
+TEST(raw_ostreamTest, FormatMemory) {
+  EXPECT_EQ("410 B 512 B 1.00 KiB", format_memory_pow2(1));
+  EXPECT_EQ("409.60 KiB 0.50 MiB 1.00 MiB", format_memory_pow2(2));
+  EXPECT_EQ("409.60 MiB 0.50 GiB 1.00 GiB", format_memory_pow2(3));
+  EXPECT_EQ("409.60 GiB 0.50 TiB 1.00 TiB", format_memory_pow2(4));
+  EXPECT_EQ("409.60 TiB 0.50 PiB 1.00 PiB", format_memory_pow2(5));
+  EXPECT_EQ("409.60 PiB 0.50 EiB 1.00 EiB", format_memory_pow2(6));
+  EXPECT_EQ("409.60 EiB 0.50 ZiB 1.00 ZiB", format_memory_pow2(7));
+  EXPECT_EQ("409.60 ZiB 0.50 YiB 1.00 YiB", format_memory_pow2(8));
+
+  EXPECT_EQ("400 B 500 B 1.00 kB", format_memory_pow10(1));
+  EXPECT_EQ("400.00 kB 0.50 MB 1.00 MB", format_memory_pow10(2));
+  EXPECT_EQ("400.00 MB 0.50 GB 1.00 GB", format_memory_pow10(3));
+  EXPECT_EQ("400.00 GB 0.50 TB 1.00 TB", format_memory_pow10(4));
+  EXPECT_EQ("400.00 TB 0.50 PB 1.00 PB", format_memory_pow10(5));
+  EXPECT_EQ("400.00 PB 0.50 EB 1.00 EB", format_memory_pow10(6));
+  EXPECT_EQ("400.00 EB 0.50 ZB 1.00 ZB", format_memory_pow10(7));
+  EXPECT_EQ("400.00 ZB 0.50 YB 1.00 YB", format_memory_pow10(8));
+
+  EXPECT_EQ("256.00 kB",
+            printToString(llvm::format_memory_size_customary(262144)));
+  EXPECT_EQ("0.50 MB",
+            printToString(llvm::format_memory_size_customary(262144 * 2)));
+
+  EXPECT_EQ("0.50 PB", printToString(llvm::format_memory_size_customary(
+                           std::pow(1024, 5) * 0.5)));
+  EXPECT_EQ("409.60 TB", printToString(llvm::format_memory_size_customary(
+                             std::pow(1024, 5) * 0.4)));
+
+  EXPECT_EQ("0.5010 TB", printToString(llvm::format_memory_size_customary(
+                             std::pow(1024, 3) + std::pow(1024, 4) * 0.5, 4)));
+}
+
 TEST(raw_ostreamTest, FormatHex) {  
   EXPECT_EQ("0x1234",     printToString(format_hex(0x1234, 6), 6));
   EXPECT_EQ("0x001234",   printToString(format_hex(0x1234, 8), 8));
   EXPECT_EQ("0x00001234", printToString(format_hex(0x1234, 10), 10));
   EXPECT_EQ("0x1234",     printToString(format_hex(0x1234, 4), 6));
   EXPECT_EQ("0xff",       printToString(format_hex(255, 4), 4));
   EXPECT_EQ("0xFF",       printToString(format_hex(255, 4, true), 4));
   EXPECT_EQ("0x1",        printToString(format_hex(1, 3), 3));