Index: include/llvm/Support/Compression.h =================================================================== --- include/llvm/Support/Compression.h +++ include/llvm/Support/Compression.h @@ -14,10 +14,12 @@ #ifndef LLVM_SUPPORT_COMPRESSION_H #define LLVM_SUPPORT_COMPRESSION_H +#include "llvm/ADT/ArrayRef.h" #include "llvm/Support/DataTypes.h" namespace llvm { template class SmallVectorImpl; +template class Expected; class Error; class StringRef; @@ -44,6 +46,34 @@ uint32_t crc32(StringRef Buffer); +/// StreamCompression class implements API that allows streaming compression. +class StreamCompression final { +public: + /// Output function callback prototype. Caller is able to receive compressed + /// data part by part from this array. + typedef std::function)> OutFn; + + ~StreamCompression(); + + /// Creates instance of this class. + /// OutBufSize - size of internal buffer for decompression. + static Expected> create(size_t OutBufSize); + + /// Compressed input chunk of data. + /// InChuck - chunk to compress. + /// IsLastChunk - flag that should be set if input chunk is the last. That + /// allows streaming to finalize compression. + /// Out - callback that allows user code to receive compressed data. + Error compress(ArrayRef InChunk, bool IsLastChunk, OutFn Out); + +private: + StreamCompression(size_t ChunkSize); + + std::vector OutBuff; + + void *ZStreamImpl; +}; + } // End of namespace zlib } // End of namespace llvm Index: lib/Support/Compression.cpp =================================================================== --- lib/Support/Compression.cpp +++ lib/Support/Compression.cpp @@ -98,6 +98,60 @@ return ::crc32(0, (const Bytef *)Buffer.data(), Buffer.size()); } +zlib::StreamCompression::StreamCompression(size_t OutBufSize) + : OutBuff(OutBufSize) { + ZStreamImpl = new z_stream(); +} + +zlib::StreamCompression::~StreamCompression() { + delete (z_stream *)ZStreamImpl; +} + +Expected> +zlib::StreamCompression::create(size_t OutBufSize) { + std::unique_ptr Ret(new StreamCompression(OutBufSize)); + + z_stream *ZStream = (z_stream *)Ret->ZStreamImpl; + ZStream->zalloc = Z_NULL; + ZStream->zfree = Z_NULL; + ZStream->opaque = (voidpf)(0); + + int Res = deflateInit(ZStream, Z_DEFAULT_COMPRESSION); + if (Res != Z_OK) + return createError(convertZlibCodeToString(Res)); + return std::move(Ret); +} + +Error zlib::StreamCompression::compress(ArrayRef InChunk, + bool IsLastChunk, OutFn Out) { + z_stream *ZStream = (z_stream *)ZStreamImpl; + ZStream->avail_in = InChunk.size(); + ZStream->next_in = (Bytef *)InChunk.data(); + + while (true) { + ZStream->avail_out = OutBuff.size(); + ZStream->next_out = &OutBuff[0]; + int Res = deflate(ZStream, IsLastChunk ? Z_FINISH : Z_NO_FLUSH); + + // Z_BUF_ERROR is a normal possible status for deflate() call. + // It just indicates here that deflate() could not consume more input. + if (Res != Z_OK && Res != Z_STREAM_END && Res != Z_BUF_ERROR) { + deflateEnd(ZStream); + return createError(convertZlibCodeToString(Res)); + } + + ArrayRef Data(OutBuff.data(), OutBuff.size() - ZStream->avail_out); + if (Data.empty()) + break; + Out(Data); + } + + if (IsLastChunk) + deflateEnd(ZStream); + + return Error::success(); +} + #else bool zlib::isAvailable() { return false; } Error zlib::compress(StringRef InputBuffer, @@ -117,5 +171,21 @@ uint32_t zlib::crc32(StringRef Buffer) { llvm_unreachable("zlib::crc32 is unavailable"); } +zlib::StreamCompression::StreamCompression(size_t) { + llvm_unreachable("zlib::compress is unavailable"); +} + +zlib::StreamCompression::~StreamCompression() { + llvm_unreachable("zlib::compress is unavailable"); +} + +Expected> +zlib::StreamCompression::create(size_t) { + llvm_unreachable("zlib::compress is unavailable"); +} + +Error zlib::StreamCompression::compress(ArrayRef, bool, OutFn) { + llvm_unreachable("zlib::compress is unavailable"); +} #endif Index: unittests/Support/CompressionTest.cpp =================================================================== --- unittests/Support/CompressionTest.cpp +++ unittests/Support/CompressionTest.cpp @@ -72,6 +72,40 @@ zlib::crc32(StringRef("The quick brown fox jumps over the lazy dog"))); } +TEST(CompressionTest, ZlibStreaming) { + // 1. Create input chunk. It will be used 4 times, + // total size of uncompressed data will be 4 * sizeof(chunk). + const size_t Size = 512; + std::vector In(512); + for (size_t I = 0; I < Size; ++I) + In[I] = I & 255; + ArrayRef InChunk(In); + + // 2. Feed streaming compression with chunks created. + SmallString<32> Compressed; + Expected> SC = + zlib::StreamCompressing::create(128); + EXPECT_TRUE((bool)SC); + for (int I = 0; I < 4; ++I) { + Error E = + (*SC)->compress(InChunk, I == 3 /*Last*/, [&](ArrayRef C) { + Compressed.insert(Compressed.end(), C.begin(), C.end()); + }); + EXPECT_FALSE(E); + consumeError(std::move(E)); + } + + // 3. Decompress using regular decompression API. + SmallString<32> Uncompressed; + Error E = zlib::uncompress(Compressed, Uncompressed, Size * 4); + EXPECT_FALSE(E); + consumeError(std::move(E)); + + // 4. Check that decompressed data equals to compressed. + for (size_t I = 0; I < Size * 4; ++I) + EXPECT_EQ((uint8_t)Uncompressed[I], I & 255); +} + #endif }