Index: include/llvm/Support/Compression.h =================================================================== --- include/llvm/Support/Compression.h +++ include/llvm/Support/Compression.h @@ -14,10 +14,12 @@ #ifndef LLVM_SUPPORT_COMPRESSION_H #define LLVM_SUPPORT_COMPRESSION_H +#include "llvm/ADT/ArrayRef.h" #include "llvm/Support/DataTypes.h" namespace llvm { template class SmallVectorImpl; +template class Expected; class Error; class StringRef; @@ -44,6 +46,37 @@ uint32_t crc32(StringRef Buffer); +/// StreamCompression class implements API that allows streaming compression. +class StreamCompression final { +public: + StreamCompression(StreamCompression&& O); + ~StreamCompression(); + + /// Creates instance of this class. + /// OutBufSize - size of internal buffer for decompression. + static Expected create(size_t OutBufSize); + + /// Compressed input chunk of data. + /// InChuck - chunk to compress. + /// IsLastChunk - flag that should be set if input chunk is the last. That + /// allows streaming to finalize compression. + /// Out - vector to receive compressed data. + Error compressChunk(ArrayRef InChunk, SmallVectorImpl &Out); + + /// Should be called after all chunks were proccessed to finalize compression. + /// Out - vector to receive compressed data. + Error endCompress(SmallVectorImpl &Out); + +private: + StreamCompression(size_t ChunkSize); + + Error doDeflate(SmallVectorImpl &Out, bool Flush); + + SmallVector OutBuff; + + void *ZStreamImpl = nullptr; +}; + } // End of namespace zlib } // End of namespace llvm Index: lib/Support/Compression.cpp =================================================================== --- lib/Support/Compression.cpp +++ lib/Support/Compression.cpp @@ -98,6 +98,77 @@ return ::crc32(0, (const Bytef *)Buffer.data(), Buffer.size()); } +zlib::StreamCompression::StreamCompression(size_t OutBufSize) + : OutBuff(OutBufSize) { + ZStreamImpl = new z_stream(); +} + +zlib::StreamCompression::StreamCompression(StreamCompression &&O) { + std::swap(ZStreamImpl, O.ZStreamImpl); + std::swap(OutBuff, O.OutBuff); +} + +zlib::StreamCompression::~StreamCompression() { + delete (z_stream *)ZStreamImpl; +} + +Expected +zlib::StreamCompression::create(size_t OutBufSize) { + zlib::StreamCompression Ret(OutBufSize); + + z_stream *ZStream = (z_stream *)Ret.ZStreamImpl; + ZStream->zalloc = Z_NULL; + ZStream->zfree = Z_NULL; + ZStream->opaque = (voidpf)(0); + + int Res = deflateInit(ZStream, Z_DEFAULT_COMPRESSION); + if (Res != Z_OK) + return createError(convertZlibCodeToString(Res)); + return Ret; +} + +Error zlib::StreamCompression::compressChunk(ArrayRef InChunk, + SmallVectorImpl &Out) { + z_stream *ZStream = (z_stream *)ZStreamImpl; + ZStream->avail_in = InChunk.size(); + ZStream->next_in = (Bytef *)InChunk.data(); + return doDeflate(Out, false /*Flush*/); +} + +Error zlib::StreamCompression::endCompress(SmallVectorImpl &Out) { + z_stream *ZStream = (z_stream *)ZStreamImpl; + ZStream->avail_in = 0; + ZStream->next_in = (Bytef *)0; + return doDeflate(Out, true /*Flush*/); +} + +Error zlib::StreamCompression::doDeflate(SmallVectorImpl &Out, + bool Flush) { + z_stream *ZStream = (z_stream *)ZStreamImpl; + + while (true) { + ZStream->avail_out = OutBuff.size(); + ZStream->next_out = &OutBuff[0]; + int Res = deflate(ZStream, Flush ? Z_FINISH : Z_NO_FLUSH); + + // Z_BUF_ERROR is a normal possible status for deflate() call. + // It just indicates here that deflate() could not consume more input. + if (Res != Z_OK && Res != Z_STREAM_END && Res != Z_BUF_ERROR) { + deflateEnd(ZStream); + return createError(convertZlibCodeToString(Res)); + } + + ArrayRef Data(OutBuff.data(), OutBuff.size() - ZStream->avail_out); + if (Data.empty()) + break; + Out.insert(Out.end(), Data.begin(), Data.end()); + } + + if (Flush) + deflateEnd(ZStream); + return Error::success(); +} + #else bool zlib::isAvailable() { return false; } Error zlib::compress(StringRef InputBuffer, @@ -117,5 +188,30 @@ uint32_t zlib::crc32(StringRef Buffer) { llvm_unreachable("zlib::crc32 is unavailable"); } +zlib::StreamCompression::StreamCompression(size_t OutBufSize) + : OutBuff(OutBufSize) { + llvm_unreachable("zlib::compress is unavailable"); +} +zlib::StreamCompression::StreamCompression(StreamCompression &&O) { + llvm_unreachable("zlib::compress is unavailable"); +} +zlib::StreamCompression::~StreamCompression() { + llvm_unreachable("zlib::compress is unavailable"); +} +Expected +zlib::StreamCompression::create(size_t OutBufSize) { + llvm_unreachable("zlib::compress is unavailable"); +} +Error zlib::StreamCompression::compressChunk(ArrayRef InChunk, + SmallVectorImpl &Out) { + llvm_unreachable("zlib::compress is unavailable"); +} +Error zlib::StreamCompression::endCompress(SmallVectorImpl &Out) { + llvm_unreachable("zlib::compress is unavailable"); +} +Error zlib::StreamCompression::doDeflate(SmallVectorImpl &Out, + bool Flush) { + llvm_unreachable("zlib::compress is unavailable"); +} #endif Index: unittests/Support/CompressionTest.cpp =================================================================== --- unittests/Support/CompressionTest.cpp +++ unittests/Support/CompressionTest.cpp @@ -72,6 +72,41 @@ zlib::crc32(StringRef("The quick brown fox jumps over the lazy dog"))); } +TEST(CompressionTest, ZlibStreaming) { + // 1. Create input chunk. It will be used 4 times, + // total size of uncompressed data will be 4 * sizeof(chunk). + const size_t Size = 512; + const size_t ChunksCnt = 4; + std::vector In(512); + for (size_t I = 0; I < Size; ++I) + In[I] = I & 255; + + // 2. Feed streaming compression with chunks created. + SmallVector Compressed; + Expected SC = zlib::StreamCompression::create(128); + EXPECT_TRUE((bool)SC); + for (int I = 0; I < ChunksCnt; ++I) { + Error E = SC->compressChunk(In, Compressed); + EXPECT_FALSE(E); + consumeError(std::move(E)); + } + Error E = SC->endCompress(Compressed); + EXPECT_FALSE(E); + consumeError(std::move(E)); + + // 3. Decompress using regular decompression API. + SmallString<32> Uncompressed; + E = zlib::uncompress( + StringRef((const char *)Compressed.begin(), Compressed.size()), + Uncompressed, Size * ChunksCnt); + EXPECT_FALSE(E); + consumeError(std::move(E)); + + // 4. Check that decompressed data equals to compressed. + for (size_t I = 0; I < Size * ChunksCnt; ++I) + EXPECT_EQ((uint8_t)Uncompressed[I], In[I % Size]); +} + #endif }