diff --git a/lld/ELF/LTO.cpp b/lld/ELF/LTO.cpp --- a/lld/ELF/LTO.cpp +++ b/lld/ELF/LTO.cpp @@ -57,6 +57,20 @@ return ret; } +// The merged bitcode after LTO is large. Try openning a file stream that +// supports reading, seeking and writing. Such a file allows BitcodeWriter to +// flush buffered data to reduce memory comsuption. If this fails, open a file +// stream that supports only write. +static std::unique_ptr openLTOOutputFile(StringRef file) { + std::error_code ec; + std::unique_ptr fs = + std::make_unique(file, ec); + if (!ec && fs->supportsSeeking()) { + return fs; + } + return openFile(file); +} + static std::string getThinLTOOutputFile(StringRef modulePath) { return lto::getThinLTOOutputFile( std::string(modulePath), std::string(config->thinLTOPrefixReplace.first), @@ -151,7 +165,8 @@ if (config->emitLLVM) { c.PostInternalizeModuleHook = [](size_t task, const Module &m) { - if (std::unique_ptr os = openFile(config->outputFile)) + if (std::unique_ptr os = + openLTOOutputFile(config->outputFile)) WriteBitcodeToFile(m, *os, false); return false; }; diff --git a/llvm/include/llvm/Bitcode/BitcodeWriter.h b/llvm/include/llvm/Bitcode/BitcodeWriter.h --- a/llvm/include/llvm/Bitcode/BitcodeWriter.h +++ b/llvm/include/llvm/Bitcode/BitcodeWriter.h @@ -47,7 +47,7 @@ public: /// Create a BitcodeWriter that writes to Buffer. - BitcodeWriter(SmallVectorImpl &Buffer); + BitcodeWriter(SmallVectorImpl &Buffer, raw_ostream *OS = nullptr); ~BitcodeWriter(); diff --git a/llvm/include/llvm/Bitstream/BitstreamWriter.h b/llvm/include/llvm/Bitstream/BitstreamWriter.h --- a/llvm/include/llvm/Bitstream/BitstreamWriter.h +++ b/llvm/include/llvm/Bitstream/BitstreamWriter.h @@ -20,17 +20,25 @@ #include "llvm/ADT/StringRef.h" #include "llvm/Bitstream/BitCodes.h" #include "llvm/Support/Endian.h" +#include "llvm/Support/raw_ostream.h" #include namespace llvm { +class raw_fd_ostream; + class BitstreamWriter { + /// Out - The buffer that keeps unflushed bytes. SmallVectorImpl &Out; + /// FS - The file stream that Out flushes to. If FS is nullptr, it does not + /// support read or seek, Out cannot be flushed until all data are written. + raw_fd_stream *FS; + /// CurBit - Always between 0 and 31 inclusive, specifies the next bit to use. unsigned CurBit; - /// CurValue - The current value. Only bits < CurBit are valid. + /// CurValue - The current value. Only bits < CurBit are valid. uint32_t CurValue; /// CurCodeSize - This is the declared size of code values used for the @@ -64,15 +72,19 @@ void WriteByte(unsigned char Value) { Out.push_back(Value); + FlushToFile(); } void WriteWord(unsigned Value) { Value = support::endian::byte_swap(Value); Out.append(reinterpret_cast(&Value), reinterpret_cast(&Value + 1)); + FlushToFile(); } - size_t GetBufferOffset() const { return Out.size(); } + uint64_t GetNumOfFlushedBytes() const { return FS ? FS->tell() : 0; } + + size_t GetBufferOffset() const { return Out.size() + GetNumOfFlushedBytes(); } size_t GetWordIndex() const { size_t Offset = GetBufferOffset(); @@ -80,9 +92,24 @@ return Offset / 4; } + /// If the related file stream supports reading, seeking and writing, flush + /// the buffer if its size is above a threshold. + void FlushToFile() { + constexpr uint64_t FLUSH_THRESHOLD = 1L << 29; // 512M + if (!FS) { + return; + } + if (Out.size() < FLUSH_THRESHOLD) { + return; + } + FS->write((char *)&Out.front(), Out.size()); + Out.clear(); + } + public: - explicit BitstreamWriter(SmallVectorImpl &O) - : Out(O), CurBit(0), CurValue(0), CurCodeSize(2) {} + BitstreamWriter(SmallVectorImpl &O, raw_ostream *OS = nullptr) + : Out(O), FS(dynamic_cast(OS)), CurBit(0), CurValue(0), + CurCodeSize(2) {} ~BitstreamWriter() { assert(CurBit == 0 && "Unflushed data remaining"); @@ -104,11 +131,55 @@ void BackpatchWord(uint64_t BitNo, unsigned NewWord) { using namespace llvm::support; uint64_t ByteNo = BitNo / 8; + uint64_t StartBit = BitNo & 7; + uint64_t NumOfFlushedBytes = GetNumOfFlushedBytes(); + + if (ByteNo >= NumOfFlushedBytes) { + assert((!endian::readAtBitAlignment( + &Out[ByteNo - NumOfFlushedBytes], StartBit)) && + "Expected to be patching over 0-value placeholders"); + endian::writeAtBitAlignment( + &Out[ByteNo - NumOfFlushedBytes], NewWord, StartBit); + return; + } + + // If the byte offset to backpatch is flushed, use seek to backfill data. + // First, save the file position to restore later. + uint64_t CurPos = FS->tell(); + + // Copy data to update into Bytes from the file FS and the buffer Out. + char Bytes[8]; + size_t BytesNum = StartBit ? 8 : 4; + size_t BytesFromDisk = std::min(BytesNum, NumOfFlushedBytes - ByteNo); + size_t BytesFromBuffer = std::max(0ul, BytesNum - BytesFromDisk); + +#ifdef NDEBUG + if (StartBit) +#endif + { + FS->seek(ByteNo); + ssize_t BytesRead = FS->read(Bytes, BytesFromDisk); + assert(BytesRead == BytesFromDisk); + for (size_t i = 0; i < BytesFromBuffer; ++i) + Bytes[BytesFromDisk + i] = Out[i]; + } + + // Update Bytes in terms of bit offset and value. assert((!endian::readAtBitAlignment( - &Out[ByteNo], BitNo & 7)) && + Bytes, StartBit)) && "Expected to be patching over 0-value placeholders"); - endian::writeAtBitAlignment( - &Out[ByteNo], NewWord, BitNo & 7); + endian::writeAtBitAlignment(Bytes, NewWord, + StartBit); + + // Copy updated data back to the file FS and the buffer Out. + FS->seek(ByteNo); + FS->write(Bytes, BytesFromDisk); + for (size_t i = 0; i < BytesFromBuffer; ++i) { + Out[i] = Bytes[BytesFromDisk + i]; + } + + // Restore the file position. + FS->seek(CurPos); } void BackpatchWord64(uint64_t BitNo, uint64_t Val) { diff --git a/llvm/include/llvm/Support/raw_ostream.h b/llvm/include/llvm/Support/raw_ostream.h --- a/llvm/include/llvm/Support/raw_ostream.h +++ b/llvm/include/llvm/Support/raw_ostream.h @@ -436,10 +436,17 @@ /// Determine an efficient buffer size. size_t preferred_buffer_size() const override; + void anchor() override; + +protected: /// Set the flag indicating that an output error has been encountered. void error_detected(std::error_code EC) { this->EC = EC; } - void anchor() override; + /// Return the file descriptor. + int get_fd() const { return FD; } + + // Update the file position by increasing \p Delta. + void inc_pos(uint64_t Delta) { pos += Delta; } public: /// Open the specified file for writing. If an error occurs, information @@ -548,6 +555,30 @@ /// This returns a reference to a raw_ostream which simply discards output. raw_ostream &nulls(); +//===----------------------------------------------------------------------===// +// File Streams +//===----------------------------------------------------------------------===// + +/// A raw_ostream that reads from and writes to a file descriptor. +/// +class raw_fd_stream : public raw_fd_ostream { +public: + /// Open the specified file for reading/writing. If an error occurs, + /// information about the error is put into EC, and the stream should be + /// immediately destroyed. + raw_fd_stream(StringRef Filename, std::error_code &EC); + + /// This reads the \p Size bytes into a buffer pointed by \p Ptr. + /// + /// \param Ptr The start of the buffer to hold data to be read. + /// + /// \param Size The number of bytes to be read. + /// + /// On success, the number of bytes read is returned, and the file position is + // advanced by this number. On error, -1 is returned, and EC is set. + ssize_t read(const char *Ptr, size_t Size); +}; + //===----------------------------------------------------------------------===// // Output Stream Adaptors //===----------------------------------------------------------------------===// diff --git a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp --- a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp +++ b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp @@ -4453,8 +4453,8 @@ Stream.Emit(0xD, 4); } -BitcodeWriter::BitcodeWriter(SmallVectorImpl &Buffer) - : Buffer(Buffer), Stream(new BitstreamWriter(Buffer)) { +BitcodeWriter::BitcodeWriter(SmallVectorImpl &Buffer, raw_ostream *OS) + : Buffer(Buffer), Stream(new BitstreamWriter(Buffer, OS)) { writeBitcodeHeader(*Stream); } @@ -4565,7 +4565,7 @@ if (TT.isOSDarwin() || TT.isOSBinFormatMachO()) Buffer.insert(Buffer.begin(), BWH_HeaderSize, 0); - BitcodeWriter Writer(Buffer); + BitcodeWriter Writer(Buffer, &Out); Writer.writeModule(M, ShouldPreserveUseListOrder, Index, GenerateHash, ModHash); Writer.writeSymtab(); @@ -4575,7 +4575,9 @@ emitDarwinBCHeaderAndTrailer(Buffer, TT); // Write the generated bitstream to "Out". - Out.write((char*)&Buffer.front(), Buffer.size()); + if (!Buffer.empty()) { + Out.write((char *)&Buffer.front(), Buffer.size()); + } } void IndexBitcodeWriter::write() { diff --git a/llvm/lib/Support/raw_ostream.cpp b/llvm/lib/Support/raw_ostream.cpp --- a/llvm/lib/Support/raw_ostream.cpp +++ b/llvm/lib/Support/raw_ostream.cpp @@ -904,6 +904,27 @@ return S; } +//===----------------------------------------------------------------------===// +// File Streams +//===----------------------------------------------------------------------===// + +raw_fd_stream::raw_fd_stream(StringRef Filename, std::error_code &EC) + : raw_fd_ostream(Filename, EC, sys::fs::CD_CreateAlways, + sys::fs::FA_Write | sys::fs::FA_Read, sys::fs::OF_None) { + assert(Filename != "-" && "Do not support standard output."); +} + +ssize_t raw_fd_stream::read(const char *Ptr, size_t Size) { + assert(get_fd() >= 0 && "File already closed."); + ssize_t ret = ::read(get_fd(), (void *)Ptr, Size); + if (ret >= 0) { + inc_pos(ret); + } else { + error_detected(std::error_code(errno, std::generic_category())); + } + return ret; +} + //===----------------------------------------------------------------------===// // raw_string_ostream //===----------------------------------------------------------------------===//