Index: include/llvm/DebugInfo/CodeView/StreamArray.h =================================================================== --- /dev/null +++ include/llvm/DebugInfo/CodeView/StreamArray.h @@ -0,0 +1,141 @@ +//===- StreamArray.h - Array backed by an arbitrary stream ----------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_DEBUGINFO_CODEVIEW_STREAMARRAY_H +#define LLVM_DEBUGINFO_CODEVIEW_STREAMARRAY_H + +#include "llvm/DebugInfo/CodeView/StreamView.h" + +#include + +namespace llvm { +namespace codeview { + +/// VarStreamArray represents an array of variable length records backed by a +/// stream. This could be a contiguous sequence of bytes in memory, it could +/// be a file on disk, or it could be a PDB stream where bytes are stored as +/// discontiguous blocks on a filesystem. Usually it is desirable to treat +/// arrays as contiguous blocks of memory, but doing so with large PDB files, +/// for example, could mean allocating huge amounts of memory just to allow +/// re-ordering of stream data to be contiguous before iterating over it. By +/// abstracting this out, we need not duplicate this memory, and we can +/// iterate over arrays in arbitrarily formatted streams. +class VarStreamArray { + typedef std::function + LengthFuncType; + struct CacheItem { + CacheItem(uint32_t Off, uint32_t Len) : Off(Off), Len(Len) {} + uint32_t Off; + uint32_t Len; + }; + +public: + template + VarStreamArray(const StreamInterface &Stream, uint32_t Count, + const LengthFunc &Len) + : Stream(Stream), Offset(Offset), ByteLength(ByteLength), Count(Count), + Len(Len) {} + + uint32_t getCount() const { return Count; } + + ArrayRef operator[](uint32_t Index); + +private: + void cacheRecordsUpTo(uint32_t Index); + + std::vector TempItem; + StreamView Stream; + std::vector RecordOffsets; // Cached record offsets + + uint32_t Count; // The number of items + LengthFuncType Len; // Function used to calculate legth of a record +}; + +template class FixedStreamArrayIterator; + +template class FixedStreamArray { + friend class FixedStreamArrayIterator; + +public: + FixedStreamArray() : Stream(), TempItem() {} + FixedStreamArray(StreamView Stream) : Stream(Stream), TempItem() {} + + const T &operator[](uint32_t Index) const { + assert(Index < size()); + uint32_t Off = Index * sizeof(T); + ArrayRef Data; + if (auto EC = Stream.getArrayRef(Off, Data, sizeof(T))) { + consumeError(std::move(EC)); + MutableArrayRef Buf(reinterpret_cast(&TempItem), + sizeof(T)); + if (EC = Stream.readBytes(Off, Buf)) { + consumeError(std::move(EC)); + return TempItem; + } + } + return *reinterpret_cast(Data.data()); + } + + uint32_t size() const { return Stream.getLength() / sizeof(T); } + + FixedStreamArrayIterator begin() const { + return FixedStreamArrayIterator(*this, 0); + } + FixedStreamArrayIterator end() const { + return FixedStreamArrayIterator(*this); + } + +private: + StreamView Stream; + mutable T TempItem; +}; + +template class FixedStreamArrayIterator { +public: + FixedStreamArrayIterator(const FixedStreamArray &Array) + : Array(Array), Index(uint32_t(-1)) {} + FixedStreamArrayIterator(const FixedStreamArray &Array, uint32_t Index) + : Array(Array), Index(Index) {} + + friend bool operator==(const FixedStreamArrayIterator &L, + const FixedStreamArrayIterator &R) { + assert(&L.Array == &R.Array); + return L.Index == R.Index; + } + + friend bool operator!=(const FixedStreamArrayIterator &L, + const FixedStreamArrayIterator &R) { + return !(L == R); + } + + const T &operator*() const { return Array[Index]; } + + FixedStreamArrayIterator &operator++() { + if (Index == uint32_t(-1)) + return *this; + if (++Index >= Array.size()) + Index = uint32_t(-1); + return *this; + } + + FixedStreamArrayIterator operator++(int) { + FixedStreamArrayIterator Original = *this; + ++*this; + return Original; + } + +private: + const FixedStreamArray &Array; + uint32_t Index; +}; + +} // namespace codeview +} // namespace llvm + +#endif // LLVM_DEBUGINFO_CODEVIEW_STREAMARRAY_H Index: include/llvm/DebugInfo/CodeView/StreamReader.h =================================================================== --- include/llvm/DebugInfo/CodeView/StreamReader.h +++ include/llvm/DebugInfo/CodeView/StreamReader.h @@ -11,6 +11,8 @@ #define LLVM_DEBUGINFO_CODEVIEW_STREAMREADER_H #include "llvm/ADT/ArrayRef.h" +#include "llvm/DebugInfo/CodeView/CodeViewError.h" +#include "llvm/DebugInfo/CodeView/StreamArray.h" #include "llvm/DebugInfo/CodeView/StreamInterface.h" #include "llvm/Support/Endian.h" #include "llvm/Support/Error.h" @@ -20,13 +22,20 @@ namespace llvm { namespace codeview { +class StreamString; +class StreamView; + class StreamReader { public: StreamReader(const StreamInterface &S); Error readBytes(MutableArrayRef Buffer); + Error readInteger(uint16_t &Dest); Error readInteger(uint32_t &Dest); Error readZeroString(std::string &Dest); + Error readZeroString(StreamString &Dest); + Error readStreamView(StreamView &View); + Error readStreamView(StreamView &View, uint32_t Length); template Error readObject(T *Dest) { MutableArrayRef Buffer(reinterpret_cast(Dest), @@ -34,6 +43,17 @@ return readBytes(Buffer); } + template + Error readArray(FixedStreamArray &Array, uint32_t NumItems) { + uint32_t Length = NumItems * sizeof(T); + if (Offset + Length >= Stream.getLength()) + return make_error(cv_error_code::insufficient_buffer); + StreamView View(Stream, Offset, Length); + Array = FixedStreamArray(View); + Offset += Length; + return Error::success(); + } + template Error readArray(MutableArrayRef Array) { MutableArrayRef Casted(reinterpret_cast(Array.data()), Array.size() * sizeof(T)); Index: include/llvm/DebugInfo/CodeView/StreamString.h =================================================================== --- /dev/null +++ include/llvm/DebugInfo/CodeView/StreamString.h @@ -0,0 +1,35 @@ +//===- StreamString.h - A string backed by an arbitrary stream ------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_DEBUGINFO_CODEVIEW_STREAMSTRING_H +#define LLVM_DEBUGINFO_CODEVIEW_STREAMSTRING_H + +#include "llvm/ADT/StringRef.h" +#include "llvm/DebugInfo/CodeView/StreamInterface.h" + +#include + +namespace llvm { +namespace codeview { +class StreamString { +public: + StreamString() {} + StreamString(const std::string &Value) : Storage(Value), Ref(Storage) {} + StreamString(StringRef Value) : Ref(Value) {} + + StringRef str() const { return Ref; } + +private: + std::string Storage; + StringRef Ref; +}; +} // codeview +} // llvm + +#endif // LLVM_DEBUGINFO_CODEVIEW_STREAMSTRING_H Index: include/llvm/DebugInfo/CodeView/StreamView.h =================================================================== --- /dev/null +++ include/llvm/DebugInfo/CodeView/StreamView.h @@ -0,0 +1,48 @@ +//===- StreamView.h - A stream providing a view over another ----*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_DEBUGINFO_CODEVIEW_STREAMVIEW_H +#define LLVM_DEBUGINFO_CODEVIEW_STREAMVIEW_H + +#include "llvm/DebugInfo/CodeView/StreamInterface.h" + +namespace llvm { +namespace codeview { + +class StreamView : public StreamInterface { +public: + StreamView() : Stream(nullptr), ViewOffset(0), Length(0) {} + StreamView(const StreamInterface &Stream) + : Stream(&Stream), ViewOffset(0), Length(Stream.getLength()) {} + StreamView(const StreamInterface &Stream, uint32_t Offset, uint32_t Length) + : Stream(&Stream), ViewOffset(Offset), Length(Length) {} + StreamView(const StreamView &Other) + : Stream(Other.Stream), ViewOffset(Other.ViewOffset), + Length(Other.Length) {} + + Error readBytes(uint32_t Offset, + MutableArrayRef Buffer) const override { + return Stream->readBytes(ViewOffset + Offset, Buffer); + } + Error getArrayRef(uint32_t Offset, ArrayRef &Buffer, + uint32_t Length) const override { + return Stream->getArrayRef(ViewOffset + Offset, Buffer, Length); + } + + uint32_t getLength() const override { return Length; } + +private: + const StreamInterface *Stream; + uint32_t ViewOffset; + uint32_t Length; +}; +} +} + +#endif // LLVM_DEBUGINFO_CODEVIEW_STREAMVIEW_H \ No newline at end of file Index: include/llvm/DebugInfo/PDB/Raw/DbiStream.h =================================================================== --- include/llvm/DebugInfo/PDB/Raw/DbiStream.h +++ include/llvm/DebugInfo/PDB/Raw/DbiStream.h @@ -11,6 +11,7 @@ #define LLVM_DEBUGINFO_PDB_RAW_PDBDBISTREAM_H #include "llvm/DebugInfo/CodeView/ByteStream.h" +#include "llvm/DebugInfo/CodeView/StreamView.h" #include "llvm/DebugInfo/PDB/PDBTypes.h" #include "llvm/DebugInfo/PDB/Raw/MappedBlockStream.h" #include "llvm/DebugInfo/PDB/Raw/ModInfo.h" @@ -63,12 +64,12 @@ NameHashTable ECNames; codeview::ByteStream ModInfoSubstream; - codeview::ByteStream SecContrSubstream; - codeview::ByteStream SecMapSubstream; - codeview::ByteStream FileInfoSubstream; - codeview::ByteStream TypeServerMapSubstream; - codeview::ByteStream ECSubstream; - codeview::ByteStream DbgHeader; + codeview::StreamView SecContrSubstream; + codeview::StreamView SecMapSubstream; + codeview::StreamView FileInfoSubstream; + codeview::StreamView TypeServerMapSubstream; + codeview::StreamView ECSubstream; + codeview::StreamView DbgHeader; std::unique_ptr Header; }; Index: include/llvm/DebugInfo/PDB/Raw/ModInfo.h =================================================================== --- include/llvm/DebugInfo/PDB/Raw/ModInfo.h +++ include/llvm/DebugInfo/PDB/Raw/ModInfo.h @@ -11,6 +11,7 @@ #define LLVM_DEBUGINFO_PDB_RAW_MODINFO_H #include "llvm/ADT/StringRef.h" +#include "llvm/DebugInfo/CodeView/StreamString.h" #include #include @@ -46,7 +47,7 @@ ModuleInfoEx(ModInfo Module) : Info(Module) {} ModInfo Info; - std::vector SourceFiles; + std::vector SourceFiles; }; class ModInfoIterator { Index: lib/DebugInfo/CodeView/CMakeLists.txt =================================================================== --- lib/DebugInfo/CodeView/CMakeLists.txt +++ lib/DebugInfo/CodeView/CMakeLists.txt @@ -7,7 +7,9 @@ MemoryTypeTableBuilder.cpp MethodListRecordBuilder.cpp RecordSerialization.cpp + StreamArray.cpp StreamReader.cpp + StreamString.cpp SymbolDumper.cpp TypeDumper.cpp TypeRecord.cpp Index: lib/DebugInfo/CodeView/StreamArray.cpp =================================================================== --- /dev/null +++ lib/DebugInfo/CodeView/StreamArray.cpp @@ -0,0 +1,51 @@ +//===- StreamArray.cpp - Array backed by an arbitrary stream --------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "llvm/DebugInfo/CodeView/StreamArray.h" + +#include "llvm/DebugInfo/CodeView/StreamInterface.h" + +using namespace llvm; +using namespace llvm::codeview; + +ArrayRef VarStreamArray::operator[](uint32_t Index) { + cacheRecordsUpTo(Index); + + const CacheItem &Item = RecordOffsets[Index]; + ArrayRef Data; + if (auto EC = Stream.getArrayRef(Item.Off, Data, Item.Len)) { + consumeError(std::move(EC)); + TempItem.resize(Item.Len); + Stream.readBytes(Item.Off, TempItem); + return TempItem; + } + + return Data; +} + +void VarStreamArray::cacheRecordsUpTo(uint32_t Index) { + assert(Index < getCount()); + // If this item is already cached, return + if (Index < RecordOffsets.size()) + return; + + uint32_t FirstUncachedItem = RecordOffsets.size(); + uint32_t StartOff = 0; + if (FirstUncachedItem > 0) { + const CacheItem &LastCachedItem = RecordOffsets[FirstUncachedItem - 1]; + StartOff = LastCachedItem.Off + LastCachedItem.Len; + } + + RecordOffsets.reserve(Index + 1); + for (uint32_t I = FirstUncachedItem; I <= Index; ++I) { + uint32_t L = Len(Stream, StartOff); + RecordOffsets.emplace_back(StartOff, L); + StartOff += L; + } +} \ No newline at end of file Index: lib/DebugInfo/CodeView/StreamReader.cpp =================================================================== --- lib/DebugInfo/CodeView/StreamReader.cpp +++ lib/DebugInfo/CodeView/StreamReader.cpp @@ -10,6 +10,8 @@ #include "llvm/DebugInfo/CodeView/StreamReader.h" #include "llvm/DebugInfo/CodeView/CodeViewError.h" +#include "llvm/DebugInfo/CodeView/StreamString.h" +#include "llvm/DebugInfo/CodeView/StreamView.h" using namespace llvm; using namespace llvm::codeview; @@ -23,6 +25,14 @@ return Error::success(); } +Error StreamReader::readInteger(uint16_t &Dest) { + support::ulittle16_t P; + if (auto EC = readObject(&P)) + return EC; + Dest = P; + return Error::success(); +} + Error StreamReader::readInteger(uint32_t &Dest) { support::ulittle32_t P; if (auto EC = readObject(&P)) @@ -43,6 +53,38 @@ return Error::success(); } +Error StreamReader::readZeroString(StreamString &Dest) { + std::string S; + uint32_t OldOff = getOffset(); + if (auto EC = readZeroString(S)) + return EC; + uint32_t NewOff = getOffset(); + setOffset(OldOff); + ArrayRef DataRef; + if (auto EC = getArrayRef(DataRef, S.size())) { + consumeError(std::move(EC)); + Dest = StreamString(S); + } else { + StringRef SR(reinterpret_cast(DataRef.data()), + DataRef.size()); + Dest = StreamString(SR); + } + setOffset(NewOff); + return Error::success(); +} + +Error StreamReader::readStreamView(StreamView &View) { + return readStreamView(View, bytesRemaining()); +} + +Error StreamReader::readStreamView(StreamView &View, uint32_t Length) { + if (bytesRemaining() < Length) + return make_error(cv_error_code::insufficient_buffer); + View = StreamView(Stream, Offset, Length); + Offset += Length; + return Error::success(); +} + Error StreamReader::getArrayRef(ArrayRef &Array, uint32_t Length) { if (auto EC = Stream.getArrayRef(Offset, Array, Length)) return EC; Index: lib/DebugInfo/CodeView/StreamString.cpp =================================================================== --- /dev/null +++ lib/DebugInfo/CodeView/StreamString.cpp @@ -0,0 +1,10 @@ +//===- StreamString.cpp - A string backed by an arbitrary stream ---------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "llvm/DebugInfo/CodeView/StreamString.h" Index: lib/DebugInfo/PDB/Raw/DbiStream.cpp =================================================================== --- lib/DebugInfo/PDB/Raw/DbiStream.cpp +++ lib/DebugInfo/PDB/Raw/DbiStream.cpp @@ -9,6 +9,7 @@ #include "llvm/DebugInfo/PDB/Raw/DbiStream.h" +#include "llvm/DebugInfo/CodeView/StreamArray.h" #include "llvm/DebugInfo/CodeView/StreamReader.h" #include "llvm/DebugInfo/PDB/Raw/InfoStream.h" #include "llvm/DebugInfo/PDB/Raw/ModInfo.h" @@ -148,19 +149,19 @@ for (auto Info : Range) ModuleInfos.push_back(ModuleInfoEx(Info)); - if (auto EC = - SecContrSubstream.initialize(Reader, Header->SecContrSubstreamSize)) + if (auto EC = Reader.readStreamView(SecContrSubstream, + Header->SecContrSubstreamSize)) return EC; - if (auto EC = SecMapSubstream.initialize(Reader, Header->SectionMapSize)) + if (auto EC = Reader.readStreamView(SecMapSubstream, Header->SectionMapSize)) return EC; - if (auto EC = FileInfoSubstream.initialize(Reader, Header->FileInfoSize)) + if (auto EC = Reader.readStreamView(FileInfoSubstream, Header->FileInfoSize)) return EC; if (auto EC = - TypeServerMapSubstream.initialize(Reader, Header->TypeServerSize)) + Reader.readStreamView(TypeServerMapSubstream, Header->TypeServerSize)) return EC; - if (auto EC = ECSubstream.initialize(Reader, Header->ECSubstreamSize)) + if (auto EC = Reader.readStreamView(ECSubstream, Header->ECSubstreamSize)) return EC; - if (auto EC = DbgHeader.initialize(Reader, Header->OptionalDbgHdrSize)) + if (auto EC = Reader.readStreamView(DbgHeader, Header->OptionalDbgHdrSize)) return EC; if (auto EC = initializeFileInfo()) @@ -247,25 +248,30 @@ // with the caveat that `NumSourceFiles` cannot be trusted, so // it is computed by summing `ModFileCounts`. // - const uint8_t *Buf = &FileInfoSubstream.data().front(); - auto FI = reinterpret_cast(Buf); - Buf += sizeof(FileInfoSubstreamHeader); + FileInfoSubstreamHeader Header; + codeview::StreamReader FISR(FileInfoSubstream); + if (auto EC = FISR.readObject(&Header)) + return EC; + // The number of modules in the stream should be the same as reported by // the FileInfoSubstreamHeader. - if (FI->NumModules != ModuleInfos.size()) + if (Header.NumModules != ModuleInfos.size()) return make_error(raw_error_code::corrupt_file, "FileInfo substream count doesn't match DBI."); + codeview::FixedStreamArray ModIndexArray; + codeview::FixedStreamArray ModFileCountArray; + codeview::FixedStreamArray FileNameOffsets; + // First is an array of `NumModules` module indices. This is not used for the // same reason that `NumSourceFiles` is not used. It's an array of uint16's, // but it's possible there are more than 64k source files, which would imply // more than 64k modules (e.g. object files) as well. So we ignore this // field. - llvm::ArrayRef ModIndexArray( - reinterpret_cast(Buf), ModuleInfos.size()); - - llvm::ArrayRef ModFileCountArray(ModIndexArray.end(), - ModuleInfos.size()); + if (auto EC = FISR.readArray(ModIndexArray, ModuleInfos.size())) + return EC; + if (auto EC = FISR.readArray(ModFileCountArray, ModuleInfos.size())) + return EC; // Compute the real number of source files. uint32_t NumSourceFiles = 0; @@ -280,11 +286,13 @@ // them in `ModuleInfoEx`. The value written to and read from the file is // not used anyway, it is only there as a way to store the offsets for the // purposes of later accessing the names at runtime. - llvm::ArrayRef FileNameOffsets( - reinterpret_cast(ModFileCountArray.end()), - NumSourceFiles); + if (auto EC = FISR.readArray(FileNameOffsets, NumSourceFiles)) + return EC; - const char *Names = reinterpret_cast(FileNameOffsets.end()); + codeview::StreamView NamesBufferView; + if (auto EC = FISR.readStreamView(NamesBufferView)) + return EC; + codeview::StreamReader Names(NamesBufferView); // We go through each ModuleInfo, determine the number N of source files for // that module, and then get the next N offsets from the Offsets array, using @@ -295,8 +303,10 @@ uint32_t NumFiles = ModFileCountArray[I]; ModuleInfos[I].SourceFiles.resize(NumFiles); for (size_t J = 0; J < NumFiles; ++J, ++NextFileIndex) { - uint32_t FileIndex = FileNameOffsets[NextFileIndex]; - ModuleInfos[I].SourceFiles[J] = StringRef(Names + FileIndex); + uint32_t FileOffset = FileNameOffsets[NextFileIndex]; + Names.setOffset(FileOffset); + if (auto EC = Names.readZeroString(ModuleInfos[I].SourceFiles[J])) + return EC; } } Index: tools/llvm-pdbdump/llvm-pdbdump.cpp =================================================================== --- tools/llvm-pdbdump/llvm-pdbdump.cpp +++ tools/llvm-pdbdump/llvm-pdbdump.cpp @@ -466,7 +466,7 @@ to_string(Modi.SourceFiles.size()) + " Contributing Source Files"; ListScope LL(P, FileListName); for (auto File : Modi.SourceFiles) - P.printString(File); + P.printString(File.str()); } bool HasModuleDI = (Modi.Info.getModuleStreamIndex() < File.getNumStreams());