Index: llvm/include/llvm/DebugInfo/CodeView/RandomAccessTypeVisitor.h =================================================================== --- /dev/null +++ llvm/include/llvm/DebugInfo/CodeView/RandomAccessTypeVisitor.h @@ -0,0 +1,118 @@ +//===- RandomAccessTypeVisitor.h ------------------------------ *- C++ --*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_DEBUGINFO_CODEVIEW_RANDOMACCESSTYPEVISITOR_H +#define LLVM_DEBUGINFO_CODEVIEW_RANDOMACCESSTYPEVISITOR_H + +#include "llvm/ADT/TinyPtrVector.h" +#include "llvm/DebugInfo/CodeView/CVTypeVisitor.h" +#include "llvm/DebugInfo/CodeView/TypeDatabase.h" +#include "llvm/DebugInfo/CodeView/TypeDatabaseVisitor.h" +#include "llvm/DebugInfo/CodeView/TypeDeserializer.h" +#include "llvm/DebugInfo/CodeView/TypeIndex.h" +#include "llvm/DebugInfo/CodeView/TypeRecord.h" +#include "llvm/DebugInfo/CodeView/TypeVisitorCallbackPipeline.h" +#include "llvm/Support/Error.h" + +namespace llvm { +namespace codeview { + +class TypeDatabase; +class TypeServerHandler; +class TypeVisitorCallbacks; + +/// \brief Provides amortized O(1) random access to a CodeView type stream. +/// Normally to access a type from a type stream, you must know its byte +/// offset into the type stream, because type records are variable-lengthed. +/// However, this is not the way we prefer to access them. For example, given +/// a symbol record one of the fields may be the TypeIndex of the symbol's +/// type record. Or given a type record such as an array type, there might +/// be a TypeIndex for the element type. Sequential access is perfect when +/// we're just dumping every entry, but it's very poor for real world usage. +/// +/// Type streams in PDBs contain an additional field which is a list of pairs +/// containing indices and their corresponding offsets, roughly every ~8KB of +/// record data. This general idea need not be confined to PDBs though. By +/// supplying such an array, the producer of a type stream can allow the +/// consumer much better access time, because the consumer can find the nearest +/// index in this array, and do a linear scan forward only from there. +/// +/// RandomAccessTypeVisitor implements this algorithm, but additionally goes one +/// step further by caching offsets of every record that has been visited at +/// least once. This way, even repeated visits of the same record will never +/// require more than one linear scan. For a type stream of N elements divided +/// into M chunks of roughly equal size, this yields a worst case lookup time +/// of O(N/M) and an amortized time of O(1). +class RandomAccessTypeVisitor { + typedef FixedStreamArray PartialOffsetArray; + +public: + RandomAccessTypeVisitor(const CVTypeArray &Types, uint32_t NumRecords, + PartialOffsetArray PartialOffsets, + TypeVisitorCallbacks &Callbacks); + + void addTypeServerHandler(TypeServerHandler &Handler); + + Error visitTypeIndex(TypeIndex Index); + + const RandomAccessTypeDatabase &database() const { return Database; } + +private: + bool isOffsetKnown(TypeIndex TI) const; + uint32_t getOffset(TypeIndex TI) const; + Error visitRange(TypeIndex First, TypeIndex Last); + + PartialOffsetArray::Iterator findRangeStartIter(TypeIndex TI) const; + TypeIndex findRangeStartIndex(TypeIndex TI) const; + + // Visited records get automatically added to the type database. + RandomAccessTypeDatabase Database; + + /// The type array to allow random access visitation of. + const CVTypeArray &Types; + + // The database visitor which adds new records to the database. + TypeDatabaseVisitor DatabaseVisitor; + + // The deserializer which deserializes new records. + TypeDeserializer Deserializer; + + // The visitation callback pipeline to use. By default this contains a + // deserializer and a type database visitor. But the callback specified + // in the constructor is also added. + TypeVisitorCallbackPipeline Pipeline; + + // The visitor used to visit the internal pipeline for deserialization and + // database maintenance. + CVTypeVisitor InternalVisitor; + + // The visitor used to visit the user's custom visitation callback. This is + // separate from the normal pipeline because we want to visit them on + // different occasions. + CVTypeVisitor UserVisitor; + + // A vector mapping type indices to type offset. For every record that has + // been visited, contains the absolute offset of that record in the record + // array. + std::vector KnownOffsets; + + // An array of index offsets for the given type stream, allowing log(N) + // lookups of a type record by index. Similar to KnownOffsets but only + // contains offsets for some type indices, some of which may not have + // ever been visited. + PartialOffsetArray PartialOffsets; + + /// The list of TypeServer handlers (usually empty). + TinyPtrVector Handlers; +}; + +} // end namespace codeview +} // end namespace llvm + +#endif // LLVM_DEBUGINFO_CODEVIEW_RANDOMACCESSTYPEVISITOR_H Index: llvm/include/llvm/DebugInfo/CodeView/TypeDatabase.h =================================================================== --- llvm/include/llvm/DebugInfo/CodeView/TypeDatabase.h +++ llvm/include/llvm/DebugInfo/CodeView/TypeDatabase.h @@ -40,13 +40,11 @@ const CVType &getTypeRecord(TypeIndex Index) const; CVType &getTypeRecord(TypeIndex Index); - bool containsTypeIndex(TypeIndex Index) const; + bool contains(TypeIndex Index) const; uint32_t size() const; protected: - uint32_t toArrayIndex(TypeIndex Index) const; - BumpPtrAllocator Allocator; /// All user defined type records in .debug$T live in here. Type indices @@ -72,9 +70,12 @@ const CVType &getTypeRecord(TypeIndex Index) const; CVType &getTypeRecord(TypeIndex Index); - bool containsTypeIndex(TypeIndex Index) const; + bool contains(TypeIndex Index) const; uint32_t size() const; + uint32_t capacity() const; + + Optional findPrevious(TypeIndex PriorTo) const; private: BitVector ValidRecords; Index: llvm/include/llvm/DebugInfo/CodeView/TypeDeserializer.h =================================================================== --- llvm/include/llvm/DebugInfo/CodeView/TypeDeserializer.h +++ llvm/include/llvm/DebugInfo/CodeView/TypeDeserializer.h @@ -46,6 +46,10 @@ return Mapping->Mapping.visitTypeBegin(Record); } + Error visitTypeBegin(CVType &Record, TypeIndex Index) override { + return visitTypeBegin(Record); + } + Error visitTypeEnd(CVType &Record) override { assert(Mapping && "Not in a type mapping!"); auto EC = Mapping->Mapping.visitTypeEnd(Record); Index: llvm/include/llvm/DebugInfo/CodeView/TypeIndex.h =================================================================== --- llvm/include/llvm/DebugInfo/CodeView/TypeIndex.h +++ llvm/include/llvm/DebugInfo/CodeView/TypeIndex.h @@ -242,6 +242,13 @@ support::ulittle32_t Index; }; +// Used for pseudo-indexing an array of type records. An array of such records +// sorted by TypeIndex can allow log(N) lookups even though such a type record +// stream does not provide random access. +struct TypeIndexOffset { + TypeIndex Type; + support::ulittle32_t Offset; +}; } } Index: llvm/include/llvm/DebugInfo/PDB/Native/RawTypes.h =================================================================== --- llvm/include/llvm/DebugInfo/PDB/Native/RawTypes.h +++ llvm/include/llvm/DebugInfo/PDB/Native/RawTypes.h @@ -73,13 +73,6 @@ support::ulittle32_t SecByteLength; // Byte count of the segment or group. }; -// Used for serialized hash table in TPI stream. -// In the reference, it is an array of TI and cbOff pair. -struct TypeIndexOffset { - codeview::TypeIndex Type; - support::ulittle32_t Offset; -}; - /// Some of the values are stored in bitfields. Since this needs to be portable /// across compilers and architectures (big / little endian in particular) we /// can't use the actual structures below, but must instead do the shifting Index: llvm/include/llvm/DebugInfo/PDB/Native/TpiStream.h =================================================================== --- llvm/include/llvm/DebugInfo/PDB/Native/TpiStream.h +++ llvm/include/llvm/DebugInfo/PDB/Native/TpiStream.h @@ -47,7 +47,7 @@ uint32_t getHashKeySize() const; uint32_t getNumHashBuckets() const; FixedStreamArray getHashValues() const; - FixedStreamArray getTypeIndexOffsets() const; + FixedStreamArray getTypeIndexOffsets() const; HashTable &getHashAdjusters(); codeview::CVTypeRange types(bool *HadError) const; @@ -62,7 +62,7 @@ std::unique_ptr HashStream; FixedStreamArray HashValues; - FixedStreamArray TypeIndexOffsets; + FixedStreamArray TypeIndexOffsets; HashTable HashAdjusters; const TpiStreamHeader *Header; Index: llvm/include/llvm/DebugInfo/PDB/Native/TpiStreamBuilder.h =================================================================== --- llvm/include/llvm/DebugInfo/PDB/Native/TpiStreamBuilder.h +++ llvm/include/llvm/DebugInfo/PDB/Native/TpiStreamBuilder.h @@ -75,7 +75,7 @@ Optional VerHeader; std::vector> TypeRecords; std::vector TypeHashes; - std::vector TypeIndexOffsets; + std::vector TypeIndexOffsets; uint32_t HashStreamIndex = kInvalidStreamIndex; std::unique_ptr HashValueStream; Index: llvm/include/llvm/Support/BinaryStreamArray.h =================================================================== --- llvm/include/llvm/Support/BinaryStreamArray.h +++ llvm/include/llvm/Support/BinaryStreamArray.h @@ -139,6 +139,7 @@ } uint32_t offset() const { return AbsOffset; } + uint32_t getRecordLength() const { return ThisLen; } private: void moveToEnd() { @@ -294,6 +295,8 @@ friend class FixedStreamArrayIterator; public: + typedef FixedStreamArrayIterator Iterator; + FixedStreamArray() = default; explicit FixedStreamArray(BinaryStreamRef Stream) : Stream(Stream) { assert(Stream.getLength() % sizeof(T) == 0); @@ -371,7 +374,7 @@ } FixedStreamArrayIterator &operator-=(std::ptrdiff_t N) { - assert(Index >= N); + assert(std::ptrdiff_t(Index) >= N); Index -= N; return *this; } Index: llvm/lib/DebugInfo/CodeView/CMakeLists.txt =================================================================== --- llvm/lib/DebugInfo/CodeView/CMakeLists.txt +++ llvm/lib/DebugInfo/CodeView/CMakeLists.txt @@ -13,6 +13,7 @@ ModuleDebugFragmentVisitor.cpp ModuleDebugInlineeLinesFragment.cpp ModuleDebugLineFragment.cpp + RandomAccessTypeVisitor.cpp RecordSerialization.cpp StringTable.cpp SymbolRecordMapping.cpp Index: llvm/lib/DebugInfo/CodeView/CVTypeVisitor.cpp =================================================================== --- llvm/lib/DebugInfo/CodeView/CVTypeVisitor.cpp +++ llvm/lib/DebugInfo/CodeView/CVTypeVisitor.cpp @@ -26,8 +26,7 @@ : Callbacks(Callbacks) {} template -static Error visitKnownRecord(CVTypeVisitor &Visitor, CVType &Record, - TypeVisitorCallbacks &Callbacks) { +static Error visitKnownRecord(CVType &Record, TypeVisitorCallbacks &Callbacks) { TypeRecordKind RK = static_cast(Record.Type); T KnownRecord(RK); if (auto EC = Callbacks.visitKnownRecord(Record, KnownRecord)) @@ -107,7 +106,7 @@ break; #define TYPE_RECORD(EnumName, EnumVal, Name) \ case EnumName: { \ - if (auto EC = visitKnownRecord(*this, Record, Callbacks)) \ + if (auto EC = visitKnownRecord(Record, Callbacks)) \ return EC; \ break; \ } Index: llvm/lib/DebugInfo/CodeView/RandomAccessTypeVisitor.cpp =================================================================== --- /dev/null +++ llvm/lib/DebugInfo/CodeView/RandomAccessTypeVisitor.cpp @@ -0,0 +1,155 @@ +//===- RandomAccessTypeVisitor.cpp ---------------------------- *- C++ --*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "llvm/DebugInfo/CodeView/RandomAccessTypeVisitor.h" + +#include "llvm/DebugInfo/CodeView/TypeDatabase.h" +#include "llvm/DebugInfo/CodeView/TypeServerHandler.h" +#include "llvm/DebugInfo/CodeView/TypeVisitorCallbacks.h" + +using namespace llvm; +using namespace llvm::codeview; + +RandomAccessTypeVisitor::RandomAccessTypeVisitor( + const CVTypeArray &Types, uint32_t NumRecords, + PartialOffsetArray PartialOffsets, TypeVisitorCallbacks &Callbacks) + : Database(NumRecords), Types(Types), DatabaseVisitor(Database), + InternalVisitor(Pipeline), UserVisitor(Callbacks), + PartialOffsets(PartialOffsets) { + Pipeline.addCallbackToPipeline(Deserializer); + Pipeline.addCallbackToPipeline(DatabaseVisitor); + + KnownOffsets.resize(Database.capacity()); + + // Do not add `Callbacks` to the internal pipeline. If the user specifies + // that they want to visit TypeIndex 42, then we may need to first visit all + // type indices between 0 and 41 before we get to 42. The user should not be + // notified of those. Only once 42 has been reached, deserialized, and added + // to the database should the user's callback be invoked, and only for that + // one single record, not all records in between. That is why we maintain a + // separate `UserVisitor` for invoking the callback handler that the user + // specifies. +} + +void RandomAccessTypeVisitor::addTypeServerHandler(TypeServerHandler &Handler) { + Handlers.push_back(&Handler); + UserVisitor.addTypeServerHandler(Handler); +} + +RandomAccessTypeVisitor::PartialOffsetArray::Iterator +RandomAccessTypeVisitor::findRangeStartIter(TypeIndex TI) const { + auto Iter = std::upper_bound(PartialOffsets.begin(), PartialOffsets.end(), TI, + [](TypeIndex Value, const TypeIndexOffset &IO) { + return Value < IO.Type; + }); + assert(Iter != PartialOffsets.begin()); + --Iter; + return Iter; +} + +TypeIndex RandomAccessTypeVisitor::findRangeStartIndex(TypeIndex TI) const { + auto Iter = findRangeStartIter(TI); + assert(Iter != PartialOffsets.end()); + return Iter->Type; +} + +Error RandomAccessTypeVisitor::visitTypeIndex(TypeIndex TI) { + assert(TI.toArrayIndex() < Database.capacity()); + + if (!Database.contains(TI)) { + TypeIndex RS = findRangeStartIndex(TI); + assert(RS <= TI); + if (auto EC = visitRange(RS, TI)) + return EC; + } + + auto &Record = Database.getTypeRecord(TI); + return UserVisitor.visitTypeRecord(Record, TI); +} + +bool RandomAccessTypeVisitor::isOffsetKnown(TypeIndex TI) const { + assert(TI.toArrayIndex() < Database.capacity()); + + // It's in the KnownOffsets array + if (Database.contains(TI)) + return true; + + // It's in the PartialOffsets array. + auto Iter = findRangeStartIter(TI); + if (Iter->Type == TI) + return true; + + return false; +} + +uint32_t RandomAccessTypeVisitor::getOffset(TypeIndex TI) const { + assert(isOffsetKnown(TI)); + + // It's in the KnownOffsets array + if (Database.contains(TI)) + return KnownOffsets[TI.toArrayIndex()]; + + // It's in the PartialOffsets array. + auto Iter = findRangeStartIter(TI); + assert(Iter != PartialOffsets.end()); + assert(Iter->Type == TI); + return Iter->Offset; +} + +Error RandomAccessTypeVisitor::visitRange(TypeIndex First, TypeIndex Last) { + assert(Last.toArrayIndex() < Database.capacity()); + assert(isOffsetKnown(First)); + assert(!Database.contains(Last)); + + // Find the last set (i.e. visited) record before `Last`. Even within a + // single block, some records may have already been visited. Skip all of + // those and go as far as we can to minimize the amount of records we have to + // deserialize. + CVTypeArray::Iterator Iter; + + Optional PrevKnown = Database.findPrevious(Last); + if (PrevKnown && *PrevKnown >= First) { + assert(Database.contains(*PrevKnown)); + uint32_t O = KnownOffsets[PrevKnown->toArrayIndex()]; + // We found a visited record in the range. Skip it and we should be at the + // beginning of the unvisited range. + Iter = Types.at(O); + ++Iter; + First = *PrevKnown + 1; + } else { + // We didn't find a visited record in the range. The first item of the + // range should show up in the PartialOffsets array. + auto PartialIter = findRangeStartIter(First); + assert(PartialIter != PartialOffsets.end()); + Iter = Types.at(PartialIter->Offset); + } + + assert(!Database.contains(First)); + assert(Iter != Types.end()); + + // At this point every record in the range [First, Last] should be unvisited + TypeIndex Current = First; + + uint32_t Offset = Iter.offset(); + while (Current <= Last) { + assert(!Database.contains(Current)); + + KnownOffsets[Current.toArrayIndex()] = Offset; + + CVType &Record = *Iter; + if (auto EC = InternalVisitor.visitTypeRecord(Record, Current)) + return EC; + + Offset += Iter.getRecordLength(); + ++Current; + ++Iter; + } + + return Error::success(); +} Index: llvm/lib/DebugInfo/CodeView/TypeDatabase.cpp =================================================================== --- llvm/lib/DebugInfo/CodeView/TypeDatabase.cpp +++ llvm/lib/DebugInfo/CodeView/TypeDatabase.cpp @@ -112,24 +112,19 @@ } const CVType &TypeDatabase::getTypeRecord(TypeIndex Index) const { - return TypeRecords[toArrayIndex(Index)]; + return TypeRecords[Index.toArrayIndex()]; } CVType &TypeDatabase::getTypeRecord(TypeIndex Index) { - return TypeRecords[toArrayIndex(Index)]; + return TypeRecords[Index.toArrayIndex()]; } -bool TypeDatabase::containsTypeIndex(TypeIndex Index) const { - return toArrayIndex(Index) < CVUDTNames.size(); +bool TypeDatabase::contains(TypeIndex Index) const { + return Index.toArrayIndex() < CVUDTNames.size(); } uint32_t TypeDatabase::size() const { return CVUDTNames.size(); } -uint32_t TypeDatabase::toArrayIndex(TypeIndex Index) const { - assert(Index.getIndex() >= TypeIndex::FirstNonSimpleIndex); - return Index.getIndex() - TypeIndex::FirstNonSimpleIndex; -} - RandomAccessTypeDatabase::RandomAccessTypeDatabase(uint32_t ExpectedSize) : TypeDatabase(ExpectedSize) { ValidRecords.resize(ExpectedSize); @@ -139,7 +134,7 @@ void RandomAccessTypeDatabase::recordType(StringRef Name, TypeIndex Index, const CVType &Data) { - assert(!containsTypeIndex(Index)); + assert(!contains(Index)); uint32_t ZI = Index.getIndex() - TypeIndex::FirstNonSimpleIndex; CVUDTNames[ZI] = Name; @@ -148,25 +143,34 @@ } StringRef RandomAccessTypeDatabase::getTypeName(TypeIndex Index) const { - assert(containsTypeIndex(Index)); return TypeDatabase::getTypeName(Index); } const CVType &RandomAccessTypeDatabase::getTypeRecord(TypeIndex Index) const { - assert(containsTypeIndex(Index)); + assert(contains(Index)); return TypeDatabase::getTypeRecord(Index); } CVType &RandomAccessTypeDatabase::getTypeRecord(TypeIndex Index) { - assert(containsTypeIndex(Index)); + assert(contains(Index)); return TypeDatabase::getTypeRecord(Index); } -bool RandomAccessTypeDatabase::containsTypeIndex(TypeIndex Index) const { - if (Index.isSimple()) - return true; - - return ValidRecords.test(toArrayIndex(Index)); +bool RandomAccessTypeDatabase::contains(TypeIndex Index) const { + return ValidRecords.test(Index.toArrayIndex()); } uint32_t RandomAccessTypeDatabase::size() const { return ValidRecords.count(); } + +uint32_t RandomAccessTypeDatabase::capacity() const { + return ValidRecords.size(); +} + +Optional +RandomAccessTypeDatabase::findPrevious(TypeIndex PriorTo) const { + uint32_t ZI = PriorTo.toArrayIndex(); + int N = ValidRecords.find_prev(ZI); + if (N == -1) + return None; + return TypeIndex::fromArrayIndex(N); +} Index: llvm/lib/DebugInfo/CodeView/TypeDatabaseVisitor.cpp =================================================================== --- llvm/lib/DebugInfo/CodeView/TypeDatabaseVisitor.cpp +++ llvm/lib/DebugInfo/CodeView/TypeDatabaseVisitor.cpp @@ -16,8 +16,6 @@ using namespace llvm::codeview; Error TypeDatabaseVisitor::visitTypeBegin(CVType &Record) { - assert(TypeDB.is()); - assert(!IsInFieldList); // Reset Name to the empty string. If the visitor sets it, we know it. Name = ""; @@ -31,18 +29,18 @@ } StringRef TypeDatabaseVisitor::getTypeName(TypeIndex Index) const { - if (auto DB = TypeDB.get()) + if (auto DB = TypeDB.dyn_cast()) return DB->getTypeName(Index); - else if (auto DB = TypeDB.get()) + else if (auto DB = TypeDB.dyn_cast()) return DB->getTypeName(Index); llvm_unreachable("Invalid TypeDB Kind!"); } StringRef TypeDatabaseVisitor::saveTypeName(StringRef Name) { - if (auto DB = TypeDB.get()) + if (auto DB = TypeDB.dyn_cast()) return DB->saveTypeName(Name); - else if (auto DB = TypeDB.get()) + else if (auto DB = TypeDB.dyn_cast()) return DB->saveTypeName(Name); llvm_unreachable("Invalid TypeDB Kind!"); @@ -69,9 +67,9 @@ // CVUDTNames is indexed by type index, and must have one entry for every // type. Field list members are not recorded, and are only referenced by // their containing field list record. - if (auto DB = TypeDB.get()) + if (auto DB = TypeDB.dyn_cast()) DB->recordType(Name, CVR); - else if (auto DB = TypeDB.get()) + else if (auto DB = TypeDB.dyn_cast()) DB->recordType(Name, CurrentTypeIndex, CVR); return Error::success(); Index: llvm/lib/DebugInfo/PDB/Native/TpiStreamBuilder.cpp =================================================================== --- llvm/lib/DebugInfo/PDB/Native/TpiStreamBuilder.cpp +++ llvm/lib/DebugInfo/PDB/Native/TpiStreamBuilder.cpp @@ -109,7 +109,7 @@ } uint32_t TpiStreamBuilder::calculateIndexOffsetSize() const { - return TypeIndexOffsets.size() * sizeof(TypeIndexOffset); + return TypeIndexOffsets.size() * sizeof(codeview::TypeIndexOffset); } Error TpiStreamBuilder::finalizeMsfLayout() { Index: llvm/tools/llvm-pdbdump/LLVMOutputStyle.cpp =================================================================== --- llvm/tools/llvm-pdbdump/LLVMOutputStyle.cpp +++ llvm/tools/llvm-pdbdump/LLVMOutputStyle.cpp @@ -180,7 +180,7 @@ CompactTypeDumpVisitor CTDV(DB, Index, &P); CVTypeVisitor Visitor(CTDV); DictScope D(P, Label); - if (DB.containsTypeIndex(Index)) { + if (DB.contains(Index)) { CVType &Type = DB.getTypeRecord(Index); if (auto EC = Visitor.visitTypeRecord(Type)) return EC; Index: llvm/unittests/DebugInfo/CMakeLists.txt =================================================================== --- llvm/unittests/DebugInfo/CMakeLists.txt +++ llvm/unittests/DebugInfo/CMakeLists.txt @@ -1,3 +1,3 @@ - +add_subdirectory(CodeView) add_subdirectory(DWARF) add_subdirectory(PDB) Index: llvm/unittests/DebugInfo/CodeView/CMakeLists.txt =================================================================== --- /dev/null +++ llvm/unittests/DebugInfo/CodeView/CMakeLists.txt @@ -0,0 +1,11 @@ +set(LLVM_LINK_COMPONENTS + DebugInfoCodeView + ) + +set(DebugInfoCodeViewSources + RandomAccessVisitorTest.cpp + ) + +add_llvm_unittest(DebugInfoCodeViewTests + ${DebugInfoCodeViewSources} + ) Index: llvm/unittests/DebugInfo/CodeView/ErrorChecking.h =================================================================== --- /dev/null +++ llvm/unittests/DebugInfo/CodeView/ErrorChecking.h @@ -0,0 +1,61 @@ +//===- ErrorChecking.h - Helpers for verifying llvm::Errors -----*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_UNITTESTS_DEBUGINFO_PDB_ERRORCHECKING_H +#define LLVM_UNITTESTS_DEBUGINFO_PDB_ERRORCHECKING_H + +#define EXPECT_NO_ERROR(Err) \ + { \ + auto E = Err; \ + EXPECT_FALSE(static_cast(E)); \ + if (E) \ + consumeError(std::move(E)); \ + } + +#define EXPECT_ERROR(Err) \ + { \ + auto E = Err; \ + EXPECT_TRUE(static_cast(E)); \ + if (E) \ + consumeError(std::move(E)); \ + } + +#define EXPECT_EXPECTED(Exp) \ + { \ + auto E = Exp.takeError(); \ + EXPECT_FALSE(static_cast(E)); \ + if (E) { \ + consumeError(std::move(E)); \ + return; \ + } \ + } + +#define EXPECT_EXPECTED_EQ(Val, Exp) \ + { \ + auto Result = Exp; \ + auto E = Result.takeError(); \ + EXPECT_FALSE(static_cast(E)); \ + if (E) { \ + consumeError(std::move(E)); \ + return; \ + } \ + EXPECT_EQ(Val, *Result); \ + } + +#define EXPECT_UNEXPECTED(Exp) \ + { \ + auto E = Exp.takeError(); \ + EXPECT_TRUE(static_cast(E)); \ + if (E) { \ + consumeError(std::move(E)); \ + return; \ + } \ + } + +#endif Index: llvm/unittests/DebugInfo/CodeView/RandomAccessVisitorTest.cpp =================================================================== --- /dev/null +++ llvm/unittests/DebugInfo/CodeView/RandomAccessVisitorTest.cpp @@ -0,0 +1,353 @@ +//===- llvm/unittest/DebugInfo/PDB/TypeServerHandlerTest.cpp --------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "ErrorChecking.h" + +#include "llvm/ADT/SmallBitVector.h" +#include "llvm/DebugInfo/CodeView/CVTypeVisitor.h" +#include "llvm/DebugInfo/CodeView/RandomAccessTypeVisitor.h" +#include "llvm/DebugInfo/CodeView/TypeRecord.h" +#include "llvm/DebugInfo/CodeView/TypeRecordMapping.h" +#include "llvm/DebugInfo/CodeView/TypeSerializer.h" +#include "llvm/DebugInfo/CodeView/TypeServerHandler.h" +#include "llvm/DebugInfo/CodeView/TypeTableBuilder.h" +#include "llvm/DebugInfo/CodeView/TypeVisitorCallbackPipeline.h" +#include "llvm/DebugInfo/CodeView/TypeVisitorCallbacks.h" +#include "llvm/DebugInfo/PDB/Native/RawTypes.h" +#include "llvm/Support/Allocator.h" +#include "llvm/Support/BinaryItemStream.h" +#include "llvm/Support/Error.h" + +#include "gtest/gtest.h" + +using namespace llvm; +using namespace llvm::codeview; +using namespace llvm::pdb; + +namespace llvm { +namespace codeview { +inline bool operator==(const ArrayRecord &R1, const ArrayRecord &R2) { + if (R1.ElementType != R2.ElementType) + return false; + if (R1.IndexType != R2.IndexType) + return false; + if (R1.Name != R2.Name) + return false; + if (R1.Size != R2.Size) + return false; + return true; +} +inline bool operator!=(const ArrayRecord &R1, const ArrayRecord &R2) { + return !(R1 == R2); +} + +inline bool operator==(const CVType &R1, const CVType &R2) { + if (R1.Type != R2.Type) + return false; + if (R1.RecordData != R2.RecordData) + return false; + return true; +} +inline bool operator!=(const CVType &R1, const CVType &R2) { + return !(R1 == R2); +} +} +} + +namespace llvm { +template <> struct BinaryItemTraits { + static size_t length(const CVType &Item) { return Item.length(); } + static ArrayRef bytes(const CVType &Item) { return Item.data(); } +}; +} + +namespace { + +class MockCallbacks : public TypeVisitorCallbacks { +public: + virtual Error visitTypeBegin(CVType &CVR, TypeIndex Index) { + Indices.push_back(Index); + return Error::success(); + } + virtual Error visitKnownRecord(CVType &CVR, ArrayRecord &AR) { + VisitedRecords.push_back(AR); + RawRecords.push_back(CVR); + return Error::success(); + } + + uint32_t count() const { + assert(Indices.size() == RawRecords.size()); + assert(Indices.size() == VisitedRecords.size()); + return Indices.size(); + } + std::vector Indices; + std::vector RawRecords; + std::vector VisitedRecords; +}; + +class RandomAccessVisitorTest : public testing::Test { +public: + RandomAccessVisitorTest() {} + + static void SetUpTestCase() { + GlobalState = llvm::make_unique(); + + TypeTableBuilder Builder(GlobalState->Allocator); + + uint32_t Offset = 0; + for (int I = 0; I < 11; ++I) { + ArrayRecord AR(TypeRecordKind::Array); + AR.ElementType = TypeIndex::Int32(); + AR.IndexType = TypeIndex::UInt32(); + AR.Size = I; + std::string Name; + raw_string_ostream Stream(Name); + Stream << "Array [" << I << "]"; + AR.Name = GlobalState->Strings.save(Stream.str()); + GlobalState->Records.push_back(AR); + GlobalState->Indices.push_back(Builder.writeKnownType(AR)); + + CVType Type(TypeLeafKind::LF_ARRAY, Builder.records().back()); + GlobalState->TypeVector.push_back(Type); + + GlobalState->AllOffsets.push_back( + {GlobalState->Indices.back(), ulittle32_t(Offset)}); + Offset += Type.length(); + } + + GlobalState->ItemStream.setItems(GlobalState->TypeVector); + GlobalState->TypeArray = VarStreamArray(GlobalState->ItemStream); + } + + static void TearDownTestCase() { GlobalState.reset(); } + + void SetUp() override { + TestState = llvm::make_unique(); + + TestState->Pipeline.addCallbackToPipeline(TestState->Deserializer); + TestState->Pipeline.addCallbackToPipeline(TestState->Callbacks); + } + + void TearDown() override { TestState.reset(); } + +protected: + bool ValidateDatabaseRecord(const RandomAccessTypeVisitor &Visitor, + uint32_t Index) { + TypeIndex TI = TypeIndex::fromArrayIndex(Index); + if (!Visitor.database().contains(TI)) + return false; + if (GlobalState->TypeVector[Index] != Visitor.database().getTypeRecord(TI)) + return false; + return true; + } + + bool ValidateVisitedRecord(uint32_t VisitationOrder, + uint32_t GlobalArrayIndex) { + TypeIndex TI = TypeIndex::fromArrayIndex(GlobalArrayIndex); + if (TI != TestState->Callbacks.Indices[VisitationOrder]) + return false; + + if (GlobalState->TypeVector[TI.toArrayIndex()] != + TestState->Callbacks.RawRecords[VisitationOrder]) + return false; + + if (GlobalState->Records[TI.toArrayIndex()] != + TestState->Callbacks.VisitedRecords[VisitationOrder]) + return false; + + return true; + } + + struct GlobalTestState { + GlobalTestState() : Strings(Allocator), ItemStream(llvm::support::little) {} + + BumpPtrAllocator Allocator; + StringSaver Strings; + + std::vector Records; + std::vector Indices; + std::vector AllOffsets; + std::vector TypeVector; + BinaryItemStream ItemStream; + VarStreamArray TypeArray; + + MutableBinaryByteStream Stream; + }; + + struct PerTestState { + FixedStreamArray Offsets; + + TypeVisitorCallbackPipeline Pipeline; + TypeDeserializer Deserializer; + MockCallbacks Callbacks; + }; + + FixedStreamArray + createPartialOffsets(MutableBinaryByteStream &Storage, + std::initializer_list Indices) { + + uint32_t Count = Indices.size(); + uint32_t Size = Count * sizeof(TypeIndexOffset); + uint8_t *Buffer = GlobalState->Allocator.Allocate(Size); + MutableArrayRef Bytes(Buffer, Size); + Storage = MutableBinaryByteStream(Bytes, support::little); + BinaryStreamWriter Writer(Storage); + for (const auto I : Indices) + consumeError(Writer.writeObject(GlobalState->AllOffsets[I])); + + BinaryStreamReader Reader(Storage); + FixedStreamArray Result; + consumeError(Reader.readArray(Result, Count)); + return Result; + } + + static std::unique_ptr GlobalState; + std::unique_ptr TestState; +}; + +std::unique_ptr + RandomAccessVisitorTest::GlobalState; +} + +TEST_F(RandomAccessVisitorTest, MultipleVisits) { + TestState->Offsets = createPartialOffsets(GlobalState->Stream, {0, 8}); + RandomAccessTypeVisitor Visitor(GlobalState->TypeArray, + GlobalState->TypeVector.size(), + TestState->Offsets, TestState->Pipeline); + + std::vector IndicesToVisit = {5, 5, 5}; + + for (uint32_t I : IndicesToVisit) { + TypeIndex TI = TypeIndex::fromArrayIndex(I); + EXPECT_NO_ERROR(Visitor.visitTypeIndex(TI)); + } + + // 0, 1, 2, 3, 4, 5 + EXPECT_EQ(6, Visitor.database().size()); + for (uint32_t I = 0; I <= 5; ++I) + EXPECT_TRUE(ValidateDatabaseRecord(Visitor, I)); + + // 5, 5, 5 + EXPECT_EQ(3, TestState->Callbacks.count()); + for (auto I : enumerate(IndicesToVisit)) + EXPECT_TRUE(ValidateVisitedRecord(I.index(), I.value())); +} + +TEST_F(RandomAccessVisitorTest, DescendingWithinChunk) { + // Visit multiple items from the same "chunk" in reverse order. In this + // example, it's 7 then 4 then 2. At the end, all records from 0 to 7 should + // be known by the database, but only 2, 4, and 7 should have been visited. + TestState->Offsets = createPartialOffsets(GlobalState->Stream, {0, 8}); + + std::vector IndicesToVisit = {7, 4, 2}; + + RandomAccessTypeVisitor Visitor(GlobalState->TypeArray, + GlobalState->TypeVector.size(), + TestState->Offsets, TestState->Pipeline); + + for (uint32_t I : IndicesToVisit) { + TypeIndex TI = TypeIndex::fromArrayIndex(I); + EXPECT_NO_ERROR(Visitor.visitTypeIndex(TI)); + } + + // [0, 7] + EXPECT_EQ(8, Visitor.database().size()); + for (uint32_t I = 0; I < 8; ++I) + EXPECT_TRUE(ValidateDatabaseRecord(Visitor, I)); + + // 2, 4, 7 + EXPECT_EQ(3, TestState->Callbacks.count()); + for (auto I : enumerate(IndicesToVisit)) + EXPECT_TRUE(ValidateVisitedRecord(I.index(), I.value())); +} + +TEST_F(RandomAccessVisitorTest, AscendingWithinChunk) { + // * Visit multiple items from the same chunk in ascending order, ensuring + // that intermediate items are not visited. In the below example, it's + // 5 -> 6 -> 7 which come from the [4,8) chunk. + TestState->Offsets = createPartialOffsets(GlobalState->Stream, {0, 8}); + + std::vector IndicesToVisit = {2, 4, 7}; + + RandomAccessTypeVisitor Visitor(GlobalState->TypeArray, + GlobalState->TypeVector.size(), + TestState->Offsets, TestState->Pipeline); + + for (uint32_t I : IndicesToVisit) { + TypeIndex TI = TypeIndex::fromArrayIndex(I); + EXPECT_NO_ERROR(Visitor.visitTypeIndex(TI)); + } + + // [0, 7] + EXPECT_EQ(8, Visitor.database().size()); + for (uint32_t I = 0; I < 8; ++I) + EXPECT_TRUE(ValidateDatabaseRecord(Visitor, I)); + + // 2, 4, 7 + EXPECT_EQ(3, TestState->Callbacks.count()); + for (auto &I : enumerate(IndicesToVisit)) + EXPECT_TRUE(ValidateVisitedRecord(I.index(), I.value())); +} + +TEST_F(RandomAccessVisitorTest, StopPrematurelyInChunk) { + // * Don't visit the last item in one chunk, ensuring that visitation stops + // at the record you specify, and the chunk is only partially visited. + // In the below example, this is tested by visiting 0 and 1 but not 2, + // all from the [0,3) chunk. + TestState->Offsets = createPartialOffsets(GlobalState->Stream, {0, 8}); + + std::vector IndicesToVisit = {0, 1, 2}; + + RandomAccessTypeVisitor Visitor(GlobalState->TypeArray, + GlobalState->TypeVector.size(), + TestState->Offsets, TestState->Pipeline); + + for (uint32_t I : IndicesToVisit) { + TypeIndex TI = TypeIndex::fromArrayIndex(I); + EXPECT_NO_ERROR(Visitor.visitTypeIndex(TI)); + } + + // [0, 2] + EXPECT_EQ(3, Visitor.database().size()); + for (uint32_t I = 0; I < 3; ++I) + EXPECT_TRUE(ValidateDatabaseRecord(Visitor, I)); + + // [0, 2] + EXPECT_EQ(3, TestState->Callbacks.count()); + for (auto I : enumerate(IndicesToVisit)) + EXPECT_TRUE(ValidateVisitedRecord(I.index(), I.value())); +} + +TEST_F(RandomAccessVisitorTest, InnerChunk) { + // Test that when a request comes from a chunk in the middle of the partial + // offsets array, that items from surrounding chunks are not visited or + // added to the database. + TestState->Offsets = createPartialOffsets(GlobalState->Stream, {0, 4, 9}); + + std::vector IndicesToVisit = {5, 7}; + + RandomAccessTypeVisitor Visitor(GlobalState->TypeArray, + GlobalState->TypeVector.size(), + TestState->Offsets, TestState->Pipeline); + + for (uint32_t I : IndicesToVisit) { + TypeIndex TI = TypeIndex::fromArrayIndex(I); + EXPECT_NO_ERROR(Visitor.visitTypeIndex(TI)); + } + + // [4, 7] + EXPECT_EQ(4, Visitor.database().size()); + for (uint32_t I = 4; I < 7; ++I) + EXPECT_TRUE(ValidateDatabaseRecord(Visitor, I)); + + // 5, 7 + EXPECT_EQ(2, TestState->Callbacks.count()); + for (auto &I : enumerate(IndicesToVisit)) + EXPECT_TRUE(ValidateVisitedRecord(I.index(), I.value())); +}