Index: clang/include/clang/AST/VTableBuilder.h =================================================================== --- clang/include/clang/AST/VTableBuilder.h +++ clang/include/clang/AST/VTableBuilder.h @@ -234,6 +234,9 @@ typedef std::pair VTableThunkTy; struct AddressPointLocation { unsigned VTableIndex, AddressPointIndex; + // This flag indicates whether the vtable containing this address point + // has vcall offsets for the type associated with this address point. + bool HasVCallOffsets; }; typedef llvm::DenseMap AddressPointsMapTy; Index: clang/include/clang/Basic/ABI.h =================================================================== --- clang/include/clang/Basic/ABI.h +++ clang/include/clang/Basic/ABI.h @@ -37,6 +37,8 @@ Dtor_Comdat ///< The COMDAT used for dtors }; +class CXXRecordDecl; + /// A return adjustment. struct ReturnAdjustment { /// The non-virtual adjustment from the derived object to its @@ -81,7 +83,13 @@ } } Virtual; - ReturnAdjustment() : NonVirtual(0) {} + /// The type for which VirtualAdjustment is applied. + /// This field is only valid when the virtual return adjust is non-zero. + const CXXRecordDecl *VirtuallyAdjustedType; + + ReturnAdjustment() : NonVirtual(0), VirtuallyAdjustedType(nullptr) {} + ReturnAdjustment(const CXXRecordDecl *Type) + : NonVirtual(0), VirtuallyAdjustedType(Type) {} bool isEmpty() const { return !NonVirtual && Virtual.isEmpty(); } @@ -149,7 +157,13 @@ } } Virtual; - ThisAdjustment() : NonVirtual(0) { } + /// The virtual base type for which VirtualAdjustment is applied. + /// This field is only valid when the virtual this adjustment is non-zero. + const CXXRecordDecl *VirtuallyAdjustedType; + + ThisAdjustment() : NonVirtual(0), VirtuallyAdjustedType(nullptr) {} + ThisAdjustment(const CXXRecordDecl *Type) + : NonVirtual(0), VirtuallyAdjustedType(Type) {} bool isEmpty() const { return !NonVirtual && Virtual.isEmpty(); } Index: clang/include/clang/Basic/CodeGenOptions.def =================================================================== --- clang/include/clang/Basic/CodeGenOptions.def +++ clang/include/clang/Basic/CodeGenOptions.def @@ -367,6 +367,9 @@ /// Whether to emit unused static constants. CODEGENOPT(KeepStaticConsts, 1, 0) +/// Whether to interleave VTables. +CODEGENOPT(VTableInterleaving, 1, 0) + #undef CODEGENOPT #undef ENUM_CODEGENOPT #undef VALUE_CODEGENOPT Index: clang/include/clang/Driver/CC1Options.td =================================================================== --- clang/include/clang/Driver/CC1Options.td +++ clang/include/clang/Driver/CC1Options.td @@ -402,6 +402,8 @@ Values<"a_key,b_key">; def mbranch_target_enforce : Flag<["-"], "mbranch-target-enforce">; def fno_dllexport_inlines : Flag<["-"], "fno-dllexport-inlines">; +def vtable_interleaving : Flag<["-"], "vtable-interleaving">, + HelpText<"Enable VTable interleaving">; //===----------------------------------------------------------------------===// // Dependency Output Options Index: clang/lib/AST/VTableBuilder.cpp =================================================================== --- clang/lib/AST/VTableBuilder.cpp +++ clang/lib/AST/VTableBuilder.cpp @@ -566,13 +566,20 @@ /// (Can be null when we're not building a vtable of the most derived class). const FinalOverriders *Overriders; + /// VCallOffsetBases - The set of BaseSubobjects whose vtables contain vcall + /// offsets. + llvm::DenseSet VCallOffsetBases; + /// AddVCallAndVBaseOffsets - Add vcall offsets and vbase offsets for the /// given base subobject. void AddVCallAndVBaseOffsets(BaseSubobject Base, bool BaseIsVirtual, CharUnits RealBaseOffset); /// AddVCallOffsets - Add vcall offsets for the given base subobject. - void AddVCallOffsets(BaseSubobject Base, CharUnits VBaseOffset); + /// VBase is the virtual base whose vtable contains the vcall offsets that are + /// collected during this function. + void AddVCallOffsets(BaseSubobject Base, CharUnits VBaseOffset, + BaseSubobject VBase); /// AddVBaseOffsets - Add vbase offsets for the given class. void AddVBaseOffsets(const CXXRecordDecl *Base, @@ -604,6 +611,11 @@ const VBaseOffsetOffsetsMapTy &getVBaseOffsetOffsets() const { return VBaseOffsetOffsets; } + + /// isVCallOffsetBase - Check whether Base's vtable has vcall offsets. + bool isVCallOffsetBase(BaseSubobject Base) { + return VCallOffsetBases.find(Base) != VCallOffsetBases.end(); + } }; void @@ -652,7 +664,7 @@ // We only want to add vcall offsets for virtual bases. if (BaseIsVirtual) - AddVCallOffsets(Base, RealBaseOffset); + AddVCallOffsets(Base, RealBaseOffset, Base); } CharUnits VCallAndVBaseOffsetBuilder::getCurrentOffsetOffset() const { @@ -669,7 +681,8 @@ } void VCallAndVBaseOffsetBuilder::AddVCallOffsets(BaseSubobject Base, - CharUnits VBaseOffset) { + CharUnits VBaseOffset, + BaseSubobject VBase) { const CXXRecordDecl *RD = Base.getBase(); const ASTRecordLayout &Layout = Context.getASTRecordLayout(RD); @@ -684,7 +697,7 @@ "Primary base should have a zero offset!"); AddVCallOffsets(BaseSubobject(PrimaryBase, Base.getBaseOffset()), - VBaseOffset); + VBaseOffset, VBase); } // Add the vcall offsets. @@ -695,6 +708,9 @@ CharUnits OffsetOffset = getCurrentOffsetOffset(); + // Now we know that VBase at least has one vcall offset in its vtable. + VCallOffsetBases.insert(VBase); + // Don't add a vcall offset if we already have one for this member function // signature. if (!VCallOffsets.AddVCallOffset(MD, OffsetOffset)) @@ -729,8 +745,7 @@ CharUnits BaseOffset = Base.getBaseOffset() + Layout.getBaseClassOffset(BaseDecl); - AddVCallOffsets(BaseSubobject(BaseDecl, BaseOffset), - VBaseOffset); + AddVCallOffsets(BaseSubobject(BaseDecl, BaseOffset), VBaseOffset, VBase); } } @@ -1175,7 +1190,7 @@ ReturnAdjustment ItaniumVTableBuilder::ComputeReturnAdjustment(BaseOffset Offset) { - ReturnAdjustment Adjustment; + ReturnAdjustment Adjustment(Offset.DerivedClass); if (!Offset.isEmpty()) { if (Offset.VirtualBase) { @@ -1262,7 +1277,7 @@ if (Offset.isEmpty()) return ThisAdjustment(); - ThisAdjustment Adjustment; + ThisAdjustment Adjustment(Offset.VirtualBase); if (Offset.VirtualBase) { // Get the vcall offset map for this virtual base. @@ -1684,11 +1699,13 @@ // Add all address points. while (true) { - AddressPoints.insert( - std::make_pair(BaseSubobject(RD, OffsetInLayoutClass), - VTableLayout::AddressPointLocation{ - unsigned(VTableIndices.size() - 1), - unsigned(AddressPoint - VTableIndex)})); + BaseSubobject CurBaseSubObj(RD, OffsetInLayoutClass); + bool HasVCallOffset = Builder.isVCallOffsetBase(CurBaseSubObj); + AddressPoints.insert(std::make_pair( + CurBaseSubObj, + VTableLayout::AddressPointLocation{unsigned(VTableIndices.size() - 1), + unsigned(AddressPoint - VTableIndex), + HasVCallOffset})); const ASTRecordLayout &Layout = Context.getASTRecordLayout(RD); const CXXRecordDecl *PrimaryBase = Layout.getPrimaryBase(); Index: clang/lib/CodeGen/CGCXXABI.h =================================================================== --- clang/lib/CodeGen/CGCXXABI.h +++ clang/lib/CodeGen/CGCXXABI.h @@ -444,8 +444,7 @@ virtual void setThunkLinkage(llvm::Function *Thunk, bool ForVTable, GlobalDecl GD, bool ReturnAdjustment) = 0; - virtual llvm::Value *performThisAdjustment(CodeGenFunction &CGF, - Address This, + virtual llvm::Value *performThisAdjustment(CodeGenFunction &CGF, Address This, const ThisAdjustment &TA) = 0; virtual llvm::Value *performReturnAdjustment(CodeGenFunction &CGF, Index: clang/lib/CodeGen/CGClass.cpp =================================================================== --- clang/lib/CodeGen/CGClass.cpp +++ clang/lib/CodeGen/CGClass.cpp @@ -2795,7 +2795,8 @@ } llvm::Value *CodeGenFunction::EmitVTableTypeCheckedLoad( - const CXXRecordDecl *RD, llvm::Value *VTable, uint64_t VTableByteOffset) { + const CXXRecordDecl *RD, llvm::Value *CastedVTable, llvm::Type *ReturnType, + llvm::Value *VTableByteOffset) { SanitizerScope SanScope(this); EmitSanitizerStatReport(llvm::SanStat_CFI_VCall); @@ -2804,19 +2805,16 @@ CGM.CreateMetadataIdentifierForType(QualType(RD->getTypeForDecl(), 0)); llvm::Value *TypeId = llvm::MetadataAsValue::get(CGM.getLLVMContext(), MD); - llvm::Value *CastedVTable = Builder.CreateBitCast(VTable, Int8PtrTy); - llvm::Value *CheckedLoad = Builder.CreateCall( - CGM.getIntrinsic(llvm::Intrinsic::type_checked_load), - {CastedVTable, llvm::ConstantInt::get(Int32Ty, VTableByteOffset), - TypeId}); + llvm::Value *CheckedLoad = + Builder.CreateCall(CGM.getIntrinsic(llvm::Intrinsic::type_checked_load), + {CastedVTable, VTableByteOffset, TypeId}); llvm::Value *CheckResult = Builder.CreateExtractValue(CheckedLoad, 1); EmitCheck(std::make_pair(CheckResult, SanitizerKind::CFIVCall), SanitizerHandler::CFICheckFail, nullptr, nullptr); - return Builder.CreateBitCast( - Builder.CreateExtractValue(CheckedLoad, 0), - cast(VTable->getType())->getElementType()); + return Builder.CreateBitCast(Builder.CreateExtractValue(CheckedLoad, 0), + ReturnType); } void CodeGenFunction::EmitForwardingCallToLambda( Index: clang/lib/CodeGen/CGVTables.cpp =================================================================== --- clang/lib/CodeGen/CGVTables.cpp +++ clang/lib/CodeGen/CGVTables.cpp @@ -23,6 +23,8 @@ #include "llvm/Transforms/Utils/Cloning.h" #include #include +#include +#include using namespace clang; using namespace CodeGen; @@ -89,9 +91,8 @@ auto ClassDecl = ResultType->getPointeeType()->getAsCXXRecordDecl(); auto ClassAlign = CGF.CGM.getClassPointerAlignment(ClassDecl); - ReturnValue = CGF.CGM.getCXXABI().performReturnAdjustment(CGF, - Address(ReturnValue, ClassAlign), - Thunk.Return); + ReturnValue = CGF.CGM.getCXXABI().performReturnAdjustment( + CGF, Address(ReturnValue, ClassAlign), Thunk.Return); if (NullCheckValue) { CGF.Builder.CreateBr(AdjustEnd); @@ -298,9 +299,9 @@ // Adjust the 'this' pointer if necessary llvm::Value *AdjustedThisPtr = - Thunk ? CGM.getCXXABI().performThisAdjustment( - *this, LoadCXXThisAddress(), Thunk->This) - : LoadCXXThis(); + Thunk ? CGM.getCXXABI().performThisAdjustment(*this, LoadCXXThisAddress(), + Thunk->This) + : LoadCXXThis(); // If perfect forwarding is required a variadic method, a method using // inalloca, or an unprototyped thunk, use musttail. Emit an error if this @@ -1048,12 +1049,19 @@ CharUnits PointerWidth = Context.toCharUnitsFromBits(Context.getTargetInfo().getPointerWidth(0)); - typedef std::pair AddressPoint; - std::vector AddressPoints; + typedef std::tuple AddressPoint; + + // We use a set first to filter out duplicates. + std::set AddressPointSet; for (auto &&AP : VTLayout.getAddressPoints()) - AddressPoints.push_back(std::make_pair( - AP.first.getBase(), VTLayout.getVTableOffset(AP.second.VTableIndex) + - AP.second.AddressPointIndex)); + AddressPointSet.insert( + std::make_tuple(AP.first.getBase(), + VTLayout.getVTableOffset(AP.second.VTableIndex) + + AP.second.AddressPointIndex, + AP.second.HasVCallOffsets)); + + std::vector AddressPoints(AddressPointSet.begin(), + AddressPointSet.end()); // Sort the address points for determinism. llvm::sort(AddressPoints, [this](const AddressPoint &AP1, @@ -1061,30 +1069,78 @@ if (&AP1 == &AP2) return false; - std::string S1; - llvm::raw_string_ostream O1(S1); - getCXXABI().getMangleContext().mangleTypeName( - QualType(AP1.first->getTypeForDecl(), 0), O1); - O1.flush(); - - std::string S2; - llvm::raw_string_ostream O2(S2); - getCXXABI().getMangleContext().mangleTypeName( - QualType(AP2.first->getTypeForDecl(), 0), O2); - O2.flush(); - - if (S1 < S2) - return true; - if (S1 != S2) - return false; - - return AP1.second < AP2.second; + if (getCodeGenOpts().VTableInterleaving) { + // When vtable interleaving is enabled, sort the address points by: + // 1) the increasing order of the byte offset; + // 2) (for the address points at the same offset) the order from less + // derived to more derived. + if (std::get<1>(AP1) != std::get<1>(AP2)) + return std::get<1>(AP1) < std::get<1>(AP2); + return std::get<0>(AP2)->isDerivedFrom(std::get<0>(AP1)); + } else { + std::string S1; + llvm::raw_string_ostream O1(S1); + getCXXABI().getMangleContext().mangleTypeName( + QualType(std::get<0>(AP1)->getTypeForDecl(), 0), O1); + O1.flush(); + + std::string S2; + llvm::raw_string_ostream O2(S2); + getCXXABI().getMangleContext().mangleTypeName( + QualType(std::get<0>(AP2)->getTypeForDecl(), 0), O2); + O2.flush(); + + if (S1 < S2) + return true; + if (S1 != S2) + return false; + + return std::get<1>(AP1) < std::get<1>(AP2); + } }); + // When vtable interleaving is enabled, we need to keep inheritance + // relationships among types that have address points at the same address. + // Note that the types that have address points at the same offset in the same + // vtable are guaranteed to form a linear inheritance hierarchy. To retain + // inheritance hierarchy among types in IR, we append a "level" field to each + // piece of type metadata. For the type metadata associated with the same + // offset in the same vtable, the higher the level value is, the more derived + // the associated type is. The minimum value for level is 1. + + // Map from each address point offset to the current level of that address. + std::map CurLevelMap; + ArrayRef Comps = VTLayout.vtable_components(); for (auto AP : AddressPoints) { + const CXXRecordDecl *Type = std::get<0>(AP); + unsigned Offset = std::get<1>(AP); + bool HasVCallOffsets = std::get<2>(AP); + + if (getCodeGenOpts().VTableInterleaving) { + if (CurLevelMap.find(Offset) == CurLevelMap.end()) + CurLevelMap[Offset] = 2; + + // Create type metadata for the address point. + AddVTableTypeMetadata(VTable, PointerWidth * Offset, Type, false, + CurLevelMap[Offset]++); + + // If the address point of some type T is in a vtable that contains vcall + // offsets for T, this address in the vtable is also an address point for + // the new type T.vcall_offset, which is intended to group all the vtables + // of T that contain vcall offsets together for interleaving. + if (HasVCallOffsets) + AddVTableTypeMetadata(VTable, PointerWidth * Offset, Type, true, + CurLevelMap[Offset]++); + + // When vtable interleaving is enabled, we don't emit type metadata for + // virtual member functions because the interleaving pass uses type + // metadata to determine address points of vtables. + continue; + } + // Create type metadata for the address point. - AddVTableTypeMetadata(VTable, PointerWidth * AP.second, AP.first); + AddVTableTypeMetadata(VTable, PointerWidth * Offset, Type); // The class associated with each address point could also potentially be // used for indirect calls via a member function pointer, so we need to @@ -1096,8 +1152,8 @@ llvm::Metadata *MD = CreateMetadataIdentifierForVirtualMemPtrType( Context.getMemberPointerType( Comps[I].getFunctionDecl()->getType(), - Context.getRecordType(AP.first).getTypePtr())); - VTable->addTypeMetadata((PointerWidth * I).getQuantity(), MD); + Context.getRecordType(std::get<0>(AP)).getTypePtr())); + VTable->addTypeMetadata((PointerWidth * I).getQuantity(), MD, 0); } } } Index: clang/lib/CodeGen/CodeGenFunction.h =================================================================== --- clang/lib/CodeGen/CodeGenFunction.h +++ clang/lib/CodeGen/CodeGenFunction.h @@ -1965,9 +1965,12 @@ /// true when both vcall CFI and whole-program-vtables are enabled. bool ShouldEmitVTableTypeCheckedLoad(const CXXRecordDecl *RD); - /// Emit a type checked load from the given vtable. - llvm::Value *EmitVTableTypeCheckedLoad(const CXXRecordDecl *RD, llvm::Value *VTable, - uint64_t VTableByteOffset); + /// Emit a type checked load from the given vtable. CastedVTable + /// is of int8ptr type and VTableByteOffset is a byte offset. + llvm::Value *EmitVTableTypeCheckedLoad(const CXXRecordDecl *RD, + llvm::Value *CastedVTable, + llvm::Type *EntryType, + llvm::Value *VTableByteOffset); /// EnterDtorCleanups - Enter the cleanups necessary to complete the /// given phase of destruction for a destructor. The end result Index: clang/lib/CodeGen/CodeGenModule.h =================================================================== --- clang/lib/CodeGen/CodeGenModule.h +++ clang/lib/CodeGen/CodeGenModule.h @@ -546,6 +546,7 @@ MetadataTypeMap MetadataIdMap; MetadataTypeMap VirtualMetadataIdMap; MetadataTypeMap GeneralizedMetadataIdMap; + MetadataTypeMap VCallOffsetMetadataIdMap; public: CodeGenModule(ASTContext &C, const HeaderSearchOptions &headersearchopts, @@ -1296,6 +1297,10 @@ /// unnamed MDNode (for internal identifiers). llvm::Metadata *CreateMetadataIdentifierGeneralized(QualType T); + /// Create a metadata identifier that is intended to be used to group + /// T's vtables containing vcall offsets together for vtable interleaving. + llvm::Metadata *CreateMetadataIdentifierForVCallOffsetType(QualType T); + /// Create and attach type metadata to the given function. void CreateFunctionTypeMetadataForIcall(const FunctionDecl *FD, llvm::Function *F); @@ -1305,7 +1310,9 @@ /// Create and attach type metadata for the given vtable. void AddVTableTypeMetadata(llvm::GlobalVariable *VTable, CharUnits Offset, - const CXXRecordDecl *RD); + const CXXRecordDecl *RD, + bool IsVCallOffsetType = false, + unsigned Level = 0); /// Return a vector of most-base classes for RD. This is used to implement /// control flow integrity checks for member function pointers. Index: clang/lib/CodeGen/CodeGenModule.cpp =================================================================== --- clang/lib/CodeGen/CodeGenModule.cpp +++ clang/lib/CodeGen/CodeGenModule.cpp @@ -1632,7 +1632,7 @@ llvm::Metadata *Id = CreateMetadataIdentifierForType(Context.getMemberPointerType( MD->getType(), Context.getRecordType(Base).getTypePtr())); - F->addTypeMetadata(0, Id); + F->addTypeMetadata(0, Id, 0); } } } @@ -1779,13 +1779,13 @@ return; llvm::Metadata *MD = CreateMetadataIdentifierForType(FD->getType()); - F->addTypeMetadata(0, MD); - F->addTypeMetadata(0, CreateMetadataIdentifierGeneralized(FD->getType())); + F->addTypeMetadata(0, MD, 0); + F->addTypeMetadata(0, CreateMetadataIdentifierGeneralized(FD->getType()), 0); // Emit a hash-based bit set entry for cross-DSO calls. if (CodeGenOpts.SanitizeCfiCrossDso) if (auto CrossDsoTypeId = CreateCrossDsoCfiTypeId(MD)) - F->addTypeMetadata(0, llvm::ConstantAsMetadata::get(CrossDsoTypeId)); + F->addTypeMetadata(0, llvm::ConstantAsMetadata::get(CrossDsoTypeId), 0); } void CodeGenModule::SetFunctionAttributes(GlobalDecl GD, llvm::Function *F, @@ -5767,6 +5767,12 @@ GeneralizedMetadataIdMap, ".generalized"); } +llvm::Metadata * +CodeGenModule::CreateMetadataIdentifierForVCallOffsetType(QualType T) { + return CreateMetadataIdentifierImpl(T, VCallOffsetMetadataIdMap, + ".vcall_offset"); +} + /// Returns whether this module needs the "all-vtables" type identifier. bool CodeGenModule::NeedAllVtablesTypeId() const { // Returns true if at least one of vtable-based CFI checkers is enabled and @@ -5783,19 +5789,29 @@ void CodeGenModule::AddVTableTypeMetadata(llvm::GlobalVariable *VTable, CharUnits Offset, - const CXXRecordDecl *RD) { + const CXXRecordDecl *RD, + bool IsVCallOffsetType, + unsigned Level) { + + // We only create type metadata with the level field if Level > 0, + // which indicates that vtable interleaving is enabled. llvm::Metadata *MD = - CreateMetadataIdentifierForType(QualType(RD->getTypeForDecl(), 0)); - VTable->addTypeMetadata(Offset.getQuantity(), MD); + IsVCallOffsetType + ? CreateMetadataIdentifierForVCallOffsetType( + QualType(RD->getTypeForDecl(), 0)) + : CreateMetadataIdentifierForType(QualType(RD->getTypeForDecl(), 0)); + VTable->addTypeMetadata(Offset.getQuantity(), MD, Level); if (CodeGenOpts.SanitizeCfiCrossDso) if (auto CrossDsoTypeId = CreateCrossDsoCfiTypeId(MD)) VTable->addTypeMetadata(Offset.getQuantity(), - llvm::ConstantAsMetadata::get(CrossDsoTypeId)); + llvm::ConstantAsMetadata::get(CrossDsoTypeId), + Level); + // all-vtables's level is always 1, which is the minimum value for level. if (NeedAllVtablesTypeId()) { llvm::Metadata *MD = llvm::MDString::get(getLLVMContext(), "all-vtables"); - VTable->addTypeMetadata(Offset.getQuantity(), MD); + VTable->addTypeMetadata(Offset.getQuantity(), MD, Level > 0 ? 1 : 0); } } Index: clang/lib/CodeGen/ItaniumCXXABI.cpp =================================================================== --- clang/lib/CodeGen/ItaniumCXXABI.cpp +++ clang/lib/CodeGen/ItaniumCXXABI.cpp @@ -24,14 +24,15 @@ #include "CodeGenFunction.h" #include "CodeGenModule.h" #include "TargetInfo.h" -#include "clang/CodeGen/ConstantInitBuilder.h" #include "clang/AST/Mangle.h" -#include "clang/AST/Type.h" #include "clang/AST/StmtCXX.h" +#include "clang/AST/Type.h" +#include "clang/CodeGen/ConstantInitBuilder.h" #include "llvm/IR/DataLayout.h" #include "llvm/IR/GlobalValue.h" #include "llvm/IR/Instructions.h" #include "llvm/IR/Intrinsics.h" +#include "llvm/IR/Metadata.h" #include "llvm/IR/Value.h" #include "llvm/Support/ScopedPrinter.h" @@ -42,6 +43,10 @@ class ItaniumCXXABI : public CodeGen::CGCXXABI { /// VTables - All the vtables which have been defined. llvm::DenseMap VTables; + /// Map each entry group to the corresponding offset global variable. + /// This map is needed for vtable interleaving. + llvm::DenseMap, llvm::GlobalVariable *> + OffsetMap; /// All the thread wrapper functions that have been used. llvm::SmallVector, 8> @@ -57,12 +62,10 @@ } public: - ItaniumCXXABI(CodeGen::CodeGenModule &CGM, - bool UseARMMethodPtrABI = false, - bool UseARMGuardVarABI = false) : - CGCXXABI(CGM), UseARMMethodPtrABI(UseARMMethodPtrABI), - UseARMGuardVarABI(UseARMGuardVarABI), - Use32BitVTableOffsetABI(false) { } + ItaniumCXXABI(CodeGen::CodeGenModule &CGM, bool UseARMMethodPtrABI = false, + bool UseARMGuardVarABI = false) + : CGCXXABI(CGM), UseARMMethodPtrABI(UseARMMethodPtrABI), + UseARMGuardVarABI(UseARMGuardVarABI), Use32BitVTableOffsetABI(false) {} bool classifyReturnType(CGFunctionInfo &FI) const override; @@ -285,6 +288,16 @@ bool exportThunk() override { return true; } + // VirtuallyAdjustedType is used for vtable interleaving. It is the type + // of the vtable in which the vcall offset entry (if it is a this adjustment) + // or the virtual base entry (if it is a return adjustment) is defined. For a + // return adjustment, VirtuallyAdjustedType is the derived type; For a this + // adjustment, VirtuallyAdjustedType is the virtual base type. + llvm::Value *performTypeAdjustment(CodeGenFunction &CGF, Address InitialPtr, + int64_t NonVirtualAdjustment, + const CXXRecordDecl *VirtuallyAdjustedType, + int64_t VirtualAdjustment, + bool IsReturnAdjustment); llvm::Value *performThisAdjustment(CodeGenFunction &CGF, Address This, const ThisAdjustment &TA) override; @@ -367,6 +380,16 @@ bool NeedsVTTParameter(GlobalDecl GD) override; + bool shouldInterleaveVTables(const CXXRecordDecl *RD); + llvm::Constant *getOffsetConstant(const CXXRecordDecl *RD, int64_t Offset, + llvm::Type *OffsetType, + bool AccessVCallOffset = false); + llvm::Value *getVTableEntryPointer(CodeGenFunction &CGF, + const CXXRecordDecl *RD, + bool AccessVCallOffset, + llvm::Value *VTable, int64_t Offset, + const llvm::Twine &Name = ""); + /**************************** RTTI Uniqueness ******************************/ protected: @@ -536,7 +559,8 @@ // pointers. return new ItaniumCXXABI(CGM, /*UseARMMethodPtrABI=*/true); } - return new ItaniumCXXABI(CGM); + return new ItaniumCXXABI(CGM, /* UseARMMethodPtrABI = */ false, + /* UseARMGuardVarABI = */ false); case TargetCXXABI::Microsoft: llvm_unreachable("Microsoft ABI is not Itanium-based"); @@ -651,7 +675,8 @@ llvm::Constant *CheckSourceLocation; llvm::Constant *CheckTypeDesc; bool ShouldEmitCFICheck = CGF.SanOpts.has(SanitizerKind::CFIMFCall) && - CGM.HasHiddenLTOVisibility(RD); + CGM.HasHiddenLTOVisibility(RD) && + !CGM.getCodeGenOpts().VTableInterleaving; if (ShouldEmitCFICheck) { CodeGenFunction::SanitizerScope SanScope(&CGF); @@ -921,6 +946,86 @@ return BuildMemberPointer(MD, CharUnits::Zero()); } +/// We should only interleave vtables when the type has cfi-vcall enabled, has +/// the hidden LTO visibility, and VTableInterleaving is set. +bool ItaniumCXXABI::shouldInterleaveVTables(const CXXRecordDecl *RD) { + // For now vtable interleaving relies on cfi-vcall. + SanitizerMask M = SanitizerKind::CFIVCall; + std::string TypeName = RD->getQualifiedNameAsString(); + if (CGM.getContext().getSanitizerBlacklist().isBlacklistedType(M, TypeName)) + return false; + + return CGM.HasHiddenLTOVisibility(RD) && + CGM.getCodeGenOpts().VTableInterleaving; +} + +/// When interleaving is disabled, this function simply returns +/// a constant of the offset. Otherwise, it create or returns the +/// offset placeholder of the specified type for the entry group identified +/// by the pair (TypeId, Offset). An entry group is a set of vtable entries +/// that must have the same offset from their corresponding address points. +/// To interleave vtables, we replace offsets into vtables with offset +/// placeholders, which will be replaced with real offsets once the interleaved +/// layout is decided. +llvm::Constant *ItaniumCXXABI::getOffsetConstant(const CXXRecordDecl *RD, + int64_t Offset, + llvm::Type *OffsetType, + bool IsVCallOffset) { + if (!shouldInterleaveVTables(RD)) + return llvm::ConstantInt::get(OffsetType, Offset, true); + + // AccessVCallOffset indicates whether we are getting the offset to a vcall + // offset or not. If it is true, we need to use the type id of the added vcall + // offset subtype of RD whose vtables are ones containing vcall offsets of RD. + llvm::Metadata *TypeId = IsVCallOffset + ? CGM.CreateMetadataIdentifierForVCallOffsetType( + QualType(RD->getTypeForDecl(), 0)) + : CGM.CreateMetadataIdentifierForType( + QualType(RD->getTypeForDecl(), 0)); + std::pair P = std::make_pair(TypeId, Offset); + + // The offset global variable for the entry group. + llvm::GlobalVariable *OffsetGV; + if (OffsetMap.find(P) != OffsetMap.end()) + OffsetGV = OffsetMap[P]; + else { + // If we haven't seen this offset, create an offset global variable for it. + llvm::Type *GVType = llvm::ArrayType::get(CGM.Int32Ty, 0); + // The name of a offset global variable has the format "__[type + // id]$[offset]". + std::string Name = + "__" + RD->getNameAsString() + "$" + std::to_string(Offset); + OffsetGV = new llvm::GlobalVariable( + CGM.getModule(), GVType, + /*Constant=*/true, llvm::GlobalValue::InternalLinkage, + llvm::Constant::getNullValue(GVType), Name); + + // Add offset.type metadata to the newly created offset global variable. + OffsetGV->addMetadata( + llvm::LLVMContext::MD_offset_type, + *llvm::MDTuple::get( + CGM.getLLVMContext(), + {TypeId, llvm::ConstantAsMetadata::get( + llvm::ConstantInt::get(CGM.Int64Ty, Offset))})); + + OffsetMap[P] = OffsetGV; + } + return llvm::ConstantExpr::getPtrToInt(OffsetGV, OffsetType); +} + +/// Return a pointer to the specified vtable entry. When vtable interleaving +/// is enabled, the calculation of this pointer is based on a vtable offset +/// placeholder, which will be replaced when the interleaved layout is decided. +llvm::Value *ItaniumCXXABI::getVTableEntryPointer( + CodeGenFunction &CGF, const CXXRecordDecl *RD, bool AccessVCallOffset, + llvm::Value *VTablePtr, int64_t Offset, const llvm::Twine &Name) { + llvm::Constant *OffsetConstant = + getOffsetConstant(RD, Offset, CGM.Int64Ty, AccessVCallOffset); + // Use the placeholder to calculate the pointer to the entry. + return CGF.Builder.CreateInBoundsGEP(nullptr, VTablePtr, OffsetConstant, + Name); +} + llvm::Constant *ItaniumCXXABI::BuildMemberPointer(const CXXMethodDecl *MD, CharUnits ThisAdjustment) { assert(MD->isInstance() && "Member function must not be static!"); @@ -936,6 +1041,7 @@ CharUnits PointerWidth = Context.toCharUnitsFromBits(Context.getTargetInfo().getPointerWidth(0)); uint64_t VTableOffset = (Index * PointerWidth.getQuantity()); + const CXXRecordDecl *RD = MD->getParent(); if (UseARMMethodPtrABI) { // ARM C++ ABI 3.2.1: @@ -944,7 +1050,7 @@ // least significant bit of adj then makes exactly the same // discrimination as the least significant bit of ptr does for // Itanium. - MemPtr[0] = llvm::ConstantInt::get(CGM.PtrDiffTy, VTableOffset); + MemPtr[0] = getOffsetConstant(RD, (int64_t)VTableOffset, CGM.PtrDiffTy); MemPtr[1] = llvm::ConstantInt::get(CGM.PtrDiffTy, 2 * ThisAdjustment.getQuantity() + 1); } else { @@ -952,7 +1058,13 @@ // For a virtual function, [the pointer field] is 1 plus the // virtual table offset (in bytes) of the function, // represented as a ptrdiff_t. - MemPtr[0] = llvm::ConstantInt::get(CGM.PtrDiffTy, VTableOffset + 1); + if (shouldInterleaveVTables(RD)) { + llvm::Constant *Placeholder = + getOffsetConstant(RD, (int64_t)VTableOffset, CGM.PtrDiffTy); + llvm::Constant *ConstantOne = llvm::ConstantInt::get(CGM.PtrDiffTy, 1); + MemPtr[0] = llvm::ConstantExpr::getAdd(Placeholder, ConstantOne); + } else + MemPtr[0] = llvm::ConstantInt::get(CGM.PtrDiffTy, VTableOffset + 1); MemPtr[1] = llvm::ConstantInt::get(CGM.PtrDiffTy, ThisAdjustment.getQuantity()); } @@ -1449,9 +1561,9 @@ CGM.getItaniumVTableContext().getVirtualBaseOffsetOffset(ClassDecl, BaseClassDecl); - llvm::Value *VBaseOffsetPtr = - CGF.Builder.CreateConstGEP1_64(VTablePtr, VBaseOffsetOffset.getQuantity(), - "vbase.offset.ptr"); + llvm::Value *VBaseOffsetPtr = getVTableEntryPointer( + CGF, ClassDecl, false, VTablePtr, VBaseOffsetOffset.getQuantity(), + "vbase.offset.ptr"); VBaseOffsetPtr = CGF.Builder.CreateBitCast(VBaseOffsetPtr, CGM.PtrDiffTy->getPointerTo()); @@ -1672,8 +1784,13 @@ llvm::ConstantInt::get(CGM.Int32Ty, AddressPoint.AddressPointIndex), }; + // The address point calculated by vtable_start + offset where vtable_start is + // the base address of the gep. When interleaving is enabled, we replace + // vtable_start with the new address point in the interleaved layout minus + // offset, which may not be an in bound address of the object. + bool InBounds = shouldInterleaveVTables(VTableClass) ? false : true; return llvm::ConstantExpr::getGetElementPtr(VTable->getValueType(), VTable, - Indices, /*InBounds=*/true, + Indices, /*InBounds=*/InBounds, /*InRangeIndex=*/1); } @@ -1740,21 +1857,25 @@ Address This, llvm::Type *Ty, SourceLocation Loc) { - Ty = Ty->getPointerTo()->getPointerTo(); auto *MethodDecl = cast(GD.getDecl()); - llvm::Value *VTable = CGF.GetVTablePtr(This, Ty, MethodDecl->getParent()); + auto *RecordDecl = MethodDecl->getParent(); + llvm::Value *VTable = CGF.GetVTablePtr(This, CGM.Int8PtrTy, RecordDecl); uint64_t VTableIndex = CGM.getItaniumVTableContext().getMethodVTableIndex(GD); + int64_t ByteOffset = + VTableIndex * CGM.getContext().getTargetInfo().getPointerWidth(0) / 8; + llvm::Value *OffsetConstant = + getOffsetConstant(RecordDecl, ByteOffset, CGM.Int32Ty); llvm::Value *VFunc; - if (CGF.ShouldEmitVTableTypeCheckedLoad(MethodDecl->getParent())) { - VFunc = CGF.EmitVTableTypeCheckedLoad( - MethodDecl->getParent(), VTable, - VTableIndex * CGM.getContext().getTargetInfo().getPointerWidth(0) / 8); - } else { - CGF.EmitTypeMetadataCodeForVCall(MethodDecl->getParent(), VTable, Loc); - + if (CGF.ShouldEmitVTableTypeCheckedLoad(RecordDecl)) + VFunc = CGF.EmitVTableTypeCheckedLoad(RecordDecl, VTable, + Ty->getPointerTo(), OffsetConstant); + else { + CGF.EmitTypeMetadataCodeForVCall(RecordDecl, VTable, Loc); llvm::Value *VFuncPtr = - CGF.Builder.CreateConstInBoundsGEP1_64(VTable, VTableIndex, "vfn"); + CGF.Builder.CreateInBoundsGEP(nullptr, VTable, OffsetConstant, "vfn"); + VFuncPtr = + CGF.Builder.CreateBitCast(VFuncPtr, Ty->getPointerTo()->getPointerTo()); auto *VFuncLoad = CGF.Builder.CreateAlignedLoad(VFuncPtr, CGF.getPointerAlign()); @@ -1867,11 +1988,11 @@ return true; } -static llvm::Value *performTypeAdjustment(CodeGenFunction &CGF, - Address InitialPtr, - int64_t NonVirtualAdjustment, - int64_t VirtualAdjustment, - bool IsReturnAdjustment) { + +llvm::Value *ItaniumCXXABI::performTypeAdjustment( + CodeGenFunction &CGF, Address InitialPtr, int64_t NonVirtualAdjustment, + const CXXRecordDecl *VirtuallyAdjustedType, int64_t VirtualAdjustment, + bool IsReturnAdjustment) { if (!NonVirtualAdjustment && !VirtualAdjustment) return InitialPtr.getPointer(); @@ -1886,15 +2007,19 @@ // Perform the virtual adjustment if we have one. llvm::Value *ResultPtr; if (VirtualAdjustment) { + assert(VirtuallyAdjustedType != nullptr && + "The type being virtually adjusted is not available"); + llvm::Type *PtrDiffTy = CGF.ConvertType(CGF.getContext().getPointerDiffType()); Address VTablePtrPtr = CGF.Builder.CreateElementBitCast(V, CGF.Int8PtrTy); llvm::Value *VTablePtr = CGF.Builder.CreateLoad(VTablePtrPtr); - + // If there is virtual adjustment and this is a this adjustment, we know + // that we read the vcall offset from the vtable. llvm::Value *OffsetPtr = - CGF.Builder.CreateConstInBoundsGEP1_64(VTablePtr, VirtualAdjustment); - + getVTableEntryPointer(CGF, VirtuallyAdjustedType, !IsReturnAdjustment, + VTablePtr, VirtualAdjustment); OffsetPtr = CGF.Builder.CreateBitCast(OffsetPtr, PtrDiffTy->getPointerTo()); // Load the adjustment offset from the vtable. @@ -1922,6 +2047,7 @@ Address This, const ThisAdjustment &TA) { return performTypeAdjustment(CGF, This, TA.NonVirtual, + TA.VirtuallyAdjustedType, TA.Virtual.Itanium.VCallOffsetOffset, /*IsReturnAdjustment=*/false); } @@ -1930,6 +2056,7 @@ ItaniumCXXABI::performReturnAdjustment(CodeGenFunction &CGF, Address Ret, const ReturnAdjustment &RA) { return performTypeAdjustment(CGF, Ret, RA.NonVirtual, + RA.VirtuallyAdjustedType, RA.Virtual.Itanium.VBaseOffsetOffset, /*IsReturnAdjustment=*/true); } @@ -2720,7 +2847,7 @@ class ItaniumRTTIBuilder { CodeGenModule &CGM; // Per-module state. llvm::LLVMContext &VMContext; - const ItaniumCXXABI &CXXABI; // Per-module state. + ItaniumCXXABI &CXXABI; // Per-module state. /// Fields - The fields of the RTTI descriptor currently being built. SmallVector Fields; @@ -2740,6 +2867,13 @@ /// inheritance, according to the Itanium C++ ABI, 2.9.5p6b. void BuildSIClassTypeInfo(const CXXRecordDecl *RD); + /// GetOffsetFlag - Build a constant representing __offset_flags. + /// When vtable interleaving is enabled and Offset is an virtual base offset, + /// this constant will be constructed from the corresponding offset + /// placeholder. + llvm::Constant *GetOffsetFlag(const CXXRecordDecl *RD, int64_t Offset, + uint64_t LowerBits, llvm::Type *OffsetFlagsTy); + /// BuildVMIClassTypeInfo - Build an abi::__vmi_class_type_info, used for /// classes with bases that do not satisfy the abi::__si_class_type_info /// constraints, according ti the Itanium C++ ABI, 2.9.5p5c. @@ -2758,7 +2892,7 @@ void BuildPointerToMemberTypeInfo(const MemberPointerType *Ty); public: - ItaniumRTTIBuilder(const ItaniumCXXABI &ABI) + ItaniumRTTIBuilder(ItaniumCXXABI &ABI) : CGM(ABI.CGM), VMContext(CGM.getModule().getContext()), CXXABI(ABI) {} // Pointer type info flags. @@ -3119,6 +3253,11 @@ static const char * const VMIClassTypeInfo = "_ZTVN10__cxxabiv121__vmi_class_type_infoE"; + llvm::Type *VTableTy1 = + llvm::StructType::get(llvm::ArrayType::get(CGM.Int8PtrTy, 7)); + llvm::Type *VTableTy2 = + llvm::StructType::get(llvm::ArrayType::get(CGM.Int8PtrTy, 10)); + llvm::Type *CurVTableTy = nullptr; const char *VTableName = nullptr; switch (Ty->getTypeClass()) { @@ -3151,6 +3290,7 @@ case Type::BlockPointer: // abi::__fundamental_type_info. VTableName = "_ZTVN10__cxxabiv123__fundamental_type_infoE"; + CurVTableTy = VTableTy1; break; case Type::ConstantArray: @@ -3158,17 +3298,20 @@ case Type::VariableArray: // abi::__array_type_info. VTableName = "_ZTVN10__cxxabiv117__array_type_infoE"; + CurVTableTy = VTableTy1; break; case Type::FunctionNoProto: case Type::FunctionProto: // abi::__function_type_info. VTableName = "_ZTVN10__cxxabiv120__function_type_infoE"; + CurVTableTy = VTableTy1; break; case Type::Enum: // abi::__enum_type_info. VTableName = "_ZTVN10__cxxabiv116__enum_type_infoE"; + CurVTableTy = VTableTy1; break; case Type::Record: { @@ -3182,7 +3325,7 @@ } else { VTableName = VMIClassTypeInfo; } - + CurVTableTy = VTableTy2; break; } @@ -3193,6 +3336,7 @@ // Handle id and Class. if (isa(Ty)) { VTableName = ClassTypeInfo; + CurVTableTy = VTableTy2; break; } @@ -3205,33 +3349,52 @@ } else { VTableName = ClassTypeInfo; } + CurVTableTy = VTableTy2; break; case Type::ObjCObjectPointer: case Type::Pointer: // abi::__pointer_type_info. VTableName = "_ZTVN10__cxxabiv119__pointer_type_infoE"; + CurVTableTy = VTableTy1; break; case Type::MemberPointer: // abi::__pointer_to_member_type_info. VTableName = "_ZTVN10__cxxabiv129__pointer_to_member_type_infoE"; + CurVTableTy = VTableTy1; break; } - llvm::Constant *VTable = - CGM.getModule().getOrInsertGlobal(VTableName, CGM.Int8PtrTy); - CGM.setDSOLocal(cast(VTable->stripPointerCasts())); - - llvm::Type *PtrDiffTy = - CGM.getTypes().ConvertType(CGM.getContext().getPointerDiffType()); + llvm::Constant *VTable; + // FIXME When interleaving is enabled, we assume that + // libc++abi is used and we use vtable types for typeinfo classes + // defined in libc++abi. + if (CGM.getCodeGenOpts().VTableInterleaving) { + VTable = CGM.getModule().getOrInsertGlobal(VTableName, CurVTableTy); + CGM.setDSOLocal(cast(VTable->stripPointerCasts())); + llvm::Value *Indices[] = { + llvm::ConstantInt::get(CGM.Int32Ty, 0), + llvm::ConstantInt::get(CGM.Int32Ty, 0), + llvm::ConstantInt::get(CGM.Int32Ty, 2), + }; + VTable = llvm::ConstantExpr::getGetElementPtr(CurVTableTy, VTable, Indices, + /*InBounds=*/true, + /*InRangeIndex=*/1); + VTable = llvm::ConstantExpr::getBitCast(VTable, CGM.Int8PtrTy); + } else { + VTable = CGM.getModule().getOrInsertGlobal(VTableName, CGM.Int8PtrTy); + CGM.setDSOLocal(cast(VTable->stripPointerCasts())); - // The vtable address point is 2. - llvm::Constant *Two = llvm::ConstantInt::get(PtrDiffTy, 2); - VTable = - llvm::ConstantExpr::getInBoundsGetElementPtr(CGM.Int8PtrTy, VTable, Two); - VTable = llvm::ConstantExpr::getBitCast(VTable, CGM.Int8PtrTy); + llvm::Type *PtrDiffTy = + CGM.getTypes().ConvertType(CGM.getContext().getPointerDiffType()); + // The vtable address point is 2. + llvm::Constant *Two = llvm::ConstantInt::get(PtrDiffTy, 2); + VTable = llvm::ConstantExpr::getInBoundsGetElementPtr(CGM.Int8PtrTy, VTable, + Two); + VTable = llvm::ConstantExpr::getBitCast(VTable, CGM.Int8PtrTy); + } Fields.push_back(VTable); } @@ -3606,6 +3769,28 @@ return Flags; } +llvm::Constant *ItaniumRTTIBuilder::GetOffsetFlag(const CXXRecordDecl *RD, + int64_t Offset, + uint64_t LowerBits, + llvm::Type *OffsetFlagsTy) { + // If interleaving is not enabled or Offset is not a virtual base offset, + // we don't need to create a offset placeholder for vtable interleaving. + if (!CXXABI.shouldInterleaveVTables(RD) || !(LowerBits & BCTI_Virtual)) { + uint64_t UpperBits = (uint64_t)Offset << 8; + uint64_t OffsetFlags = UpperBits | LowerBits; + return llvm::ConstantInt::get(OffsetFlagsTy, OffsetFlags); + } + + llvm::Constant *OffsetConstant = + CXXABI.getOffsetConstant(RD, Offset, OffsetFlagsTy); + llvm::Constant *ShiftAmount = llvm::ConstantInt::get(OffsetFlagsTy, 8); + llvm::Constant *UpperBitsConstant = + llvm::ConstantExpr::getNSWShl(OffsetConstant, ShiftAmount); + llvm::Constant *LowerBitsConstant = + llvm::ConstantInt::get(OffsetFlagsTy, LowerBits); + return llvm::ConstantExpr::getOr(UpperBitsConstant, LowerBitsConstant); +} + /// BuildVMIClassTypeInfo - Build an abi::__vmi_class_type_info, used for /// classes with bases that do not satisfy the abi::__si_class_type_info /// constraints, according ti the Itanium C++ ABI, 2.9.5p5c. @@ -3665,8 +3850,6 @@ const CXXRecordDecl *BaseDecl = cast(Base.getType()->getAs()->getDecl()); - int64_t OffsetFlags = 0; - // All but the lower 8 bits of __offset_flags are a signed offset. // For a non-virtual base, this is the offset in the object of the base // subobject. For a virtual base, this is the offset in the virtual table of @@ -3680,16 +3863,17 @@ Offset = Layout.getBaseClassOffset(BaseDecl); }; - OffsetFlags = uint64_t(Offset.getQuantity()) << 8; + int64_t OffsetInt = Offset.getQuantity(); + uint64_t Flags = 0; // The low-order byte of __offset_flags contains flags, as given by the // masks from the enumeration __offset_flags_masks. if (Base.isVirtual()) - OffsetFlags |= BCTI_Virtual; + Flags |= BCTI_Virtual; if (Base.getAccessSpecifier() == AS_public) - OffsetFlags |= BCTI_Public; + Flags |= BCTI_Public; - Fields.push_back(llvm::ConstantInt::get(OffsetFlagsLTy, OffsetFlags)); + Fields.push_back(GetOffsetFlag(RD, OffsetInt, Flags, OffsetFlagsLTy)); } } Index: clang/lib/CodeGen/MicrosoftCXXABI.cpp =================================================================== --- clang/lib/CodeGen/MicrosoftCXXABI.cpp +++ clang/lib/CodeGen/MicrosoftCXXABI.cpp @@ -1859,13 +1859,14 @@ llvm::Type *Ty, SourceLocation Loc) { CGBuilderTy &Builder = CGF.Builder; - - Ty = Ty->getPointerTo()->getPointerTo(); + llvm::Type *VTablePtrTy = Ty->getPointerTo()->getPointerTo(); + llvm::Type *FuncPtrTy = Ty->getPointerTo(); Address VPtr = adjustThisArgumentForVirtualFunctionCall(CGF, GD, This, true); auto *MethodDecl = cast(GD.getDecl()); - llvm::Value *VTable = CGF.GetVTablePtr(VPtr, Ty, MethodDecl->getParent()); + llvm::Value *VTable = + CGF.GetVTablePtr(VPtr, VTablePtrTy, MethodDecl->getParent()); MicrosoftVTableContext &VFTContext = CGM.getMicrosoftVTableContext(); MethodVFTableLocation ML = VFTContext.getMethodVFTableLocation(GD); @@ -1884,9 +1885,13 @@ llvm::Value *VFunc; if (CGF.ShouldEmitVTableTypeCheckedLoad(MethodDecl->getParent())) { - VFunc = CGF.EmitVTableTypeCheckedLoad( - getObjectWithVPtr(), VTable, - ML.Index * CGM.getContext().getTargetInfo().getPointerWidth(0) / 8); + int64_t ByteOffset = + ML.Index * CGM.getContext().getTargetInfo().getPointerWidth(0) / 8; + llvm::Value *OffsetConstant = + llvm::ConstantInt::get(CGM.Int32Ty, ByteOffset); + llvm::Value *CastedVTable = Builder.CreateBitCast(VTable, CGM.Int8PtrTy); + VFunc = CGF.EmitVTableTypeCheckedLoad(getObjectWithVPtr(), CastedVTable, + FuncPtrTy, OffsetConstant); } else { if (CGM.getCodeGenOpts().PrepareForLTO) CGF.EmitTypeMetadataCodeForVCall(getObjectWithVPtr(), VTable, Loc); Index: clang/lib/Frontend/CompilerInvocation.cpp =================================================================== --- clang/lib/Frontend/CompilerInvocation.cpp +++ clang/lib/Frontend/CompilerInvocation.cpp @@ -1403,6 +1403,8 @@ Opts.SymbolPartition = Args.getLastArgValue(OPT_fsymbol_partition_EQ); + Opts.VTableInterleaving = Args.hasArg(OPT_vtable_interleaving); + return Success; } Index: clang/test/CodeGen/tbaa-for-vptr.cpp =================================================================== --- clang/test/CodeGen/tbaa-for-vptr.cpp +++ clang/test/CodeGen/tbaa-for-vptr.cpp @@ -1,10 +1,10 @@ -// RUN: %clang_cc1 -triple %itanium_abi_triple -emit-llvm -o - -fsanitize=thread %s | FileCheck %s --check-prefixes=CHECK,OLD-PATH -// RUN: %clang_cc1 -triple %itanium_abi_triple -emit-llvm -o - -O1 -fno-experimental-new-pass-manager %s | FileCheck %s --check-prefixes=CHECK,OLD-PATH -// RUN: %clang_cc1 -triple %itanium_abi_triple -emit-llvm -o - -O1 -fno-experimental-new-pass-manager -relaxed-aliasing -fsanitize=thread %s | FileCheck %s --check-prefixes=CHECK,OLD-PATH +// RUN: %clang_cc1 -triple %itanium_abi_triple -emit-llvm -o - -fsanitize=thread %s | FileCheck %s --check-prefixes=CHECK,OLD-PATH,UNOPTIMIZED +// RUN: %clang_cc1 -triple %itanium_abi_triple -emit-llvm -o - -O1 -fno-experimental-new-pass-manager %s | FileCheck %s --check-prefixes=CHECK,OLD-PATH,OPTIMIZED +// RUN: %clang_cc1 -triple %itanium_abi_triple -emit-llvm -o - -O1 -fno-experimental-new-pass-manager -relaxed-aliasing -fsanitize=thread %s | FileCheck %s --check-prefixes=CHECK,OLD-PATH,OPTIMIZED // -// RUN: %clang_cc1 -triple %itanium_abi_triple -emit-llvm -new-struct-path-tbaa -o - -fsanitize=thread %s | FileCheck %s --check-prefixes=CHECK,NEW-PATH -// RUN: %clang_cc1 -triple %itanium_abi_triple -emit-llvm -new-struct-path-tbaa -o - -O1 -fno-experimental-new-pass-manager %s | FileCheck %s --check-prefixes=CHECK,NEW-PATH -// RUN: %clang_cc1 -triple %itanium_abi_triple -emit-llvm -new-struct-path-tbaa -o - -O1 -fno-experimental-new-pass-manager -relaxed-aliasing -fsanitize=thread %s | FileCheck %s --check-prefixes=CHECK,NEW-PATH +// RUN: %clang_cc1 -triple %itanium_abi_triple -emit-llvm -new-struct-path-tbaa -o - -fsanitize=thread %s | FileCheck %s --check-prefixes=CHECK,NEW-PATH,UNOPTIMIZED +// RUN: %clang_cc1 -triple %itanium_abi_triple -emit-llvm -new-struct-path-tbaa -o - -O1 -fno-experimental-new-pass-manager %s | FileCheck %s --check-prefixes=CHECK,NEW-PATH,OPTIMIZED +// RUN: %clang_cc1 -triple %itanium_abi_triple -emit-llvm -new-struct-path-tbaa -o - -O1 -fno-experimental-new-pass-manager -relaxed-aliasing -fsanitize=thread %s | FileCheck %s --check-prefixes=CHECK,NEW-PATH,OPTIMIZED // // RUN: %clang_cc1 -triple %itanium_abi_triple -emit-llvm -o - %s | FileCheck %s --check-prefix=NOTBAA // RUN: %clang_cc1 -triple %itanium_abi_triple -emit-llvm -o - -O2 -fno-experimental-new-pass-manager -relaxed-aliasing %s | FileCheck %s --check-prefix=NOTBAA @@ -27,7 +27,9 @@ } // CHECK-LABEL: @_Z7CallFoo -// CHECK: %{{.*}} = load i32 (%struct.A*)**, {{.*}} !tbaa ![[NUM:[0-9]+]] +// UNOPTIMIZED: [[R0:%[^ ]*]] = bitcast %struct.A* {{.*}} to i8** +// UNOPTIMIZED: %{{.*}} = load i8*, i8** [[R0]], {{.*}} !tbaa ![[NUM:[0-9]+]] +// OPTIMIZED: %{{.*}} = load i32 (%struct.A*)**, {{.*}} !tbaa ![[NUM:[0-9]+]] // CHECK: br i1 // CHECK: load i8*, {{.*}}, !tbaa ![[NUM]] // Index: clang/test/CodeGenCXX/alignment.cpp =================================================================== --- clang/test/CodeGenCXX/alignment.cpp +++ clang/test/CodeGenCXX/alignment.cpp @@ -226,7 +226,7 @@ // CHECK: [[B_P:%.*]] = load [[B:%.*]]*, [[B]]** // CHECK: [[VPTR_P:%.*]] = bitcast [[B]]* [[B_P]] to i8** // CHECK: [[VPTR:%.*]] = load i8*, i8** [[VPTR_P]], align 8 - // CHECK: [[T0:%.*]] = getelementptr i8, i8* [[VPTR]], i64 -24 + // CHECK: [[T0:%.*]] = getelementptr inbounds i8, i8* [[VPTR]], i64 -24 // CHECK: [[OFFSET_P:%.*]] = bitcast i8* [[T0]] to i64* // CHECK: [[OFFSET:%.*]] = load i64, i64* [[OFFSET_P]], align 8 // CHECK: [[T0:%.*]] = bitcast [[B]]* [[B_P]] to i8* @@ -280,7 +280,7 @@ // CHECK: [[RESULT:%.*]] = alloca [[ARRAY]], align 64 // CHECK: [[VPTR_P:%.*]] = bitcast [[D]]* [[D_P]] to i8** // CHECK: [[VPTR:%.*]] = load i8*, i8** [[VPTR_P]], align 16 - // CHECK: [[T0:%.*]] = getelementptr i8, i8* [[VPTR]], i64 -24 + // CHECK: [[T0:%.*]] = getelementptr inbounds i8, i8* [[VPTR]], i64 -24 // CHECK: [[OFFSET_P:%.*]] = bitcast i8* [[T0]] to i64* // CHECK: [[OFFSET:%.*]] = load i64, i64* [[OFFSET_P]], align 8 // CHECK: [[T0:%.*]] = bitcast [[D]]* [[D_P]] to i8* Index: clang/test/CodeGenCXX/arm.cpp =================================================================== --- clang/test/CodeGenCXX/arm.cpp +++ clang/test/CodeGenCXX/arm.cpp @@ -278,11 +278,12 @@ // CHECK-NEXT: [[V:%.*]] = load [[A]]*, [[A]]** [[AVAR]], align 4 // CHECK-NEXT: [[ISNULL:%.*]] = icmp eq [[A]]* [[V]], null // CHECK-NEXT: br i1 [[ISNULL]] - // CHECK: [[T0:%.*]] = bitcast [[A]]* [[V]] to void ([[A]]*)*** - // CHECK-NEXT: [[T1:%.*]] = load void ([[A]]*)**, void ([[A]]*)*** [[T0]] - // CHECK-NEXT: [[T2:%.*]] = getelementptr inbounds void ([[A]]*)*, void ([[A]]*)** [[T1]], i64 1 - // CHECK-NEXT: [[T3:%.*]] = load void ([[A]]*)*, void ([[A]]*)** [[T2]] - // CHECK-NEXT: call void [[T3]]([[A]]* [[V]]) + // CHECK: [[T0:%.*]] = bitcast [[A]]* [[V]] to i8** + // CHECK-NEXT: [[T1:%.*]] = load i8*, i8** [[T0]] + // CHECK-NEXT: [[T2:%.*]] = getelementptr inbounds i8, i8* [[T1]], i32 4 + // CHECK: [[T3:%.*]] = bitcast i8* [[T2]] to void ([[A]]*)** + // CHECK-NEXT: [[T4:%.*]] = load void ([[A]]*)*, void ([[A]]*)** [[T3]] + // CHECK-NEXT: call void [[T4]]([[A]]* [[V]]) // CHECK-NEXT: br label // CHECK: ret void delete a; Index: clang/test/CodeGenCXX/cfi-cross-dso.cpp =================================================================== --- clang/test/CodeGenCXX/cfi-cross-dso.cpp +++ clang/test/CodeGenCXX/cfi-cross-dso.cpp @@ -28,8 +28,12 @@ // MS: @[[B_VTABLE:.*]] = private unnamed_addr constant { [2 x i8*] } {{.*}}@"??_R4B@?A0x{{[^@]*}}@@6B@"{{.*}}@"?f@B@?A0x{{[^@]*}}@@UEAAXXZ" -// CHECK: %[[VT:.*]] = load void (%struct.A*)**, void (%struct.A*)*** -// CHECK: %[[VT2:.*]] = bitcast {{.*}}%[[VT]] to i8*, !nosanitize +// CHECK: %[[R0:.*]] = load %struct.A*, %struct.A** %a.addr +// ITANIUM: %[[R1:.*]] = bitcast %struct.A* %[[R0]] to i8** +// ITANIUM: %[[VT2:.*]] = load i8*, i8** %[[R1]] +// MS: %[[R1:.*]] = bitcast %struct.A* %[[R0]] to void (%struct.A*)*** +// MS: %[[VT:.*]] = load void (%struct.A*)**, void (%struct.A*)*** %[[R1]] +// MS: %[[VT2:.*]] = bitcast {{.*}}%[[VT]] to i8* // ITANIUM: %[[TEST:.*]] = call i1 @llvm.type.test(i8* %[[VT2]], metadata !"_ZTS1A"), !nosanitize // MS: %[[TEST:.*]] = call i1 @llvm.type.test(i8* %[[VT2]], metadata !"?AUA@@"), !nosanitize // CHECK: br i1 %[[TEST]], label %[[CONT:.*]], label %[[SLOW:.*]], {{.*}} !nosanitize Index: clang/test/CodeGenCXX/constructor-destructor-return-this.cpp =================================================================== --- clang/test/CodeGenCXX/constructor-destructor-return-this.cpp +++ clang/test/CodeGenCXX/constructor-destructor-return-this.cpp @@ -131,7 +131,8 @@ // Verify that virtual calls to destructors are not marked with a 'returned' // this parameter at the call site... -// CHECKARM: [[VFN:%.*]] = getelementptr inbounds %class.E* (%class.E*)*, %class.E* (%class.E*)** +// CHECKARM: [[VFPTR:%.*]] = getelementptr inbounds i8, i8* +// CHECKARM: [[VFN:%.*]] = bitcast i8* [[VFPTR]] to %class.E* (%class.E*)** // CHECKARM: [[THUNK:%.*]] = load %class.E* (%class.E*)*, %class.E* (%class.E*)** [[VFN]] // CHECKARM: call %class.E* [[THUNK]](%class.E* % Index: clang/test/CodeGenCXX/constructor-init.cpp =================================================================== --- clang/test/CodeGenCXX/constructor-init.cpp +++ clang/test/CodeGenCXX/constructor-init.cpp @@ -96,8 +96,9 @@ // CHECK-LABEL: define void @_ZN10InitVTable1BC2Ev(%"struct.InitVTable::B"* %this) unnamed_addr // CHECK: [[T0:%.*]] = bitcast [[B:%.*]]* [[THIS:%.*]] to i32 (...)*** // CHECK-NEXT: store i32 (...)** bitcast (i8** getelementptr inbounds ({ [3 x i8*] }, { [3 x i8*] }* @_ZTVN10InitVTable1BE, i32 0, inrange i32 0, i32 2) to i32 (...)**), i32 (...)*** [[T0]] - // CHECK: [[VTBL:%.*]] = load i32 ([[B]]*)**, i32 ([[B]]*)*** {{%.*}} - // CHECK-NEXT: [[FNP:%.*]] = getelementptr inbounds i32 ([[B]]*)*, i32 ([[B]]*)** [[VTBL]], i64 0 + // CHECK: [[VTBL:%.*]] = load i8*, i8** {{%.*}} + // CHECK-NEXT: [[FNBP:%.*]] = getelementptr inbounds i8, i8* [[VTBL]], i32 0 + // CHECK-NEXT: [[FNP:%.*]] = bitcast i8* [[FNBP]] to i32 ([[B]]*)** // CHECK-NEXT: [[FN:%.*]] = load i32 ([[B]]*)*, i32 ([[B]]*)** [[FNP]] // CHECK-NEXT: [[ARG:%.*]] = call i32 [[FN]]([[B]]* [[THIS]]) // CHECK-NEXT: call void @_ZN10InitVTable1AC2Ei({{.*}}* {{%.*}}, i32 [[ARG]]) Index: clang/test/CodeGenCXX/delete.cpp =================================================================== --- clang/test/CodeGenCXX/delete.cpp +++ clang/test/CodeGenCXX/delete.cpp @@ -123,10 +123,11 @@ // CHECK-NEXT: [[ALLOCATED:%.*]] = getelementptr inbounds i8, i8* [[T0]], i64 [[OFFSET]] // Load the complete-object destructor (not the deleting destructor) // and call it. - // CHECK-NEXT: [[T0:%.*]] = bitcast [[X:%.*]]* [[XP:%.*]] to void ([[X]]*)*** - // CHECK-NEXT: [[VTABLE:%.*]] = load void ([[X]]*)**, void ([[X]]*)*** [[T0]] - // CHECK-NEXT: [[T0:%.*]] = getelementptr inbounds void ([[X]]*)*, void ([[X]]*)** [[VTABLE]], i64 0 - // CHECK-NEXT: [[DTOR:%.*]] = load void ([[X]]*)*, void ([[X]]*)** [[T0]] + // CHECK-NEXT: [[T0:%.*]] = bitcast [[X:%.*]]* [[XP:%.*]] to i8** + // CHECK-NEXT: [[VTABLE:%.*]] = load i8*, i8** [[T0]] + // CHECK-NEXT: [[T0:%.*]] = getelementptr inbounds i8, i8* [[VTABLE]], i32 0 + // CHECK-NEXT: [[T1:%.*]] = bitcast i8* [[T0]] to void ([[X]]*)** + // CHECK-NEXT: [[DTOR:%.*]] = load void ([[X]]*)*, void ([[X]]*)** [[T1]] // CHECK-NEXT: call void [[DTOR]]([[X]]* [[OBJ:%.*]]) // Call the global operator delete. // CHECK-NEXT: call void @_ZdlPv(i8* [[ALLOCATED]]) [[NUW:#[0-9]+]] Index: clang/test/CodeGenCXX/devirtualize-virtual-function-calls-final.cpp =================================================================== --- clang/test/CodeGenCXX/devirtualize-virtual-function-calls-final.cpp +++ clang/test/CodeGenCXX/devirtualize-virtual-function-calls-final.cpp @@ -102,6 +102,7 @@ // FIXME: It should be possible to devirtualize this case, but that is // not implemented yet. // CHECK: getelementptr + // CHECK-NEXT: bitcast // CHECK-NEXT: %[[FUNC:.*]] = load // CHECK-NEXT: call void %[[FUNC]] static_cast(d)->f(); @@ -111,6 +112,7 @@ // FIXME: It should be possible to devirtualize this case, but that is // not implemented yet. // CHECK: getelementptr + // CHECK-NEXT: bitcast // CHECK-NEXT: %[[FUNC:.*]] = load // CHECK-NEXT: call i32 %[[FUNC]] -static_cast(*d); @@ -220,7 +222,8 @@ // CHECK: [[F_PTR_RA:%.+]] = bitcast // CHECK: [[VTABLE:%.+]] = load {{.+}} [[F_PTR_RA]] // CHECK: [[VFN:%.+]] = getelementptr inbounds {{.+}} [[VTABLE]], i{{[0-9]+}} 0 - // CHECK-NEXT: %[[FUNC:.*]] = load {{.+}} [[VFN]] + // CHECK-NEXT: [[VFP:%.*]] = bitcast {{.+}} [[VFN]] + // CHECK-NEXT: %[[FUNC:.*]] = load {{.+}} [[VFP]] // CHECK-NEXT: = call {{.*}} %[[FUNC]] return static_cast(x)->f(); } @@ -232,8 +235,9 @@ // CHECK: bitcast // CHECK: [[F_PTR_RA:%.+]] = bitcast // CHECK: [[VTABLE:%.+]] = load {{.+}} [[F_PTR_RA]] - // CHECK: [[VFN:%.+]] = getelementptr inbounds {{.+}} [[VTABLE]], i{{[0-9]+}} 1 - // CHECK-NEXT: %[[FUNC:.*]] = load {{.+}} [[VFN]] + // CHECK: [[VFN:%.+]] = getelementptr inbounds {{.+}} [[VTABLE]], i{{[0-9]+}} 4 + // CHECK-NEXT: [[VFP:%.*]] = bitcast {{.+}} [[VFN]] + // CHECK-NEXT: %[[FUNC:.*]] = load {{.+}} [[VFP]] // CHECK-NEXT: = call {{.*}} %[[FUNC]] return -static_cast(*x); } Index: clang/test/CodeGenCXX/interleaving-member-function-pointers.cpp =================================================================== --- /dev/null +++ clang/test/CodeGenCXX/interleaving-member-function-pointers.cpp @@ -0,0 +1,180 @@ +// This file tests if accesses to vbase entries are correctly generated when vtable interleaving is enabled. +// This test is based on member-function-pointers.cpp. + +// RUN: %clang_cc1 %s -flto -fvisibility hidden -fsanitize=cfi-vcall -vtable-interleaving -emit-llvm -o - -triple=x86_64-unknown-unknown | FileCheck -check-prefix CODE-LP64 %s +// RUN: %clang_cc1 %s -flto -fvisibility hidden -fsanitize=cfi-vcall -vtable-interleaving -emit-llvm -o - -triple=i386-unknown-unknown | FileCheck -check-prefix CODE-LP32 %s +// RUN: %clang_cc1 %s -flto -fvisibility hidden -fsanitize=cfi-vcall -vtable-interleaving -emit-llvm -o - -triple=x86_64-unknown-unknown | FileCheck -check-prefix GLOBAL-LP64 %s +// RUN: %clang_cc1 %s -flto -fvisibility hidden -fsanitize=cfi-vcall -vtable-interleaving -emit-llvm -o - -triple=i386-unknown-unknown | FileCheck -check-prefix GLOBAL-LP32 %s +// RUN: %clang_cc1 %s -flto -fvisibility hidden -fsanitize=cfi-vcall -vtable-interleaving -emit-llvm -o - -triple=armv7-unknown-unknown | FileCheck -check-prefix GLOBAL-ARM %s + +// PNaCl uses the same representation of method pointers as ARM. +// RUN: %clang_cc1 %s -flto -fvisibility hidden -fsanitize=cfi-vcall -vtable-interleaving -emit-llvm -o - -triple=le32-unknown-nacl | FileCheck -check-prefix GLOBAL-ARM %s +// MIPS uses the same representation of method pointers as ARM. +// RUN: %clang_cc1 %s -flto -fvisibility hidden -fsanitize=cfi-vcall -vtable-interleaving -emit-llvm -o - -triple=mips-unknown-linux-gnu | FileCheck -check-prefix GLOBAL-ARM %s +// WebAssembly uses the same representation of method pointers as ARM. +// RUN: %clang_cc1 %s -flto -fvisibility hidden -fsanitize=cfi-vcall -vtable-interleaving -emit-llvm -o - -triple=wasm32-unknown-unknown | FileCheck -check-prefix GLOBAL-ARM %s + +namespace test1 { +struct A { + int a; + void f(); + virtual void vf1(); + virtual void vf2(); +}; +struct B { + int b; + virtual void g(); +}; +struct C : B, A {}; + +void (A::*pa)(); +void (A::*volatile vpa)(); +void (B::*pb)(); +void (C::*pc)(); + +// GLOBAL-LP64: @_ZN5test13pa2E = hidden global { i64, i64 } { i64 ptrtoint (void (%"struct.test1::A"*)* @_ZN5test11A1fEv to i64), i64 0 }, align 8 +void (A::*pa2)() = &A::f; + +// GLOBAL-LP64: @_ZN5test13pa3E = hidden global { i64, i64 } { i64 add (i64 ptrtoint ([0 x i32]* @"__A$0" to i64), i64 1), i64 0 }, align 8 +// GLOBAL-LP32: @_ZN5test13pa3E = hidden global { i32, i32 } { i32 add (i32 ptrtoint ([0 x i32]* @"__A$0" to i32), i32 1), i32 0 }, align 4 +void (A::*pa3)() = &A::vf1; + +// GLOBAL-LP64: @_ZN5test13pa4E = hidden global { i64, i64 } { i64 add (i64 ptrtoint ([0 x i32]* @"__A$8" to i64), i64 1), i64 0 }, align 8 +// GLOBAL-LP32: @_ZN5test13pa4E = hidden global { i32, i32 } { i32 add (i32 ptrtoint ([0 x i32]* @"__A$4" to i32), i32 1), i32 0 }, align 4 +void (A::*pa4)() = &A::vf2; + +// GLOBAL-LP64: @_ZN5test13pc2E = hidden global { i64, i64 } { i64 ptrtoint (void (%"struct.test1::A"*)* @_ZN5test11A1fEv to i64), i64 16 }, align 8 +void (C::*pc2)() = &C::f; + +// GLOBAL-LP64: @_ZN5test13pc3E = hidden global { i64, i64 } { i64 add (i64 ptrtoint ([0 x i32]* @"__A$0" to i64), i64 1), i64 0 }, align 8 +void (A::*pc3)() = &A::vf1; + +void f() { + // CODE-LP64: store { i64, i64 } zeroinitializer, { i64, i64 }* @_ZN5test12paE + pa = 0; + + // Is this okay? What are LLVM's volatile semantics for structs? + // CODE-LP64: store volatile { i64, i64 } zeroinitializer, { i64, i64 }* @_ZN5test13vpaE + vpa = 0; + + // CODE-LP64: [[TMP:%.*]] = load { i64, i64 }, { i64, i64 }* @_ZN5test12paE, align 8 + // CODE-LP64: [[TMPADJ:%.*]] = extractvalue { i64, i64 } [[TMP]], 1 + // CODE-LP64: [[ADJ:%.*]] = add nsw i64 [[TMPADJ]], 16 + // CODE-LP64: [[RES:%.*]] = insertvalue { i64, i64 } [[TMP]], i64 [[ADJ]], 1 + // CODE-LP64: store { i64, i64 } [[RES]], { i64, i64 }* @_ZN5test12pcE, align 8 + pc = pa; + + // CODE-LP64: [[TMP:%.*]] = load { i64, i64 }, { i64, i64 }* @_ZN5test12pcE, align 8 + // CODE-LP64: [[TMPADJ:%.*]] = extractvalue { i64, i64 } [[TMP]], 1 + // CODE-LP64: [[ADJ:%.*]] = sub nsw i64 [[TMPADJ]], 16 + // CODE-LP64: [[RES:%.*]] = insertvalue { i64, i64 } [[TMP]], i64 [[ADJ]], 1 + // CODE-LP64: store { i64, i64 } [[RES]], { i64, i64 }* @_ZN5test12paE, align 8 + pa = static_cast(pc); +} + +void f2() { + // CODE-LP64: store { i64, i64 } { i64 ptrtoint (void (%"struct.test1::A"*)* @_ZN5test11A1fEv to i64), i64 0 } + void (A::*pa2)() = &A::f; + + // CODE-LP64: store { i64, i64 } { i64 add (i64 ptrtoint ([0 x i32]* @"__A$0" to i64), i64 1), i64 0 } + // CODE-LP32: store { i32, i32 } { i32 add (i32 ptrtoint ([0 x i32]* @"__A$0" to i32), i32 1), i32 0 } + void (A::*pa3)() = &A::vf1; + + // CODE-LP64: store { i64, i64 } { i64 add (i64 ptrtoint ([0 x i32]* @"__A$8" to i64), i64 1), i64 0 } + // CODE-LP32: store { i32, i32 } { i32 add (i32 ptrtoint ([0 x i32]* @"__A$4" to i32), i32 1), i32 0 } + void (A::*pa4)() = &A::vf2; +} + +void f3(A *a, A &ar) { + (a->*pa)(); + (ar.*pa)(); +} + +bool f4() { + return pa; +} +} // namespace test1 + +namespace test2 { +struct A { + void foo(); + virtual void vfoo(); +}; +struct B { + void foo(); + virtual void vfoo(); +}; +struct C : A, B { + void foo(); + virtual void vfoo(); +}; + +// GLOBAL-ARM: @_ZN5test24ptr0E = hidden global {{.*}} { i32 ptrtoint ({{.*}}* @_ZN5test21A3fooEv to i32), i32 0 } +// GLOBAL-ARM: @_ZN5test24ptr1E = hidden global {{.*}} { i32 ptrtoint ({{.*}}* @_ZN5test21B3fooEv to i32), i32 8 } +// GLOBAL-ARM: @_ZN5test24ptr2E = hidden global {{.*}} { i32 ptrtoint ({{.*}}* @_ZN5test21C3fooEv to i32), i32 0 } +// GLOBAL-ARM: @_ZN5test24ptr3E = hidden global {{.*}} { i32 ptrtoint ([0 x i32]* @"__A$0.1" to i32), i32 1 } +// GLOBAL-ARM: @_ZN5test24ptr4E = hidden global {{.*}} { i32 ptrtoint ([0 x i32]* @"__B$0" to i32), i32 9 } +// GLOBAL-ARM: @_ZN5test24ptr5E = hidden global {{.*}} { i32 ptrtoint ([0 x i32]* @"__C$0" to i32), i32 1 } +void (C::*ptr0)() = &A::foo; +void (C::*ptr1)() = &B::foo; +void (C::*ptr2)() = &C::foo; +void (C::*ptr3)() = &A::vfoo; +void (C::*ptr4)() = &B::vfoo; +void (C::*ptr5)() = &C::vfoo; +} // namespace test2 + +// rdar://problem/10815683 - Verify that we can emit reinterprets of +// member pointers as constant initializers. For added trickiness, +// we also add some non-trivial adjustments. +namespace test3 { +struct A { + int nonEmpty; + void foo(); +}; +struct B : public A { + virtual void requireNonZeroAdjustment(); +}; +struct C { + int nonEmpty; +}; +struct D : public C { + virtual void requireNonZeroAdjustment(); +}; + +// It's not that the offsets are doubled on ARM, it's that they're left-shifted by 1. + +// GLOBAL-LP64: @_ZN5test31aE = hidden global { i64, i64 } { i64 ptrtoint (void (%"struct.test3::A"*)* @_ZN5test31A3fooEv to i64), i64 0 }, align 8 +// GLOBAL-LP32: @_ZN5test31aE = hidden global { i32, i32 } { i32 ptrtoint (void (%"struct.test3::A"*)* @_ZN5test31A3fooEv to i32), i32 0 }, align 4 +// GLOBAL-ARM: @_ZN5test31aE = hidden global { i32, i32 } { i32 ptrtoint (void (%"struct.test3::A"*)* @_ZN5test31A3fooEv to i32), i32 0 }, align 4 +void (A::*a)() = &A::foo; + +// GLOBAL-LP64: @_ZN5test31bE = hidden global { i64, i64 } { i64 ptrtoint (void (%"struct.test3::A"*)* @_ZN5test31A3fooEv to i64), i64 8 }, align 8 +// GLOBAL-LP32: @_ZN5test31bE = hidden global { i32, i32 } { i32 ptrtoint (void (%"struct.test3::A"*)* @_ZN5test31A3fooEv to i32), i32 4 }, align 4 +// GLOBAL-ARM: @_ZN5test31bE = hidden global { i32, i32 } { i32 ptrtoint (void (%"struct.test3::A"*)* @_ZN5test31A3fooEv to i32), i32 8 }, align 4 +void (B::*b)() = (void (B::*)()) & A::foo; + +// GLOBAL-LP64: @_ZN5test31cE = hidden global { i64, i64 } { i64 ptrtoint (void (%"struct.test3::A"*)* @_ZN5test31A3fooEv to i64), i64 8 }, align 8 +// GLOBAL-LP32: @_ZN5test31cE = hidden global { i32, i32 } { i32 ptrtoint (void (%"struct.test3::A"*)* @_ZN5test31A3fooEv to i32), i32 4 }, align 4 +// GLOBAL-ARM: @_ZN5test31cE = hidden global { i32, i32 } { i32 ptrtoint (void (%"struct.test3::A"*)* @_ZN5test31A3fooEv to i32), i32 8 }, align 4 +void (C::*c)() = (void (C::*)())(void (B::*)()) & A::foo; + +// GLOBAL-LP64: @_ZN5test31dE = hidden global { i64, i64 } { i64 ptrtoint (void (%"struct.test3::A"*)* @_ZN5test31A3fooEv to i64), i64 16 }, align 8 +// GLOBAL-LP32: @_ZN5test31dE = hidden global { i32, i32 } { i32 ptrtoint (void (%"struct.test3::A"*)* @_ZN5test31A3fooEv to i32), i32 8 }, align 4 +// GLOBAL-ARM: @_ZN5test31dE = hidden global { i32, i32 } { i32 ptrtoint (void (%"struct.test3::A"*)* @_ZN5test31A3fooEv to i32), i32 16 }, align 4 +void (D::*d)() = (void (C::*)())(void (B::*)()) & A::foo; +} // namespace test3 + +namespace test4 { +struct A { + virtual void a(); +}; +struct B : A {}; +struct C : B { + virtual void a(); +}; +void (C::*x)() = &C::a; + +// GLOBAL-LP64: @_ZN5test41xE = hidden global { i64, i64 } { i64 add (i64 ptrtoint ([0 x i32]* @"__C$0.2" to i64), i64 1), i64 0 } +// GLOBAL-LP32: @_ZN5test41xE = hidden global { i32, i32 } { i32 add (i32 ptrtoint ([0 x i32]* @"__C$0.2" to i32), i32 1), i32 0 } +// GLOBAL-ARM: @_ZN5test41xE = hidden global { i32, i32 } { i32 ptrtoint ([0 x i32]* @"__C$0.2" to i32), i32 1 } +} // namespace test4 Index: clang/test/CodeGenCXX/interleaving-virtual-base.cpp =================================================================== --- /dev/null +++ clang/test/CodeGenCXX/interleaving-virtual-base.cpp @@ -0,0 +1,72 @@ +// This file tests if accesses to vbase entries are correctly generated when vtable interleaving is enabled. +// This test is based on virtual-base-cast.cpp. + +// RUN: %clang_cc1 -flto -fvisibility hidden -fsanitize=cfi-vcall -vtable-interleaving -emit-llvm %s -o - -triple i686-pc-linux-gnu | FileCheck %s + +struct A { + int a; + virtual int aa(); +}; +struct B { + int b; + virtual int bb(); +}; +struct C : virtual A, virtual B { + int c; + virtual int aa(); + virtual int bb(); +}; +struct AA { + int a; + virtual int aa(); +}; +struct BB { + int b; + virtual int bb(); +}; +struct CC : AA, BB { + virtual int aa(); + virtual int bb(); + virtual int cc(); +}; +struct D : virtual C, virtual CC { + int e; +}; + +D *x; + +A *a() { return x; } +// CHECK: @_Z1av() [[NUW:#[0-9]+]] +// CHECK: [[VBASEOFFSETPTRA:%[a-zA-Z0-9\.]+]] = getelementptr inbounds i8, i8* {{.*}}, i64 ptrtoint ([0 x i32]* @"__D$-16" to i64) +// CHECK: [[CASTVBASEOFFSETPTRA:%[a-zA-Z0-9\.]+]] = bitcast i8* [[VBASEOFFSETPTRA]] to i32* +// CHECK: load i32, i32* [[CASTVBASEOFFSETPTRA]] +// CHECK: } + +B *b() { return x; } +// CHECK: @_Z1bv() [[NUW]] +// CHECK: [[VBASEOFFSETPTRA:%[a-zA-Z0-9\.]+]] = getelementptr inbounds i8, i8* {{.*}}, i64 ptrtoint ([0 x i32]* @"__D$-20" to i64) +// CHECK: [[CASTVBASEOFFSETPTRA:%[a-zA-Z0-9\.]+]] = bitcast i8* [[VBASEOFFSETPTRA]] to i32* +// CHECK: load i32, i32* [[CASTVBASEOFFSETPTRA]] +// CHECK: } + +BB *c() { return x; } +// CHECK: @_Z1cv() [[NUW]] +// CHECK: [[VBASEOFFSETPTRC:%[a-zA-Z0-9\.]+]] = getelementptr inbounds i8, i8* {{.*}}, i64 ptrtoint ([0 x i32]* @"__D$-24" to i64) +// CHECK: [[CASTVBASEOFFSETPTRC:%[a-zA-Z0-9\.]+]] = bitcast i8* [[VBASEOFFSETPTRC]] to i32* +// CHECK: [[VBASEOFFSETC:%[a-zA-Z0-9\.]+]] = load i32, i32* [[CASTVBASEOFFSETPTRC]] +// CHECK: add i32 [[VBASEOFFSETC]], 8 +// CHECK: } + +// Put the vbptr at a non-zero offset inside a non-virtual base. +struct E { + int e; +}; +struct F : E, D { + int f; +}; + +F *y; + +BB *d() { return y; } + +// CHECK: attributes [[NUW]] = { noinline nounwind{{.*}} } Index: clang/test/CodeGenCXX/interleaving-virtual-calls.cpp =================================================================== --- /dev/null +++ clang/test/CodeGenCXX/interleaving-virtual-calls.cpp @@ -0,0 +1,38 @@ +// This file tests if accesses to virtual function pointer entries are correctly generated when vtable interleaving is enabled. + +// RUN: %clang_cc1 -vtable-interleaving -flto -fvisibility hidden -fsanitize=cfi-vcall -emit-llvm %s -o - -triple x86_64-unknown-linux | FileCheck %s + +struct A { + int a; + virtual int foo1(); + virtual int foo2(); +}; +struct B : A { + int b; + virtual int foo1(); +}; +struct C { + int c; + virtual int foo3(); + virtual int foo4(); +}; +struct D : B, C { + virtual int foo2(); + virtual int foo3(); +}; + +// CHECK: @_Z5test1P1A +int test1(A *a) { + // CHECK: [[R0:%.*]] = bitcast %struct.A* {{.*}} to i8** + // CHECK: [[VTABLE:%.*]] = load i8*, i8** [[R0]] + // CHECK: getelementptr inbounds i8, i8* [[VTABLE]], i32 ptrtoint ([0 x i32]* @"__A$0" to i32) + return a->foo1(); +} + +// CHECK: @_Z5test2P1C +int test2(C *c) { + // CHECK: [[R0:%.*]] = bitcast %struct.C* {{.*}} to i8** + // CHECK: [[VTABLE:%.*]] = load i8*, i8** [[R0]] + // CHECK: getelementptr inbounds i8, i8* [[VTABLE]], i32 ptrtoint ([0 x i32]* @"__C$8" to i32) + return c->foo4(); +} Index: clang/test/CodeGenCXX/type-metadata.cpp =================================================================== --- clang/test/CodeGenCXX/type-metadata.cpp +++ clang/test/CodeGenCXX/type-metadata.cpp @@ -12,10 +12,15 @@ // RUN: %clang_cc1 -flto -flto-unit -triple x86_64-unknown-linux -fvisibility hidden -fsanitize=cfi-vcall -fsanitize-trap=cfi-vcall -fwhole-program-vtables -emit-llvm -o - %s | FileCheck --check-prefix=CFI --check-prefix=CFI-VT --check-prefix=ITANIUM --check-prefix=TC-ITANIUM %s // RUN: %clang_cc1 -flto -flto-unit -triple x86_64-pc-windows-msvc -fsanitize=cfi-vcall -fsanitize-trap=cfi-vcall -fwhole-program-vtables -emit-llvm -o - %s | FileCheck --check-prefix=CFI --check-prefix=CFI-VT --check-prefix=MS --check-prefix=TC-MS %s +// Tests for the cfi-vcall with vtable interleaving: +// RUN: %clang_cc1 -flto -flto-unit -triple x86_64-unknown-linux -fvisibility hidden -fsanitize=cfi-vcall -fsanitize-trap=cfi-vcall -vtable-interleaving -emit-llvm -o - %s | FileCheck --check-prefix=CFI --check-prefix=CFI-NVT --check-prefix=INTER --check-prefix=TT-ITANIUM --check-prefix=NDIAG %s + // ITANIUM: @_ZTV1A = {{[^!]*}}, !type [[A16:![0-9]+]] // ITANIUM-DIAG-SAME: !type [[ALL16:![0-9]+]] // ITANIUM-SAME: !type [[AF16:![0-9]+]] +// INTER: @_ZTV1A = {{[^!]*}}, !type [[A16:![0-9]+]] + // ITANIUM: @_ZTV1B = {{[^!]*}}, !type [[A32:![0-9]+]] // ITANIUM-DIAG-SAME: !type [[ALL32:![0-9]+]] // ITANIUM-SAME: !type [[AF32:![0-9]+]] @@ -27,6 +32,10 @@ // ITANIUM-SAME: !type [[BF40:![0-9]+]] // ITANIUM-SAME: !type [[BF48:![0-9]+]] +// INTER: @_ZTV1B = {{[^!]*}}, !type [[A32:![0-9]+]] +// INTER-SAME: !type [[A_VCALL_OFFSET32:![0-9]+]] +// INTER-SAME: !type [[B32:![0-9]+]] + // ITANIUM: @_ZTV1C = {{[^!]*}}, !type [[A32]] // ITANIUM-DIAG-SAME: !type [[ALL32]] // ITANIUM-SAME: !type [[AF32]] @@ -34,6 +43,10 @@ // ITANIUM-DIAG-SAME: !type [[ALL32]] // ITANIUM-SAME: !type [[CF32:![0-9]+]] +// INTER: @_ZTV1C = {{[^!]*}}, !type [[A32]] +// INTER-SAME: !type [[A_VCALL_OFFSET32]] +// INTER-SAME: !type [[C32_4:![0-9]+]] + // DIAG: @[[SRC:.*]] = private unnamed_addr constant [{{.*}} x i8] c"{{.*}}type-metadata.cpp\00", align 1 // DIAG: @[[TYPE:.*]] = private unnamed_addr constant { i16, i16, [4 x i8] } { i16 -1, i16 0, [4 x i8] c"'A'\00" } // DIAG: @[[BADTYPESTATIC:.*]] = private unnamed_addr global { i8, { [{{.*}} x i8]*, i32, i32 }, { i16, i16, [4 x i8] }* } { i8 0, { [{{.*}} x i8]*, i32, i32 } { [{{.*}} x i8]* @[[SRC]], i32 123, i32 3 }, { i16, i16, [4 x i8] }* @[[TYPE]] } @@ -59,11 +72,21 @@ // ITANIUM-SAME: !type [[DF40:![0-9]+]] // ITANIUM-SAME: !type [[DF48:![0-9]+]] +// INTER: @_ZTVN12_GLOBAL__N_11DE = {{[^!]*}}, !type [[A32]] +// INTER-SAME: !type [[A_VCALL_OFFSET32]] +// INTER-SAME: !type [[B32]] +// INTER-SAME: !type [[D32:![0-9]+]] +// INTER-SAME: !type [[C88:![0-9]+]] + // ITANIUM: @_ZTCN12_GLOBAL__N_11DE0_1B = {{[^!]*}}, !type [[A32]] // ITANIUM-DIAG-SAME: !type [[ALL32]] // ITANIUM-SAME: !type [[B32]] // ITANIUM-DIAG-SAME: !type [[ALL32]] +// INTER: @_ZTCN12_GLOBAL__N_11DE0_1B = {{[^!]*}}, !type [[A32]] +// INTER-SAME: !type [[A_VCALL_OFFSET32]] +// INTER-SAME: !type [[B32]] + // ITANIUM: @_ZTCN12_GLOBAL__N_11DE8_1C = {{[^!]*}}, !type [[A64:![0-9]+]] // ITANIUM-DIAG-SAME: !type [[ALL64:![0-9]+]] // ITANIUM-SAME: !type [[AF64:![0-9]+]] @@ -71,6 +94,10 @@ // ITANIUM-DIAG-SAME: !type [[ALL32]] // ITANIUM-SAME: !type [[CF64:![0-9]+]] +// INTER: @_ZTCN12_GLOBAL__N_11DE8_1C = {{[^!]*}}, !type [[C32_2:![0-9]+]] +// INTER-SAME: !type [[A64:![0-9]+]] +// INTER-SAME: !type [[A_VCALL_OFFSET64:![0-9]+]] + // ITANIUM: @_ZTVZ3foovE2FA = {{[^!]*}}, !type [[A16]] // ITANIUM-DIAG-SAME: !type [[ALL16]] // ITANIUM-SAME: !type [[AF16]] @@ -78,6 +105,9 @@ // ITANIUM-DIAG-SAME: !type [[ALL16]] // ITANIUM-SAME: !type [[FAF16:![0-9]+]] +// INTER: @_ZTVZ3foovE2FA = {{[^!]*}}, !type [[A16]] +// INTER-SAME: !type [[FA16:![0-9]+]] + // MS: comdat($"??_7A@@6B@"), !type [[A8:![0-9]+]] // MS: comdat($"??_7B@@6B0@@"), !type [[B8:![0-9]+]] // MS: comdat($"??_7B@@6BA@@@"), !type [[A8]] @@ -129,6 +159,7 @@ } // ITANIUM: define hidden void @_Z2afP1A +// INTER: define hidden void @_Z2afP1A // MS: define dso_local void @"?af@@YAXPEAUA@@@Z" void af(A *a) { // TT-ITANIUM: [[P:%[^ ]*]] = call i1 @llvm.type.test(i8* [[VT:%[^ ]*]], metadata !"_ZTS1A") @@ -161,6 +192,7 @@ } // ITANIUM: define internal void @_Z3df1PN12_GLOBAL__N_11DE +// INTER: define internal void @_Z3df1PN12_GLOBAL__N_11DE // MS: define internal void @"?df1@@YAXPEAUD@?A0x{{[^@]*}}@@@Z" void df1(D *d) { // TT-ITANIUM: {{%[^ ]*}} = call i1 @llvm.type.test(i8* {{%[^ ]*}}, metadata ![[DTYPE:[0-9]+]]) @@ -171,6 +203,7 @@ } // ITANIUM: define internal void @_Z3dg1PN12_GLOBAL__N_11DE +// INTER: define internal void @_Z3dg1PN12_GLOBAL__N_11DE // MS: define internal void @"?dg1@@YAXPEAUD@?A0x{{[^@]*}}@@@Z" void dg1(D *d) { // TT-ITANIUM: {{%[^ ]*}} = call i1 @llvm.type.test(i8* {{%[^ ]*}}, metadata !"_ZTS1B") @@ -181,6 +214,7 @@ } // ITANIUM: define internal void @_Z3dh1PN12_GLOBAL__N_11DE +// INTER: define internal void @_Z3dh1PN12_GLOBAL__N_11DE // MS: define internal void @"?dh1@@YAXPEAUD@?A0x{{[^@]*}}@@@Z" void dh1(D *d) { // TT-ITANIUM: {{%[^ ]*}} = call i1 @llvm.type.test(i8* {{%[^ ]*}}, metadata ![[DTYPE]]) @@ -191,6 +225,7 @@ } // ITANIUM: define internal void @_Z3df2PN12_GLOBAL__N_11DE +// INTER: define internal void @_Z3df2PN12_GLOBAL__N_11DE // MS: define internal void @"?df2@@YAXPEAUD@?A0x{{[^@]*}}@@@Z" __attribute__((no_sanitize("cfi"))) void df2(D *d) { @@ -201,6 +236,7 @@ } // ITANIUM: define internal void @_Z3df3PN12_GLOBAL__N_11DE +// INTER: define internal void @_Z3df3PN12_GLOBAL__N_11DE // MS: define internal void @"?df3@@YAXPEAUD@?A0x{{[^@]*}}@@@Z" __attribute__((no_sanitize("address"))) __attribute__((no_sanitize("cfi-vcall"))) void df3(D *d) { @@ -239,6 +275,7 @@ }; // ITANIUM: define hidden void @_ZN5test21fEPNS_1DE +// INTER: define hidden void @_ZN5test21fEPNS_1DE // MS: define dso_local void @"?f@test2@@YAXPEAUD@1@@Z" void f(D *d) { // TT-ITANIUM: {{%[^ ]*}} = call i1 @llvm.type.test(i8* {{%[^ ]*}}, metadata !"_ZTSN5test21DE") @@ -283,6 +320,20 @@ // ITANIUM: [[FAF16]] = !{i64 16, [[FAF_ID:![0-9]+]]} // ITANIUM: [[FAF_ID]] = distinct !{} +// INTER: [[A16]] = !{i64 16, !"_ZTS1A", i64 2} +// INTER: [[A32]] = !{i64 32, !"_ZTS1A", i64 2} +// INTER: [[A_VCALL_OFFSET32]] = !{i64 32, !"_ZTS1A.vcall_offset", i64 3} +// INTER: [[B32]] = !{i64 32, !"_ZTS1B", i64 4} +// INTER: [[C32_4]] = !{i64 32, !"_ZTS1C", i64 4} +// INTER: [[D32]] = !{i64 32, [[D_ID:![0-9]+]], i64 5} +// INTER: [[D_ID]] = distinct !{} +// INTER: [[C88]] = !{i64 88, !"_ZTS1C", i64 2} +// INTER: [[C32_2]] = !{i64 32, !"_ZTS1C", i64 2} +// INTER: [[A64]] = !{i64 64, !"_ZTS1A", i64 2} +// INTER: [[A_VCALL_OFFSET64]] = !{i64 64, !"_ZTS1A.vcall_offset", i64 3} +// INTER: [[FA16]] = !{i64 16, [[FA_ID:![0-9]+]], i64 3} +// INTER: [[FA_ID]] = distinct !{} + // MS: [[A8]] = !{i64 8, !"?AUA@@"} // MS: [[B8]] = !{i64 8, !"?AUB@@"} // MS: [[D8]] = !{i64 8, [[D_ID:![0-9]+]]} Index: clang/test/CodeGenCXX/ubsan-vtable-checks.cpp =================================================================== --- clang/test/CodeGenCXX/ubsan-vtable-checks.cpp +++ clang/test/CodeGenCXX/ubsan-vtable-checks.cpp @@ -25,7 +25,10 @@ // CHECK-NULL-NEXT: br i1 [[UBSAN_CMP_RES]], label %{{.*}}, label %{{.*}} // CHECK-NULL: call void @__ubsan_handle_type_mismatch_v1_abort // Second, we check that vtable is actually loaded once the type check is done. - // CHECK-NULL: load {{.*}} (%struct.T*{{.*}})**, {{.*}} (%struct.T*{{.*}})*** + // ITANIUM: [[R0:%.*]] = bitcast %struct.T* {{.*}} to i8** + // ITANIUM-NEXT: load i8*, i8** [[R0]] + // MSABI: [[R0:%.*]] = bitcast %struct.T* {{.*}} to i32 (%struct.T*)*** + // MSABI-NEXT: load i32 (%struct.T*)**, i32 (%struct.T*)*** [[R0]] return t->v(); } @@ -37,7 +40,10 @@ // CHECK-VPTR: br i1 {{.*}} label %{{.*}} // CHECK-VPTR: call void @__ubsan_handle_dynamic_type_cache_miss_abort // Second, we check that vtable is actually loaded once the type check is done. - // CHECK-VPTR: load {{.*}} (%struct.T*{{.*}})**, {{.*}} (%struct.T*{{.*}})*** + // ITANIUM: [[vtable:%.*]] = bitcast %struct.T* {{.*}} to i8** + // ITANIUM-NEXT: load i8*, i8** [[vtable]] + // MSABI: [[R0:%.*]] = bitcast %struct.T* {{.*}} to i8* (%struct.T*, i32)*** + // MSABI-NEXT: load i8* (%struct.T*, i32)**, i8* (%struct.T*, i32)*** [[R0]] delete t; } Index: clang/test/CodeGenCXX/virtual-base-cast.cpp =================================================================== --- clang/test/CodeGenCXX/virtual-base-cast.cpp +++ clang/test/CodeGenCXX/virtual-base-cast.cpp @@ -13,7 +13,7 @@ A* a() { return x; } // CHECK: @_Z1av() [[NUW:#[0-9]+]] -// CHECK: [[VBASEOFFSETPTRA:%[a-zA-Z0-9\.]+]] = getelementptr i8, i8* {{.*}}, i64 -16 +// CHECK: [[VBASEOFFSETPTRA:%[a-zA-Z0-9\.]+]] = getelementptr inbounds i8, i8* {{.*}}, i64 -16 // CHECK: [[CASTVBASEOFFSETPTRA:%[a-zA-Z0-9\.]+]] = bitcast i8* [[VBASEOFFSETPTRA]] to i32* // CHECK: load i32, i32* [[CASTVBASEOFFSETPTRA]] // CHECK: } @@ -29,7 +29,7 @@ B* b() { return x; } // CHECK: @_Z1bv() [[NUW]] -// CHECK: [[VBASEOFFSETPTRA:%[a-zA-Z0-9\.]+]] = getelementptr i8, i8* {{.*}}, i64 -20 +// CHECK: [[VBASEOFFSETPTRA:%[a-zA-Z0-9\.]+]] = getelementptr inbounds i8, i8* {{.*}}, i64 -20 // CHECK: [[CASTVBASEOFFSETPTRA:%[a-zA-Z0-9\.]+]] = bitcast i8* [[VBASEOFFSETPTRA]] to i32* // CHECK: load i32, i32* [[CASTVBASEOFFSETPTRA]] // CHECK: } @@ -47,7 +47,7 @@ BB* c() { return x; } // CHECK: @_Z1cv() [[NUW]] -// CHECK: [[VBASEOFFSETPTRC:%[a-zA-Z0-9\.]+]] = getelementptr i8, i8* {{.*}}, i64 -24 +// CHECK: [[VBASEOFFSETPTRC:%[a-zA-Z0-9\.]+]] = getelementptr inbounds i8, i8* {{.*}}, i64 -24 // CHECK: [[CASTVBASEOFFSETPTRC:%[a-zA-Z0-9\.]+]] = bitcast i8* [[VBASEOFFSETPTRC]] to i32* // CHECK: [[VBASEOFFSETC:%[a-zA-Z0-9\.]+]] = load i32, i32* [[CASTVBASEOFFSETPTRC]] // CHECK: add i32 [[VBASEOFFSETC]], 8 Index: compiler-rt/test/cfi/CMakeLists.txt =================================================================== --- compiler-rt/test/cfi/CMakeLists.txt +++ compiler-rt/test/cfi/CMakeLists.txt @@ -1,6 +1,6 @@ set(CFI_TESTSUITES) -macro (add_cfi_test_suites lld thinlto newpm) +macro (add_cfi_test_suites lld thinlto newpm interleaving) set(suffix) if (${lld}) set(suffix ${suffix}-lld) @@ -11,11 +11,15 @@ if (${newpm}) set(suffix ${suffix}-newpm) endif() + if (${interleaving}) + set(suffix ${suffix}-interleaving) + endif() set(suffix ${suffix}-${CFI_TEST_TARGET_ARCH}) set(CFI_TEST_USE_LLD ${lld}) set(CFI_TEST_USE_THINLTO ${thinlto}) set(CFI_TEST_USE_NEWPM ${newpm}) + set(CFI_TEST_USE_INTERLEAVING ${interleaving}) set(CFI_LIT_TEST_MODE Standalone) set(CFI_TEST_CONFIG_SUFFIX -standalone${suffix}) @@ -44,19 +48,19 @@ get_test_cc_for_arch(${arch} CFI_TEST_TARGET_CC CFI_TEST_TARGET_CFLAGS) if (APPLE) # FIXME: enable ThinLTO tests after fixing http://llvm.org/pr32741 - add_cfi_test_suites(False False False) + add_cfi_test_suites(False False False False) elseif(WIN32) - add_cfi_test_suites(True False False) - add_cfi_test_suites(True True False) + add_cfi_test_suites(True False False False) + add_cfi_test_suites(True True False False) else() - add_cfi_test_suites(False False False) - add_cfi_test_suites(False True False) - add_cfi_test_suites(False False True) - add_cfi_test_suites(False True True) - if (COMPILER_RT_HAS_LLD AND NOT arch STREQUAL "i386") - add_cfi_test_suites(True False False) - add_cfi_test_suites(True True False) - endif() + add_cfi_test_suites(False False False False) + add_cfi_test_suites(False True False False) + add_cfi_test_suites(False False True False) + add_cfi_test_suites(False True True False) + add_cfi_test_suites(True False False False) + add_cfi_test_suites(True True False False) + add_cfi_test_suites(True False False True) + add_cfi_test_suites(False False False True) endif() endforeach() Index: compiler-rt/test/cfi/anon-namespace.cpp =================================================================== --- compiler-rt/test/cfi/anon-namespace.cpp +++ compiler-rt/test/cfi/anon-namespace.cpp @@ -26,7 +26,7 @@ // RUN: %clangxx_cfi_diag -c -DTU1 -o %t1.o %s // RUN: %clangxx_cfi_diag -c -DTU2 -o %t2.o %S/../cfi/anon-namespace.cpp // RUN: %clangxx_cfi_diag -o %t6 %t1.o %t2.o -// RUN: %run %t6 2>&1 | FileCheck --check-prefix=CFI-DIAG %s +// RUN: %interleave_diag %run %t6 2>&1 | FileCheck --check-prefix=CFI-DIAG %s // Tests that the CFI mechanism treats classes in the anonymous namespace in // different translation units as having distinct identities. This is done by Index: compiler-rt/test/cfi/create-derivers.test =================================================================== --- compiler-rt/test/cfi/create-derivers.test +++ compiler-rt/test/cfi/create-derivers.test @@ -1,21 +1,22 @@ REQUIRES: asserts +UNSUPPORTED: interleaving %% Explicit -flto to override possible -flto=thin in %clangxx_cfi RUN: %clangxx_cfi -flto -c -o %t1.o %S/simple-fail.cpp -RUN: opt -lowertypetests -debug-only=lowertypetests -o /dev/null %t1.o 2>&1 | FileCheck --check-prefix=B0 %s +RUN: opt -%opt_flag -debug-only=%opt_flag -o /dev/null %t1.o 2>&1 | FileCheck --check-prefix=B0 %s B0: {{1B|B@@}}: {{.*}} size 1 RUN: %clangxx_cfi -DB32 -flto -c -o %t2.o %S/simple-fail.cpp -RUN: opt -lowertypetests -debug-only=lowertypetests -o /dev/null %t2.o 2>&1 | FileCheck --check-prefix=B32 %s +RUN: opt -%opt_flag -debug-only=%opt_flag -o /dev/null %t2.o 2>&1 | FileCheck --check-prefix=B32 %s B32: {{1B|B@@}}: {{.*}} size 2{{3|4}} B32-NOT: all-ones RUN: %clangxx_cfi -DB64 -flto -c -o %t3.o %S/simple-fail.cpp -RUN: opt -lowertypetests -debug-only=lowertypetests -o /dev/null %t3.o 2>&1 | FileCheck --check-prefix=B64 %s +RUN: opt -%opt_flag -debug-only=%opt_flag -o /dev/null %t3.o 2>&1 | FileCheck --check-prefix=B64 %s B64: {{1B|B@@}}: {{.*}} size 5{{3|4}} B64-NOT: all-ones RUN: %clangxx_cfi -DBM -flto -c -o %t4.o %S/simple-fail.cpp -RUN: opt -lowertypetests -debug-only=lowertypetests -o /dev/null %t4.o 2>&1 | FileCheck --check-prefix=BM %s +RUN: opt -%opt_flag -debug-only=%opt_flag -o /dev/null %t4.o 2>&1 | FileCheck --check-prefix=BM %s BM: {{1B|B@@}}: {{.*}} size 8{{3|4}} BM-NOT: all-ones Index: compiler-rt/test/cfi/cross-dso/icall/lit.local.cfg.py =================================================================== --- compiler-rt/test/cfi/cross-dso/icall/lit.local.cfg.py +++ compiler-rt/test/cfi/cross-dso/icall/lit.local.cfg.py @@ -1,3 +1,6 @@ # The cfi-icall checker is only supported on x86 and x86_64 for now. if config.root.host_arch not in ['x86', 'x86_64']: config.unsupported = True + +if 'interleaving' in config.available_features: + config.unsupported = True \ No newline at end of file Index: compiler-rt/test/cfi/cross-dso/lit.local.cfg.py =================================================================== --- compiler-rt/test/cfi/cross-dso/lit.local.cfg.py +++ compiler-rt/test/cfi/cross-dso/lit.local.cfg.py @@ -11,3 +11,6 @@ # Android O (API level 26) has support for cross-dso cfi in libdl.so. if config.android and 'android-26' not in config.available_features: config.unsupported = True + +if 'interleaving' in config.available_features: + config.unsupported = True \ No newline at end of file Index: compiler-rt/test/cfi/lit.cfg.py =================================================================== --- compiler-rt/test/cfi/lit.cfg.py +++ compiler-rt/test/cfi/lit.cfg.py @@ -19,7 +19,10 @@ if config.cfi_lit_test_mode == "Devirt": config.available_features.add('devirt') clang_cfi += '-fwhole-program-vtables ' - config.substitutions.append((r"%expect_crash_unless_devirt ", "")) + if config.use_interleaving: + config.substitutions.append((r"%expect_crash_unless_devirt ", config.expect_crash)) + else: + config.substitutions.append((r"%expect_crash_unless_devirt ", "")) else: config.substitutions.append((r"%expect_crash_unless_devirt ", config.expect_crash)) @@ -27,14 +30,28 @@ diag = '-fno-sanitize-trap=cfi -fsanitize-recover=cfi ' non_dso = '-fvisibility=hidden ' dso = '-fsanitize-cfi-cross-dso -fvisibility=default ' + interleave = '-Xclang -vtable-interleaving -Wl,-plugin-opt,-vtable-interleaving -fno-sanitize=cfi-mfcall ' if config.android: dso += '-include ' + config.test_source_root + '/cross-dso/util/cfi_stubs.h ' + config.substitutions.append((r"%clang_cfi ", clang_cfi + non_dso)) - config.substitutions.append((r"%clangxx_cfi ", clang_cfi + cxx + non_dso)) config.substitutions.append((r"%clang_cfi_diag ", clang_cfi + non_dso + diag)) - config.substitutions.append((r"%clangxx_cfi_diag ", clang_cfi + cxx + non_dso + diag)) - config.substitutions.append((r"%clangxx_cfi_dso ", clang_cfi + cxx + dso)) - config.substitutions.append((r"%clangxx_cfi_dso_diag ", clang_cfi + cxx + dso + diag)) + + if config.use_interleaving: + config.available_features.add('interleaving') + config.substitutions.append((r"%clangxx_cfi ", clang_cfi + cxx + non_dso + interleave)) + config.substitutions.append((r"%clangxx_cfi_diag ", clang_cfi + cxx + non_dso + diag + interleave)) + config.substitutions.append((r"%interleave_diag ", "not ")) + else: + config.substitutions.append((r"%clangxx_cfi ", clang_cfi + cxx + non_dso)) + config.substitutions.append((r"%clangxx_cfi_dso ", clang_cfi + cxx + dso)) + config.substitutions.append((r"%clangxx_cfi_dso_diag ", clang_cfi + cxx + dso + diag)) + config.substitutions.append((r"%clangxx_cfi_diag ", clang_cfi + cxx + non_dso + diag)) + config.substitutions.append((r"%interleave_diag ", "")) + config.substitutions.append((r"%opt_flag ", "lowertypetests ")) + + config.substitutions.append((r"%clangxx_cfi_no_interleaving ", clang_cfi + cxx + non_dso)) + config.substitutions.append((r"%debug_info_flags", ' '.join(config.debug_info_flags))) else: config.unsupported = True @@ -44,3 +61,4 @@ if lit_config.params.get('check_supported', None) and config.unsupported: raise BaseException("Tests unsupported") + Index: compiler-rt/test/cfi/lit.site.cfg.py.in =================================================================== --- compiler-rt/test/cfi/lit.site.cfg.py.in +++ compiler-rt/test/cfi/lit.site.cfg.py.in @@ -8,6 +8,7 @@ config.use_lto = True # CFI *requires* LTO. config.use_thinlto = @CFI_TEST_USE_THINLTO@ config.use_newpm = @CFI_TEST_USE_NEWPM@ +config.use_interleaving = @CFI_TEST_USE_INTERLEAVING@ lit_config.load_config(config, "@COMPILER_RT_BINARY_DIR@/test/lit.common.configured") lit_config.load_config(config, "@CMAKE_CURRENT_SOURCE_DIR@/lit.cfg.py") Index: compiler-rt/test/cfi/mfcall.cpp =================================================================== --- compiler-rt/test/cfi/mfcall.cpp +++ compiler-rt/test/cfi/mfcall.cpp @@ -1,4 +1,4 @@ -// UNSUPPORTED: windows-msvc +// UNSUPPORTED: windows-msvc, interleaving // RUN: %clangxx_cfi -o %t %s // RUN: %expect_crash %run %t a Index: compiler-rt/test/cfi/multiple-inheritance.cpp =================================================================== --- compiler-rt/test/cfi/multiple-inheritance.cpp +++ compiler-rt/test/cfi/multiple-inheritance.cpp @@ -19,8 +19,8 @@ // RUN: %run %t5 x 2>&1 | FileCheck --check-prefix=NCFI %s // RUN: %clangxx_cfi_diag -o %t6 %s -// RUN: %run %t6 2>&1 | FileCheck --check-prefix=CFI-DIAG2 %s -// RUN: %run %t6 x 2>&1 | FileCheck --check-prefix=CFI-DIAG1 %s +// RUN: %interleave_diag %run %t6 2>&1 | FileCheck --check-prefix=CFI-DIAG2 %s +// RUN: %interleave_diag %run %t6 x 2>&1 | FileCheck --check-prefix=CFI-DIAG1 %s // Tests that the CFI mechanism is sensitive to multiple inheritance and only // permits calls via virtual tables for the correct base class. Index: compiler-rt/test/cfi/simple-fail.cpp =================================================================== --- compiler-rt/test/cfi/simple-fail.cpp +++ compiler-rt/test/cfi/simple-fail.cpp @@ -47,12 +47,12 @@ // RUN: %expect_crash %run %t16 2>&1 | FileCheck --check-prefix=CFI %s // RUN: %clangxx_cfi_diag -o %t17 %s -// RUN: %run %t17 2>&1 | FileCheck --check-prefix=CFI-DIAG %s +// RUN: %interleave_diag %run %t17 2>&1 | FileCheck --check-prefix=CFI-DIAG %s // RUN: %clangxx -o %t18 %s // RUN: %run %t18 2>&1 | FileCheck --check-prefix=NCFI %s -// RUN: %clangxx_cfi -DCHECK_NO_SANITIZE_CFI -o %t19 %s +// RUN: %clangxx_cfi_no_interleaving -DCHECK_NO_SANITIZE_CFI -o %t19 %s // RUN: %run %t19 2>&1 | FileCheck --check-prefix=NCFI %s // Tests that the CFI mechanism crashes the program when making a virtual call @@ -95,6 +95,7 @@ // CFI-DIAG-NEXT: note: vtable is of type '{{(struct )?}}A' ((B *)a)->f(); // UB here + // INTERLEAVING-NCFI-NOT: {{^2$}} // CFI-NOT: {{^2$}} // NCFI: {{^2$}} fprintf(stderr, "2\n"); Index: compiler-rt/test/cfi/vdtor.cpp =================================================================== --- compiler-rt/test/cfi/vdtor.cpp +++ compiler-rt/test/cfi/vdtor.cpp @@ -14,7 +14,7 @@ // RUN: %run %t5 2>&1 | FileCheck --check-prefix=NCFI %s // RUN: %clangxx_cfi_diag -o %t6 %s -// RUN: %run %t6 2>&1 | FileCheck --check-prefix=CFI-DIAG %s +// RUN: %interleave_diag %run %t6 2>&1 | FileCheck --check-prefix=CFI-DIAG %s // Tests that the CFI enforcement also applies to virtual destructor calls made // via 'delete'. Index: compiler-rt/test/lit.common.configured.in =================================================================== --- compiler-rt/test/lit.common.configured.in +++ compiler-rt/test/lit.common.configured.in @@ -36,6 +36,7 @@ set_default("use_thinlto", False) set_default("use_lto", config.use_thinlto) set_default("use_newpm", False) +set_default("use_interleaving", False) set_default("android", @ANDROID_PYBOOL@) set_default("android_ndk_version", @ANDROID_NDK_VERSION@) set_default("android_serial", "@ANDROID_SERIAL_FOR_TESTING@")