Index: clang/include/clang/Driver/CC1Options.td =================================================================== --- clang/include/clang/Driver/CC1Options.td +++ clang/include/clang/Driver/CC1Options.td @@ -351,7 +351,10 @@ HelpText<"Prints debug information for the new pass manager">; def fno_debug_pass_manager : Flag<["-"], "fno-debug-pass-manager">, HelpText<"Disables debug printing for the new pass manager">; - +def enable_vtable_interleaving : Flag<["-"], "enable-vtable-interleaving">, + HelpText<"Enable VTable interleaving">; +def disable_vtable_interleaving : Flag<["-"], "disable-vtable-interleaving">, + HelpText<"Disable VTable interleaving">; //===----------------------------------------------------------------------===// // Dependency Output Options //===----------------------------------------------------------------------===// Index: clang/include/clang/Frontend/CodeGenOptions.h =================================================================== --- clang/include/clang/Frontend/CodeGenOptions.h +++ clang/include/clang/Frontend/CodeGenOptions.h @@ -268,6 +268,11 @@ /// Set of XRay instrumentation kinds to emit. XRayInstrSet XRayInstrumentationBundle; + /// When this flag is true, ItaniumCXXABI will generate a global variable + /// for each referenced offset of vtables, which will be replaced once + /// the interleaving layout is decided. + bool EnableVTableInterleaving; + public: // Define accessors/mutators for code generation options of enumeration type. #define CODEGENOPT(Name, Bits, Default) Index: clang/lib/CodeGen/CGCXXABI.h =================================================================== --- clang/lib/CodeGen/CGCXXABI.h +++ clang/lib/CodeGen/CGCXXABI.h @@ -445,13 +445,14 @@ virtual void setThunkLinkage(llvm::Function *Thunk, bool ForVTable, GlobalDecl GD, bool ReturnAdjustment) = 0; - virtual llvm::Value *performThisAdjustment(CodeGenFunction &CGF, - Address This, - const ThisAdjustment &TA) = 0; + virtual llvm::Value *performThisAdjustment(CodeGenFunction &CGF, Address This, + const ThisAdjustment &TA, + const CXXRecordDecl *RD) = 0; virtual llvm::Value *performReturnAdjustment(CodeGenFunction &CGF, Address Ret, - const ReturnAdjustment &RA) = 0; + const ReturnAdjustment &RA, + const CXXRecordDecl *RD) = 0; virtual void EmitReturnFromThunk(CodeGenFunction &CGF, RValue RV, QualType ResultType); Index: clang/lib/CodeGen/CGClass.cpp =================================================================== --- clang/lib/CodeGen/CGClass.cpp +++ clang/lib/CodeGen/CGClass.cpp @@ -2759,7 +2759,8 @@ } llvm::Value *CodeGenFunction::EmitVTableTypeCheckedLoad( - const CXXRecordDecl *RD, llvm::Value *VTable, uint64_t VTableByteOffset) { + const CXXRecordDecl *RD, llvm::Value *CastedVTable, llvm::Type *ReturnType, + llvm::Value *VTableByteOffset) { SanitizerScope SanScope(this); EmitSanitizerStatReport(llvm::SanStat_CFI_VCall); @@ -2768,19 +2769,16 @@ CGM.CreateMetadataIdentifierForType(QualType(RD->getTypeForDecl(), 0)); llvm::Value *TypeId = llvm::MetadataAsValue::get(CGM.getLLVMContext(), MD); - llvm::Value *CastedVTable = Builder.CreateBitCast(VTable, Int8PtrTy); - llvm::Value *CheckedLoad = Builder.CreateCall( - CGM.getIntrinsic(llvm::Intrinsic::type_checked_load), - {CastedVTable, llvm::ConstantInt::get(Int32Ty, VTableByteOffset), - TypeId}); + llvm::Value *CheckedLoad = + Builder.CreateCall(CGM.getIntrinsic(llvm::Intrinsic::type_checked_load), + {CastedVTable, VTableByteOffset, TypeId}); llvm::Value *CheckResult = Builder.CreateExtractValue(CheckedLoad, 1); EmitCheck(std::make_pair(CheckResult, SanitizerKind::CFIVCall), SanitizerHandler::CFICheckFail, nullptr, nullptr); - return Builder.CreateBitCast( - Builder.CreateExtractValue(CheckedLoad, 0), - cast(VTable->getType())->getElementType()); + return Builder.CreateBitCast(Builder.CreateExtractValue(CheckedLoad, 0), + ReturnType); } void CodeGenFunction::EmitForwardingCallToLambda( Index: clang/lib/CodeGen/CGVTables.cpp =================================================================== --- clang/lib/CodeGen/CGVTables.cpp +++ clang/lib/CodeGen/CGVTables.cpp @@ -90,9 +90,8 @@ auto ClassDecl = ResultType->getPointeeType()->getAsCXXRecordDecl(); auto ClassAlign = CGF.CGM.getClassPointerAlignment(ClassDecl); - ReturnValue = CGF.CGM.getCXXABI().performReturnAdjustment(CGF, - Address(ReturnValue, ClassAlign), - Thunk.Return); + ReturnValue = CGF.CGM.getCXXABI().performReturnAdjustment( + CGF, Address(ReturnValue, ClassAlign), Thunk.Return, ClassDecl); if (NullCheckValue) { CGF.Builder.CreateBr(AdjustEnd); @@ -158,6 +157,7 @@ const CGFunctionInfo &FnInfo, GlobalDecl GD, const ThunkInfo &Thunk) { const CXXMethodDecl *MD = cast(GD.getDecl()); + const CXXRecordDecl *RD = MD->getParent(); const FunctionProtoType *FPT = MD->getType()->getAs(); QualType ResultType = FPT->getReturnType(); @@ -201,7 +201,7 @@ // Adjust "this", if necessary. Builder.SetInsertPoint(&*ThisStore); llvm::Value *AdjustedThisPtr = - CGM.getCXXABI().performThisAdjustment(*this, ThisPtr, Thunk.This); + CGM.getCXXABI().performThisAdjustment(*this, ThisPtr, Thunk.This, RD); ThisStore->setOperand(0, AdjustedThisPtr); if (!Thunk.Return.isEmpty()) { @@ -285,12 +285,13 @@ assert(isa(CurGD.getDecl()) && "Please use a new CGF for this thunk"); const CXXMethodDecl *MD = cast(CurGD.getDecl()); + const CXXRecordDecl *RD = MD->getParent(); // Adjust the 'this' pointer if necessary llvm::Value *AdjustedThisPtr = - Thunk ? CGM.getCXXABI().performThisAdjustment( - *this, LoadCXXThisAddress(), Thunk->This) - : LoadCXXThis(); + Thunk ? CGM.getCXXABI().performThisAdjustment(*this, LoadCXXThisAddress(), + Thunk->This, RD) + : LoadCXXThis(); if (CurFnInfo->usesInAlloca() || IsUnprototyped) { // We don't handle return adjusting thunks, because they require us to call @@ -1052,6 +1053,12 @@ // Create type metadata for the address point. AddVTableTypeMetadata(VTable, PointerWidth * AP.second, AP.first); + // When vtable interleaving is enabled, we don't emit type metadata for + // virtual member functions because the interleaving pass uses type metadata + // to determine address points of vtables. + if (getCodeGenOpts().EnableVTableInterleaving) + continue; + // The class associated with each address point could also potentially be // used for indirect calls via a member function pointer, so we need to // annotate the address of each function pointer with the appropriate member Index: clang/lib/CodeGen/CodeGenFunction.h =================================================================== --- clang/lib/CodeGen/CodeGenFunction.h +++ clang/lib/CodeGen/CodeGenFunction.h @@ -1938,9 +1938,12 @@ /// true when both vcall CFI and whole-program-vtables are enabled. bool ShouldEmitVTableTypeCheckedLoad(const CXXRecordDecl *RD); - /// Emit a type checked load from the given vtable. - llvm::Value *EmitVTableTypeCheckedLoad(const CXXRecordDecl *RD, llvm::Value *VTable, - uint64_t VTableByteOffset); + /// Emit a type checked load from the given vtable. CastedVTable + /// is of int8ptr type and VTableByteOffset is a byte offset. + llvm::Value *EmitVTableTypeCheckedLoad(const CXXRecordDecl *RD, + llvm::Value *CastedVTable, + llvm::Type *EntryType, + llvm::Value *VTableByteOffset); /// EnterDtorCleanups - Enter the cleanups necessary to complete the /// given phase of destruction for a destructor. The end result Index: clang/lib/CodeGen/CodeGenModule.cpp =================================================================== --- clang/lib/CodeGen/CodeGenModule.cpp +++ clang/lib/CodeGen/CodeGenModule.cpp @@ -5210,8 +5210,8 @@ /// Returns whether this module needs the "all-vtables" type identifier. bool CodeGenModule::NeedAllVtablesTypeId() const { - // Returns true if at least one of vtable-based CFI checkers is enabled and - // is not in the trapping mode. + // Returns true if vtable interleaving is disabled and at least one of + // vtable-based CFI checkers is enabled and is not in the trapping mode. return ((LangOpts.Sanitize.has(SanitizerKind::CFIVCall) && !CodeGenOpts.SanitizeTrap.has(SanitizerKind::CFIVCall)) || (LangOpts.Sanitize.has(SanitizerKind::CFINVCall) && Index: clang/lib/CodeGen/ItaniumCXXABI.cpp =================================================================== --- clang/lib/CodeGen/ItaniumCXXABI.cpp +++ clang/lib/CodeGen/ItaniumCXXABI.cpp @@ -25,16 +25,19 @@ #include "CodeGenFunction.h" #include "CodeGenModule.h" #include "TargetInfo.h" -#include "clang/CodeGen/ConstantInitBuilder.h" #include "clang/AST/Mangle.h" -#include "clang/AST/Type.h" #include "clang/AST/StmtCXX.h" +#include "clang/AST/Type.h" +#include "clang/CodeGen/ConstantInitBuilder.h" #include "llvm/IR/CallSite.h" +#include "llvm/IR/Constants.h" #include "llvm/IR/DataLayout.h" #include "llvm/IR/GlobalValue.h" #include "llvm/IR/Instructions.h" #include "llvm/IR/Intrinsics.h" +#include "llvm/IR/Metadata.h" #include "llvm/IR/Value.h" +#include "llvm/Support/ErrorHandling.h" #include "llvm/Support/ScopedPrinter.h" using namespace clang; @@ -44,6 +47,10 @@ class ItaniumCXXABI : public CodeGen::CGCXXABI { /// VTables - All the vtables which have been defined. llvm::DenseMap VTables; + /// Map each entry group to the corresponding offset global variable. + /// This map is needed for vtable interleaving. + llvm::DenseMap, llvm::GlobalVariable *> + OffsetMap; protected: bool UseARMMethodPtrABI; @@ -55,12 +62,10 @@ } public: - ItaniumCXXABI(CodeGen::CodeGenModule &CGM, - bool UseARMMethodPtrABI = false, - bool UseARMGuardVarABI = false) : - CGCXXABI(CGM), UseARMMethodPtrABI(UseARMMethodPtrABI), - UseARMGuardVarABI(UseARMGuardVarABI), - Use32BitVTableOffsetABI(false) { } + ItaniumCXXABI(CodeGen::CodeGenModule &CGM, bool UseARMMethodPtrABI = false, + bool UseARMGuardVarABI = false) + : CGCXXABI(CGM), UseARMMethodPtrABI(UseARMMethodPtrABI), + UseARMGuardVarABI(UseARMGuardVarABI), Use32BitVTableOffsetABI(false) {} bool classifyReturnType(CGFunctionInfo &FI) const override; @@ -299,11 +304,17 @@ bool exportThunk() override { return true; } + llvm::Value * + performTypeAdjustment(CodeGenFunction &CGF, const CXXRecordDecl *RD, + Address InitialPtr, int64_t NonVirtualAdjustment, + int64_t VirtualAdjustment, bool IsReturnAdjustment); llvm::Value *performThisAdjustment(CodeGenFunction &CGF, Address This, - const ThisAdjustment &TA) override; + const ThisAdjustment &TA, + const CXXRecordDecl *RD) override; llvm::Value *performReturnAdjustment(CodeGenFunction &CGF, Address Ret, - const ReturnAdjustment &RA) override; + const ReturnAdjustment &RA, + const CXXRecordDecl *RD) override; size_t getSrcArgforCopyCtor(const CXXConstructorDecl *, FunctionArgList &Args) const override { @@ -345,6 +356,14 @@ bool NeedsVTTParameter(GlobalDecl GD) override; + bool shouldInterleaveVTables(const CXXRecordDecl *RD); + llvm::Constant *getOffsetConstant(const CXXRecordDecl *RD, int64_t Offset, + llvm::Type *OffsetType); + llvm::Value *getVTableEntryPointer(CodeGenFunction &CGF, + const CXXRecordDecl *RD, + llvm::Value *VTable, int64_t Offset, + const llvm::Twine &Name = ""); + /**************************** RTTI Uniqueness ******************************/ protected: @@ -431,9 +450,9 @@ class ARMCXXABI : public ItaniumCXXABI { public: - ARMCXXABI(CodeGen::CodeGenModule &CGM) : - ItaniumCXXABI(CGM, /* UseARMMethodPtrABI = */ true, - /* UseARMGuardVarABI = */ true) {} + ARMCXXABI(CodeGen::CodeGenModule &CGM) + : ItaniumCXXABI(CGM, /* UseARMMethodPtrABI = */ true, + /* UseARMGuardVarABI = */ true) {} bool HasThisReturn(GlobalDecl GD) const override { return (isa(GD.getDecl()) || ( @@ -501,7 +520,8 @@ /* UseARMGuardVarABI = */ true); case TargetCXXABI::GenericMIPS: - return new ItaniumCXXABI(CGM, /* UseARMMethodPtrABI = */ true); + return new ItaniumCXXABI(CGM, /* UseARMMethodPtrABI = */ true, + /* UseARMGuardVarABI = */ false); case TargetCXXABI::WebAssembly: return new WebAssemblyCXXABI(CGM); @@ -515,7 +535,8 @@ return new ItaniumCXXABI(CGM, /* UseARMMethodPtrABI = */ true, /* UseARMGuardVarABI = */ false); } - return new ItaniumCXXABI(CGM); + return new ItaniumCXXABI(CGM, /* UseARMMethodPtrABI = */ false, + /* UseARMGuardVarABI = */ false); case TargetCXXABI::Microsoft: llvm_unreachable("Microsoft ABI is not Itanium-based"); @@ -630,7 +651,8 @@ llvm::Constant *CheckSourceLocation; llvm::Constant *CheckTypeDesc; bool ShouldEmitCFICheck = CGF.SanOpts.has(SanitizerKind::CFIMFCall) && - CGM.HasHiddenLTOVisibility(RD); + CGM.HasHiddenLTOVisibility(RD) && + !CGM.getCodeGenOpts().EnableVTableInterleaving; if (ShouldEmitCFICheck) { CodeGenFunction::SanitizerScope SanScope(&CGF); @@ -900,6 +922,76 @@ return BuildMemberPointer(MD, CharUnits::Zero()); } +/// We should only interleave vtables when the module has the hidden +/// LTO visibility, cfi-vcall is enabled and EnableVTableInterleaving +/// is set. +bool ItaniumCXXABI::shouldInterleaveVTables(const CXXRecordDecl *RD) { + return CGM.HasHiddenLTOVisibility(RD) && + CGM.getLangOpts().Sanitize.has(SanitizerKind::CFIVCall) && + CGM.getCodeGenOpts().EnableVTableInterleaving; +} + +/// When interleaving is disabled, this function simply returns +/// a constant of the offset. Otherwise, it create or returns the +/// offset placeholder of the specified type for the entry group identified +/// by the pair (TypeId, Offset). An entry group is a set of vtable entries +/// that must have the same offset from their corresponding address points. +/// To interleave vtables, we replace offsets into vtables with offset +/// placeholders, which will be replaced with real offsets once the interleaved +/// layout is decided. +llvm::Constant *ItaniumCXXABI::getOffsetConstant(const CXXRecordDecl *RD, + int64_t Offset, + llvm::Type *OffsetType) { + if (!shouldInterleaveVTables(RD)) + return llvm::ConstantInt::get(OffsetType, Offset, true); + + llvm::Metadata *TypeId = + CGM.CreateMetadataIdentifierForType(QualType(RD->getTypeForDecl(), 0)); + std::pair P = std::make_pair(TypeId, Offset); + + // The offset global variable for the entry group. + llvm::GlobalVariable *OffsetGV; + if (OffsetMap.find(P) != OffsetMap.end()) + OffsetGV = OffsetMap[P]; + else { + // If we haven't seen this offset, create an offset global variable for it. + llvm::Type *GVType = llvm::ArrayType::get(CGM.Int32Ty, 0); + // The name of a offset global variable has the format "__[type + // id]$[offset]". + std::string Name = + "__" + RD->getNameAsString() + "$" + std::to_string(Offset); + OffsetGV = new llvm::GlobalVariable( + CGM.getModule(), GVType, + /*Constant=*/true, llvm::GlobalValue::InternalLinkage, + llvm::Constant::getNullValue(GVType), Name); + + // Add offset.type metadata to the newly created offset global variable. + OffsetGV->addMetadata( + llvm::LLVMContext::MD_offset_type, + *llvm::MDTuple::get( + CGM.getLLVMContext(), + {TypeId, llvm::ConstantAsMetadata::get( + llvm::ConstantInt::get(CGM.Int64Ty, Offset))})); + + OffsetMap[P] = OffsetGV; + } + return llvm::ConstantExpr::getPtrToInt(OffsetGV, OffsetType); +} + +/// Return a pointer to the specified vtable entry. When vtable interleaving +/// is enabled, the calculation of this pointer is based on a vtable offset +/// placeholder, which will be replaced when the interleaved layout is decided. +llvm::Value *ItaniumCXXABI::getVTableEntryPointer(CodeGenFunction &CGF, + const CXXRecordDecl *RD, + llvm::Value *VTablePtr, + int64_t Offset, + const llvm::Twine &Name) { + llvm::Constant *OffsetConstant = getOffsetConstant(RD, Offset, CGM.Int64Ty); + // Use the placeholder to calculate the pointer to the entry. + return CGF.Builder.CreateInBoundsGEP(nullptr, VTablePtr, OffsetConstant, + Name); +} + llvm::Constant *ItaniumCXXABI::BuildMemberPointer(const CXXMethodDecl *MD, CharUnits ThisAdjustment) { assert(MD->isInstance() && "Member function must not be static!"); @@ -923,7 +1015,8 @@ // least significant bit of adj then makes exactly the same // discrimination as the least significant bit of ptr does for // Itanium. - MemPtr[0] = llvm::ConstantInt::get(CGM.PtrDiffTy, VTableOffset); + MemPtr[0] = getOffsetConstant(MD->getParent(), (int64_t)VTableOffset, + CGM.PtrDiffTy); MemPtr[1] = llvm::ConstantInt::get(CGM.PtrDiffTy, 2 * ThisAdjustment.getQuantity() + 1); } else { @@ -931,7 +1024,14 @@ // For a virtual function, [the pointer field] is 1 plus the // virtual table offset (in bytes) of the function, // represented as a ptrdiff_t. - MemPtr[0] = llvm::ConstantInt::get(CGM.PtrDiffTy, VTableOffset + 1); + const CXXRecordDecl *RD = MD->getParent(); + if (shouldInterleaveVTables(RD)) { + llvm::Constant *Placeholder = + getOffsetConstant(RD, (int64_t)VTableOffset, CGM.PtrDiffTy); + llvm::Constant *ConstantOne = llvm::ConstantInt::get(CGM.PtrDiffTy, 1); + MemPtr[0] = llvm::ConstantExpr::getAdd(Placeholder, ConstantOne); + } else + MemPtr[0] = llvm::ConstantInt::get(CGM.PtrDiffTy, VTableOffset + 1); MemPtr[1] = llvm::ConstantInt::get(CGM.PtrDiffTy, ThisAdjustment.getQuantity()); } @@ -1426,9 +1526,9 @@ CGM.getItaniumVTableContext().getVirtualBaseOffsetOffset(ClassDecl, BaseClassDecl); - llvm::Value *VBaseOffsetPtr = - CGF.Builder.CreateConstGEP1_64(VTablePtr, VBaseOffsetOffset.getQuantity(), - "vbase.offset.ptr"); + llvm::Value *VBaseOffsetPtr = getVTableEntryPointer( + CGF, ClassDecl, VTablePtr, VBaseOffsetOffset.getQuantity(), + "vbase.offset.ptr"); VBaseOffsetPtr = CGF.Builder.CreateBitCast(VBaseOffsetPtr, CGM.PtrDiffTy->getPointerTo()); @@ -1649,8 +1749,13 @@ llvm::ConstantInt::get(CGM.Int32Ty, AddressPoint.AddressPointIndex), }; + // The address point calculated by vtable_start + offset where vtable_start is + // the base address of the gep. When interleaving is enabled, we replace + // vtable_start with the new address point in the interleaved layout minus + // offset, which may not be an in bound address of the object. + bool InBounds = shouldInterleaveVTables(VTableClass) ? false : true; return llvm::ConstantExpr::getGetElementPtr(VTable->getValueType(), VTable, - Indices, /*InBounds=*/true, + Indices, /*InBounds=*/InBounds, /*InRangeIndex=*/1); } @@ -1717,21 +1822,25 @@ Address This, llvm::Type *Ty, SourceLocation Loc) { - Ty = Ty->getPointerTo()->getPointerTo(); auto *MethodDecl = cast(GD.getDecl()); - llvm::Value *VTable = CGF.GetVTablePtr(This, Ty, MethodDecl->getParent()); + auto *RecordDecl = MethodDecl->getParent(); + llvm::Value *VTable = CGF.GetVTablePtr(This, CGM.Int8PtrTy, RecordDecl); uint64_t VTableIndex = CGM.getItaniumVTableContext().getMethodVTableIndex(GD); + int64_t ByteOffset = + VTableIndex * CGM.getContext().getTargetInfo().getPointerWidth(0) / 8; + llvm::Value *OffsetConstant = + getOffsetConstant(RecordDecl, ByteOffset, CGM.Int32Ty); llvm::Value *VFunc; - if (CGF.ShouldEmitVTableTypeCheckedLoad(MethodDecl->getParent())) { - VFunc = CGF.EmitVTableTypeCheckedLoad( - MethodDecl->getParent(), VTable, - VTableIndex * CGM.getContext().getTargetInfo().getPointerWidth(0) / 8); - } else { - CGF.EmitTypeMetadataCodeForVCall(MethodDecl->getParent(), VTable, Loc); - + if (CGF.ShouldEmitVTableTypeCheckedLoad(RecordDecl)) + VFunc = CGF.EmitVTableTypeCheckedLoad(RecordDecl, VTable, + Ty->getPointerTo(), OffsetConstant); + else { + CGF.EmitTypeMetadataCodeForVCall(RecordDecl, VTable, Loc); llvm::Value *VFuncPtr = - CGF.Builder.CreateConstInBoundsGEP1_64(VTable, VTableIndex, "vfn"); + CGF.Builder.CreateInBoundsGEP(nullptr, VTable, OffsetConstant, "vfn"); + VFuncPtr = + CGF.Builder.CreateBitCast(VFuncPtr, Ty->getPointerTo()->getPointerTo()); auto *VFuncLoad = CGF.Builder.CreateAlignedLoad(VFuncPtr, CGF.getPointerAlign()); @@ -1798,11 +1907,13 @@ // can emit definition of the inline functions. return !hasAnyUnusedVirtualInlineFunction(RD); } -static llvm::Value *performTypeAdjustment(CodeGenFunction &CGF, - Address InitialPtr, - int64_t NonVirtualAdjustment, - int64_t VirtualAdjustment, - bool IsReturnAdjustment) { + +llvm::Value *ItaniumCXXABI::performTypeAdjustment(CodeGenFunction &CGF, + const CXXRecordDecl *RD, + Address InitialPtr, + int64_t NonVirtualAdjustment, + int64_t VirtualAdjustment, + bool IsReturnAdjustment) { if (!NonVirtualAdjustment && !VirtualAdjustment) return InitialPtr.getPointer(); @@ -1822,10 +1933,8 @@ Address VTablePtrPtr = CGF.Builder.CreateElementBitCast(V, CGF.Int8PtrTy); llvm::Value *VTablePtr = CGF.Builder.CreateLoad(VTablePtrPtr); - llvm::Value *OffsetPtr = - CGF.Builder.CreateConstInBoundsGEP1_64(VTablePtr, VirtualAdjustment); - + getVTableEntryPointer(CGF, RD, VTablePtr, VirtualAdjustment); OffsetPtr = CGF.Builder.CreateBitCast(OffsetPtr, PtrDiffTy->getPointerTo()); // Load the adjustment offset from the vtable. @@ -1851,16 +1960,18 @@ llvm::Value *ItaniumCXXABI::performThisAdjustment(CodeGenFunction &CGF, Address This, - const ThisAdjustment &TA) { - return performTypeAdjustment(CGF, This, TA.NonVirtual, + const ThisAdjustment &TA, + const CXXRecordDecl *RD) { + return performTypeAdjustment(CGF, RD, This, TA.NonVirtual, TA.Virtual.Itanium.VCallOffsetOffset, /*IsReturnAdjustment=*/false); } -llvm::Value * -ItaniumCXXABI::performReturnAdjustment(CodeGenFunction &CGF, Address Ret, - const ReturnAdjustment &RA) { - return performTypeAdjustment(CGF, Ret, RA.NonVirtual, +llvm::Value *ItaniumCXXABI::performReturnAdjustment(CodeGenFunction &CGF, + Address Ret, + const ReturnAdjustment &RA, + const CXXRecordDecl *RD) { + return performTypeAdjustment(CGF, RD, Ret, RA.NonVirtual, RA.Virtual.Itanium.VBaseOffsetOffset, /*IsReturnAdjustment=*/true); } @@ -2617,7 +2728,7 @@ class ItaniumRTTIBuilder { CodeGenModule &CGM; // Per-module state. llvm::LLVMContext &VMContext; - const ItaniumCXXABI &CXXABI; // Per-module state. + ItaniumCXXABI &CXXABI; // Per-module state. /// Fields - The fields of the RTTI descriptor currently being built. SmallVector Fields; @@ -2637,6 +2748,12 @@ /// inheritance, according to the Itanium C++ ABI, 2.9.5p6b. void BuildSIClassTypeInfo(const CXXRecordDecl *RD); + /// GetOffsetFlag - Build a constant representing __offset_flags. When vtable + /// interleaving is enabled, this constant will be constructed from the + /// corresponding offset placeholder. + llvm::Constant *GetOffsetFlag(const CXXRecordDecl *RD, int64_t Offset, + uint64_t LowerBits, llvm::Type *OffsetFlagsTy); + /// BuildVMIClassTypeInfo - Build an abi::__vmi_class_type_info, used for /// classes with bases that do not satisfy the abi::__si_class_type_info /// constraints, according ti the Itanium C++ ABI, 2.9.5p5c. @@ -2655,7 +2772,7 @@ void BuildPointerToMemberTypeInfo(const MemberPointerType *Ty); public: - ItaniumRTTIBuilder(const ItaniumCXXABI &ABI) + ItaniumRTTIBuilder(ItaniumCXXABI &ABI) : CGM(ABI.CGM), VMContext(CGM.getModule().getContext()), CXXABI(ABI) {} // Pointer type info flags. @@ -3010,6 +3127,11 @@ static const char * const VMIClassTypeInfo = "_ZTVN10__cxxabiv121__vmi_class_type_infoE"; + llvm::Type *VTableTy1 = + llvm::StructType::get(llvm::ArrayType::get(CGM.Int8PtrTy, 7)); + llvm::Type *VTableTy2 = + llvm::StructType::get(llvm::ArrayType::get(CGM.Int8PtrTy, 10)); + llvm::Type *CurVTableTy = nullptr; const char *VTableName = nullptr; switch (Ty->getTypeClass()) { @@ -3042,6 +3164,7 @@ case Type::BlockPointer: // abi::__fundamental_type_info. VTableName = "_ZTVN10__cxxabiv123__fundamental_type_infoE"; + CurVTableTy = VTableTy1; break; case Type::ConstantArray: @@ -3049,17 +3172,20 @@ case Type::VariableArray: // abi::__array_type_info. VTableName = "_ZTVN10__cxxabiv117__array_type_infoE"; + CurVTableTy = VTableTy1; break; case Type::FunctionNoProto: case Type::FunctionProto: // abi::__function_type_info. VTableName = "_ZTVN10__cxxabiv120__function_type_infoE"; + CurVTableTy = VTableTy1; break; case Type::Enum: // abi::__enum_type_info. VTableName = "_ZTVN10__cxxabiv116__enum_type_infoE"; + CurVTableTy = VTableTy1; break; case Type::Record: { @@ -3073,7 +3199,7 @@ } else { VTableName = VMIClassTypeInfo; } - + CurVTableTy = VTableTy2; break; } @@ -3084,6 +3210,7 @@ // Handle id and Class. if (isa(Ty)) { VTableName = ClassTypeInfo; + CurVTableTy = VTableTy2; break; } @@ -3096,33 +3223,52 @@ } else { VTableName = ClassTypeInfo; } + CurVTableTy = VTableTy2; break; case Type::ObjCObjectPointer: case Type::Pointer: // abi::__pointer_type_info. VTableName = "_ZTVN10__cxxabiv119__pointer_type_infoE"; + CurVTableTy = VTableTy1; break; case Type::MemberPointer: // abi::__pointer_to_member_type_info. VTableName = "_ZTVN10__cxxabiv129__pointer_to_member_type_infoE"; + CurVTableTy = VTableTy1; break; } - llvm::Constant *VTable = - CGM.getModule().getOrInsertGlobal(VTableName, CGM.Int8PtrTy); - CGM.setDSOLocal(cast(VTable->stripPointerCasts())); - - llvm::Type *PtrDiffTy = - CGM.getTypes().ConvertType(CGM.getContext().getPointerDiffType()); + llvm::Constant *VTable; + // FIXME When interleaving is enabled, we assume that + // libc++abi is used and we use vtable types for typeinfo classes + // defined in libc++abi. + if (CGM.getCodeGenOpts().EnableVTableInterleaving) { + VTable = CGM.getModule().getOrInsertGlobal(VTableName, CurVTableTy); + CGM.setDSOLocal(cast(VTable->stripPointerCasts())); + llvm::Value *Indices[] = { + llvm::ConstantInt::get(CGM.Int32Ty, 0), + llvm::ConstantInt::get(CGM.Int32Ty, 0), + llvm::ConstantInt::get(CGM.Int32Ty, 2), + }; + VTable = llvm::ConstantExpr::getGetElementPtr(CurVTableTy, VTable, Indices, + /*InBounds=*/true, + /*InRangeIndex=*/1); + VTable = llvm::ConstantExpr::getBitCast(VTable, CGM.Int8PtrTy); + } else { + VTable = CGM.getModule().getOrInsertGlobal(VTableName, CGM.Int8PtrTy); + CGM.setDSOLocal(cast(VTable->stripPointerCasts())); - // The vtable address point is 2. - llvm::Constant *Two = llvm::ConstantInt::get(PtrDiffTy, 2); - VTable = - llvm::ConstantExpr::getInBoundsGetElementPtr(CGM.Int8PtrTy, VTable, Two); - VTable = llvm::ConstantExpr::getBitCast(VTable, CGM.Int8PtrTy); + llvm::Type *PtrDiffTy = + CGM.getTypes().ConvertType(CGM.getContext().getPointerDiffType()); + // The vtable address point is 2. + llvm::Constant *Two = llvm::ConstantInt::get(PtrDiffTy, 2); + VTable = llvm::ConstantExpr::getInBoundsGetElementPtr(CGM.Int8PtrTy, VTable, + Two); + VTable = llvm::ConstantExpr::getBitCast(VTable, CGM.Int8PtrTy); + } Fields.push_back(VTable); } @@ -3494,6 +3640,26 @@ return Flags; } +llvm::Constant *ItaniumRTTIBuilder::GetOffsetFlag(const CXXRecordDecl *RD, + int64_t Offset, + uint64_t LowerBits, + llvm::Type *OffsetFlagsTy) { + if (!CXXABI.shouldInterleaveVTables(RD)) { + uint64_t UpperBits = (uint64_t)Offset << 8; + uint64_t OffsetFlags = UpperBits | LowerBits; + return llvm::ConstantInt::get(OffsetFlagsTy, OffsetFlags); + } + + llvm::Constant *OffsetConstant = + CXXABI.getOffsetConstant(RD, Offset, OffsetFlagsTy); + llvm::Constant *ShiftAmount = llvm::ConstantInt::get(OffsetFlagsTy, 8); + llvm::Constant *UpperBitsConstant = + llvm::ConstantExpr::getNSWShl(OffsetConstant, ShiftAmount); + llvm::Constant *LowerBitsConstant = + llvm::ConstantInt::get(OffsetFlagsTy, LowerBits); + return llvm::ConstantExpr::getOr(UpperBitsConstant, LowerBitsConstant); +} + /// BuildVMIClassTypeInfo - Build an abi::__vmi_class_type_info, used for /// classes with bases that do not satisfy the abi::__si_class_type_info /// constraints, according ti the Itanium C++ ABI, 2.9.5p5c. @@ -3553,8 +3719,6 @@ const CXXRecordDecl *BaseDecl = cast(Base.getType()->getAs()->getDecl()); - int64_t OffsetFlags = 0; - // All but the lower 8 bits of __offset_flags are a signed offset. // For a non-virtual base, this is the offset in the object of the base // subobject. For a virtual base, this is the offset in the virtual table of @@ -3568,16 +3732,17 @@ Offset = Layout.getBaseClassOffset(BaseDecl); }; - OffsetFlags = uint64_t(Offset.getQuantity()) << 8; + int64_t OffsetInt = Offset.getQuantity(); + uint64_t Flags = 0; // The low-order byte of __offset_flags contains flags, as given by the // masks from the enumeration __offset_flags_masks. if (Base.isVirtual()) - OffsetFlags |= BCTI_Virtual; + Flags |= BCTI_Virtual; if (Base.getAccessSpecifier() == AS_public) - OffsetFlags |= BCTI_Public; + Flags |= BCTI_Public; - Fields.push_back(llvm::ConstantInt::get(OffsetFlagsLTy, OffsetFlags)); + Fields.push_back(GetOffsetFlag(RD, OffsetInt, Flags, OffsetFlagsLTy)); } } Index: clang/lib/CodeGen/MicrosoftCXXABI.cpp =================================================================== --- clang/lib/CodeGen/MicrosoftCXXABI.cpp +++ clang/lib/CodeGen/MicrosoftCXXABI.cpp @@ -378,10 +378,12 @@ bool exportThunk() override { return false; } llvm::Value *performThisAdjustment(CodeGenFunction &CGF, Address This, - const ThisAdjustment &TA) override; + const ThisAdjustment &TA, + const CXXRecordDecl *RD) override; llvm::Value *performReturnAdjustment(CodeGenFunction &CGF, Address Ret, - const ReturnAdjustment &RA) override; + const ReturnAdjustment &RA, + const CXXRecordDecl *RD) override; void EmitThreadLocalInitFuncs( CodeGenModule &CGM, ArrayRef CXXThreadLocals, @@ -1835,13 +1837,14 @@ llvm::Type *Ty, SourceLocation Loc) { CGBuilderTy &Builder = CGF.Builder; - - Ty = Ty->getPointerTo()->getPointerTo(); + llvm::Type *VTablePtrTy = Ty->getPointerTo()->getPointerTo(); + llvm::Type *FuncPtrTy = Ty->getPointerTo(); Address VPtr = adjustThisArgumentForVirtualFunctionCall(CGF, GD, This, true); auto *MethodDecl = cast(GD.getDecl()); - llvm::Value *VTable = CGF.GetVTablePtr(VPtr, Ty, MethodDecl->getParent()); + llvm::Value *VTable = + CGF.GetVTablePtr(VPtr, VTablePtrTy, MethodDecl->getParent()); MicrosoftVTableContext &VFTContext = CGM.getMicrosoftVTableContext(); MethodVFTableLocation ML = VFTContext.getMethodVFTableLocation(GD); @@ -1860,9 +1863,13 @@ llvm::Value *VFunc; if (CGF.ShouldEmitVTableTypeCheckedLoad(MethodDecl->getParent())) { - VFunc = CGF.EmitVTableTypeCheckedLoad( - getObjectWithVPtr(), VTable, - ML.Index * CGM.getContext().getTargetInfo().getPointerWidth(0) / 8); + int64_t ByteOffset = + ML.Index * CGM.getContext().getTargetInfo().getPointerWidth(0) / 8; + llvm::Value *OffsetConstant = + llvm::ConstantInt::get(CGM.Int32Ty, ByteOffset); + llvm::Value *CastedVTable = Builder.CreateBitCast(VTable, CGM.Int8PtrTy); + VFunc = CGF.EmitVTableTypeCheckedLoad(getObjectWithVPtr(), CastedVTable, + FuncPtrTy, OffsetConstant); } else { if (CGM.getCodeGenOpts().PrepareForLTO) CGF.EmitTypeMetadataCodeForVCall(getObjectWithVPtr(), VTable, Loc); @@ -2092,7 +2099,8 @@ llvm::Value *MicrosoftCXXABI::performThisAdjustment(CodeGenFunction &CGF, Address This, - const ThisAdjustment &TA) { + const ThisAdjustment &TA, + const CXXRecordDecl *RD) { if (TA.isEmpty()) return This.getPointer(); @@ -2144,7 +2152,8 @@ llvm::Value * MicrosoftCXXABI::performReturnAdjustment(CodeGenFunction &CGF, Address Ret, - const ReturnAdjustment &RA) { + const ReturnAdjustment &RA, + const CXXRecordDecl *RD) { if (RA.isEmpty()) return Ret.getPointer(); Index: clang/lib/Frontend/CompilerInvocation.cpp =================================================================== --- clang/lib/Frontend/CompilerInvocation.cpp +++ clang/lib/Frontend/CompilerInvocation.cpp @@ -1147,6 +1147,7 @@ Opts.SpeculativeLoadHardening = Args.hasArg(OPT_mspeculative_load_hardening); + Opts.EnableVTableInterleaving = Args.hasArg(OPT_enable_vtable_interleaving); return Success; } Index: clang/test/CodeGen/tbaa-for-vptr.cpp =================================================================== --- clang/test/CodeGen/tbaa-for-vptr.cpp +++ clang/test/CodeGen/tbaa-for-vptr.cpp @@ -1,10 +1,10 @@ -// RUN: %clang_cc1 -triple %itanium_abi_triple -emit-llvm -o - -fsanitize=thread %s | FileCheck %s --check-prefixes=CHECK,OLD-PATH -// RUN: %clang_cc1 -triple %itanium_abi_triple -emit-llvm -o - -O1 %s | FileCheck %s --check-prefixes=CHECK,OLD-PATH -// RUN: %clang_cc1 -triple %itanium_abi_triple -emit-llvm -o - -O1 -relaxed-aliasing -fsanitize=thread %s | FileCheck %s --check-prefixes=CHECK,OLD-PATH +// RUN: %clang_cc1 -triple %itanium_abi_triple -emit-llvm -o - -fsanitize=thread %s | FileCheck %s --check-prefixes=CHECK,OLD-PATH,UNOPTIMIZED +// RUN: %clang_cc1 -triple %itanium_abi_triple -emit-llvm -o - -O1 %s | FileCheck %s --check-prefixes=CHECK,OLD-PATH,OPTIMIZED +// RUN: %clang_cc1 -triple %itanium_abi_triple -emit-llvm -o - -O1 -relaxed-aliasing -fsanitize=thread %s | FileCheck %s --check-prefixes=CHECK,OLD-PATH,OPTIMIZED // -// RUN: %clang_cc1 -triple %itanium_abi_triple -emit-llvm -new-struct-path-tbaa -o - -fsanitize=thread %s | FileCheck %s --check-prefixes=CHECK,NEW-PATH -// RUN: %clang_cc1 -triple %itanium_abi_triple -emit-llvm -new-struct-path-tbaa -o - -O1 %s | FileCheck %s --check-prefixes=CHECK,NEW-PATH -// RUN: %clang_cc1 -triple %itanium_abi_triple -emit-llvm -new-struct-path-tbaa -o - -O1 -relaxed-aliasing -fsanitize=thread %s | FileCheck %s --check-prefixes=CHECK,NEW-PATH +// RUN: %clang_cc1 -triple %itanium_abi_triple -emit-llvm -new-struct-path-tbaa -o - -fsanitize=thread %s | FileCheck %s --check-prefixes=CHECK,NEW-PATH,UNOPTIMIZED +// RUN: %clang_cc1 -triple %itanium_abi_triple -emit-llvm -new-struct-path-tbaa -o - -O1 %s | FileCheck %s --check-prefixes=CHECK,NEW-PATH,OPTIMIZED +// RUN: %clang_cc1 -triple %itanium_abi_triple -emit-llvm -new-struct-path-tbaa -o - -O1 -relaxed-aliasing -fsanitize=thread %s | FileCheck %s --check-prefixes=CHECK,NEW-PATH,OPTIMIZED // // RUN: %clang_cc1 -triple %itanium_abi_triple -emit-llvm -o - %s | FileCheck %s --check-prefix=NOTBAA // RUN: %clang_cc1 -triple %itanium_abi_triple -emit-llvm -o - -O2 -relaxed-aliasing %s | FileCheck %s --check-prefix=NOTBAA @@ -27,7 +27,9 @@ } // CHECK-LABEL: @_Z7CallFoo -// CHECK: %{{.*}} = load i32 (%struct.A*)**, {{.*}} !tbaa ![[NUM:[0-9]+]] +// UNOPTIMIZED: [[R0:%[^ ]*]] = bitcast %struct.A* {{.*}} to i8** +// UNOPTIMIZED: %{{.*}} = load i8*, i8** [[R0]], {{.*}} !tbaa ![[NUM:[0-9]+]] +// OPTIMIZED: %{{.*}} = load i32 (%struct.A*)**, {{.*}} !tbaa ![[NUM:[0-9]+]] // CHECK: br i1 // CHECK: load i8*, {{.*}}, !tbaa ![[NUM]] // Index: clang/test/CodeGenCXX/alignment.cpp =================================================================== --- clang/test/CodeGenCXX/alignment.cpp +++ clang/test/CodeGenCXX/alignment.cpp @@ -226,7 +226,7 @@ // CHECK: [[B_P:%.*]] = load [[B:%.*]]*, [[B]]** // CHECK: [[VPTR_P:%.*]] = bitcast [[B]]* [[B_P]] to i8** // CHECK: [[VPTR:%.*]] = load i8*, i8** [[VPTR_P]], align 8 - // CHECK: [[T0:%.*]] = getelementptr i8, i8* [[VPTR]], i64 -24 + // CHECK: [[T0:%.*]] = getelementptr inbounds i8, i8* [[VPTR]], i64 -24 // CHECK: [[OFFSET_P:%.*]] = bitcast i8* [[T0]] to i64* // CHECK: [[OFFSET:%.*]] = load i64, i64* [[OFFSET_P]], align 8 // CHECK: [[T0:%.*]] = bitcast [[B]]* [[B_P]] to i8* @@ -280,7 +280,7 @@ // CHECK: [[RESULT:%.*]] = alloca [[ARRAY]], align 64 // CHECK: [[VPTR_P:%.*]] = bitcast [[D]]* [[D_P]] to i8** // CHECK: [[VPTR:%.*]] = load i8*, i8** [[VPTR_P]], align 16 - // CHECK: [[T0:%.*]] = getelementptr i8, i8* [[VPTR]], i64 -24 + // CHECK: [[T0:%.*]] = getelementptr inbounds i8, i8* [[VPTR]], i64 -24 // CHECK: [[OFFSET_P:%.*]] = bitcast i8* [[T0]] to i64* // CHECK: [[OFFSET:%.*]] = load i64, i64* [[OFFSET_P]], align 8 // CHECK: [[T0:%.*]] = bitcast [[D]]* [[D_P]] to i8* Index: clang/test/CodeGenCXX/arm.cpp =================================================================== --- clang/test/CodeGenCXX/arm.cpp +++ clang/test/CodeGenCXX/arm.cpp @@ -278,11 +278,12 @@ // CHECK-NEXT: [[V:%.*]] = load [[A]]*, [[A]]** [[AVAR]], align 4 // CHECK-NEXT: [[ISNULL:%.*]] = icmp eq [[A]]* [[V]], null // CHECK-NEXT: br i1 [[ISNULL]] - // CHECK: [[T0:%.*]] = bitcast [[A]]* [[V]] to void ([[A]]*)*** - // CHECK-NEXT: [[T1:%.*]] = load void ([[A]]*)**, void ([[A]]*)*** [[T0]] - // CHECK-NEXT: [[T2:%.*]] = getelementptr inbounds void ([[A]]*)*, void ([[A]]*)** [[T1]], i64 1 - // CHECK-NEXT: [[T3:%.*]] = load void ([[A]]*)*, void ([[A]]*)** [[T2]] - // CHECK-NEXT: call void [[T3]]([[A]]* [[V]]) + // CHECK: [[T0:%.*]] = bitcast [[A]]* [[V]] to i8** + // CHECK-NEXT: [[T1:%.*]] = load i8*, i8** [[T0]] + // CHECK-NEXT: [[T2:%.*]] = getelementptr inbounds i8, i8* [[T1]], i32 4 + // CHECK: [[T3:%.*]] = bitcast i8* [[T2]] to void ([[A]]*)** + // CHECK-NEXT: [[T4:%.*]] = load void ([[A]]*)*, void ([[A]]*)** [[T3]] + // CHECK-NEXT: call void [[T4]]([[A]]* [[V]]) // CHECK-NEXT: br label // CHECK: ret void delete a; Index: clang/test/CodeGenCXX/cfi-cross-dso.cpp =================================================================== --- clang/test/CodeGenCXX/cfi-cross-dso.cpp +++ clang/test/CodeGenCXX/cfi-cross-dso.cpp @@ -28,8 +28,12 @@ // MS: @[[B_VTABLE:.*]] = private unnamed_addr constant { [2 x i8*] } {{.*}}@"??_R4B@?A0x{{[^@]*}}@@6B@"{{.*}}@"?f@B@?A0x{{[^@]*}}@@UEAAXXZ" -// CHECK: %[[VT:.*]] = load void (%struct.A*)**, void (%struct.A*)*** -// CHECK: %[[VT2:.*]] = bitcast {{.*}}%[[VT]] to i8*, !nosanitize +// CHECK: %[[R0:.*]] = load %struct.A*, %struct.A** %a.addr +// ITANIUM: %[[R1:.*]] = bitcast %struct.A* %[[R0]] to i8** +// ITANIUM: %[[VT2:.*]] = load i8*, i8** %[[R1]] +// MS: %[[R1:.*]] = bitcast %struct.A* %[[R0]] to void (%struct.A*)*** +// MS: %[[VT:.*]] = load void (%struct.A*)**, void (%struct.A*)*** %[[R1]] +// MS: %[[VT2:.*]] = bitcast {{.*}}%[[VT]] to i8* // ITANIUM: %[[TEST:.*]] = call i1 @llvm.type.test(i8* %[[VT2]], metadata !"_ZTS1A"), !nosanitize // MS: %[[TEST:.*]] = call i1 @llvm.type.test(i8* %[[VT2]], metadata !"?AUA@@"), !nosanitize // CHECK: br i1 %[[TEST]], label %[[CONT:.*]], label %[[SLOW:.*]], {{.*}} !nosanitize Index: clang/test/CodeGenCXX/constructor-destructor-return-this.cpp =================================================================== --- clang/test/CodeGenCXX/constructor-destructor-return-this.cpp +++ clang/test/CodeGenCXX/constructor-destructor-return-this.cpp @@ -131,7 +131,8 @@ // Verify that virtual calls to destructors are not marked with a 'returned' // this parameter at the call site... -// CHECKARM: [[VFN:%.*]] = getelementptr inbounds %class.E* (%class.E*)*, %class.E* (%class.E*)** +// CHECKARM: [[VFPTR:%.*]] = getelementptr inbounds i8, i8* +// CHECKARM: [[VFN:%.*]] = bitcast i8* [[VFPTR]] to %class.E* (%class.E*)** // CHECKARM: [[THUNK:%.*]] = load %class.E* (%class.E*)*, %class.E* (%class.E*)** [[VFN]] // CHECKARM: call %class.E* [[THUNK]](%class.E* % Index: clang/test/CodeGenCXX/constructor-init.cpp =================================================================== --- clang/test/CodeGenCXX/constructor-init.cpp +++ clang/test/CodeGenCXX/constructor-init.cpp @@ -96,8 +96,9 @@ // CHECK-LABEL: define void @_ZN10InitVTable1BC2Ev(%"struct.InitVTable::B"* %this) unnamed_addr // CHECK: [[T0:%.*]] = bitcast [[B:%.*]]* [[THIS:%.*]] to i32 (...)*** // CHECK-NEXT: store i32 (...)** bitcast (i8** getelementptr inbounds ({ [3 x i8*] }, { [3 x i8*] }* @_ZTVN10InitVTable1BE, i32 0, inrange i32 0, i32 2) to i32 (...)**), i32 (...)*** [[T0]] - // CHECK: [[VTBL:%.*]] = load i32 ([[B]]*)**, i32 ([[B]]*)*** {{%.*}} - // CHECK-NEXT: [[FNP:%.*]] = getelementptr inbounds i32 ([[B]]*)*, i32 ([[B]]*)** [[VTBL]], i64 0 + // CHECK: [[VTBL:%.*]] = load i8*, i8** {{%.*}} + // CHECK-NEXT: [[FNBP:%.*]] = getelementptr inbounds i8, i8* [[VTBL]], i32 0 + // CHECK-NEXT: [[FNP:%.*]] = bitcast i8* [[FNBP]] to i32 ([[B]]*)** // CHECK-NEXT: [[FN:%.*]] = load i32 ([[B]]*)*, i32 ([[B]]*)** [[FNP]] // CHECK-NEXT: [[ARG:%.*]] = call i32 [[FN]]([[B]]* [[THIS]]) // CHECK-NEXT: call void @_ZN10InitVTable1AC2Ei({{.*}}* {{%.*}}, i32 [[ARG]]) Index: clang/test/CodeGenCXX/delete.cpp =================================================================== --- clang/test/CodeGenCXX/delete.cpp +++ clang/test/CodeGenCXX/delete.cpp @@ -123,10 +123,11 @@ // CHECK-NEXT: [[ALLOCATED:%.*]] = getelementptr inbounds i8, i8* [[T0]], i64 [[OFFSET]] // Load the complete-object destructor (not the deleting destructor) // and call it. - // CHECK-NEXT: [[T0:%.*]] = bitcast [[X:%.*]]* [[XP:%.*]] to void ([[X]]*)*** - // CHECK-NEXT: [[VTABLE:%.*]] = load void ([[X]]*)**, void ([[X]]*)*** [[T0]] - // CHECK-NEXT: [[T0:%.*]] = getelementptr inbounds void ([[X]]*)*, void ([[X]]*)** [[VTABLE]], i64 0 - // CHECK-NEXT: [[DTOR:%.*]] = load void ([[X]]*)*, void ([[X]]*)** [[T0]] + // CHECK-NEXT: [[T0:%.*]] = bitcast [[X:%.*]]* [[XP:%.*]] to i8** + // CHECK-NEXT: [[VTABLE:%.*]] = load i8*, i8** [[T0]] + // CHECK-NEXT: [[T0:%.*]] = getelementptr inbounds i8, i8* [[VTABLE]], i32 0 + // CHECK-NEXT: [[T1:%.*]] = bitcast i8* [[T0]] to void ([[X]]*)** + // CHECK-NEXT: [[DTOR:%.*]] = load void ([[X]]*)*, void ([[X]]*)** [[T1]] // CHECK-NEXT: call void [[DTOR]]([[X]]* [[OBJ:%.*]]) // Call the global operator delete. // CHECK-NEXT: call void @_ZdlPv(i8* [[ALLOCATED]]) [[NUW:#[0-9]+]] Index: clang/test/CodeGenCXX/devirtualize-virtual-function-calls-final.cpp =================================================================== --- clang/test/CodeGenCXX/devirtualize-virtual-function-calls-final.cpp +++ clang/test/CodeGenCXX/devirtualize-virtual-function-calls-final.cpp @@ -89,6 +89,7 @@ // FIXME: It should be possible to devirtualize this case, but that is // not implemented yet. // CHECK: getelementptr + // CHECK-NEXT: bitcast // CHECK-NEXT: %[[FUNC:.*]] = load // CHECK-NEXT: call void %[[FUNC]] static_cast(d)->f(); @@ -98,6 +99,7 @@ // FIXME: It should be possible to devirtualize this case, but that is // not implemented yet. // CHECK: getelementptr + // CHECK-NEXT: bitcast // CHECK-NEXT: %[[FUNC:.*]] = load // CHECK-NEXT: call i32 %[[FUNC]] -static_cast(*d); @@ -207,7 +209,8 @@ // CHECK: [[F_PTR_RA:%.+]] = bitcast // CHECK: [[VTABLE:%.+]] = load {{.+}} [[F_PTR_RA]] // CHECK: [[VFN:%.+]] = getelementptr inbounds {{.+}} [[VTABLE]], i{{[0-9]+}} 0 - // CHECK-NEXT: %[[FUNC:.*]] = load {{.+}} [[VFN]] + // CHECK-NEXT: [[VFP:%.*]] = bitcast {{.+}} [[VFN]] + // CHECK-NEXT: %[[FUNC:.*]] = load {{.+}} [[VFP]] // CHECK-NEXT: = call {{.*}} %[[FUNC]] return static_cast(x)->f(); } @@ -219,8 +222,9 @@ // CHECK: bitcast // CHECK: [[F_PTR_RA:%.+]] = bitcast // CHECK: [[VTABLE:%.+]] = load {{.+}} [[F_PTR_RA]] - // CHECK: [[VFN:%.+]] = getelementptr inbounds {{.+}} [[VTABLE]], i{{[0-9]+}} 1 - // CHECK-NEXT: %[[FUNC:.*]] = load {{.+}} [[VFN]] + // CHECK: [[VFN:%.+]] = getelementptr inbounds {{.+}} [[VTABLE]], i{{[0-9]+}} 4 + // CHECK-NEXT: [[VFP:%.*]] = bitcast {{.+}} [[VFN]] + // CHECK-NEXT: %[[FUNC:.*]] = load {{.+}} [[VFP]] // CHECK-NEXT: = call {{.*}} %[[FUNC]] return -static_cast(*x); } Index: clang/test/CodeGenCXX/interleaving-member-function-pointers.cpp =================================================================== --- /dev/null +++ clang/test/CodeGenCXX/interleaving-member-function-pointers.cpp @@ -0,0 +1,180 @@ +// This file tests if accesses to vbase entries are correctly generated when vtable interleaving is enabled. +// This test is based on member-function-pointers.cpp. + +// RUN: %clang_cc1 %s -flto -fvisibility hidden -fsanitize=cfi-vcall -enable-vtable-interleaving -emit-llvm -o - -triple=x86_64-unknown-unknown | FileCheck -check-prefix CODE-LP64 %s +// RUN: %clang_cc1 %s -flto -fvisibility hidden -fsanitize=cfi-vcall -enable-vtable-interleaving -emit-llvm -o - -triple=i386-unknown-unknown | FileCheck -check-prefix CODE-LP32 %s +// RUN: %clang_cc1 %s -flto -fvisibility hidden -fsanitize=cfi-vcall -enable-vtable-interleaving -emit-llvm -o - -triple=x86_64-unknown-unknown | FileCheck -check-prefix GLOBAL-LP64 %s +// RUN: %clang_cc1 %s -flto -fvisibility hidden -fsanitize=cfi-vcall -enable-vtable-interleaving -emit-llvm -o - -triple=i386-unknown-unknown | FileCheck -check-prefix GLOBAL-LP32 %s +// RUN: %clang_cc1 %s -flto -fvisibility hidden -fsanitize=cfi-vcall -enable-vtable-interleaving -emit-llvm -o - -triple=armv7-unknown-unknown | FileCheck -check-prefix GLOBAL-ARM %s + +// PNaCl uses the same representation of method pointers as ARM. +// RUN: %clang_cc1 %s -flto -fvisibility hidden -fsanitize=cfi-vcall -enable-vtable-interleaving -emit-llvm -o - -triple=le32-unknown-nacl | FileCheck -check-prefix GLOBAL-ARM %s +// MIPS uses the same representation of method pointers as ARM. +// RUN: %clang_cc1 %s -flto -fvisibility hidden -fsanitize=cfi-vcall -enable-vtable-interleaving -emit-llvm -o - -triple=mips-unknown-linux-gnu | FileCheck -check-prefix GLOBAL-ARM %s +// WebAssembly uses the same representation of method pointers as ARM. +// RUN: %clang_cc1 %s -flto -fvisibility hidden -fsanitize=cfi-vcall -enable-vtable-interleaving -emit-llvm -o - -triple=wasm32-unknown-unknown | FileCheck -check-prefix GLOBAL-ARM %s + +namespace test1 { +struct A { + int a; + void f(); + virtual void vf1(); + virtual void vf2(); +}; +struct B { + int b; + virtual void g(); +}; +struct C : B, A {}; + +void (A::*pa)(); +void (A::*volatile vpa)(); +void (B::*pb)(); +void (C::*pc)(); + +// GLOBAL-LP64: @_ZN5test13pa2E = hidden global { i64, i64 } { i64 ptrtoint (void (%"struct.test1::A"*)* @_ZN5test11A1fEv to i64), i64 0 }, align 8 +void (A::*pa2)() = &A::f; + +// GLOBAL-LP64: @_ZN5test13pa3E = hidden global { i64, i64 } { i64 add (i64 ptrtoint ([0 x i32]* @"__A$0" to i64), i64 1), i64 0 }, align 8 +// GLOBAL-LP32: @_ZN5test13pa3E = hidden global { i32, i32 } { i32 add (i32 ptrtoint ([0 x i32]* @"__A$0" to i32), i32 1), i32 0 }, align 4 +void (A::*pa3)() = &A::vf1; + +// GLOBAL-LP64: @_ZN5test13pa4E = hidden global { i64, i64 } { i64 add (i64 ptrtoint ([0 x i32]* @"__A$8" to i64), i64 1), i64 0 }, align 8 +// GLOBAL-LP32: @_ZN5test13pa4E = hidden global { i32, i32 } { i32 add (i32 ptrtoint ([0 x i32]* @"__A$4" to i32), i32 1), i32 0 }, align 4 +void (A::*pa4)() = &A::vf2; + +// GLOBAL-LP64: @_ZN5test13pc2E = hidden global { i64, i64 } { i64 ptrtoint (void (%"struct.test1::A"*)* @_ZN5test11A1fEv to i64), i64 16 }, align 8 +void (C::*pc2)() = &C::f; + +// GLOBAL-LP64: @_ZN5test13pc3E = hidden global { i64, i64 } { i64 add (i64 ptrtoint ([0 x i32]* @"__A$0" to i64), i64 1), i64 0 }, align 8 +void (A::*pc3)() = &A::vf1; + +void f() { + // CODE-LP64: store { i64, i64 } zeroinitializer, { i64, i64 }* @_ZN5test12paE + pa = 0; + + // Is this okay? What are LLVM's volatile semantics for structs? + // CODE-LP64: store volatile { i64, i64 } zeroinitializer, { i64, i64 }* @_ZN5test13vpaE + vpa = 0; + + // CODE-LP64: [[TMP:%.*]] = load { i64, i64 }, { i64, i64 }* @_ZN5test12paE, align 8 + // CODE-LP64: [[TMPADJ:%.*]] = extractvalue { i64, i64 } [[TMP]], 1 + // CODE-LP64: [[ADJ:%.*]] = add nsw i64 [[TMPADJ]], 16 + // CODE-LP64: [[RES:%.*]] = insertvalue { i64, i64 } [[TMP]], i64 [[ADJ]], 1 + // CODE-LP64: store { i64, i64 } [[RES]], { i64, i64 }* @_ZN5test12pcE, align 8 + pc = pa; + + // CODE-LP64: [[TMP:%.*]] = load { i64, i64 }, { i64, i64 }* @_ZN5test12pcE, align 8 + // CODE-LP64: [[TMPADJ:%.*]] = extractvalue { i64, i64 } [[TMP]], 1 + // CODE-LP64: [[ADJ:%.*]] = sub nsw i64 [[TMPADJ]], 16 + // CODE-LP64: [[RES:%.*]] = insertvalue { i64, i64 } [[TMP]], i64 [[ADJ]], 1 + // CODE-LP64: store { i64, i64 } [[RES]], { i64, i64 }* @_ZN5test12paE, align 8 + pa = static_cast(pc); +} + +void f2() { + // CODE-LP64: store { i64, i64 } { i64 ptrtoint (void (%"struct.test1::A"*)* @_ZN5test11A1fEv to i64), i64 0 } + void (A::*pa2)() = &A::f; + + // CODE-LP64: store { i64, i64 } { i64 add (i64 ptrtoint ([0 x i32]* @"__A$0" to i64), i64 1), i64 0 } + // CODE-LP32: store { i32, i32 } { i32 add (i32 ptrtoint ([0 x i32]* @"__A$0" to i32), i32 1), i32 0 } + void (A::*pa3)() = &A::vf1; + + // CODE-LP64: store { i64, i64 } { i64 add (i64 ptrtoint ([0 x i32]* @"__A$8" to i64), i64 1), i64 0 } + // CODE-LP32: store { i32, i32 } { i32 add (i32 ptrtoint ([0 x i32]* @"__A$4" to i32), i32 1), i32 0 } + void (A::*pa4)() = &A::vf2; +} + +void f3(A *a, A &ar) { + (a->*pa)(); + (ar.*pa)(); +} + +bool f4() { + return pa; +} +} // namespace test1 + +namespace test2 { +struct A { + void foo(); + virtual void vfoo(); +}; +struct B { + void foo(); + virtual void vfoo(); +}; +struct C : A, B { + void foo(); + virtual void vfoo(); +}; + +// GLOBAL-ARM: @_ZN5test24ptr0E = hidden global {{.*}} { i32 ptrtoint ({{.*}}* @_ZN5test21A3fooEv to i32), i32 0 } +// GLOBAL-ARM: @_ZN5test24ptr1E = hidden global {{.*}} { i32 ptrtoint ({{.*}}* @_ZN5test21B3fooEv to i32), i32 8 } +// GLOBAL-ARM: @_ZN5test24ptr2E = hidden global {{.*}} { i32 ptrtoint ({{.*}}* @_ZN5test21C3fooEv to i32), i32 0 } +// GLOBAL-ARM: @_ZN5test24ptr3E = hidden global {{.*}} { i32 ptrtoint ([0 x i32]* @"__A$0.1" to i32), i32 1 } +// GLOBAL-ARM: @_ZN5test24ptr4E = hidden global {{.*}} { i32 ptrtoint ([0 x i32]* @"__B$0" to i32), i32 9 } +// GLOBAL-ARM: @_ZN5test24ptr5E = hidden global {{.*}} { i32 ptrtoint ([0 x i32]* @"__C$0" to i32), i32 1 } +void (C::*ptr0)() = &A::foo; +void (C::*ptr1)() = &B::foo; +void (C::*ptr2)() = &C::foo; +void (C::*ptr3)() = &A::vfoo; +void (C::*ptr4)() = &B::vfoo; +void (C::*ptr5)() = &C::vfoo; +} // namespace test2 + +// rdar://problem/10815683 - Verify that we can emit reinterprets of +// member pointers as constant initializers. For added trickiness, +// we also add some non-trivial adjustments. +namespace test3 { +struct A { + int nonEmpty; + void foo(); +}; +struct B : public A { + virtual void requireNonZeroAdjustment(); +}; +struct C { + int nonEmpty; +}; +struct D : public C { + virtual void requireNonZeroAdjustment(); +}; + +// It's not that the offsets are doubled on ARM, it's that they're left-shifted by 1. + +// GLOBAL-LP64: @_ZN5test31aE = hidden global { i64, i64 } { i64 ptrtoint (void (%"struct.test3::A"*)* @_ZN5test31A3fooEv to i64), i64 0 }, align 8 +// GLOBAL-LP32: @_ZN5test31aE = hidden global { i32, i32 } { i32 ptrtoint (void (%"struct.test3::A"*)* @_ZN5test31A3fooEv to i32), i32 0 }, align 4 +// GLOBAL-ARM: @_ZN5test31aE = hidden global { i32, i32 } { i32 ptrtoint (void (%"struct.test3::A"*)* @_ZN5test31A3fooEv to i32), i32 0 }, align 4 +void (A::*a)() = &A::foo; + +// GLOBAL-LP64: @_ZN5test31bE = hidden global { i64, i64 } { i64 ptrtoint (void (%"struct.test3::A"*)* @_ZN5test31A3fooEv to i64), i64 8 }, align 8 +// GLOBAL-LP32: @_ZN5test31bE = hidden global { i32, i32 } { i32 ptrtoint (void (%"struct.test3::A"*)* @_ZN5test31A3fooEv to i32), i32 4 }, align 4 +// GLOBAL-ARM: @_ZN5test31bE = hidden global { i32, i32 } { i32 ptrtoint (void (%"struct.test3::A"*)* @_ZN5test31A3fooEv to i32), i32 8 }, align 4 +void (B::*b)() = (void (B::*)()) & A::foo; + +// GLOBAL-LP64: @_ZN5test31cE = hidden global { i64, i64 } { i64 ptrtoint (void (%"struct.test3::A"*)* @_ZN5test31A3fooEv to i64), i64 8 }, align 8 +// GLOBAL-LP32: @_ZN5test31cE = hidden global { i32, i32 } { i32 ptrtoint (void (%"struct.test3::A"*)* @_ZN5test31A3fooEv to i32), i32 4 }, align 4 +// GLOBAL-ARM: @_ZN5test31cE = hidden global { i32, i32 } { i32 ptrtoint (void (%"struct.test3::A"*)* @_ZN5test31A3fooEv to i32), i32 8 }, align 4 +void (C::*c)() = (void (C::*)())(void (B::*)()) & A::foo; + +// GLOBAL-LP64: @_ZN5test31dE = hidden global { i64, i64 } { i64 ptrtoint (void (%"struct.test3::A"*)* @_ZN5test31A3fooEv to i64), i64 16 }, align 8 +// GLOBAL-LP32: @_ZN5test31dE = hidden global { i32, i32 } { i32 ptrtoint (void (%"struct.test3::A"*)* @_ZN5test31A3fooEv to i32), i32 8 }, align 4 +// GLOBAL-ARM: @_ZN5test31dE = hidden global { i32, i32 } { i32 ptrtoint (void (%"struct.test3::A"*)* @_ZN5test31A3fooEv to i32), i32 16 }, align 4 +void (D::*d)() = (void (C::*)())(void (B::*)()) & A::foo; +} // namespace test3 + +namespace test4 { +struct A { + virtual void a(); +}; +struct B : A {}; +struct C : B { + virtual void a(); +}; +void (C::*x)() = &C::a; + +// GLOBAL-LP64: @_ZN5test41xE = hidden global { i64, i64 } { i64 add (i64 ptrtoint ([0 x i32]* @"__C$0.2" to i64), i64 1), i64 0 } +// GLOBAL-LP32: @_ZN5test41xE = hidden global { i32, i32 } { i32 add (i32 ptrtoint ([0 x i32]* @"__C$0.2" to i32), i32 1), i32 0 } +// GLOBAL-ARM: @_ZN5test41xE = hidden global { i32, i32 } { i32 ptrtoint ([0 x i32]* @"__C$0.2" to i32), i32 1 } +} // namespace test4 Index: clang/test/CodeGenCXX/interleaving-virtual-base.cpp =================================================================== --- /dev/null +++ clang/test/CodeGenCXX/interleaving-virtual-base.cpp @@ -0,0 +1,72 @@ +// This file tests if accesses to vbase entries are correctly generated when vtable interleaving is enabled. +// This test is based on virtual-base-cast.cpp. + +// RUN: %clang_cc1 -flto -fvisibility hidden -fsanitize=cfi-vcall -enable-vtable-interleaving -emit-llvm %s -o - -triple i686-pc-linux-gnu | FileCheck %s + +struct A { + int a; + virtual int aa(); +}; +struct B { + int b; + virtual int bb(); +}; +struct C : virtual A, virtual B { + int c; + virtual int aa(); + virtual int bb(); +}; +struct AA { + int a; + virtual int aa(); +}; +struct BB { + int b; + virtual int bb(); +}; +struct CC : AA, BB { + virtual int aa(); + virtual int bb(); + virtual int cc(); +}; +struct D : virtual C, virtual CC { + int e; +}; + +D *x; + +A *a() { return x; } +// CHECK: @_Z1av() [[NUW:#[0-9]+]] +// CHECK: [[VBASEOFFSETPTRA:%[a-zA-Z0-9\.]+]] = getelementptr inbounds i8, i8* {{.*}}, i64 ptrtoint ([0 x i32]* @"__D$-16" to i64) +// CHECK: [[CASTVBASEOFFSETPTRA:%[a-zA-Z0-9\.]+]] = bitcast i8* [[VBASEOFFSETPTRA]] to i32* +// CHECK: load i32, i32* [[CASTVBASEOFFSETPTRA]] +// CHECK: } + +B *b() { return x; } +// CHECK: @_Z1bv() [[NUW]] +// CHECK: [[VBASEOFFSETPTRA:%[a-zA-Z0-9\.]+]] = getelementptr inbounds i8, i8* {{.*}}, i64 ptrtoint ([0 x i32]* @"__D$-20" to i64) +// CHECK: [[CASTVBASEOFFSETPTRA:%[a-zA-Z0-9\.]+]] = bitcast i8* [[VBASEOFFSETPTRA]] to i32* +// CHECK: load i32, i32* [[CASTVBASEOFFSETPTRA]] +// CHECK: } + +BB *c() { return x; } +// CHECK: @_Z1cv() [[NUW]] +// CHECK: [[VBASEOFFSETPTRC:%[a-zA-Z0-9\.]+]] = getelementptr inbounds i8, i8* {{.*}}, i64 ptrtoint ([0 x i32]* @"__D$-24" to i64) +// CHECK: [[CASTVBASEOFFSETPTRC:%[a-zA-Z0-9\.]+]] = bitcast i8* [[VBASEOFFSETPTRC]] to i32* +// CHECK: [[VBASEOFFSETC:%[a-zA-Z0-9\.]+]] = load i32, i32* [[CASTVBASEOFFSETPTRC]] +// CHECK: add i32 [[VBASEOFFSETC]], 8 +// CHECK: } + +// Put the vbptr at a non-zero offset inside a non-virtual base. +struct E { + int e; +}; +struct F : E, D { + int f; +}; + +F *y; + +BB *d() { return y; } + +// CHECK: attributes [[NUW]] = { noinline nounwind{{.*}} } Index: clang/test/CodeGenCXX/interleaving-virtual-calls.cpp =================================================================== --- /dev/null +++ clang/test/CodeGenCXX/interleaving-virtual-calls.cpp @@ -0,0 +1,38 @@ +// This file tests if accesses to virtual function pointer entries are correctly generated when vtable interleaving is enabled. + +// RUN: %clang_cc1 -enable-vtable-interleaving -flto -fvisibility hidden -fsanitize=cfi-vcall -emit-llvm %s -o - -triple x86_64-unknown-linux | FileCheck %s + +struct A { + int a; + virtual int foo1(); + virtual int foo2(); +}; +struct B : A { + int b; + virtual int foo1(); +}; +struct C { + int c; + virtual int foo3(); + virtual int foo4(); +}; +struct D : B, C { + virtual int foo2(); + virtual int foo3(); +}; + +// CHECK: @_Z5test1P1A +int test1(A *a) { + // CHECK: [[R0:%.*]] = bitcast %struct.A* {{.*}} to i8** + // CHECK: [[VTABLE:%.*]] = load i8*, i8** [[R0]] + // CHECK: getelementptr inbounds i8, i8* [[VTABLE]], i32 ptrtoint ([0 x i32]* @"__A$0" to i32) + return a->foo1(); +} + +// CHECK: @_Z5test2P1C +int test2(C *c) { + // CHECK: [[R0:%.*]] = bitcast %struct.C* {{.*}} to i8** + // CHECK: [[VTABLE:%.*]] = load i8*, i8** [[R0]] + // CHECK: getelementptr inbounds i8, i8* [[VTABLE]], i32 ptrtoint ([0 x i32]* @"__C$8" to i32) + return c->foo4(); +} Index: clang/test/CodeGenCXX/ubsan-vtable-checks.cpp =================================================================== --- clang/test/CodeGenCXX/ubsan-vtable-checks.cpp +++ clang/test/CodeGenCXX/ubsan-vtable-checks.cpp @@ -25,7 +25,10 @@ // CHECK-NULL-NEXT: br i1 [[UBSAN_CMP_RES]], label %{{.*}}, label %{{.*}} // CHECK-NULL: call void @__ubsan_handle_type_mismatch_v1_abort // Second, we check that vtable is actually loaded once the type check is done. - // CHECK-NULL: load {{.*}} (%struct.T*{{.*}})**, {{.*}} (%struct.T*{{.*}})*** + // ITANIUM: [[R0:%.*]] = bitcast %struct.T* {{.*}} to i8** + // ITANIUM-NEXT: load i8*, i8** [[R0]] + // MSABI: [[R0:%.*]] = bitcast %struct.T* {{.*}} to i32 (%struct.T*)*** + // MSABI-NEXT: load i32 (%struct.T*)**, i32 (%struct.T*)*** [[R0]] return t->v(); } @@ -37,7 +40,10 @@ // CHECK-VPTR: br i1 {{.*}} label %{{.*}} // CHECK-VPTR: call void @__ubsan_handle_dynamic_type_cache_miss_abort // Second, we check that vtable is actually loaded once the type check is done. - // CHECK-VPTR: load {{.*}} (%struct.T*{{.*}})**, {{.*}} (%struct.T*{{.*}})*** + // ITANIUM: [[vtable:%.*]] = bitcast %struct.T* {{.*}} to i8** + // ITANIUM-NEXT: load i8*, i8** [[vtable]] + // MSABI: [[R0:%.*]] = bitcast %struct.T* {{.*}} to i8* (%struct.T*, i32)*** + // MSABI-NEXT: load i8* (%struct.T*, i32)**, i8* (%struct.T*, i32)*** [[R0]] delete t; } Index: clang/test/CodeGenCXX/virtual-base-cast.cpp =================================================================== --- clang/test/CodeGenCXX/virtual-base-cast.cpp +++ clang/test/CodeGenCXX/virtual-base-cast.cpp @@ -13,7 +13,7 @@ A* a() { return x; } // CHECK: @_Z1av() [[NUW:#[0-9]+]] -// CHECK: [[VBASEOFFSETPTRA:%[a-zA-Z0-9\.]+]] = getelementptr i8, i8* {{.*}}, i64 -16 +// CHECK: [[VBASEOFFSETPTRA:%[a-zA-Z0-9\.]+]] = getelementptr inbounds i8, i8* {{.*}}, i64 -16 // CHECK: [[CASTVBASEOFFSETPTRA:%[a-zA-Z0-9\.]+]] = bitcast i8* [[VBASEOFFSETPTRA]] to i32* // CHECK: load i32, i32* [[CASTVBASEOFFSETPTRA]] // CHECK: } @@ -29,7 +29,7 @@ B* b() { return x; } // CHECK: @_Z1bv() [[NUW]] -// CHECK: [[VBASEOFFSETPTRA:%[a-zA-Z0-9\.]+]] = getelementptr i8, i8* {{.*}}, i64 -20 +// CHECK: [[VBASEOFFSETPTRA:%[a-zA-Z0-9\.]+]] = getelementptr inbounds i8, i8* {{.*}}, i64 -20 // CHECK: [[CASTVBASEOFFSETPTRA:%[a-zA-Z0-9\.]+]] = bitcast i8* [[VBASEOFFSETPTRA]] to i32* // CHECK: load i32, i32* [[CASTVBASEOFFSETPTRA]] // CHECK: } @@ -47,7 +47,7 @@ BB* c() { return x; } // CHECK: @_Z1cv() [[NUW]] -// CHECK: [[VBASEOFFSETPTRC:%[a-zA-Z0-9\.]+]] = getelementptr i8, i8* {{.*}}, i64 -24 +// CHECK: [[VBASEOFFSETPTRC:%[a-zA-Z0-9\.]+]] = getelementptr inbounds i8, i8* {{.*}}, i64 -24 // CHECK: [[CASTVBASEOFFSETPTRC:%[a-zA-Z0-9\.]+]] = bitcast i8* [[VBASEOFFSETPTRC]] to i32* // CHECK: [[VBASEOFFSETC:%[a-zA-Z0-9\.]+]] = load i32, i32* [[CASTVBASEOFFSETPTRC]] // CHECK: add i32 [[VBASEOFFSETC]], 8 Index: compiler-rt/test/cfi/CMakeLists.txt =================================================================== --- compiler-rt/test/cfi/CMakeLists.txt +++ compiler-rt/test/cfi/CMakeLists.txt @@ -1,6 +1,6 @@ set(CFI_TESTSUITES) -macro (add_cfi_test_suites lld thinlto newpm) +macro (add_cfi_test_suites lld thinlto newpm interleaving) set(suffix) if (${lld}) set(suffix ${suffix}-lld) @@ -11,11 +11,15 @@ if (${newpm}) set(suffix ${suffix}-newpm) endif() + if (${interleaving}) + set(suffix ${suffix}-interleaving) + endif() set(suffix ${suffix}-${CFI_TEST_TARGET_ARCH}) set(CFI_TEST_USE_LLD ${lld}) set(CFI_TEST_USE_THINLTO ${thinlto}) set(CFI_TEST_USE_NEWPM ${newpm}) + set(CFI_TEST_USE_INTERLEAVING ${interleaving}) set(CFI_LIT_TEST_MODE Standalone) set(CFI_TEST_CONFIG_SUFFIX -standalone${suffix}) @@ -44,19 +48,19 @@ get_test_cc_for_arch(${arch} CFI_TEST_TARGET_CC CFI_TEST_TARGET_CFLAGS) if (APPLE) # FIXME: enable ThinLTO tests after fixing http://llvm.org/pr32741 - add_cfi_test_suites(False False False) + add_cfi_test_suites(False False False False) elseif(WIN32) - add_cfi_test_suites(True False False) - add_cfi_test_suites(True True False) + add_cfi_test_suites(True False False False) + add_cfi_test_suites(True True False False) else() - add_cfi_test_suites(False False False) - add_cfi_test_suites(False True False) - add_cfi_test_suites(False False True) - add_cfi_test_suites(False True True) - if (COMPILER_RT_HAS_LLD AND NOT arch STREQUAL "i386") - add_cfi_test_suites(True False False) - add_cfi_test_suites(True True False) - endif() + add_cfi_test_suites(False False False False) + add_cfi_test_suites(False True False False) + add_cfi_test_suites(False False True False) + add_cfi_test_suites(False True True False) + add_cfi_test_suites(True False False False) + add_cfi_test_suites(True True False False) + add_cfi_test_suites(True False False True) + add_cfi_test_suites(False False False True) endif() endforeach() Index: compiler-rt/test/cfi/anon-namespace.cpp =================================================================== --- compiler-rt/test/cfi/anon-namespace.cpp +++ compiler-rt/test/cfi/anon-namespace.cpp @@ -26,7 +26,7 @@ // RUN: %clangxx_cfi_diag -c -DTU1 -o %t1.o %s // RUN: %clangxx_cfi_diag -c -DTU2 -o %t2.o %S/../cfi/anon-namespace.cpp // RUN: %clangxx_cfi_diag -o %t6 %t1.o %t2.o -// RUN: %run %t6 2>&1 | FileCheck --check-prefix=CFI-DIAG %s +// RUN: %interleave_diag %run %t6 2>&1 | FileCheck --check-prefix=CFI-DIAG %s // Tests that the CFI mechanism treats classes in the anonymous namespace in // different translation units as having distinct identities. This is done by Index: compiler-rt/test/cfi/cross-dso/lit.local.cfg =================================================================== --- compiler-rt/test/cfi/cross-dso/lit.local.cfg +++ compiler-rt/test/cfi/cross-dso/lit.local.cfg @@ -11,3 +11,6 @@ # Android O (API level 26) has support for cross-dso cfi in libdl.so. if config.android and 'android-26' not in config.available_features: config.unsupported = True + +if 'interleaving' in config.available_features: + config.unsupported = True \ No newline at end of file Index: compiler-rt/test/cfi/icall/lit.local.cfg =================================================================== --- compiler-rt/test/cfi/icall/lit.local.cfg +++ compiler-rt/test/cfi/icall/lit.local.cfg @@ -1,3 +1,6 @@ # The cfi-icall checker is only supported on x86 and x86_64 for now. if config.root.host_arch not in ['x86', 'x86_64']: config.unsupported = True + +if 'interleaving' in config.available_features: + config.unsupported = True \ No newline at end of file Index: compiler-rt/test/cfi/lit.cfg =================================================================== --- compiler-rt/test/cfi/lit.cfg +++ compiler-rt/test/cfi/lit.cfg @@ -19,7 +19,10 @@ if config.cfi_lit_test_mode == "Devirt": config.available_features.add('devirt') clang_cfi += '-fwhole-program-vtables ' - config.substitutions.append((r"%expect_crash_unless_devirt ", "")) + if config.use_interleaving: + config.substitutions.append((r"%expect_crash_unless_devirt ", config.expect_crash)) + else: + config.substitutions.append((r"%expect_crash_unless_devirt ", "")) else: config.substitutions.append((r"%expect_crash_unless_devirt ", config.expect_crash)) @@ -27,14 +30,27 @@ diag = '-fno-sanitize-trap=cfi -fsanitize-recover=cfi ' non_dso = '-fvisibility=hidden ' dso = '-fsanitize-cfi-cross-dso -fvisibility=default ' + interleave = '-Xclang -enable-vtable-interleaving -Wl,-plugin-opt,-enable-vtable-interleaving -fno-sanitize=cfi-mfcall ' if config.android: dso += '-include ' + config.test_source_root + '/cross-dso/util/cfi_stubs.h ' + config.substitutions.append((r"%clang_cfi ", clang_cfi + non_dso)) - config.substitutions.append((r"%clangxx_cfi ", clang_cfi + cxx + non_dso)) config.substitutions.append((r"%clang_cfi_diag ", clang_cfi + non_dso + diag)) - config.substitutions.append((r"%clangxx_cfi_diag ", clang_cfi + cxx + non_dso + diag)) - config.substitutions.append((r"%clangxx_cfi_dso ", clang_cfi + cxx + dso)) - config.substitutions.append((r"%clangxx_cfi_dso_diag ", clang_cfi + cxx + dso + diag)) + + if config.use_interleaving: + config.available_features.add('interleaving') + config.substitutions.append((r"%clangxx_cfi ", clang_cfi + cxx + non_dso + interleave)) + config.substitutions.append((r"%clangxx_cfi_diag ", clang_cfi + cxx + non_dso + diag + interleave)) + config.substitutions.append((r"%interleave_diag ", "not ")) + else: + config.substitutions.append((r"%clangxx_cfi ", clang_cfi + cxx + non_dso)) + config.substitutions.append((r"%clangxx_cfi_dso ", clang_cfi + cxx + dso)) + config.substitutions.append((r"%clangxx_cfi_dso_diag ", clang_cfi + cxx + dso + diag)) + config.substitutions.append((r"%clangxx_cfi_diag ", clang_cfi + cxx + non_dso + diag)) + config.substitutions.append((r"%interleave_diag ", "")) + + config.substitutions.append((r"%clangxx_cfi_no_interleaving ", clang_cfi + cxx + non_dso)) + config.substitutions.append((r"%debug_info_flags", ' '.join(config.debug_info_flags))) else: config.unsupported = True @@ -44,3 +60,4 @@ if lit_config.params.get('check_supported', None) and config.unsupported: raise BaseException("Tests unsupported") + \ No newline at end of file Index: compiler-rt/test/cfi/lit.site.cfg.in =================================================================== --- compiler-rt/test/cfi/lit.site.cfg.in +++ compiler-rt/test/cfi/lit.site.cfg.in @@ -8,6 +8,7 @@ config.use_lto = True # CFI *requires* LTO. config.use_thinlto = @CFI_TEST_USE_THINLTO@ config.use_newpm = @CFI_TEST_USE_NEWPM@ +config.use_interleaving = @CFI_TEST_USE_INTERLEAVING@ lit_config.load_config(config, "@COMPILER_RT_BINARY_DIR@/test/lit.common.configured") lit_config.load_config(config, "@CMAKE_CURRENT_SOURCE_DIR@/lit.cfg") Index: compiler-rt/test/cfi/mfcall.cpp =================================================================== --- compiler-rt/test/cfi/mfcall.cpp +++ compiler-rt/test/cfi/mfcall.cpp @@ -1,4 +1,4 @@ -// UNSUPPORTED: windows-msvc +// UNSUPPORTED: windows-msvc, interleaving // RUN: %clangxx_cfi -o %t %s // RUN: %expect_crash %run %t a Index: compiler-rt/test/cfi/multiple-inheritance.cpp =================================================================== --- compiler-rt/test/cfi/multiple-inheritance.cpp +++ compiler-rt/test/cfi/multiple-inheritance.cpp @@ -20,7 +20,7 @@ // RUN: %clangxx_cfi_diag -o %t6 %s // RUN: %run %t6 2>&1 | FileCheck --check-prefix=CFI-DIAG2 %s -// RUN: %run %t6 x 2>&1 | FileCheck --check-prefix=CFI-DIAG1 %s +// RUN: %interleave_diag %run %t6 x 2>&1 | FileCheck --check-prefix=CFI-DIAG1 %s // Tests that the CFI mechanism is sensitive to multiple inheritance and only // permits calls via virtual tables for the correct base class. Index: compiler-rt/test/cfi/simple-fail.cpp =================================================================== --- compiler-rt/test/cfi/simple-fail.cpp +++ compiler-rt/test/cfi/simple-fail.cpp @@ -47,12 +47,12 @@ // RUN: %expect_crash %run %t16 2>&1 | FileCheck --check-prefix=CFI %s // RUN: %clangxx_cfi_diag -o %t17 %s -// RUN: %run %t17 2>&1 | FileCheck --check-prefix=CFI-DIAG %s +// RUN: %interleave_diag %run %t17 2>&1 | FileCheck --check-prefix=CFI-DIAG %s // RUN: %clangxx -o %t18 %s // RUN: %run %t18 2>&1 | FileCheck --check-prefix=NCFI %s -// RUN: %clangxx_cfi -DCHECK_NO_SANITIZE_CFI -o %t19 %s +// RUN: %clangxx_cfi_no_interleaving -DCHECK_NO_SANITIZE_CFI -o %t19 %s // RUN: %run %t19 2>&1 | FileCheck --check-prefix=NCFI %s // Tests that the CFI mechanism crashes the program when making a virtual call @@ -95,6 +95,7 @@ // CFI-DIAG-NEXT: note: vtable is of type '{{(struct )?}}A' ((B *)a)->f(); // UB here + // INTERLEAVING-NCFI-NOT: {{^2$}} // CFI-NOT: {{^2$}} // NCFI: {{^2$}} fprintf(stderr, "2\n"); Index: compiler-rt/test/cfi/vdtor.cpp =================================================================== --- compiler-rt/test/cfi/vdtor.cpp +++ compiler-rt/test/cfi/vdtor.cpp @@ -14,7 +14,7 @@ // RUN: %run %t5 2>&1 | FileCheck --check-prefix=NCFI %s // RUN: %clangxx_cfi_diag -o %t6 %s -// RUN: %run %t6 2>&1 | FileCheck --check-prefix=CFI-DIAG %s +// RUN: %interleave_diag %run %t6 2>&1 | FileCheck --check-prefix=CFI-DIAG %s // Tests that the CFI enforcement also applies to virtual destructor calls made // via 'delete'. Index: compiler-rt/test/lit.common.configured.in =================================================================== --- compiler-rt/test/lit.common.configured.in +++ compiler-rt/test/lit.common.configured.in @@ -35,6 +35,7 @@ set_default("use_thinlto", False) set_default("use_lto", config.use_thinlto) set_default("use_newpm", False) +set_default("use_interleaving", False) set_default("android", @ANDROID_PYBOOL@) config.available_features.add('target-is-%s' % config.target_arch)