[cfi] Cross-DSO CFI diagnostic mode (clang part)

[eugenis]

Summary

• Runtime diagnostic data for cfi-icall changed to match the rest of cfi checks
• Layout of all CFI diagnostic data changed to put Kind at the beginning. There is no ABI stability promise yet.
• call cfi_slowpath_diag instead of cfi_slowpath when needed.
• emit __cfi_check_fail function, which dispatches a CFI check faliure according to trap/recover settings of the current module.
• a tiny driver change to match the way the new handlers are done in compiler-rt.

[pcc]

lib/CodeGen/CGExpr.cpp

2620

This looks like CodeGenFunction::EmitTrapCheck.

2654–2678

Can't you replace all this code with a call to EmitCheck passing icmps for each of the check kinds? Sure, it wouldn't be a switch, but we don't care about the performance of this code anyway.

lib/CodeGen/CodeGenFunction.h

1386

No need to rename, these are still all type checks.

test/CodeGen/cfi-icall-cross-dso.c

28

Are you still testing the __cfi_slowpath code path?

[eugenis]

lib/CodeGen/CGExpr.cpp

2654–2678

Done. The code became a bit simpler, but the test looks awful now.

test/CodeGen/cfi-icall-cross-dso.c

28

added a test for the non-diag path

[eugenis]

ping

[pcc]

LGTM

test/CodeGen/cfi-check-fail.c

18

Does this match in a -asserts build?

[eugenis]

test/CodeGen/cfi-check-fail.c

19

Right.
It fails.
I've removed the ":" after all label names.

[eugenis]

Thanks! Committed as
http://reviews.llvm.org/rL258745

lib/CodeGen/CGClass.cpp

LeastDerivedClassWithSameLayout(const CXXRecordDecl *RD) {
   }
 
   return LeastDerivedClassWithSameLayout(
       RD->bases_begin()->getType()->getAsCXXRecordDecl());
 }
 
 void CodeGenFunction::EmitVTablePtrCheckForCall(const CXXMethodDecl *MD,
                                                 llvm::Value *VTable,
-                                                CFITypeCheckKind TCK,
+                                                CFICheckKind TCK,
                                                 SourceLocation Loc) {
   const CXXRecordDecl *ClassDecl = MD->getParent();
   if (!SanOpts.has(SanitizerKind::CFICastStrict))
     ClassDecl = LeastDerivedClassWithSameLayout(ClassDecl);
 
   EmitVTablePtrCheck(ClassDecl, VTable, TCK, Loc);
 }
 
 void CodeGenFunction::EmitVTablePtrCheckForCast(QualType T,
                                                 llvm::Value *Derived,
                                                 bool MayBeNull,
-                                                CFITypeCheckKind TCK,
+                                                CFICheckKind TCK,
                                                 SourceLocation Loc) {
   if (!getLangOpts().CPlusPlus)
     return;
 
   auto *ClassTy = T->getAs<RecordType>();
   if (!ClassTy)
     return;
 
void CodeGenFunction::EmitVTablePtrCheckForCast(QualType T,
   if (MayBeNull) {
     Builder.CreateBr(ContBlock);
     EmitBlock(ContBlock);
   }
 }
 
 void CodeGenFunction::EmitVTablePtrCheck(const CXXRecordDecl *RD,
                                          llvm::Value *VTable,
-                                         CFITypeCheckKind TCK,
+                                         CFICheckKind TCK,
                                          SourceLocation Loc) {
   if (CGM.IsCFIBlacklistedRecord(RD))
     return;
 
   SanitizerScope SanScope(this);
 
   llvm::Metadata *MD =
       CGM.CreateMetadataIdentifierForType(QualType(RD->getTypeForDecl(), 0));
   llvm::Value *BitSetName = llvm::MetadataAsValue::get(getLLVMContext(), MD);
 
   llvm::Value *CastedVTable = Builder.CreateBitCast(VTable, Int8PtrTy);
   llvm::Value *BitSetTest =
       Builder.CreateCall(CGM.getIntrinsic(llvm::Intrinsic::bitset_test),
                          {CastedVTable, BitSetName});
 
-  if (CGM.getCodeGenOpts().SanitizeCfiCrossDso) {
-    if (auto TypeId = CGM.CreateCfiIdForTypeMetadata(MD)) {
-      EmitCfiSlowPathCheck(BitSetTest, TypeId, CastedVTable);
-      return;
-    }
-  }
-
   SanitizerMask M;
   switch (TCK) {
   case CFITCK_VCall:
     M = SanitizerKind::CFIVCall;
     break;
   case CFITCK_NVCall:
     M = SanitizerKind::CFINVCall;
     break;
   case CFITCK_DerivedCast:
     M = SanitizerKind::CFIDerivedCast;
     break;
   case CFITCK_UnrelatedCast:
     M = SanitizerKind::CFIUnrelatedCast;
     break;
+  case CFITCK_ICall:
+    llvm_unreachable("not expecting CFITCK_ICall");
   }
 
   llvm::Constant *StaticData[] = {
+      llvm::ConstantInt::get(Int8Ty, TCK),
       EmitCheckSourceLocation(Loc),
       EmitCheckTypeDescriptor(QualType(RD->getTypeForDecl(), 0)),
-      llvm::ConstantInt::get(Int8Ty, TCK),
   };
-  EmitCheck(std::make_pair(BitSetTest, M), "cfi_bad_type", StaticData,
+
+  auto TypeId = CGM.CreateCfiIdForTypeMetadata(MD);
+  if (CGM.getCodeGenOpts().SanitizeCfiCrossDso && TypeId) {
+    EmitCfiSlowPathCheck(M, BitSetTest, TypeId, CastedVTable, StaticData);
+  } else {
+    EmitCheck(std::make_pair(BitSetTest, M), "cfi_check_fail", StaticData,
               CastedVTable);
   }
+}
 
 // FIXME: Ideally Expr::IgnoreParenNoopCasts should do this, but it doesn't do
 // quite what we want.
 static const Expr *skipNoOpCastsAndParens(const Expr *E) {
   while (true) {
     if (const ParenExpr *PE = dyn_cast<ParenExpr>(E)) {
       E = PE->getSubExpr();
       continue;

lib/CodeGen/CGExpr.cpp

if (!FatalCond || !RecoverableCond) {
     EmitBlock(NonFatalHandlerBB);
     emitCheckHandlerCall(*this, FnType, Args, CheckName, RecoverKind, false,
                          Cont);
   }
 
   EmitBlock(Cont);
 }
 
-void CodeGenFunction::EmitCfiSlowPathCheck(llvm::Value *Cond,
-                                           llvm::ConstantInt *TypeId,
-                                           llvm::Value *Ptr) {
-  auto &Ctx = getLLVMContext();
+void CodeGenFunction::EmitCfiSlowPathCheck(
+    SanitizerMask Kind, llvm::Value *Cond, llvm::ConstantInt *TypeId,
+    llvm::Value *Ptr, ArrayRef<llvm::Constant *> StaticArgs) {
   llvm::BasicBlock *Cont = createBasicBlock("cfi.cont");
 
   llvm::BasicBlock *CheckBB = createBasicBlock("cfi.slowpath");
   llvm::BranchInst *BI = Builder.CreateCondBr(Cond, Cont, CheckBB);
 
   llvm::MDBuilder MDHelper(getLLVMContext());
   llvm::MDNode *Node = MDHelper.createBranchWeights((1U << 20) - 1, 1);
   BI->setMetadata(llvm::LLVMContext::MD_prof, Node);
 
   EmitBlock(CheckBB);
 
+  bool WithDiag = !CGM.getCodeGenOpts().SanitizeTrap.has(Kind);
+
+  llvm::CallInst *CheckCall;
+  if (WithDiag) {
+    llvm::Constant *Info = llvm::ConstantStruct::getAnon(StaticArgs);
+    auto *InfoPtr =
+        new llvm::GlobalVariable(CGM.getModule(), Info->getType(), false,
+                                 llvm::GlobalVariable::PrivateLinkage, Info);
+    InfoPtr->setUnnamedAddr(true);
+    CGM.getSanitizerMetadata()->disableSanitizerForGlobal(InfoPtr);
+
+    llvm::Constant *SlowPathDiagFn = CGM.getModule().getOrInsertFunction(
+        "__cfi_slowpath_diag",
+        llvm::FunctionType::get(VoidTy, {Int64Ty, Int8PtrTy, Int8PtrTy},
+                                false));
+    CheckCall = Builder.CreateCall(
+        SlowPathDiagFn,
+        {TypeId, Ptr, Builder.CreateBitCast(InfoPtr, Int8PtrTy)});
+  } else {
     llvm::Constant *SlowPathFn = CGM.getModule().getOrInsertFunction(
         "__cfi_slowpath",
-      llvm::FunctionType::get(
-          llvm::Type::getVoidTy(Ctx),
-          {llvm::Type::getInt64Ty(Ctx),
-           llvm::PointerType::getUnqual(llvm::Type::getInt8Ty(Ctx))},
-          false));
-  llvm::CallInst *CheckCall = Builder.CreateCall(SlowPathFn, {TypeId, Ptr});
+        llvm::FunctionType::get(VoidTy, {Int64Ty, Int8PtrTy}, false));
+    CheckCall = Builder.CreateCall(SlowPathFn, {TypeId, Ptr});
+  }
+
   CheckCall->setDoesNotThrow();
 
   EmitBlock(Cont);
 }
 
+// This function is basically a switch over the CFI failure kind, which is
+// extracted from CFICheckFailData (1st function argument). Each case is either
+// llvm.trap or a call to one of the two runtime handlers, based on
+// -fsanitize-trap and -fsanitize-recover settings.  Default case (invalid
+// failure kind) traps, but this should really never happen.  CFICheckFailData
+// can be nullptr if the calling module has -fsanitize-trap behavior for this
+// check kind; in this case __cfi_check_fail traps as well.
+void CodeGenFunction::EmitCfiCheckFail() {
+  FunctionArgList Args;
+  ImplicitParamDecl ArgData(getContext(), nullptr, SourceLocation(), nullptr,
+                            getContext().VoidPtrTy);
+  ImplicitParamDecl ArgAddr(getContext(), nullptr, SourceLocation(), nullptr,
+                            getContext().VoidPtrTy);
+  Args.push_back(&ArgData);
+  Args.push_back(&ArgAddr);
+
+  const CGFunctionInfo &FI = CGM.getTypes().arrangeFreeFunctionDeclaration(
+      getContext().VoidTy, Args, FunctionType::ExtInfo(), /*variadic=*/false);
+
+  llvm::Function *F = llvm::Function::Create(
+      llvm::FunctionType::get(VoidTy, {VoidPtrTy, VoidPtrTy}, false),
+      llvm::GlobalValue::WeakODRLinkage, "__cfi_check_fail", &CGM.getModule());
+  F->setVisibility(llvm::GlobalValue::HiddenVisibility);
+
+  StartFunction(GlobalDecl(), CGM.getContext().VoidTy, F, FI, Args,
+                SourceLocation());
+
+  llvm::Value *Data =
+      EmitLoadOfScalar(GetAddrOfLocalVar(&ArgData), /*Volatile=*/false,
+                       CGM.getContext().VoidPtrTy, ArgData.getLocation());
+  llvm::Value *Addr =
+      EmitLoadOfScalar(GetAddrOfLocalVar(&ArgAddr), /*Volatile=*/false,
+                       CGM.getContext().VoidPtrTy, ArgAddr.getLocation());
+
+  auto ExitBB = createBasicBlock("exit");
+  auto TrapBB = createBasicBlock("trap");
+  auto ContBB = createBasicBlock("cont");
+
+  // Data == nullptr means the calling module has trap behaviour for this check.
+  llvm::Value *DataIsNullPtr =
+      Builder.CreateICmpEQ(Data, llvm::ConstantPointerNull::get(Int8PtrTy));
+  Builder.CreateCondBr(DataIsNullPtr, TrapBB, ContBB);
+
+  EmitBlock(ContBB);
+  llvm::StructType *SourceLocationTy =
+      llvm::StructType::get(VoidPtrTy, Int32Ty, Int32Ty, nullptr);
+  llvm::StructType *CfiCheckFailDataTy =
+      llvm::StructType::get(Int8Ty, SourceLocationTy, VoidPtrTy, nullptr);
+
+  llvm::Value *V = Builder.CreateConstGEP2_32(
+      CfiCheckFailDataTy,
+      Builder.CreatePointerCast(Data, CfiCheckFailDataTy->getPointerTo(0)), 0,
+      0);
+  Address CheckKindAddr(V, getIntAlign());
+  llvm::Value *CheckKind = Builder.CreateLoad(CheckKindAddr);
+  llvm::SwitchInst *SI =
+      Builder.CreateSwitch(CheckKind, TrapBB, /*NumCases*/ 5);
+
+  EmitBlock(TrapBB);
+  llvm::CallInst *TrapCall = EmitTrapCall(llvm::Intrinsic::trap);
+  TrapCall->setDoesNotReturn();
+  TrapCall->setDoesNotThrow();
+  Builder.CreateUnreachable();
+
+  llvm::BasicBlock *RecoverBB = nullptr;
+  llvm::BasicBlock *FatalBB = nullptr;
+
+  constexpr std::pair<int, SanitizerMask> CheckKinds[] = {
+      {CFITCK_VCall, SanitizerKind::CFIVCall},
+      {CFITCK_NVCall, SanitizerKind::CFINVCall},
+      {CFITCK_DerivedCast, SanitizerKind::CFIDerivedCast},
+      {CFITCK_UnrelatedCast, SanitizerKind::CFIUnrelatedCast},
+      {CFITCK_ICall, SanitizerKind::CFIICall}};
+
+  for (auto CheckKindMaskPair : CheckKinds) {
+    int Kind = CheckKindMaskPair.first;
+    SanitizerMask Mask = CheckKindMaskPair.second;
+    // All CFI checks are recoverable.
+    assert(getRecoverableKind(Mask) == CheckRecoverableKind::Recoverable);
+    if (CGM.getCodeGenOpts().SanitizeTrap.has(Mask)) {
+      SI->addCase(llvm::ConstantInt::get(Int8Ty, Kind), TrapBB);
+    } else if (CGM.getCodeGenOpts().SanitizeRecover.has(Mask)) {
+      if (!RecoverBB) {
+        RecoverBB = createBasicBlock("non_fatal");
+        EmitBlock(RecoverBB);
+        emitCheckHandlerCall(*this, F->getFunctionType(), {Data, Addr},
+                             "cfi_check_fail",
+                             CheckRecoverableKind::Recoverable, false, ExitBB);
+      }
+      SI->addCase(llvm::ConstantInt::get(Int8Ty, Kind), RecoverBB);
+    } else {
+      if (!FatalBB) {
+        FatalBB = createBasicBlock("fatal");
+        EmitBlock(FatalBB);
+        emitCheckHandlerCall(*this, F->getFunctionType(), {Data, Addr},
+                             "cfi_check_fail",
+                             CheckRecoverableKind::Recoverable, true, ExitBB);
+      }
+      SI->addCase(llvm::ConstantInt::get(Int8Ty, Kind), FatalBB);
+    }
+  }
+
+  EmitBlock(ExitBB);
+  FinishFunction();
+  // The only reference to this function will be created during LTO link.
+  // Make sure it survives until then.
+  CGM.addUsedGlobal(F);
+}
+
 void CodeGenFunction::EmitTrapCheck(llvm::Value *Checked) {
   llvm::BasicBlock *Cont = createBasicBlock("cont");
 
   // If we're optimizing, collapse all calls to trap down to just one per
   // function to save on code size.
   if (!CGM.getCodeGenOpts().OptimizationLevel || !TrapBB) {
     TrapBB = createBasicBlock("trap");
     Builder.CreateCondBr(Checked, Cont, TrapBB);
if (SanOpts.has(SanitizerKind::CFIICall) &&
     llvm::Value *BitSetName = llvm::MetadataAsValue::get(getLLVMContext(), MD);
 
     llvm::Value *CastedCallee = Builder.CreateBitCast(Callee, Int8PtrTy);
     llvm::Value *BitSetTest =
         Builder.CreateCall(CGM.getIntrinsic(llvm::Intrinsic::bitset_test),
                            {CastedCallee, BitSetName});
 
     auto TypeId = CGM.CreateCfiIdForTypeMetadata(MD);
-    if (CGM.getCodeGenOpts().SanitizeCfiCrossDso && TypeId) {
-      EmitCfiSlowPathCheck(BitSetTest, TypeId, CastedCallee);
-    } else {
-      llvm::Constant *StaticData[] = {
-          EmitCheckSourceLocation(E->getLocStart()),
-          EmitCheckTypeDescriptor(QualType(FnType, 0)),
-      };
+        llvm::ConstantInt::get(Int8Ty, CFITCK_ICall),
+    if (CGM.getCodeGenOpts().SanitizeCfiCrossDso && TypeId) {
+      EmitCfiSlowPathCheck(SanitizerKind::CFIICall, BitSetTest, TypeId,
+                           CastedCallee, StaticData);
+    } else {
       EmitCheck(std::make_pair(BitSetTest, SanitizerKind::CFIICall),
-                "cfi_bad_icall", StaticData, CastedCallee);
+                "cfi_check_fail", StaticData, CastedCallee);
     }
   }
 
   CallArgList Args;
   if (Chain)
     Args.add(RValue::get(Builder.CreateBitCast(Chain, CGM.VoidPtrTy)),
              CGM.getContext().VoidPtrTy);
   EmitCallArgs(Args, dyn_cast<FunctionProtoType>(FnType), E->arguments(),

lib/CodeGen/CodeGenFunction.h

public:
 
   void InitializeVTablePointers(const CXXRecordDecl *ClassDecl);
 
   /// GetVTablePtr - Return the Value of the vtable pointer member pointed
   /// to by This.
   llvm::Value *GetVTablePtr(Address This, llvm::Type *VTableTy,
                             const CXXRecordDecl *VTableClass);
 
-  enum CFITypeCheckKind {
+  enum CFICheckKind {
     CFITCK_VCall,
     CFITCK_NVCall,
     CFITCK_DerivedCast,
     CFITCK_UnrelatedCast,
+    CFITCK_ICall,
   };
 
   /// \brief Derived is the presumed address of an object of type T after a
   /// cast. If T is a polymorphic class type, emit a check that the virtual
   /// table for Derived belongs to a class derived from T.
   void EmitVTablePtrCheckForCast(QualType T, llvm::Value *Derived,
-                                 bool MayBeNull, CFITypeCheckKind TCK,
+                                 bool MayBeNull, CFICheckKind TCK,
                                  SourceLocation Loc);
 
   /// EmitVTablePtrCheckForCall - Virtual method MD is being called via VTable.
   /// If vptr CFI is enabled, emit a check that VTable is valid.
   void EmitVTablePtrCheckForCall(const CXXMethodDecl *MD, llvm::Value *VTable,
-                                 CFITypeCheckKind TCK, SourceLocation Loc);
+                                 CFICheckKind TCK, SourceLocation Loc);
 
   /// EmitVTablePtrCheck - Emit a check that VTable is a valid virtual table for
   /// RD using llvm.bitset.test.
   void EmitVTablePtrCheck(const CXXRecordDecl *RD, llvm::Value *VTable,
-                          CFITypeCheckKind TCK, SourceLocation Loc);
+                          CFICheckKind TCK, SourceLocation Loc);
 
   /// CanDevirtualizeMemberFunctionCalls - Checks whether virtual calls on given
   /// expr can be devirtualized.
   bool CanDevirtualizeMemberFunctionCall(const Expr *Base,
                                          const CXXMethodDecl *MD);
 
   /// EnterDtorCleanups - Enter the cleanups necessary to complete the
   /// given phase of destruction for a destructor.  The end result
public:
   /// sanitizer runtime with the provided arguments, and create a conditional
   /// branch to it.
   void EmitCheck(ArrayRef<std::pair<llvm::Value *, SanitizerMask>> Checked,
                  StringRef CheckName, ArrayRef<llvm::Constant *> StaticArgs,
                  ArrayRef<llvm::Value *> DynamicArgs);
 
   /// \brief Emit a slow path cross-DSO CFI check which calls __cfi_slowpath
   /// if Cond if false.
-  void EmitCfiSlowPathCheck(llvm::Value *Cond, llvm::ConstantInt *TypeId,
-                            llvm::Value *Ptr);
+  void EmitCfiSlowPathCheck(SanitizerMask Kind, llvm::Value *Cond,
+                            llvm::ConstantInt *TypeId, llvm::Value *Ptr,
+                            ArrayRef<llvm::Constant *> StaticArgs);
 
   /// \brief Create a basic block that will call the trap intrinsic, and emit a
   /// conditional branch to it, for the -ftrapv checks.
   void EmitTrapCheck(llvm::Value *Checked);
 
   /// \brief Emit a call to trap or debugtrap and attach function attribute
   /// "trap-func-name" if specified.
   llvm::CallInst *EmitTrapCall(llvm::Intrinsic::ID IntrID);
 
+  /// \brief Emit a cross-DSO CFI failure handling function.
+  void EmitCfiCheckFail();
+
   /// \brief Create a check for a function parameter that may potentially be
   /// declared as non-null.
   void EmitNonNullArgCheck(RValue RV, QualType ArgType, SourceLocation ArgLoc,
                            const FunctionDecl *FD, unsigned ParmNum);
 
   /// EmitCallArg - Emit a single call argument.
   void EmitCallArg(CallArgList &args, const Expr *E, QualType ArgType);
 

lib/CodeGen/CodeGenModule.cpp

void CodeGenModule::Release() {
   }
   EmitCtorList(GlobalCtors, "llvm.global_ctors");
   EmitCtorList(GlobalDtors, "llvm.global_dtors");
   EmitGlobalAnnotations();
   EmitStaticExternCAliases();
   EmitDeferredUnusedCoverageMappings();
   if (CoverageMapping)
     CoverageMapping->emit();
+  if (CodeGenOpts.SanitizeCfiCrossDso)
+    CodeGenFunction(*this).EmitCfiCheckFail();
   emitLLVMUsed();
 
   if (CodeGenOpts.Autolink &&
       (Context.getLangOpts().Modules || !LinkerOptionsMetadata.empty())) {
     EmitModuleLinkOptions();
   }
   if (CodeGenOpts.DwarfVersion) {
     // We actually want the latest version when there are conflicts.

lib/Driver/Tools.cpp

collectSanitizerRuntimes(const ToolChain &TC, const ArgList &Args,
   }
   if (SanArgs.needsUbsanRt()) {
     StaticRuntimes.push_back("ubsan_standalone");
     if (SanArgs.linkCXXRuntimes())
       StaticRuntimes.push_back("ubsan_standalone_cxx");
   }
   if (SanArgs.needsSafeStackRt())
     StaticRuntimes.push_back("safestack");
-  if (SanArgs.needsCfiRt())
+  if (SanArgs.needsCfiRt()) {
     StaticRuntimes.push_back("cfi");
-  if (SanArgs.needsCfiDiagRt())
+  }
+  if (SanArgs.needsCfiDiagRt()) {
     StaticRuntimes.push_back("cfi_diag");
+    if (SanArgs.linkCXXRuntimes())
+      StaticRuntimes.push_back("ubsan_standalone_cxx");
+  }
 }
 
 // Should be called before we add system libraries (C++ ABI, libstdc++/libc++,
 // C runtime, etc). Returns true if sanitizer system deps need to be linked in.
 static bool addSanitizerRuntimes(const ToolChain &TC, const ArgList &Args,
                                  ArgStringList &CmdArgs) {
   SmallVector<StringRef, 4> SharedRuntimes, StaticRuntimes,
       HelperStaticRuntimes;

test/CodeGen/cfi-check-fail.c

+// RUN: %clang_cc1 -triple x86_64-unknown-linux -O0 -fsanitize=cfi-icall -fsanitize-cfi-cross-dso \
+// RUN:     -fsanitize-trap=cfi-icall,cfi-nvcall -fsanitize-recover=cfi-vcall,cfi-unrelated-cast \
+// RUN:     -emit-llvm -o - %s | FileCheck %s
+
+void caller(void (*f)()) {
+  f();
+}
+
+// CHECK: define weak_odr hidden void @__cfi_check_fail(i8*, i8*) {
+// CHECK: store i8* %0, i8** %[[ALLOCA0:.*]], align 8
+// CHECK: store i8* %1, i8** %[[ALLOCA1:.*]], align 8
+// CHECK: %[[DATA:.*]] = load i8*, i8** %[[ALLOCA0]], align 8
+// CHECK: %[[ADDR:.*]] = load i8*, i8** %[[ALLOCA1]], align 8
+// CHECK: %[[ICMP_NULL:.*]] = icmp eq i8* %[[DATA]], null
+// CHECK: br i1 %[[ICMP_NULL]], label %[[TRAP:.*]], label %[[CONT:.*]]
+
+// CHECK:      [[CONT]]:
+// CHECK:        %[[A:.*]] = bitcast i8* %[[DATA]] to { i8, { i8*, i32, i32 }, i8* }*
+// CHECK:        %[[KINDPTR:.*]] = getelementptr {{.*}} %[[A]], i32 0, i32 0
+// CHECK:        %[[KIND:.*]] = load i8, i8* %[[KINDPTR]], align 4
+// CHECK:        switch i8 %[[KIND]], label %[[TRAP]] [
+// CHECK-NEXT:     i8 0, label %[[NON_FATAL:.*]]
+// CHECK-NEXT:     i8 1, label %[[TRAP]]
+// CHECK-NEXT:     i8 2, label %[[FATAL:.*]]
+// CHECK-NEXT:     i8 3, label %[[NON_FATAL]]
+// CHECK-NEXT:     i8 4, label %[[TRAP]]
+// CHECK-NEXT:   ]
+
+// CHECK: [[TRAP]]:
+// CHECK:   call void @llvm.trap()
+// CHECK:   unreachable
+
+// CHECK: [[NON_FATAL]]:
+// CHECK:   call void @__ubsan_handle_cfi_check_fail(i8* %[[DATA]], i8* %[[ADDR]])
+// CHECK:   br label %[[EXIT:.*]]
+
+// CHECK: [[FATAL]]:
+// CHECK:   call void @__ubsan_handle_cfi_check_fail_abort(i8* %[[DATA]], i8* %[[ADDR]])
+// CHECK:   unreachable
+
+// CHECK: [[EXIT]]:
+// CHECK:   ret void

test/CodeGen/cfi-icall-cross-dso.c

 }
 Fn h1() {
   return &h;
 }
 
 inline void foo() {}
 void bar() { foo(); }
 
+// CHECK: @[[SRC:.*]] = private unnamed_addr constant {{.*}}cfi-icall-cross-dso.c\00
+// CHECK: @[[TYPE:.*]] = private unnamed_addr constant { i16, i16, [{{.*}} x i8] } { i16 -1, i16 0, [{{.*}} x i8] c"'void ()'\00"
+// CHECK: @[[DATA:.*]] = private unnamed_addr global {{.*}}@[[SRC]]{{.*}}@[[TYPE]]
+
+
 // ITANIUM: call i1 @llvm.bitset.test(i8* %{{.*}}, metadata !"_ZTSFvE"), !nosanitize
-// ITANIUM: call void @__cfi_slowpath(i64 6588678392271548388, i8* %{{.*}}) {{.*}}, !nosanitize
+// ITANIUM: call void @__cfi_slowpath_diag(i64 6588678392271548388, i8* %{{.*}}, {{.*}}@[[DATA]]{{.*}}, !nosanitize
 
 // MS: call i1 @llvm.bitset.test(i8* %{{.*}}, metadata !"?6AX@Z"), !nosanitize
-// MS: call void @__cfi_slowpath(i64 4195979634929632483, i8* %{{.*}}) {{.*}}, !nosanitize
+// MS: call void @__cfi_slowpath_diag(i64 4195979634929632483, i8* %{{.*}}, {{.*}}@[[DATA]]{{.*}}, !nosanitize
 
 // ITANIUM: define available_externally void @foo()
 // MS: define linkonce_odr void @foo()
 
 // Check that we emit both string and hash based bit set entries for static void g(),
 // and don't emit them for the declaration of h().
 
 // CHECK-NOT: !{!"{{.*}}", void ()* @h, i64 0}

test/CodeGenCXX/cfi-cross-dso.cpp

 // MS: @[[B_VTABLE:.*]] = private unnamed_addr constant [2 x i8*] {{.*}}@"\01??_R4B@?A@@6B@"{{.*}}@"\01?f@B@?A@@UEAAXXZ"
 
 // CHECK:   %[[VT:.*]] = load void (%struct.A*)**, void (%struct.A*)***
 // CHECK:   %[[VT2:.*]] = bitcast {{.*}}%[[VT]] to i8*, !nosanitize
 // ITANIUM:   %[[TEST:.*]] = call i1 @llvm.bitset.test(i8* %[[VT2]], metadata !"_ZTS1A"), !nosanitize
 // MS:   %[[TEST:.*]] = call i1 @llvm.bitset.test(i8* %[[VT2]], metadata !"?AUA@@"), !nosanitize
 // CHECK:   br i1 %[[TEST]], label %[[CONT:.*]], label %[[SLOW:.*]], {{.*}} !nosanitize
 // CHECK: [[SLOW]]
-// ITANIUM:   call void @__cfi_slowpath(i64 7004155349499253778, i8* %[[VT2]]) {{.*}} !nosanitize
-// MS:   call void @__cfi_slowpath(i64 -8005289897957287421, i8* %[[VT2]]) {{.*}} !nosanitize
+// ITANIUM:   call void @__cfi_slowpath_diag(i64 7004155349499253778, i8* %[[VT2]], {{.*}}) {{.*}} !nosanitize
+// MS:   call void @__cfi_slowpath_diag(i64 -8005289897957287421, i8* %[[VT2]], {{.*}}) {{.*}} !nosanitize
 // CHECK:   br label %[[CONT]], !nosanitize
 // CHECK: [[CONT]]
 // CHECK:   call void %{{.*}}(%struct.A* %{{.*}})
 
 // No hash-based bit set entry for (anonymous namespace)::B
 // ITANIUM-NOT: !{i64 {{.*}}, [3 x i8*]* @_ZTVN12_GLOBAL__N_11BE,
 // MS-NOT: !{i64 {{.*}}, [2 x i8*]* @[[B_VTABLE]],

test/CodeGenCXX/cfi-vcall.cpp

 void D::f() {
 }
 
 void D::h() {
 }
 
 // DIAG: @[[SRC:.*]] = private unnamed_addr constant [{{.*}} x i8] c"{{.*}}cfi-vcall.cpp\00", align 1
 // DIAG: @[[TYPE:.*]] = private unnamed_addr constant { i16, i16, [4 x i8] } { i16 -1, i16 0, [4 x i8] c"'A'\00" }
-// DIAG: @[[BADTYPESTATIC:.*]] = private unnamed_addr global { { [{{.*}} x i8]*, i32, i32 }, { i16, i16, [4 x i8] }*, i8 } { { [{{.*}} x i8]*, i32, i32 } { [{{.*}} x i8]* @[[SRC]], i32 [[@LINE+21]], i32 3 }, { i16, i16, [4 x i8] }* @[[TYPE]], i8 0 }
+// DIAG: @[[BADTYPESTATIC:.*]] = private unnamed_addr global { i8, { [{{.*}} x i8]*, i32, i32 }, { i16, i16, [4 x i8] }* } { i8 0, { [{{.*}} x i8]*, i32, i32 } { [{{.*}} x i8]* @[[SRC]], i32 [[@LINE+21]], i32 3 }, { i16, i16, [4 x i8] }* @[[TYPE]] }
 
 // ITANIUM: define void @_Z2afP1A
 // MS: define void @"\01?af@@YAXPEAUA@@@Z"
 void af(A *a) {
   // ITANIUM: [[P:%[^ ]*]] = call i1 @llvm.bitset.test(i8* [[VT:%[^ ]*]], metadata !"_ZTS1A")
   // MS: [[P:%[^ ]*]] = call i1 @llvm.bitset.test(i8* [[VT:%[^ ]*]], metadata !"?AUA@@")
   // CHECK-NEXT: br i1 [[P]], label %[[CONTBB:[^ ,]*]], label %[[TRAPBB:[^ ,]*]]
   // CHECK-NEXT: {{^$}}
 
   // CHECK: [[TRAPBB]]
   // NDIAG-NEXT: call void @llvm.trap()
   // NDIAG-NEXT: unreachable
   // DIAG-NEXT: [[VTINT:%[^ ]*]] = ptrtoint i8* [[VT]] to i64
-  // DIAG-ABORT-NEXT: call void @__ubsan_handle_cfi_bad_type_abort(i8* bitcast ({{.*}} @[[BADTYPESTATIC]] to i8*), i64 [[VTINT]])
+  // DIAG-ABORT-NEXT: call void @__ubsan_handle_cfi_check_fail_abort(i8* getelementptr inbounds ({{.*}} @[[BADTYPESTATIC]], i32 0, i32 0), i64 [[VTINT]])
   // DIAG-ABORT-NEXT: unreachable
-  // DIAG-RECOVER-NEXT: call void @__ubsan_handle_cfi_bad_type(i8* bitcast ({{.*}} @[[BADTYPESTATIC]] to i8*), i64 [[VTINT]])
+  // DIAG-RECOVER-NEXT: call void @__ubsan_handle_cfi_check_fail(i8* getelementptr inbounds ({{.*}} @[[BADTYPESTATIC]], i32 0, i32 0), i64 [[VTINT]])
   // DIAG-RECOVER-NEXT: br label %[[CONTBB]]
 
   // CHECK: [[CONTBB]]
   // CHECK: call void %
   a->f();
 }
 
 // ITANIUM: define internal void @_Z3df1PN12_GLOBAL__N_11DE