[asan] Fixed a bug causing a crash when redzone optimization kicked in on X86 with -asan-optimize-callbacks flag on.

[kstoimenov]

Summary

Added addtional check in PrologEpilogInserter.cpp which disables redzone optimization when ASan intrinsics are found in the code.

[kstoimenov]

After rebase.

llvm/lib/CodeGen/PrologEpilogInserter.cpp

for (MachineBasicBlock::iterator I = BB.begin(); I != BB.end(); ++I)
         if (Size > MaxCallFrameSize) MaxCallFrameSize = Size;
         AdjustsStack = true;
         FrameSDOps.push_back(I);
       } else if (I->isInlineAsm()) {
         // Some inline asm's need a stack frame, as indicated by operand 1.
         unsigned ExtraInfo = I->getOperand(InlineAsm::MIOp_ExtraInfo).getImm();
         if (ExtraInfo & InlineAsm::Extra_IsAlignStack)
           AdjustsStack = true;
+      } else if (TII.isStackAdjustIntrinsic(*I)) {
+        AdjustsStack = true;
       }
 
   assert(!MFI.isMaxCallFrameSizeComputed() ||
          (MFI.getMaxCallFrameSize() == MaxCallFrameSize &&
           MFI.adjustsStack() == AdjustsStack));
   MFI.setAdjustsStack(AdjustsStack);
   MFI.setMaxCallFrameSize(MaxCallFrameSize);
 

llvm/test/CodeGen/X86/asan-check-memaccess-add.ll

 ; RUN: llc < %s | FileCheck %s
 
 target triple = "x86_64-unknown-linux-gnu"
 
 define void @load1(i8* nocapture readonly %x) {
-; CHECK:              callq   __asan_check_load1_rn[[RN1:.*]]
-; CHECK:              callq   __asan_check_store1_rn[[RN1]]
+; CHECK:              pushq   %rax
+; CHECK-NEXT:         .cfi_def_cfa_offset 16
+; CHECK-NEXT:         callq   __asan_check_load1_rn[[RN1:.*]]
+; CHECK-NEXT:         callq   __asan_check_store1_rn[[RN1]]
+; CHECK-NEXT:         popq    %rax
+; CHECK-NEXT:         .cfi_def_cfa_offset 8
 ; CHECK-NEXT:         retq
   call void @llvm.asan.check.memaccess(i8* %x, i32 0)
   call void @llvm.asan.check.memaccess(i8* %x, i32 32)
   ret void
 }
 
 define void @load2(i16* nocapture readonly %x) {
-; CHECK:              callq   __asan_check_load2_rn[[RN2:.*]]
-; CHECK:              callq   __asan_check_store2_rn[[RN2]]
+; CHECK:              pushq   %rax
+; CHECK-NEXT:         .cfi_def_cfa_offset 16
+; CHECK-NEXT:         callq   __asan_check_load2_rn[[RN2:.*]]
+; CHECK-NEXT:         callq   __asan_check_store2_rn[[RN2]]
+; CHECK-NEXT:         popq    %rax
+; CHECK-NEXT:         .cfi_def_cfa_offset 8
 ; CHECK-NEXT:         retq
   %1 = ptrtoint i16* %x to i64
   %2 = bitcast i16* %x to i8*
   call void @llvm.asan.check.memaccess(i8* %2, i32 2)
   call void @llvm.asan.check.memaccess(i8* %2, i32 34)
   ret void
 }
 
 define void @load4(i32* nocapture readonly %x) {
-; CHECK:              callq   __asan_check_load4_rn[[RN4:.*]]
-; CHECK:              callq   __asan_check_store4_rn[[RN4]]
+; CHECK:              pushq   %rax
+; CHECK-NEXT:         .cfi_def_cfa_offset 16
+; CHECK-NEXT:         callq   __asan_check_load4_rn[[RN4:.*]]
+; CHECK-NEXT:         callq   __asan_check_store4_rn[[RN4]]
+; CHECK-NEXT:         popq    %rax
+; CHECK-NEXT:         .cfi_def_cfa_offset 8
 ; CHECK-NEXT:         retq
   %1 = ptrtoint i32* %x to i64
   %2 = bitcast i32* %x to i8*
   call void @llvm.asan.check.memaccess(i8* %2, i32 4)
   call void @llvm.asan.check.memaccess(i8* %2, i32 36)
   ret void
 }
 define void @load8(i64* nocapture readonly %x) {
-; CHECK:              callq   __asan_check_load8_rn[[RN8:.*]]
-; CHECK:              callq   __asan_check_store8_rn[[RN8]]
+; CHECK:              pushq   %rax
+; CHECK-NEXT:         .cfi_def_cfa_offset 16
+; CHECK-NEXT:         callq   __asan_check_load8_rn[[RN8:.*]]
+; CHECK-NEXT:         callq   __asan_check_store8_rn[[RN8]]
+; CHECK-NEXT:         popq    %rax
+; CHECK-NEXT:         .cfi_def_cfa_offset 8
 ; CHECK-NEXT:         retq
   %1 = ptrtoint i64* %x to i64
   %2 = bitcast i64* %x to i8*
   call void @llvm.asan.check.memaccess(i8* %2, i32 6)
   call void @llvm.asan.check.memaccess(i8* %2, i32 38)
   ret void
 }
 
 define void @load16(i128* nocapture readonly %x) {
-; CHECK:              callq   __asan_check_load16_rn[[RN16:.*]]
-; CHECK:              callq   __asan_check_store16_rn[[RN16]]
+; CHECK:              pushq   %rax
+; CHECK-NEXT:         .cfi_def_cfa_offset 16
+; CHECK-NEXT:         callq   __asan_check_load16_rn[[RN16:.*]]
+; CHECK-NEXT:         callq   __asan_check_store16_rn[[RN16]]
+; CHECK-NEXT:         popq    %rax
+; CHECK-NEXT:         .cfi_def_cfa_offset 8
 ; CHECK-NEXT:         retq
   %1 = ptrtoint i128* %x to i64
   %2 = bitcast i128* %x to i8*
   call void @llvm.asan.check.memaccess(i8* %2, i32 8)
   call void @llvm.asan.check.memaccess(i8* %2, i32 40)
   ret void
 }
 

llvm/test/CodeGen/X86/asan-check-memaccess-or.ll

 ; RUN: llc < %s | FileCheck %s
 
 target triple = "x86_64-pc-win"
 
 define void @load1(i8* nocapture readonly %x) {
-; CHECK:              callq   __asan_check_load1_rn[[RN1:.*]]
-; CHECK:              callq   __asan_check_store1_rn[[RN1]]
+; CHECK:              pushq   %rax
+; CHECK-NEXT:         .cfi_def_cfa_offset 16
+; CHECK-NEXT:         callq   __asan_check_load1_rn[[RN1:.*]]
+; CHECK-NEXT:         callq   __asan_check_store1_rn[[RN1]]
+; CHECK-NEXT:         popq    %rax
+; CHECK-NEXT:         .cfi_def_cfa_offset 8
 ; CHECK-NEXT:         retq
   call void @llvm.asan.check.memaccess(i8* %x, i32 0)
   call void @llvm.asan.check.memaccess(i8* %x, i32 32)
   ret void
 }
 
 define void @load2(i16* nocapture readonly %x) {
-; CHECK:              callq   __asan_check_load2_rn[[RN2:.*]]
-; CHECK:              callq   __asan_check_store2_rn[[RN2]]
+; CHECK:              pushq   %rax
+; CHECK-NEXT:         .cfi_def_cfa_offset 16
+; CHECK-NEXT:         callq   __asan_check_load2_rn[[RN2:.*]]
+; CHECK-NEXT:         callq   __asan_check_store2_rn[[RN2]]
+; CHECK-NEXT:         popq    %rax
+; CHECK-NEXT:         .cfi_def_cfa_offset 8
 ; CHECK-NEXT:         retq
   %1 = ptrtoint i16* %x to i64
   %2 = bitcast i16* %x to i8*
   call void @llvm.asan.check.memaccess(i8* %2, i32 2)
   call void @llvm.asan.check.memaccess(i8* %2, i32 34)
   ret void
 }
 
 define void @load4(i32* nocapture readonly %x) {
-; CHECK:              callq   __asan_check_load4_rn[[RN4:.*]]
-; CHECK:              callq   __asan_check_store4_rn[[RN4]]
+; CHECK:              pushq   %rax
+; CHECK-NEXT:         .cfi_def_cfa_offset 16
+; CHECK-NEXT:         callq   __asan_check_load4_rn[[RN4:.*]]
+; CHECK-NEXT:         callq   __asan_check_store4_rn[[RN4]]
+; CHECK-NEXT:         popq    %rax
+; CHECK-NEXT:         .cfi_def_cfa_offset 8
 ; CHECK-NEXT:         retq
   %1 = ptrtoint i32* %x to i64
   %2 = bitcast i32* %x to i8*
   call void @llvm.asan.check.memaccess(i8* %2, i32 4)
   call void @llvm.asan.check.memaccess(i8* %2, i32 36)
   ret void
 }
 define void @load8(i64* nocapture readonly %x) {
-; CHECK:              callq   __asan_check_load8_rn[[RN8:.*]]
-; CHECK:              callq   __asan_check_store8_rn[[RN8]]
+; CHECK:              pushq   %rax
+; CHECK-NEXT:         .cfi_def_cfa_offset 16
+; CHECK-NEXT:         callq   __asan_check_load8_rn[[RN8:.*]]
+; CHECK-NEXT:         callq   __asan_check_store8_rn[[RN8]]
+; CHECK-NEXT:         popq    %rax
+; CHECK-NEXT:         .cfi_def_cfa_offset 8
 ; CHECK-NEXT:         retq
   %1 = ptrtoint i64* %x to i64
   %2 = bitcast i64* %x to i8*
   call void @llvm.asan.check.memaccess(i8* %2, i32 6)
   call void @llvm.asan.check.memaccess(i8* %2, i32 38)
   ret void
 }
 
 define void @load16(i128* nocapture readonly %x) {
-; CHECK:              callq   __asan_check_load16_rn[[RN16:.*]]
-; CHECK:              callq   __asan_check_store16_rn[[RN16]]
+; CHECK:              pushq   %rax
+; CHECK-NEXT:         .cfi_def_cfa_offset 16
+; CHECK-NEXT:         callq   __asan_check_load16_rn[[RN16:.*]]
+; CHECK-NEXT:         callq   __asan_check_store16_rn[[RN16]]
+; CHECK-NEXT:         popq    %rax
+; CHECK-NEXT:         .cfi_def_cfa_offset 8
 ; CHECK-NEXT:         retq
   %1 = ptrtoint i128* %x to i64
   %2 = bitcast i128* %x to i8*
   call void @llvm.asan.check.memaccess(i8* %2, i32 8)
   call void @llvm.asan.check.memaccess(i8* %2, i32 40)
   ret void
 }
-
 ; CHECK:              .type   __asan_check_load1_rn[[RN1]],@function
 ; CHECK-NEXT:         .weak   __asan_check_load1_rn[[RN1]]
 ; CHECK-NEXT:         .hidden __asan_check_load1_rn[[RN1]]
 ; CHECK-NEXT: __asan_check_load1_rn[[RN1]]:
 ; CHECK-NEXT:         movq    [[REG:.*]], %r8
 ; CHECK-NEXT:         shrq    $3, %r8
 ; CHECK-NEXT:         orq     $17592186044416, %r8{{.*}}
 ; CHECK-NEXT:         movb    (%r8), %r8b