[asan] Fixed a bug causing a crash when redzone optimization kicked in on X86 with -asan-optimize-callbacks flag on.

[kstoimenov]

Summary

This change adds the ASan intrinsic to the list whihc are setting hasCopyImplyingStackAdjustment.

[kstoimenov]

Added tests.

[kstoimenov]

Switched to using AdjustsStack instead of HasCopyImplyingStackAdjustment.

[kstoimenov]

Removed dead code.

[kstoimenov]

Switched to AdjustsStack. PTAL.

[eugenis]

llvm/lib/Target/X86/X86ISelLowering.cpp

27014

Does the intrinsic have to be x86-specific? AFAIK it is fine to have x86-specific handling of a generic intrinsic.

llvm/test/CodeGen/X86/asan-check-memaccess-or.ll

8

CHECK-NEXT, or CHECK-NOT: something-%rbp to make sure we are not force-enabling the frame pointer

[kstoimenov]

Moved back to generic intrinsic.

[kstoimenov]

After rebase.

[kstoimenov]

Added a blank line back.

[kstoimenov]

llvm/lib/Target/X86/X86ISelLowering.cpp

27014

I reverted it back to a generic intrinsic.

[eugenis]

LGTM

llvm/lib/Target/X86/X86ISelLowering.cpp

27015

This comment simply repeats what the next line says, in the exact same words.
It would be better to briefly explain why this is needed instead.

[kstoimenov]

Fixed a comment.

[kstoimenov]

Fixed a comment.

llvm/lib/Target/X86/X86ISelLowering.cpp

case llvm::Intrinsic::x86_wrpkru: {
       SDLoc dl(Op);
       // Create a WRPKRU node, pass the input to the EAX parameter,  and pass 0
       // to the EDX and ECX parameters.
       return DAG.getNode(X86ISD::WRPKRU, dl, MVT::Other,
                          Op.getOperand(0), Op.getOperand(2),
                          DAG.getConstant(0, dl, MVT::i32),
                          DAG.getConstant(0, dl, MVT::i32));
     }
+    case llvm::Intrinsic::asan_check_memaccess: {
+      // Mark this as adjustsStack it will be lowered to a call.
+      DAG.getMachineFunction().getFrameInfo().setAdjustsStack(true);
+      // Don't do anything here, we will expand these intrinsics out later.
+      return Op;
+    }
     case llvm::Intrinsic::x86_flags_read_u32:
     case llvm::Intrinsic::x86_flags_read_u64:
     case llvm::Intrinsic::x86_flags_write_u32:
     case llvm::Intrinsic::x86_flags_write_u64: {
       // We need a frame pointer because this will get lowered to a PUSH/POP
       // sequence.
       MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
       MFI.setHasCopyImplyingStackAdjustment(true);

llvm/test/CodeGen/X86/asan-check-memaccess-add.ll

 ; RUN: llc < %s | FileCheck %s
 
 target triple = "x86_64-unknown-linux-gnu"
 
 define void @load1(i8* nocapture readonly %x) {
+; CHECK:              pushq   %rax
+; CHECK-NOT:          push    %rbp
 ; CHECK:              callq   __asan_check_load1_rn[[RN1:.*]]
 ; CHECK:              callq   __asan_check_store1_rn[[RN1]]
-; CHECK-NEXT:         retq
+; CHECK-NOT:          pop     %rbp
+; CHECK:              popq    %rax
   call void @llvm.asan.check.memaccess(i8* %x, i32 0)
   call void @llvm.asan.check.memaccess(i8* %x, i32 32)
   ret void
 }
 
 define void @load2(i16* nocapture readonly %x) {
+; CHECK:              pushq   %rax
+; CHECK-NOT:          push    %rbp
 ; CHECK:              callq   __asan_check_load2_rn[[RN2:.*]]
 ; CHECK:              callq   __asan_check_store2_rn[[RN2]]
-; CHECK-NEXT:         retq
+; CHECK-NOT:          pop     %rbp
+; CHECK:              popq    %rax
   %1 = ptrtoint i16* %x to i64
   %2 = bitcast i16* %x to i8*
   call void @llvm.asan.check.memaccess(i8* %2, i32 2)
   call void @llvm.asan.check.memaccess(i8* %2, i32 34)
   ret void
 }
 
 define void @load4(i32* nocapture readonly %x) {
+; CHECK:              pushq   %rax
+; CHECK-NOT:          push    %rbp
 ; CHECK:              callq   __asan_check_load4_rn[[RN4:.*]]
 ; CHECK:              callq   __asan_check_store4_rn[[RN4]]
-; CHECK-NEXT:         retq
+; CHECK-NOT:          pop     %rbp
+; CHECK:              popq    %rax
   %1 = ptrtoint i32* %x to i64
   %2 = bitcast i32* %x to i8*
   call void @llvm.asan.check.memaccess(i8* %2, i32 4)
   call void @llvm.asan.check.memaccess(i8* %2, i32 36)
   ret void
 }
 define void @load8(i64* nocapture readonly %x) {
+; CHECK:              pushq   %rax
+; CHECK-NOT:          push    %rbp
 ; CHECK:              callq   __asan_check_load8_rn[[RN8:.*]]
 ; CHECK:              callq   __asan_check_store8_rn[[RN8]]
-; CHECK-NEXT:         retq
+; CHECK-NOT:          pop     %rbp
+; CHECK:              popq    %rax
   %1 = ptrtoint i64* %x to i64
   %2 = bitcast i64* %x to i8*
   call void @llvm.asan.check.memaccess(i8* %2, i32 6)
   call void @llvm.asan.check.memaccess(i8* %2, i32 38)
   ret void
 }
 
 define void @load16(i128* nocapture readonly %x) {
+; CHECK:              pushq   %rax
+; CHECK-NOT:          push    %rbp
 ; CHECK:              callq   __asan_check_load16_rn[[RN16:.*]]
 ; CHECK:              callq   __asan_check_store16_rn[[RN16]]
-; CHECK-NEXT:         retq
+; CHECK-NOT:          pop     %rbp
+; CHECK:              popq    %rax
   %1 = ptrtoint i128* %x to i64
   %2 = bitcast i128* %x to i8*
   call void @llvm.asan.check.memaccess(i8* %2, i32 8)
   call void @llvm.asan.check.memaccess(i8* %2, i32 40)
   ret void
 }
 
 ; CHECK:              .type   __asan_check_load1_rn[[RN1]],@function

llvm/test/CodeGen/X86/asan-check-memaccess-or.ll

 ; RUN: llc < %s | FileCheck %s
 
 target triple = "x86_64-pc-win"
 
 define void @load1(i8* nocapture readonly %x) {
+; CHECK:              pushq   %rax
+; CHECK-NOT:          push    %rbp
 ; CHECK:              callq   __asan_check_load1_rn[[RN1:.*]]
 ; CHECK:              callq   __asan_check_store1_rn[[RN1]]
-; CHECK-NEXT:         retq
+; CHECK-NOT:          pop     %rbp
+; CHECK:              popq    %rax
   call void @llvm.asan.check.memaccess(i8* %x, i32 0)
   call void @llvm.asan.check.memaccess(i8* %x, i32 32)
   ret void
 }
 
 define void @load2(i16* nocapture readonly %x) {
+; CHECK:              pushq   %rax
+; CHECK-NOT:          push    %rbp
 ; CHECK:              callq   __asan_check_load2_rn[[RN2:.*]]
 ; CHECK:              callq   __asan_check_store2_rn[[RN2]]
-; CHECK-NEXT:         retq
+; CHECK-NOT:          pop     %rbp
+; CHECK:              popq    %rax
   %1 = ptrtoint i16* %x to i64
   %2 = bitcast i16* %x to i8*
   call void @llvm.asan.check.memaccess(i8* %2, i32 2)
   call void @llvm.asan.check.memaccess(i8* %2, i32 34)
   ret void
 }
 
 define void @load4(i32* nocapture readonly %x) {
+; CHECK:              pushq   %rax
+; CHECK-NOT:          push    %rbp
 ; CHECK:              callq   __asan_check_load4_rn[[RN4:.*]]
 ; CHECK:              callq   __asan_check_store4_rn[[RN4]]
-; CHECK-NEXT:         retq
+; CHECK-NOT:          pop     %rbp
+; CHECK:              popq    %rax
   %1 = ptrtoint i32* %x to i64
   %2 = bitcast i32* %x to i8*
   call void @llvm.asan.check.memaccess(i8* %2, i32 4)
   call void @llvm.asan.check.memaccess(i8* %2, i32 36)
   ret void
 }
 define void @load8(i64* nocapture readonly %x) {
+; CHECK:              pushq   %rax
+; CHECK-NOT:          push    %rbp
 ; CHECK:              callq   __asan_check_load8_rn[[RN8:.*]]
 ; CHECK:              callq   __asan_check_store8_rn[[RN8]]
-; CHECK-NEXT:         retq
+; CHECK-NOT:          pop     %rbp
+; CHECK:              popq    %rax
   %1 = ptrtoint i64* %x to i64
   %2 = bitcast i64* %x to i8*
   call void @llvm.asan.check.memaccess(i8* %2, i32 6)
   call void @llvm.asan.check.memaccess(i8* %2, i32 38)
   ret void
 }
 
 define void @load16(i128* nocapture readonly %x) {
+; CHECK:              pushq   %rax
+; CHECK-NOT:          push    %rbp
 ; CHECK:              callq   __asan_check_load16_rn[[RN16:.*]]
 ; CHECK:              callq   __asan_check_store16_rn[[RN16]]
-; CHECK-NEXT:         retq
+; CHECK-NOT:          pop     %rbp
+; CHECK:              popq    %rax
   %1 = ptrtoint i128* %x to i64
   %2 = bitcast i128* %x to i8*
   call void @llvm.asan.check.memaccess(i8* %2, i32 8)
   call void @llvm.asan.check.memaccess(i8* %2, i32 40)
   ret void
 }
 
 ; CHECK:              .type   __asan_check_load1_rn[[RN1]],@function