[ASan] Don't use stack malloc for 32-bit functions using inline asm

[rnk]

Summary

This prevents us from running out of registers in the backend.

Introducing stack malloc calls prevents the backend from recognizing the
inline asm operands as stack objects. When the backend recognizes a
stack object, it doesn't need to materialize the address of the memory
in a physical register. Instead it generates a simple SP-based memory
operand. Introducing a stack malloc forces the backend to find a free
register for every memory operand. 32-bit x86 simply doesn't have enough
registers for this to succeed in most cases.

[samsonov]

LGTM

lib/Transforms/Instrumentation/AddressSanitizer.cpp

void FunctionStackPoisoner::poisonStack() {
   // i.e. 32 bytes on 64-bit platforms and 16 bytes in 32-bit platforms.
   size_t MinHeaderSize = ASan.LongSize / 2;
   ASanStackFrameLayout L;
   ComputeASanStackFrameLayout(SVD, 1UL << Mapping.Scale, MinHeaderSize, &L);
   DEBUG(dbgs() << L.DescriptionString << " --- " << L.FrameSize << "\n");
   uint64_t LocalStackSize = L.FrameSize;
   bool DoStackMalloc =
       ClUseAfterReturn && LocalStackSize <= kMaxStackMallocSize;
-  // Don't do dynamic alloca in presence of inline asm: too often it
-  // makes assumptions on which registers are available.
+  // Don't do dynamic alloca in presence of inline asm: too often it makes
+  // assumptions on which registers are available. Don't do stack malloc in the
+  // presence of inline asm on 32-bit platforms for the same reason.
   bool DoDynamicAlloca = ClDynamicAllocaStack && !HasNonEmptyInlineAsm;
+  DoStackMalloc &= !HasNonEmptyInlineAsm || ASan.LongSize != 32;
 
   Value *StaticAlloca =
       DoDynamicAlloca ? nullptr : createAllocaForLayout(IRB, L, false);
 
   Value *FakeStack;
   Value *LocalStackBase;
 
   if (DoStackMalloc) {

test/Instrumentation/AddressSanitizer/X86/asm_cpuid.ll

+; RUN: opt < %s -asan -S -o %t.ll
+; RUN: FileCheck %s < %t.ll
+; RUN: llc < %t.ll | FileCheck %s --check-prefix=ASM
+
+target datalayout = "e-m:x-p:32:32-i64:64-f80:32-n8:16:32-S32"
+target triple = "i386-pc-windows-msvc"
+
+define void @MyCPUID(i32 %fxn, i32* %out) sanitize_address {
+  %fxn.ptr = alloca i32
+  %a.ptr = alloca i32
+  %b.ptr = alloca i32
+  %c.ptr = alloca i32
+  %d.ptr = alloca i32
+  store i32 %fxn, i32* %fxn.ptr
+  call void asm sideeffect inteldialect "xchg ebx, esi\0A\09mov eax, dword ptr $4\0A\09cpuid\0A\09mov dword ptr $0, eax\0A\09mov dword ptr $1, ebx\0A\09mov dword ptr $2, ecx\0A\09mov dword ptr $3, edx\0A\09xchg ebx, esi", "=*m,=*m,=*m,=*m,*m,~{eax},~{ebx},~{ecx},~{edx},~{esi},~{dirflag},~{fpsr},~{flags}"(i32* %a.ptr, i32* %b.ptr, i32* %c.ptr, i32* %d.ptr, i32* %fxn.ptr)
+
+  %a = load i32, i32* %a.ptr
+  %a.out = getelementptr inbounds i32, i32* %out, i32 0
+  store i32 %a, i32* %a.out
+
+  %b = load i32, i32* %b.ptr
+  %b.out = getelementptr inbounds i32, i32* %out, i32 1
+  store i32 %b, i32* %b.out
+
+  %c = load i32, i32* %c.ptr
+  %c.out = getelementptr inbounds i32, i32* %out, i32 2
+  store i32 %c, i32* %c.out
+
+  %d = load i32, i32* %d.ptr
+  %d.out = getelementptr inbounds i32, i32* %out, i32 3
+  store i32 %d, i32* %d.out
+
+  ret void
+}
+
+; We used to introduce stack mallocs for UAR detection, but that makes LLVM run
+; out of registers on 32-bit platforms. Therefore, we don't do stack malloc on
+; such functions.
+
+; CHECK-LABEL: define void @MyCPUID(i32 %fxn, i32* %out)
+; CHECK: %MyAlloca = alloca [96 x i8], align 32
+; CHECK-NOT: call {{.*}} @__asan_stack_malloc
+
+; The code generator should recognize that all operands are just stack memory.
+; This is important with MS inline asm where operand lists are implicit and all
+; local variables can be referenced freely.
+
+; ASM-LABEL: MyCPUID:
+; ASM:      cpuid
+; ASM-NEXT: movl    %eax, {{[0-9]+}}(%esp)
+; ASM-NEXT: movl    %ebx, {{[0-9]+}}(%esp)
+; ASM-NEXT: movl    %ecx, {{[0-9]+}}(%esp)
+; ASM-NEXT: movl    %edx, {{[0-9]+}}(%esp)