[X86] Reject registers that require a REX prefix in inline asm constraints in 32-bit mode

[craig.topper]

Summary

We don't currently reject r8-r15 or xmm8-32 in 32-bit mode.

I think we're probably also missing blocking sil/dil/spl/bpl but that probably requires a different fix strategy.

[craig.topper]

Fix spl/bpl/sil/dil as well

Use existing isX86_64ExtendedReg and isX86_64NonExtLowByteReg to detect the registers we need to flag.

[RKSimon]

Can we check that x32 build still work?

[craig.topper]

Add RUN line for gnux32

[RKSimon]

LGTM - thanks

llvm/trunk/lib/Target/X86/X86ISelLowering.cpp

if (Constraint == "A") {
         Res.first = X86::EAX;
         Res.second = &X86::GR32_ADRegClass;
       }
       return Res;
     }
     return Res;
   }
 
+  // Make sure it isn't a register that requires 64-bit mode.
+  if (!Subtarget.is64Bit() &&
+      (isFRClass(*Res.second) || isGRClass(*Res.second)) &&
+      TRI->getEncodingValue(Res.first) >= 8) {
+    // Register requires REX prefix, but we're in 32-bit mode.
+    Res.first = 0;
+    Res.second = nullptr;
+    return Res;
+  }
+
   // Make sure it isn't a register that requires AVX512.
   if (!Subtarget.hasAVX512() && isFRClass(*Res.second) &&
       TRI->getEncodingValue(Res.first) & 0x10) {
     // Register requires EVEX prefix.
     Res.first = 0;
     Res.second = nullptr;
     return Res;
   }

llvm/trunk/test/CodeGen/X86/asm-reject-rex.ll

+; RUN: not llc -o /dev/null %s -mtriple=i386-unknown-unknown 2>&1 | FileCheck %s
+; Make sure X32 still works.
+; RUN: llc -o /dev/null %s -mtriple=x86_64-linux-gnux32
+
+; CHECK: error: couldn't allocate output register for constraint '{xmm8}'
+define i64 @blup() {
+  %v = tail call i64 asm "", "={xmm8},0"(i64 0)
+  ret i64 %v
+}
+
+; CHECK: error: couldn't allocate output register for constraint '{r8d}'
+define i32 @foo() {
+  %v = tail call i32 asm "", "={r8d},0"(i32 0)
+  ret i32 %v
+}
+
+; CHECK: error: couldn't allocate output register for constraint '{rax}'
+define i64 @bar() {
+  %v = tail call i64 asm "", "={rax},0"(i64 0)
+  ret i64 %v
+}

llvm/trunk/test/CodeGen/X86/asm-reject-xmm16.ll

-; RUN: not llc -o /dev/null %s 2>&1 | FileCheck %s
-target triple = "x86_64--"
+; RUN: not llc -o /dev/null %s -mtriple=x86_64-unknown-unknown 2>&1 | FileCheck %s
+; RUN: not llc -o /dev/null %s -mtriple=i386-unknown-unknown -mattr=avx512vl 2>&1 | FileCheck %s
 
 ; CHECK: error: couldn't allocate output register for constraint '{xmm16}'
 define i64 @blup() {
   %v = tail call i64 asm "", "={xmm16},0"(i64 0)
   ret i64 %v
 }