CodeGen: introduce support for Windows SSP on x86/x64

[compnerd]

Summary

This adds MSVCRT style SSP support (/GS). This is incomplete as the cases where
it triggers is not as comprehensive as what MSVC does. This simply sets up the
prologue and epilogue appropriately for the use of the "Buffer Security Checks"
in MSVCRT. It triggers only on MSVCRT style Windows targets (i.e. it excludes
MinGW/cygwin).

The use of the llvm.stackprotector intrinsics was avoided as the implementation
details of SSP is different on Windows. A future change could rename the
current SDAG node and construct a new one specific for the windows behaviour if
the SDAG approach is preferable. This seems just as clean as we can fully
describe the necessary behaviour in LLVM IR.

A special thanks to David Majnemer for pointing out the need for inreg on
parameters for fastcall.

[rnk]

I don't think calling llvm.read_register("esp/rsp") in LLVM IR is going to work. There's no guarantee that the results of those calls will be the same in the prologue and the epilogue due to dynamic allocas.

If we want to get the right SP, we need to get CodeGen so we can put the check at the right point in the epilogue, prologue, and tail call sequence.

lib/CodeGen/StackProtector.cpp

if (TLI->getStackCookieLocation(AddressSpace, Offset)) {
 
     StackGuardVar =
         ConstantExpr::getIntToPtr(OffsetVal, PointerType::get(PtrTy,
                                                               AddressSpace));
   } else if (TT.isOSOpenBSD()) {
     StackGuardVar = M->getOrInsertGlobal("__guard_local", PtrTy);
     cast<GlobalValue>(StackGuardVar)
         ->setVisibility(GlobalValue::HiddenVisibility);
+  } else if (TT.isOSMSVCRT()) {
+    LLVMContext &Context = RI->getContext();
+    Type *ArgTy = Type::getInt32Ty(Context);
+    MDString *SPReg = MDString::get(Context, "esp");
+    Function *RRTy =
+        Intrinsic::getDeclaration(M, Intrinsic::read_register, ArgTy);
+
+    switch (TT.getArch()) {
+    default:
+      report_fatal_error("unsupported arch for Windows stack protection");
+    case Triple::x86_64:
+      ArgTy = Type::getInt64Ty(Context);
+      SPReg = MDString::get(Context, "rsp");
+      RRTy = Intrinsic::getDeclaration(M, Intrinsic::read_register, ArgTy);
+      /* fallthrough */
+    case Triple::x86: {
+      Value *Cookie = M->getOrInsertGlobal("__security_cookie", ArgTy);
+
+      IRBuilder<> IRB(&F->getEntryBlock().front());
+
+      /// %StackGuardSlot = alloca [i32|i64]
+      /// %SecurityCookie = load [i32|i64]* @__security_cookie
+      /// %StackPointer = call [i32|i64]* @llvm.read_register.[i32|i64](metadata !0)
+      /// %Canary = xor [i32|i64] %SecurityCookie, %StackPointer
+      /// store [i32|i64] %Canary, [i32|i64]* %StackGuardSlot
+      AI = IRB.CreateAlloca(ArgTy, nullptr, "StackGuardSlot");
+      LoadInst *SecurityCookie = IRB.CreateLoad(Cookie, "SecurityCookie");
+      Value *R = MetadataAsValue::get(Context, MDNode::get(Context, SPReg));
+      CallInst *StackPointer = IRB.CreateCall(RRTy, R, "StackPointer");
+      Value *Canary = IRB.CreateXor(SecurityCookie, StackPointer, "Canary");
+      IRB.CreateStore(Canary, AI);
+
+      break;
+    }
+    }
+
+    return false;
   } else {
     SupportsSelectionDAGSP = true;
     StackGuardVar = M->getOrInsertGlobal("__stack_chk_guard", PtrTy);
   }
 
   IRBuilder<> B(&F->getEntryBlock().front());
   AI = B.CreateAlloca(PtrTy, nullptr, "StackGuardSlot");
   LoadInst *LI = B.CreateLoad(StackGuardVar, "StackGuard");
for (Function::iterator I = F->begin(), E = F->end(); I != E;) {
 
     if (!HasPrologue) {
       HasPrologue = true;
       SupportsSelectionDAGSP &=
           CreatePrologue(F, M, RI, TLI, Trip, AI, StackGuardVar);
     }
 
     if (SupportsSelectionDAGSP) {
+      assert(!Trip.isOSMSVCRT() && "MSVCRT SSP cannot be lowered via SelDAG");
+
       // Since we have a potential tail call, insert the special stack check
       // intrinsic.
       Instruction *InsertionPt = nullptr;
       if (CallInst *CI = FindPotentialTailCall(BB, RI, TLI)) {
         InsertionPt = CI;
       } else {
         InsertionPt = RI;
         // At this point we know that BB has a return statement so it *DOES*
         // have a terminator.
         assert(InsertionPt != nullptr &&
                "BB must have a terminator instruction at this point.");
       }
 
       Function *Intrinsic =
           Intrinsic::getDeclaration(M, Intrinsic::stackprotectorcheck);
       CallInst::Create(Intrinsic, StackGuardVar, "", InsertionPt);
     } else {
+      LLVMContext &Context = RI->getContext();
+      Type *ArgTy = Type::getInt32Ty(Context);
+      MDString *SPReg = MDString::get(Context, "esp");
+      Function *RRTy =
+          Intrinsic::getDeclaration(M, Intrinsic::read_register, ArgTy);
+
+      if (Trip.isOSMSVCRT()) {
+        switch (Trip.getArch()) {
+        default:
+          report_fatal_error("unsupported arch for Windows stack protection");
+        case Triple::x86_64:
+          ArgTy = Type::getInt64Ty(RI->getContext());
+          SPReg = MDString::get(Context, "rsp");
+          RRTy = Intrinsic::getDeclaration(M, Intrinsic::read_register, ArgTy);
+          /* fallthrough */
+        case Triple::x86: {
+          Constant *SecurityCheckCookie =
+              M->getOrInsertFunction("__security_check_cookie",
+                                     Type::getVoidTy(Context), ArgTy, nullptr);
+          if (Trip.getArch() == Triple::x86) {
+            Function *F = cast<Function>(SecurityCheckCookie);
+            F->setCallingConv(CallingConv::X86_FastCall);
+            F->addAttribute(1, Attribute::InReg);
+          }
+
+          IRBuilder<> IRB(RI);
+
+          /// %SavedCanary = load [i32|i64]* %StackGuardSlot
+          /// %StackPointer = call [i32|i64] @llvm.read_register.[i32|o64](metadata !0)
+          /// %Cookie = xor [i32|i64] %SavedCanary, %StackPointer
+          /// call void @__security_check_cookie([i32|i64] %Cookie)
+          LoadInst *SavedCanary = IRB.CreateLoad(AI, "SavedCanary");
+          Value *R = MetadataAsValue::get(Context, MDNode::get(Context, SPReg));
+          CallInst *StackPointer = IRB.CreateCall(RRTy, R, "StackPointer");
+          Value *Cookie = IRB.CreateXor(SavedCanary, StackPointer, "Cookie");
+          CallInst *Check = IRB.CreateCall(SecurityCheckCookie, Cookie);
+          if (Trip.getArch() == Triple::x86) {
+            Check->setCallingConv(CallingConv::X86_FastCall);
+            Check->addAttribute(1, Attribute::InReg);
+          }
+        }
+        }
+        continue;
+      }
+
       // If we do not support SelectionDAG based tail calls, generate IR level
       // tail calls.
       //
       // For each block with a return instruction, convert this:
       //
       //   return:
       //     ...
       //     ret ...

test/CodeGen/X86/windows-ssp-x64.ll

+; RUN: llc -mtriple x86_64-windows-msvc -filetype asm -o - %s | FileCheck %s
+
+target triple = "x86_64--windows-msvc"
+
+@__security_cookie = external global i64
+
+declare i64 @llvm.read_register.i64(metadata)
+declare void @__security_check_cookie(i64)
+declare void @memset(i8*, i32, i64)
+
+define void @function() {
+  %StackGuardSlot = alloca i64
+  %SecurityCookie = load i64* @__security_cookie
+  %StackPointer = call i64 @llvm.read_register.i64(metadata !0)
+  %Canary = xor i64 %SecurityCookie, %StackPointer
+  store i64 %Canary, i64* %StackGuardSlot
+
+  %buffer = alloca [8 x i8], align 1
+  %arraydecay = bitcast [8 x i8]* %buffer to i8*
+  call void @memset(i8* %arraydecay, i32 0, i64 9)
+
+  %SavedCanary = load i64* %StackGuardSlot
+  %StackPointer1 = call i64 @llvm.read_register.i64(metadata !0)
+  %Cookie = xor i64 %SavedCanary, %StackPointer1
+  call void @__security_check_cookie(i64 %Cookie)
+
+  ret void
+}
+
+; CHECK-LABEL: function
+
+; CHECK: movq %rsp, %[[RSI:r..]]
+; CHECK-NEXT: movq __security_cookie(%rip), %[[RAX:r..]]
+; CHECK-NEXT: xorq %[[RSI]], %[[RAX]]
+; CHECK-NEXT: movq %[[RAX]], [[OFFSET:[0-9]+]](%rsp)
+
+; CHECK: callq memset
+
+; CHECK: xorq [[OFFSET]](%rsp), %[[RSI]]
+; CHECK-NEXT: movq %[[RSI]], %rcx
+; CHECK-NEXT: callq __security_check_cookie
+
+define i32 @main() {
+  call void @function()
+  ret i32 0
+}
+
+!0 = !{!"rsp\00"}
+

test/CodeGen/X86/windows-ssp-x86.ll

+; RUN: llc -mtriple i686-windows-msvc -filetype asm -o - %s | FileCheck %s
+
+target triple = "i686--windows-msvc"
+
+@__security_cookie = external global i32
+
+declare i32 @llvm.read_register.i32(metadata)
+declare x86_fastcallcc void @__security_check_cookie(i32 inreg)
+declare void @memset(i8*, i32, i32)
+
+define void @function() {
+  %StackGuardSlot = alloca i32
+  %SecurityCookie = load i32* @__security_cookie
+  %StackPointer = call i32 @llvm.read_register.i32(metadata !0)
+  %Canary = xor i32 %SecurityCookie, %StackPointer
+  store i32 %Canary, i32* %StackGuardSlot
+
+  %buffer = alloca [8 x i8], align 1
+  %arraydecay = bitcast [8 x i8]* %buffer to i8*
+  call void @memset(i8* %arraydecay, i32 0, i32 9)
+
+  %SavedCanary = load i32* %StackGuardSlot
+  %StackPointer1 = call i32 @llvm.read_register.i32(metadata !0)
+  %Cookie = xor i32 %SavedCanary, %StackPointer1
+  call x86_fastcallcc void @__security_check_cookie(i32 inreg %Cookie)
+
+  ret void
+}
+
+; CHECK-LABEL: _function
+; CHECK: movl ___security_cookie, %[[EAX:e..]]
+; CHECK-NEXT: xorl %[[SP:e..]], %[[EAX]]
+; CHECK-NEXT: movl %[[EAX]], [[OFFSET:[0-9]+]](%esp)
+
+; CHECK: calll _memset
+
+; CHECK: xorl [[OFFSET]](%esp), %[[SP]]
+; CHECK-NEXT: movl %[[SP]], %ecx
+; CHECK-NEXT: calll @__security_check_cookie@4
+
+define i32 @main() {
+  call void @function()
+  ret i32 0
+}
+
+!0 = !{!"esp\00"}
+