Make __shiftleft128 / __shiftright128 real compiler built-ins.

[thakis]

Summary

r337619 added __shiftleft128 / __shiftright128 as functions in intrin.h. Microsoft's STL plans on using these functions, and they're using intrin0.h which just has declarations of built-ins to not pull in the huge intrin.h header in the standard library headers. That requires that these functions are real built-ins.

[thakis]

(I tried using funnelshift for this as mentioned in the FIXME, but it looks like it still produces a bunch of unnecessary code if the shift amount isn't constant.)

[thakis]

Oh, and I verified that this still gets compiled to shld / shrd:

Nicos-MacBook-Pro:llvm-mono thakis$ cat test.c
#include <intrin.h>

unsigned __int64 test__shiftleft128(unsigned __int64 l, unsigned __int64 h,
                                    unsigned char d) {
  return __shiftleft128(l, h, d);
}

unsigned __int64 test__shiftright128(unsigned __int64 l, unsigned __int64 h,
                                     unsigned char d) {
  return __shiftright128(l, h, d);
}
Nicos-MacBook-Pro:llvm-mono thakis$ ./clang-cl.sh /O2 /c test.c   /FA ; cat test.asm
	.text
	.intel_syntax noprefix
	.def	 test__shiftleft128;
	.scl	2;
	.type	32;
	.endef
	.section	.text,"xr",one_only,test__shiftleft128
	.globl	test__shiftleft128      # -- Begin function test__shiftleft128
	.p2align	4, 0x90
test__shiftleft128:                     # @test__shiftleft128
# %bb.0:                                # %entry
	mov	rax, rcx
	mov	ecx, r8d
	shld	rdx, rax, cl
	mov	rax, rdx
	ret
                                        # -- End function
	.def	 test__shiftright128;
	.scl	2;
	.type	32;
	.endef
	.section	.text,"xr",one_only,test__shiftright128
	.globl	test__shiftright128     # -- Begin function test__shiftright128
	.p2align	4, 0x90
test__shiftright128:                    # @test__shiftright128
# %bb.0:                                # %entry
	mov	rax, rcx
	mov	ecx, r8d
	shrd	rax, rdx, cl
	ret
                                        # -- End function
	.section	.drectve,"yn"
	.ascii	" /DEFAULTLIB:libcmt.lib"
	.ascii	" /DEFAULTLIB:oldnames.lib"

[thakis]

Forgot to add cfe-commits :-( Doing that now.

[craig.topper]

LGTM

[thakis]

r340048, thanks!

[rnk]

lgtm

clang/include/clang/Basic/BuiltinsX86_64.def

 TARGET_HEADER_BUILTIN(_BitScanReverse64, "UcUNi*ULLi", "nh", "intrin.h", ALL_MS_LANGUAGES, "")
 
 TARGET_HEADER_BUILTIN(__mulh,  "LLiLLiLLi",    "nch", "intrin.h", ALL_MS_LANGUAGES, "")
 TARGET_HEADER_BUILTIN(__umulh, "ULLiULLiULLi", "nch", "intrin.h", ALL_MS_LANGUAGES, "")
 TARGET_HEADER_BUILTIN(_mul128, "LLiLLiLLiLLi*",      "nch",   "intrin.h", ALL_MS_LANGUAGES, "")
 TARGET_HEADER_BUILTIN(_umul128, "ULLiULLiULLiULLi*", "nch",   "intrin.h", ALL_MS_LANGUAGES, "")
 
 TARGET_HEADER_BUILTIN(__faststorefence, "v", "nh", "intrin.h", ALL_MS_LANGUAGES, "")
+TARGET_HEADER_BUILTIN(__shiftleft128, "ULLiULLiULLiUc", "nch", "intrin.h", ALL_MS_LANGUAGES, "")
+TARGET_HEADER_BUILTIN(__shiftright128, "ULLiULLiULLiUc", "nch", "intrin.h", ALL_MS_LANGUAGES, "")
 
 TARGET_HEADER_BUILTIN(_InterlockedAnd64,         "LLiLLiD*LLi", "nh", "intrin.h", ALL_MS_LANGUAGES, "")
 TARGET_HEADER_BUILTIN(_InterlockedDecrement64,   "LLiLLiD*",    "nh", "intrin.h", ALL_MS_LANGUAGES, "")
 TARGET_HEADER_BUILTIN(_InterlockedExchange64,    "LLiLLiD*LLi", "nh", "intrin.h", ALL_MS_LANGUAGES, "")
 TARGET_HEADER_BUILTIN(_InterlockedExchangeAdd64, "LLiLLiD*LLi", "nh", "intrin.h", ALL_MS_LANGUAGES, "")
 TARGET_HEADER_BUILTIN(_InterlockedExchangeSub64, "LLiLLiD*LLi", "nh", "intrin.h", ALL_MS_LANGUAGES, "")
 TARGET_HEADER_BUILTIN(_InterlockedIncrement64,   "LLiLLiD*",    "nh", "intrin.h", ALL_MS_LANGUAGES, "")
 TARGET_HEADER_BUILTIN(_InterlockedOr64,          "LLiLLiD*LLi", "nh", "intrin.h", ALL_MS_LANGUAGES, "")

clang/lib/CodeGen/CGBuiltin.cpp

case X86::BI_umul128: {
     Builder.CreateStore(HigherBits, HighBitsAddress);
     return Builder.CreateIntCast(MulResult, ResType, IsSigned);
   }
 
   case X86::BI__faststorefence: {
     return Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent,
                                llvm::SyncScope::System);
   }
+  case X86::BI__shiftleft128:
+  case X86::BI__shiftright128: {
+    // FIXME: Once fshl/fshr no longer add an unneeded and and cmov, do this:
+    // llvm::Function *F = CGM.getIntrinsic(
+    //   BuiltinID == X86::BI__shiftleft128 ? Intrinsic::fshl : Intrinsic::fshr,
+    //   Int64Ty);
+    // Ops[2] = Builder.CreateZExt(Ops[2], Int64Ty);
+    // return Builder.CreateCall(F, Ops);
+    llvm::Type *Int128Ty = Builder.getInt128Ty();
+    Value *Val = Builder.CreateOr(
+        Builder.CreateShl(Builder.CreateZExt(Ops[1], Int128Ty), 64),
+        Builder.CreateZExt(Ops[0], Int128Ty));
+    Value *Amt = Builder.CreateAnd(Builder.CreateZExt(Ops[2], Int128Ty),
+                                   llvm::ConstantInt::get(Int128Ty, 0x3f));
+    Value *Res;
+    if (BuiltinID == X86::BI__shiftleft128)
+      Res = Builder.CreateLShr(Builder.CreateShl(Val, Amt), 64);
+    else
+      Res = Builder.CreateLShr(Val, Amt);
+    return Builder.CreateTrunc(Res, Int64Ty);
+  }
   case X86::BI_ReadWriteBarrier:
   case X86::BI_ReadBarrier:
   case X86::BI_WriteBarrier: {
     return Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent,
                                llvm::SyncScope::SingleThread);
   }
   case X86::BI_BitScanForward:
   case X86::BI_BitScanForward64:

clang/lib/Headers/intrin.h

 __halt(void) {
   __asm__ volatile ("hlt");
 }
 static __inline__ void __DEFAULT_FN_ATTRS
 __nop(void) {
   __asm__ volatile ("nop");
 }
 #endif
-#if defined(__x86_64__)
-static __inline__ unsigned __int64 __DEFAULT_FN_ATTRS
-__shiftleft128(unsigned __int64 __l, unsigned __int64 __h, unsigned char __d) {
-  unsigned __int128 __val = ((unsigned __int128)__h << 64) | __l;
-  unsigned __int128 __res = __val << (__d & 63);
-  return (unsigned __int64)(__res >> 64);
-}
-static __inline__ unsigned __int64 __DEFAULT_FN_ATTRS
-__shiftright128(unsigned __int64 __l, unsigned __int64 __h, unsigned char __d) {
-  unsigned __int128 __val = ((unsigned __int128)__h << 64) | __l;
-  unsigned __int128 __res = __val >> (__d & 63);
-  return (unsigned __int64)__res;
-}
-#endif
 
 /*----------------------------------------------------------------------------*\
 |* Privileged intrinsics
 \*----------------------------------------------------------------------------*/
 #if defined(__i386__) || defined(__x86_64__)
 static __inline__ unsigned __int64 __DEFAULT_FN_ATTRS
 __readmsr(unsigned long __register) {
   // Loads the contents of a 64-bit model specific register (MSR) specified in

clang/test/CodeGen/ms-x86-intrinsics.c

unsigned __int64 test_umul128(unsigned __int64 Multiplier,
   return _umul128(Multiplier, Multiplicand, HighProduct);
 }
 // CHECK-X64-LABEL: define dso_local i64 @test_umul128(i64 %Multiplier, i64 %Multiplicand, i64*{{[a-z_ ]*}}%HighProduct)
 // CHECK-X64: = zext i64 %Multiplier to i128
 // CHECK-X64: = zext i64 %Multiplicand to i128
 // CHECK-X64: = mul nuw i128 %
 // CHECK-X64: store i64 %
 // CHECK-X64: ret i64 %
-#endif
+
+unsigned __int64 test__shiftleft128(unsigned __int64 l, unsigned __int64 h,
+                                    unsigned char d) {
+  return __shiftleft128(l, h, d);
+}
+// CHECK-X64-LABEL: define dso_local i64 @test__shiftleft128(i64 %l, i64 %h, i8 %d)
+// CHECK-X64  = zext i64 %h to i128
+// CHECK-X64  = shl nuw i128 %0, 64
+// CHECK-X64  = zext i64 %l to i128
+// CHECK-X64  = or i128 %1, %2
+// CHECK-X64  = and i8 %d, 63
+// CHECK-X64  = shl i128 %
+// CHECK-X64  = lshr i128 %
+// CHECK-X64  = trunc i128 %
+// CHECK-X64  ret i64 %
+
+unsigned __int64 test__shiftright128(unsigned __int64 l, unsigned __int64 h,
+                                     unsigned char d) {
+  return __shiftright128(l, h, d);
+}
+// CHECK-X64-LABEL: define dso_local i64 @test__shiftright128(i64 %l, i64 %h, i8 %d)
+// CHECK-X64  = zext i64 %h to i128
+// CHECK-X64  = shl nuw i128 %
+// CHECK-X64  = zext i64 %l to i128
+// CHECK-X64  = or i128 %
+// CHECK-X64  = and i8 %d, 63
+// CHECK-X64  = lshr i128 %
+// CHECK-X64  = trunc i128 %
+// CHECK-X64  ret i64 %
+
+#endif // defined(__x86_64__)

clang/test/Headers/ms-intrin.cpp

void f() {
   __movsb(0, 0, 0);
   __movsd(0, 0, 0);
   __movsw(0, 0, 0);
 
   __stosd(0, 0, 0);
   __stosw(0, 0, 0);
 
 #ifdef _M_X64
-  __shiftleft128(1, 2, 3);
-  __shiftright128(1, 2, 3);
   __movsq(0, 0, 0);
   __stosq(0, 0, 0);
 #endif
 
   int info[4];
   __cpuid(info, 0);
   __cpuidex(info, 0, 0);
   _xgetbv(0);