[X86][LWP] Add llvm support for LWP instructions.

[RKSimon]

Summary

This patch adds support for the the LightWeight Profiling (LWP) instructions which are available on all AMD Bulldozer class CPUs (bdver1 to bdver4).

[RKSimon]

clang patch: https://reviews.llvm.org/D32770

[craig.topper]

test/CodeGen/X86/lwp-intrinsics-x86_64.ll

15

That immediate seems way out of range of 32-bit value. Is that intentional?

[RKSimon]

test/CodeGen/X86/lwp-intrinsics-x86_64.ll

15

Yes, I was trying to check that it truncated correctly. Should I comment it? Add it as a separate test?

[filcab]

lib/Target/X86/X86ISelLowering.cpp

20313

Can you split this refactor, please?

test/CodeGen/X86/lwp-intrinsics-x86_64.ll

15

I'm not sure this is useful, does it even make it past the IR reader?
Truncating a too large constant seems like it should be tested somewhere else.

[RKSimon]

test/CodeGen/X86/lwp-intrinsics-x86_64.ll

15

OK, I'll strip them down.

[RKSimon]

Rebased and removed implicit truncation from tests' immediate value arguments

[andreadb]

I noticed that according to X86GenInstrInfo.inc, the new LWPINS instructions don't have "UnmodeledSideEffects".
However, all the other LWP instructions have flag UnmodeledSideEffects set.
My understanding is that this is because LWPVAL/LLWPCB/SLWPCB are directly matched from x86 intrinsics with side effects, while LWPINS* are firstly lowered to X86ISD::LWPINS nodes.

Shouldn't we set hasSideEffects = 1 for the LWPINS variants too? After all, those instructions modify memory by pushing new events in the ring buffer. So, those can both read and write.

[RKSimon]

In D32769#744477, @andreadb wrote:

I noticed that according to X86GenInstrInfo.inc, the new LWPINS instructions don't have "UnmodeledSideEffects".
However, all the other LWP instructions have flag UnmodeledSideEffects set.
My understanding is that this is because LWPVAL/LLWPCB/SLWPCB are directly matched from x86 intrinsics with side effects, while LWPINS* are firstly lowered to X86ISD::LWPINS nodes.

Shouldn't we set hasSideEffects = 1 for the LWPINS variants too? After all, those instructions modify memory by pushing new events in the ring buffer. So, those can both read and write.

I think I just need to add SDNPSideEffect to X86lwpins.

[RKSimon]

Fixed lwpins sideffects issue and added disassembler tests

[andreadb]

LGTM. Thanks!

include/llvm/IR/IntrinsicsX86.td

def int_x86_xop_vpshlq :
                         [IntrNoMem]>;
   def int_x86_xop_vpshlw :
               GCCBuiltin<"__builtin_ia32_vpshlw">,
               Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty],
                         [IntrNoMem]>;
 }
 
 //===----------------------------------------------------------------------===//
+// LWP
+let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
+  def int_x86_llwpcb :
+              GCCBuiltin<"__builtin_ia32_llwpcb">,
+              Intrinsic<[], [llvm_ptr_ty], []>;
+  def int_x86_slwpcb :
+              GCCBuiltin<"__builtin_ia32_slwpcb">,
+              Intrinsic<[llvm_ptr_ty], [], []>;
+  def int_x86_lwpins32 :
+              GCCBuiltin<"__builtin_ia32_lwpins32">,
+              Intrinsic<[llvm_i8_ty], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], []>;
+  def int_x86_lwpins64 :
+              GCCBuiltin<"__builtin_ia32_lwpins64">,
+              Intrinsic<[llvm_i8_ty], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty], []>;
+  def int_x86_lwpval32 :
+              GCCBuiltin<"__builtin_ia32_lwpval32">,
+              Intrinsic<[], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], []>;
+  def int_x86_lwpval64 :
+              GCCBuiltin<"__builtin_ia32_lwpval64">,
+              Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty], []>;
+}
+
+//===----------------------------------------------------------------------===//
 // MMX
 
 // Empty MMX state op.
 let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
   def int_x86_mmx_emms  : GCCBuiltin<"__builtin_ia32_emms">,
               Intrinsic<[], [], []>;
   def int_x86_mmx_femms : GCCBuiltin<"__builtin_ia32_femms">,
               Intrinsic<[], [], []>;

lib/Support/Host.cpp

bool sys::getHostCPUFeatures(StringMap<bool> &Features) {
   getX86CpuIDAndInfo(0x80000000, &MaxExtLevel, &EBX, &ECX, &EDX);
 
   bool HasExtLeaf1 = MaxExtLevel >= 0x80000001 &&
                      !getX86CpuIDAndInfo(0x80000001, &EAX, &EBX, &ECX, &EDX);
   Features["lzcnt"] = HasExtLeaf1 && ((ECX >> 5) & 1);
   Features["sse4a"] = HasExtLeaf1 && ((ECX >> 6) & 1);
   Features["prfchw"] = HasExtLeaf1 && ((ECX >> 8) & 1);
   Features["xop"] = HasExtLeaf1 && ((ECX >> 11) & 1) && HasAVXSave;
+  Features["lwp"] = HasExtLeaf1 && ((ECX >> 15) & 1);
   Features["fma4"] = HasExtLeaf1 && ((ECX >> 16) & 1) && HasAVXSave;
   Features["tbm"] = HasExtLeaf1 && ((ECX >> 21) & 1);
   Features["mwaitx"] = HasExtLeaf1 && ((ECX >> 29) & 1);
 
   bool HasExtLeaf8 = MaxExtLevel >= 0x80000008 &&
                      !getX86CpuIDAndInfoEx(0x80000008,0x0, &EAX, &EBX, &ECX, &EDX);
   Features["clzero"] = HasExtLeaf8 && ((EBX >> 0) & 1);
 

lib/Target/X86/X86.td

 def FeatureSSEUnalignedMem : SubtargetFeature<"sse-unaligned-mem",
                                           "HasSSEUnalignedMem", "true",
                       "Allow unaligned memory operands with SSE instructions">;
 def FeatureAES     : SubtargetFeature<"aes", "HasAES", "true",
                                       "Enable AES instructions",
                                       [FeatureSSE2]>;
 def FeatureTBM     : SubtargetFeature<"tbm", "HasTBM", "true",
                                       "Enable TBM instructions">;
+def FeatureLWP     : SubtargetFeature<"lwp", "HasLWP", "true",
+                                      "Enable LWP instructions">;
 def FeatureMOVBE   : SubtargetFeature<"movbe", "HasMOVBE", "true",
                                       "Support MOVBE instruction">;
 def FeatureRDRAND  : SubtargetFeature<"rdrnd", "HasRDRAND", "true",
                                       "Support RDRAND instruction">;
 def FeatureF16C    : SubtargetFeature<"f16c", "HasF16C", "true",
                        "Support 16-bit floating point conversion instructions",
                        [FeatureAVX]>;
 def FeatureFSGSBase : SubtargetFeature<"fsgsbase", "HasFSGSBase", "true",
def : Proc<"bdver1", [
   FeaturePCLMUL,
   FeatureMMX,
   FeatureAVX,
   FeatureFXSR,
   FeatureSSE4A,
   FeatureLZCNT,
   FeaturePOPCNT,
   FeatureXSAVE,
+  FeatureLWP,
   FeatureSlowSHLD,
   FeatureLAHFSAHF
 ]>;
 // Piledriver
 def : Proc<"bdver2", [
   FeatureX87,
   FeatureXOP,
   FeatureFMA4,
   FeatureCMPXCHG16B,
   FeatureAES,
   FeaturePRFCHW,
   FeaturePCLMUL,
   FeatureMMX,
   FeatureAVX,
   FeatureFXSR,
   FeatureSSE4A,
   FeatureF16C,
   FeatureLZCNT,
   FeaturePOPCNT,
   FeatureXSAVE,
   FeatureBMI,
   FeatureTBM,
+  FeatureLWP,
   FeatureFMA,
   FeatureSlowSHLD,
   FeatureLAHFSAHF
 ]>;
 
 // Steamroller
 def : Proc<"bdver3", [
   FeatureX87,
   FeatureXOP,
   FeatureFMA4,
   FeatureCMPXCHG16B,
   FeatureAES,
   FeaturePRFCHW,
   FeaturePCLMUL,
   FeatureMMX,
   FeatureAVX,
   FeatureFXSR,
   FeatureSSE4A,
   FeatureF16C,
   FeatureLZCNT,
   FeaturePOPCNT,
   FeatureXSAVE,
   FeatureBMI,
   FeatureTBM,
+  FeatureLWP,
   FeatureFMA,
   FeatureXSAVEOPT,
   FeatureSlowSHLD,
   FeatureFSGSBase,
   FeatureLAHFSAHF
 ]>;
 
 // Excavator
def : Proc<"bdver4", [
   FeaturePCLMUL,
   FeatureF16C,
   FeatureLZCNT,
   FeaturePOPCNT,
   FeatureXSAVE,
   FeatureBMI,
   FeatureBMI2,
   FeatureTBM,
+  FeatureLWP,
   FeatureFMA,
   FeatureXSAVEOPT,
   FeatureSlowSHLD,
   FeatureFSGSBase,
   FeatureLAHFSAHF,
   FeatureMWAITX
 ]>;
 

lib/Target/X86/X86ISelLowering.h

enum NodeType : unsigned {
       XTEST,
 
       // ERI instructions.
       RSQRT28, RSQRT28S, RCP28, RCP28S, EXP2,
 
       // Conversions between float and half-float.
       CVTPS2PH, CVTPH2PS,
 
+      // LWP insert record.
+      LWPINS,
+
       // Compare and swap.
       LCMPXCHG_DAG = ISD::FIRST_TARGET_MEMORY_OPCODE,
       LCMPXCHG8_DAG,
       LCMPXCHG16_DAG,
       LCMPXCHG8_SAVE_EBX_DAG,
       LCMPXCHG16_SAVE_RBX_DAG,
 
       /// LOCK-prefixed arithmetic read-modify-write instructions.

lib/Target/X86/X86ISelLowering.cpp

if (!IntrData) {
     switch (IntNo) {
     case llvm::Intrinsic::x86_seh_ehregnode:
       return MarkEHRegistrationNode(Op, DAG);
     case llvm::Intrinsic::x86_seh_ehguard:
       return MarkEHGuard(Op, DAG);
     case llvm::Intrinsic::x86_flags_read_u32:
     case llvm::Intrinsic::x86_flags_read_u64:
     case llvm::Intrinsic::x86_flags_write_u32:
     case llvm::Intrinsic::x86_flags_write_u64: {
       // We need a frame pointer because this will get lowered to a PUSH/POP
       // sequence.
       MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
       MFI.setHasCopyImplyingStackAdjustment(true);
       // Don't do anything here, we will expand these intrinsics out later
       // during ExpandISelPseudos in EmitInstrWithCustomInserter.
       return SDValue();
     }
+    case Intrinsic::x86_lwpins32:
+    case Intrinsic::x86_lwpins64: {
+      SDLoc dl(Op);
+      SDValue Chain = Op->getOperand(0);
+      SDVTList VTs = DAG.getVTList(MVT::i32, MVT::Other);
+      SDValue LwpIns =
+          DAG.getNode(X86ISD::LWPINS, dl, VTs, Chain, Op->getOperand(2),
+                      Op->getOperand(3), Op->getOperand(4));
+      SDValue SetCC = getSETCC(X86::COND_B, LwpIns.getValue(0), dl, DAG);
+      SDValue Result = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i8, SetCC);
+      return DAG.getNode(ISD::MERGE_VALUES, dl, Op->getVTList(), Result,
+                         LwpIns.getValue(1));
+    }
     }
     return SDValue();
   }
 
   SDLoc dl(Op);
   switch(IntrData->Type) {
   default: llvm_unreachable("Unknown Intrinsic Type");
   case RDSEED:
const char *X86TargetLowering::getTargetNodeName(unsigned Opcode) const {
   case X86ISD::CVTPS2PH:           return "X86ISD::CVTPS2PH";
   case X86ISD::CVTPH2PS:           return "X86ISD::CVTPH2PS";
   case X86ISD::CVTP2SI:            return "X86ISD::CVTP2SI";
   case X86ISD::CVTP2UI:            return "X86ISD::CVTP2UI";
   case X86ISD::CVTP2SI_RND:        return "X86ISD::CVTP2SI_RND";
   case X86ISD::CVTP2UI_RND:        return "X86ISD::CVTP2UI_RND";
   case X86ISD::CVTS2SI_RND:        return "X86ISD::CVTS2SI_RND";
   case X86ISD::CVTS2UI_RND:        return "X86ISD::CVTS2UI_RND";
+  case X86ISD::LWPINS:             return "X86ISD::LWPINS";
   }
   return nullptr;
 }
 
 /// Return true if the addressing mode represented by AM is legal for this
 /// target, for a load/store of the specified type.
 bool X86TargetLowering::isLegalAddressingMode(const DataLayout &DL,
                                               const AddrMode &AM, Type *Ty,

lib/Target/X86/X86InstrInfo.td

def X86WinAlloca : SDNode<"X86ISD::WIN_ALLOCA", SDT_X86WIN_ALLOCA,
                           [SDNPHasChain, SDNPOutGlue]>;
 
 def X86SegAlloca : SDNode<"X86ISD::SEG_ALLOCA", SDT_X86SEG_ALLOCA,
                           [SDNPHasChain]>;
 
 def X86TLSCall : SDNode<"X86ISD::TLSCALL", SDT_X86TLSCALL,
                         [SDNPHasChain, SDNPOptInGlue, SDNPOutGlue]>;
 
+def X86lwpins : SDNode<"X86ISD::LWPINS",
+                       SDTypeProfile<1, 3, [SDTCisVT<0, i32>, SDTCisInt<1>,
+                                            SDTCisVT<2, i32>, SDTCisVT<3, i32>]>,
+                       [SDNPHasChain, SDNPSideEffect]>;
+
 //===----------------------------------------------------------------------===//
 // X86 Operand Definitions.
 //
 
 // A version of ptr_rc which excludes SP, ESP, and RSP. This is used for
 // the index operand of an address, to conform to x86 encoding restrictions.
 def ptr_rc_nosp : PointerLikeRegClass<1>;
 

 def HasXSAVEOPT  : Predicate<"Subtarget->hasXSAVEOPT()">;
 def HasXSAVEC    : Predicate<"Subtarget->hasXSAVEC()">;
 def HasXSAVES    : Predicate<"Subtarget->hasXSAVES()">;
 def HasPCLMUL    : Predicate<"Subtarget->hasPCLMUL()">;
 def HasFMA       : Predicate<"Subtarget->hasFMA()">;
 def HasFMA4      : Predicate<"Subtarget->hasFMA4()">;
 def HasXOP       : Predicate<"Subtarget->hasXOP()">;
 def HasTBM       : Predicate<"Subtarget->hasTBM()">;
+def HasLWP       : Predicate<"Subtarget->hasLWP()">;
 def HasMOVBE     : Predicate<"Subtarget->hasMOVBE()">;
 def HasRDRAND    : Predicate<"Subtarget->hasRDRAND()">;
 def HasF16C      : Predicate<"Subtarget->hasF16C()">;
 def NoF16C       : Predicate<"!Subtarget->hasF16C()">;
 def HasFSGSBase  : Predicate<"Subtarget->hasFSGSBase()">;
 def HasLZCNT     : Predicate<"Subtarget->hasLZCNT()">;
 def HasBMI       : Predicate<"Subtarget->hasBMI()">;
 def HasBMI2      : Predicate<"Subtarget->hasBMI2()">;

 defm BLCS    : tbm_binary_intr<0x01, "blcs", MRM3r, MRM3m>;
 defm BLSFILL : tbm_binary_intr<0x01, "blsfill", MRM2r, MRM2m>;
 defm BLSIC   : tbm_binary_intr<0x01, "blsic", MRM6r, MRM6m>;
 defm T1MSKC  : tbm_binary_intr<0x01, "t1mskc", MRM7r, MRM7m>;
 defm TZMSK   : tbm_binary_intr<0x01, "tzmsk", MRM4r, MRM4m>;
 } // HasTBM, EFLAGS
 
 //===----------------------------------------------------------------------===//
+// Lightweight Profiling Instructions
+
+let Predicates = [HasLWP] in {
+
+def LLWPCB : I<0x12, MRM0r, (outs), (ins GR32:$src), "llwpcb\t$src",
+               [(int_x86_llwpcb GR32:$src)], IIC_LWP>,
+               XOP, XOP9, Requires<[Not64BitMode]>;
+def SLWPCB : I<0x12, MRM1r, (outs GR32:$dst), (ins), "slwpcb\t$dst",
+               [(set GR32:$dst, (int_x86_slwpcb))], IIC_LWP>,
+               XOP, XOP9, Requires<[Not64BitMode]>;
+
+def LLWPCB64 : I<0x12, MRM0r, (outs), (ins GR64:$src), "llwpcb\t$src",
+                 [(int_x86_llwpcb GR64:$src)], IIC_LWP>,
+                 XOP, XOP9, VEX_W, Requires<[In64BitMode]>;
+def SLWPCB64 : I<0x12, MRM1r, (outs GR64:$dst), (ins), "slwpcb\t$dst",
+                 [(set GR64:$dst, (int_x86_slwpcb))], IIC_LWP>,
+                 XOP, XOP9, VEX_W, Requires<[In64BitMode]>;
+
+multiclass lwpins_intr<RegisterClass RC> {
+  def rri : Ii32<0x12, MRM0r, (outs), (ins RC:$src0, GR32:$src1, i32imm:$cntl),
+                 "lwpins\t{$cntl, $src1, $src0|$src0, $src1, $cntl}",
+                 [(set EFLAGS, (X86lwpins RC:$src0, GR32:$src1, imm:$cntl))]>,
+                 XOP_4V, XOPA;
+  let mayLoad = 1 in
+  def rmi : Ii32<0x12, MRM0m, (outs), (ins RC:$src0, i32mem:$src1, i32imm:$cntl),
+                 "lwpins\t{$cntl, $src1, $src0|$src0, $src1, $cntl}",
+                 [(set EFLAGS, (X86lwpins RC:$src0, (loadi32 addr:$src1), imm:$cntl))]>,
+                 XOP_4V, XOPA;
+}
+
+let Defs = [EFLAGS] in {
+  defm LWPINS32 : lwpins_intr<GR32>;
+  defm LWPINS64 : lwpins_intr<GR64>, VEX_W;
+} // EFLAGS
+
+multiclass lwpval_intr<RegisterClass RC, Intrinsic Int> {
+  def rri : Ii32<0x12, MRM1r, (outs), (ins RC:$src0, GR32:$src1, i32imm:$cntl),
+                 "lwpval\t{$cntl, $src1, $src0|$src0, $src1, $cntl}",
+                 [(Int RC:$src0, GR32:$src1, imm:$cntl)], IIC_LWP>,
+                 XOP_4V, XOPA;
+  let mayLoad = 1 in
+  def rmi : Ii32<0x12, MRM1m, (outs), (ins RC:$src0, i32mem:$src1, i32imm:$cntl),
+                 "lwpval\t{$cntl, $src1, $src0|$src0, $src1, $cntl}",
+                 [(Int RC:$src0, (loadi32 addr:$src1), imm:$cntl)], IIC_LWP>,
+                 XOP_4V, XOPA;
+}
+
+defm LWPVAL32 : lwpval_intr<GR32, int_x86_lwpval32>;
+defm LWPVAL64 : lwpval_intr<GR64, int_x86_lwpval64>, VEX_W;
+
+} // HasLWP
+
+//===----------------------------------------------------------------------===//
 // MONITORX/MWAITX Instructions
 //
 let SchedRW = [ WriteSystem ] in {
   let usesCustomInserter = 1 in {
     def MONITORX : PseudoI<(outs), (ins i32mem:$src1, GR32:$src2, GR32:$src3),
                            [(int_x86_monitorx addr:$src1, GR32:$src2, GR32:$src3)]>,
                    Requires<[ HasMWAITX ]>;
   }

lib/Target/X86/X86Schedule.td

 def IIC_SWAPGS : InstrItinClass;
 def IIC_SYSCALL : InstrItinClass;
 def IIC_SYS_ENTER_EXIT : InstrItinClass;
 def IIC_IN_RR : InstrItinClass;
 def IIC_IN_RI : InstrItinClass;
 def IIC_OUT_RR : InstrItinClass;
 def IIC_OUT_IR : InstrItinClass;
 def IIC_INS : InstrItinClass;
+def IIC_LWP : InstrItinClass;
 def IIC_MOV_REG_DR : InstrItinClass;
 def IIC_MOV_DR_REG : InstrItinClass;
 def IIC_MOV_REG_CR : InstrItinClass;
 def IIC_MOV_CR_REG : InstrItinClass;
 def IIC_MOV_REG_SR : InstrItinClass;
 def IIC_MOV_MEM_SR : InstrItinClass;
 def IIC_MOV_SR_REG : InstrItinClass;
 def IIC_MOV_SR_MEM : InstrItinClass;

lib/Target/X86/X86Subtarget.h

protected:
   bool HasFMA4;
 
   /// Target has XOP instructions
   bool HasXOP;
 
   /// Target has TBM instructions.
   bool HasTBM;
 
+  /// Target has LWP instructions
+  bool HasLWP;
+
   /// True if the processor has the MOVBE instruction.
   bool HasMOVBE;
 
   /// True if the processor has the RDRAND instruction.
   bool HasRDRAND;
 
   /// Processor has 16-bit floating point conversion instructions.
   bool HasF16C;
public:
   bool hasPCLMUL() const { return HasPCLMUL; }
   // Prefer FMA4 to FMA - its better for commutation/memory folding and
   // has equal or better performance on all supported targets.
   bool hasFMA() const { return (HasFMA || hasAVX512()) && !HasFMA4; }
   bool hasFMA4() const { return HasFMA4; }
   bool hasAnyFMA() const { return hasFMA() || hasFMA4(); }
   bool hasXOP() const { return HasXOP; }
   bool hasTBM() const { return HasTBM; }
+  bool hasLWP() const { return HasLWP; }
   bool hasMOVBE() const { return HasMOVBE; }
   bool hasRDRAND() const { return HasRDRAND; }
   bool hasF16C() const { return HasF16C; }
   bool hasFSGSBase() const { return HasFSGSBase; }
   bool hasLZCNT() const { return HasLZCNT; }
   bool hasBMI() const { return HasBMI; }
   bool hasBMI2() const { return HasBMI2; }
   bool hasVBMI() const { return HasVBMI; }

lib/Target/X86/X86Subtarget.cpp

void X86Subtarget::initializeEnvironment() {
   HasXSAVEOPT = false;
   HasXSAVEC = false;
   HasXSAVES = false;
   HasPCLMUL = false;
   HasFMA = false;
   HasFMA4 = false;
   HasXOP = false;
   HasTBM = false;
+  HasLWP = false;
   HasMOVBE = false;
   HasRDRAND = false;
   HasF16C = false;
   HasFSGSBase = false;
   HasLZCNT = false;
   HasBMI = false;
   HasBMI2 = false;
   HasVBMI = false;

test/CodeGen/X86/lwp-intrinsics-x86_64.ll

+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+lwp | FileCheck %s --check-prefix=X64
+; RUN: llc < %s -mtriple=x86_64-unknown -mcpu=bdver1 | FileCheck %s --check-prefix=X64
+; RUN: llc < %s -mtriple=x86_64-unknown -mcpu=bdver2 | FileCheck %s --check-prefix=X64
+; RUN: llc < %s -mtriple=x86_64-unknown -mcpu=bdver3 | FileCheck %s --check-prefix=X64
+; RUN: llc < %s -mtriple=x86_64-unknown -mcpu=bdver4 | FileCheck %s --check-prefix=X64
+
+define i8 @test_lwpins64_rri(i64 %a0, i32 %a1) nounwind {
+; X64-LABEL: test_lwpins64_rri:
+; X64:       # BB#0:
+; X64-NEXT:    lwpins $-1985229329, %esi, %rdi # imm = 0x89ABCDEF
+; X64-NEXT:    setb %al
+; X64-NEXT:    retq
+  %1 = tail call i8 @llvm.x86.lwpins64(i64 %a0, i32 %a1, i32 2309737967)
+  ret i8 %1
+}
+
+define i8 @test_lwpins64_rmi(i64 %a0, i32 *%p1) nounwind {
+; X64-LABEL: test_lwpins64_rmi:
+; X64:       # BB#0:
+; X64-NEXT:    lwpins $1985229328, (%rsi), %rdi # imm = 0x76543210
+; X64-NEXT:    setb %al
+; X64-NEXT:    retq
+  %a1 = load i32, i32 *%p1
+  %1 = tail call i8 @llvm.x86.lwpins64(i64 %a0, i32 %a1, i32 1985229328)
+  ret i8 %1
+}
+
+define void @test_lwpval64_rri(i64 %a0, i32 %a1) nounwind {
+; X64-LABEL: test_lwpval64_rri:
+; X64:       # BB#0:
+; X64-NEXT:    lwpval $-19088744, %esi, %rdi # imm = 0xFEDCBA98
+; X64-NEXT:    retq
+  tail call void @llvm.x86.lwpval64(i64 %a0, i32 %a1, i32 4275878552)
+  ret void
+}
+
+define void @test_lwpval64_rmi(i64 %a0, i32 *%p1) nounwind {
+; X64-LABEL: test_lwpval64_rmi:
+; X64:       # BB#0:
+; X64-NEXT:    lwpval $305419896, (%rsi), %rdi # imm = 0x12345678
+; X64-NEXT:    retq
+  %a1 = load i32, i32 *%p1
+  tail call void @llvm.x86.lwpval64(i64 %a0, i32 %a1, i32 305419896)
+  ret void
+}
+
+declare i8 @llvm.x86.lwpins64(i64, i32, i32) nounwind
+declare void @llvm.x86.lwpval64(i64, i32, i32) nounwind

test/CodeGen/X86/lwp-intrinsics.ll

+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-unknown -mattr=+lwp | FileCheck %s --check-prefix=X86
+; RUN: llc < %s -mtriple=i686-unknown -mcpu=bdver1 | FileCheck %s --check-prefix=X86
+; RUN: llc < %s -mtriple=i686-unknown -mcpu=bdver2 | FileCheck %s --check-prefix=X86
+; RUN: llc < %s -mtriple=i686-unknown -mcpu=bdver3 | FileCheck %s --check-prefix=X86
+; RUN: llc < %s -mtriple=i686-unknown -mcpu=bdver4 | FileCheck %s --check-prefix=X86
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+lwp | FileCheck %s --check-prefix=X64
+; RUN: llc < %s -mtriple=x86_64-unknown -mcpu=bdver1 | FileCheck %s --check-prefix=X64
+; RUN: llc < %s -mtriple=x86_64-unknown -mcpu=bdver2 | FileCheck %s --check-prefix=X64
+; RUN: llc < %s -mtriple=x86_64-unknown -mcpu=bdver3 | FileCheck %s --check-prefix=X64
+; RUN: llc < %s -mtriple=x86_64-unknown -mcpu=bdver4 | FileCheck %s --check-prefix=X64
+
+define void @test_llwpcb(i8 *%a0) nounwind {
+; X86-LABEL: test_llwpcb:
+; X86:       # BB#0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    llwpcb %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: test_llwpcb:
+; X64:       # BB#0:
+; X64-NEXT:    llwpcb %rdi
+; X64-NEXT:    retq
+  tail call void @llvm.x86.llwpcb(i8 *%a0)
+  ret void
+}
+
+define i8* @test_slwpcb(i8 *%a0) nounwind {
+; X86-LABEL: test_slwpcb:
+; X86:       # BB#0:
+; X86-NEXT:    slwpcb %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: test_slwpcb:
+; X64:       # BB#0:
+; X64-NEXT:    slwpcb %rax
+; X64-NEXT:    retq
+  %1 = tail call i8* @llvm.x86.slwpcb()
+  ret i8 *%1
+}
+
+define i8 @test_lwpins32_rri(i32 %a0, i32 %a1) nounwind {
+; X86-LABEL: test_lwpins32_rri:
+; X86:       # BB#0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    addl %ecx, %ecx
+; X86-NEXT:    lwpins $-1985229329, %ecx, %eax # imm = 0x89ABCDEF
+; X86-NEXT:    setb %al
+; X86-NEXT:    retl
+;
+; X64-LABEL: test_lwpins32_rri:
+; X64:       # BB#0:
+; X64-NEXT:    addl %esi, %esi
+; X64-NEXT:    lwpins $-1985229329, %esi, %edi # imm = 0x89ABCDEF
+; X64-NEXT:    setb %al
+; X64-NEXT:    retq
+  %1 = add i32 %a1, %a1
+  %2 = tail call i8 @llvm.x86.lwpins32(i32 %a0, i32 %1, i32 2309737967)
+  ret i8 %2
+}
+
+define i8 @test_lwpins32_rmi(i32 %a0, i32 *%p1) nounwind {
+; X86-LABEL: test_lwpins32_rmi:
+; X86:       # BB#0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    lwpins $1985229328, (%eax), %ecx # imm = 0x76543210
+; X86-NEXT:    setb %al
+; X86-NEXT:    retl
+;
+; X64-LABEL: test_lwpins32_rmi:
+; X64:       # BB#0:
+; X64-NEXT:    lwpins $1985229328, (%rsi), %edi # imm = 0x76543210
+; X64-NEXT:    setb %al
+; X64-NEXT:    retq
+  %a1 = load i32, i32 *%p1
+  %1 = tail call i8 @llvm.x86.lwpins32(i32 %a0, i32 %a1, i32 1985229328)
+  ret i8 %1
+}
+
+define void @test_lwpval32_rri(i32 %a0, i32 %a1) nounwind {
+; X86-LABEL: test_lwpval32_rri:
+; X86:       # BB#0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    addl %ecx, %ecx
+; X86-NEXT:    lwpval $-19088744, %ecx, %eax # imm = 0xFEDCBA98
+; X86-NEXT:    retl
+;
+; X64-LABEL: test_lwpval32_rri:
+; X64:       # BB#0:
+; X64-NEXT:    addl %esi, %esi
+; X64-NEXT:    lwpval $-19088744, %esi, %edi # imm = 0xFEDCBA98
+; X64-NEXT:    retq
+  %1 = add i32 %a1, %a1
+  tail call void @llvm.x86.lwpval32(i32 %a0, i32 %1, i32 4275878552)
+  ret void
+}
+
+define void @test_lwpval32_rmi(i32 %a0, i32 *%p1) nounwind {
+; X86-LABEL: test_lwpval32_rmi:
+; X86:       # BB#0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    lwpval $305419896, (%eax), %ecx # imm = 0x12345678
+; X86-NEXT:    retl
+;
+; X64-LABEL: test_lwpval32_rmi:
+; X64:       # BB#0:
+; X64-NEXT:    lwpval $305419896, (%rsi), %edi # imm = 0x12345678
+; X64-NEXT:    retq
+  %a1 = load i32, i32 *%p1
+  tail call void @llvm.x86.lwpval32(i32 %a0, i32 %a1, i32 305419896)
+  ret void
+}
+
+declare void @llvm.x86.llwpcb(i8*) nounwind
+declare i8* @llvm.x86.slwpcb() nounwind
+declare i8 @llvm.x86.lwpins32(i32, i32, i32) nounwind
+declare void @llvm.x86.lwpval32(i32, i32, i32) nounwind

test/MC/Disassembler/X86/x86-32.txt

 0x82 0x2D 0x38 0x87 0x50 0x00 0x26
 # CHECK: xorb $38, 5277496
 0x82 0x35 0x38 0x87 0x50 0x00 0x26
 # CHECK: cmpb $38, 5277496
 0x82 0x3d 0x38 0x87 0x50 0x00 0x26
 
 #CHECK: getsec
 0x0f 0x37
+
+#CHECK: llwpcb %ecx
+0x8f 0xe9 0x78 0x12 0xc1
+
+#CHECK: slwpcb %ecx
+0x8f 0xe9 0x78 0x12 0xc9
+
+# CHECK: lwpins $305419896, %ebx, %eax
+0x8f 0xea 0x78 0x12 0xc3 0x78 0x56 0x34 0x12
+
+# CHECK: lwpins $591751049, (%esp), %edx
+0x8f 0xea 0x68 0x12 0x04 0x24 0x89 0x67 0x45 0x23
+
+# CHECK: lwpval $1737075661, %ebx, %eax
+0x8f 0xea 0x78 0x12 0xcb 0xcd 0xab 0x89 0x67
+
+# CHECK: lwpval $2309737967, (%esp), %edx
+0x8f 0xea 0x68 0x12 0x0c 0x24 0xef 0xcd 0xab 0x89

test/MC/Disassembler/X86/x86-64.txt

 # FIXME: vaddps (%rdx,%rcx), %zmm20, %zmm15
 0x62 0x71 0x5c 0x40 0x58 0x3c 0x0a
 
 # CHECK: callq 32767
 0xe8 0xff 0x7f 0x00 0x00
 
 # CHECK: callq -32769
 0xe8 0xff 0x7f 0xff 0xff
+
+# CHECK: llwpcb %rax
+0x8f 0xe9 0xf8 0x12 0xc0
+
+# CHECK: slwpcb %rax
+0x8f 0xe9 0xf8 0x12 0xc8
+
+# CHECK: lwpins $305419896, %ebx, %rax
+0x8f 0xea 0xf8 0x12 0xc3 0x78 0x56 0x34 0x12
+
+# CHECK: lwpins $591751049, (%rsp), %rdx
+0x8f 0xea 0xe8 0x12 0x04 0x24 0x89 0x67 0x45 0x23
+
+# CHECK: lwpins $591751049, (%esp), %edx
+0x67 0x8f 0xea 0x68 0x12 0x04 0x24 0x89 0x67 0x45 0x23
+
+# CHECK: lwpval $1737075661, %ebx, %rax
+0x8f 0xea 0xf8 0x12 0xcb 0xcd 0xab 0x89 0x67
+
+# CHECK: lwpval $2309737967, (%rsp), %rdx
+0x8f 0xea 0xe8 0x12 0x0c 0x24 0xef 0xcd 0xab 0x89
+
+# CHECK: lwpval $2309737967, (%esp), %edx
+0x67 0x8f 0xea 0x68 0x12 0x0c 0x24 0xef 0xcd 0xab 0x89

test/MC/X86/lwp-x86_64.s

+# RUN: llvm-mc -triple x86_64-unknown-unknown --show-encoding %s | FileCheck %s --check-prefix=CHECK
+
+llwpcb %rcx
+# CHECK: llwpcb %rcx
+# CHECK: encoding: [0x8f,0xe9,0xf8,0x12,0xc1]
+
+slwpcb %rax
+# CHECK: slwpcb %rax
+# CHECK: encoding: [0x8f,0xe9,0xf8,0x12,0xc8]
+
+lwpins $305419896, %ebx, %rax
+# CHECK: lwpins $305419896, %ebx, %rax
+# CHECK: encoding: [0x8f,0xea,0xf8,0x12,0xc3,0x78,0x56,0x34,0x12]
+
+lwpins $591751049, (%rsp), %rdx
+# CHECK: lwpins $591751049, (%rsp), %rdx
+# CHECK: encoding: [0x8f,0xea,0xe8,0x12,0x04,0x24,0x89,0x67,0x45,0x23]
+
+lwpval $1737075661, %ebx, %rax
+# CHECK: lwpval $1737075661, %ebx, %rax
+# CHECK: encoding: [0x8f,0xea,0xf8,0x12,0xcb,0xcd,0xab,0x89,0x67]
+
+lwpval $2309737967, (%rsp), %rdx
+# CHECK: lwpval $2309737967, (%rsp), %rdx
+# CHECK: encoding: [0x8f,0xea,0xe8,0x12,0x0c,0x24,0xef,0xcd,0xab,0x89]

test/MC/X86/lwp.s

+# RUN: llvm-mc -triple i686-unknown-unknown --show-encoding %s | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-X86
+# RUN: llvm-mc -triple x86_64-unknown-unknown --show-encoding %s | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-X64
+
+llwpcb %ecx
+# CHECK: llwpcb %ecx
+# CHECK-X86: encoding: [0x8f,0xe9,0x78,0x12,0xc1]
+# CHECK-X64: encoding: [0x8f,0xe9,0x78,0x12,0xc1]
+
+slwpcb %eax
+# CHECK: slwpcb %eax
+# CHECK-X86: encoding: [0x8f,0xe9,0x78,0x12,0xc8]
+# CHECK-X64: encoding: [0x8f,0xe9,0x78,0x12,0xc8]
+
+lwpins $305419896, %ebx, %eax
+# CHECK: lwpins $305419896, %ebx, %eax
+# CHECK-X86: encoding: [0x8f,0xea,0x78,0x12,0xc3,0x78,0x56,0x34,0x12]
+# CHECK-X64: encoding: [0x8f,0xea,0x78,0x12,0xc3,0x78,0x56,0x34,0x12]
+
+lwpins $591751049, (%esp), %edx
+# CHECK: lwpins $591751049, (%esp), %edx
+# CHECK-X86: encoding: [0x8f,0xea,0x68,0x12,0x04,0x24,0x89,0x67,0x45,0x23]
+# CHECK-X64: encoding: [0x67,0x8f,0xea,0x68,0x12,0x04,0x24,0x89,0x67,0x45,0x23]
+
+lwpval $1737075661, %ebx, %eax
+# CHECK: lwpval $1737075661, %ebx, %eax
+# CHECK-X86: encoding: [0x8f,0xea,0x78,0x12,0xcb,0xcd,0xab,0x89,0x67]
+# CHECK-X64: encoding: [0x8f,0xea,0x78,0x12,0xcb,0xcd,0xab,0x89,0x67]
+
+lwpval $2309737967, (%esp), %edx
+# CHECK: lwpval $2309737967, (%esp), %edx
+# CHECK-X86: encoding: [0x8f,0xea,0x68,0x12,0x0c,0x24,0xef,0xcd,0xab,0x89]
+# CHECK-X64: encoding: [0x67,0x8f,0xea,0x68,0x12,0x0c,0x24,0xef,0xcd,0xab,0x89]