[X86] WaitPKG instructions

[GBuella]

Summary

Three new instructions:

umonitor - Sets up a linear address range to be
monitored by hardware and activates the monitor.
The address range should be a writeback memory
caching type.

umwait - A hint that allows the processor to
stop instruction execution and enter an
implementation-dependent optimized state
until occurrence of a class of events.

tpause - Directs the processor to enter an
implementation-dependent optimized state
until the TSC reaches the value in EDX:EAX.

Also modifying the description of the mfence
instruction, as the rep prefix (0xF3) was allowed
before, which would conflict with umonitor during
disassembly.

Before this patch
$ echo 0xf3,0x0f,0xae,0xf0 | llvm-mc -disassemble

.text
mfence

After this patch
$ echo 0xf3,0x0f,0xae,0xf0 | llvm-mc -disassemble

.text
umonitor        %rax

[craig.topper]

lib/Target/X86/X86InstrInfo.td

2664

Line these up with the 0xAE on the first line.

2667

Line these up.

2689

Based on where these landed in the opcode map, you should change the TB on MFENCE in X86InstrSSE.td to PS.

Before this patch, this sequence of bytes disassembled as mfence, but it should have been invalid. The PS will fix that.
0xf3,0x41,0x0f,0xae,0xf0

I suspect LFENCE should probably be PS as well, but we haven't gotten a collision there yet. SFENCE was fixed when PCOMMIT was added. Though PCOMMIT was later removed.

[GBuella]

Fixed mfence description

[lebedev.ri]

Please always upload patches with full context (-U99999)

[GBuella]

Supply diff context.

[craig.topper]

lib/Target/X86/X86InstrInfo.td

2642

What is the llvm-mc assembler expectation here? As this is defined it means that 64-bit mode assembly must alway use a 64-bit register and 32-bit mode must always use a 32-bit register. The documentation itself mentions being able to use a 16-bit register and that the 0x67 prefix has an effect, but that's not captured here.

2660

Again this also has a weird assembler expectation the register size is dependent on the mode.

test/CodeGen/X86/waitpkg-intrinsics-64.ll

15

Can these tests be simplified to not use so many loads and stores? Is this the -O0 clang output? Can we use -O2 output instead.

[GBuella]

Fixed umonitor 16 bit decoding.
Fixed umwait, tpause instruction description - modified the corresponding intrinsics, thus D45254 needs to be updated.

[craig.topper]

LGTM

[craig.topper]

lib/Target/X86/X86InstrInfo.td

2647

I think UMWAIT and TPAUSE Def EFLAGS.

[craig.topper]

test/CodeGen/X86/waitpkg-intrinsics-64.ll

1

Can we merge the two tests now that the types aren't dependent on mode?

[craig.topper]

LGTM

include/llvm/IR/IntrinsicsX86.td

 
 //===----------------------------------------------------------------------===//
 // Cache-line demote
 
 let TargetPrefix = "x86" in {
   def int_x86_cldemote : GCCBuiltin<"__builtin_ia32_cldemote">,
       Intrinsic<[], [llvm_ptr_ty], []>;
 }
+
+//===----------------------------------------------------------------------===//
+// Wait and pause enhancements
+let TargetPrefix = "x86" in {
+  def int_x86_umonitor : GCCBuiltin<"__builtin_ia32_umonitor">,
+              Intrinsic<[], [llvm_ptr_ty], []>;
+  def int_x86_umwait : GCCBuiltin<"__builtin_ia32_umwait">,
+              Intrinsic<[llvm_i8_ty], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], []>;
+  def int_x86_tpause : GCCBuiltin<"__builtin_ia32_tpause">,
+              Intrinsic<[llvm_i8_ty], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], []>;
+}

lib/Support/Host.cpp

bool sys::getHostCPUFeatures(StringMap<bool> &Features) {
   Features["avx512cd"]   = HasLeaf7 && ((EBX >> 28) & 1) && HasAVX512Save;
   Features["sha"]        = HasLeaf7 && ((EBX >> 29) & 1);
   Features["avx512bw"]   = HasLeaf7 && ((EBX >> 30) & 1) && HasAVX512Save;
   Features["avx512vl"]   = HasLeaf7 && ((EBX >> 31) & 1) && HasAVX512Save;
 
   Features["prefetchwt1"]     = HasLeaf7 && ((ECX >>  0) & 1);
   Features["avx512vbmi"]      = HasLeaf7 && ((ECX >>  1) & 1) && HasAVX512Save;
   Features["pku"]             = HasLeaf7 && ((ECX >>  4) & 1);
+  Features["waitpkg"]         = HasLeaf7 && ((ECX >>  5) & 1);
   Features["avx512vbmi2"]     = HasLeaf7 && ((ECX >>  6) & 1) && HasAVX512Save;
   Features["shstk"]           = HasLeaf7 && ((ECX >>  7) & 1);
   Features["gfni"]            = HasLeaf7 && ((ECX >>  8) & 1);
   Features["vaes"]            = HasLeaf7 && ((ECX >>  9) & 1) && HasAVXSave;
   Features["vpclmulqdq"]      = HasLeaf7 && ((ECX >> 10) & 1) && HasAVXSave;
   Features["avx512vnni"]      = HasLeaf7 && ((ECX >> 11) & 1) && HasAVX512Save;
   Features["avx512bitalg"]    = HasLeaf7 && ((ECX >> 12) & 1) && HasAVX512Save;
   Features["avx512vpopcntdq"] = HasLeaf7 && ((ECX >> 14) & 1) && HasAVX512Save;

lib/Target/X86/Disassembler/X86DisassemblerDecoder.cpp

case 0x66:
       if (insn->mode != MODE_16BIT)
         attrMask |= ATTR_OPSIZE;
       break;
     case 0x67:
       attrMask |= ATTR_ADSIZE;
       break;
     }
 
-    if (insn->hasAdSize)
-      attrMask |= ATTR_ADSIZE;
   }
 
   if (insn->rexPrefix & 0x08) {
     attrMask |= ATTR_REXW;
     attrMask &= ~ATTR_ADSIZE;
   }
 
   /*
if ((insn->vectorExtensionType == TYPE_EVEX &&
         insn->instructionID = instructionIDWithREXW;
         insn->spec = specifierForUID(instructionIDWithREXW);
         return 0;
       }
     }
   }
 
   /*
-   * Absolute moves need special handling.
+   * Absolute moves and umonitor need special handling.
    * -For 16-bit mode because the meaning of the AdSize and OpSize prefixes are
    *  inverted w.r.t.
    * -For 32-bit mode we need to ensure the ADSIZE prefix is observed in
    *  any position.
    */
-  if (insn->opcodeType == ONEBYTE && ((insn->opcode & 0xFC) == 0xA0)) {
+  if ((insn->opcodeType == ONEBYTE && ((insn->opcode & 0xFC) == 0xA0)) ||
+      (insn->opcodeType == TWOBYTE && (insn->opcode == 0xAE))) {
     /* Make sure we observed the prefixes in any position. */
     if (insn->hasAdSize)
       attrMask |= ATTR_ADSIZE;
     if (insn->hasOpSize)
       attrMask |= ATTR_OPSIZE;
 
     /* In 16-bit, invert the attributes. */
-    if (insn->mode == MODE_16BIT)
-      attrMask ^= ATTR_ADSIZE | ATTR_OPSIZE;
+    if (insn->mode == MODE_16BIT) {
+      attrMask ^= ATTR_ADSIZE;
+      /* The OpSize attribute is only valid with the absolute moves. */
+      if (insn->opcodeType == ONEBYTE && ((insn->opcode & 0xFC) == 0xA0))
+        attrMask ^= ATTR_OPSIZE;
+    }
 
     if (getIDWithAttrMask(&instructionID, insn, attrMask))
       return -1;
 
     insn->instructionID = instructionID;
     insn->spec = specifierForUID(instructionID);
     return 0;
   }

lib/Target/X86/X86.td

 def FeatureCLFLUSHOPT : SubtargetFeature<"clflushopt", "HasCLFLUSHOPT", "true",
                                       "Flush A Cache Line Optimized">;
 def FeatureCLWB    : SubtargetFeature<"clwb", "HasCLWB", "true",
                                       "Cache Line Write Back">;
 def FeatureWBNOINVD    : SubtargetFeature<"wbnoinvd", "HasWBNOINVD", "true",
                                       "Write Back No Invalidate">;
 def FeatureRDPID : SubtargetFeature<"rdpid", "HasRDPID", "true",
                                     "Support RDPID instructions">;
+def FeatureWAITPKG  : SubtargetFeature<"waitpkg", "HasWAITPKG", "true",
+                                      "Wait and pause enhancements">;
 // On some processors, instructions that implicitly take two memory operands are
 // slow. In practice, this means that CALL, PUSH, and POP with memory operands
 // should be avoided in favor of a MOV + register CALL/PUSH/POP.
 def FeatureSlowTwoMemOps : SubtargetFeature<"slow-two-mem-ops",
                                      "SlowTwoMemOps", "true",
                                      "Two memory operand instructions are slow">;
 def FeatureLEAUsesAG : SubtargetFeature<"lea-uses-ag", "LEAUsesAG", "true",
                                    "LEA instruction needs inputs at AG stage">;

 def : GoldmontPlusProc<"goldmont-plus">;
 
 class TremontProc<string Name> : ProcModel<Name, SLMModel,
       GLMFeatures.Value, [
   ProcIntelTRM,
   FeatureCLDEMOTE,
   FeatureGFNI,
   FeatureRDPID,
-  FeatureSGX
+  FeatureSGX,
+  FeatureWAITPKG
 ]>;
 def : TremontProc<"tremont">;
 
 // "Arrandale" along with corei3 and corei5
 class NehalemProc<string Name> : ProcessorModel<Name, SandyBridgeModel, [
   FeatureX87,
   FeatureMMX,
   FeatureSSE42,

lib/Target/X86/X86ISelLowering.h

enum NodeType : unsigned {
       CVTPS2PH, CVTPH2PS, CVTPH2PS_RND,
 
       // Galois Field Arithmetic Instructions
       GF2P8AFFINEINVQB, GF2P8AFFINEQB, GF2P8MULB,
 
       // LWP insert record.
       LWPINS,
 
+      // User level wait
+      UMWAIT, TPAUSE,
+
       // Compare and swap.
       LCMPXCHG_DAG = ISD::FIRST_TARGET_MEMORY_OPCODE,
       LCMPXCHG8_DAG,
       LCMPXCHG16_DAG,
       LCMPXCHG8_SAVE_EBX_DAG,
       LCMPXCHG16_SAVE_RBX_DAG,
 
       /// LOCK-prefixed arithmetic read-modify-write instructions.

lib/Target/X86/X86ISelLowering.cpp

case llvm::Intrinsic::x86_flags_write_u64: {
       // sequence.
       MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
       MFI.setHasCopyImplyingStackAdjustment(true);
       // Don't do anything here, we will expand these intrinsics out later
       // during ExpandISelPseudos in EmitInstrWithCustomInserter.
       return SDValue();
     }
     case Intrinsic::x86_lwpins32:
-    case Intrinsic::x86_lwpins64: {
+    case Intrinsic::x86_lwpins64:
+    case Intrinsic::x86_umwait:
+    case Intrinsic::x86_tpause: {
       SDLoc dl(Op);
       SDValue Chain = Op->getOperand(0);
       SDVTList VTs = DAG.getVTList(MVT::i32, MVT::Other);
-      SDValue LwpIns =
-          DAG.getNode(X86ISD::LWPINS, dl, VTs, Chain, Op->getOperand(2),
+      unsigned Opcode;
+
+      switch (IntNo) {
+      default: llvm_unreachable("Impossible intrinsic");
+      case Intrinsic::x86_umwait:
+        Opcode = X86ISD::UMWAIT;
+        break;
+      case Intrinsic::x86_tpause:
+        Opcode = X86ISD::TPAUSE;
+        break;
+      case Intrinsic::x86_lwpins32:
+      case Intrinsic::x86_lwpins64:
+        Opcode = X86ISD::LWPINS;
+        break;
+      }
+
+      SDValue Operation =
+          DAG.getNode(Opcode, dl, VTs, Chain, Op->getOperand(2),
                       Op->getOperand(3), Op->getOperand(4));
-      SDValue SetCC = getSETCC(X86::COND_B, LwpIns.getValue(0), dl, DAG);
+      SDValue SetCC = getSETCC(X86::COND_B, Operation.getValue(0), dl, DAG);
       SDValue Result = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i8, SetCC);
       return DAG.getNode(ISD::MERGE_VALUES, dl, Op->getVTList(), Result,
-                         LwpIns.getValue(1));
+                         Operation.getValue(1));
     }
     }
     return SDValue();
   }
 
   SDLoc dl(Op);
   switch(IntrData->Type) {
   default: llvm_unreachable("Unknown Intrinsic Type");
const char *X86TargetLowering::getTargetNodeName(unsigned Opcode) const {
   case X86ISD::VPDPWSSD:           return "X86ISD::VPDPWSSD";
   case X86ISD::VPDPWSSDS:          return "X86ISD::VPDPWSSDS";
   case X86ISD::VPSHUFBITQMB:       return "X86ISD::VPSHUFBITQMB";
   case X86ISD::GF2P8MULB:          return "X86ISD::GF2P8MULB";
   case X86ISD::GF2P8AFFINEQB:      return "X86ISD::GF2P8AFFINEQB";
   case X86ISD::GF2P8AFFINEINVQB:   return "X86ISD::GF2P8AFFINEINVQB";
   case X86ISD::NT_CALL:            return "X86ISD::NT_CALL";
   case X86ISD::NT_BRIND:           return "X86ISD::NT_BRIND";
+  case X86ISD::UMWAIT:             return "X86ISD::UMWAIT";
+  case X86ISD::TPAUSE:             return "X86ISD::TPAUSE";
   }
   return nullptr;
 }
 
 /// Return true if the addressing mode represented by AM is legal for this
 /// target, for a load/store of the specified type.
 bool X86TargetLowering::isLegalAddressingMode(const DataLayout &DL,
                                               const AddrMode &AM, Type *Ty,

lib/Target/X86/X86InstrInfo.td

 def X86TLSCall : SDNode<"X86ISD::TLSCALL", SDT_X86TLSCALL,
                         [SDNPHasChain, SDNPOptInGlue, SDNPOutGlue]>;
 
 def X86lwpins : SDNode<"X86ISD::LWPINS",
                        SDTypeProfile<1, 3, [SDTCisVT<0, i32>, SDTCisInt<1>,
                                             SDTCisVT<2, i32>, SDTCisVT<3, i32>]>,
                        [SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPSideEffect]>;
 
+def X86umwait : SDNode<"X86ISD::UMWAIT",
+                       SDTypeProfile<1, 3, [SDTCisVT<0, i32>, SDTCisInt<1>,
+                                            SDTCisVT<2, i32>, SDTCisVT<3, i32>]>,
+                       [SDNPHasChain, SDNPSideEffect]>;
+
+def X86tpause : SDNode<"X86ISD::TPAUSE",
+                       SDTypeProfile<1, 3, [SDTCisVT<0, i32>, SDTCisInt<1>,
+                                            SDTCisVT<2, i32>, SDTCisVT<3, i32>]>,
+                       [SDNPHasChain, SDNPSideEffect]>;
+
 //===----------------------------------------------------------------------===//
 // X86 Operand Definitions.
 //
 
 // A version of ptr_rc which excludes SP, ESP, and RSP. This is used for
 // the index operand of an address, to conform to x86 encoding restrictions.
 def ptr_rc_nosp : PointerLikeRegClass<1>;
 

 def FPStackf64   : Predicate<"!Subtarget->hasSSE2()">;
 def HasMPX       : Predicate<"Subtarget->hasMPX()">;
 def HasSHSTK     : Predicate<"Subtarget->hasSHSTK()">;
 def HasIBT       : Predicate<"Subtarget->hasIBT()">;
 def HasCLFLUSHOPT : Predicate<"Subtarget->hasCLFLUSHOPT()">;
 def HasCLWB      : Predicate<"Subtarget->hasCLWB()">;
 def HasWBNOINVD  : Predicate<"Subtarget->hasWBNOINVD()">;
 def HasRDPID     : Predicate<"Subtarget->hasRDPID()">;
+def HasWAITPKG   : Predicate<"Subtarget->hasWAITPKG()">;
 def HasCmpxchg16b: Predicate<"Subtarget->hasCmpxchg16b()">;
 def Not64BitMode : Predicate<"!Subtarget->is64Bit()">,
                              AssemblerPredicate<"!Mode64Bit", "Not 64-bit mode">;
 def In64BitMode  : Predicate<"Subtarget->is64Bit()">,
                              AssemblerPredicate<"Mode64Bit", "64-bit mode">;
 def IsLP64  : Predicate<"Subtarget->isTarget64BitLP64()">;
 def NotLP64 : Predicate<"!Subtarget->isTarget64BitLP64()">;
 def In16BitMode  : Predicate<"Subtarget->is16Bit()">,
def : InstAlias<"mwaitx\t{%rax, %rcx, %rbx|rbx, rcx, rax}", (MWAITXrrr)>,
       Requires<[ In64BitMode ]>;
 
 def : InstAlias<"monitorx\t{%eax, %ecx, %edx|edx, ecx, eax}", (MONITORXrrr)>,
       Requires<[ Not64BitMode ]>;
 def : InstAlias<"monitorx\t{%rax, %rcx, %rdx|rdx, rcx, rax}", (MONITORXrrr)>,
       Requires<[ In64BitMode ]>;
 
 //===----------------------------------------------------------------------===//
+// WAITPKG Instructions
+//
+let SchedRW = [WriteSystem] in {
+  def UMONITOR16 : I<0xAE, MRM6r, (outs), (ins GR16:$src),
+                     "umonitor\t$src", [(int_x86_umonitor GR16:$src)]>,
+                     XS, AdSize16, Requires<[HasWAITPKG, Not64BitMode]>;
+  def UMONITOR32 : I<0xAE, MRM6r, (outs), (ins GR32:$src),
+                     "umonitor\t$src", [(int_x86_umonitor GR32:$src)]>,
+                     XS, AdSize32, Requires<[HasWAITPKG]>;
+  def UMONITOR64 : I<0xAE, MRM6r, (outs), (ins GR64:$src),
+                     "umonitor\t$src", [(int_x86_umonitor GR64:$src)]>,
+                     XS, AdSize64, Requires<[HasWAITPKG, In64BitMode]>;
+  let Uses = [EAX, EDX], Defs = [EFLAGS] in {
+    def UMWAIT : I<0xAE, MRM6r,
+                     (outs), (ins GR32orGR64:$src), "umwait\t$src",
+                     [(set EFLAGS, (X86umwait GR32orGR64:$src, EDX, EAX))]>,
+                     XD, Requires<[HasWAITPKG]>;
+    def TPAUSE : I<0xAE, MRM6r,
+                     (outs), (ins GR32orGR64:$src), "tpause\t$src",
+                     [(set EFLAGS, (X86tpause GR32orGR64:$src, EDX, EAX))]>,
+                     PD, Requires<[HasWAITPKG]>;
+  }
+} // SchedRW
+
+//===----------------------------------------------------------------------===//
 // CLZERO Instruction
 //
 let SchedRW = [WriteSystem] in {
   let Uses = [EAX] in
   def CLZEROr : I<0x01, MRM_FC, (outs), (ins), "clzero", []>,
                 TB, Requires<[HasCLZERO]>;
 
   let usesCustomInserter = 1 in {
   def CLZERO : PseudoI<(outs), (ins i32mem:$src1),
                        [(int_x86_clzero addr:$src1)]>, Requires<[HasCLZERO]>;
   }
 } // SchedRW
 
 def : InstAlias<"clzero\t{%eax|eax}", (CLZEROr)>, Requires<[Not64BitMode]>;
 def : InstAlias<"clzero\t{%rax|rax}", (CLZEROr)>, Requires<[In64BitMode]>;
 
 //===----------------------------------------------------------------------===//
 // Pattern fragments to auto generate TBM instructions.
 //===----------------------------------------------------------------------===//
 
 let Predicates = [HasTBM] in {
   // FIXME: patterns for the load versions are not implemented
   def : Pat<(and GR32:$src, (add GR32:$src, 1)),
             (BLCFILL32rr GR32:$src)>;
   def : Pat<(and GR64:$src, (add GR64:$src, 1)),
             (BLCFILL64rr GR64:$src)>;
 
   def : Pat<(or GR32:$src, (not (add GR32:$src, 1))),
             (BLCI32rr GR32:$src)>;
   def : Pat<(or GR64:$src, (not (add GR64:$src, 1))),
             (BLCI64rr GR64:$src)>;
 
   // Extra patterns because opt can optimize the above patterns to this.
   def : Pat<(or GR32:$src, (sub -2, GR32:$src)),
             (BLCI32rr GR32:$src)>;
   def : Pat<(or GR64:$src, (sub -2, GR64:$src)),
             (BLCI64rr GR64:$src)>;
 
   def : Pat<(and (not GR32:$src), (add GR32:$src, 1)),

lib/Target/X86/X86InstrSSE.td

 // Load, store, and memory fence
 // TODO: As with mfence, we may want to ease the availablity of sfence/lfence
 // to include any 64-bit target.
 def SFENCE : I<0xAE, MRM_F8, (outs), (ins), "sfence", [(int_x86_sse_sfence)]>,
                PS, Requires<[HasSSE1]>;
 def LFENCE : I<0xAE, MRM_E8, (outs), (ins), "lfence", [(int_x86_sse2_lfence)]>,
                TB, Requires<[HasSSE2]>;
 def MFENCE : I<0xAE, MRM_F0, (outs), (ins), "mfence", [(int_x86_sse2_mfence)]>,
-               TB, Requires<[HasMFence]>;
+               PS, Requires<[HasMFence]>;
 } // SchedRW
 
 def : Pat<(X86MFence), (MFENCE)>;
 
 //===----------------------------------------------------------------------===//
 // SSE 1 & 2 - Load/Store XCSR register
 //===----------------------------------------------------------------------===//
 

lib/Target/X86/X86Subtarget.h

protected:
   bool HasCLWB;
 
   /// Processor supports Write Back No Invalidate instruction
   bool HasWBNOINVD;
 
   /// Processor support RDPID instruction
   bool HasRDPID;
 
+  /// Processor supports WaitPKG instructions
+  bool HasWAITPKG;
+
   /// Use a retpoline thunk rather than indirect calls to block speculative
   /// execution.
   bool UseRetpoline;
 
   /// When using a retpoline thunk, call an externally provided thunk rather
   /// than emitting one inside the compiler.
   bool UseRetpolineExternalThunk;
 
public:
   bool hasBITALG() const { return HasBITALG; }
   bool hasMPX() const { return HasMPX; }
   bool hasSHSTK() const { return HasSHSTK; }
   bool hasIBT() const { return HasIBT; }
   bool hasCLFLUSHOPT() const { return HasCLFLUSHOPT; }
   bool hasCLWB() const { return HasCLWB; }
   bool hasWBNOINVD() const { return HasWBNOINVD; }
   bool hasRDPID() const { return HasRDPID; }
+  bool hasWAITPKG() const { return HasWAITPKG; }
   bool useRetpoline() const { return UseRetpoline; }
   bool useRetpolineExternalThunk() const { return UseRetpolineExternalThunk; }
 
   unsigned getPreferVectorWidth() const { return PreferVectorWidth; }
   unsigned getRequiredVectorWidth() const { return RequiredVectorWidth; }
 
   // Helper functions to determine when we should allow widening to 512-bit
   // during codegen.

lib/Target/X86/X86Subtarget.cpp

void X86Subtarget::initializeEnvironment() {
   HasMPX = false;
   HasSHSTK = false;
   HasIBT = false;
   HasSGX = false;
   HasCLFLUSHOPT = false;
   HasCLWB = false;
   HasWBNOINVD = false;
   HasRDPID = false;
+  HasWAITPKG = false;
   UseRetpoline = false;
   UseRetpolineExternalThunk = false;
   IsPMULLDSlow = false;
   IsSHLDSlow = false;
   IsUAMem16Slow = false;
   IsUAMem32Slow = false;
   HasSSEUnalignedMem = false;
   HasCmpxchg16b = false;

test/CodeGen/X86/waitpkg-intrinsics-32.ll

+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i386-pc-linux -mattr=+waitpkg | FileCheck %s --check-prefix=X32
+
+define void @test_umonitor(i8* %address) {
+; X32-LABEL: test_umonitor:
+; X32:       # %bb.0: # %entry
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT:    umonitor %eax
+; X32-NEXT:    retl
+entry:
+  call void @llvm.x86.umonitor(i8* %address)
+  ret void
+}
+
+define i8 @test_umwait(i32 %control, i32 %counter_high, i32 %counter_low) {
+; X32-LABEL: test_umwait:
+; X32:       # %bb.0: # %entry
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT:    umwait %ecx
+; X32-NEXT:    setb %al
+; X32-NEXT:    retl
+entry:
+  %0 = call i8 @llvm.x86.umwait(i32 %control, i32 %counter_high, i32 %counter_low)
+  ret i8 %0
+}
+
+define i8 @test_tpause(i32 %control, i32 %counter_high, i32 %counter_low) {
+; X32-LABEL: test_tpause:
+; X32:       # %bb.0: # %entry
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT:    tpause %ecx
+; X32-NEXT:    setb %al
+; X32-NEXT:    retl
+entry:
+  %0 = call i8 @llvm.x86.tpause(i32 %control, i32 %counter_high, i32 %counter_low)
+  ret i8 %0
+}
+
+declare void @llvm.x86.umonitor(i8*)
+declare i8 @llvm.x86.umwait(i32, i32, i32)
+declare i8 @llvm.x86.tpause(i32, i32, i32)

test/CodeGen/X86/waitpkg-intrinsics-64.ll

+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-linux -mattr=+waitpkg | FileCheck %s --check-prefix=X64
+
+define void @test_umonitor(i8* %address) {
+; X64-LABEL: test_umonitor:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    umonitor %rdi
+; X64-NEXT:    retq
+entry:
+  call void @llvm.x86.umonitor(i8* %address)
+  ret void
+}
+
+define i8 @test_umwait(i32 %control, i32 %counter_high, i32 %counter_low) {
+; X64-LABEL: test_umwait:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    movl %edx, %eax
+; X64-NEXT:    movl %esi, %edx
+; X64-NEXT:    umwait %edi
+; X64-NEXT:    setb %al
+; X64-NEXT:    retq
+entry:
+  call i8 @llvm.x86.umwait(i32 %control, i32 %counter_high, i32 %counter_low)
+  ret i8 %0
+}
+
+define i8 @test_tpause(i32 %control, i32 %counter_high, i32 %counter_low) {
+; X64-LABEL: test_tpause:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    movl %edx, %eax
+; X64-NEXT:    movl %esi, %edx
+; X64-NEXT:    tpause %edi
+; X64-NEXT:    setb %al
+; X64-NEXT:    retq
+entry:
+  call i8 @llvm.x86.tpause(i32 %control, i32 %counter_high, i32 %counter_low)
+  ret i8 %0
+}
+
+declare void @llvm.x86.umonitor(i8*)
+declare i8 @llvm.x86.umwait(i32, i32, i32)
+declare i8 @llvm.x86.tpause(i32, i32, i32)

test/MC/Disassembler/X86/x86-16.txt

 # CHECK: lretl
 0x66 0xcb
 
 # CHECK: callw  -1
 0xe8 0xff 0xff
 
 # CHECK: wbnoinvd
 0xf3 0x0f 0x09
+
+# CHECK: umonitor %ax
+0xf3 0x0f 0xae 0xf0
+
+# CHECK: umonitor %eax
+0x67 0xf3 0x0f 0xae 0xf0

test/MC/Disassembler/X86/x86-32.txt

 # CHECK: movswl (%eax), %eax
 0x0f 0xbf 0x00
 # CHECK: movsww (%eax), %ax
 0x66 0x0f 0xbf 0x00
 # CHECK: movzwl (%eax), %eax
 0x0f 0xb7 0x00
 # CHECK: movzww (%eax), %ax
 0x66 0x0f 0xb7 0x00
+
+# CHECK: umonitor %eax
+0xf3 0x0f 0xae 0xf0
+
+# CHECK: umonitor %ax
+0x67 0xf3 0x0f 0xae 0xf0
+
+# CHECK: umwait %eax
+0xf2 0x0f 0xae 0xf0
+
+# CHECK: tpause %eax
+0x66 0x0f 0xae 0xf0

test/MC/Disassembler/X86/x86-64.txt

 # CHECK: wbnoinvd
 0xf3 0x0f 0x09
 
 # CHECK: cldemote 4(%rax)
 0x0f 0x1c 0x40 0x04
 
 # CHECK: cldemote -559038737(%rbx,%rcx,8)
 0x0f,0x1c,0x84,0xcb,0xef,0xbe,0xad,0xde
+
+# CHECK: umonitor %rax
+0xf3 0x0f 0xae 0xf0
+
+# CHECK: umonitor %eax
+0x67 0xf3 0x0f 0xae 0xf0
+
+# CHECK: umonitor %r13
+0xf3 0x41 0x0f 0xae 0xf5
+
+# CHECK: umwait %r15
+0xf2 0x41 0x0f 0xae 0xf7
+
+# CHECK: tpause %r15
+0x66 0x41 0x0f 0xae 0xf7

test/MC/X86/x86-16.s

 // CHECK: encoding: [0x66]
 // CHECK: lgdtw 4(%eax)
 // CHECK:  encoding: [0x67,0x0f,0x01,0x50,0x04]
 data32 lgdt 4(%eax)
 
 // CHECK: wbnoinvd
 // CHECK:  encoding: [0xf3,0x0f,0x09]
 wbnoinvd
+
+// CHECK: umonitor %ax
+// CHECK:  encoding: [0xf3,0x0f,0xae,0xf0]
+umonitor %ax
+
+// CHECK: umonitor %eax
+// CHECK:  encoding: [0x67,0xf3,0x0f,0xae,0xf0]
+umonitor %eax

test/MC/X86/x86-32-coverage.s

 
 // CHECK: cldemote 4(%eax)
 // CHECK:  encoding: [0x0f,0x1c,0x40,0x04]
           cldemote 4(%eax)
 
 // CHECK: cldemote 3735928559(%ebx,%ecx,8)
 // CHECK:  encoding: [0x0f,0x1c,0x84,0xcb,0xef,0xbe,0xad,0xde]
           cldemote 0xdeadbeef(%ebx,%ecx,8)
+
+// CHECK: umonitor %eax
+// CHECK:  encoding: [0xf3,0x0f,0xae,0xf0]
+  umonitor %eax
+
+// CHECK: umonitor %ax
+// CHECK:  encoding: [0x67,0xf3,0x0f,0xae,0xf0]
+  umonitor %ax
+
+// CHECK: umwait %eax
+// CHECK:  encoding: [0xf2,0x0f,0xae,0xf0]
+  umwait %eax
+
+// CHECK: tpause %eax
+// CHECK:  encoding: [0x66,0x0f,0xae,0xf0]
+  tpause %eax

test/MC/X86/x86-64.s

 // CHECK: cldemote 4(%rax)
 // CHECK:  encoding: [0x0f,0x1c,0x40,0x04]
 cldemote 4(%rax)
 
 // CHECK: cldemote 3735928559(%rbx,%rcx,8)
 // CHECK:  encoding: [0x0f,0x1c,0x84,0xcb,0xef,0xbe,0xad,0xde]
 cldemote 0xdeadbeef(%rbx,%rcx,8)
 
+// CHECK: umonitor %r13
+// CHECK:  encoding: [0xf3,0x41,0x0f,0xae,0xf5]
+umonitor %r13
+
+// CHECK: umonitor %rax
+// CHECK:  encoding: [0xf3,0x0f,0xae,0xf0]
+umonitor %rax
+
+// CHECK: umonitor %eax
+// CHECK:  encoding: [0x67,0xf3,0x0f,0xae,0xf0]
+umonitor %eax
+
+// CHECK: umwait %r15
+// CHECK:  encoding: [0xf2,0x41,0x0f,0xae,0xf7]
+umwait %r15
+
+// CHECK: umwait %ebx
+// CHECK:  encoding: [0xf2,0x0f,0xae,0xf3]
+umwait %ebx
+
+// CHECK: tpause %r15
+// CHECK:  encoding: [0x66,0x41,0x0f,0xae,0xf7]
+tpause %r15
+
+// CHECK: tpause %ebx
+// CHECK:  encoding: [0x66,0x0f,0xae,0xf3]
+tpause %ebx
+
 //  __asm __volatile(
 //    "pushf        \n\t"
 //    "popf       \n\t"
 //    "rep        \n\t"
 //    ".byte  0x0f, 0xa7, 0xd0"
 //  );
 // CHECK: pushfq
 // CHECK-NEXT: popfq