[X86] Use BEXTR/BEXTRI for 64-bit 'and' with a large mask

[craig.topper]

Summary

The 64-bit 'and' with immediate instruction only supports a 32-bit immediate. So for larger constants we have to load the constant into a register first. If the immediate happens to be a mask we can use the BEXTRI instruction to perform the masking. We already do something similar using the BZHI instruction from the BMI2 instruction set.

[RKSimon]

Nice - useful for BMI1 as well do you think? Would use a second register like the AND case but requires a smaller immediate and would allow more load folding opportunities.

[craig.topper]

Are there any CPUs that support BMI1 without TBM or BMI2?

[RKSimon]

In D36129#827503, @craig.topper wrote:

Are there any CPUs that support BMI1 without TBM or BMI2?

Yes, Jaguar (btver2)

[craig.topper]

Added BEXTR for BMI1 only.

[RKSimon]

LGTM - thanks.

llvm/trunk/lib/Target/X86/X86InstrInfo.td

 def HasXSAVEOPT  : Predicate<"Subtarget->hasXSAVEOPT()">;
 def HasXSAVEC    : Predicate<"Subtarget->hasXSAVEC()">;
 def HasXSAVES    : Predicate<"Subtarget->hasXSAVES()">;
 def HasPCLMUL    : Predicate<"Subtarget->hasPCLMUL()">;
 def HasFMA       : Predicate<"Subtarget->hasFMA()">;
 def HasFMA4      : Predicate<"Subtarget->hasFMA4()">;
 def HasXOP       : Predicate<"Subtarget->hasXOP()">;
 def HasTBM       : Predicate<"Subtarget->hasTBM()">;
+def NoTBM        : Predicate<"!Subtarget->hasTBM()">;
 def HasLWP       : Predicate<"Subtarget->hasLWP()">;
 def HasMOVBE     : Predicate<"Subtarget->hasMOVBE()">;
 def HasRDRAND    : Predicate<"Subtarget->hasRDRAND()">;
 def HasF16C      : Predicate<"Subtarget->hasF16C()">;
 def NoF16C       : Predicate<"!Subtarget->hasF16C()">;
 def HasFSGSBase  : Predicate<"Subtarget->hasFSGSBase()">;
 def HasLZCNT     : Predicate<"Subtarget->hasLZCNT()">;
 def HasBMI       : Predicate<"Subtarget->hasBMI()">;
 def HasBMI2      : Predicate<"Subtarget->hasBMI2()">;
+def NoBMI2       : Predicate<"!Subtarget->hasBMI2()">;
 def HasVBMI      : Predicate<"Subtarget->hasVBMI()">,
                      AssemblerPredicate<"FeatureVBMI", "AVX-512 VBMI ISA">;
 def HasIFMA      : Predicate<"Subtarget->hasIFMA()">,
                      AssemblerPredicate<"FeatureIFMA", "AVX-512 IFMA ISA">;
 def HasRTM       : Predicate<"Subtarget->hasRTM()">;
 def HasADX       : Predicate<"Subtarget->hasADX()">;
 def HasSHA       : Predicate<"Subtarget->hasSHA()">;
 def HasPRFCHW    : Predicate<"Subtarget->hasPRFCHW()">;

 }
 
 
 def CountTrailingOnes : SDNodeXForm<imm, [{
   // Count the trailing ones in the immediate.
   return getI8Imm(countTrailingOnes(N->getZExtValue()), SDLoc(N));
 }]>;
 
-def BZHIMask : ImmLeaf<i64, [{
-  return isMask_64(Imm) && (countTrailingOnes<uint64_t>(Imm) > 32);
-}]>;
-
-let Predicates = [HasBMI2] in {
-  def : Pat<(and GR64:$src, BZHIMask:$mask),
+def BEXTRMaskXForm : SDNodeXForm<imm, [{
+  unsigned Length = countTrailingOnes(N->getZExtValue());
+  return getI32Imm(Length << 8, SDLoc(N));
+}]>;
+
+def AndMask64 : ImmLeaf<i64, [{
+  return isMask_64(Imm) && Imm > UINT32_MAX;
+}]>;
+
+// Use BEXTR for 64-bit 'and' with large immediate 'mask'.
+let Predicates = [HasBMI, NoBMI2, NoTBM] in {
+  def : Pat<(and GR64:$src, AndMask64:$mask),
+            (BEXTR64rr GR64:$src,
+              (SUBREG_TO_REG (i64 0),
+                             (MOV32ri (BEXTRMaskXForm imm:$mask)), sub_32bit))>;
+  def : Pat<(and (loadi64 addr:$src), AndMask64:$mask),
+            (BEXTR64rm addr:$src,
+              (SUBREG_TO_REG (i64 0),
+                             (MOV32ri (BEXTRMaskXForm imm:$mask)), sub_32bit))>;
+}
+
+// Use BZHI for 64-bit 'and' with large immediate 'mask'.
+let Predicates = [HasBMI2, NoTBM] in {
+  def : Pat<(and GR64:$src, AndMask64:$mask),
             (BZHI64rr GR64:$src,
               (INSERT_SUBREG (i64 (IMPLICIT_DEF)),
                              (MOV8ri (CountTrailingOnes imm:$mask)), sub_8bit))>;
-  def : Pat<(and (loadi64 addr:$src), BZHIMask:$mask),
+  def : Pat<(and (loadi64 addr:$src), AndMask64:$mask),
             (BZHI64rm addr:$src,
               (INSERT_SUBREG (i64 (IMPLICIT_DEF)),
                              (MOV8ri (CountTrailingOnes imm:$mask)), sub_8bit))>;
+}
 
+let Predicates = [HasBMI2] in {
   def : Pat<(and GR32:$src, (add (shl 1, GR8:$lz), -1)),
             (BZHI32rr GR32:$src,
               (INSERT_SUBREG (i32 (IMPLICIT_DEF)), GR8:$lz, sub_8bit))>;
 
   def : Pat<(and (loadi32 addr:$src), (add (shl 1, GR8:$lz), -1)),
             (BZHI32rm addr:$src,
               (INSERT_SUBREG (i32 (IMPLICIT_DEF)), GR8:$lz, sub_8bit))>;
 

 defm BLCMSK  : tbm_binary_intr<0x02, "blcmsk", MRM1r, MRM1m>;
 defm BLCS    : tbm_binary_intr<0x01, "blcs", MRM3r, MRM3m>;
 defm BLSFILL : tbm_binary_intr<0x01, "blsfill", MRM2r, MRM2m>;
 defm BLSIC   : tbm_binary_intr<0x01, "blsic", MRM6r, MRM6m>;
 defm T1MSKC  : tbm_binary_intr<0x01, "t1mskc", MRM7r, MRM7m>;
 defm TZMSK   : tbm_binary_intr<0x01, "tzmsk", MRM4r, MRM4m>;
 } // HasTBM, EFLAGS
 
+// Use BEXTRI for 64-bit 'and' with large immediate 'mask'.
+let Predicates = [HasTBM] in {
+  def : Pat<(and GR64:$src, AndMask64:$mask),
+            (BEXTRI64ri GR64:$src, (BEXTRMaskXForm imm:$mask))>;
+
+  def : Pat<(and (loadi64 addr:$src), AndMask64:$mask),
+            (BEXTRI64mi addr:$src, (BEXTRMaskXForm imm:$mask))>;
+}
+
 //===----------------------------------------------------------------------===//
 // Lightweight Profiling Instructions
 
 let Predicates = [HasLWP] in {
 
 def LLWPCB : I<0x12, MRM0r, (outs), (ins GR32:$src), "llwpcb\t$src",
                [(int_x86_llwpcb GR32:$src)], IIC_LWP>,
                XOP, XOP9, Requires<[Not64BitMode]>;

llvm/trunk/test/CodeGen/X86/bmi.ll

entry:
   %shl = shl i64 %a, %sh_prom
   %shr = lshr i64 %shl, %sh_prom
   ret i64 %shr
 }
 
 define i64 @bzhi64_constant_mask(i64 %x) {
 ; BMI1-LABEL: bzhi64_constant_mask:
 ; BMI1:       # BB#0: # %entry
-; BMI1-NEXT:    movabsq $4611686018427387903, %rax # imm = 0x3FFFFFFFFFFFFFFF
-; BMI1-NEXT:    andq %rdi, %rax
+; BMI1-NEXT:    movl $15872, %eax # imm = 0x3E00
+; BMI1-NEXT:    bextrq %rax, %rdi, %rax
 ; BMI1-NEXT:    retq
 ;
 ; BMI2-LABEL: bzhi64_constant_mask:
 ; BMI2:       # BB#0: # %entry
 ; BMI2-NEXT:    movb $62, %al
 ; BMI2-NEXT:    bzhiq %rax, %rdi, %rax
 ; BMI2-NEXT:    retq
 entry:
   %and = and i64 %x, 4611686018427387903
   ret i64 %and
 }
 
 define i64 @bzhi64_constant_mask_load(i64* %x) {
 ; BMI1-LABEL: bzhi64_constant_mask_load:
 ; BMI1:       # BB#0: # %entry
-; BMI1-NEXT:    movabsq $4611686018427387903, %rax # imm = 0x3FFFFFFFFFFFFFFF
-; BMI1-NEXT:    andq (%rdi), %rax
+; BMI1-NEXT:    movl $15872, %eax # imm = 0x3E00
+; BMI1-NEXT:    bextrq %rax, (%rdi), %rax
 ; BMI1-NEXT:    retq
 ;
 ; BMI2-LABEL: bzhi64_constant_mask_load:
 ; BMI2:       # BB#0: # %entry
 ; BMI2-NEXT:    movb $62, %al
 ; BMI2-NEXT:    bzhiq %rax, (%rdi), %rax
 ; BMI2-NEXT:    retq
 entry:

llvm/trunk/test/CodeGen/X86/tbm_patterns.ll

 ; CHECK-NEXT:    tzmsk %rdi, %rax
 ; CHECK-NEXT:    retq
   %t0 = xor i64 %a, -1
   %t1 = add i64 %a, -1
   %t2 = and i64 %t0, %t1
   ret i64 %t2
 }
 
+define i64 @test_and_large_constant_mask(i64 %x) {
+; CHECK-LABEL: test_and_large_constant_mask:
+; CHECK:       # BB#0: # %entry
+; CHECK-NEXT:    bextr $15872, %rdi, %rax # imm = 0x3E00
+; CHECK-NEXT:    retq
+entry:
+  %and = and i64 %x, 4611686018427387903
+  ret i64 %and
+}
+
+define i64 @test_and_large_constant_mask_load(i64* %x) {
+; CHECK-LABEL: test_and_large_constant_mask_load:
+; CHECK:       # BB#0: # %entry
+; CHECK-NEXT:    bextr $15872, (%rdi), %rax # imm = 0x3E00
+; CHECK-NEXT:    retq
+entry:
+  %x1 = load i64, i64* %x
+  %and = and i64 %x1, 4611686018427387903
+  ret i64 %and
+}