[PowerPC] Generate lxsd instead of the ld->mtvsrd sequence for vector loads

[amyk]

Summary

This patch addresses:

• Implementation within PPCISelLowering.cpp to check if we should use direct load into vector instructions (such as lxsd/lfd ) when the scalar_to_vector function is used; which will allow us to catch as many cases of the scalar_to_vector uses as possible to translate the ld->mtvsrd sequence into lxsd.
• Test cases to exhibit the behaviour of emitting lxsd/lfd.

[nemanjai]

llvm/test/CodeGen/PowerPC/pre-inc-disable.ll

11

I don't think this is a meaningful test since the corresponding RUN line doesn't include something like -debug-only=isel.

[amyk]

Updated the patch to address:

• Making the test cases a little bit less exhaustive (removing CHECK-NEXT's)
• unnecessary whitespaces

[nemanjai]

LGTM.

llvm/test/CodeGen/PowerPC/pre-inc-disable.ll

10

This is no longer required since you are not checking for anything that requires an asserts build.

[amyk]

This comment was removed by amyk.

[amyk]

Updated the test case again, removing the assert line as previously addressed.

llvm/lib/Target/PowerPC/PPCISelLowering.cpp

bool PPCTargetLowering::SelectAddressRegRegOnly(SDValue N, SDValue &Base,
 
   // Otherwise, do it the hard way, using R0 as the base register.
   Base = DAG.getRegister(Subtarget.isPPC64() ? PPC::ZERO8 : PPC::ZERO,
                          N.getValueType());
   Index = N;
   return true;
 }
 
+/// Returns true if we should use a direct load into vector instruction
+/// such as lxsd or lfd instead of a load into gpr + direct move sequence.  
+static bool usePartialVectorLoads(SDNode *N) {
+  if (!N->hasOneUse())
+    return false;
+
+  // If there are any other uses other than scalar to vector, then we should
+  // keep it as a scalar load -> direct move pattern to prevent multiple
+  // loads.  Currently, only check for i64 since we have lxsd/lfd to do this 
+  // efficiently, but no update equivalent.
+  if (LoadSDNode *LD = dyn_cast<LoadSDNode>(N)) {
+    EVT MemVT = LD->getMemoryVT();
+    if (MemVT.isSimple() && MemVT.getSimpleVT().SimpleTy == MVT::i64) {
+      SDNode *User = *(LD->use_begin());
+      if (User->getOpcode() == ISD::SCALAR_TO_VECTOR)
+        return true;
+    }
+  }
+
+  return false;
+}
+
 /// getPreIndexedAddressParts - returns true by value, base pointer and
 /// offset pointer and addressing mode by reference if the node's address
 /// can be legally represented as pre-indexed load / store address.
 bool PPCTargetLowering::getPreIndexedAddressParts(SDNode *N, SDValue &Base,
                                                   SDValue &Offset,
                                                   ISD::MemIndexedMode &AM,
                                                   SelectionDAG &DAG) const {
   if (DisablePPCPreinc) return false;
bool PPCTargetLowering::getPreIndexedAddressParts(SDNode *N, SDValue &Base,
   } else if (StoreSDNode *ST = dyn_cast<StoreSDNode>(N)) {
     Ptr = ST->getBasePtr();
     VT  = ST->getMemoryVT();
     Alignment = ST->getAlignment();
     isLoad = false;
   } else
     return false;
 
+  // Do not generate pre-inc forms for specific loads that feed scalar_to_vector
+  // instructions because we can fold these into a more efficient instruction
+  // instead, such as LXSD.
+  if (isLoad && usePartialVectorLoads(N)) {
+    return false;
+  }
+
   // PowerPC doesn't have preinc load/store instructions for vectors (except
   // for QPX, which does have preinc r+r forms).
   if (VT.isVector()) {
     if (!Subtarget.hasQPX() || (VT != MVT::v4f64 && VT != MVT::v4f32)) {
       return false;
     } else if (SelectAddressRegRegOnly(Ptr, Offset, Base, DAG)) {
       AM = ISD::PRE_INC;
       return true;

llvm/test/CodeGen/PowerPC/pre-inc-disable.ll

+; RUN: llc -mcpu=pwr9 -O3 -verify-machineinstrs -ppc-vsr-nums-as-vr \
+; RUN:     -ppc-asm-full-reg-names -mtriple=powerpc64le-unknown-linux-gnu \
+; RUN:     < %s | FileCheck %s
+
+; RUN: llc -mcpu=pwr9 -O3 -verify-machineinstrs -ppc-vsr-nums-as-vr \              
+; RUN:     -ppc-asm-full-reg-names -mtriple=powerpc64-unknown-linux-gnu \        
+; RUN:     < %s | FileCheck %s --check-prefix=P9BE
+
+; REQUIRES: asserts
+
+; CHECK-NOT: i64,i64,ch = load<(load 8 from {{.*}}), <pre-inc>>
+
+; Function Attrs: norecurse nounwind readonly
+define signext i32 @test_pre_inc_disable_1(i8* nocapture readonly %pix1, i32 signext %i_stride_pix1, i8* nocapture readonly %pix2) {
+; CHECK-LABEL: test_pre_inc_disable_1:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    addis r6, r2, .LCPI0_0@toc@ha
+; CHECK-NEXT:    addis r7, r2, .LCPI0_1@toc@ha
+; CHECK-NEXT:    lfd f0, 0(r5)
+; CHECK-NEXT:    xxlxor v4, v4, v4
+; CHECK-NEXT:    li r5, 4
+; CHECK-NEXT:    addi r12, r6, .LCPI0_0@toc@l
+; CHECK-NEXT:    addi r6, r7, .LCPI0_1@toc@l
+; CHECK-NEXT:    mtctr r5
+; CHECK-NEXT:    li r5, 0
+; CHECK-NEXT:    lxvx v3, 0, r6
+; CHECK-NEXT:    lxvx v2, 0, r12
+; CHECK-NEXT:    xxpermdi v5, f0, f0, 2
+; CHECK-NEXT:    li r6, 0
+; CHECK-NEXT:    vperm v0, v4, v5, v2
+; CHECK-NEXT:    vperm v5, v5, v4, v3
+; CHECK-NEXT:    xvnegsp v5, v5
+; CHECK-NEXT:    xvnegsp v0, v0
+; CHECK-NEXT:    .p2align 4
+; CHECK-NEXT:  .LBB0_1: # %for.cond1.preheader
+; CHECK:         lfd f0, 0(r3)
+; CHECK-NEXT:    xxpermdi v1, f0, f0, 2
+; CHECK-NEXT:    vperm v6, v1, v4, v3
+; CHECK-NEXT:    vperm v1, v4, v1, v2
+; CHECK-NEXT:    xvnegsp v6, v6
+; CHECK-NEXT:    xvnegsp v1, v1
+; CHECK-NEXT:    vabsduw v6, v6, v5
+; CHECK-NEXT:    vabsduw v1, v1, v0
+; CHECK-NEXT:    vadduwm v1, v6, v1
+; CHECK-NEXT:    xxswapd v6, v1
+; CHECK-NEXT:    vadduwm v1, v1, v6
+; CHECK-NEXT:    xxspltw v6, v1, 2
+; CHECK-NEXT:    vadduwm v1, v1, v6
+; CHECK-NEXT:    vextuwrx r7, r6, v1
+; CHECK-NEXT:    ldux r8, r3, r4
+; CHECK-NEXT:    add r3, r3, r4
+; CHECK-NEXT:    add r5, r7, r5
+; CHECK-NEXT:    mtvsrd f0, r8
+; CHECK-NEXT:    xxswapd v18, vs0
+; CHECK-NEXT:    vperm v1, v4, v18, v2
+; CHECK-NEXT:    vperm v19, v18, v4, v3
+; CHECK-NEXT:    xvnegsp v6, v19
+; CHECK-NEXT:    xvnegsp v1, v1
+; CHECK-NEXT:    vabsduw v1, v1, v0
+; CHECK-NEXT:    vabsduw v6, v6, v5
+; CHECK-NEXT:    vadduwm v1, v6, v1
+; CHECK-NEXT:    xxswapd v6, v1
+; CHECK-NEXT:    vadduwm v1, v1, v6
+; CHECK-NEXT:    xxspltw v6, v1, 2
+; CHECK-NEXT:    vadduwm v1, v1, v6
+; CHECK-NEXT:    vextuwrx r12, r6, v1
+; CHECK-NEXT:    add r5, r12, r5
+; CHECK-NEXT:    bdnz .LBB0_1
+; CHECK-NEXT:  # %bb.2: # %for.cond.cleanup
+; CHECK-NEXT:    extsw r3, r5
+; CHECK-NEXT:    blr
+
+; P9BE-LABEL: test_pre_inc_disable_1:                                          
+; P9BE:       # %bb.0: # %entry                                                
+; P9BE-NEXT:    addis r6, r2, .LCPI0_0@toc@ha                                  
+; P9BE-NEXT:    addis r7, r2, .LCPI0_1@toc@ha                                  
+; P9BE-NEXT:    lfd f0, 0(r5)                                                  
+; P9BE-NEXT:    xxlxor v4, v4, v4                                              
+; P9BE-NEXT:    li r5, 4                                                       
+; P9BE-NEXT:    addi r12, r6, .LCPI0_0@toc@l                                   
+; P9BE-NEXT:    addi r6, r7, .LCPI0_1@toc@l                                    
+; P9BE-NEXT:    mtctr r5                                                       
+; P9BE-NEXT:    li r5, 0                                                       
+; P9BE-NEXT:    lxvx v3, 0, r6                                                 
+; P9BE-NEXT:    lxvx v2, 0, r12                                                
+; P9BE-NEXT:    xxlor v5, vs0, vs0                                         
+; P9BE-NEXT:    li r6, 0                                                       
+; P9BE-NEXT:    vperm v0, v4, v5, v2                                           
+; P9BE-NEXT:    vperm v5, v4, v5, v3                                           
+; P9BE-NEXT:    xvnegsp v5, v5                                                 
+; P9BE-NEXT:    xvnegsp v0, v0                                                 
+; P9BE-NEXT:    .p2align 4                                                     
+; P9BE-NEXT:  .LBB0_1: # %for.cond1.preheader                                  
+; P9BE:         lfd f0, 0(r3)                                                  
+; P9BE-NEXT:    xxlor v1, vs0, vs0                                          
+; P9BE-NEXT:    vperm v6, v4, v1, v3                                           
+; P9BE-NEXT:    vperm v1, v4, v1, v2                                           
+; P9BE-NEXT:    xvnegsp v6, v6                                                 
+; P9BE-NEXT:    xvnegsp v1, v1                                                 
+; P9BE-NEXT:    vabsduw v1, v1, v0                                             
+; P9BE-NEXT:    vabsduw v6, v6, v5                                             
+; P9BE-NEXT:    vadduwm v1, v6, v1                                             
+; P9BE-NEXT:    xxswapd v6, v1                                                 
+; P9BE-NEXT:    vadduwm v1, v1, v6                                             
+; P9BE-NEXT:    xxspltw v6, v1, 1                                              
+; P9BE-NEXT:    vadduwm v1, v1, v6                                             
+; P9BE-NEXT:    vextuwlx r7, r6, v1                                            
+; P9BE-NEXT:    ldux r8, r3, r4                                                
+; P9BE-NEXT:    add r3, r3, r4                                                 
+; P9BE-NEXT:    add r5, r7, r5                                                 
+; P9BE-NEXT:    mtvsrd v1, r8                                                  
+; P9BE-NEXT:    vperm v19, v4, v1, v3                                               
+; P9BE-NEXT:    vperm v1, v4, v1, v2                                         
+; P9BE-NEXT:    xvnegsp v6, v19                                                
+; P9BE-NEXT:    xvnegsp v1, v1                                                 
+; P9BE-NEXT:    vabsduw v1, v1, v0                                             
+; P9BE-NEXT:    vabsduw v6, v6, v5                                             
+; P9BE-NEXT:    vadduwm v1, v6, v1                                             
+; P9BE-NEXT:    xxswapd v6, v1                                                 
+; P9BE-NEXT:    vadduwm v1, v1, v6                                             
+; P9BE-NEXT:    xxspltw v6, v1, 1                                              
+; P9BE-NEXT:    vadduwm v1, v1, v6                                             
+; P9BE-NEXT:    vextuwlx r12, r6, v1                                           
+; P9BE-NEXT:    add r5, r12, r5                                                
+; P9BE-NEXT:    bdnz .LBB0_1                                                   
+; P9BE-NEXT:  # %bb.2: # %for.cond.cleanup                                     
+; P9BE-NEXT:    extsw r3, r5                                                   
+; P9BE-NEXT:    blr 
+entry:
+  %idx.ext = sext i32 %i_stride_pix1 to i64
+  %0 = bitcast i8* %pix2 to <8 x i8>*
+  %1 = load <8 x i8>, <8 x i8>* %0, align 1
+  %2 = zext <8 x i8> %1 to <8 x i32>
+  br label %for.cond1.preheader
+
+for.cond1.preheader:                              ; preds = %for.cond1.preheader, %entry
+  %y.024 = phi i32 [ 0, %entry ], [ %inc9.1, %for.cond1.preheader ]
+  %i_sum.023 = phi i32 [ 0, %entry ], [ %op.extra.1, %for.cond1.preheader ]
+  %pix1.addr.022 = phi i8* [ %pix1, %entry ], [ %add.ptr.1, %for.cond1.preheader ]
+  %3 = bitcast i8* %pix1.addr.022 to <8 x i8>*
+  %4 = load <8 x i8>, <8 x i8>* %3, align 1
+  %5 = zext <8 x i8> %4 to <8 x i32>
+  %6 = sub nsw <8 x i32> %5, %2
+  %7 = icmp slt <8 x i32> %6, zeroinitializer
+  %8 = sub nsw <8 x i32> zeroinitializer, %6
+  %9 = select <8 x i1> %7, <8 x i32> %8, <8 x i32> %6
+  %rdx.shuf = shufflevector <8 x i32> %9, <8 x i32> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
+  %bin.rdx = add nsw <8 x i32> %9, %rdx.shuf
+  %rdx.shuf32 = shufflevector <8 x i32> %bin.rdx, <8 x i32> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %bin.rdx33 = add nsw <8 x i32> %bin.rdx, %rdx.shuf32
+  %rdx.shuf34 = shufflevector <8 x i32> %bin.rdx33, <8 x i32> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %bin.rdx35 = add nsw <8 x i32> %bin.rdx33, %rdx.shuf34
+  %10 = extractelement <8 x i32> %bin.rdx35, i32 0
+  %op.extra = add nsw i32 %10, %i_sum.023
+  %add.ptr = getelementptr inbounds i8, i8* %pix1.addr.022, i64 %idx.ext
+  %11 = bitcast i8* %add.ptr to <8 x i8>*
+  %12 = load <8 x i8>, <8 x i8>* %11, align 1
+  %13 = zext <8 x i8> %12 to <8 x i32>
+  %14 = sub nsw <8 x i32> %13, %2
+  %15 = icmp slt <8 x i32> %14, zeroinitializer
+  %16 = sub nsw <8 x i32> zeroinitializer, %14
+  %17 = select <8 x i1> %15, <8 x i32> %16, <8 x i32> %14
+  %rdx.shuf.1 = shufflevector <8 x i32> %17, <8 x i32> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
+  %bin.rdx.1 = add nsw <8 x i32> %17, %rdx.shuf.1
+  %rdx.shuf32.1 = shufflevector <8 x i32> %bin.rdx.1, <8 x i32> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %bin.rdx33.1 = add nsw <8 x i32> %bin.rdx.1, %rdx.shuf32.1
+  %rdx.shuf34.1 = shufflevector <8 x i32> %bin.rdx33.1, <8 x i32> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %bin.rdx35.1 = add nsw <8 x i32> %bin.rdx33.1, %rdx.shuf34.1
+  %18 = extractelement <8 x i32> %bin.rdx35.1, i32 0
+  %op.extra.1 = add nsw i32 %18, %op.extra
+  %add.ptr.1 = getelementptr inbounds i8, i8* %add.ptr, i64 %idx.ext
+  %inc9.1 = add nuw nsw i32 %y.024, 2
+  %exitcond.1 = icmp eq i32 %inc9.1, 8
+  br i1 %exitcond.1, label %for.cond.cleanup, label %for.cond1.preheader
+
+for.cond.cleanup:                                 ; preds = %for.cond1.preheader
+  ret i32 %op.extra.1
+}
+
+; Function Attrs: norecurse nounwind readonly
+define signext i32 @test_pre_inc_disable_2(i8* nocapture readonly %pix1, i8* nocapture readonly %pix2) {
+; CHECK-LABEL: test_pre_inc_disable_2:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    addis r5, r2, .LCPI1_0@toc@ha
+; CHECK-NEXT:    addis r6, r2, .LCPI1_1@toc@ha
+; CHECK-NEXT:    lfd f0, 0(r3)
+; CHECK-NEXT:    lfd f1, 0(r4)
+; CHECK-NEXT:    xxlxor v0, v0, v0
+; CHECK-NEXT:    li r3, 0
+; CHECK-NEXT:    addi r12, r5, .LCPI1_0@toc@l
+; CHECK-NEXT:    addi r4, r6, .LCPI1_1@toc@l
+; CHECK-NEXT:    lxvx v2, 0, r12
+; CHECK-NEXT:    lxvx v3, 0, r4
+; CHECK-NEXT:    xxpermdi v4, f0, f0, 2
+; CHECK-NEXT:    xxpermdi v5, f1, f1, 2
+; CHECK-NEXT:    vperm v1, v4, v0, v2
+; CHECK-NEXT:    vperm v4, v0, v4, v3
+; CHECK-NEXT:    vperm v2, v5, v0, v2
+; CHECK-NEXT:    vperm v3, v0, v5, v3
+; CHECK-NEXT:    xvnegsp v19, v1
+; CHECK-NEXT:    xvnegsp v4, v4
+; CHECK-NEXT:    xvnegsp v2, v2
+; CHECK-NEXT:    xvnegsp v3, v3
+; CHECK-NEXT:    vabsduw v3, v4, v3
+; CHECK-NEXT:    vabsduw v2, v19, v2
+; CHECK-NEXT:    vadduwm v2, v2, v3
+; CHECK-NEXT:    xxswapd v3, v2
+; CHECK-NEXT:    vadduwm v2, v2, v3
+; CHECK-NEXT:    xxspltw v3, v2, 2
+; CHECK-NEXT:    vadduwm v2, v2, v3
+; CHECK-NEXT:    vextuwrx r3, r3, v2
+; CHECK-NEXT:    extsw r3, r3
+; CHECK-NEXT:    blr
+
+; P9BE-LABEL: test_pre_inc_disable_2:                                          
+; P9BE:       # %bb.0: # %entry                                                
+; P9BE-NEXT:    addis r5, r2, .LCPI1_0@toc@ha                                  
+; P9BE-NEXT:    addis r6, r2, .LCPI1_1@toc@ha                                  
+; P9BE-NEXT:    lfd f1, 0(r4)                                                  
+; P9BE-NEXT:    lfd f0, 0(r3)                                                  
+; P9BE-NEXT:    xxlxor v5, v5, v5                                              
+; P9BE-NEXT:    li r3, 0                                                       
+; P9BE-NEXT:    addi r12, r5, .LCPI1_0@toc@l                                   
+; P9BE-NEXT:    addi r4, r6, .LCPI1_1@toc@l                                    
+; P9BE-NEXT:    lxvx v2, 0, r12                                                
+; P9BE-NEXT:    lxvx v3, 0, r4                                                 
+; P9BE-NEXT:    xxlor v4, vs0, vs0                                         
+; P9BE-NEXT:    xxlor v0, vs1, vs1                                         
+; P9BE-NEXT:    vperm v1, v5, v4, v2                                           
+; P9BE-NEXT:    vperm v4, v5, v4, v3                                           
+; P9BE-NEXT:    vperm v2, v5, v0, v2                                           
+; P9BE-NEXT:    vperm v3, v5, v0, v3                                           
+; P9BE-NEXT:    xvnegsp v19, v1                                                
+; P9BE-NEXT:    xvnegsp v4, v4                                                 
+; P9BE-NEXT:    xvnegsp v2, v2                                                 
+; P9BE-NEXT:    xvnegsp v3, v3                                                 
+; P9BE-NEXT:    vabsduw v3, v4, v3                                             
+; P9BE-NEXT:    vabsduw v2, v19, v2                                            
+; P9BE-NEXT:    vadduwm v2, v2, v3                                             
+; P9BE-NEXT:    xxswapd v3, v2                                                 
+; P9BE-NEXT:    vadduwm v2, v2, v3                                             
+; P9BE-NEXT:    xxspltw v3, v2, 1                                              
+; P9BE-NEXT:    vadduwm v2, v2, v3                                             
+; P9BE-NEXT:    vextuwlx r3, r3, v2                                            
+; P9BE-NEXT:    extsw r3, r3                                                   
+; P9BE-NEXT:    blr  
+entry:
+  %0 = bitcast i8* %pix1 to <8 x i8>*
+  %1 = load <8 x i8>, <8 x i8>* %0, align 1
+  %2 = zext <8 x i8> %1 to <8 x i32>
+  %3 = bitcast i8* %pix2 to <8 x i8>*
+  %4 = load <8 x i8>, <8 x i8>* %3, align 1
+  %5 = zext <8 x i8> %4 to <8 x i32>
+  %6 = sub nsw <8 x i32> %2, %5
+  %7 = icmp slt <8 x i32> %6, zeroinitializer
+  %8 = sub nsw <8 x i32> zeroinitializer, %6
+  %9 = select <8 x i1> %7, <8 x i32> %8, <8 x i32> %6
+  %rdx.shuf = shufflevector <8 x i32> %9, <8 x i32> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
+  %bin.rdx = add nsw <8 x i32> %9, %rdx.shuf
+  %rdx.shuf12 = shufflevector <8 x i32> %bin.rdx, <8 x i32> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %bin.rdx13 = add nsw <8 x i32> %bin.rdx, %rdx.shuf12
+  %rdx.shuf14 = shufflevector <8 x i32> %bin.rdx13, <8 x i32> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %bin.rdx15 = add nsw <8 x i32> %bin.rdx13, %rdx.shuf14
+  %10 = extractelement <8 x i32> %bin.rdx15, i32 0
+  ret i32 %10
+}
+
+
+; Generated from C source:
+;
+;#include <stdint.h>
+;#include <stdlib.h>
+;int test_pre_inc_disable_1( uint8_t *pix1, int i_stride_pix1, uint8_t *pix2 ) {                                                   
+;    int i_sum = 0;                                  
+;    for( int y = 0; y < 8; y++ )                   
+;    {                                               
+;        for( int x = 0; x < 8; x++ )               
+;        {                                           
+;            i_sum += abs( pix1[x] - pix2[x] );      
+;        }                                           
+;        pix1 += i_stride_pix1;                                            
+;    }                                               
+;    return i_sum;                                   
+;}
+
+;int test_pre_inc_disable_2( uint8_t *pix1, uint8_t *pix2 ) {                       
+;  int i_sum = 0;                                                                   
+;  for( int x = 0; x < 8; x++ ) {                                                   
+;    i_sum += abs( pix1[x] - pix2[x] );                                             
+;  }                                                                                
+;                                                                                   
+;  return i_sum;                                                                    
+;}
+