[ARM] Add FP handling for MVE lane interleaving

[dmgreen]

Summary

FP16 to FP32 converts can be handled in MVE lane interleaving, much like the sext/zext lowering we do. This expands the pass with fpext and fptrunc handling, and basic fp operations allowing more efficient lowering of fp vectors.

[dmgreen]

Rebase

[SjoerdMeijer]

llvm/lib/Target/ARM/MVELaneInterleavingPass.cpp

127

Can you clarify this a bit? I am no following the "extra VCVT's" => "cheap" reasoning. I am guessing you meant expensive?

[dmgreen]

Yep. "cheap" -> "beneficial to convert"

[SjoerdMeijer]

Thanks, LGTM

llvm/lib/Target/ARM/MVELaneInterleavingPass.cpp

static bool isProfitableToInterleave(SmallSetVector<Instruction *, 4> &Exts,
   // Truncs are usually the same number of instructions,
   //  VSTRH.32(A);VSTRH.32(B) vs VSTRH.16(VMOVNT A, B) with interleaving
   // Exts are unfortunately more instructions in the general case:
   //  A=VLDRH.32; B=VLDRH.32;
   // vs with interleaving:
   //  T=VLDRH.16; A=VMOVNB T; B=VMOVNT T
   // But those VMOVL may be folded into a VMULL.
 
-  // But expensive extends/truncs are always good to remove.
-  for (auto *E : Exts)
-    if (!isa<LoadInst>(E->getOperand(0))) {
+  // But expensive extends/truncs are always good to remove. FPExts always
+  // involve extra VCVT's so are always considered to be beneficial to convert.
+  for (auto *E : Exts) {
+    if (isa<FPExtInst>(E) || !isa<LoadInst>(E->getOperand(0))) {
       LLVM_DEBUG(dbgs() << "Beneficial due to " << *E << "\n");
       return true;
     }
-  for (auto *T : Truncs)
+  }
+  for (auto *T : Truncs) {
     if (T->hasOneUse() && !isa<StoreInst>(*T->user_begin())) {
       LLVM_DEBUG(dbgs() << "Beneficial due to " << *T << "\n");
       return true;
     }
+  }
 
   // Otherwise, we know we have a load(ext), see if any of the Extends are a
   // vmull. This is a simple heuristic and certainly not perfect.
   for (auto *E : Exts) {
     if (!E->hasOneUse() ||
         cast<Instruction>(*E->user_begin())->getOpcode() != Instruction::Mul) {
       LLVM_DEBUG(dbgs() << "Not beneficial due to " << *E << "\n");
       return false;
static bool tryInterleave(Instruction *Start,
 
   while (!Worklist.empty()) {
     Instruction *I = Worklist.back();
     Worklist.pop_back();
 
     switch (I->getOpcode()) {
     // Truncs
     case Instruction::Trunc:
+    case Instruction::FPTrunc:
       if (Truncs.count(I))
         continue;
       Truncs.insert(I);
       Visited.insert(I);
       break;
 
     // Extend leafs
     case Instruction::SExt:
     case Instruction::ZExt:
+    case Instruction::FPExt:
       if (Exts.count(I))
         continue;
       for (auto *Use : I->users())
         Worklist.push_back(cast<Instruction>(Use));
       Exts.insert(I);
       break;
 
     // Binary/tertiary ops
     case Instruction::Add:
     case Instruction::Sub:
     case Instruction::Mul:
     case Instruction::AShr:
     case Instruction::LShr:
     case Instruction::Shl:
     case Instruction::ICmp:
+    case Instruction::FCmp:
+    case Instruction::FAdd:
+    case Instruction::FMul:
     case Instruction::Select:
       if (Ops.count(I))
         continue;
       Ops.insert(I);
 
       for (Use &Op : I->operands()) {
         if (isa<Instruction>(Op))
           Worklist.push_back(cast<Instruction>(&Op));
for (unsigned i = 0; i < BaseElts / 2; i++) {
       TruncMask.push_back(Base + i + BaseElts / 2);
     }
   }
 
   for (Instruction *I : Exts) {
     LLVM_DEBUG(dbgs() << "Replacing ext " << *I << "\n");
     Builder.SetInsertPoint(I);
     Value *Shuffle = Builder.CreateShuffleVector(I->getOperand(0), LeafMask);
+    bool FPext = isa<FPExtInst>(I);
     bool Sext = isa<SExtInst>(I);
-    Value *Ext = Sext ? Builder.CreateSExt(Shuffle, I->getType())
+    Value *Ext = FPext ? Builder.CreateFPExt(Shuffle, I->getType())
+                       : Sext ? Builder.CreateSExt(Shuffle, I->getType())
                               : Builder.CreateZExt(Shuffle, I->getType());
     I->replaceAllUsesWith(Ext);
     LLVM_DEBUG(dbgs() << "  with " << *Shuffle << "\n");
   }
 
   for (Use *I : OtherLeafs) {
     LLVM_DEBUG(dbgs() << "Replacing leaf " << *I << "\n");
     Builder.SetInsertPoint(cast<Instruction>(I->getUser()));
     Value *Shuffle = Builder.CreateShuffleVector(I->get(), LeafMask);

llvm/test/CodeGen/Thumb2/mve-fp16convertloops.ll

 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    .save {r7, lr}
 ; CHECK-NEXT:    push {r7, lr}
 ; CHECK-NEXT:    mov.w lr, #128
 ; CHECK-NEXT:    adr r2, .LCPI7_0
 ; CHECK-NEXT:    vldrw.u32 q0, [r2]
 ; CHECK-NEXT:  .LBB7_1: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    vldrh.u32 q1, [r0, #8]
-; CHECK-NEXT:    vcvtb.f32.f16 q1, q1
-; CHECK-NEXT:    vmul.f32 q1, q1, q0
-; CHECK-NEXT:    vcvtb.f16.f32 q1, q1
-; CHECK-NEXT:    vstrh.32 q1, [r1, #8]
-; CHECK-NEXT:    vldrh.u32 q1, [r0], #16
-; CHECK-NEXT:    vcvtb.f32.f16 q1, q1
+; CHECK-NEXT:    vldrh.u16 q1, [r0], #16
+; CHECK-NEXT:    vcvtb.f32.f16 q2, q1
+; CHECK-NEXT:    vcvtt.f32.f16 q1, q1
+; CHECK-NEXT:    vmul.f32 q2, q2, q0
 ; CHECK-NEXT:    vmul.f32 q1, q1, q0
-; CHECK-NEXT:    vcvtb.f16.f32 q1, q1
-; CHECK-NEXT:    vstrh.32 q1, [r1], #16
+; CHECK-NEXT:    vcvtb.f16.f32 q2, q2
+; CHECK-NEXT:    vcvtt.f16.f32 q2, q1
+; CHECK-NEXT:    vstrb.8 q2, [r1], #16
 ; CHECK-NEXT:    le lr, .LBB7_1
 ; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
 ; CHECK-NEXT:    pop {r7, pc}
 ; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  @ %bb.3:
 ; CHECK-NEXT:  .LCPI7_0:
 ; CHECK-NEXT:    .long 0x40066666 @ float 2.0999999
 ; CHECK-NEXT:    .long 0x40066666 @ float 2.0999999

 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    .save {r7, lr}
 ; CHECK-NEXT:    push {r7, lr}
 ; CHECK-NEXT:    mov.w lr, #64
 ; CHECK-NEXT:    adr r2, .LCPI8_0
 ; CHECK-NEXT:    vldrw.u32 q0, [r2]
 ; CHECK-NEXT:  .LBB8_1: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    vldrh.u32 q1, [r0, #24]
-; CHECK-NEXT:    vcvtb.f32.f16 q1, q1
-; CHECK-NEXT:    vmul.f32 q1, q1, q0
-; CHECK-NEXT:    vcvtb.f16.f32 q1, q1
-; CHECK-NEXT:    vstrh.32 q1, [r1, #24]
-; CHECK-NEXT:    vldrh.u32 q1, [r0, #16]
-; CHECK-NEXT:    vcvtb.f32.f16 q1, q1
-; CHECK-NEXT:    vmul.f32 q1, q1, q0
-; CHECK-NEXT:    vcvtb.f16.f32 q1, q1
-; CHECK-NEXT:    vstrh.32 q1, [r1, #16]
-; CHECK-NEXT:    vldrh.u32 q1, [r0, #8]
-; CHECK-NEXT:    vcvtb.f32.f16 q1, q1
+; CHECK-NEXT:    vldrh.u16 q1, [r0, #16]
+; CHECK-NEXT:    vcvtb.f32.f16 q2, q1
+; CHECK-NEXT:    vcvtt.f32.f16 q1, q1
+; CHECK-NEXT:    vmul.f32 q2, q2, q0
 ; CHECK-NEXT:    vmul.f32 q1, q1, q0
-; CHECK-NEXT:    vcvtb.f16.f32 q1, q1
-; CHECK-NEXT:    vstrh.32 q1, [r1, #8]
-; CHECK-NEXT:    vldrh.u32 q1, [r0], #32
-; CHECK-NEXT:    vcvtb.f32.f16 q1, q1
+; CHECK-NEXT:    vcvtb.f16.f32 q2, q2
+; CHECK-NEXT:    vcvtt.f16.f32 q2, q1
+; CHECK-NEXT:    vldrh.u16 q1, [r0], #32
+; CHECK-NEXT:    vstrh.16 q2, [r1, #16]
+; CHECK-NEXT:    vcvtb.f32.f16 q2, q1
+; CHECK-NEXT:    vcvtt.f32.f16 q1, q1
+; CHECK-NEXT:    vmul.f32 q2, q2, q0
 ; CHECK-NEXT:    vmul.f32 q1, q1, q0
-; CHECK-NEXT:    vcvtb.f16.f32 q1, q1
-; CHECK-NEXT:    vstrh.32 q1, [r1], #32
+; CHECK-NEXT:    vcvtb.f16.f32 q2, q2
+; CHECK-NEXT:    vcvtt.f16.f32 q2, q1
+; CHECK-NEXT:    vstrh.16 q2, [r1], #32
 ; CHECK-NEXT:    le lr, .LBB8_1
 ; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
 ; CHECK-NEXT:    pop {r7, pc}
 ; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  @ %bb.3:
 ; CHECK-NEXT:  .LCPI8_0:
 ; CHECK-NEXT:    .long 0x40066666 @ float 2.0999999
 ; CHECK-NEXT:    .long 0x40066666 @ float 2.0999999