[RISCV] Implement prefetch locality by NTLH

[BeMg]

Summary

We add the MemOperand then backend will generate NTLH automatically.

__builtin_prefetch(ptr,  0 /* rw==read */, 0 /* locality */); => ntl.all + prefetch.r (ptr)
__builtin_prefetch(ptr,  0 /* rw==read */, 1 /* locality */); => ntl.pall + prefetch.r (ptr)
__builtin_prefetch(ptr,  0 /* rw==read */, 2 /* locality */); => ntl.p1 + prefetch.r (ptr)
__builtin_prefetch(ptr,  0 /* rw==read */, 3 /* locality */); => prefetch.r (ptr)

[BeMg]

rebase

[fakepaper56]

llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp

2126

Replace to Node->getConstantOperandVal(3)?

[BeMg]

Use getConstantOperandVal

[craig.topper]

LGTM

[asb]

I just mentioned this in https://github.com/riscv-non-isa/riscv-c-api-doc/pull/46, but given the NTL instructions are all just hints, perhaps we should be unconditionally emitting them even if zihintntl isn't known to be present (though printing it as add x0, x0, x? in .s form in that case). Not a blocker for landing this patch, but thought I'd mention.

llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp

case RISCVISD::VFMV_V_F_VL: {
     // Update the chain.
     ReplaceUses(Src.getValue(1), SDValue(Load, 1));
     // Record the mem-refs
     CurDAG->setNodeMemRefs(Load, {Ld->getMemOperand()});
     // Replace the splat with the vlse.
     ReplaceNode(Node, Load);
     return;
   }
+  case ISD::PREFETCH:
+    unsigned Locality =
+        cast<ConstantSDNode>(Node->getOperand(3))->getZExtValue();
+    if (Locality > 2)
+      break;
+
+    if (auto *LoadStoreMem = dyn_cast<MemSDNode>(Node)) {
+      MachineMemOperand *MMO = LoadStoreMem->getMemOperand();
+      MMO->setFlags(MachineMemOperand::MONonTemporal);
+
+      int NontemporalLevel = 0;
+      switch (Locality) {
+      case 0:
+        NontemporalLevel = 3; // NTL.ALL
+        break;
+      case 1:
+        NontemporalLevel = 1; // NTL.PALL
+        break;
+      case 2:
+        NontemporalLevel = 0; // NTL.P1
+        break;
+      default:
+        llvm_unreachable("unexpected locality value.");
+      }
+
+      if (NontemporalLevel & 0b1)
+        MMO->setFlags(MONontemporalBit0);
+      if (NontemporalLevel & 0b10)
+        MMO->setFlags(MONontemporalBit1);
+    }
+    break;
   }
 
   // Select the default instruction.
   SelectCode(Node);
 }
 
 bool RISCVDAGToDAGISel::SelectInlineAsmMemoryOperand(
     const SDValue &Op, unsigned ConstraintID, std::vector<SDValue> &OutOps) {

llvm/test/CodeGen/RISCV/prefetch.ll

 ;
 ; RV64ZICBOP-LABEL: test_prefetch_read_locality_0:
 ; RV64ZICBOP:       # %bb.0:
 ; RV64ZICBOP-NEXT:    prefetch.r 0(a0)
 ; RV64ZICBOP-NEXT:    ret
 ;
 ; RV64ZICBOPZIHINTNTL-LABEL: test_prefetch_read_locality_0:
 ; RV64ZICBOPZIHINTNTL:       # %bb.0:
+; RV64ZICBOPZIHINTNTL-NEXT:    ntl.all
 ; RV64ZICBOPZIHINTNTL-NEXT:    prefetch.r 0(a0)
 ; RV64ZICBOPZIHINTNTL-NEXT:    ret
   call void @llvm.prefetch(ptr %a, i32 0, i32 0, i32 1)
   ret void
 }
 
 define void @test_prefetch_write_locality_0(ptr %a) nounwind {
 ; RV32I-LABEL: test_prefetch_write_locality_0:

 ;
 ; RV64ZICBOP-LABEL: test_prefetch_write_locality_0:
 ; RV64ZICBOP:       # %bb.0:
 ; RV64ZICBOP-NEXT:    prefetch.w 0(a0)
 ; RV64ZICBOP-NEXT:    ret
 ;
 ; RV64ZICBOPZIHINTNTL-LABEL: test_prefetch_write_locality_0:
 ; RV64ZICBOPZIHINTNTL:       # %bb.0:
+; RV64ZICBOPZIHINTNTL-NEXT:    ntl.all
 ; RV64ZICBOPZIHINTNTL-NEXT:    prefetch.w 0(a0)
 ; RV64ZICBOPZIHINTNTL-NEXT:    ret
   call void @llvm.prefetch(ptr %a, i32 1, i32 0, i32 1)
   ret void
 }
 
 define void @test_prefetch_instruction_locality_0(ptr %a) nounwind {
 ; RV32I-LABEL: test_prefetch_instruction_locality_0:

 ;
 ; RV64ZICBOP-LABEL: test_prefetch_instruction_locality_0:
 ; RV64ZICBOP:       # %bb.0:
 ; RV64ZICBOP-NEXT:    prefetch.i 0(a0)
 ; RV64ZICBOP-NEXT:    ret
 ;
 ; RV64ZICBOPZIHINTNTL-LABEL: test_prefetch_instruction_locality_0:
 ; RV64ZICBOPZIHINTNTL:       # %bb.0:
+; RV64ZICBOPZIHINTNTL-NEXT:    ntl.all
 ; RV64ZICBOPZIHINTNTL-NEXT:    prefetch.i 0(a0)
 ; RV64ZICBOPZIHINTNTL-NEXT:    ret
   call void @llvm.prefetch(ptr %a, i32 0, i32 0, i32 0)
   ret void
 }
 
 define void @test_prefetch_read_locality_1(ptr %a) nounwind {
 ; RV32I-LABEL: test_prefetch_read_locality_1:

 ;
 ; RV64ZICBOP-LABEL: test_prefetch_read_locality_1:
 ; RV64ZICBOP:       # %bb.0:
 ; RV64ZICBOP-NEXT:    prefetch.r 0(a0)
 ; RV64ZICBOP-NEXT:    ret
 ;
 ; RV64ZICBOPZIHINTNTL-LABEL: test_prefetch_read_locality_1:
 ; RV64ZICBOPZIHINTNTL:       # %bb.0:
+; RV64ZICBOPZIHINTNTL-NEXT:    ntl.pall
 ; RV64ZICBOPZIHINTNTL-NEXT:    prefetch.r 0(a0)
 ; RV64ZICBOPZIHINTNTL-NEXT:    ret
   call void @llvm.prefetch(ptr %a, i32 0, i32 1, i32 1)
   ret void
 }
 
 define void @test_prefetch_write_locality_1(ptr %a) nounwind {
 ; RV32I-LABEL: test_prefetch_write_locality_1:

 ;
 ; RV64ZICBOP-LABEL: test_prefetch_write_locality_1:
 ; RV64ZICBOP:       # %bb.0:
 ; RV64ZICBOP-NEXT:    prefetch.w 0(a0)
 ; RV64ZICBOP-NEXT:    ret
 ;
 ; RV64ZICBOPZIHINTNTL-LABEL: test_prefetch_write_locality_1:
 ; RV64ZICBOPZIHINTNTL:       # %bb.0:
+; RV64ZICBOPZIHINTNTL-NEXT:    ntl.pall
 ; RV64ZICBOPZIHINTNTL-NEXT:    prefetch.w 0(a0)
 ; RV64ZICBOPZIHINTNTL-NEXT:    ret
   call void @llvm.prefetch(ptr %a, i32 1, i32 1, i32 1)
   ret void
 }
 
 define void @test_prefetch_instruction_locality_1(ptr %a) nounwind {
 ; RV32I-LABEL: test_prefetch_instruction_locality_1:

 ;
 ; RV64ZICBOP-LABEL: test_prefetch_instruction_locality_1:
 ; RV64ZICBOP:       # %bb.0:
 ; RV64ZICBOP-NEXT:    prefetch.i 0(a0)
 ; RV64ZICBOP-NEXT:    ret
 ;
 ; RV64ZICBOPZIHINTNTL-LABEL: test_prefetch_instruction_locality_1:
 ; RV64ZICBOPZIHINTNTL:       # %bb.0:
+; RV64ZICBOPZIHINTNTL-NEXT:    ntl.pall
 ; RV64ZICBOPZIHINTNTL-NEXT:    prefetch.i 0(a0)
 ; RV64ZICBOPZIHINTNTL-NEXT:    ret
   call void @llvm.prefetch(ptr %a, i32 0, i32 1, i32 0)
   ret void
 }
 
 define void @test_prefetch_read_locality_2(ptr %a) nounwind {
 ; RV32I-LABEL: test_prefetch_read_locality_2:

 ;
 ; RV64ZICBOP-LABEL: test_prefetch_read_locality_2:
 ; RV64ZICBOP:       # %bb.0:
 ; RV64ZICBOP-NEXT:    prefetch.r 0(a0)
 ; RV64ZICBOP-NEXT:    ret
 ;
 ; RV64ZICBOPZIHINTNTL-LABEL: test_prefetch_read_locality_2:
 ; RV64ZICBOPZIHINTNTL:       # %bb.0:
+; RV64ZICBOPZIHINTNTL-NEXT:    ntl.p1
 ; RV64ZICBOPZIHINTNTL-NEXT:    prefetch.r 0(a0)
 ; RV64ZICBOPZIHINTNTL-NEXT:    ret
   call void @llvm.prefetch(ptr %a, i32 0, i32 2, i32 1)
   ret void
 }
 
 define void @test_prefetch_write_locality_2(ptr %a) nounwind {
 ; RV32I-LABEL: test_prefetch_write_locality_2:

 ;
 ; RV64ZICBOP-LABEL: test_prefetch_write_locality_2:
 ; RV64ZICBOP:       # %bb.0:
 ; RV64ZICBOP-NEXT:    prefetch.w 0(a0)
 ; RV64ZICBOP-NEXT:    ret
 ;
 ; RV64ZICBOPZIHINTNTL-LABEL: test_prefetch_write_locality_2:
 ; RV64ZICBOPZIHINTNTL:       # %bb.0:
+; RV64ZICBOPZIHINTNTL-NEXT:    ntl.p1
 ; RV64ZICBOPZIHINTNTL-NEXT:    prefetch.w 0(a0)
 ; RV64ZICBOPZIHINTNTL-NEXT:    ret
   call void @llvm.prefetch(ptr %a, i32 1, i32 2, i32 1)
   ret void
 }
 
 define void @test_prefetch_instruction_locality_2(ptr %a) nounwind {
 ; RV32I-LABEL: test_prefetch_instruction_locality_2:

 ;
 ; RV64ZICBOP-LABEL: test_prefetch_instruction_locality_2:
 ; RV64ZICBOP:       # %bb.0:
 ; RV64ZICBOP-NEXT:    prefetch.i 0(a0)
 ; RV64ZICBOP-NEXT:    ret
 ;
 ; RV64ZICBOPZIHINTNTL-LABEL: test_prefetch_instruction_locality_2:
 ; RV64ZICBOPZIHINTNTL:       # %bb.0:
+; RV64ZICBOPZIHINTNTL-NEXT:    ntl.p1
 ; RV64ZICBOPZIHINTNTL-NEXT:    prefetch.i 0(a0)
 ; RV64ZICBOPZIHINTNTL-NEXT:    ret
   call void @llvm.prefetch(ptr %a, i32 0, i32 2, i32 0)
   ret void
 }
 
 
 define void @test_prefetch_read_locality_3(ptr %a) nounwind {

 ; RV64ZICBOP-NEXT:    ret
 ;
 ; RV64ZICBOPZIHINTNTL-LABEL: test_prefetch_instruction_locality_3:
 ; RV64ZICBOPZIHINTNTL:       # %bb.0:
 ; RV64ZICBOPZIHINTNTL-NEXT:    prefetch.i 0(a0)
 ; RV64ZICBOPZIHINTNTL-NEXT:    ret
   call void @llvm.prefetch(ptr %a, i32 0, i32 3, i32 0)
   ret void
-}
+}