[AArch64] Support SLC in ACLE prefetch intrinsics

[lenary]

Summary

This change:

• Modifies the ACLE code to allow the new SLC value (3) for the prefetch target.

• Introduces a new intrinsic, @llvm.aarch64.prefetch which matches the PRFM family instructions much more closely, and can represent all values for the PRFM immediate.
  
  The target-independent @llvm.prefetch intrinsic does not have enough information for us to be able to lower to it from the ACLE intrinsics correctly.

• Lowers the acle calls to the new intrinsic on aarch64 (the ARM lowering is unchanged).

• Implements code generation for the new intrinsic in both SelectionDAG and GlobalISel. We specifically choose to continue to support lowering the target-independent @llvm.prefetch intrinsic so that other frontends can continue to use it.

[lenary]

@sdesmalen adding you as I've moved around the custom lowering of INTRINSIC_VOID to be its own function, and as we need to handle it for @llvm.arm.prefetch all the time, it's no longer conditional on SME.

[tschuett]

llvm/test/CodeGen/AArch64/arm64-prefetch-new.ll

2

I believe you also have to turn on GISL: -global-isel.

See test in https://reviews.llvm.org/D109827.

[lenary]

llvm/test/CodeGen/AArch64/arm64-prefetch-new.ll

2

I'm not sure this is necessary, global isel is enabled at O0 on AArch64, and e.g. RegBankSelect appears in the output of llc -mtriple=aarch64 -mattr=+v8.9a -O0 < llvm/test/CodeGen/AArch64/arm64-prefetch-new.ll -debug-pass=Structure -o arm64-prefetch-new.ll

Would you find it clearer as:

; RUN: llc -mtriple=aarch64 -mattr=+v8.9a --global-isel=0 < %s | FileCheck %s
; RUN: llc -mtriple=aarch64 -mattr=+v8.9a --global-isel=1 --global-isel-abort=1 < %s | FileCheck %s

?

[tschuett]

llvm/test/CodeGen/AArch64/arm64-prefetch-new.ll

2

I didn't know or forgot. I believe --global-isel=1 makes the intent clearer.

[lenary]

Adding Jessica and Amara as this affects GlobalISel for AArch64.

[paquette]

GISel part LGTM

[pratlucas]

LGTM.

clang/lib/CodeGen/CGBuiltin.cpp

case clang::AArch64::BI__sevl:
     break;
   }
 
   if (HintID != static_cast<unsigned>(-1)) {
     Function *F = CGM.getIntrinsic(Intrinsic::aarch64_hint);
     return Builder.CreateCall(F, llvm::ConstantInt::get(Int32Ty, HintID));
   }
 
-  if (BuiltinID == clang::AArch64::BI__builtin_arm_prefetch) {
-    Value *Address         = EmitScalarExpr(E->getArg(0));
-    Value *RW              = EmitScalarExpr(E->getArg(1));
-    Value *CacheLevel      = EmitScalarExpr(E->getArg(2));
-    Value *RetentionPolicy = EmitScalarExpr(E->getArg(3));
-    Value *IsData          = EmitScalarExpr(E->getArg(4));
-
-    Value *Locality = nullptr;
-    if (cast<llvm::ConstantInt>(RetentionPolicy)->isZero()) {
-      // Temporal fetch, needs to convert cache level to locality.
-      Locality = llvm::ConstantInt::get(Int32Ty,
-        -cast<llvm::ConstantInt>(CacheLevel)->getValue() + 3);
-    } else {
-      // Streaming fetch.
-      Locality = llvm::ConstantInt::get(Int32Ty, 0);
-    }
-
-    // FIXME: We need AArch64 specific LLVM intrinsic if we want to specify
-    // PLDL3STRM or PLDL2STRM.
-    Function *F = CGM.getIntrinsic(Intrinsic::prefetch, Address->getType());
-    return Builder.CreateCall(F, {Address, RW, Locality, IsData});
-  }
-
   if (BuiltinID == clang::AArch64::BI__builtin_arm_rbit) {
     assert((getContext().getTypeSize(E->getType()) == 32) &&
            "rbit of unusual size!");
     llvm::Value *Arg = EmitScalarExpr(E->getArg(0));
     return Builder.CreateCall(
         CGM.getIntrinsic(Intrinsic::bitreverse, Arg->getType()), Arg, "rbit");
   }
   if (BuiltinID == clang::AArch64::BI__builtin_arm_rbit64) {

clang/lib/Sema/SemaChecking.cpp

if (BuiltinID == AArch64::BI__builtin_arm_ldrex ||
       BuiltinID == AArch64::BI__builtin_arm_ldaex ||
       BuiltinID == AArch64::BI__builtin_arm_strex ||
       BuiltinID == AArch64::BI__builtin_arm_stlex) {
     return CheckARMBuiltinExclusiveCall(BuiltinID, TheCall, 128);
   }
 
   if (BuiltinID == AArch64::BI__builtin_arm_prefetch) {
     return SemaBuiltinConstantArgRange(TheCall, 1, 0, 1) ||
-      SemaBuiltinConstantArgRange(TheCall, 2, 0, 2) ||
+           SemaBuiltinConstantArgRange(TheCall, 2, 0, 3) ||
            SemaBuiltinConstantArgRange(TheCall, 3, 0, 1) ||
            SemaBuiltinConstantArgRange(TheCall, 4, 0, 1);
   }
 
   if (BuiltinID == AArch64::BI__builtin_arm_rsr64 ||
       BuiltinID == AArch64::BI__builtin_arm_wsr64)
     return SemaBuiltinARMSpecialReg(BuiltinID, TheCall, 0, 5, true);
 
   // Memory Tagging Extensions (MTE) Intrinsics
   if (BuiltinID == AArch64::BI__builtin_arm_irg ||

clang/test/CodeGen/arm_acle.c

 // AArch64-NEXT:    ret void
 //
 void test_swp(uint32_t x, volatile void *p) {
   __swp(x, p);
 }
 
 /* 8.6 Memory prefetch intrinsics */
 /* 8.6.1 Data prefetch */
-// ARM-LABEL: @test_pld(
-// ARM-NEXT:  entry:
-// ARM-NEXT:    call void @llvm.prefetch.p0(ptr null, i32 0, i32 3, i32 1)
-// ARM-NEXT:    ret void
+// AArch32-LABEL: @test_pld(
+// AArch32-NEXT:  entry:
+// AArch32-NEXT:    call void @llvm.prefetch.p0(ptr null, i32 0, i32 3, i32 1)
+// AArch32-NEXT:    ret void
+//
+// AArch64-LABEL: @test_pld(
+// AArch64-NEXT:  entry:
+// AArch64-NEXT:    call void @llvm.aarch64.prefetch(ptr null, i32 0, i32 0, i32 0, i32 1)
+// AArch64-NEXT:    ret void
 //
 void test_pld() {
   __pld(0);
 }
 
 // AArch32-LABEL: @test_pldx(
 // AArch32-NEXT:  entry:
 // AArch32-NEXT:    call void @llvm.prefetch.p0(ptr null, i32 1, i32 3, i32 1)
 // AArch32-NEXT:    ret void
 //
 // AArch64-LABEL: @test_pldx(
 // AArch64-NEXT:  entry:
-// AArch64-NEXT:    call void @llvm.prefetch.p0(ptr null, i32 1, i32 1, i32 1)
+// AArch64-NEXT:    call void @llvm.aarch64.prefetch(ptr null, i32 1, i32 2, i32 0, i32 1)
 // AArch64-NEXT:    ret void
 //
 void test_pldx() {
   __pldx(1, 2, 0, 0);
 }
 
 /* 8.6.2 Instruction prefetch */
-// ARM-LABEL: @test_pli(
-// ARM-NEXT:  entry:
-// ARM-NEXT:    call void @llvm.prefetch.p0(ptr null, i32 0, i32 3, i32 0)
-// ARM-NEXT:    ret void
+// AArch32-LABEL: @test_pli(
+// AArch32-NEXT:  entry:
+// AArch32-NEXT:    call void @llvm.prefetch.p0(ptr null, i32 0, i32 3, i32 0)
+// AArch32-NEXT:    ret void
+//
+// AArch64-LABEL: @test_pli(
+// AArch64-NEXT:  entry:
+// AArch64-NEXT:    call void @llvm.aarch64.prefetch(ptr null, i32 0, i32 0, i32 0, i32 0)
+// AArch64-NEXT:    ret void
 //
 void test_pli() {
   __pli(0);
 }
 
 // AArch32-LABEL: @test_plix(
 // AArch32-NEXT:  entry:
 // AArch32-NEXT:    call void @llvm.prefetch.p0(ptr null, i32 0, i32 3, i32 0)
 // AArch32-NEXT:    ret void
 //
 // AArch64-LABEL: @test_plix(
 // AArch64-NEXT:  entry:
-// AArch64-NEXT:    call void @llvm.prefetch.p0(ptr null, i32 0, i32 1, i32 0)
+// AArch64-NEXT:    call void @llvm.aarch64.prefetch(ptr null, i32 0, i32 2, i32 0, i32 0)
 // AArch64-NEXT:    ret void
 //
 void test_plix() {
   __plix(2, 0, 0);
 }
 
 /* 8.7 NOP */
 // AArch32-LABEL: @test_nop(

clang/test/CodeGen/builtins-arm64.c

 void barriers(void) {
   __builtin_arm_dmb(1);  //CHECK: call {{.*}} @llvm.aarch64.dmb(i32 1)
   __builtin_arm_dsb(2);  //CHECK: call {{.*}} @llvm.aarch64.dsb(i32 2)
   __builtin_arm_isb(3);  //CHECK: call {{.*}} @llvm.aarch64.isb(i32 3)
 }
 
 void prefetch(void) {
   __builtin_arm_prefetch(0, 1, 2, 0, 1); // pstl3keep
-  // CHECK: call {{.*}} @llvm.prefetch.p0(ptr null, i32 1, i32 1, i32 1)
+  // CHECK: call {{.*}} @llvm.aarch64.prefetch(ptr null, i32 1, i32 2, i32 0, i32 1)
 
   __builtin_arm_prefetch(0, 0, 0, 1, 1); // pldl1keep
-  // CHECK: call {{.*}} @llvm.prefetch.p0(ptr null, i32 0, i32 0, i32 1)
+  // CHECK: call {{.*}} @llvm.aarch64.prefetch(ptr null, i32 0, i32 0, i32 1, i32 1)
 
   __builtin_arm_prefetch(0, 0, 0, 1, 1); // pldl1strm
-  // CHECK: call {{.*}} @llvm.prefetch.p0(ptr null, i32 0, i32 0, i32 1)
+  // CHECK: call {{.*}} @llvm.aarch64.prefetch(ptr null, i32 0, i32 0, i32 1, i32 1)
 
   __builtin_arm_prefetch(0, 0, 0, 0, 0); // plil1keep
-  // CHECK: call {{.*}} @llvm.prefetch.p0(ptr null, i32 0, i32 3, i32 0)
+  // CHECK: call {{.*}} @llvm.aarch64.prefetch(ptr null, i32 0, i32 0, i32 0, i32 0)
+
+  __builtin_arm_prefetch(0, 0, 3, 0, 1); // pldslckeep
+  // CHECK: call {{.*}} @llvm.aarch64.prefetch(ptr null, i32 0, i32 3, i32 0, i32 1)
 }
 
 __attribute__((target("v8.5a")))
 int32_t jcvt(double v) {
   //CHECK-LABEL: @jcvt(
   //CHECK: call i32 @llvm.aarch64.fjcvtzs
   return __builtin_arm_jcvt(v);
 }

clang/test/Sema/builtins-arm64.c

 void test_memory_barriers(void) {
   __builtin_arm_dmb(16); // expected-error-re {{argument value {{.*}} is outside the valid range}}
   __builtin_arm_dsb(17); // expected-error-re {{argument value {{.*}} is outside the valid range}}
   __builtin_arm_isb(18); // expected-error-re {{argument value {{.*}} is outside the valid range}}
 }
 
 void test_prefetch(void) {
   __builtin_arm_prefetch(0, 2, 0, 0, 0); // expected-error-re {{argument value {{.*}} is outside the valid range}}
-  __builtin_arm_prefetch(0, 0, 3, 0, 0); // expected-error-re {{argument value {{.*}} is outside the valid range}}
+  __builtin_arm_prefetch(0, 0, 4, 0, 0); // expected-error-re {{argument value {{.*}} is outside the valid range}}
   __builtin_arm_prefetch(0, 0, 0, 2, 0); // expected-error-re {{argument value {{.*}} is outside the valid range}}
   __builtin_arm_prefetch(0, 0, 0, 0, 2); // expected-error-re {{argument value {{.*}} is outside the valid range}}
 }

llvm/include/llvm/IR/IntrinsicsAArch64.td

 //===----------------------------------------------------------------------===//
 // HINT
 
 def int_aarch64_hint : DefaultAttrsIntrinsic<[], [llvm_i32_ty]>;
 
 def int_aarch64_break : Intrinsic<[], [llvm_i32_ty],
     [IntrNoMem, IntrHasSideEffects, IntrNoReturn, IntrCold, ImmArg<ArgIndex<0>>]>;
 
+
+def int_aarch64_prefetch : Intrinsic<[],
+    [llvm_ptr_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty],
+    [IntrInaccessibleMemOrArgMemOnly, IntrWillReturn, ReadOnly<ArgIndex<0>>,
+     ImmArg<ArgIndex<1>>, ImmArg<ArgIndex<2>>, ImmArg<ArgIndex<3>>, ImmArg<ArgIndex<4>>
+     ]>,
+    ClangBuiltin<"__builtin_arm_prefetch">;
+
 //===----------------------------------------------------------------------===//
 // Data Barrier Instructions
 
 def int_aarch64_dmb : ClangBuiltin<"__builtin_arm_dmb">, MSBuiltin<"__dmb">,
                       Intrinsic<[], [llvm_i32_ty], [IntrNoFree, IntrWillReturn]>;
 def int_aarch64_dsb : ClangBuiltin<"__builtin_arm_dsb">, MSBuiltin<"__dsb">,
                       Intrinsic<[], [llvm_i32_ty], [IntrNoFree, IntrWillReturn]>;
 def int_aarch64_isb : ClangBuiltin<"__builtin_arm_isb">, MSBuiltin<"__isb">,

llvm/lib/IR/Verifier.cpp

#include "llvm/IR/ConstrainedOps.def"
   case Intrinsic::arm_stlex:
   case Intrinsic::arm_strex: {
     Type *ElemTy = Call.getAttributes().getParamElementType(1);
     Check(ElemTy,
           "Intrinsic requires elementtype attribute on second argument.",
           &Call);
     break;
   }
+  case Intrinsic::aarch64_prefetch: {
+    Check(cast<ConstantInt>(Call.getArgOperand(1))->getZExtValue() < 2,
+          "write argument to llvm.aarch64.prefetch must be 0 or 1", Call);
+    Check(cast<ConstantInt>(Call.getArgOperand(2))->getZExtValue() < 4,
+          "target argument to llvm.aarch64.prefetch must be 0-3", Call);
+    Check(cast<ConstantInt>(Call.getArgOperand(3))->getZExtValue() < 2,
+          "stream argument to llvm.aarch64.prefetch must be 0 or 1", Call);
+    Check(cast<ConstantInt>(Call.getArgOperand(4))->getZExtValue() < 2,
+          "isdata argument to llvm.aarch64.prefetch must be 0 or 1", Call);
+    break;
+  }
   };
 }
 
 /// Carefully grab the subprogram from a local scope.
 ///
 /// This carefully grabs the subprogram from a local scope, avoiding the
 /// built-in assertions that would typically fire.
 static DISubprogram *getSubprogram(Metadata *LocalScope) {

llvm/lib/Target/AArch64/AArch64ISelLowering.h

private:
 
   SDValue LowerMGATHER(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerMSCATTER(SDValue Op, SelectionDAG &DAG) const;
 
   SDValue LowerMLOAD(SDValue Op, SelectionDAG &DAG) const;
 
   SDValue LowerINTRINSIC_W_CHAIN(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG) const;
+  SDValue LowerINTRINSIC_VOID(SDValue Op, SelectionDAG &DAG) const;
 
   bool
   isEligibleForTailCallOptimization(const CallLoweringInfo &CLI) const;
 
   /// Finds the incoming stack arguments which overlap the given fixed stack
   /// object and incorporates their load into the current chain. This prevents
   /// an upcoming store from clobbering the stack argument before it's used.
   SDValue addTokenForArgument(SDValue Chain, SelectionDAG &DAG,

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

#undef LCALLNAME5
   if (Subtarget->hasSVEorSME()) {
     for (auto VT :
          {MVT::nxv16i1, MVT::nxv8i1, MVT::nxv4i1, MVT::nxv2i1, MVT::nxv1i1}) {
       setOperationAction(ISD::SPLAT_VECTOR, VT, Custom);
       setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
     }
   }
 
-  if (Subtarget->hasSME())
-    setOperationAction(ISD::INTRINSIC_VOID, MVT::Other, Custom);
-
   if (Subtarget->hasSVE()) {
     for (auto VT : {MVT::nxv16i8, MVT::nxv8i16, MVT::nxv4i32, MVT::nxv2i64}) {
       setOperationAction(ISD::BITREVERSE, VT, Custom);
       setOperationAction(ISD::BSWAP, VT, Custom);
       setOperationAction(ISD::CTLZ, VT, Custom);
       setOperationAction(ISD::CTPOP, VT, Custom);
       setOperationAction(ISD::CTTZ, VT, Custom);
       setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom);
if (Subtarget->hasSVE()) {
     setOperationAction(ISD::VSCALE, MVT::i32, Custom);
   }
 
   if (Subtarget->hasMOPS() && Subtarget->hasMTE()) {
     // Only required for llvm.aarch64.mops.memset.tag
     setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::i8, Custom);
   }
 
+  setOperationAction(ISD::INTRINSIC_VOID, MVT::Other, Custom);
+
   PredictableSelectIsExpensive = Subtarget->predictableSelectIsExpensive();
 
   IsStrictFPEnabled = true;
 }
 
 void AArch64TargetLowering::addTypeForNEON(MVT VT) {
   assert(VT.isVector() && "VT should be a vector type");
 
if (auto *ES = dyn_cast<ExternalSymbolSDNode>(V)) {
     if (S == "__arm_sme_state" || S == "__arm_tpidr2_save")
       return SMEAttrs(SMEAttrs::SM_Compatible | SMEAttrs::ZA_Preserved);
     if (S == "__arm_tpidr2_restore")
       return SMEAttrs(SMEAttrs::SM_Compatible | SMEAttrs::ZA_Shared);
   }
   return std::nullopt;
 }
 
+SDValue AArch64TargetLowering::LowerINTRINSIC_VOID(SDValue Op,
+                                                   SelectionDAG &DAG) const {
+  unsigned IntNo = Op.getConstantOperandVal(1);
+  SDLoc DL(Op);
+  switch (IntNo) {
+  default:
+    return SDValue(); // Don't custom lower most intrinsics.
+  case Intrinsic::aarch64_prefetch: {
+    SDValue Chain = Op.getOperand(0);
+    SDValue Addr = Op.getOperand(2);
+
+    unsigned IsWrite = cast<ConstantSDNode>(Op.getOperand(3))->getZExtValue();
+    unsigned Locality = cast<ConstantSDNode>(Op.getOperand(4))->getZExtValue();
+    unsigned IsStream = cast<ConstantSDNode>(Op.getOperand(5))->getZExtValue();
+    unsigned IsData = cast<ConstantSDNode>(Op.getOperand(6))->getZExtValue();
+    unsigned PrfOp = (IsWrite << 4) |    // Load/Store bit
+                     (!IsData << 3) |    // IsDataCache bit
+                     (Locality << 1) |   // Cache level bits
+                     (unsigned)IsStream; // Stream bit
+
+    return DAG.getNode(AArch64ISD::PREFETCH, DL, MVT::Other, Chain,
+                       DAG.getTargetConstant(PrfOp, DL, MVT::i32), Addr);
+  }
+  case Intrinsic::aarch64_sme_za_enable:
+    return DAG.getNode(
+        AArch64ISD::SMSTART, DL, MVT::Other,
+        Op->getOperand(0), // Chain
+        DAG.getTargetConstant((int32_t)(AArch64SVCR::SVCRZA), DL, MVT::i32),
+        DAG.getConstant(0, DL, MVT::i64), DAG.getConstant(1, DL, MVT::i64));
+  case Intrinsic::aarch64_sme_za_disable:
+    return DAG.getNode(
+        AArch64ISD::SMSTOP, DL, MVT::Other,
+        Op->getOperand(0), // Chain
+        DAG.getTargetConstant((int32_t)(AArch64SVCR::SVCRZA), DL, MVT::i32),
+        DAG.getConstant(0, DL, MVT::i64), DAG.getConstant(1, DL, MVT::i64));
+  }
+}
+
 SDValue AArch64TargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op,
                                                       SelectionDAG &DAG) const {
   unsigned IntNo = Op.getConstantOperandVal(1);
   SDLoc DL(Op);
   switch (IntNo) {
   default:
     return SDValue(); // Don't custom lower most intrinsics.
   case Intrinsic::aarch64_mops_memset_tag: {
SDValue MS =
                      Size, Alignment, IsVol, DstPtrInfo, MachinePointerInfo{});
 
     // MOPS_MEMSET_TAGGING has 3 results (DstWb, SizeWb, Chain) whereas the
     // intrinsic has 2. So hide SizeWb using MERGE_VALUES. Otherwise
     // LowerOperationWrapper will complain that the number of results has
     // changed.
     return DAG.getMergeValues({MS.getValue(0), MS.getValue(2)}, DL);
   }
-  case Intrinsic::aarch64_sme_za_enable:
-    return DAG.getNode(
-        AArch64ISD::SMSTART, DL, MVT::Other,
-        Op->getOperand(0), // Chain
-        DAG.getTargetConstant((int32_t)(AArch64SVCR::SVCRZA), DL, MVT::i32),
-        DAG.getConstant(0, DL, MVT::i64), DAG.getConstant(1, DL, MVT::i64));
-  case Intrinsic::aarch64_sme_za_disable:
-    return DAG.getNode(
-        AArch64ISD::SMSTOP, DL, MVT::Other,
-        Op->getOperand(0), // Chain
-        DAG.getTargetConstant((int32_t)(AArch64SVCR::SVCRZA), DL, MVT::i32),
-        DAG.getConstant(0, DL, MVT::i64), DAG.getConstant(1, DL, MVT::i64));
   }
 }
 
 SDValue AArch64TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
                                                      SelectionDAG &DAG) const {
   unsigned IntNo = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
   SDLoc dl(Op);
   switch (IntNo) {
SDValue AArch64TargetLowering::LowerOperation(SDValue Op,
   case ISD::SET_ROUNDING:
     return LowerSET_ROUNDING(Op, DAG);
   case ISD::MUL:
     return LowerMUL(Op, DAG);
   case ISD::MULHS:
     return LowerToPredicatedOp(Op, DAG, AArch64ISD::MULHS_PRED);
   case ISD::MULHU:
     return LowerToPredicatedOp(Op, DAG, AArch64ISD::MULHU_PRED);
-  case ISD::INTRINSIC_VOID:
   case ISD::INTRINSIC_W_CHAIN:
     return LowerINTRINSIC_W_CHAIN(Op, DAG);
   case ISD::INTRINSIC_WO_CHAIN:
     return LowerINTRINSIC_WO_CHAIN(Op, DAG);
+  case ISD::INTRINSIC_VOID:
+    return LowerINTRINSIC_VOID(Op, DAG);
   case ISD::ATOMIC_STORE:
     if (cast<MemSDNode>(Op)->getMemoryVT() == MVT::i128) {
       assert(Subtarget->hasLSE2());
       return LowerStore128(Op, DAG);
     }
     return SDValue();
   case ISD::STORE:
     return LowerSTORE(Op, DAG);

llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp

case Intrinsic::prefetch: {
 
     unsigned PrfOp =
         (IsWrite << 4) | (!IsData << 3) | (Locality << 1) | IsStream;
 
     MIB.buildInstr(AArch64::G_PREFETCH).addImm(PrfOp).add(AddrVal);
     MI.eraseFromParent();
     return true;
   }
+  case Intrinsic::aarch64_prefetch: {
+    MachineIRBuilder MIB(MI);
+    auto &AddrVal = MI.getOperand(1);
+
+    int64_t IsWrite = MI.getOperand(2).getImm();
+    int64_t Target = MI.getOperand(3).getImm();
+    int64_t IsStream = MI.getOperand(4).getImm();
+    int64_t IsData = MI.getOperand(5).getImm();
+
+    unsigned PrfOp = (IsWrite << 4) |    // Load/Store bit
+                     (!IsData << 3) |    // IsDataCache bit
+                     (Target << 1) |     // Cache level bits
+                     (unsigned)IsStream; // Stream bit
+
+    MIB.buildInstr(AArch64::G_PREFETCH).addImm(PrfOp).add(AddrVal);
+    MI.eraseFromParent();
+    return true;
+  }
   }
 
   return true;
 }
 
 bool AArch64LegalizerInfo::legalizeShlAshrLshr(
     MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &MIRBuilder,
     GISelChangeObserver &Observer) const {

llvm/test/CodeGen/AArch64/arm64-prefetch-new.ll

+; RUN: llc -mtriple=aarch64 -mattr=+v8.9a < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64 -mattr=+v8.9a -O0 --global-isel-abort=1 < %s | FileCheck %s
+
+@a = internal global ptr null, align 8
+@b = external global ptr, align 8
+
+define void @test(ptr %i, i32 %j) nounwind ssp {
+entry:
+  ; CHECK-LABEL: @test
+  %j.addr = alloca i32, align 4
+  store i32 %j, ptr %j.addr, align 4, !tbaa !0
+  %tmp = bitcast ptr %j.addr to ptr
+
+  %i.next = getelementptr i8, ptr %i, i64 2
+
+  ; Verify prefetching works for all the different kinds of pointers we might
+  ; want to prefetch.
+
+  ; CHECK: prfm pldl1keep,
+  call void @llvm.aarch64.prefetch(ptr null, i32 0, i32 0, i32 0, i32 1)
+
+  ; CHECK: prfum pldl1keep,
+  call void @llvm.aarch64.prefetch(ptr %tmp, i32 0, i32 0, i32 0, i32 1)
+
+  ; CHECK: prfm pldl1keep,
+  call void @llvm.aarch64.prefetch(ptr %i, i32 0, i32 0, i32 0, i32 1)
+
+  ; CHECK: prfum pldl1keep,
+  call void @llvm.aarch64.prefetch(ptr %i.next, i32 0, i32 0, i32 0, i32 1)
+
+  ; CHECK: prfm pldl1keep,
+  call void @llvm.aarch64.prefetch(ptr @a, i32 0, i32 0, i32 0, i32 1)
+
+  ; CHECK: prfm pldl1keep,
+  call void @llvm.aarch64.prefetch(ptr @b, i32 0, i32 0, i32 0, i32 1)
+
+  ; Verify that we can generate every single valid prefetch value.
+
+  ; CHECK: prfm pstl1keep,
+  call void @llvm.aarch64.prefetch(ptr null, i32 1, i32 0, i32 0, i32 1)
+
+  ; CHECK: prfm pldl2keep,
+  call void @llvm.aarch64.prefetch(ptr null, i32 0, i32 1, i32 0, i32 1)
+
+  ; CHECK: prfm pldl3keep,
+  call void @llvm.aarch64.prefetch(ptr null, i32 0, i32 2, i32 0, i32 1)
+
+  ; CHECK: prfm pldslckeep,
+  call void @llvm.aarch64.prefetch(ptr null, i32 0, i32 3, i32 0, i32 1)
+
+  ; CHECK: prfm pldl1strm,
+  call void @llvm.aarch64.prefetch(ptr null, i32 0, i32 0, i32 1, i32 1)
+
+  ; CHECK: prfm plil1keep,
+  call void @llvm.aarch64.prefetch(ptr null, i32 0, i32 0, i32 0, i32 0)
+
+  ret void
+}
+
+declare void @llvm.aarch64.prefetch(ptr readonly, i32 immarg, i32 immarg, i32 immarg, i32 immarg) #0
+
+attributes #0 = { inaccessiblemem_or_argmemonly nounwind willreturn }
+
+!0 = !{!"int", !1}
+!1 = !{!"omnipotent char", !2}
+!2 = !{!"Simple C/C++ TBAA"}
+!3 = !{!"any pointer", !1}