[AArch64][GlobalISel] Emit bzero on Darwin

[paquette]

Summary

Darwin platforms for both AArch64 and X86 can provide optimized bzero() routines. In this case, it may be preferable to use bzero in place of a memset of 0.

This adds a G_BZERO generic opcode, similar to G_MEMSET et al. This opcode can be generated by platforms which may want to use bzero.

To emit the G_BZERO, this adds a pre-legalize combine for AArch64. The conditions for this are largely a port of the bzero case in AArch64SelectionDAGInfo::EmitTargetCodeForMemset.

The only difference in comparison to the SelectionDAG code is that, when compiling for minsize, this will fire for all memsets of 0. The original code notes that it's not beneficial to do this for small memsets; however, using bzero here will save a mov from wzr. For minsize, I think that it's preferable to prioritise omitting the mov.

This also fixes a bug in the libcall legalization code which would delete instructions which could not be legalized. It also adds a check to make sure that we actually get a libcall name.

Code size improvements (Darwin):

• CTMark -Os: -0.0% geomean (-0.1% on pairlocalalign)
• CTMark -Oz: -0.2% geomean (-0.5% on bullet)

[aemerson]

We should also have MachineVerifier checks and tests, like we do for G_MEMSET.

[paquette]

Add verifier tests (and actually copy the memoperand from the memset)

[aemerson]

LGTM.

llvm/include/llvm/Support/TargetOpcodes.def

 /// llvm.memcpy intrinsic
 HANDLE_TARGET_OPCODE(G_MEMCPY)
 
 /// llvm.memmove intrinsic
 HANDLE_TARGET_OPCODE(G_MEMMOVE)
 
 /// llvm.memset intrinsic
 HANDLE_TARGET_OPCODE(G_MEMSET)
+HANDLE_TARGET_OPCODE(G_BZERO)
 
 /// Vector reductions
 HANDLE_TARGET_OPCODE(G_VECREDUCE_SEQ_FADD)
 HANDLE_TARGET_OPCODE(G_VECREDUCE_SEQ_FMUL)
 HANDLE_TARGET_OPCODE(G_VECREDUCE_FADD)
 HANDLE_TARGET_OPCODE(G_VECREDUCE_FMUL)
 HANDLE_TARGET_OPCODE(G_VECREDUCE_FMAX)
 HANDLE_TARGET_OPCODE(G_VECREDUCE_FMIN)

llvm/include/llvm/Target/GenericOpcodes.td

 
 def G_MEMSET : GenericInstruction {
   let OutOperandList = (outs);
   let InOperandList = (ins ptype0:$dst_addr, type1:$value, type2:$size, untyped_imm_0:$tailcall);
   let hasSideEffects = false;
   let mayStore = true;
 }
 
+def G_BZERO : GenericInstruction {
+  let OutOperandList = (outs);
+  let InOperandList = (ins ptype0:$dst_addr, type1:$size, untyped_imm_0:$tailcall);
+  let hasSideEffects = false;
+  let mayStore = true;
+}
+
 //------------------------------------------------------------------------------
 // Bitfield extraction.
 //------------------------------------------------------------------------------
 
 // Generic signed bitfield extraction.
 def G_SBFX : GenericInstruction {
   let OutOperandList = (outs type0:$dst);
   let InOperandList = (ins type0:$src, type0:$lsb, type0:$width);

llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp

for (unsigned i = 0; i < MI.getNumOperands() - 1; ++i) {
     else
       OpTy = IntegerType::get(Ctx, OpLLT.getSizeInBits());
     Args.push_back({Reg, OpTy});
   }
 
   auto &CLI = *MIRBuilder.getMF().getSubtarget().getCallLowering();
   auto &TLI = *MIRBuilder.getMF().getSubtarget().getTargetLowering();
   RTLIB::Libcall RTLibcall;
-  switch (MI.getOpcode()) {
+  unsigned Opc = MI.getOpcode();
+  switch (Opc) {
+  case TargetOpcode::G_BZERO:
+    RTLibcall = RTLIB::BZERO;
+    break;
   case TargetOpcode::G_MEMCPY:
     RTLibcall = RTLIB::MEMCPY;
     break;
   case TargetOpcode::G_MEMMOVE:
     RTLibcall = RTLIB::MEMMOVE;
     break;
   case TargetOpcode::G_MEMSET:
     RTLibcall = RTLIB::MEMSET;
     break;
   default:
     return LegalizerHelper::UnableToLegalize;
   }
   const char *Name = TLI.getLibcallName(RTLibcall);
 
+  // Unsupported libcall on the target.
+  if (!Name) {
+    LLVM_DEBUG(dbgs() << ".. .. Could not find libcall name for "
+                      << MIRBuilder.getTII().getName(Opc) << "\n");
+    return LegalizerHelper::UnableToLegalize;
+  }
+
   CallLowering::CallLoweringInfo Info;
   Info.CallConv = TLI.getLibcallCallingConv(RTLibcall);
   Info.Callee = MachineOperand::CreateES(Name);
   Info.OrigRet = CallLowering::ArgInfo({0}, Type::getVoidTy(Ctx));
   Info.IsTailCall = MI.getOperand(MI.getNumOperands() - 1).getImm() &&
                     isLibCallInTailPosition(MIRBuilder.getTII(), MI);
 
   std::copy(Args.begin(), Args.end(), std::back_inserter(Info.OrigArgs));
case TargetOpcode::G_UITOFP: {
     LegalizeResult Status = conversionLibcall(
         MI, MIRBuilder,
         ToSize == 64 ? Type::getDoubleTy(Ctx) : Type::getFloatTy(Ctx),
         FromSize == 32 ? Type::getInt32Ty(Ctx) : Type::getInt64Ty(Ctx));
     if (Status != Legalized)
       return Status;
     break;
   }
+  case TargetOpcode::G_BZERO:
   case TargetOpcode::G_MEMCPY:
   case TargetOpcode::G_MEMMOVE:
   case TargetOpcode::G_MEMSET: {
-    LegalizeResult Result = createMemLibcall(MIRBuilder, *MIRBuilder.getMRI(), MI);
+    LegalizeResult Result =
+        createMemLibcall(MIRBuilder, *MIRBuilder.getMRI(), MI);
+    if (Result != Legalized)
+      return Result;
     MI.eraseFromParent();
     return Result;
   }
   }
 
   MI.eraseFromParent();
   return Legalized;
 }

llvm/lib/CodeGen/MachineVerifier.cpp

case TargetOpcode::G_MEMMOVE: {
 
     if (DstPtrTy.getAddressSpace() != MMOs[0]->getAddrSpace())
       report("inconsistent store address space", MI);
     if (SrcPtrTy.getAddressSpace() != MMOs[1]->getAddrSpace())
       report("inconsistent load address space", MI);
 
     break;
   }
+  case TargetOpcode::G_BZERO:
   case TargetOpcode::G_MEMSET: {
     ArrayRef<MachineMemOperand *> MMOs = MI->memoperands();
+    std::string Name = Opc == TargetOpcode::G_MEMSET ? "memset" : "bzero";
     if (MMOs.size() != 1) {
-      report("memset must have 1 memory operand", MI);
+      report(Twine(Name, " must have 1 memory operand"), MI);
       break;
     }
 
     if ((!MMOs[0]->isStore() || MMOs[0]->isLoad())) {
-      report("memset memory operand must be a store", MI);
+      report(Twine(Name, " memory operand must be a store"), MI);
       break;
     }
 
     LLT DstPtrTy = MRI->getType(MI->getOperand(0).getReg());
     if (!DstPtrTy.isPointer()) {
-      report("memset operand must be a pointer", MI);
+      report(Twine(Name, " operand must be a pointer"), MI);
       break;
     }
 
     if (DstPtrTy.getAddressSpace() != MMOs[0]->getAddrSpace())
-      report("inconsistent memset address space", MI);
+      report("inconsistent " + Twine(Name, " address space"), MI);
 
     break;
   }
   case TargetOpcode::G_VECREDUCE_SEQ_FADD:
   case TargetOpcode::G_VECREDUCE_SEQ_FMUL: {
     LLT DstTy = MRI->getType(MI->getOperand(0).getReg());
     LLT Src1Ty = MRI->getType(MI->getOperand(1).getReg());
     LLT Src2Ty = MRI->getType(MI->getOperand(2).getReg());

llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp

AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST)
   getActionDefinitionsBuilder(G_JUMP_TABLE).legalFor({{p0}, {s64}});
 
   getActionDefinitionsBuilder(G_BRJT).legalIf([=](const LegalityQuery &Query) {
     return Query.Types[0] == p0 && Query.Types[1] == s64;
   });
 
   getActionDefinitionsBuilder(G_DYN_STACKALLOC).lower();
 
-  getActionDefinitionsBuilder({G_MEMCPY, G_MEMMOVE, G_MEMSET}).libcall();
+  getActionDefinitionsBuilder({G_BZERO, G_MEMCPY, G_MEMMOVE, G_MEMSET})
+      .libcall();
 
   getActionDefinitionsBuilder(G_ABS).lowerIf(
       [=](const LegalityQuery &Query) { return Query.Types[0].isScalar(); });
 
   getActionDefinitionsBuilder(G_VECREDUCE_FADD)
       // We only have FADDP to do reduction-like operations. Lower the rest.
       .legalFor({{s32, v2s32}, {s64, v2s64}})
       .lower();

llvm/lib/Target/AArch64/GISel/AArch64PreLegalizerCombiner.cpp

static bool applyFoldGlobalOffset(MachineInstr &MI, MachineRegisterInfo &MRI,
   MI.getOperand(0).setReg(NewGVDst);
   Observer.changedInstr(MI);
   B.buildPtrAdd(
       Dst, NewGVDst,
       B.buildConstant(LLT::scalar(64), -static_cast<int64_t>(MinOffset)));
   return true;
 }
 
+/// Replace a G_MEMSET with a value of 0 with a G_BZERO instruction if it is
+/// supported and beneficial to do so.
+///
+/// \note This only applies on Darwin.
+///
+/// \returns true if \p MI was replaced with a G_BZERO.
+static bool tryEmitBZero(MachineInstr &MI, MachineIRBuilder &MIRBuilder,
+                         bool MinSize) {
+  assert(MI.getOpcode() == TargetOpcode::G_MEMSET);
+  MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
+  auto &TLI = *MIRBuilder.getMF().getSubtarget().getTargetLowering();
+  if (!TLI.getLibcallName(RTLIB::BZERO))
+    return false;
+  auto Zero = getConstantVRegValWithLookThrough(MI.getOperand(1).getReg(), MRI);
+  if (!Zero || Zero->Value.getSExtValue() != 0)
+    return false;
+
+  // It's not faster to use bzero rather than memset for sizes <= 256.
+  // However, it *does* save us a mov from wzr, so if we're going for
+  // minsize, use bzero even if it's slower.
+  if (!MinSize) {
+    // If the size is known, check it. If it is not known, assume using bzero is
+    // better.
+    if (auto Size =
+            getConstantVRegValWithLookThrough(MI.getOperand(2).getReg(), MRI)) {
+      if (Size->Value.getSExtValue() <= 256)
+        return false;
+    }
+  }
+
+  MIRBuilder.setInstrAndDebugLoc(MI);
+  MIRBuilder
+      .buildInstr(TargetOpcode::G_BZERO, {},
+                  {MI.getOperand(0), MI.getOperand(2)})
+      .addImm(MI.getOperand(3).getImm())
+      .addMemOperand(*MI.memoperands_begin());
+  MI.eraseFromParent();
+  return true;
+}
+
 class AArch64PreLegalizerCombinerHelperState {
 protected:
   CombinerHelper &Helper;
 
 public:
   AArch64PreLegalizerCombinerHelperState(CombinerHelper &Helper)
       : Helper(Helper) {}
 };
bool AArch64PreLegalizerCombinerInfo::combine(GISelChangeObserver &Observer,
                                               MachineInstr &MI,
                                               MachineIRBuilder &B) const {
   CombinerHelper Helper(Observer, B, KB, MDT);
   AArch64GenPreLegalizerCombinerHelper Generated(GeneratedRuleCfg, Helper);
 
   if (Generated.tryCombineAll(Observer, MI, B))
     return true;
 
-  switch (MI.getOpcode()) {
+  unsigned Opc = MI.getOpcode();
+  switch (Opc) {
   case TargetOpcode::G_CONCAT_VECTORS:
     return Helper.tryCombineConcatVectors(MI);
   case TargetOpcode::G_SHUFFLE_VECTOR:
     return Helper.tryCombineShuffleVector(MI);
   case TargetOpcode::G_MEMCPY:
   case TargetOpcode::G_MEMMOVE:
   case TargetOpcode::G_MEMSET: {
     // If we're at -O0 set a maxlen of 32 to inline, otherwise let the other
     // heuristics decide.
     unsigned MaxLen = EnableOpt ? 0 : 32;
     // Try to inline memcpy type calls if optimizations are enabled.
-    return !EnableMinSize ? Helper.tryCombineMemCpyFamily(MI, MaxLen) : false;
+    if (!EnableMinSize && Helper.tryCombineMemCpyFamily(MI, MaxLen))
+      return true;
+    if (Opc == TargetOpcode::G_MEMSET)
+      return tryEmitBZero(MI, B, EnableMinSize);
+    return false;
   }
   }
 
   return false;
 }
 
 #define AARCH64PRELEGALIZERCOMBINERHELPER_GENCOMBINERHELPER_CPP
 #include "AArch64GenPreLegalizeGICombiner.inc"

llvm/test/CodeGen/AArch64/GlobalISel/legalize-bzero-unsupported.mir

+# RUN: not llc -mtriple=aarch64 -global-isel-abort=1 -run-pass=legalizer -verify-machineinstrs %s -o /dev/null 2>&1 | FileCheck %s
+# RUN: not llc -mtriple=aarch64-linux-gnu -global-isel-abort=1 -run-pass=legalizer -verify-machineinstrs %s -o /dev/null 2>&1 | FileCheck %s
+...
+---
+name:            bzero
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    # CHECK: LLVM ERROR: unable to legalize instruction: G_BZERO
+    liveins: $x0, $x1
+    %ptr:_(p0) = COPY $x0
+    %width:_(s64) = COPY $x1
+    G_BZERO %ptr(p0), %width(s64), 0 :: (store 4)
+    RET_ReallyLR

llvm/test/CodeGen/AArch64/GlobalISel/legalize-bzero.mir

+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=aarch64-apple-ios -run-pass=legalizer -verify-machineinstrs %s -o - | FileCheck %s
+
+# Check that we can legalize G_BZERO on Darwin.
+
+...
+---
+name:            bzero
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $x0, $x1
+    ; CHECK-LABEL: name: bzero
+    ; CHECK: liveins: $x0, $x1
+    ; CHECK: %ptr:_(p0) = COPY $x0
+    ; CHECK: %width:_(s64) = COPY $x1
+    ; CHECK: ADJCALLSTACKDOWN 0, 0, implicit-def $sp, implicit $sp
+    ; CHECK: $x0 = COPY %ptr(p0)
+    ; CHECK: $x1 = COPY %width(s64)
+    ; CHECK: BL &bzero, csr_darwin_aarch64_aapcs, implicit-def $lr, implicit $sp, implicit $x0, implicit $x1
+    ; CHECK: ADJCALLSTACKUP 0, 0, implicit-def $sp, implicit $sp
+    ; CHECK: RET_ReallyLR
+    %ptr:_(p0) = COPY $x0
+    %width:_(s64) = COPY $x1
+    G_BZERO %ptr(p0), %width(s64), 0 :: (store 4)
+    RET_ReallyLR
+
+...
+---
+name:            bzero_tail_call
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $x0, $x1
+    ; CHECK-LABEL: name: bzero_tail_call
+    ; CHECK: liveins: $x0, $x1
+    ; CHECK: %ptr:_(p0) = COPY $x0
+    ; CHECK: %width:_(s64) = COPY $x1
+    ; CHECK: $x0 = COPY %ptr(p0)
+    ; CHECK: $x1 = COPY %width(s64)
+    ; CHECK: TCRETURNdi &bzero, 0, csr_darwin_aarch64_aapcs, implicit $sp, implicit $x0, implicit $x1
+    %ptr:_(p0) = COPY $x0
+    %width:_(s64) = COPY $x1
+    G_BZERO %ptr(p0), %width(s64), 1 :: (store 4)
+    RET_ReallyLR

llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir

 # DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
 # DEBUG-NEXT: G_READ_REGISTER (opcode {{[0-9]+}}): 1 type index, 0 imm indices
 # DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
 # DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
 # DEBUG-NEXT: G_WRITE_REGISTER (opcode {{[0-9]+}}): 1 type index, 0 imm indices
 # DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
 # DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
 # DEBUG-NEXT: G_MEMCPY (opcode {{[0-9]+}}): 3 type indices, 1 imm index
+# DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}}
 # DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
 # DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
 # DEBUG-NEXT: G_MEMMOVE (opcode {{[0-9]+}}): 3 type indices, 1 imm index
 # DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}}
 # DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
 # DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
 # DEBUG-NEXT: G_MEMSET (opcode {{[0-9]+}}): 3 type indices, 1 imm index
 # DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}}
 # DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
 # DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
+# DEBUG-NEXT: G_BZERO (opcode {{[0-9]+}}): 2 type indices, 1 imm index
+# DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
+# DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
 # DEBUG-NEXT: G_VECREDUCE_SEQ_FADD (opcode {{[0-9]+}}): 3 type indices, 0 imm indices
 # DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
 # DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
 # DEBUG-NEXT: G_VECREDUCE_SEQ_FMUL (opcode {{[0-9]+}}): 3 type indices, 0 imm indices
 # DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
 # DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
 # DEBUG-NEXT: G_VECREDUCE_FADD (opcode {{[0-9]+}}): 2 type indices, 0 imm indices
 # DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected

llvm/test/CodeGen/AArch64/GlobalISel/prelegalizercombiner-bzero.mir

+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple aarch64-apple-ios -run-pass=aarch64-prelegalizer-combiner -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=DARWIN
+# RUN: llc -mtriple aarch64 -run-pass=aarch64-prelegalizer-combiner -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=UNKNOWN
+#
+# Check that on Darwin we can combine to G_BZERO. Without Darwin, this should
+# stay as memset.
+
+--- |
+  define void @bzero_unknown_width() { unreachable }
+  define void @bzero_tail_unknown_width() { unreachable }
+  define void @bzero_constant_width() { unreachable }
+  define void @bzero_constant_width_minsize() minsize { unreachable }
+  define void @not_zero() minsize { unreachable }
+...
+---
+name:            bzero_unknown_width
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $x0, $x1
+    ; Always use G_BZERO when the memset width is unknown on Darwin.
+
+    ; DARWIN-LABEL: name: bzero_unknown_width
+    ; DARWIN: liveins: $x0, $x1
+    ; DARWIN: %ptr:_(p0) = COPY $x0
+    ; DARWIN: %width:_(s64) = COPY $x1
+    ; DARWIN: G_BZERO %ptr(p0), %width(s64), 0 :: (store 4)
+    ; DARWIN: RET_ReallyLR
+    ; UNKNOWN-LABEL: name: bzero_unknown_width
+    ; UNKNOWN: liveins: $x0, $x1
+    ; UNKNOWN: %ptr:_(p0) = COPY $x0
+    ; UNKNOWN: %zero:_(s8) = G_CONSTANT i8 0
+    ; UNKNOWN: %width:_(s64) = COPY $x1
+    ; UNKNOWN: G_MEMSET %ptr(p0), %zero(s8), %width(s64), 0 :: (store 4)
+    ; UNKNOWN: RET_ReallyLR
+    %ptr:_(p0) = COPY $x0
+    %zero:_(s8) = G_CONSTANT i8 0
+    %width:_(s64) = COPY $x1
+    G_MEMSET %ptr(p0), %zero(s8), %width(s64), 0 :: (store 4)
+    RET_ReallyLR
+...
+---
+name:            bzero_tail_unknown_width
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $x0, $x1
+    ; DARWIN-LABEL: name: bzero_tail_unknown_width
+    ; DARWIN: liveins: $x0, $x1
+    ; DARWIN: %ptr:_(p0) = COPY $x0
+    ; DARWIN: %width:_(s64) = COPY $x1
+    ; DARWIN: G_BZERO %ptr(p0), %width(s64), 1 :: (store 4)
+    ; DARWIN: RET_ReallyLR
+    ; UNKNOWN-LABEL: name: bzero_tail_unknown_width
+    ; UNKNOWN: liveins: $x0, $x1
+    ; UNKNOWN: %ptr:_(p0) = COPY $x0
+    ; UNKNOWN: %zero:_(s8) = G_CONSTANT i8 0
+    ; UNKNOWN: %width:_(s64) = COPY $x1
+    ; UNKNOWN: G_MEMSET %ptr(p0), %zero(s8), %width(s64), 1 :: (store 4)
+    ; UNKNOWN: RET_ReallyLR
+    %ptr:_(p0) = COPY $x0
+    %zero:_(s8) = G_CONSTANT i8 0
+    %width:_(s64) = COPY $x1
+    G_MEMSET %ptr(p0), %zero(s8), %width(s64), 1 :: (store 4)
+    RET_ReallyLR
+...
+---
+name:            bzero_constant_width
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $x0, $x1
+    ; For values >256, we should use G_BZERO on Darwin.
+
+    ; DARWIN-LABEL: name: bzero_constant_width
+    ; DARWIN: liveins: $x0, $x1
+    ; DARWIN: %ptr:_(p0) = COPY $x0
+    ; DARWIN: %width:_(s64) = G_CONSTANT i64 1024
+    ; DARWIN: G_BZERO %ptr(p0), %width(s64), 0 :: (store 4)
+    ; DARWIN: RET_ReallyLR
+    ; UNKNOWN-LABEL: name: bzero_constant_width
+    ; UNKNOWN: liveins: $x0, $x1
+    ; UNKNOWN: %ptr:_(p0) = COPY $x0
+    ; UNKNOWN: %zero:_(s8) = G_CONSTANT i8 0
+    ; UNKNOWN: %width:_(s64) = G_CONSTANT i64 1024
+    ; UNKNOWN: G_MEMSET %ptr(p0), %zero(s8), %width(s64), 0 :: (store 4)
+    ; UNKNOWN: RET_ReallyLR
+    %ptr:_(p0) = COPY $x0
+    %zero:_(s8) = G_CONSTANT i8 0
+    %width:_(s64) = G_CONSTANT i64 1024
+    G_MEMSET %ptr(p0), %zero(s8), %width(s64), 0 :: (store 4)
+    RET_ReallyLR
+...
+---
+name:            bzero_constant_width_minsize
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $x0, $x1
+    ; With minsize, we should always use G_BZERO to avoid a copy from wzr.
+
+    ; DARWIN-LABEL: name: bzero_constant_width_minsize
+    ; DARWIN: liveins: $x0, $x1
+    ; DARWIN: %ptr:_(p0) = COPY $x0
+    ; DARWIN: %width:_(s64) = G_CONSTANT i64 256
+    ; DARWIN: G_BZERO %ptr(p0), %width(s64), 0 :: (store 4)
+    ; DARWIN: RET_ReallyLR
+    ; UNKNOWN-LABEL: name: bzero_constant_width_minsize
+    ; UNKNOWN: liveins: $x0, $x1
+    ; UNKNOWN: %ptr:_(p0) = COPY $x0
+    ; UNKNOWN: %zero:_(s8) = G_CONSTANT i8 0
+    ; UNKNOWN: %width:_(s64) = G_CONSTANT i64 256
+    ; UNKNOWN: G_MEMSET %ptr(p0), %zero(s8), %width(s64), 0 :: (store 4)
+    ; UNKNOWN: RET_ReallyLR
+    %ptr:_(p0) = COPY $x0
+    %zero:_(s8) = G_CONSTANT i8 0
+    %width:_(s64) = G_CONSTANT i64 256
+    G_MEMSET %ptr(p0), %zero(s8), %width(s64), 0 :: (store 4)
+    RET_ReallyLR
+...
+---
+name:            not_zero
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $x0, $x1
+    ; When the value isn't 0, don't create G_BZERO.
+
+    ; DARWIN-LABEL: name: not_zero
+    ; DARWIN: liveins: $x0, $x1
+    ; DARWIN: %ptr:_(p0) = COPY $x0
+    ; DARWIN: %not_zero:_(s8) = G_CONSTANT i8 1
+    ; DARWIN: %width:_(s64) = G_CONSTANT i64 256
+    ; DARWIN: G_MEMSET %ptr(p0), %not_zero(s8), %width(s64), 0 :: (store 4)
+    ; DARWIN: RET_ReallyLR
+    ; UNKNOWN-LABEL: name: not_zero
+    ; UNKNOWN: liveins: $x0, $x1
+    ; UNKNOWN: %ptr:_(p0) = COPY $x0
+    ; UNKNOWN: %not_zero:_(s8) = G_CONSTANT i8 1
+    ; UNKNOWN: %width:_(s64) = G_CONSTANT i64 256
+    ; UNKNOWN: G_MEMSET %ptr(p0), %not_zero(s8), %width(s64), 0 :: (store 4)
+    ; UNKNOWN: RET_ReallyLR
+    %ptr:_(p0) = COPY $x0
+    %not_zero:_(s8) = G_CONSTANT i8 1
+    %width:_(s64) = G_CONSTANT i64 256
+    G_MEMSET %ptr(p0), %not_zero(s8), %width(s64), 0 :: (store 4)
+    RET_ReallyLR

llvm/test/MachineVerifier/test_g_bzero.mir

+#RUN: not --crash llc -o - -march=arm64 -run-pass=none -verify-machineinstrs %s 2>&1 | FileCheck %s
+# REQUIRES: aarch64-registered-target
+---
+name:            test_bzero
+legalized:       true
+regBankSelected: false
+selected:        false
+tracksRegLiveness: true
+liveins:
+body:             |
+  bb.0:
+
+    %ptr:_(p0) = G_IMPLICIT_DEF
+    %cst1:_(s64) = G_CONSTANT i64 4
+    %cst2:_(s8) = G_CONSTANT i8 7
+
+    ; CHECK: *** Bad machine code: bzero must have 1 memory operand ***
+    G_BZERO %ptr, %cst2, 0
+
+    ; CHECK: *** Bad machine code: bzero memory operand must be a store ***
+    G_BZERO %ptr, %cst2, 0 :: (load 4)
+
+    ; CHECK: *** Bad machine code: Missing mayLoad flag ***
+    ; CHECK: *** Bad machine code: bzero memory operand must be a store ***
+    G_BZERO %ptr, %cst2, 0 :: (load store 4)
+
+    ; CHECK: *** Bad machine code: inconsistent bzero address space ***
+    G_BZERO %ptr, %cst2, 0 :: (store 4, addrspace 1)
+
+   ; CHECK: *** Bad machine code: bzero operand must be a pointer ***
+    G_BZERO %cst1, %cst2, 0 :: (store 4)
+
+...