[RISCV] Add Stackmap/Statepoint/Patchpoint support with targets

[Zeavee]

Summary

This patch adds stackmap support for RISC-V with call targets.

Depends on D123496

[Zeavee]

Update tests that do not check data directive using update_llc_test_checks.py.

Avoid RISC-V fixups for StkMapRecord Instruction Offset as they them to be 0.

[jrtc27]

llvm/lib/Target/RISCV/MCTargetDesc/RISCVELFStreamer.cpp

262 ↗

(On Diff #445140)

This seems extremely dubious

llvm/lib/Target/RISCV/RISCVAsmPrinter.cpp

166

This seems like a lot of duplication of what already exists for expanding PseudoLI

llvm/lib/Target/RISCV/RISCVInstrInfo.cpp

1306

Why initialise to 0? And why have this large scope when it's only for one case?

llvm/test/CodeGen/RISCV/rv64-patchpoint.ll

42

Why the blank lines?

53

These look like some rather machine-generated names; if you don't want to name them, make them %0/1/2/...

101–112

What's the actual point of this test? It seems like a bit of a mess of random arithmetic, pointer casting and memory operations, with a return of 10 for unknown reasons. Are the instructions themselves interesting? If not, make them simpler.

[Zeavee]

llvm/lib/Target/RISCV/MCTargetDesc/RISCVELFStreamer.cpp

262 ↗

(On Diff #445140)

I understand, and it is. I tried to look at where exactly the OffsetExpr is reduced to 0 in the fixups, but I did not find it. I will search more.

llvm/lib/Target/RISCV/RISCVAsmPrinter.cpp

166

I will check if emitting a PseudoLI works correctly.

llvm/lib/Target/RISCV/RISCVInstrInfo.cpp

1306

I will put it in a smaller scope.

llvm/test/CodeGen/RISCV/rv64-patchpoint.ll

42

They seem to have been added by update_llc_test_checks.py, as I do not have them in previous versions.

101–112

To be honest, I used tests from AArch64, because I thought they would better than test I would create myself. I do not think the instructions themselves are interesting, so I will modify it.

[Zeavee]

Remove useless test and refactor generateInstSeq by adding generateMCInstSeq to avoid code duplication between PseudoLI and Stackmap.

[Zeavee]

Update rv64-patchpoint.ll test.

[Zeavee]

Remove dubious check in RISCVELFStreamer.cpp.

[Zeavee]

Rebased the patch on main. For context, this patch has been used by the LLVM backend of GraalVM's Native Image project in production for around 4 months with no major issues.

[Zeavee]

Hello, now that D123496 has been merged, would it be possible to have a new review on this patch?

[Zeavee]

Gentle ping

[Zeavee]

Friendly ping

llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp

void RISCVAsmParser::emitToStreamer(MCStreamer &S, const MCInst &Inst) {
   bool Res = RISCVRVC::compress(CInst, Inst, getSTI());
   if (Res)
     ++RISCVNumInstrsCompressed;
   S.emitInstruction((Res ? CInst : Inst), getSTI());
 }
 
 void RISCVAsmParser::emitLoadImm(MCRegister DestReg, int64_t Value,
                                  MCStreamer &Out) {
-  RISCVMatInt::InstSeq Seq =
-      RISCVMatInt::generateInstSeq(Value, getSTI().getFeatureBits());
+  SmallVector<MCInst, 8> Seq =
+      RISCVMatInt::generateMCInstSeq(Value, getSTI().getFeatureBits(), DestReg);
 
-  MCRegister SrcReg = RISCV::X0;
-  for (const RISCVMatInt::Inst &Inst : Seq) {
-    switch (Inst.getOpndKind()) {
-    case RISCVMatInt::Imm:
-      emitToStreamer(Out,
-                     MCInstBuilder(Inst.getOpcode()).addReg(DestReg).addImm(Inst.getImm()));
-      break;
-    case RISCVMatInt::RegX0:
-      emitToStreamer(
-          Out, MCInstBuilder(Inst.getOpcode()).addReg(DestReg).addReg(SrcReg).addReg(
-                   RISCV::X0));
-      break;
-    case RISCVMatInt::RegReg:
-      emitToStreamer(
-          Out, MCInstBuilder(Inst.getOpcode()).addReg(DestReg).addReg(SrcReg).addReg(
-                   SrcReg));
-      break;
-    case RISCVMatInt::RegImm:
-      emitToStreamer(
-          Out, MCInstBuilder(Inst.getOpcode()).addReg(DestReg).addReg(SrcReg).addImm(
-                   Inst.getImm()));
-      break;
-    }
-
-    // Only the first instruction has X0 as its source.
-    SrcReg = DestReg;
+  for (MCInst &Inst : Seq) {
+    emitToStreamer(Out, Inst);
   }
 }
 
 void RISCVAsmParser::emitAuipcInstPair(MCOperand DestReg, MCOperand TmpReg,
                                        const MCExpr *Symbol,
                                        RISCVMCExpr::VariantKind VKHi,
                                        unsigned SecondOpcode, SMLoc IDLoc,
                                        MCStreamer &Out) {

llvm/lib/Target/RISCV/MCTargetDesc/RISCVMatInt.h

 //===- RISCVMatInt.h - Immediate materialisation ---------------*- C++ -*--===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #ifndef LLVM_LIB_TARGET_RISCV_MCTARGETDESC_MATINT_H
 #define LLVM_LIB_TARGET_RISCV_MCTARGETDESC_MATINT_H
 
 #include "llvm/ADT/SmallVector.h"
+#include "llvm/MC/MCInst.h"
+#include "llvm/MC/MCRegisterInfo.h"
 #include "llvm/TargetParser/SubtargetFeature.h"
 #include <cstdint>
 
 namespace llvm {
 class APInt;
 
 namespace RISCVMatInt {
 

 // Helper to generate an instruction sequence that can materialize the given
 // immediate value into a register using an additional temporary register. This
 // handles cases where the constant can be generated by (ADD (SLLI X, C), X) or
 // (ADD_UW (SLLI X, C) X). The sequence to generate X is returned. ShiftAmt is
 // provides the SLLI and AddOpc indicates ADD or ADD_UW.
 InstSeq generateTwoRegInstSeq(int64_t Val, const FeatureBitset &ActiveFeatures,
                               unsigned &ShiftAmt, unsigned &AddOpc);
 
+// Helper to generate the generateInstSeq instruction sequence using MCInsts
+SmallVector<MCInst, 8> generateMCInstSeq(int64_t Val,
+                                         const FeatureBitset &ActiveFeatures,
+                                         MCRegister DestReg);
+
 // Helper to estimate the number of instructions required to materialise the
 // given immediate value into a register. This estimate does not account for
 // `Val` possibly fitting into an immediate, and so may over-estimate.
 //
 // This will attempt to produce instructions to materialise `Val` as an
 // `Size`-bit immediate.
 //
 // If CompressionCost is true it will use a different cost calculation if RVC is
 // enabled. This should be used to compare two different sequences to determine
 // which is more compressible.
 int getIntMatCost(const APInt &Val, unsigned Size,
                   const FeatureBitset &ActiveFeatures,
                   bool CompressionCost = false);
 } // namespace RISCVMatInt
 } // namespace llvm
 #endif

llvm/lib/Target/RISCV/MCTargetDesc/RISCVMatInt.cpp

 //===- RISCVMatInt.cpp - Immediate materialisation -------------*- C++ -*--===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "RISCVMatInt.h"
 #include "MCTargetDesc/RISCVMCTargetDesc.h"
 #include "llvm/ADT/APInt.h"
+#include "llvm/MC/MCInstBuilder.h"
 #include "llvm/Support/MathExtras.h"
 using namespace llvm;
 
 static int getInstSeqCost(RISCVMatInt::InstSeq &Res, bool HasRVC) {
   if (!HasRVC)
     return Res.size();
 
   int Cost = 0;
if (ActiveFeatures[RISCV::FeatureStdExtZba] && Lo_32(Val) == Hi_32(Val)) {
     ShiftAmt = 32;
     AddOpc = RISCV::ADD_UW;
     return RISCVMatInt::generateInstSeq(LoVal, ActiveFeatures);
   }
 
   return RISCVMatInt::InstSeq();
 }
 
+SmallVector<MCInst, 8> generateMCInstSeq(int64_t Val,
+                                         const FeatureBitset &ActiveFeatures,
+                                         MCRegister DestReg) {
+  RISCVMatInt::InstSeq Seq = RISCVMatInt::generateInstSeq(Val, ActiveFeatures);
+
+  SmallVector<MCInst, 8> instructions;
+
+  MCRegister SrcReg = RISCV::X0;
+  for (RISCVMatInt::Inst &Inst : Seq) {
+    switch (Inst.getOpndKind()) {
+    case RISCVMatInt::Imm:
+      instructions.push_back(MCInstBuilder(Inst.getOpcode())
+                                 .addReg(DestReg)
+                                 .addImm(Inst.getImm()));
+      break;
+    case RISCVMatInt::RegX0:
+      instructions.push_back(MCInstBuilder(Inst.getOpcode())
+                                 .addReg(DestReg)
+                                 .addReg(SrcReg)
+                                 .addReg(RISCV::X0));
+      break;
+    case RISCVMatInt::RegReg:
+      instructions.push_back(MCInstBuilder(Inst.getOpcode())
+                                 .addReg(DestReg)
+                                 .addReg(SrcReg)
+                                 .addReg(SrcReg));
+      break;
+    case RISCVMatInt::RegImm:
+      instructions.push_back(MCInstBuilder(Inst.getOpcode())
+                                 .addReg(DestReg)
+                                 .addReg(SrcReg)
+                                 .addImm(Inst.getImm()));
+      break;
+    }
+
+    // Only the first instruction has X0 as its source.
+    SrcReg = DestReg;
+  }
+  return instructions;
+}
+
 int getIntMatCost(const APInt &Val, unsigned Size,
                   const FeatureBitset &ActiveFeatures, bool CompressionCost) {
   bool IsRV64 = ActiveFeatures[RISCV::Feature64Bit];
   bool HasRVC = CompressionCost && (ActiveFeatures[RISCV::FeatureStdExtC] ||
                                     ActiveFeatures[RISCV::FeatureStdExtZca]);
   int PlatRegSize = IsRV64 ? 64 : 32;
 
   // Split the constant into platform register sized chunks, and calculate cost

llvm/lib/Target/RISCV/RISCVAsmPrinter.cpp

 //===-- RISCVAsmPrinter.cpp - RISC-V LLVM assembly writer -----------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 // This file contains a printer that converts from our internal representation
 // of machine-dependent LLVM code to the RISC-V assembly language.
 //
 //===----------------------------------------------------------------------===//
 
 #include "MCTargetDesc/RISCVBaseInfo.h"
 #include "MCTargetDesc/RISCVInstPrinter.h"
 #include "MCTargetDesc/RISCVMCExpr.h"
+#include "MCTargetDesc/RISCVMatInt.h"
 #include "MCTargetDesc/RISCVTargetStreamer.h"
 #include "RISCV.h"
 #include "RISCVMachineFunctionInfo.h"
 #include "RISCVTargetMachine.h"
 #include "TargetInfo/RISCVTargetInfo.h"
 #include "llvm/ADT/APInt.h"
 #include "llvm/ADT/Statistic.h"
 #include "llvm/BinaryFormat/ELF.h"
void RISCVAsmPrinter::LowerPATCHPOINT(MCStreamer &OutStreamer, StackMaps &SM,
 
   auto &Ctx = OutStreamer.getContext();
   MCSymbol *MILabel = Ctx.createTempSymbol();
   OutStreamer.emitLabel(MILabel);
   SM.recordPatchPoint(*MILabel, MI);
 
   PatchPointOpers Opers(&MI);
 
+  const MachineOperand &CalleeMO = Opers.getCallTarget();
   unsigned EncodedBytes = 0;
 
+  if (CalleeMO.isImm()) {
+    uint64_t CallTarget = CalleeMO.getImm();
+    if (CallTarget) {
+      assert((CallTarget & 0xFFFF'FFFF'FFFF) == CallTarget &&
+             "High 16 bits of call target should be zero.");
+      // Materialize the jump address:
+      SmallVector<MCInst, 8> Seq = RISCVMatInt::generateMCInstSeq(
+          CallTarget, STI->getFeatureBits(), RISCV::X1);
+      for (MCInst &Inst : Seq) {
+        EmitToStreamer(OutStreamer, Inst);
+      }
+      EncodedBytes += Seq.size() * 4;
+      EmitToStreamer(OutStreamer, MCInstBuilder(RISCV::JALR)
+                                      .addReg(RISCV::X1)
+                                      .addReg(RISCV::X1)
+                                      .addImm(0));
+      EncodedBytes += 4;
+    }
+  } else if (CalleeMO.isGlobal()) {
+    MCOperand CallTargetMCOp;
+    lowerOperand(CalleeMO, CallTargetMCOp);
+    EmitToStreamer(OutStreamer,
+                   MCInstBuilder(RISCV::PseudoCALL).addOperand(CallTargetMCOp));
+    EncodedBytes += 8;
+  }
+
   // Emit padding.
   unsigned NumBytes = Opers.getNumPatchBytes();
   assert(NumBytes >= EncodedBytes &&
          "Patchpoint can't request size less than the length of a call.");
   assert((NumBytes - EncodedBytes) % NOPBytes == 0 &&
          "Invalid number of NOP bytes requested!");
   emitNops((NumBytes - EncodedBytes) / NOPBytes);
 }
 
 void RISCVAsmPrinter::LowerSTATEPOINT(MCStreamer &OutStreamer, StackMaps &SM,
                                       const MachineInstr &MI) {
   unsigned NOPBytes = STI->getFeatureBits()[RISCV::FeatureStdExtC] ? 2 : 4;
 
   StatepointOpers SOpers(&MI);
   if (unsigned PatchBytes = SOpers.getNumPatchBytes()) {
     assert(PatchBytes % NOPBytes == 0 &&
            "Invalid number of NOP bytes requested!");
     emitNops(PatchBytes / NOPBytes);
+  } else {
+    // Lower call target and choose correct opcode
+    const MachineOperand &CallTarget = SOpers.getCallTarget();
+    MCOperand CallTargetMCOp;
+    switch (CallTarget.getType()) {
+    case MachineOperand::MO_GlobalAddress:
+    case MachineOperand::MO_ExternalSymbol:
+      lowerOperand(CallTarget, CallTargetMCOp);
+      EmitToStreamer(
+          OutStreamer,
+          MCInstBuilder(RISCV::PseudoCALL).addOperand(CallTargetMCOp));
+      break;
+    case MachineOperand::MO_Immediate:
+      CallTargetMCOp = MCOperand::createImm(CallTarget.getImm());
+      EmitToStreamer(OutStreamer, MCInstBuilder(RISCV::JAL)
+                                      .addReg(RISCV::X1)
+                                      .addOperand(CallTargetMCOp));
+      break;
+    case MachineOperand::MO_Register:
+      CallTargetMCOp = MCOperand::createReg(CallTarget.getReg());
+      EmitToStreamer(OutStreamer, MCInstBuilder(RISCV::JALR)
+                                      .addReg(RISCV::X1)
+                                      .addOperand(CallTargetMCOp)
+                                      .addImm(0));
+      break;
+    default:
+      llvm_unreachable("Unsupported operand type in statepoint call target");
+      break;
+    }
   }
 
   auto &Ctx = OutStreamer.getContext();
   MCSymbol *MILabel = Ctx.createTempSymbol();
   OutStreamer.emitLabel(MILabel);
   SM.recordStatepoint(*MILabel, MI);
 }
 

llvm/lib/Target/RISCV/RISCVISelLowering.cpp

#define PseudoVFCVT_RM_CASE_MF8(RMOpc, Opc)                                    \
   case RISCV::PseudoFROUND_H_INX:
   case RISCV::PseudoFROUND_S:
   case RISCV::PseudoFROUND_S_INX:
   case RISCV::PseudoFROUND_D:
   case RISCV::PseudoFROUND_D_INX:
   case RISCV::PseudoFROUND_D_IN32X:
     return emitFROUND(MI, BB, Subtarget);
   case TargetOpcode::STATEPOINT:
+    // STATEPOINT is a pseudo instruction which has no implicit defs/uses
+    // while jal call instruction (where statepoint will be lowered at the end)
+    // has implicit def. This def is early-clobber as it will be set at
+    // the moment of the call and earlier than any use is read.
+    // Add this implicit dead def here as a workaround.
+    MI.addOperand(*MI.getMF(),
+                  MachineOperand::CreateReg(
+                      RISCV::X1, /*isDef*/ true,
+                      /*isImp*/ true, /*isKill*/ false, /*isDead*/ true,
+                      /*isUndef*/ false, /*isEarlyClobber*/ true));
+    [[fallthrough]];
   case TargetOpcode::STACKMAP:
   case TargetOpcode::PATCHPOINT:
     if (!Subtarget.is64Bit())
       report_fatal_error("STACKMAP, PATCHPOINT and STATEPOINT are only "
                          "supported on 64-bit targets");
     return emitPatchPoint(MI, BB);
   }
 }

llvm/lib/Target/RISCV/RISCVInstrInfo.cpp

unsigned RISCVInstrInfo::getInstSizeInBytes(const MachineInstr &MI) const {
   if (Opcode == TargetOpcode::BUNDLE)
     return getInstBundleLength(MI);
 
   if (MI.getParent() && MI.getParent()->getParent()) {
     if (isCompressibleInst(MI, STI))
       return 2;
   }
 
   switch (Opcode) {
   case TargetOpcode::STACKMAP:
     // The upper bound for a stackmap intrinsic is the full length of its shadow
     return StackMapOpers(&MI).getNumPatchBytes();
   case TargetOpcode::PATCHPOINT:
     // The size of the patchpoint intrinsic is the number of bytes requested
     return PatchPointOpers(&MI).getNumPatchBytes();
-  case TargetOpcode::STATEPOINT:
+  case TargetOpcode::STATEPOINT: {
     // The size of the statepoint intrinsic is the number of bytes requested
-    return StatepointOpers(&MI).getNumPatchBytes();
+    unsigned NumBytes = StatepointOpers(&MI).getNumPatchBytes();
+    // A statepoint is at least a PseudoCALL
+    if (NumBytes < 8)
+      NumBytes = 8;
+    return NumBytes;
+  }
   default:
     return get(Opcode).getSize();
   }
 }
 
 unsigned RISCVInstrInfo::getInstBundleLength(const MachineInstr &MI) const {
   unsigned Size = 0;
   MachineBasicBlock::const_instr_iterator I = MI.getIterator();

llvm/test/CodeGen/RISCV/rv64-patchpoint.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=riscv64 -debug-entry-values -enable-misched=0 < %s | FileCheck %s
 
+; Trivial patchpoint codegen
+;
+define i64 @trivial_patchpoint_codegen(i64 %p1, i64 %p2, i64 %p3, i64 %p4) {
+; CHECK-LABEL: trivial_patchpoint_codegen:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    addi sp, sp, -16
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    sd s0, 8(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    sd s1, 0(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    .cfi_offset s0, -8
+; CHECK-NEXT:    .cfi_offset s1, -16
+; CHECK-NEXT:    mv s0, a0
+; CHECK-NEXT:  .Ltmp0:
+; CHECK-NEXT:    lui ra, 3563
+; CHECK-NEXT:    addiw ra, ra, -577
+; CHECK-NEXT:    slli ra, ra, 12
+; CHECK-NEXT:    addi ra, ra, -259
+; CHECK-NEXT:    slli ra, ra, 12
+; CHECK-NEXT:    addi ra, ra, -1282
+; CHECK-NEXT:    jalr ra
+; CHECK-NEXT:    mv s1, a0
+; CHECK-NEXT:    mv a0, s0
+; CHECK-NEXT:    mv a1, s1
+; CHECK-NEXT:  .Ltmp1:
+; CHECK-NEXT:    lui ra, 3563
+; CHECK-NEXT:    addiw ra, ra, -577
+; CHECK-NEXT:    slli ra, ra, 12
+; CHECK-NEXT:    addi ra, ra, -259
+; CHECK-NEXT:    slli ra, ra, 12
+; CHECK-NEXT:    addi ra, ra, -1281
+; CHECK-NEXT:    jalr ra
+; CHECK-NEXT:    mv a0, s1
+; CHECK-NEXT:    ld s0, 8(sp) # 8-byte Folded Reload
+; CHECK-NEXT:    ld s1, 0(sp) # 8-byte Folded Reload
+; CHECK-NEXT:    addi sp, sp, 16
+; CHECK-NEXT:    ret
+entry:
+  %resolveCall2 = inttoptr i64 244837814094590 to i8*
+  %result = tail call i64 (i64, i32, i8*, i32, ...) @llvm.experimental.patchpoint.i64(i64 2, i32 28, i8* %resolveCall2, i32 4, i64 %p1, i64 %p2, i64 %p3, i64 %p4)
+  %resolveCall3 = inttoptr i64 244837814094591 to i8*
+  tail call void (i64, i32, i8*, i32, ...) @llvm.experimental.patchpoint.void(i64 3, i32 28, i8* %resolveCall3, i32 2, i64 %p1, i64 %result)
+  ret i64 %result
+}
+
 ; Test small patchpoints that don't emit calls.
 define void @small_patchpoint_codegen(i64 %p1, i64 %p2, i64 %p3, i64 %p4) {
 ; CHECK-LABEL: small_patchpoint_codegen:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    .cfi_def_cfa_offset 0
-; CHECK-NEXT:  .Ltmp0:
+; CHECK-NEXT:  .Ltmp2:
 ; CHECK-NEXT:    nop
 ; CHECK-NEXT:    nop
 ; CHECK-NEXT:    nop
 ; CHECK-NEXT:    nop
 ; CHECK-NEXT:    nop
 ; CHECK-NEXT:    ret
 entry:
   %result = tail call i64 (i64, i32, i8*, i32, ...) @llvm.experimental.patchpoint.i64(i64 5, i32 20, i8* null, i32 2, i64 %p1, i64 %p2)
   ret void
 }
 
 declare void @llvm.experimental.stackmap(i64, i32, ...)
 declare void @llvm.experimental.patchpoint.void(i64, i32, i8*, i32, ...)
 declare i64 @llvm.experimental.patchpoint.i64(i64, i32, i8*, i32, ...)