[ARM] Add support for the MVE long shift instructions

[samtebbs]

Summary

MVE adds the lsll, lsrl and asrl instructions, which perform a shift on a 64 bit value separated into two 32 bit registers.

The Expand64BitShift function is modified to accept ISD::SHL, ISD::SRL and ISD::SRA and convert it into the appropriate opcode in ARMISD. An SHL is converted into an lsll, an SRL is converted into an lsrl for the immediate form and a negation and lsll for the register form, and SRA is converted into an asrl.

test/CodeGen/ARM/shift_parts.ll is added to test the logic of emitting these instructions.

[samparker]

llvm/lib/Target/ARM/ARMISelLowering.cpp

5537

Unnecessary opcode checks, the above assert is good enough.

5553

This isn't immediately obvious to me why this is correct, so could you please expand in the comment?

llvm/lib/Target/ARM/ARMInstrInfo.td

181

Why do we have Glue flags here?

[samtebbs]

llvm/lib/Target/ARM/ARMInstrInfo.td

181

They were copied from existing patterns but I've verified that they are unnecessary so will remove.

[dmgreen]

llvm/lib/Target/ARM/ARMISelLowering.cpp

5553

Maybe be explicit that there isn't a t2LSRLr instruction, which is why the negative t2LSLLr is needed.

llvm/test/CodeGen/ARM/shift_parts.ll

168

You can remove this comment.

[samparker]

LGTM.

llvm/lib/Target/ARM/ARMISelLowering.h

enum NodeType : unsigned {
       BRCOND,       // Conditional branch.
       BR_JT,        // Jumptable branch.
       BR2_JT,       // Jumptable branch (2 level - jumptable entry is a jump).
       RET_FLAG,     // Return with a flag operand.
       INTRET_FLAG,  // Interrupt return with an LR-offset and a flag operand.
 
       PIC_ADD,      // Add with a PC operand and a PIC label.
 
+      ASRL,         // MVE long arithmetic shift right.
+      LSRL,         // MVE long shift right.
+      LSLL,         // MVE long shift left.
+
       CMP,          // ARM compare instructions.
       CMN,          // ARM CMN instructions.
       CMPZ,         // ARM compare that sets only Z flag.
       CMPFP,        // ARM VFP compare instruction, sets FPSCR.
       CMPFPw0,      // ARM VFP compare against zero instruction, sets FPSCR.
       FMSTAT,       // ARM fmstat instruction.
 
       CMOV,         // ARM conditional move instructions.

llvm/lib/Target/ARM/ARMISelLowering.cpp

ARMTargetLowering::ARMTargetLowering(const TargetMachine &TM,
 
   setOperationAction(ISD::SHL_PARTS, MVT::i32, Custom);
   setOperationAction(ISD::SRA_PARTS, MVT::i32, Custom);
   setOperationAction(ISD::SRL_PARTS, MVT::i32, Custom);
   setOperationAction(ISD::SRL,       MVT::i64, Custom);
   setOperationAction(ISD::SRA,       MVT::i64, Custom);
   setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i64, Custom);
 
+  // MVE lowers 64 bit shifts to lsll and lsrl
+  // assuming that ISD::SRL and SRA of i64 are already marked custom
+  if (Subtarget->hasMVEIntegerOps())
+    setOperationAction(ISD::SHL, MVT::i64, Custom);
+
   // Expand to __aeabi_l{lsl,lsr,asr} calls for Thumb1.
   if (Subtarget->isThumb1Only()) {
     setOperationAction(ISD::SHL_PARTS, MVT::i32, Expand);
     setOperationAction(ISD::SRA_PARTS, MVT::i32, Expand);
     setOperationAction(ISD::SRL_PARTS, MVT::i32, Expand);
   }
 
   if (!Subtarget->isThumb1Only() && Subtarget->hasV6T2Ops())
const char *ARMTargetLowering::getTargetNodeName(unsigned Opcode) const {
   case ARMISD::FMSTAT:        return "ARMISD::FMSTAT";
 
   case ARMISD::CMOV:          return "ARMISD::CMOV";
   case ARMISD::SUBS:          return "ARMISD::SUBS";
 
   case ARMISD::SSAT:          return "ARMISD::SSAT";
   case ARMISD::USAT:          return "ARMISD::USAT";
 
+  case ARMISD::ASRL:          return "ARMISD::ASRL";
+  case ARMISD::LSRL:          return "ARMISD::LSRL";
+  case ARMISD::LSLL:          return "ARMISD::LSLL";
+
   case ARMISD::SRL_FLAG:      return "ARMISD::SRL_FLAG";
   case ARMISD::SRA_FLAG:      return "ARMISD::SRA_FLAG";
   case ARMISD::RRX:           return "ARMISD::RRX";
 
   case ARMISD::ADDC:          return "ARMISD::ADDC";
   case ARMISD::ADDE:          return "ARMISD::ADDE";
   case ARMISD::SUBC:          return "ARMISD::SUBC";
   case ARMISD::SUBE:          return "ARMISD::SUBE";
static SDValue Expand64BitShift(SDNode *N, SelectionDAG &DAG,
                                 const ARMSubtarget *ST) {
   EVT VT = N->getValueType(0);
   SDLoc dl(N);
 
   // We can get here for a node like i32 = ISD::SHL i32, i64
   if (VT != MVT::i64)
     return SDValue();
 
-  assert((N->getOpcode() == ISD::SRL || N->getOpcode() == ISD::SRA) &&
+  assert((N->getOpcode() == ISD::SRL || N->getOpcode() == ISD::SRA ||
+          N->getOpcode() == ISD::SHL) &&
          "Unknown shift to lower!");
 
+  unsigned ShOpc = N->getOpcode();
+  if (ST->hasMVEIntegerOps()) {
+    SDValue ShAmt = N->getOperand(1);
+    unsigned ShPartsOpc = ARMISD::LSLL;
+    ConstantSDNode *Con = dyn_cast<ConstantSDNode>(ShAmt);
+
+    // If the shift amount is greater than 32 then do the default optimisation
+    if (Con && Con->getZExtValue() > 32)
+      return SDValue();
+
+    // Extract the lower 32 bits of the shift amount if it's an i64
+    if (ShAmt->getValueType(0) == MVT::i64)
+      ShAmt = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, ShAmt,
+                          DAG.getConstant(0, dl, MVT::i32));
+
+    if (ShOpc == ISD::SRL) {
+      if (!Con)
+        // If the shift amount is in a register then negate it and perform an
+        // lsll to emulate a right shift
+        ShAmt = DAG.getNode(ISD::SUB, dl, MVT::i32,
+                            DAG.getConstant(0, dl, MVT::i32), ShAmt);
+      else
+        // Else generate an lsrl on the immediate shift amount
+        ShPartsOpc = ARMISD::LSRL;
+    } else if (ShOpc == ISD::SRA)
+      ShPartsOpc = ARMISD::ASRL;
+
+    // Lower 32 bits of the destination/source
+    SDValue Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, N->getOperand(0),
+                             DAG.getConstant(0, dl, MVT::i32));
+    // Upper 32 bits of the destination/source
+    SDValue Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, N->getOperand(0),
+                             DAG.getConstant(1, dl, MVT::i32));
+
+    // Generate the shift operation as computed above
+    Lo = DAG.getNode(ShPartsOpc, dl, DAG.getVTList(MVT::i32, MVT::i32), Lo, Hi,
+                     ShAmt);
+    // The upper 32 bits come from the second return value of lsll
+    Hi = SDValue(Lo.getNode(), 1);
+    return DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, Lo, Hi);
+  }
+
   // We only lower SRA, SRL of 1 here, all others use generic lowering.
-  if (!isOneConstant(N->getOperand(1)))
+  if (!isOneConstant(N->getOperand(1)) || N->getOpcode() == ISD::SHL)
     return SDValue();
 
   // If we are in thumb mode, we don't have RRX.
   if (ST->isThumb1Only()) return SDValue();
 
   // Okay, we have a 64-bit SRA or SRL of 1.  Lower this to an RRX expr.
   SDValue Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, N->getOperand(0),
                            DAG.getConstant(0, dl, MVT::i32));
void ARMTargetLowering::ReplaceNodeResults(SDNode *N,
   case ISD::READ_REGISTER:
     ExpandREAD_REGISTER(N, Results, DAG);
     break;
   case ISD::BITCAST:
     Res = ExpandBITCAST(N, DAG, Subtarget);
     break;
   case ISD::SRL:
   case ISD::SRA:
+  case ISD::SHL:
     Res = Expand64BitShift(N, DAG, Subtarget);
     break;
   case ISD::SREM:
   case ISD::UREM:
     Res = LowerREM(N, DAG);
     break;
   case ISD::SDIVREM:
   case ISD::UDIVREM:

llvm/lib/Target/ARM/ARMInstrInfo.td

 
 def SDT_LongMac  : SDTypeProfile<2, 4, [SDTCisVT<0, i32>,
                                         SDTCisSameAs<0, 1>,
                                         SDTCisSameAs<0, 2>,
                                         SDTCisSameAs<0, 3>,
                                         SDTCisSameAs<0, 4>,
                                         SDTCisSameAs<0, 5>]>;
 
+// ARMlsll, ARMlsrl, ARMasrl
+def SDT_ARMIntShiftParts : SDTypeProfile<2, 3, [SDTCisSameAs<0, 1>,
+                                              SDTCisSameAs<0, 2>,
+                                              SDTCisSameAs<0, 3>,
+                                              SDTCisInt<0>,
+                                              SDTCisInt<4>]>;
+
 def ARMSmlald        : SDNode<"ARMISD::SMLALD", SDT_LongMac>;
 def ARMSmlaldx       : SDNode<"ARMISD::SMLALDX", SDT_LongMac>;
 def ARMSmlsld        : SDNode<"ARMISD::SMLSLD", SDT_LongMac>;
 def ARMSmlsldx       : SDNode<"ARMISD::SMLSLDX", SDT_LongMac>;
 
 def SDT_MulHSR       : SDTypeProfile<1, 3, [SDTCisVT<0,i32>,
                                             SDTCisSameAs<0, 1>,
                                             SDTCisSameAs<0, 2>,

 def ARMcmn           : SDNode<"ARMISD::CMN", SDT_ARMCmp,
                               [SDNPOutGlue]>;
 
 def ARMcmpZ          : SDNode<"ARMISD::CMPZ", SDT_ARMCmp,
                               [SDNPOutGlue, SDNPCommutative]>;
 
 def ARMpic_add       : SDNode<"ARMISD::PIC_ADD", SDT_ARMPICAdd>;
 
+def ARMasrl          : SDNode<"ARMISD::ASRL", SDT_ARMIntShiftParts, []>;
+def ARMlsrl          : SDNode<"ARMISD::LSRL", SDT_ARMIntShiftParts, []>;
+def ARMlsll          : SDNode<"ARMISD::LSLL", SDT_ARMIntShiftParts, []>;
+
 def ARMsrl_flag      : SDNode<"ARMISD::SRL_FLAG", SDTIntUnaryOp, [SDNPOutGlue]>;
 def ARMsra_flag      : SDNode<"ARMISD::SRA_FLAG", SDTIntUnaryOp, [SDNPOutGlue]>;
 def ARMrrx           : SDNode<"ARMISD::RRX"     , SDTIntUnaryOp, [SDNPInGlue ]>;
 
 def ARMaddc          : SDNode<"ARMISD::ADDC",  SDTBinaryArithWithFlags,
                               [SDNPCommutative]>;
 def ARMsubc          : SDNode<"ARMISD::SUBC",  SDTBinaryArithWithFlags>;
 def ARMadde          : SDNode<"ARMISD::ADDE",  SDTBinaryArithWithFlagsInOut>;

llvm/lib/Target/ARM/ARMInstrMVE.td

class t2MVEShiftDRegReg<string iname, bit op5, bit op16, list<dag> pattern=[]>
   // Custom decoder method because of the following overlapping encodings:
   // ASRL and SQRSHR
   // LSLL and UQRSHL
   // SQRSHRL and SQRSHR
   // UQRSHLL and UQRSHL
   let DecoderMethod = "DecodeMVEOverlappingLongShift";
 }
 
-def t2ASRLr   : t2MVEShiftDRegReg<"asrl",    0b1,  0b0>;
-def t2ASRLi   : t2MVEShiftDRegImm<"asrl",    0b10, ?>;
-def t2LSLLr   : t2MVEShiftDRegReg<"lsll",    0b0,  0b0>;
-def t2LSLLi   : t2MVEShiftDRegImm<"lsll",    0b00, ?>;
-def t2LSRL    : t2MVEShiftDRegImm<"lsrl",    0b01, ?>;
+def t2ASRLr   : t2MVEShiftDRegReg<"asrl",    0b1,  0b0, [(set tGPREven:$RdaLo, tGPROdd:$RdaHi,
+                                        (ARMasrl tGPREven:$RdaLo_src,
+                                        tGPROdd:$RdaHi_src, rGPR:$Rm))]>;
+def t2ASRLi   : t2MVEShiftDRegImm<"asrl",    0b10, ?, [(set tGPREven:$RdaLo, tGPROdd:$RdaHi,
+                                        (ARMasrl tGPREven:$RdaLo_src,
+                                        tGPROdd:$RdaHi_src, (i32 imm:$imm)))]>;
+def t2LSLLr   : t2MVEShiftDRegReg<"lsll",    0b0,  0b0, [(set tGPREven:$RdaLo, tGPROdd:$RdaHi,
+                                        (ARMlsll tGPREven:$RdaLo_src,
+                                        tGPROdd:$RdaHi_src, rGPR:$Rm))]>;
+def t2LSLLi   : t2MVEShiftDRegImm<"lsll",    0b00, ?, [(set tGPREven:$RdaLo, tGPROdd:$RdaHi,
+                                        (ARMlsll tGPREven:$RdaLo_src,
+                                        tGPROdd:$RdaHi_src, (i32 imm:$imm)))]>;
+def t2LSRL    : t2MVEShiftDRegImm<"lsrl",    0b01, ?, [(set tGPREven:$RdaLo, tGPROdd:$RdaHi,
+                                        (ARMlsrl tGPREven:$RdaLo_src,
+                                        tGPROdd:$RdaHi_src, (i32 imm:$imm)))]>;
 
 def t2SQRSHRL : t2MVEShiftDRegReg<"sqrshrl", 0b1,  0b1>;
 def t2SQSHLL  : t2MVEShiftDRegImm<"sqshll",  0b11, 0b1>;
 def t2SRSHRL  : t2MVEShiftDRegImm<"srshrl",  0b10, 0b1>;
 
 def t2UQRSHLL : t2MVEShiftDRegReg<"uqrshll", 0b0,  0b1>;
 def t2UQSHLL  : t2MVEShiftDRegImm<"uqshll",  0b00, 0b1>;
 def t2URSHRL  : t2MVEShiftDRegImm<"urshrl",  0b01, 0b1>;

llvm/test/CodeGen/ARM/shift_parts.ll

+; RUN: llc --verify-machineinstrs -mtriple=thumbv8.1-m.main-none-eabi -mattr=+mve %s -o - | FileCheck %s -check-prefix=CHECK --check-prefix=CHECK-MVE
+; RUN: llc --verify-machineinstrs -mtriple=thumbv8.1-m.main-none-eabi %s -o - | FileCheck %s -check-prefix=CHECK --check-prefix=CHECK-NON-MVE
+
+define i64 @shift_left_reg(i64 %x, i64 %y) {
+; CHECK-MVE-LABEL: shift_left_reg:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    lsll r0, r1, r2
+;
+; CHECK-NON-MVE-LABEL: shift_left_reg:
+; CHECK-NON-MVE:       @ %bb.0: @ %entry
+; CHECK-NON-MVE-NEXT:    .save {r7, lr}
+; CHECK-NON-MVE-NEXT:    push {r7, lr}
+; CHECK-NON-MVE-NEXT:    bl __aeabi_llsl
+entry:
+  %shl = shl i64 %x, %y
+  ret i64 %shl
+}
+
+define i64 @shift_left_imm(i64 %x) {
+; CHECK-MVE-LABEL: shift_left_imm:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    lsll r0, r1, #3
+;
+; CHECK-NON-MVE-LABEL: shift_left_imm:
+; CHECK-NON-MVE:       @ %bb.0: @ %entry
+; CHECK-NON-MVE-NEXT:    lsrs r2, r0, #29
+; CHECK-NON-MVE-NEXT:    lsls r1, r1, #3
+; CHECK-NON-MVE-NEXT:    adds r1, r1, r2
+; CHECK-NON-MVE-NEXT:    lsls r0, r0, #3
+entry:
+  %shl = shl i64 %x, 3
+  ret i64 %shl
+}
+
+define i64 @shift_left_imm_big(i64 %x) {
+; CHECK-LABEL: shift_left_imm_big:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    lsls r1, r0, #16
+; CHECK-NEXT:    movs r0, #0
+entry:
+  %shl = shl i64 %x, 48
+  ret i64 %shl
+}
+
+define i64 @shift_left_imm_big2(i64 %x) {
+; CHECK-MVE-LABEL: shift_left_imm_big2:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    lsll r0, r1, #32
+;
+; CHECK-NON-MVE-LABEL: shift_left_imm_big2:
+; CHECK-NON-MVE:       @ %bb.0: @ %entry
+; CHECK-NON-MVE-NEXT:    movs r1, r0
+; CHECK-NON-MVE-NEXT:    movs r0, #0
+entry:
+  %shl = shl i64 %x, 32
+  ret i64 %shl
+}
+
+define i64 @shift_left_imm_big3(i64 %x) {
+; CHECK-LABEL: shift_left_imm_big3:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    lsls r1, r0, #1
+; CHECK-NEXT:    movs r0, #0
+entry:
+  %shl = shl i64 %x, 33
+  ret i64 %shl
+}
+
+define i64 @shift_right_reg(i64 %x, i64 %y) {
+; CHECK-MVE-LABEL: shift_right_reg:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    rsbs r2, r2, #0
+; CHECK-MVE-NEXT:    lsll r0, r1, r2
+;
+; CHECK-NON-MVE-LABEL: shift_right_reg:
+; CHECK-NON-MVE:       @ %bb.0: @ %entry
+; CHECK-NON-MVE-NEXT:    .save {r7, lr}
+; CHECK-NON-MVE-NEXT:    push {r7, lr}
+; CHECK-NON-MVE-NEXT:    bl __aeabi_llsr
+entry:
+  %shr = lshr i64 %x, %y
+  ret i64 %shr
+}
+
+define i64 @shift_right_imm(i64 %x) {
+; CHECK-MVE-LABEL: shift_right_imm:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    lsrl r0, r1, #3
+;
+; CHECK-NON-MVE-LABEL: shift_right_imm:
+; CHECK-NON-MVE:       @ %bb.0: @ %entry
+; CHECK-NON-MVE-NEXT:    lsls r2, r1, #29
+; CHECK-NON-MVE-NEXT:    lsrs r0, r0, #3
+; CHECK-NON-MVE-NEXT:    adds r0, r0, r2
+; CHECK-NON-MVE-NEXT:    lsrs r1, r1, #3
+entry:
+  %shr = lshr i64 %x, 3
+  ret i64 %shr
+}
+
+define i64 @shift_right_imm_big(i64 %x) {
+; CHECK-LABEL: shift_right_imm_big:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    lsrs r0, r1, #16
+; CHECK-NEXT:    movs r1, #0
+entry:
+  %shr = lshr i64 %x, 48
+  ret i64 %shr
+}
+
+define i64 @shift_right_imm_big2(i64 %x) {
+; CHECK-MVE-LABEL: shift_right_imm_big2:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    lsrl r0, r1, #32
+;
+; CHECK-NON-MVE-LABEL: shift_right_imm_big2:
+; CHECK-NON-MVE:       @ %bb.0: @ %entry
+; CHECK-NON-MVE-NEXT:    movs r0, r1
+; CHECK-NON-MVE-NEXT:    movs r1, #0
+entry:
+  %shr = lshr i64 %x, 32
+  ret i64 %shr
+}
+
+define i64 @shift_right_imm_big3(i64 %x) {
+; CHECK-LABEL: shift_right_imm_big3:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    lsrs r0, r1, #1
+; CHECK-NEXT:    movs r1, #0
+entry:
+  %shr = lshr i64 %x, 33
+  ret i64 %shr
+}
+
+define i64 @shift_arithmetic_right_reg(i64 %x, i64 %y) {
+; CHECK-MVE-LABEL: shift_arithmetic_right_reg:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    asrl r0, r1, r2
+;
+; CHECK-NON-MVE-LABEL: shift_arithmetic_right_reg:
+; CHECK-NON-MVE:       @ %bb.0: @ %entry
+; CHECK-NON-MVE-NEXT:    .save {r7, lr}
+; CHECK-NON-MVE-NEXT:    push {r7, lr}
+; CHECK-NON-MVE-NEXT:    bl __aeabi_lasr
+entry:
+  %shr = ashr i64 %x, %y
+  ret i64 %shr
+}
+
+define i64 @shift_arithmetic_right_imm(i64 %x) {
+; CHECK-MVE-LABEL: shift_arithmetic_right_imm:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    asrl r0, r1, #3
+;
+; CHECK-NON-MVE-LABEL: shift_arithmetic_right_imm:
+; CHECK-NON-MVE:       @ %bb.0: @ %entry
+; CHECK-NON-MVE-NEXT:    lsls r2, r1, #29
+; CHECK-NON-MVE-NEXT:    lsrs r0, r0, #3
+; CHECK-NON-MVE-NEXT:    adds r0, r0, r2
+; CHECK-NON-MVE-NEXT:    asrs r1, r1, #3
+entry:
+  %shr = ashr i64 %x, 3
+  ret i64 %shr
+}
+
+%struct.bar = type { i16, i8, [5 x i8] }
+
+; Function Attrs: norecurse nounwind
+define arm_aapcs_vfpcc void @fn1(%struct.bar* nocapture %a) {
+; CHECK-MVE-LABEL: fn1:
+; CHECK-MVE:       @ %bb.0: @ %entry
+; CHECK-MVE-NEXT:    ldr r2, [r0, #4]
+; CHECK-MVE-NEXT:    movs r1, #0
+; CHECK-MVE-NEXT:    lsll r2, r1, #8
+; CHECK-MVE-NEXT:    strb r1, [r0, #7]
+; CHECK-MVE-NEXT:    str.w r2, [r0, #3]
+;
+; CHECK-NON-MVE-LABEL: fn1:
+; CHECK-NON-MVE:       @ %bb.0: @ %entry
+; CHECK-NON-MVE-NEXT:    ldr r1, [r0, #4]
+; CHECK-NON-MVE-NEXT:    lsls r2, r1, #8
+; CHECK-NON-MVE-NEXT:    movs r3, #3
+; CHECK-NON-MVE-NEXT:    str r2, [r0, r3]
+; CHECK-NON-MVE-NEXT:    lsrs r1, r1, #24
+; CHECK-NON-MVE-NEXT:    strb r1, [r0, #7]
+entry:
+  %carey = getelementptr inbounds %struct.bar, %struct.bar* %a, i32 0, i32 2
+  %0 = bitcast [5 x i8]* %carey to i40*
+  %bf.load = load i40, i40* %0, align 1
+  %bf.clear = and i40 %bf.load, -256
+  store i40 %bf.clear, i40* %0, align 1
+  ret void
+}