[llvm-mca] Add pipeline stall events.

[andreadb]

Summary

This patch introduces a new class named HWStallEvent (see HWEventListener.h), and updates the event listener interface. A HWStallEvent represents a pipeline stall caused by the lack of hardware resources. Similarly to HWInstructionEvent, the event type is an unsigned, and the exact meaning depends on the subtarget.
At the moment, HWStallEvent supports a few generic dispatch events.

The main goals of this patch is to remove the logic that counts dispatch stalls from the DispatchUnit to the BackendStatistics view.

Previously, DispatchUnit was responsible for counting and classifying dispatch stall events. With this patch, we delegate the task of counting and classifying stall events to the listeners (i.e. in our case, it is view "BackendStatistics"). So, the DispatchUnit doesn't have to do extra (unnecessary) bookkeeping.

This patch also helps futher simplifying the Backend interface. Now class BackendStatistics no longer has to query the Backend interface to obtain the number of dispatch stalls. As a consequence, we can get rid of all the 'getNumXXX()' methods from class Backend.
The long term goal is to remove all the remaining dependencies between the Backend and the BackendStatistics interface.

No functional change intended.

[courbet]

Only minor nits, otherwise LGTM.

tools/llvm-mca/Dispatch.cpp

289 ↗

(On Diff #138891)

Any reason not to inline this ?

301 ↗

(On Diff #138891)

ditto

[andreadb]

tools/llvm-mca/Dispatch.cpp

289 ↗

(On Diff #138891)

Not really... If I remember correctly, the statement was still using two lines (due to the 80-col). When I wrote this patch, I thought it was probably more readable if I did split it into two statements.
That being said, I am okay with inlining it. I will fix it (and the other case).

llvm/trunk/tools/llvm-mca/Backend.h

 #include "InstrBuilder.h"
 #include "Scheduler.h"
 #include "SourceMgr.h"
 
 namespace mca {
 
 class HWEventListener;
 class HWInstructionEvent;
+class HWStallEvent;
 
 /// \brief An out of order backend for a specific subtarget.
 ///
 /// It emulates an out-of-order execution of instructions. Instructions are
 /// fetched from a MCInst sequence managed by an object of class SourceMgr.
 /// Instructions are firstly dispatched to the schedulers and then executed.
 /// This class tracks the lifetime of an instruction from the moment where
 /// it gets dispatched to the schedulers, to the moment where it finishes
public:
   }
   unsigned getMaxUsedRegisterMappings() const {
     return DU->getMaxUsedRegisterMappings();
   }
   void getBuffersUsage(std::vector<BufferUsageEntry> &Usage) const {
     return HWS->getBuffersUsage(Usage);
   }
 
-  unsigned getNumRATStalls() const { return DU->getNumRATStalls(); }
-  unsigned getNumRCUStalls() const { return DU->getNumRCUStalls(); }
-  unsigned getNumSQStalls() const { return DU->getNumSQStalls(); }
-  unsigned getNumLDQStalls() const { return DU->getNumLDQStalls(); }
-  unsigned getNumSTQStalls() const { return DU->getNumSTQStalls(); }
-  unsigned getNumDispatchGroupStalls() const {
-    return DU->getNumDispatchGroupStalls();
-  }
-
   void addEventListener(HWEventListener *Listener);
   void notifyCycleBegin(unsigned Cycle);
   void notifyInstructionEvent(const HWInstructionEvent &Event);
+  void notifyStallEvent(const HWStallEvent &Event);
   void notifyResourceAvailable(const ResourceRef &RR);
   void notifyCycleEnd(unsigned Cycle);
 };
 
 } // namespace mca
 
 #endif

llvm/trunk/tools/llvm-mca/Backend.cpp

 void Backend::runCycle(unsigned Cycle) {
   notifyCycleBegin(Cycle);
 
   while (SM.hasNext()) {
     InstRef IR = SM.peekNext();
     std::unique_ptr<Instruction> NewIS(
         IB->createInstruction(STI, IR.first, *IR.second));
     const InstrDesc &Desc = NewIS->getDesc();
-    if (!DU->isAvailable(Desc.NumMicroOps) || !DU->canDispatch(*NewIS))
+    if (!DU->isAvailable(Desc.NumMicroOps) ||
+        !DU->canDispatch(IR.first, *NewIS))
       break;
 
     Instruction *IS = NewIS.get();
     Instructions[IR.first] = std::move(NewIS);
     IS->setRCUTokenID(DU->dispatch(IR.first, IS, STI));
     SM.updateNext();
   }
 
void Backend::notifyCycleBegin(unsigned Cycle) {
   HWS->cycleEvent(Cycle);
 }
 
 void Backend::notifyInstructionEvent(const HWInstructionEvent &Event) {
   for (HWEventListener *Listener : Listeners)
     Listener->onInstructionEvent(Event);
 }
 
+void Backend::notifyStallEvent(const HWStallEvent &Event) {
+  for (HWEventListener *Listener : Listeners)
+    Listener->onStallEvent(Event);
+}
+
 void Backend::notifyResourceAvailable(const ResourceRef &RR) {
   DEBUG(dbgs() << "[E] Resource Available: [" << RR.first << '.' << RR.second
                << "]\n");
   for (HWEventListener *Listener : Listeners)
     Listener->onResourceAvailable(RR);
 }
 
 void Backend::notifyCycleEnd(unsigned Cycle) {
   DEBUG(dbgs() << "[E] Cycle end: " << Cycle << "\n\n");
   for (HWEventListener *Listener : Listeners)
     Listener->onCycleEnd(Cycle);
 }
 
 } // namespace mca.

llvm/trunk/tools/llvm-mca/BackendStatistics.h

 ///
 //===----------------------------------------------------------------------===//
 
 #ifndef LLVM_TOOLS_LLVM_MCA_BACKENDSTATISTICS_H
 #define LLVM_TOOLS_LLVM_MCA_BACKENDSTATISTICS_H
 
 #include "Backend.h"
 #include "View.h"
+#include "llvm/ADT/SmallVector.h"
 #include "llvm/MC/MCSubtargetInfo.h"
 #include "llvm/Support/raw_ostream.h"
 #include <map>
 
 namespace mca {
 
 class BackendStatistics : public View {
   // TODO: remove the dependency from Backend.
   const Backend &B;
   const llvm::MCSubtargetInfo &STI;
 
   using Histogram = std::map<unsigned, unsigned>;
   Histogram DispatchGroupSizePerCycle;
   Histogram RetiredPerCycle;
   Histogram IssuedPerCycle;
 
   unsigned NumDispatched;
   unsigned NumIssued;
   unsigned NumRetired;
   unsigned NumCycles;
 
+  // Counts dispatch stall events caused by unavailability of resources.  There
+  // is one counter for every generic stall kind (see class HWStallEvent).
+  llvm::SmallVector<unsigned, 8> HWStalls;
+
   void updateHistograms() {
     DispatchGroupSizePerCycle[NumDispatched]++;
     IssuedPerCycle[NumIssued]++;
     RetiredPerCycle[NumRetired]++;
     NumDispatched = 0;
     NumIssued = 0;
     NumRetired = 0;
   }
 
   void printRetireUnitStatistics(llvm::raw_ostream &OS) const;
   void printDispatchUnitStatistics(llvm::raw_ostream &OS) const;
   void printSchedulerStatistics(llvm::raw_ostream &OS) const;
 
-  void printDispatchStalls(llvm::raw_ostream &OS, unsigned RATStalls,
-                           unsigned RCUStalls, unsigned SQStalls,
-                           unsigned LDQStalls, unsigned STQStalls,
-                           unsigned DGStalls) const;
+  void printDispatchStalls(llvm::raw_ostream &OS) const;
   void printRATStatistics(llvm::raw_ostream &OS, unsigned Mappings,
                           unsigned MaxUsedMappings) const;
   void printRCUStatistics(llvm::raw_ostream &OS, const Histogram &Histogram,
                           unsigned Cycles) const;
   void printDispatchUnitUsage(llvm::raw_ostream &OS, const Histogram &Stats,
                               unsigned Cycles) const;
   void printIssuePerCycle(const Histogram &IssuePerCycle,
                           unsigned TotalCycles) const;
   void printSchedulerUsage(llvm::raw_ostream &OS, const llvm::MCSchedModel &SM,
                            const llvm::ArrayRef<BufferUsageEntry> &Usage) const;
 
 public:
   BackendStatistics(const Backend &backend, const llvm::MCSubtargetInfo &sti)
       : B(backend), STI(sti), NumDispatched(0), NumIssued(0), NumRetired(0),
-        NumCycles(0) {}
+        NumCycles(0), HWStalls(HWStallEvent::LastGenericEvent) {}
 
   void onInstructionEvent(const HWInstructionEvent &Event) override;
 
   void onCycleBegin(unsigned Cycle) override { NumCycles++; }
 
   void onCycleEnd(unsigned Cycle) override { updateHistograms(); }
 
+  void onStallEvent(const HWStallEvent &Event) override {
+    if (Event.Type < HWStallEvent::LastGenericEvent)
+      HWStalls[Event.Type]++;
+  }
+
   void printView(llvm::raw_ostream &OS) const override {
-    printDispatchStalls(OS, B.getNumRATStalls(), B.getNumRCUStalls(),
-                        B.getNumSQStalls(), B.getNumLDQStalls(),
-                        B.getNumSTQStalls(), B.getNumDispatchGroupStalls());
+    printDispatchStalls(OS);
     printRATStatistics(OS, B.getTotalRegisterMappingsCreated(),
                        B.getMaxUsedRegisterMappings());
     printDispatchUnitStatistics(OS);
     printSchedulerStatistics(OS);
     printRetireUnitStatistics(OS);
 
     std::vector<BufferUsageEntry> Usage;
     B.getBuffersUsage(Usage);
     printSchedulerUsage(OS, STI.getSchedModel(), Usage);
   }
 };
 
 } // namespace mca
 
 #endif

llvm/trunk/tools/llvm-mca/BackendStatistics.cpp

void BackendStatistics::printRATStatistics(raw_ostream &OS,
   TempStream << "\n\nRegister Alias Table:";
   TempStream << "\nTotal number of mappings created: " << TotalMappings;
   TempStream << "\nMax number of mappings used:      " << MaxUsedMappings
              << '\n';
   TempStream.flush();
   OS << Buffer;
 }
 
-void BackendStatistics::printDispatchStalls(raw_ostream &OS, unsigned RATStalls,
-                                            unsigned RCUStalls,
-                                            unsigned SCHEDQStalls,
-                                            unsigned LDQStalls,
-                                            unsigned STQStalls,
-                                            unsigned DGStalls) const {
+void BackendStatistics::printDispatchStalls(raw_ostream &OS) const {
   std::string Buffer;
   raw_string_ostream TempStream(Buffer);
   TempStream << "\n\nDynamic Dispatch Stall Cycles:\n";
   TempStream << "RAT     - Register unavailable:                      "
-             << RATStalls;
+             << HWStalls[HWStallEvent::RegisterFileStall];
   TempStream << "\nRCU     - Retire tokens unavailable:                 "
-             << RCUStalls;
+             << HWStalls[HWStallEvent::RetireControlUnitStall];
   TempStream << "\nSCHEDQ  - Scheduler full:                            "
-             << SCHEDQStalls;
+             << HWStalls[HWStallEvent::SchedulerQueueFull];
   TempStream << "\nLQ      - Load queue full:                           "
-             << LDQStalls;
+             << HWStalls[HWStallEvent::LoadQueueFull];
   TempStream << "\nSQ      - Store queue full:                          "
-             << STQStalls;
+             << HWStalls[HWStallEvent::StoreQueueFull];
   TempStream << "\nGROUP   - Static restrictions on the dispatch group: "
-             << DGStalls;
+             << HWStalls[HWStallEvent::DispatchGroupStall];
   TempStream << '\n';
   TempStream.flush();
   OS << Buffer;
 }
 
 void BackendStatistics::printSchedulerUsage(
     raw_ostream &OS, const MCSchedModel &SM,
     const ArrayRef<BufferUsageEntry> &Usage) const {

llvm/trunk/tools/llvm-mca/Dispatch.h

class DispatchUnit {
   unsigned AvailableEntries;
   unsigned CarryOver;
   Scheduler *SC;
 
   std::unique_ptr<RegisterFile> RAT;
   std::unique_ptr<RetireControlUnit> RCU;
   Backend *Owner;
 
-  /// Dispatch stall event identifiers.
-  ///
-  /// The naming convention is:
-  /// * Event names starts with the "DS_" prefix
-  /// * For dynamic dispatch stalls, the "DS_" prefix is followed by the
-  ///   the unavailable resource/functional unit acronym (example: RAT)
-  /// * The last substring is the event reason (example: REG_UNAVAILABLE means
-  ///   that register renaming couldn't find enough spare registers in the
-  ///   register file).
-  ///
-  /// List of acronyms used for processor resoures:
-  /// RAT - Register Alias Table (used by the register renaming logic)
-  /// RCU - Retire Control Unit
-  /// SQ  - Scheduler's Queue
-  /// LDQ - Load Queue
-  /// STQ - Store Queue
-  enum {
-    DS_RAT_REG_UNAVAILABLE,
-    DS_RCU_TOKEN_UNAVAILABLE,
-    DS_SQ_TOKEN_UNAVAILABLE,
-    DS_LDQ_TOKEN_UNAVAILABLE,
-    DS_STQ_TOKEN_UNAVAILABLE,
-    DS_DISPATCH_GROUP_RESTRICTION,
-    DS_LAST
-  };
-
-  // The DispatchUnit track dispatch stall events caused by unavailable
-  // of hardware resources. Events are classified based on the stall kind;
-  // so we have a counter for every source of dispatch stall. Counters are
-  // stored into a vector `DispatchStall` which is always of size DS_LAST.
-  std::vector<unsigned> DispatchStalls;
-
-  bool checkRAT(const Instruction &Desc);
-  bool checkRCU(const InstrDesc &Desc);
-  bool checkScheduler(const InstrDesc &Desc);
+  bool checkRAT(unsigned Index, const Instruction &Desc);
+  bool checkRCU(unsigned Index, const InstrDesc &Desc);
+  bool checkScheduler(unsigned Index, const InstrDesc &Desc);
 
   void updateRAWDependencies(ReadState &RS, const llvm::MCSubtargetInfo &STI);
   void notifyInstructionDispatched(unsigned IID);
 
 public:
   DispatchUnit(Backend *B, const llvm::MCRegisterInfo &MRI,
                unsigned MicroOpBufferSize, unsigned RegisterFileSize,
                unsigned MaxRetirePerCycle, unsigned MaxDispatchWidth,
                Scheduler *Sched)
       : DispatchWidth(MaxDispatchWidth), AvailableEntries(MaxDispatchWidth),
         CarryOver(0U), SC(Sched),
         RAT(llvm::make_unique<RegisterFile>(MRI, RegisterFileSize)),
         RCU(llvm::make_unique<RetireControlUnit>(MicroOpBufferSize,
                                                  MaxRetirePerCycle, this)),
-        Owner(B), DispatchStalls(DS_LAST, 0) {}
+        Owner(B) {}
 
   unsigned getDispatchWidth() const { return DispatchWidth; }
 
   bool isAvailable(unsigned NumEntries) const {
     return NumEntries <= AvailableEntries || AvailableEntries == DispatchWidth;
   }
 
   bool isRCUEmpty() const { return RCU->isEmpty(); }
 
-  bool canDispatch(const Instruction &Inst) {
+  bool canDispatch(unsigned Index, const Instruction &Inst) {
     const InstrDesc &Desc = Inst.getDesc();
     assert(isAvailable(Desc.NumMicroOps));
-    return checkRCU(Desc) && checkRAT(Inst) && checkScheduler(Desc);
+    return checkRCU(Index, Desc) && checkRAT(Index, Inst) &&
+           checkScheduler(Index, Desc);
   }
 
   unsigned dispatch(unsigned IID, Instruction *NewInst,
                     const llvm::MCSubtargetInfo &STI);
 
   void collectWrites(llvm::SmallVectorImpl<WriteState *> &Vec,
                      unsigned RegID) const {
     return RAT->collectWrites(Vec, RegID);
   }
-  unsigned getNumRATStalls() const {
-    return DispatchStalls[DS_RAT_REG_UNAVAILABLE];
-  }
-  unsigned getNumRCUStalls() const {
-    return DispatchStalls[DS_RCU_TOKEN_UNAVAILABLE];
-  }
-  unsigned getNumSQStalls() const {
-    return DispatchStalls[DS_SQ_TOKEN_UNAVAILABLE];
-  }
-  unsigned getNumLDQStalls() const {
-    return DispatchStalls[DS_LDQ_TOKEN_UNAVAILABLE];
-  }
-  unsigned getNumSTQStalls() const {
-    return DispatchStalls[DS_STQ_TOKEN_UNAVAILABLE];
-  }
-  unsigned getNumDispatchGroupStalls() const {
-    return DispatchStalls[DS_DISPATCH_GROUP_RESTRICTION];
-  }
   unsigned getMaxUsedRegisterMappings(unsigned RegFileIndex = 0) const {
     return RAT->getMaxUsedRegisterMappings(RegFileIndex);
   }
   unsigned getTotalRegisterMappingsCreated(unsigned RegFileIndex = 0) const {
     return RAT->getTotalRegisterMappingsCreated(RegFileIndex);
   }
   void addNewRegisterMapping(WriteState &WS) { RAT->addRegisterMapping(WS); }
 
   void cycleEvent(unsigned Cycle) {
     RCU->cycleEvent();
     AvailableEntries =
         CarryOver >= DispatchWidth ? 0 : DispatchWidth - CarryOver;
     CarryOver = CarryOver >= DispatchWidth ? CarryOver - DispatchWidth : 0U;
   }
 
   void notifyInstructionRetired(unsigned Index);
 
+  void notifyDispatchStall(unsigned Index, unsigned EventType);
+
   void onInstructionExecuted(unsigned TokenID) {
     RCU->onInstructionExecuted(TokenID);
   }
 
   void invalidateRegisterMappings(const Instruction &Inst);
 #ifndef NDEBUG
   void dump() const;
 #endif
 };
 
 } // namespace mca
 
 #endif

llvm/trunk/tools/llvm-mca/Dispatch.cpp

 
 #ifndef NDEBUG
 void RetireControlUnit::dump() const {
   dbgs() << "Retire Unit: { Total Slots=" << Queue.size()
          << ", Available Slots=" << AvailableSlots << " }\n";
 }
 #endif
 
-bool DispatchUnit::checkRAT(const Instruction &Instr) {
-  // Collect register definitions from the WriteStates.
-  SmallVector<unsigned, 8> RegDefs;
-
-  for (const std::unique_ptr<WriteState> &Def : Instr.getDefs())
-    RegDefs.push_back(Def->getRegisterID());
-
-  unsigned RegisterMask = RAT->isAvailable(RegDefs);
+bool DispatchUnit::checkRAT(unsigned Index, const Instruction &Instr) {
+  const InstrDesc &Desc = Instr.getDesc();
+  unsigned NumWrites = Desc.Writes.size();
+  unsigned RegisterMask = RAT->isAvailable(NumWrites);
   // A mask with all zeroes means: register files are available.
   if (RegisterMask) {
-    // TODO: We currently implement a single hardware counter for all the
-    // dispatch stalls caused by the unavailability of registers in one of the
-    // register files.  In future, we want to let register files directly notify
-    // hardware listeners in the event of a dispatch stall.  This would simplify
-    // the logic in Dispatch.[h/cpp], and move all the "hardware counting logic"
-    // into a View (for example: BackendStatistics).
-    DispatchStalls[DS_RAT_REG_UNAVAILABLE]++;
+    Owner->notifyStallEvent(
+        HWStallEvent(HWStallEvent::RegisterFileStall, Index));
     return false;
   }
 
   return true;
 }
 
-bool DispatchUnit::checkRCU(const InstrDesc &Desc) {
+bool DispatchUnit::checkRCU(unsigned Index, const InstrDesc &Desc) {
   unsigned NumMicroOps = Desc.NumMicroOps;
   if (RCU->isAvailable(NumMicroOps))
     return true;
-  DispatchStalls[DS_RCU_TOKEN_UNAVAILABLE]++;
+  Owner->notifyStallEvent(
+      HWStallEvent(HWStallEvent::RetireControlUnitStall, Index));
   return false;
 }
 
-bool DispatchUnit::checkScheduler(const InstrDesc &Desc) {
+bool DispatchUnit::checkScheduler(unsigned Index, const InstrDesc &Desc) {
   // If this is a zero-latency instruction, then it bypasses
   // the scheduler.
+  HWStallEvent::GenericEventType Type = HWStallEvent::Invalid;
   switch (SC->canBeDispatched(Desc)) {
   case Scheduler::HWS_AVAILABLE:
     return true;
   case Scheduler::HWS_QUEUE_UNAVAILABLE:
-    DispatchStalls[DS_SQ_TOKEN_UNAVAILABLE]++;
+    Type = HWStallEvent::SchedulerQueueFull;
     break;
   case Scheduler::HWS_LD_QUEUE_UNAVAILABLE:
-    DispatchStalls[DS_LDQ_TOKEN_UNAVAILABLE]++;
+    Type = HWStallEvent::LoadQueueFull;
     break;
   case Scheduler::HWS_ST_QUEUE_UNAVAILABLE:
-    DispatchStalls[DS_STQ_TOKEN_UNAVAILABLE]++;
+    Type = HWStallEvent::StoreQueueFull;
     break;
   case Scheduler::HWS_DISPATCH_GROUP_RESTRICTION:
-    DispatchStalls[DS_DISPATCH_GROUP_RESTRICTION]++;
+    Type = HWStallEvent::DispatchGroupStall;
   }
 
+  Owner->notifyStallEvent(HWStallEvent(Type, Index));
   return false;
 }
 
 void DispatchUnit::updateRAWDependencies(ReadState &RS,
                                          const MCSubtargetInfo &STI) {
   SmallVector<WriteState *, 4> DependentWrites;
 
   collectWrites(DependentWrites, RS.getRegisterID());
unsigned DispatchUnit::dispatch(unsigned IID, Instruction *NewInst,
   SC->scheduleInstruction(IID, NewInst);
   return RCUTokenID;
 }
 
 #ifndef NDEBUG
 void DispatchUnit::dump() const {
   RAT->dump();
   RCU->dump();
-
-  unsigned DSRAT = DispatchStalls[DS_RAT_REG_UNAVAILABLE];
-  unsigned DSRCU = DispatchStalls[DS_RCU_TOKEN_UNAVAILABLE];
-  unsigned DSSCHEDQ = DispatchStalls[DS_SQ_TOKEN_UNAVAILABLE];
-  unsigned DSLQ = DispatchStalls[DS_LDQ_TOKEN_UNAVAILABLE];
-  unsigned DSSQ = DispatchStalls[DS_STQ_TOKEN_UNAVAILABLE];
-
-  dbgs() << "STALLS --- RAT: " << DSRAT << ", RCU: " << DSRCU
-         << ", SCHED_QUEUE: " << DSSCHEDQ << ", LOAD_QUEUE: " << DSLQ
-         << ", STORE_QUEUE: " << DSSQ << '\n';
 }
 #endif
 
 } // namespace mca

llvm/trunk/tools/llvm-mca/HWEventListener.h

public:
   HWInstructionIssuedEvent(
       unsigned Index, const llvm::ArrayRef<std::pair<ResourceRef, unsigned>> UR)
       : HWInstructionEvent(HWInstructionEvent::Issued, Index),
         UsedResources(UR) {}
 
   const llvm::ArrayRef<std::pair<ResourceRef, unsigned>> UsedResources;
 };
 
+// A HWStallEvent represents a pipeline stall caused by the lack of hardware
+// resources.
+class HWStallEvent {
+public:
+  enum GenericEventType {
+    Invalid = 0,
+    // Generic stall events generated by the DispatchUnit.
+    RegisterFileStall,
+    RetireControlUnitStall,
+    DispatchGroupStall,
+    SchedulerQueueFull,
+    LoadQueueFull,
+    StoreQueueFull,
+    LastGenericEvent
+  };
+
+  HWStallEvent(unsigned type, unsigned index) : Type(type), Index(index) {}
+
+  // The exact meaning of the stall event type depends on the subtarget.
+  const unsigned Type;
+  // The index of the instruction in the source manager.
+  const unsigned Index;
+};
+
 class HWEventListener {
 public:
   // Generic events generated by the backend pipeline.
   virtual void onCycleBegin(unsigned Cycle) {}
   virtual void onCycleEnd(unsigned Cycle) {}
 
   virtual void onInstructionEvent(const HWInstructionEvent &Event) {}
+  virtual void onStallEvent(const HWStallEvent &Event) {}
 
   using ResourceRef = std::pair<uint64_t, uint64_t>;
   virtual void onResourceAvailable(const ResourceRef &RRef) {}
 
   virtual ~HWEventListener() {}
 
 private:
   virtual void anchor();
 };
 
 } // namespace mca
 
 #endif