[test-suite] Adding miniXyce Benchmark

[homerdin]

Summary

Description:

At this time, miniXyce is a simple linear circuit simulator with a
basic parser that performs transient analysis on any circuit with
resistors (R), inductors (L), capacitors (C), and voltage/current
sources. The parser incorporated into this version of miniXyce is a
single pass parser, where the netlist is expected to be flat
(no hierarchy via subcircuits is allowed). Simulating the system of
DAEs generates a nonsymmetric linear problem, which is solved using
un-preconditioned GMRES. The time integration method used in miniXyce
is backward Euler with a constant time-step. The simulator outputs
all the solution variables at each time step in a 'prn' file.

Links:

Web: https://mantevo.org/packages/
Github: https://github.com/Mantevo/miniXyce

When run on Intel(R) Xeon(R) CPU E5-2699 v4 @ 2.20GHz:

compile_time: 76.0057 
exec_time: 0.0132 
Maximum resident set size (kbytes): 6272

[MatzeB]

• Integration looks good to me.
• Maybe put the information from the patch description into a README file in the benchmark directory?
• On my macOS x86_64 system it compiles warning free but fails to verify:

$ diff -u /Users/mbraun/dev/test-suite/build/MultiSource/Benchmarks/DOE-ProxyApps-C++/miniXyce/Output/miniXyce.test.out /Users/mbraun/dev/test-suite/MultiSource/Benchmarks/DOE-ProxyApps-C++/miniXyce/miniXyce.reference_output
--- /Users/mbraun/dev/test-suite/build/MultiSource/Benchmarks/DOE-ProxyApps-C++/miniXyce/Output/miniXyce.test.out	2017-08-15 14:10:36.000000000 -0700
+++ /Users/mbraun/dev/test-suite/MultiSource/Benchmarks/DOE-ProxyApps-C++/miniXyce/miniXyce.reference_output	2017-08-15 14:09:10.000000000 -0700
@@ -32,5 +32,5 @@
   GMRES_tolerance: 1e-06
   GMRES_subspace_dim: 10
   GMRES_average_iters: 5
-  GMRES_average_res: 5.65781e-16
+  GMRES_average_res: 5.79223e-16
 exit 0

LICENSE.TXT

87

misindentend (tabs vs spaces?)

[MatzeB]

This also seems to produce intermediate files:

• Doing I/O in a compiler benchmark is generally a bad smell: We want to benchmark the CPU not the I/O subsystem. It may be fine if the I/O part doesn't dominate the runtime and is simple enough, but if there is an easy way around it that would be nice.
• If you create new files, I'd suggest to accept a full pathname on the commandline and specifying a path using ${CMAKE_CURRENT_BINARY_DIRECTORY} placing files in the source directory can lead to race conditions if multiple builds (in different build directories) use the same sourcecode checkout. We also generally shouldn't assume the source checkout to be writable for benchmarks.

[homerdin]

I included the README file and address the indentation in LICENSE.txt. I macro'd out the production of the output file, it is meant to be used for analysis and should not be created within the test-suite. We are still looking into the GMRES_average_res issue.

[homerdin]

In D36738#842616, @MatzeB wrote:

• On my macOS x86_64 system it compiles warning free but fails to verify:

$ diff -u /Users/mbraun/dev/test-suite/build/MultiSource/Benchmarks/DOE-ProxyApps-C++/miniXyce/Output/miniXyce.test.out /Users/mbraun/dev/test-suite/MultiSource/Benchmarks/DOE-ProxyApps-C++/miniXyce/miniXyce.reference_output
--- /Users/mbraun/dev/test-suite/build/MultiSource/Benchmarks/DOE-ProxyApps-C++/miniXyce/Output/miniXyce.test.out	2017-08-15 14:10:36.000000000 -0700
+++ /Users/mbraun/dev/test-suite/MultiSource/Benchmarks/DOE-ProxyApps-C++/miniXyce/miniXyce.reference_output	2017-08-15 14:09:10.000000000 -0700
@@ -32,5 +32,5 @@
   GMRES_tolerance: 1e-06
   GMRES_subspace_dim: 10
   GMRES_average_iters: 5
-  GMRES_average_res: 5.65781e-16
+  GMRES_average_res: 5.79223e-16
 exit 0

Was able to reproduce this issue on my mac. We figured out that the issue in from the use of the sin function.

mX_source.cpp:51:  return (offset + amplitude * sin(2*pi*freq*t + phase));
mX_source.cpp:67:  return (offset + amplitude * sin(2*pi*carrier_freq*t + modulation_index*sin(2*pi*signal_freq*t)));

I rebuilt the application linking to the JuliaLang openlibm on both machines and am now getting GMRES_average_res: 5.73829e-16 on both machines.
Still looking into how to approach this issue for the test-suite.

Thanks,
Brian

LICENSE.TXT

 oggenc:             llvm-test/MultiSource/Applications/oggenc
 JM:                 llvm-test/MultiSource/Applications/JM
 ALAC:               llvm-test/MultiSource/Applications/ALAC
 ASC Sequoia:        llvm-test/MultiSource/Benchmarks/ASC_Sequoia/AMGmk
                     llvm-test/MultiSource/Benchmarks/ASC_Sequoia/IRSmk
                     llvm-test/MultiSource/Benchmarks/ASC_Sequoia/sphot
 smg2000:            llvm-test/MultiSource/Benchmarks/ASCI_Purple/SMG2000
 XSBench:            llvm-test/MultiSource/Benchmarks/DOE-ProxyApps-C/XSBench
+miniXyce:           llvm-test/MultiSource/Benchmarks/DOE-ProxyApps-C++/miniXyce
 Fhourstones:        llvm-test/MultiSource/Benchmarks/Fhourstones
 Fhourstones-3.1:    llvm-test/MultiSource/Benchmarks/Fhourstones-3.1
 McCat:              llvm-test/MultiSource/Benchmarks/McCat
 Olden:              llvm-test/MultiSource/Benchmarks/Olden
 OptimizerEval:      llvm-test/MultiSource/Benchmarks/OptimizerEval
 Ptrdist:            llvm-test/MultiSource/Benchmarks/Ptrdist
 LLUBenchmark:       llvm-test/MultiSource/Benchmarks/llubenchmark
 SIM:                llvm-test/MultiSource/Benchmarks/sim

MultiSource/Benchmarks/CMakeLists.txt

 
 if((NOT "${TARGET_OS}" STREQUAL "Darwin") OR (NOT "${ARCH}" STREQUAL "ARM"))
   add_subdirectory(TSVC)
 endif()
 if(NOT DEFINED DISABLE_CXX)
   add_subdirectory(Prolangs-C++)
   add_subdirectory(Bullet)
   add_subdirectory(tramp3d-v4)
+  add_subdirectory(DOE-ProxyApps-C++)
   if(NOT "${ARCH}" STREQUAL "XCore")
     add_subdirectory(7zip)
     add_subdirectory(PAQ8p)
   endif()
 endif()
 if(NOT DEFINED SMALL_PROBLEM_SIZE)
   add_subdirectory(mafft)
 endif()

MultiSource/Benchmarks/DOE-ProxyApps-C++/CMakeLists.txt

+add_subdirectory(miniXyce)

MultiSource/Benchmarks/DOE-ProxyApps-C++/Makefile

+# MultiSource/DOE-ProxyApps-C++ Makefile: Build all subdirectories automatically
+
+LEVEL = ../../..
+PARALLEL_DIRS = miniXyce
+
+include $(LEVEL)/Makefile.programs

MultiSource/Benchmarks/DOE-ProxyApps-C++/miniXyce/CMakeLists.txt

+set(PROG miniXyce)
+list(APPEND CXXFLAGS -DMINIXYCE_INFO=0)
+# Single Pass Circuit Simulator.  Runs very quickly.  Unsure how to scale up.
+set(RUN_OPTIONS -c ${CMAKE_CURRENT_SOURCE_DIR}/cir5.net -pf ${CMAKE_CURRENT_SOURCE_DIR}/last_used_params.txt)
+llvm_multisource()

MultiSource/Benchmarks/DOE-ProxyApps-C++/miniXyce/Makefile

+LEVEL = ../../../..
+
+PROG = miniXyce
+CPPFLAGS = -DMINIXYCE_INFO=0
+RUN_OPTIONS = -c $(PROJ_SRC_DIR)/cir5.net -pf $(PROJ_SRC_DIR)/last_used_params.txt
+include $(LEVEL)/MultiSource/Makefile.multisrc

MultiSource/Benchmarks/DOE-ProxyApps-C++/miniXyce/README

+**************************************************************************
+LLVM Test-suite Note:
+**************************************************************************
+The original source is located at https://github.com/Mantevo/miniXyce.
+Beyond this paragraph is the original README contained with the source 
+code.  The Makefile refered to within is not utilized within the 
+test-suite.  The test-suite builds a serial version (openmp and 
+mpi disabled) with its own cmake and make build system.  It does not
+produce the <ckt_name>_tran_results.prn file nor does it offer the 
+test building scripts.
+**************************************************************************
+
+miniXyce circuit/network simulation mini-application
+
+--------------------------------------
+Contents of this README file:
+1. miniXyce overview
+2. miniXyce versions
+3. building miniXyce
+4. running miniXyce
+5. generating test circuits
+6. understanding the results
+--------------------------------------
+
+--------------------------------------
+1. miniXyce overview
+
+At this time, miniXyce is a simple linear circuit simulator with a 
+basic parser that performs transient analysis on any circuit with
+resistors (R), inductors (L), capacitors (C), and voltage/current 
+sources.  The parser incorporated into this version of miniXyce is a
+single pass parser, where the netlist is expected to be flat 
+(no hierarchy via subcircuits is allowed). Simulating the system of 
+DAEs generates a nonsymmetric linear problem, which is solved using 
+un-preconditioned GMRES. The time integration method used in miniXyce 
+is backward Euler with a constant time-step.  The simulator outputs
+all the solution variables at each time step in a 'prn' file.
+
+The development of the first version of miniXyce resulted in something
+closer to a compact application than a miniapp since more focus was put 
+on the simulator returning the correct answer, than modeling performance 
+characteristics of interest. Further analysis of Xyce has called out 
+particular performance issues in each of the three phases of circuit
+simulation:  parsing, device evaluation/loading, and the solution of linear
+equations.  These issues will inspire enhancements to, and a second version 
+of, miniXyce. 
+
+Finally, miniXyce compares the computed solution against gold standard output
+from Xyce for a couple test circuits found in the 'tests' directory.
+
+--------------------------------------
+2. miniXyce versions:
+- miniXyce_ref:
+     reference version: self-contained, includes serial and MPI-parallel
+     parallelism is optional, it can be built without MPI usage.
+
+-------------------
+3. Building miniXyce:
+
+ The default case is: cd into the 'miniXyce_ref' directory and type 'make'.
+ That will build the MPI-parallel miniXyce code assuming you have the
+ mpi-compiler-wrappers mpicxx in your path.
+
+ If successful, that will create an executable called miniXyce.x.
+
+Special builds can be done for things like:
+- gnu compilers (g++), no MPI
+  type 'make -f Makefile.gnu.serial'
+- Intel compilers (icpc), no MPI
+  type 'make -f Makefile.intel.serial'
+
+If the default makefile isn't correct for your system, copy and edit and
+use a 'make -f' command similar to the above cases.
+
+Note 1:
+  'make' calls the script 'generate_info_header' to generate miniXyce_info.hpp
+  with platform and build information. If that script fails for some reason and 
+  doesn't produce a valid miniXyce_info.hpp header, you can still build miniXyce by 
+  typing
+  'make MINIXYCE_INFO=0'.
+ 
+-------------------
+4. Running miniXyce:
+
+ miniXyce can be run like this:
+   % <mpi-run-command> miniXyce.x <options>
+
+ Description of command line options:
+
+  -c, --circuit {filename}: specifies the netlist to be simulated.
+  -ti, --tstart, --t_start {double value}: specifies start time of transient simulation
+  -tf, -- tstop, --t_stop {double value}: specifies stop time of transient simulation
+  -h, --tstep, --t_step {double value}: specifies the length of each transient simulation time step
+  -tol, --tolerance {double value}: specifies the GMRES tolerance
+  -k, --restart {double value}: tells GMRES how often to restart
+  -i, --init, --init_cond, --initcond, --x0 {vector of length (num_nodes + num_voltage_sources + num_inductors)}: 
+      specifies the initial condition for the simulation. If it is not specified, then a DC operating point initial condition is assumed.
+  -pf, --paramsfile, --params_file {filename}: specifies a file from which additional simulation parameters are to be obtained.
+  --prev : this flag tells the simulator to use whatever parameter values it used last time (for those parameters that have not been specifies on the command line)
+
+ miniXyce examples:
+
+  Ex 1: Run a simulation of circuit tests/cir1.net on 3 processors, using t_start = 1e-6 and other parameters taken from the file params.txt
+
+   mpirun -np 3 miniXyce.x --circuit tests/cir1.net --t_start 1e-6 --pf params.txt
+
+  Ex 2: Repeat the above simulation, but this time with 4 processors
+
+   mpirun -np 4 miniXyce.x --prev
+
+  Ex 3: Simulate the circuit cir4.net on 2 processors with default parameters
+
+   mpirun -np 2 miniXyce.x -c tests/cir4.net
+
+  Ex 4: Repeat the above simulation, but this time use an initial condition [2 3]
+
+   mpirun -np 2 miniXyce.x --prev --init 2 3
+
+  Note: The initial condition is *not* saved in the last_used_params.txt file. Only the rest of the parameters are saved. 
+        So if you use --prev without specifying anything else, a DC operating point initial condition will be computed.
+-----------------
+5. Generating test circuits:
+
+The test circuits provided in the 'tests' directory (cir1.net, cir2.net, cir3.net, and cir4.net) can run on, at most, 4 MPI processes.
+To make it easier to generate netlists for RC and RLC Ladders, there are 2 perl scripts RC_Ladder.pl and RLC_Ladder.pl in the 'tests' 
+directory. You can use them as follows:
+
+perl RC_Ladder.pl <number of RC Ladder stages you want> ><circuit name>.net
+perl RLC_Ladder.pl <number of RLC Ladder stages you want> ><circuit name>.net
+
+Example: perl RC_Ladder.pl 5 >cir6.net
+
+This allows the user to generate arbitrarily large linear circuits to test miniXyce.
+
+-----------------
+6. Understanding the results
+
+After running miniXyce, you will be left with a text file named <ckt_name>_tran_results.prn. There will only be one, regardless of the number of processors.
+
+The first line of this file is of the form:
+
+TIME state_variable_name_1 state_variable_name_2 ... state_variable_name_k
+
+Each subsequent line in each of these files gives values of the form:
+
+time value1 value2 ... valuek
+
+where value1 ... valuek are all the components of the solution vector x. 
+In addition, each line of the text file will contain two more attributes corresponding to the number of gmres iterations and restarts respectively.
+
+Matlab can be used to plot the data in this file using the very basic plot_tran_sim_results.m, which can be found in the 'utils' directory.
+ 

MultiSource/Benchmarks/DOE-ProxyApps-C++/miniXyce/YAML_Doc.hpp

+//w
+//@HEADER
+// ************************************************************************
+// 
+//               Mantevo: A collection of mini-applications for HPC
+//                 Copyright (2008) Sandia Corporation
+// 
+// Under terms of Contract DE-AC04-94AL85000, there is a non-exclusive
+// license for use of this work by or on behalf of the U.S. Government.
+// 
+// This library is free software; you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as
+// published by the Free Software Foundation; either version 2.1 of the
+// License, or (at your option) any later version.
+//  
+// This library is distributed in the hope that it will be useful, but
+// WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+// Lesser General Public License for more details.
+//  
+// You should have received a copy of the GNU Lesser General Public
+// License along with this library; if not, write to the Free Software
+// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+// USA
+// Questions? Contact Michael A. Heroux ([e-mail]) 
+// 
+// ************************************************************************
+//@HEADER
+
+// Changelog
+//
+// Version 0.1
+// - Initial version.
+//
+/////////////////////////////////////////////////////////////////////////
+
+#ifndef YAML_DOC_H
+#define YAML_DOC_H
+#include <string>
+#include <vector>
+#include "YAML_Element.hpp"
+
+//! The Mantevo YAML_Doc class for the uniform collecting and reporting of performance data for mini-applications
+
+/*!
+
+The YAML_Doc class works in conjuction with the YAML_Element class to facilitate easy collecting and reporting of YAML-formatted
+data that can be then registered with the Mantevo results collection website.
+
+\code
+
+//EXAMPLE CODE FOR GENERATING YAML
+
+  YAML_Doc doc("hpccg","1.0");
+  doc.add("final_residual",1.4523e-13);
+  doc.add("time","4.893"); 
+ 
+//note: the following line will remove the data (4.890) associated with "time"
+  doc.get("time")->add("total",4.243);
+
+//note:  the following line will likewise remove the data (1.243) associated with "time"
+  doc.get("time")->get("total")->add("time",2.457);
+  doc.get("time")->get("total")->add("flops",4.88e5);
+  doc.get("time")->add("ddot",1.243);
+  doc.get("time")->add("sparsemv","");
+  doc.get("time")->get("sparsemv")->add("time",0.3445);
+  doc.get("time")->get("sparsemv")->add("overhead","");
+  doc.get("time")->get("sparsemv")->get("overhead")->add("time",0.0123);
+  doc.get("time")->get("sparsemv")->get("overhead")->add("percentage",0.034);
+  cout << doc.generateYAML() << endl; 
+  return 0;
+
+\endcode
+
+Below is the output generated by the above code:
+
+\verbatim
+
+final_residual: 1.4523e-13
+time: 
+  total:
+    time: 2.457
+    flops: 4.88e5
+  ddot: 1.243
+  sparsemv:
+    time: 0.3445
+    overhead:
+      time: 0.0123
+      percentage: 0.034
+
+\endverbatim
+
+\note {No value is allowed to be attached to a key that has children.  If children are added to a key, the value is simply set to "".}
+
+*/
+class YAML_Doc: public YAML_Element {
+  public:
+  //! Constructor: accepts mini-application name and version as strings, optionally accepts directory and file name for printing results.
+  /*!
+    The sole constructor for this class accepts and name and version number for the mini-application as well as optional directory 
+    and file name information for results that are generated by the generateYAML() method.
+    \param miniApp_Name (in) string containing name of the mini-application
+    \param miniApp_Version (in) string containing the version of the mini-application
+    \param destination_Directory (in, optional) path of diretory where results file will be stored, relative to current working directory. 
+           If this value is not supplied, the results file will be stored in the current working directory.  If the directory does not exist
+     it will be created.
+    \param destination_FileName (in, optional) root name of the results file.  A suffix of ".yaml" will be automatically appended.  If no
+           file name is specified the filename will be constructed by concatenating the miniAppName + miniAppVersion + ".yaml" strings.
+  */
+  YAML_Doc(const std::string& miniApp_Name, const std::string& miniApp_Version, const std::string& destination_Directory = "", const std::string& destination_FileName = "");
+  //! Destructor
+  ~YAML_Doc();
+  //! Generate YAML results to standard out and to a file using specified directory and filename, using current directory and miniAppName + miniAppVersion + ".yaml" by default
+  std::string generateYAML();
+
+protected:
+  std::string miniAppName;
+  std::string miniAppVersion;
+  std::string destinationDirectory;
+  std::string destinationFileName;
+};
+#endif /* YAML_DOC_H */
+

MultiSource/Benchmarks/DOE-ProxyApps-C++/miniXyce/YAML_Doc.cpp

+#include <ctime>
+#include <cstdlib>
+#include <ctime>
+#include <vector>
+#include <iostream>
+#include <fstream>
+#include <sstream>
+#ifdef REDSTORM
+#include <time.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#endif
+#include "YAML_Doc.hpp"
+using namespace std;
+
+//set the microapp_name and version which will become part of the YAML doc.
+YAML_Doc::YAML_Doc(const std::string& miniApp_Name, const std::string& miniApp_Version, const std::string& destination_Directory, const std::string& destination_FileName){
+  miniAppName = miniApp_Name;
+  miniAppVersion = miniApp_Version;
+  destinationDirectory = destination_Directory;
+  destinationFileName = destination_FileName;
+}
+
+//inherits the destructor from YAML_Element
+YAML_Doc::~YAML_Doc(void){
+}
+
+/*
+* generates YAML from the elements of the document and saves it
+* to a file
+*/
+string YAML_Doc::generateYAML(){
+  string yaml;
+  yaml =  yaml + "Mini-Application Name: " + miniAppName + "\n";
+  yaml =  yaml + "Mini-Application Version: " + miniAppVersion + "\n";
+  for(size_t i=0; i<children.size(); i++){
+    yaml = yaml + children[i]->printYAML("");
+  }
+  
+  time_t rawtime;
+  tm * ptm;
+  time ( &rawtime );
+  ptm = localtime(&rawtime);
+  char sdate[25];
+  //use tm_mon+1 because tm_mon is 0 .. 11 instead of 1 .. 12
+  sprintf (sdate,"%04d:%02d:%02d-%02d:%02d:%02d",ptm->tm_year + 1900, ptm->tm_mon+1,
+    ptm->tm_mday, ptm->tm_hour, ptm->tm_min,ptm->tm_sec);
+
+  string filename;
+  if (destinationFileName=="") 
+    filename = miniAppName + "-" + miniAppVersion + "_";
+  else 
+    filename = destinationFileName;
+  filename = filename + string(sdate) + ".yaml";
+  if (destinationDirectory!="" && destinationDirectory!=".") {
+    string mkdir_cmd = "mkdir " + destinationDirectory;
+#ifdef REDSTORM
+    mkdir(destinationDirectory.c_str(),0755);
+#else
+    system(mkdir_cmd.c_str());
+#endif
+    filename = destinationDirectory + "/" + destinationFileName;
+  }
+  else 
+    filename = "./" + filename;
+
+#ifdef GENERATE_YAML
+  ofstream myfile;
+  myfile.open(filename.c_str());
+  myfile << yaml;
+  myfile.close();
+#endif
+  return yaml;
+}
+
+

MultiSource/Benchmarks/DOE-ProxyApps-C++/miniXyce/YAML_Element.hpp

+//@HEADER
+// ************************************************************************
+// 
+//               Mantevo: A collection of mini-applications for HPC
+//                 Copyright (2008) Sandia Corporation
+// 
+// Under terms of Contract DE-AC04-94AL85000, there is a non-exclusive
+// license for use of this work by or on behalf of the U.S. Government.
+// 
+// This library is free software; you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as
+// published by the Free Software Foundation; either version 2.1 of the
+// License, or (at your option) any later version.
+//  
+// This library is distributed in the hope that it will be useful, but
+// WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+// Lesser General Public License for more details.
+//  
+// You should have received a copy of the GNU Lesser General Public
+// License along with this library; if not, write to the Free Software
+// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+// USA
+// Questions? Contact Michael A. Heroux ([e-mail]) 
+// 
+// ************************************************************************
+//@HEADER
+
+// Changelog
+//
+// Version 0.1
+// - Initial version.
+//
+/////////////////////////////////////////////////////////////////////////
+
+#ifndef YAML_ELEMENT_H
+#define YAML_ELEMENT_H
+#include <string>
+#include <vector>
+//! The Mantevo YAML_Element class for registering key-value pairs of performance data
+
+/*!
+  Mantevo mini-applications generate a collection of performance data for each run of the executable.  YAML_Element, and
+  the related YAML_Doc class, provide a uniform facility for gathering and reporting this data using the YAML text format.
+*/
+class YAML_Element {
+  public:
+
+  //! Default constructor.
+  YAML_Element (){key="";value="";}
+  //! Construct with known key-value pair
+  YAML_Element (const std::string& key_arg, const std::string& value_arg);
+  //! Destructor
+  ~YAML_Element ();
+  //! Key accessor method
+  std::string getKey(){return key;}
+  //! Add a child element to an element list associated with this element, value of type double
+  YAML_Element* add(const std::string& key_arg, double value_arg);
+  //! Add a child element to an element list associated with this element, value of type int
+  YAML_Element* add(const std::string& key_arg, int value_arg);
+#ifndef MINIFE_NO_LONG_LONG
+  //! Add a child element to an element list associated with this element, value of type long long
+  YAML_Element* add(const std::string& key_arg, long long value_arg);
+#endif
+  //! Add a child element to an element list associated with this element, value of type size_t
+  YAML_Element* add(const std::string& key_arg, size_t value_arg);
+  //! Add a child element to an element list associated with this element, value of type string
+  YAML_Element* add(const std::string& key_arg, const std::string& value_arg);
+  //! get the element in the list with the given key
+  YAML_Element* get(const std::string& key_arg);
+  std::string printYAML(std::string space);
+  
+protected:
+  std::string key;
+  std::string value;
+  std::vector<YAML_Element*> children;
+
+private:
+  std::string convert_double_to_string(double value_arg);
+  std::string convert_int_to_string(int value_arg);
+#ifndef MINIFE_NO_LONG_LONG
+  std::string convert_long_long_to_string(long long value_arg);
+#endif
+  std::string convert_size_t_to_string(size_t value_arg);
+};
+#endif /* YAML_ELEMENT_H */

MultiSource/Benchmarks/DOE-ProxyApps-C++/miniXyce/YAML_Element.cpp

+#include <vector>
+#include <iostream>
+#include <fstream>
+#include <sstream>
+#include "YAML_Element.hpp"
+using namespace std;
+YAML_Element::YAML_Element(const std::string& key_arg, const std::string& value_arg){
+  key = key_arg;
+  value = value_arg;
+}
+
+YAML_Element::~YAML_Element(){
+  for (size_t i=0; i<children.size(); i++) {
+    delete children[i];
+  }
+  children.clear();
+}
+
+/*
+* Add an element to the vector
+* QUESTION: if an element is not added because the key already exists,
+* will this lead to memory leakage?
+*/
+YAML_Element* YAML_Element::add(const std::string& key_arg, double value_arg) {
+  this->value = "";
+  string converted_value = convert_double_to_string(value_arg);
+  YAML_Element* element = new YAML_Element(key_arg,converted_value);
+  children.push_back(element);
+  return element;
+}
+
+YAML_Element* YAML_Element::add(const std::string& key_arg, int value_arg) {
+  this->value = "";
+  string converted_value = convert_int_to_string(value_arg);
+  YAML_Element* element = new YAML_Element(key_arg,converted_value);
+  children.push_back(element);
+  return element;
+}
+
+#ifndef MINIFE_NO_LONG_LONG
+
+YAML_Element* YAML_Element::add(const std::string& key_arg, long long value_arg) {
+  this->value = "";
+  string converted_value = convert_long_long_to_string(value_arg);
+  YAML_Element* element = new YAML_Element(key_arg,converted_value);
+  children.push_back(element);
+  return element;
+}
+
+#endif
+
+YAML_Element* YAML_Element::add(const std::string& key_arg, size_t value_arg) {
+  this->value = "";
+  string converted_value = convert_size_t_to_string(value_arg);
+  YAML_Element* element = new YAML_Element(key_arg,converted_value);
+  children.push_back(element);
+  return element;
+}
+
+YAML_Element* YAML_Element::add(const std::string& key_arg, const std::string& value_arg) {
+  this->value = "";
+  YAML_Element* element = new YAML_Element(key_arg, value_arg);
+  children.push_back(element);
+  return element;
+}
+
+/*
+* returns pointer to the YAML_Element for the given key.
+* I, cam, believe an exception should be thrown if there is no
+* element in the vector for the specified key
+*/
+YAML_Element* YAML_Element::get(const std::string& key_arg) {
+  for (size_t i=0; i<children.size(); i++) {
+    if(children[i]->getKey() == key_arg){
+      return children[i];
+    }
+  }
+  return 0;
+}
+
+/*
+* prints a line of a YAML document.  Correct YAML depends on
+* correct spacing; the parameter space should be the proper
+* amount of space for the parent element
+*/
+string YAML_Element::printYAML(std::string space){
+  string yaml_line = space + key + ": " + value + "\n";
+  for(int i=0; i<2; i++) space = space + " ";
+  for(size_t i=0; i<children.size(); i++){
+    yaml_line = yaml_line + children[i]->printYAML(space);
+  }
+  return yaml_line;
+}
+
+string YAML_Element::convert_double_to_string(double value_arg){
+  stringstream strm;
+  strm << value_arg;
+  return strm.str();
+}
+string YAML_Element::convert_int_to_string(int value_arg){
+  stringstream strm;
+  strm << value_arg;
+  return strm.str();
+}
+
+#ifndef MINIFE_NO_LONG_LONG
+
+string YAML_Element::convert_long_long_to_string(long long value_arg){
+  stringstream strm;
+  strm << value_arg;
+  return strm.str();
+}
+
+#endif
+
+string YAML_Element::convert_size_t_to_string(size_t value_arg){
+  stringstream strm;
+  strm << value_arg;
+  return strm.str();
+}

MultiSource/Benchmarks/DOE-ProxyApps-C++/miniXyce/cir5.net

+% simple circuit 5
+% includes all the components {R,L,C,V,I}
+
+R1 1 2 1
+R2 2 0 2
+L1 2 3 1e-6
+C1 3 0 1e-6
+
+V1 1 0 SINE 0 5 1e6 0
+I1 2 0 PULSE 0 2 1e-6 0 1e-6 0 1e-6
+
+
+
+

MultiSource/Benchmarks/DOE-ProxyApps-C++/miniXyce/last_used_params.txt

+circuit = tests/cir5.net
+t_start = 1e-06
+t_step = 1e-08
+t_stop = 1e-05
+tol = 1e-06
+k = 10

MultiSource/Benchmarks/DOE-ProxyApps-C++/miniXyce/mX_linear_DAE.h

+#ifndef __MX_LINEAR_DAE_H__
+#define __MX_LINEAR_DAE_H__
+
+//@HEADER
+// ************************************************************************
+// 
+//               miniXyce: A simple circuit simulation benchmark code
+//                 Copyright (2011) Sandia Corporation
+// 
+// Under terms of Contract DE-AC04-94AL85000, there is a non-exclusive
+// license for use of this work by or on behalf of the U.S. Government.
+// 
+// This library is free software; you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as
+// published by the Free Software Foundation; either version 2.1 of the
+// License, or (at your option) any later version.
+//  
+// This library is distributed in the hope that it will be useful, but
+// WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+// Lesser General Public License for more details.
+//  
+// You should have received a copy of the GNU Lesser General Public
+// License along with this library; if not, write to the Free Software
+// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+// USA
+// Questions? Contact Michael A. Heroux ([e-mail]) 
+// 
+// ************************************************************************
+//@HEADER
+
+// Author : Karthik V Aadithya
+// Mentor : Heidi K Thornquist
+// Date : July 2010
+
+#include <list>
+#include <vector>
+#include "mX_source.h"
+#include "mX_sparse_matrix.h"
+
+using namespace mX_source_utils;
+using namespace mX_matrix_utils;
+
+namespace mX_linear_DAE_utils
+{
+  struct mX_linear_DAE_RHS_entry
+  {
+    // a single entry in the RHS of a DAE of the form "A x + B x_dot = b(t)"
+      // each entry in b(t) is a list of scaled sources
+        // linear combination of multiple voltage/current sources
+
+    std::list<mX_scaled_source*> scaled_src_list;
+  };
+
+  struct mX_linear_DAE
+  {
+    // a linear DAE is of the form "A x + B x_dot = b(t)"
+      // A and B are distributed sparse matrices
+      // b(t) is a time-varying function
+        // every entry of which is a linear combination of voltage/current sources
+
+    distributed_sparse_matrix* A;
+    distributed_sparse_matrix* B;
+    std::vector<mX_linear_DAE_RHS_entry*> b;
+  };
+
+  std::vector<double> evaluate_b(double t, mX_linear_DAE* dae);
+        void destroy(mX_linear_DAE* dae);
+        void destroy_RHS(mX_linear_DAE_RHS_entry* entry);
+}
+
+#endif

MultiSource/Benchmarks/DOE-ProxyApps-C++/miniXyce/mX_linear_DAE.cpp

+//@HEADER
+// ************************************************************************
+// 
+//               miniXyce: A simple circuit simulation benchmark code
+//                 Copyright (2011) Sandia Corporation
+// 
+// Under terms of Contract DE-AC04-94AL85000, there is a non-exclusive
+// license for use of this work by or on behalf of the U.S. Government.
+// 
+// This library is free software; you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as
+// published by the Free Software Foundation; either version 2.1 of the
+// License, or (at your option) any later version.
+//  
+// This library is distributed in the hope that it will be useful, but
+// WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+// Lesser General Public License for more details.
+//  
+// You should have received a copy of the GNU Lesser General Public
+// License along with this library; if not, write to the Free Software
+// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+// USA
+// Questions? Contact Michael A. Heroux ([e-mail]) 
+// 
+// ************************************************************************
+//@HEADER
+
+// Author : Karthik V Aadithya
+// Mentor : Heidi K Thornquist
+// Date : July 2010
+
+#include "mX_linear_DAE.h"
+
+using namespace mX_source_utils;
+using namespace mX_matrix_utils;
+using namespace mX_linear_DAE_utils;
+
+std::vector<double> mX_linear_DAE_utils::evaluate_b(double t, mX_linear_DAE* dae)
+{
+  // given a linear DAE "A x + B x_dot = b(t)"
+  // and a particular time point t
+  // this function computes and returns the vector b at that time point t
+
+  std::vector<mX_linear_DAE_RHS_entry*>::iterator it2;
+  std::vector<double> result;
+
+  for (it2 = dae->b.begin(); it2 != dae->b.end(); it2++)
+  {
+    if (!(*it2))
+    {
+      result.push_back(0);
+    }
+    else
+    {
+      double sum = (double)(0);
+      std::list<mX_scaled_source*>::iterator it3;
+
+      for(it3 = (*it2)->scaled_src_list.begin(); it3 != (*it2)->scaled_src_list.end(); it3++)
+      {
+  mX_source* src = (*it3)->src;
+  sum += src->output((double)(t)) * ((*it3)->scale);
+      }
+
+      result.push_back(sum);
+    }
+  }
+  return result;
+}
+
+void mX_linear_DAE_utils::destroy_RHS(mX_linear_DAE_RHS_entry* entry)
+{
+  if (entry)
+  {
+    // delete send_instructions
+    while (!entry->scaled_src_list.empty())
+    {
+      mX_scaled_source* curr = entry->scaled_src_list.front();
+      if (curr) delete curr->src; delete curr; curr=0;
+      entry->scaled_src_list.pop_front();
+    }
+    delete entry; entry=0;
+  }
+}
+
+void mX_linear_DAE_utils::destroy(mX_linear_DAE* dae)
+{
+  // Destroy A
+  destroy_matrix( dae->A );
+
+  // Destroy B
+  destroy_matrix( dae->B );
+
+  // Destroy b
+  for (int i=0; i<dae->b.size(); ++i)
+  {
+    destroy_RHS( dae->b[i] );
+  }
+  dae->b.resize(0);
+
+  delete dae; dae=0;
+}

MultiSource/Benchmarks/DOE-ProxyApps-C++/miniXyce/mX_linear_ckt_simulator.cpp

+//@HEADER
+// ************************************************************************
+// 
+//               miniXyce: A simple circuit simulation benchmark code
+//                 Copyright (2011) Sandia Corporation
+// 
+// Under terms of Contract DE-AC04-94AL85000, there is a non-exclusive
+// license for use of this work by or on behalf of the U.S. Government.
+// 
+// This library is free software; you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as
+// published by the Free Software Foundation; either version 2.1 of the
+// License, or (at your option) any later version.
+//  
+// This library is distributed in the hope that it will be useful, but
+// WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+// Lesser General Public License for more details.
+//  
+// You should have received a copy of the GNU Lesser General Public
+// License along with this library; if not, write to the Free Software
+// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+// USA
+// Questions? Contact Michael A. Heroux ([e-mail]) 
+// 
+// ************************************************************************
+//@HEADER
+
+// Author : Karthik V Aadithya
+// Mentor : Heidi K Thornquist
+// Date : July 2010
+
+#include <string>
+#include <sstream>
+#include <fstream>
+#include <iostream>
+#include <iomanip>
+#include <vector>
+#include "mX_parser.h"
+#include "mX_source.h"
+#include "mX_sparse_matrix.h"
+#include "mX_linear_DAE.h"
+#include "mX_parms.h"
+#include "mX_timer.h"
+#include "YAML_Element.hpp"
+#include "YAML_Doc.hpp"
+
+
+#ifdef HAVE_MPI
+#include "mpi.h"
+#endif
+
+using namespace mX_parse_utils;
+using namespace mX_source_utils;
+using namespace mX_linear_DAE_utils;
+using namespace mX_parms_utils;
+
+int main(int argc, char* argv[])
+{
+  // this is of course, the actual transient simulator
+  int p=1, pid=0;
+#ifdef HAVE_MPI 
+  MPI_Init(&argc,&argv);
+  
+  MPI_Comm_size(MPI_COMM_WORLD, &p);
+  MPI_Comm_rank(MPI_COMM_WORLD, &pid);
+#endif
+        double sim_start = mX_timer();
+
+        // initialize YAML doc
+        YAML_Doc doc("miniXyce","1.0");
+
+  // initialize the simulation parameters
+
+  std::string ckt_netlist_filename;
+  double t_start, t_step, t_stop, tol, res;
+  int k, iters, restarts;
+
+  std::vector<double> x;
+  bool init_cond_specified;
+
+        double tstart = mX_timer();
+  get_parms(argc,argv,ckt_netlist_filename,t_start,t_step,t_stop,tol,k,x,init_cond_specified,p,pid);
+        double tend = mX_timer() - tstart;
+#ifdef TIMING
+        doc.add("Parameter_parsing_time",tend);
+#endif
+
+  // build the DAE from the circuit netlist
+
+        tstart = mX_timer();
+  int total_devices, total_unknowns, num_internal_nodes;
+        std::map<std::string, int> device_count;
+
+  mX_linear_DAE* dae = parse_netlist(ckt_netlist_filename,p,pid,total_devices,total_unknowns,num_internal_nodes,device_count);
+
+        tend = mX_timer() - tstart;
+#ifdef TIMING
+        doc.add("Netlist_parsing_time",tend);
+#endif
+
+        // document circuit and matrix attributes
+
+#ifdef PRINT_FILE 
+        doc.add("Netlist_file",ckt_netlist_filename.c_str());
+#endif
+
+        doc.add("Circuit_attributes","");
+        doc.get("Circuit_attributes")->add("Number_of_devices",total_devices);
+        if (device_count.find("R") != device_count.end())
+        {
+          int num_resistors = device_count["R"];
+          doc.get("Circuit_attributes")->add("Resistors_(R)",num_resistors);
+        }
+        if (device_count.find("L") != device_count.end())
+        {
+          int num_inductors = device_count["L"];
+          doc.get("Circuit_attributes")->add("Inductors_(L)",num_inductors);
+        }
+        if (device_count.find("C") != device_count.end())
+        {
+          int num_capacitors = device_count["C"];
+          doc.get("Circuit_attributes")->add("Capacitors_(C)",num_capacitors);
+        }
+        if (device_count.find("V") != device_count.end())
+        {
+          int num_voltage_sources = device_count["V"];
+          doc.get("Circuit_attributes")->add("Voltage_sources_(V)",num_voltage_sources);
+        }
+        if (device_count.find("I") != device_count.end())
+        {
+          int num_current_sources = device_count["I"];
+          doc.get("Circuit_attributes")->add("Current_sources_(I)",num_current_sources);
+        }
+
+        int num_my_rows = dae->A->end_row - dae->A->start_row + 1;
+        int num_my_nnz = dae->A->local_nnz, sum_nnz = dae->A->local_nnz;
+        int min_nnz = num_my_nnz, max_nnz = num_my_nnz;
+        int min_rows = num_my_rows, max_rows = num_my_rows, sum_rows = num_my_rows;
+
+        for (int proc = 0; proc < p; proc++)
+        {
+          if (pid == proc)
+          {
+            int num_my_sends = dae->A->send_instructions.size();
+            std::cout << "Processor : " << pid << " has " << num_my_sends << " sends: ";
+            std::list<data_transfer_instruction*>::iterator it1;
+            for (it1 = dae->A->send_instructions.begin(); it1 != dae->A->send_instructions.end(); it1++)
+            {
+              std::cout << " ( " << (*it1)->pid << " ) " << (*it1)->indices.size() << " ";
+            }
+            std::cout << std::endl;
+          }
+        }
+
+#ifdef HAVE_MPI
+        MPI_Allreduce(&num_my_nnz,&sum_nnz,1,MPI_INT,MPI_SUM,MPI_COMM_WORLD);
+        MPI_Allreduce(&num_my_nnz,&min_nnz,1,MPI_INT,MPI_MIN,MPI_COMM_WORLD);
+        MPI_Allreduce(&num_my_nnz,&max_nnz,1,MPI_INT,MPI_MAX,MPI_COMM_WORLD);
+        MPI_Allreduce(&num_my_rows,&sum_rows,1,MPI_INT,MPI_SUM,MPI_COMM_WORLD);
+        MPI_Allreduce(&num_my_rows,&min_rows,1,MPI_INT,MPI_MIN,MPI_COMM_WORLD);
+        MPI_Allreduce(&num_my_rows,&max_rows,1,MPI_INT,MPI_MAX,MPI_COMM_WORLD);
+#endif
+
+        doc.add("Matrix_attributes","");
+        doc.get("Matrix_attributes")->add("Global_rows",sum_rows);
+        doc.get("Matrix_attributes")->add("Rows_per_proc_MIN",min_rows);
+        doc.get("Matrix_attributes")->add("Rows_per_proc_MAX",max_rows);
+        doc.get("Matrix_attributes")->add("Rows_per_proc_AVG",(double)sum_rows/p);
+        doc.get("Matrix_attributes")->add("Global_NNZ",sum_nnz);
+        doc.get("Matrix_attributes")->add("NNZ_per_proc_MIN",min_nnz);
+        doc.get("Matrix_attributes")->add("NNZ_per_proc_MAX",max_nnz);
+        doc.get("Matrix_attributes")->add("NNZ_per_proc_AVG",(double)sum_nnz/p);
+        
+  // compute the initial condition if not specified by user
+
+  int start_row = dae->A->start_row;
+  int end_row = dae->A->end_row;
+        tstart = mX_timer();
+        
+  if (!init_cond_specified)
+  {
+    std::vector<double> init_cond_guess;
+
+    for (int i = 0; i<num_my_rows; i++)
+    {
+      init_cond_guess.push_back((double)(0));
+    }
+
+    std::vector<double> init_RHS = evaluate_b(t_start,dae);
+
+    gmres(dae->A,init_RHS,init_cond_guess,tol,res,k,x,iters,restarts);
+
+                doc.add("DCOP Calculation","");
+                doc.get("DCOP Calculation")->add("Init_cond_specified", false);
+                doc.get("DCOP Calculation")->add("GMRES_tolerance",tol);
+                doc.get("DCOP Calculation")->add("GMRES_subspace_dim",k);
+                doc.get("DCOP Calculation")->add("GMRES_iterations",iters);
+                doc.get("DCOP Calculation")->add("GMRES_restarts",restarts);
+                doc.get("DCOP Calculation")->add("GMRES_native_residual",res);
+  }
+        else 
+        {
+                doc.add("DCOP Calculation","");
+                doc.get("DCOP Calculation")->add("Init_cond_specified", true);
+        }
+        tend = mX_timer() - tstart;
+
+#ifdef TIMING
+        doc.get("DCOP Calculation")->add("DCOP_calculation_time",tend);
+#endif
+
+  // write the headers and computed initial condition to file
+
+        tstart = mX_timer();
+  int dot_position = ckt_netlist_filename.find_first_of('.');
+
+#ifdef OUT_FILE
+  std::string out_filename = ckt_netlist_filename.substr(0,dot_position) + "_tran_results.prn";
+  std::ofstream* outfile=0;
+#endif
+
+#ifdef HAVE_MPI 
+        // Prepare rcounts and displs for a contiguous gather of the full solution vector.
+        std::vector<int> rcounts( p, 0 ), displs( p, 0 );
+        MPI_Gather(&num_my_rows, 1, MPI_INT, &rcounts[0], 1, MPI_INT, 0, MPI_COMM_WORLD);
+        for (int i=1; i<p ; i++) displs[i] = displs[i-1] + rcounts[i-1];
+
+        std::vector<double> fullX( sum_rows, 0.0 ); 
+        MPI_Gatherv(&x[0], num_my_rows, MPI_DOUBLE, &fullX[0], &rcounts[0], &displs[0], MPI_DOUBLE, 0, MPI_COMM_WORLD);
+#endif
+
+#ifdef OUT_FILE
+        if (pid == 0)
+        {
+            outfile = new std::ofstream(out_filename.data(), std::ios::out);
+  
+          *outfile << std::setw(18) << "TIME";
+
+      for (int i = 0; i < sum_rows; i++)
+      {
+              std::stringstream ss2;
+    if (i < num_internal_nodes)
+                        ss2 << "V" << i+1;
+    else
+                        ss2 << "I" << i+1-num_internal_nodes;
+         *outfile << std::setw(18) << ss2.str();
+      }
+
+       *outfile << std::setw(20) << "num_GMRES_iters" << std::setw(20) << "num_GMRES_restarts" << std::endl;
+
+            outfile->precision(8);
+
+      *outfile << std::scientific << std::setw(18) << t_start;
+
+      for (int i = 0; i < sum_rows; i++)
+      {
+#ifdef HAVE_MPI
+    *outfile << std::setw(18) << fullX[i];
+#else
+                *outfile << std::setw(18) << x[i];
+#endif
+      }
+
+         *outfile << std::fixed << std::setw(20)  << iters << std::setw(20) << restarts << std::endl;
+  }
+#endif
+
+        double io_tend = mX_timer() - tstart;
+  
+  // from now you won't be needing any more Ax = b solves
+    // but you will be needing many (A + B/t_step)x = b solves
+    // so change A to (A + B/t_step) right now
+      // so you won't have to compute it at each time step
+
+        tstart = mX_timer();
+  distributed_sparse_matrix* A = dae->A;
+  distributed_sparse_matrix* B = dae->B;
+
+  std::vector<distributed_sparse_matrix_entry*>::iterator it1;
+  int row_idx = start_row - 1;
+
+  for (it1 = B->row_headers.begin(); it1 != B->row_headers.end(); it1++)
+  {
+    row_idx++;
+    distributed_sparse_matrix_entry* curr = *it1;
+
+    while (curr)
+    {
+      int col_idx = curr->column;
+      double value = (curr->value)/t_step;
+      
+      distributed_sparse_matrix_add_to(A,row_idx,col_idx,value,total_unknowns,p);
+
+      curr = curr->next_in_row;
+    }
+  }
+        double matrix_setup_tend = mX_timer() - tstart;
+
+  // this is where the actual transient simulation starts
+
+        tstart = mX_timer();
+  double t = t_start + t_step;
+        double total_gmres_res = 0.0;
+        int total_gmres_iters = 0;
+        int trans_steps = 0;
+
+  while (t < t_stop)
+  {
+                trans_steps++;
+
+    // new time point t => new value for b(t)
+
+    std::vector<double> b = evaluate_b(t,dae);
+
+    // build the linear system Ax = b that needs to be solved at this time point
+      // Backward Euler is used at every iteration
+
+    std::vector<double> RHS;
+    sparse_matrix_vector_product(B,x,RHS);
+
+    for (int i = 0; i < num_my_rows; i++)
+    {
+      RHS[i] /= t_step;
+      RHS[i] += b[i];
+    }
+
+    // now solve the linear system just built using GMRES(k)
+
+    gmres(A,RHS,x,tol,res,k,x,iters,restarts);
+                total_gmres_iters += iters;
+                total_gmres_res += res;
+
+    // write the results to file
+          double io_tstart = mX_timer();  
+#ifdef HAVE_MPI
+                MPI_Gatherv(&x[0], num_my_rows, MPI_DOUBLE, &fullX[0], &rcounts[0], &displs[0], MPI_DOUBLE, 0, MPI_COMM_WORLD);
+#endif
+#ifdef OUT_FILE
+                if (pid == 0)
+                {
+                  outfile->precision(8);
+                  *outfile << std::scientific << std::setw(18) << t;
+
+        for (int i = 0; i < sum_rows; i++)
+      {
+#ifdef HAVE_MPI
+            *outfile << std::setw(18) << fullX[i];
+#else
+                        *outfile << std::setw(18) << x[i];
+#endif
+      }
+
+        *outfile << std::fixed << std::setw(20)  << iters << std::setw(20) << restarts << std::endl;
+    }
+#endif
+
+                io_tend += (mX_timer() - io_tstart);
+
+    // increment t
+
+    t += t_step;
+
+  }
+
+#ifdef OUT_FILE 
+  // Hurray, the transient simulation is done!
+        if (pid ==0)
+        { 
+      outfile->close();
+          delete outfile;
+        }
+#endif        
+        // Document transient simulation performance
+
+        tend = mX_timer();
+        double sim_end = tend - sim_start;
+
+        doc.add("Transient Calculation","");
+        doc.get("Transient Calculation")->add("Number_of_time_steps", trans_steps);
+        doc.get("Transient Calculation")->add("Time_start", t_start);
+        doc.get("Transient Calculation")->add("Time_end", t_stop);
+        doc.get("Transient Calculation")->add("Time_step", t_step);
+        doc.get("Transient Calculation")->add("GMRES_tolerance",tol);
+        doc.get("Transient Calculation")->add("GMRES_subspace_dim",k);
+        doc.get("Transient Calculation")->add("GMRES_average_iters",total_gmres_iters/trans_steps);
+        doc.get("Transient Calculation")->add("GMRES_average_res",total_gmres_res/trans_steps);
+#ifdef TIMING
+        doc.get("Transient Calculation")->add("Matrix_setup_time",matrix_setup_tend);
+        doc.get("Transient Calculation")->add("Transient_calculation_time",tend-tstart);
+        doc.add("I/O File Time",io_tend);
+        doc.add("Total Simulation Time",sim_end);
+#endif
+
+
+        if (pid==0) { // Only PE 0 needs to compute and report timing results
+          std::string yaml = doc.generateYAML();
+          std::cout << yaml;
+        }
+
+        // Clean up
+        mX_linear_DAE_utils::destroy( dae );
+
+#ifdef HAVE_MPI 
+  MPI_Finalize();
+#endif
+
+  return 0;
+}

MultiSource/Benchmarks/DOE-ProxyApps-C++/miniXyce/mX_parms.h

+#ifndef __MX_PARMS_H__
+#define __MX_PARMS_H__
+
+//@HEADER
+// ************************************************************************
+// 
+//               miniXyce: A simple circuit simulation benchmark code
+//                 Copyright (2011) Sandia Corporation
+// 
+// Under terms of Contract DE-AC04-94AL85000, there is a non-exclusive
+// license for use of this work by or on behalf of the U.S. Government.
+// 
+// This library is free software; you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as
+// published by the Free Software Foundation; either version 2.1 of the
+// License, or (at your option) any later version.
+//  
+// This library is distributed in the hope that it will be useful, but
+// WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+// Lesser General Public License for more details.
+//  
+// You should have received a copy of the GNU Lesser General Public
+// License along with this library; if not, write to the Free Software
+// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+// USA
+// Questions? Contact Michael A. Heroux ([e-mail]) 
+// 
+// ************************************************************************
+//@HEADER
+
+// Author : Karthik V Aadithya
+// Mentor : Heidi K Thornquist
+// Date : July 2010
+
+#include <string>
+#include <set>
+#include <vector>
+
+namespace mX_parms_utils
+{
+  enum parm_names{CKT_FILENAME,T_START,T_STEP,T_STOP,TOL,K,INIT_COND,PARMS_FILE,PREV};
+
+  void parse_command_line(int argc, std::vector<std::string> &argv, std::string &ckt_filename, double &t_start, double &t_step, double &t_stop, double &tol, int &k, std::vector<double> &init_cond, std::string &parms_file, std::set<int> &specified_parms, int p, int pid);
+
+  std::vector<std::string> get_command_line_equivalent_from_file(std::string &filename);
+
+  void get_parms(int argc, char* argv[], std::string &ckt_filename, double &t_start, double &t_step, double &t_stop, double &tol, int &k, std::vector<double> &init_cond, bool &init_cond_specified, int p, int pid);
+}
+
+#endif

MultiSource/Benchmarks/DOE-ProxyApps-C++/miniXyce/mX_parms.cpp

+//@HEADER
+// ************************************************************************
+// 
+//               miniXyce: A simple circuit simulation benchmark code
+//                 Copyright (2011) Sandia Corporation
+// 
+// Under terms of Contract DE-AC04-94AL85000, there is a non-exclusive
+// license for use of this work by or on behalf of the U.S. Government.
+// 
+// This library is free software; you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as
+// published by the Free Software Foundation; either version 2.1 of the
+// License, or (at your option) any later version.
+//  
+// This library is distributed in the hope that it will be useful, but
+// WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+// Lesser General Public License for more details.
+//  
+// You should have received a copy of the GNU Lesser General Public
+// License along with this library; if not, write to the Free Software
+// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+// USA
+// Questions? Contact Michael A. Heroux ([e-mail]) 
+// 
+// ************************************************************************
+//@HEADER
+
+// Author : Karthik V Aadithya
+// Mentor : Heidi K Thornquist
+// Date : July 2010
+
+#include <string>
+#include <set>
+#include <vector>
+#include <cstdlib>
+#include <fstream>
+#include "mX_parms.h"
+#include <cassert>
+
+using namespace mX_parms_utils;
+
+void mX_parms_utils::parse_command_line(int argc, std::vector<std::string> &argv, std::string &ckt_filename, double &t_start, double &t_step, double &t_stop, double &tol, int &k, std::vector<double> &init_cond, std::string &parms_file, std::set<int> &specified_parms, int p, int pid)
+{
+  // take a command-line-ish argc and argv
+  // parse it to get the parms specified there
+      // change those parms if they haven't been already specified earlier
+
+  int i = 1;
+
+  while (i < argc)
+  {
+    // got to read in a new parameter and its value
+
+    std::string parm_name = argv[i];
+    i++;
+
+    int first_non_minus_index = parm_name.find_first_not_of('-');
+    assert(first_non_minus_index != 0);
+    parm_name = parm_name.substr(first_non_minus_index);
+
+    if ((parm_name == "c") || (parm_name == "circuit"))
+      {
+      if (specified_parms.find(CKT_FILENAME) == specified_parms.end())
+      {
+        specified_parms.insert(CKT_FILENAME);
+        ckt_filename = argv[i];
+      }
+      
+      i++;
+    }
+
+    if ((parm_name == "ti") || (parm_name == "t_start") || (parm_name == "tstart"))
+    {
+      if (specified_parms.find(T_START) == specified_parms.end())
+      {
+        specified_parms.insert(T_START);
+        t_start = atof(argv[i].data());
+      }
+      
+      i++;
+    }
+
+    if ((parm_name == "tf") || (parm_name == "t_stop") || (parm_name == "tstop"))
+    {
+      if (specified_parms.find(T_STOP) == specified_parms.end())
+      {
+        specified_parms.insert(T_STOP);
+        t_stop = atof(argv[i].data());
+      }
+      
+      i++;
+    }
+
+    if ((parm_name == "h") || (parm_name == "t_step") || (parm_name == "tstep"))
+    {
+      if (specified_parms.find(T_STEP) == specified_parms.end())
+      {
+        specified_parms.insert(T_STEP);
+        t_step = atof(argv[i].data());
+      }
+      
+      i++;
+    }
+
+    if ((parm_name == "tol") || (parm_name == "tolerance"))
+    {
+      if (specified_parms.find(TOL) == specified_parms.end())
+      {
+        specified_parms.insert(TOL);
+        tol = atof(argv[i].data());
+      }
+      
+      i++;
+    }
+
+    if ((parm_name == "k") || (parm_name == "restart"))
+    {
+      if (specified_parms.find(K) == specified_parms.end())
+      {
+        specified_parms.insert(K);
+        k = atoi(argv[i].data());
+      }
+
+      i++;
+    }
+
+    if ((parm_name == "pf") || (parm_name == "paramsfile") || (parm_name == "params_file"))
+    {
+      if (specified_parms.find(PARMS_FILE) == specified_parms.end())
+      {
+        specified_parms.insert(PARMS_FILE);
+        parms_file = argv[i];
+      }
+      
+      i++;
+    }
+
+    if (parm_name == "prev")
+    {
+      specified_parms.insert(PREV);
+    }
+
+    if ((parm_name == "i") || (parm_name == "init") || (parm_name == "initcond") || (parm_name == "init_cond") || (parm_name == "x0"))
+    {
+      std::vector<std::string> init;
+      int n = 0;
+
+      while (i < argc)
+      {
+        std::string next_arg = argv[i];
+
+        if (next_arg[0] == '-')
+        {
+          break;
+        }
+
+        init.push_back(next_arg);
+        i++; n++;
+      }
+
+      if (specified_parms.find(INIT_COND) == specified_parms.end())
+      {
+        specified_parms.insert(INIT_COND);
+        
+        int start_row = (n/p)*(pid) + ((pid < n%p) ? pid : n%p);
+        int end_row = start_row + (n/p) - 1 + ((pid < n%p) ? 1 : 0);
+  
+        for (int j = start_row; j <= end_row; j++)
+        {
+          if (j-start_row < init_cond.size())
+          {
+            init_cond[j-start_row] = atof(init[j].data());
+          }
+  
+          else
+          {
+            init_cond.push_back(atof(init[j].data()));
+          }
+        }
+      }
+    }
+  }
+}
+
+std::vector<std::string> mX_parms_utils::get_command_line_equivalent_from_file(std::string &filename)
+{
+  // read a file that has parameters stored in it
+    // construct a command-line-ish sequence from those parameters
+
+  std::ifstream infile;
+  infile.open(filename.data());
+
+  if (infile.fail())
+  {
+    std::vector<std::string> v;
+    return v;
+  }
+
+  std::vector<std::string> command_line_equivalent;
+  command_line_equivalent.push_back("ignore");
+
+  while (!infile.eof())
+  {
+    std::string curr_line;
+    getline(infile,curr_line);
+
+    if ((curr_line.length() == 0) || (curr_line[0] == '%'))
+    {
+      continue;
+    }
+
+    std::string LHS,RHS;
+
+    int equals_idx = curr_line.find_first_of('=');
+    LHS = curr_line.substr(0,equals_idx);
+    RHS = curr_line.substr(equals_idx+1);
+
+    int first_significant_index = LHS.find_first_not_of(" \t");
+    int last_significant_index = LHS.find_last_not_of(" \t");
+    LHS = LHS.substr(first_significant_index, last_significant_index-first_significant_index+1);
+
+    first_significant_index = RHS.find_first_not_of(" \t");
+    last_significant_index = RHS.find_last_not_of(" \t");
+    RHS = RHS.substr(first_significant_index, last_significant_index-first_significant_index+1);
+
+    command_line_equivalent.push_back("--" + LHS);
+    command_line_equivalent.push_back(RHS);
+  }
+
+  infile.close();
+
+  return command_line_equivalent;
+}
+
+void mX_parms_utils::get_parms(int argc, char* argv[], std::string &ckt_filename, double &t_start, double &t_step, double &t_stop, double &tol, int &k, std::vector<double> &init_cond, bool &init_cond_specified, int p, int pid)
+{
+  // get all the required simulation parameters
+    // first go through command line options, they get first priority
+    // after looking through the command line
+      // if the command line options specify a file
+        // open the file and read the parameters stored there
+    // then, if the command line options specify --prev
+      // read parameters from last_used_params.txt
+    // then read remaining parameters from default_params.txt
+
+  std::set<int> specified_parms;
+  std::string parms_file;
+
+  std::vector<std::string> argv_strings;
+
+  for (int i = 0; i < argc; i++)
+  {
+    argv_strings.push_back((std::string)(argv[i]));
+  }
+
+  parse_command_line(argc, argv_strings, ckt_filename, t_start, t_step, t_stop, tol, k, init_cond, parms_file, specified_parms, p, pid);
+
+  // scanned all command line parameters
+    // now scan any files that are necessary
+
+  if (specified_parms.find(PARMS_FILE) != specified_parms.end())
+  {
+    argv_strings = get_command_line_equivalent_from_file(parms_file);
+    parse_command_line(argv_strings.size(), argv_strings, ckt_filename, t_start, t_step, t_stop, tol, k, init_cond, parms_file, specified_parms, p, pid);
+  }
+
+  if (specified_parms.find(PREV) != specified_parms.end())
+  {
+    std::string filename = "last_used_params.txt";
+    argv_strings = get_command_line_equivalent_from_file(filename);
+    parse_command_line(argv_strings.size(), argv_strings, ckt_filename, t_start, t_step, t_stop, tol, k, init_cond, parms_file, specified_parms, p, pid);
+  }
+
+  std::string filename = "default_params.txt";
+  argv_strings = get_command_line_equivalent_from_file(filename);
+  parse_command_line(argv_strings.size(), argv_strings, ckt_filename, t_start, t_step, t_stop, tol, k, init_cond, parms_file, specified_parms, p, pid);
+  
+  // by now all parms must have been obtained
+
+  assert(specified_parms.find(CKT_FILENAME) != specified_parms.end());
+  assert(specified_parms.find(T_START) != specified_parms.end());
+  assert(specified_parms.find(T_STEP) != specified_parms.end());
+  assert(specified_parms.find(T_STOP) != specified_parms.end());
+  assert(specified_parms.find(TOL) != specified_parms.end());
+  assert(specified_parms.find(K) != specified_parms.end());
+
+  init_cond_specified = (specified_parms.find(INIT_COND) != specified_parms.end());
+
+  // remember to update the "last_used_params.txt" file
+
+
+#ifdef LAST_PARAMS
+  if (pid == 0)
+  {
+    filename = "last_used_params.txt";
+    std::ofstream outfile(filename.data(), std::ios::out);
+    
+    outfile << "circuit " << "= " << ckt_filename << std::endl;
+    outfile << "t_start " << "= " << t_start << std::endl;
+    outfile << "t_step " << "= " << t_step << std::endl;
+    outfile << "t_stop " << "= " << t_stop << std::endl;
+    outfile << "tol " << "= " << tol << std::endl;
+    outfile << "k " << "= " << k << std::endl;
+
+    outfile.close();
+  }
+#endif
+}

MultiSource/Benchmarks/DOE-ProxyApps-C++/miniXyce/mX_parser.h

+#ifndef __MX_PARSER_H__
+#define __MX_PARSER_H__
+
+//@HEADER
+// ************************************************************************
+// 
+//               miniXyce: A simple circuit simulation benchmark code
+//                 Copyright (2011) Sandia Corporation
+// 
+// Under terms of Contract DE-AC04-94AL85000, there is a non-exclusive
+// license for use of this work by or on behalf of the U.S. Government.
+// 
+// This library is free software; you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as
+// published by the Free Software Foundation; either version 2.1 of the
+// License, or (at your option) any later version.
+//  
+// This library is distributed in the hope that it will be useful, but
+// WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+// Lesser General Public License for more details.
+//  
+// You should have received a copy of the GNU Lesser General Public
+// License along with this library; if not, write to the Free Software
+// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+// USA
+// Questions? Contact Michael A. Heroux ([e-mail]) 
+// 
+// ************************************************************************
+//@HEADER
+
+// Author : Karthik V Aadithya
+// Mentor : Heidi K Thornquist
+// Date : July 2010
+
+#include <string>
+#include <map>
+#include "mX_source.h"
+#include "mX_linear_DAE.h"
+#include "mX_sparse_matrix.h"
+
+using namespace mX_source_utils;
+using namespace mX_matrix_utils;
+using namespace mX_linear_DAE_utils;
+
+namespace mX_parse_utils
+{
+  mX_linear_DAE* parse_netlist(std::string filename, int p, int pid, int &total_devices, int &total_unknowns, int &num_internal_nodes, std::map<std::string, int>& device_count);
+}
+#endif

MultiSource/Benchmarks/DOE-ProxyApps-C++/miniXyce/mX_parser.cpp

+//@HEADER
+// ************************************************************************
+// 
+//               miniXyce: A simple circuit simulation benchmark code
+//                 Copyright (2011) Sandia Corporation
+// 
+// Under terms of Contract DE-AC04-94AL85000, there is a non-exclusive
+// license for use of this work by or on behalf of the U.S. Government.
+// 
+// This library is free software; you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as
+// published by the Free Software Foundation; either version 2.1 of the
+// License, or (at your option) any later version.
+//  
+// This library is distributed in the hope that it will be useful, but
+// WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+// Lesser General Public License for more details.
+//  
+// You should have received a copy of the GNU Lesser General Public
+// License along with this library; if not, write to the Free Software
+// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+// USA
+// Questions? Contact Michael A. Heroux ([e-mail]) 
+// 
+// ************************************************************************
+//@HEADER
+
+// Author : Karthik V Aadithya
+// Mentor : Heidi K Thornquist
+// Date : July 2010
+
+#include <fstream>
+#include <sstream>
+#include <algorithm>
+#include <vector>
+#include "mX_source.h"
+#include "mX_linear_DAE.h"
+#include "mX_sparse_matrix.h"
+#include "mX_parser.h"
+
+#ifdef HAVE_MPI
+#include "mpi.h"
+#endif
+
+using namespace mX_source_utils;
+using namespace mX_matrix_utils;
+using namespace mX_linear_DAE_utils;
+  
+mX_linear_DAE* mX_parse_utils::parse_netlist(std::string filename, int p, int pid, int &total_devices, int &total_unknowns, int &num_internal_nodes, std::map<std::string, int>& device_count)
+{
+  // here's the netlist parser I was boasting about
+
+  // given a linear netlist description stored in a file
+        // this function will construct a DAE out of it
+  // and return that part of the DAE to be stored with processor pid
+
+  // I assume that all processors will be parsing the netlist simultaneously
+  // and each processor will only take the entries that are rightfully his own
+  // thou shall not covet your fellow processor's distributed matrix entries
+
+  std::ifstream infile;
+  infile.open(filename.data());
+  
+        // Variables for the device count
+        int num_voltage_sources = 0;
+        int num_current_sources = 0;
+        int num_resistors = 0;
+        int num_capacitors = 0;
+        int num_inductors = 0;
+
+        // Make first pass over the netlist and count the number of devices
+        // and collect the integer node names.
+        std::vector<int> node_list;
+
+        // Make first pass and perform device count
+  while (!infile.eof())
+  {
+          std::string curr_line;
+          getline(infile,curr_line);
+    
+          if ((curr_line.length() == 0) || (curr_line[0] == '%'))
+          {
+            continue; // comments begin with %
+          }
+
+          std::istringstream input_str(curr_line);
+          std::string first_word;
+          input_str >> first_word;
+
+          switch(first_word[0])
+          {
+            case 'r':
+            case 'R':
+            {
+              // seen a resistor
+              num_resistors++;
+            }
+            break;
+
+            case 'c':
+            case 'C':
+            {
+              // seen a capacitor
+              num_capacitors++;
+            }
+            break;
+           
+            case 'l':
+            case 'L':
+            {
+              // seen an inductor
+              num_inductors++;
+            }
+            break;
+
+            case 'v':
+            case 'V':
+            {
+              // seen a voltage source
+              num_voltage_sources++;
+            } 
+            break;
+
+            case 'i':
+            case 'I':
+            {
+              // seen a current source
+              num_current_sources++;
+            }
+            break;
+          }
+
+          // Save the node names, which are numbers
+          int node1, node2;
+          input_str >> node1;
+          input_str >> node2;
+          node_list.push_back( node1 );
+          node_list.push_back( node2 );
+        }
+
+        // Sort the node list and count how many nonzero nodes there are.
+        std::sort( node_list.begin(), node_list.end() );
+        node_list.erase( std::unique( node_list.begin(), node_list.end() ), node_list.end() );
+        num_internal_nodes = node_list.size();
+        if (node_list[0] == 0)
+          num_internal_nodes--;  // Get rid of ground node, which is '0', from the node count.
+
+        total_unknowns = num_internal_nodes + num_voltage_sources + num_inductors;
+        total_devices = num_voltage_sources + num_current_sources + num_resistors + num_capacitors + num_inductors;
+
+        if (num_voltage_sources)
+          device_count["V"] = num_voltage_sources;
+        if (num_current_sources)
+          device_count["I"] = num_current_sources;
+        if (num_resistors)
+          device_count["R"] = num_resistors;
+        if (num_capacitors)
+          device_count["C"] = num_capacitors;
+        if (num_inductors)
+          device_count["L"] = num_inductors;
+
+        // Closing and reopening the file, should be able to rewind the file, but it didn't work
+  infile.close();
+  infile.open(filename.data());
+        //infile.seekg( 0, infile.beg );
+
+        // Variables for the device count
+  int voltage_src_number = 0;
+  int current_src_number = 0;
+  int inductor_number = 0; 
+        int resistor_number = 0;
+        int capacitor_number = 0;
+
+  int start_row = (total_unknowns/p)*(pid) + ((pid < total_unknowns%p) ? pid : total_unknowns%p);
+  int end_row = start_row + (total_unknowns/p) - 1 + ((pid < total_unknowns%p) ? 1 : 0);
+
+  mX_linear_DAE* dae = new mX_linear_DAE();
+  dae->A = new distributed_sparse_matrix();
+  dae->B = new distributed_sparse_matrix();
+
+  distributed_sparse_matrix* A = dae->A;
+  distributed_sparse_matrix* B = dae->B;
+
+  // initialise the A, B and b parts that need to be stored in this processor
+
+  A->start_row = start_row;
+  A->end_row = end_row;
+  B->start_row = start_row;
+  B->end_row = end_row;
+
+  for (int i = start_row; i <= end_row; i++)
+  {
+    distributed_sparse_matrix_entry* null_ptr_1 = 0;
+    A->row_headers.push_back(null_ptr_1);
+    B->row_headers.push_back(null_ptr_1);
+
+    mX_linear_DAE_RHS_entry* null_ptr_2 = 0;
+    (dae->b).push_back(null_ptr_2);
+  }
+
+  // initialisation done
+  // now read each line from the input file and do what is expected
+
+  while (!infile.eof())
+  {
+    std::string curr_line;
+    getline(infile,curr_line);
+
+    if ((curr_line.length() == 0) || (curr_line[0] == '%'))
+    {
+      continue; // comments begin with %
+    }
+
+    std::istringstream input_str(curr_line);
+    std::string first_word;
+    input_str >> first_word;
+
+    switch(first_word[0])
+    {
+      case 'r':
+      case 'R':
+
+      {
+        // seen a resistor
+                                resistor_number++;      
+  
+        int node1, node2;
+        double rvalue;
+
+        input_str >> node1;
+        input_str >> node2;
+        input_str >> rvalue;
+        
+        distributed_sparse_matrix_add_to(A,node1-1,node1-1,(double)(1)/rvalue,total_unknowns,p);
+        distributed_sparse_matrix_add_to(A,node2-1,node2-1,(double)(1)/rvalue,total_unknowns,p);
+        distributed_sparse_matrix_add_to(A,node1-1,node2-1,(double)(-1)/rvalue,total_unknowns,p);
+        distributed_sparse_matrix_add_to(A,node2-1,node1-1,(double)(-1)/rvalue,total_unknowns,p);
+      }
+
+      break;
+
+      case 'c':
+      case 'C':
+
+      {
+        // seen a capacitor
+        capacitor_number++;
+
+        int node1, node2;
+        double cvalue;
+
+        input_str >> node1;
+        input_str >> node2;
+        input_str >> cvalue;
+        
+        distributed_sparse_matrix_add_to(B,node1-1,node1-1,cvalue,total_unknowns,p);
+        distributed_sparse_matrix_add_to(B,node2-1,node2-1,cvalue,total_unknowns,p);
+        distributed_sparse_matrix_add_to(B,node1-1,node2-1,-cvalue,total_unknowns,p);
+        distributed_sparse_matrix_add_to(B,node2-1,node1-1,-cvalue,total_unknowns,p);
+      }
+
+      break;
+
+      case 'l':
+      case 'L':
+
+      {
+        // seen an inductor
+        
+        inductor_number++;
+        
+        int k = num_internal_nodes + num_voltage_sources + inductor_number;
+
+        int node1, node2;
+        double lvalue;
+
+        input_str >> node1;
+        input_str >> node2;
+        input_str >> lvalue;
+        
+        distributed_sparse_matrix_add_to(A,k-1,node1-1,(double)(1),total_unknowns,p);
+        distributed_sparse_matrix_add_to(A,k-1,node2-1,(double)(-1),total_unknowns,p);
+        distributed_sparse_matrix_add_to(B,k-1,k-1,-lvalue,total_unknowns,p);
+        distributed_sparse_matrix_add_to(A,node1-1,k-1,(double)(1),total_unknowns,p);
+        distributed_sparse_matrix_add_to(A,node2-1,k-1,(double)(-1),total_unknowns,p);
+      }
+
+      break;
+
+      case 'v':
+      case 'V':
+
+      {
+        // seen a voltage source 
+        
+        voltage_src_number++;
+        
+        int k = num_internal_nodes + voltage_src_number;
+
+        int node1, node2;
+
+        input_str >> node1;
+        input_str >> node2;
+        
+        distributed_sparse_matrix_add_to(A,k-1,node1-1,(double)(1),total_unknowns,p);
+        distributed_sparse_matrix_add_to(A,k-1,node2-1,(double)(-1),total_unknowns,p);
+        distributed_sparse_matrix_add_to(A,node1-1,k-1,(double)(-1),total_unknowns,p);
+        distributed_sparse_matrix_add_to(A,node2-1,k-1,(double)(1),total_unknowns,p);
+        
+        if ((k-1 >= start_row) && (k-1 <= end_row))
+        {
+          mX_source* src = parse_source(input_str);
+        
+          mX_scaled_source* scaled_src = new mX_scaled_source();
+          scaled_src->src = src;
+          scaled_src->scale = (double)(1);
+
+          if(!(dae->b[k-1-start_row]))
+          {
+            dae->b[k-1-start_row] = new mX_linear_DAE_RHS_entry();
+          }
+
+          (dae->b[k-1-start_row])->scaled_src_list.push_back(scaled_src);
+
+        }
+
+      }
+
+      break;
+
+      case 'i':
+      case 'I':
+
+      {
+        // seen a current source 
+        current_src_number++;
+
+        int node1, node2;
+                                
+        input_str >> node1;
+        input_str >> node2;
+
+        if ((node1-1 >= start_row) && (node1-1 <= end_row))
+        {
+          mX_source* src = parse_source(input_str);
+        
+          mX_scaled_source* scaled_src = new mX_scaled_source();
+          scaled_src->src = src;
+          scaled_src->scale = (double)(1);
+
+          if(!(dae->b[node1-1-start_row]))
+          {
+            dae->b[node1-1-start_row] = new mX_linear_DAE_RHS_entry();
+          }
+
+          (dae->b[node1-1-start_row])->scaled_src_list.push_back(scaled_src);
+
+          if ((node2-1 >= start_row) && (node2-1 <= end_row))
+          {
+            mX_scaled_source* scaled_src_2 = new mX_scaled_source();
+            scaled_src_2->src = src;
+            scaled_src_2->scale = (double)(-1);
+
+            if(!(dae->b[node2-1-start_row]))
+            {
+              dae->b[node2-1-start_row] = new mX_linear_DAE_RHS_entry();
+            }
+
+            (dae->b[node2-1-start_row])->scaled_src_list.push_back(scaled_src_2);
+
+          }
+        }
+
+        else
+        {
+          if ((node2-1 >= start_row) && (node2-1 <= end_row))
+          {
+            mX_source* src = parse_source(input_str);
+          
+            mX_scaled_source* scaled_src = new mX_scaled_source();
+            scaled_src->src = src;
+            scaled_src->scale = (double)(-1);
+
+            if(!(dae->b[node2-1-start_row]))
+            {
+              dae->b[node2-1-start_row] = new mX_linear_DAE_RHS_entry();
+            }
+
+            (dae->b[node2-1-start_row])->scaled_src_list.push_back(scaled_src);
+            
+          }
+        }
+      }
+
+      break;
+    }
+  }
+
+  // whew! all the lines have been read
+    // and each processor hopefully has his correct share of the DAE
+
+  infile.close();
+
+  return dae;
+}

MultiSource/Benchmarks/DOE-ProxyApps-C++/miniXyce/mX_source.h

+#ifndef __MX_SOURCE_DEFS_H__
+#define __MX_SOURCE_DEFS_H__
+
+//@HEADER
+// ************************************************************************
+// 
+//               miniXyce: A simple circuit simulation benchmark code
+//                 Copyright (2011) Sandia Corporation
+// 
+// Under terms of Contract DE-AC04-94AL85000, there is a non-exclusive
+// license for use of this work by or on behalf of the U.S. Government.
+// 
+// This library is free software; you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as
+// published by the Free Software Foundation; either version 2.1 of the
+// License, or (at your option) any later version.
+//  
+// This library is distributed in the hope that it will be useful, but
+// WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+// Lesser General Public License for more details.
+//  
+// You should have received a copy of the GNU Lesser General Public
+// License along with this library; if not, write to the Free Software
+// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+// USA
+// Questions? Contact Michael A. Heroux ([e-mail]) 
+// 
+// ************************************************************************
+//@HEADER
+
+// Author : Karthik V Aadithya
+// Mentor : Heidi K Thornquist
+// Date : July 2010
+
+#include <vector>
+#include <sstream>
+
+namespace mX_source_utils
+{
+  class mX_source
+  {
+    // this class represents a time-varying voltage or current source
+      // all that is needed is the value of the output at time "t"
+
+    public:
+
+    virtual double output(double t) = 0;
+  };
+
+  class DC : public mX_source
+  {
+    // DC voltage/current source
+
+    public:
+
+    double val;
+
+    DC(double d)
+    {
+      val = d;
+    }
+
+    virtual double output(double t)
+    {
+      return val;
+    }
+  };
+
+  class SINE : public mX_source
+  {
+    // sinusoidal voltage/current source
+
+    public:
+
+    double offset;
+    double amplitude;
+    double freq;
+    double phase;
+
+    SINE(double off, double amp, double f, double ph);
+
+    virtual double output(double t); 
+  };
+
+  class FM : public mX_source
+  {
+    // frequency modulated voltage/current source
+
+    public:
+
+    double offset;
+    double amplitude;
+    double carrier_freq;
+    double modulation_index;
+    double signal_freq;
+
+    FM(double o, double a, double cf, double mi, double sf);
+
+    virtual double output(double t); 
+  };
+
+  class PWL : public mX_source
+  {
+    // piecewise linear voltage/current source
+
+    public:
+
+    std::vector<double> times;
+    std::vector<double> values;
+
+    PWL(std::vector<double> ts, std::vector<double> vals);
+
+    virtual double output(double t);
+  };
+
+  class PULSE : public mX_source
+  {
+    // pulse shaped voltage/current source
+
+    public:
+
+    double val1;
+    double val2;
+    double t_delay;
+    double t_rise;
+    double pulse_width;
+    double t_fall;
+    double t_stop;
+
+    PULSE(double d1, double d2, double td, double tr, double pw, double tf, double ts);
+
+    virtual double output(double t);
+  };
+
+  struct mX_scaled_source
+  {
+    // a scaled voltage/current source
+      // useful in many contexts
+        // eg: when you want -V(t) and already have a source V(t)
+
+    mX_source* src;
+    double scale;
+  };
+
+  mX_source* parse_source(std::istringstream& input_str);
+}
+#endif

MultiSource/Benchmarks/DOE-ProxyApps-C++/miniXyce/mX_source.cpp

+//@HEADER
+// ************************************************************************
+// 
+//               miniXyce: A simple circuit simulation benchmark code
+//                 Copyright (2011) Sandia Corporation
+// 
+// Under terms of Contract DE-AC04-94AL85000, there is a non-exclusive
+// license for use of this work by or on behalf of the U.S. Government.
+// 
+// This library is free software; you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as
+// published by the Free Software Foundation; either version 2.1 of the
+// License, or (at your option) any later version.
+//  
+// This library is distributed in the hope that it will be useful, but
+// WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+// Lesser General Public License for more details.
+//  
+// You should have received a copy of the GNU Lesser General Public
+// License along with this library; if not, write to the Free Software
+// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+// USA
+// Questions? Contact Michael A. Heroux ([e-mail]) 
+// 
+// ************************************************************************
+//@HEADER
+
+// Author : Karthik V Aadithya
+// Mentor : Heidi K Thornquist
+// Date : July 2010
+
+#include <string>
+#include <cmath>
+#include "mX_source.h"
+
+using namespace mX_source_utils;
+
+// ------------------------------------------------------------------
+// Implementations for the SINE class
+mX_source_utils::SINE::SINE(double off, double amp, double f, double ph)
+  : offset(off),
+    amplitude(amp),
+    freq(f),
+    phase(ph)
+{}
+
+double mX_source_utils::SINE::output(double t) 
+{
+  double pi = 3.1415926535;
+  return (offset + amplitude * sin(2*pi*freq*t + phase));
+}
+
+// ------------------------------------------------------------------
+// Implementations for the FM class
+mX_source_utils::FM::FM(double o, double a, double cf, double mi, double sf)
+  : offset(o),
+    amplitude(a),
+    carrier_freq(cf),
+    modulation_index(mi),
+    signal_freq(sf)
+{}
+
+double mX_source_utils::FM::output(double t) 
+{
+  double pi = 3.1415926535;
+  return (offset + amplitude * sin(2*pi*carrier_freq*t + modulation_index*sin(2*pi*signal_freq*t)));
+}
+
+// ------------------------------------------------------------------
+// Implementations for the PWL class
+mX_source_utils::PWL::PWL(std::vector<double> ts, std::vector<double> vals)
+  : times(ts),
+    values(vals)
+{}
+
+double mX_source_utils::PWL::output(double t) 
+{
+  if (times.size() == 1)
+  {
+    return values[0];
+  }
+
+  if (t <= times[0])
+  {
+    return values[0];
+  }
+
+  if (t >= times.back())
+  {
+    return values.back();
+  }
+
+  int start_index = 0;
+  int end_index = times.size()-1;
+  int mid_index = (start_index + end_index)/2;
+      
+  while(true)
+  {
+    if (end_index - start_index == 1)
+    {
+      break;
+    }
+    if (times[mid_index] > t)
+    {
+      end_index = mid_index;
+      mid_index = (start_index + end_index)/2;
+    }
+    else
+    {
+      if (times[mid_index] < t)
+      {
+        start_index = mid_index;
+        mid_index = (start_index + end_index)/2;
+      }
+      else
+      {
+        return values[mid_index];
+      }
+    }
+  }
+
+  return (values[start_index] + (values[end_index]-values[start_index])*((t - times[start_index])/(times[end_index] - times[start_index])));
+}
+
+// ------------------------------------------------------------------
+// Implementations for the PULSE class
+
+mX_source_utils::PULSE::PULSE(double d1, double d2, double td, double tr, double pw, double tf, double ts)
+  : val1(d1),
+    val2(d2),
+    t_delay(td),
+    t_rise(tr),
+    pulse_width(pw),
+    t_fall(tf),
+    t_stop(ts)
+{}
+
+double mX_source_utils::PULSE::output(double t)
+{
+  double t_total = t_delay + t_rise + pulse_width + t_fall + t_stop;
+  double n = floor(t/t_total);
+  double t_disp = t - n*t_total;
+
+  if ((t_disp >= 0) && (t_disp <= t_delay))
+  {
+    return val1;
+  }       
+
+  if ((t_disp >= t_delay) && (t_disp <= t_delay + t_rise))
+  {
+    return (val1 + ((val2-val1)/(t_rise))*(t_disp-t_delay));
+  }
+
+  if ((t_disp >= t_delay + t_rise) && (t_disp <= t_delay + t_rise + pulse_width))
+  {
+    return val2;
+  }
+      
+  if ((t_disp >= t_delay + t_rise + pulse_width) && (t_disp <= t_delay + t_rise + pulse_width + t_fall))
+  {
+    return val2 + ((val1-val2)/(t_fall))*(t_disp-(t_delay + t_rise + pulse_width));
+  }
+
+  return val1;
+}
+
+// ------------------------------------------------------------------
+// Implementation to parse the source type
+
+mX_source* mX_source_utils::parse_source(std::istringstream& input_str)
+{
+  // ok, so you know you have a source in the box
+    // but you don't know what kind of beast it is
+    // this function will look at the box and return the correct source object for you
+
+  std::string src_type;
+  input_str >> src_type;
+
+  if (src_type == "DC")
+  {
+    double val;
+    input_str >> val;
+    return new DC(val);
+  }
+
+  if (src_type == "SINE")
+  {
+    double offset,amp,f,ph;
+    input_str >> offset >> amp >> f >> ph;
+    return new SINE(offset,amp,f,ph);
+  }
+
+  if (src_type == "FM")
+  {
+    double offset,amp,cf,mi,sf;
+    input_str >> offset >> amp >> cf >> mi >> sf;
+    return new FM(offset,amp,cf,mi,sf);
+  }
+
+  if (src_type == "PWL")
+  {
+    int pwl_size;
+    std::vector<double> times;
+    std::vector<double> values;
+
+    double t,v;
+
+    input_str >> pwl_size;
+
+    for (int i = 0; i < pwl_size; i++)
+    {
+      input_str >> t >> v;
+      times.push_back(t);
+      values.push_back(v);
+    }
+
+    return new PWL(times,values);
+  }
+
+  if (src_type == "PULSE")
+  {
+    double v1,v2,td,tr,pw,tf,ts;
+    input_str >> v1 >> v2 >> td >> tr >> pw >> tf >> ts;
+    return new PULSE(v1,v2,td,tr,pw,tf,ts);
+  }
+
+  return 0;
+}

MultiSource/Benchmarks/DOE-ProxyApps-C++/miniXyce/mX_sparse_matrix.h

+#ifndef __MX_SPARSE_MATRIX_DEFS_H__
+#define __MX_SPARSE_MATRIX_DEFS_H__
+
+//@HEADER
+// ************************************************************************
+// 
+//               miniXyce: A simple circuit simulation benchmark code
+//                 Copyright (2011) Sandia Corporation
+// 
+// Under terms of Contract DE-AC04-94AL85000, there is a non-exclusive
+// license for use of this work by or on behalf of the U.S. Government.
+// 
+// This library is free software; you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as
+// published by the Free Software Foundation; either version 2.1 of the
+// License, or (at your option) any later version.
+//  
+// This library is distributed in the hope that it will be useful, but
+// WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+// Lesser General Public License for more details.
+//  
+// You should have received a copy of the GNU Lesser General Public
+// License along with this library; if not, write to the Free Software
+// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+// USA
+// Questions? Contact Michael A. Heroux ([e-mail]) 
+// 
+// ************************************************************************
+//@HEADER
+
+// Author : Karthik V Aadithya
+// Mentor : Heidi K Thornquist
+// Date : July 2010
+
+#include <vector>
+#include <list>
+#include <map>
+
+namespace mX_matrix_utils
+{
+  struct distributed_sparse_matrix_entry
+  {
+    int column;   // global column index
+    double value; // value stored in the matrix
+    distributed_sparse_matrix_entry* next_in_row; // pointer to next entry in the same row
+  };
+
+  struct data_transfer_instruction
+  {
+    std::list<int> indices;    // which elements of the vector to send
+    int pid;    // to which processor the data is to be sent
+  };
+
+  struct distributed_sparse_matrix
+  {
+    // the data structure for a distributed sparse matrix is a 1-d threaded list
+      // a set of pointers called row_headers point to the first entry of each row
+      // each row entry in turn points to the next entry in the same row
+
+    // but a distributed matrix needs more data than this
+      // there is a list of data transfer instructions
+        // these instructions are to be followed whenever a mat-vec product is needed
+    
+    // each processor also stores 2 entries start_row and end_row
+      // it is assumed that all processors store contiguous rows of the distributed matrix
+
+    int start_row;
+    int end_row;
+                int local_nnz;
+
+    std::vector<distributed_sparse_matrix_entry*> row_headers;
+    std::list<data_transfer_instruction*> send_instructions;
+
+                distributed_sparse_matrix();
+  };
+
+
+  void distributed_sparse_matrix_add_to(distributed_sparse_matrix* M, int row_idx, int col_idx, double val, int n, int p);
+
+  void sparse_matrix_vector_product(distributed_sparse_matrix* A, std::vector<double> &x, std::vector<double> &y);
+
+  double norm(std::vector<double> &x);
+
+  void gmres(distributed_sparse_matrix* A, std::vector<double> &b, std::vector<double> &x0, double &tol, double &err, int k, std::vector<double> &x, int &iters, int &restarts);
+
+        void destroy_matrix(distributed_sparse_matrix* A);
+
+        void print_vector(std::vector<double> &x);
+
+        void print_matrix(distributed_sparse_matrix &A);
+}
+
+#endif

MultiSource/Benchmarks/DOE-ProxyApps-C++/miniXyce/mX_sparse_matrix.cpp

+//@HEADER
+// ************************************************************************
+// 
+//               miniXyce: A simple circuit simulation benchmark code
+//                 Copyright (2011) Sandia Corporation
+// 
+// Under terms of Contract DE-AC04-94AL85000, there is a non-exclusive
+// license for use of this work by or on behalf of the U.S. Government.
+// 
+// This library is free software; you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as
+// published by the Free Software Foundation; either version 2.1 of the
+// License, or (at your option) any later version.
+//  
+// This library is distributed in the hope that it will be useful, but
+// WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+// Lesser General Public License for more details.
+//  
+// You should have received a copy of the GNU Lesser General Public
+// License along with this library; if not, write to the Free Software
+// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+// USA
+// Questions? Contact Michael A. Heroux ([e-mail]) 
+// 
+// ************************************************************************
+//@HEADER
+
+// Author : Karthik V Aadithya
+// Mentor : Heidi K Thornquist
+// Date : July 2010
+
+#ifdef HAVE_MPI
+#include <mpi.h> // If this routine is compiled with -DHAVE_MPI
+                 // then include mpi.h
+#endif
+#include "mX_sparse_matrix.h"
+#include <map>
+#include <cmath>
+#include <iostream>
+
+using namespace mX_matrix_utils;
+
+distributed_sparse_matrix::distributed_sparse_matrix()
+  : start_row(0), end_row(0), local_nnz(0)
+{}
+
+void mX_matrix_utils::distributed_sparse_matrix_add_to(distributed_sparse_matrix* M, int row_idx, int col_idx, double val, int n, int p)
+{
+  // implements M[row_idx][col_idx] += val
+    // man, in the distributed matrix world, this simple thing takes such a lot of effort!
+
+  if ((row_idx < 0) || (col_idx < 0))
+  {
+    // check for negative indices
+      // useful in cases where you want to ignore the ground
+        // because the ground is not really a node
+
+    return;
+  }
+
+  if ((row_idx >= M->start_row) && (row_idx <= M->end_row))
+  {
+                M->local_nnz++;
+
+    // ok, so the processor that's supposed to store M[row_idx][col_idx] is here
+      // navigate through the fellow's threaded list and do the needful
+
+    bool inserted = false;
+
+    distributed_sparse_matrix_entry* prev = 0;
+    distributed_sparse_matrix_entry* curr = M->row_headers[row_idx-M->start_row];
+  
+    while ((curr) && (!inserted))
+    {
+      if (curr->column < col_idx)
+      {
+        prev = curr;
+        curr = curr->next_in_row;
+      }
+
+      else
+      {
+        if (curr->column == col_idx)
+        {
+          curr->value = curr->value + val;
+          inserted = true;
+        }
+
+        else
+        {
+          distributed_sparse_matrix_entry* entry_ptr_1 = new distributed_sparse_matrix_entry();
+          entry_ptr_1->column = col_idx;
+          entry_ptr_1->value = val;
+          entry_ptr_1->next_in_row = curr;
+
+          if (prev)
+          {
+            prev->next_in_row = entry_ptr_1;
+          }
+
+          else
+          {
+            M->row_headers[row_idx-M->start_row] = entry_ptr_1;
+          }
+
+          inserted = true;
+        }
+      }
+    }
+
+    if (!inserted)
+    {
+      distributed_sparse_matrix_entry* entry_ptr_1 = new distributed_sparse_matrix_entry();
+      entry_ptr_1->column = col_idx;
+      entry_ptr_1->value = val;
+      entry_ptr_1->next_in_row = curr;
+
+      if (prev)
+      {
+        prev->next_in_row = entry_ptr_1;
+      }
+
+      else
+      {
+        M->row_headers[row_idx-M->start_row] = entry_ptr_1;
+      }
+
+      inserted = true;
+    }
+
+    return;
+  }
+
+  // but wait, if you have inserted a new non-zero in the sparse matrix
+    // it might require some additional book-keeping of communication info
+
+  if ((col_idx >= M->start_row) && (col_idx <= M->end_row))
+  {
+    // aha, here's the processor who controls the col_idx
+    // this fellow needs to send the value at col_idx to the fellow who controls the row_idx
+    // everytime a matrix vector product is needed
+    // so find out who has the row_idx
+    // and if needed, include an instruction in this fellow's list of send instructions
+
+    int pid_to_send_info;
+    bool pid_found = false;
+
+    int start_pid = 0;
+    int end_pid = p-1;
+    int mid_pid = (start_pid + end_pid)/2;
+
+    int mid_start_row = (n/p)*(mid_pid) + ((mid_pid < n%p) ? mid_pid : n%p);
+    int mid_end_row = mid_start_row + (n/p) - 1 + ((mid_pid < n%p) ? 1 : 0);
+
+    while (!pid_found)
+    {
+      if (row_idx < mid_start_row)
+      {
+        end_pid = mid_pid - 1;
+        mid_pid = (start_pid + end_pid)/2;
+      }
+
+      else
+      {
+        if (row_idx > mid_end_row)
+        {
+          start_pid = mid_pid + 1;
+          mid_pid = (start_pid + end_pid)/2;
+        }
+
+        else
+        {
+          pid_to_send_info = mid_pid;
+          pid_found = true;
+        }
+      }
+
+      mid_start_row = (n/p)*(mid_pid) + ((mid_pid < n%p) ? mid_pid : n%p);
+      mid_end_row = mid_start_row + (n/p) - 1 + ((mid_pid < n%p) ? 1 : 0);
+    }
+
+    bool send_instruction_posted = false;
+    
+    std::list<data_transfer_instruction*>::iterator it1;
+
+    for (it1 = M->send_instructions.begin(); it1 != M->send_instructions.end(); it1++)
+    {
+      if ((*it1)->pid == pid_to_send_info)
+      {
+        std::list<int>::iterator it2;
+
+        for (it2 = ((*it1)->indices).begin(); it2 != ((*it1)->indices).end(); it2++)
+        {
+          if (*it2 == col_idx)
+          {
+            send_instruction_posted = true;
+          }
+        }
+
+        if (!send_instruction_posted)
+        {
+          (*it1)->indices.push_back(col_idx);
+          send_instruction_posted = true;
+        }
+      }
+    }
+
+    if (!send_instruction_posted)
+    {
+      data_transfer_instruction* dti_ptr_1 = new data_transfer_instruction();
+      dti_ptr_1->pid = pid_to_send_info;
+      dti_ptr_1->indices.push_back(col_idx);
+
+      M->send_instructions.push_back(dti_ptr_1);
+      send_instruction_posted = true;
+    }
+  }
+
+  // that was easy!
+
+}
+
+void mX_matrix_utils::sparse_matrix_vector_product(distributed_sparse_matrix* A, std::vector<double> &x, std::vector<double> &y)
+{
+  // compute the matrix vector product A*x and return it in y
+    // assuming x contains only x[start_row] to x[end_row]
+  
+  // at the end of this function, it is guaranteed that
+    // y[0] to y[end_row-start_row] will contain the correct entries of (Ax)[start_row] to (Ax)[end_row] 
+
+        int start_row = A->start_row;
+  int end_row = A->end_row;
+
+#ifdef HAVE_MPI
+  // ok, now's the time to follow the send instructions that each pid has been maintaining
+
+  std::list<data_transfer_instruction*>::iterator it1;
+  
+  for (it1 = A->send_instructions.begin(); it1 != A->send_instructions.end(); it1++)
+  {
+    std::list<int>::iterator it2;
+
+    for (it2 = (*it1)->indices.begin(); it2 != (*it1)->indices.end(); it2++)
+    {
+      MPI_Send(&x[(*it2)-start_row],1,MPI_DOUBLE,(*it1)->pid,*it2,MPI_COMM_WORLD);
+    }
+  }
+#endif
+
+  // and everytime a processor receives an x_vec entry
+    // it stores the entry in a temporary map
+
+  std::map<int,double> x_vec_entries;
+  std::map<int,double>::iterator it3;
+
+  for (int i = start_row; i <= end_row; i++)
+  {
+    // compute the mat_vec product for the i'th row
+
+    if (y.size() > i - start_row)
+    {
+      y[i-start_row] = (double)(0);
+    }
+
+    else
+    {
+      y.push_back((double)(0));
+    }
+
+    distributed_sparse_matrix_entry* curr = A->row_headers[i-start_row];
+
+    while(curr)
+    {
+      int col_idx = curr->column;
+      
+      if ((col_idx >= start_row) && (col_idx <= end_row))
+      {
+        // aha, this processor has the correct x_vec entry locally
+
+        y[i-start_row] += (curr->value)*x[col_idx-start_row];
+      }
+
+#ifdef HAVE_MPI
+      else
+      {
+        // this processor does not have the x_vec entry locally
+          // but some other processor might have sent it to this guy
+            // in which case the entry would have been stored in the local x_vec_entries map
+            // so check if the entry is in the map
+
+        it3 = x_vec_entries.find(col_idx);
+
+        if (it3 != x_vec_entries.end())
+        {
+          y[i-start_row] += (double)(it3->second)*(curr->value);
+        }
+
+        else
+        {
+          // no, the entry is not in the map either
+            // so this processor waits until someone sends the entry
+            // and once it gets the entry, it does two things
+              // puts the entry in the map for future reference
+              // continues with the matrix vector multiplication
+
+          double x_vec_entry;
+                            MPI_Status status;
+          MPI_Recv(&x_vec_entry,1,MPI_DOUBLE,MPI_ANY_SOURCE,col_idx,MPI_COMM_WORLD,&status);
+
+          x_vec_entries[col_idx] = x_vec_entry;
+          y[i-start_row] += x_vec_entry*(curr->value);
+        }
+      }
+#endif
+      curr = curr->next_in_row;
+    }
+  }
+}
+
+double mX_matrix_utils::norm(std::vector<double> &x)
+{
+  // at last, a function that's relatively simple to implement in parallel
+
+  double global_norm;
+  double local_norm = 0.0;
+
+  for (int i = 0; i < x.size(); i++)
+  {
+    local_norm += x[i]*x[i];
+  }
+#ifdef HAVE_MPI
+  MPI_Allreduce(&local_norm,&global_norm,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
+#else
+        global_norm = local_norm;
+#endif
+
+  return std::sqrt(global_norm);
+}
+
+void mX_matrix_utils::gmres(distributed_sparse_matrix* A, std::vector<double> &b, std::vector<double> &x0, double &tol, double &err, int k, std::vector<double> &x, int &iters, int &restarts)
+{
+  // here's the star of the show, the guest of honor, none other than Mr.GMRES
+  
+  // first Mr.GMRES will compute the error in the initial guess
+    // if it's already smaller than tol, he calls it a day
+    // otherwise he settles down to work in mysterious ways his wonders to perform
+
+  int start_row = A->start_row;
+  int end_row = A-> end_row;
+
+  x = x0;
+
+  std::vector<double> temp1;
+  sparse_matrix_vector_product(A,x,temp1);
+
+  for (int i = 0; i < temp1.size(); i++)
+  {
+    temp1[i] -= b[i];
+  }
+
+  err = norm(temp1);
+    restarts = -1;
+  iters = 0;
+
+  while (err > tol)
+  {
+    // at the start of every re-start
+      // the initial guess is already stored in x
+
+    restarts++;
+    
+    std::vector<double> temp1;
+    std::vector< std::vector<double> > V;
+    sparse_matrix_vector_product(A,x,temp1);
+
+    for (int i = start_row; i <= end_row; i++)
+    {
+      temp1[i-start_row] -= b[i-start_row];
+      temp1[i-start_row] *= (double)(-1);
+
+      std::vector<double> temp2;
+      temp2.push_back(temp1[i-start_row]);
+      V.push_back(temp2);
+    }
+
+    double beta = norm(temp1);
+
+    for (int i = start_row; i <= end_row; i++)
+    {
+      V[i-start_row][0] /= beta;
+    }
+
+    err = beta;
+    iters = 0;
+
+    std::vector<double> cosines;
+    std::vector<double> sines;
+    std::vector<double> g;
+    std::vector< std::vector<double> > R;
+    
+    g.push_back(beta);
+
+    // ok, Mr.GMRES has determined the initial values for
+      // V,R,g,sines,cosines,err and iters
+      // he updates these at every iteration until
+        // either err becomes less than tol
+        // or a new restart is required
+
+    // note that Mr.GMRES does not think it necessary to store the upper Hessenberg matrix at each iteration
+      // he computes R at each iteration from the new Hessenberg matrix column
+      // and he's clever enough to determine err without having to solve for x at each iteration
+
+    while ((err > tol) && (iters < k))
+    {
+      iters++;
+
+      // Mr.GMRES is now going to update the V matrix
+        // for which he will require a matrix vector multiplication
+
+      std::vector<double> temp1;
+      std::vector<double> temp2;
+
+      for (int i = start_row; i <= end_row; i++)
+      {
+        temp1.push_back(V[i-start_row][iters-1]);  
+      }
+      sparse_matrix_vector_product(A,temp1,temp2);
+
+      // Right, Mr.GMRES now has the matrix vector product
+        // now he will orthogonalize this vector with the previous ones 
+          // with some help from Messrs Gram and Schmidt
+      
+      std::vector<double> new_col_H;
+
+      for (int i = 0; i < iters; i++)
+      {
+        double local_dot = 0.0;
+        double global_dot;
+
+        for (int j = start_row; j <= end_row; j++)
+        {
+          local_dot += temp2[j-start_row]*V[j-start_row][i];
+        }
+#ifdef HAVE_MPI
+        MPI_Allreduce(&local_dot,&global_dot,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD);
+#else
+                                global_dot = local_dot;
+#endif        
+        for (int j = start_row; j <= end_row; j++)
+        {
+          temp2[j-start_row] -= global_dot*V[j-start_row][i];
+        }
+
+        new_col_H.push_back(global_dot);
+      }
+
+      new_col_H.push_back(norm(temp2));
+
+      for (int i = start_row; i <= end_row; i++)
+      {
+        temp2[i-start_row] /= new_col_H.back();
+        V[i-start_row].push_back(temp2[i-start_row]);
+      }
+
+      // Right, Mr.GMRES has successfully updated V
+        // on the side, he has also been computing the new column of the Hessenberg matrix
+      // now he's going to get the new column of R using the current sines and cosines
+        // and he will also add a new sine and a new cosine for future use
+
+      for (int i = 0; i < iters-1; i++)
+      {
+        double old_i = new_col_H[i];
+        double old_i_plus_one = new_col_H[i+1];
+
+        new_col_H[i] = cosines[i]*old_i + sines[i]*old_i_plus_one;
+        new_col_H[i+1] = -sines[i]*old_i + cosines[i]*old_i_plus_one;
+      }
+
+      double r = std::sqrt(new_col_H[iters-1]*new_col_H[iters-1] + new_col_H[iters]*new_col_H[iters]);
+      cosines.push_back(new_col_H[iters-1]/r);
+      sines.push_back(new_col_H[iters]/r);
+      
+      double old_i = new_col_H[iters-1];
+      double old_i_plus_one = new_col_H[iters];
+
+      new_col_H[iters-1] = cosines.back()*old_i + sines.back()*old_i_plus_one;
+      new_col_H.pop_back();
+
+      R.push_back(new_col_H);
+
+      // Right, the new column of R is ready
+      // the only thing left to do is to update g
+        // which will also tell Mr.GMRES what the new error is
+
+      double old_g = g[iters-1];
+      g[iters-1] = old_g*cosines.back();
+      g.push_back(-old_g*sines.back());
+
+      err = std::abs(g.back());
+    }
+
+    // ok, so either Mr.GMRES has a solution
+      // or he's being forced to restart
+    // either way, he needs to compute x
+      // now he needs to solve Ry = g
+      // after which he will say x += (V without its last column)*y
+
+    std::vector<double> y;
+
+    for (int i = iters-1; i >= 0; i--)
+    {
+      double sum = (double)(0);
+
+      for (int j = iters-1; j > i; j--)
+      {
+        sum += R[j][i]*y[iters-1-j];
+      }
+
+      y.push_back((g[i] - sum)/R[i][i]);
+    }
+
+    // ok, so y is ready (although it's stored upside down)
+
+    for (int i = start_row; i <= end_row; i++)
+    {
+      double sum = (double)(0);
+
+      for (int j = iters-1; j >= 0; j--)
+      {
+        sum += y[iters-1-j]*V[i-start_row][j];
+      }
+
+      x[i-start_row] += sum;
+    }
+
+    // the new x is also ready
+      // either return it or use it as an initial guess for the next restart
+  }
+  
+  // if Mr.GMRES ever reaches here, it means he's solved the problem
+
+  if (restarts < 0)
+  {
+    restarts = 0;
+  }
+}
+
+void mX_matrix_utils::destroy_matrix(distributed_sparse_matrix* A)
+{
+  if (A)
+  {
+      // delete row_headers
+      for (int j=A->start_row, cnt=0; j<=A->end_row; ++j, ++cnt)
+      {
+        distributed_sparse_matrix_entry* curr = (*A).row_headers[cnt], *next = 0;
+        
+        while (curr)
+        {
+          next = curr->next_in_row;
+          delete curr; curr=0;
+          curr = next;
+        }
+      }
+      A->row_headers.resize(0);  
+
+      // delete send_instructions
+      while (!A->send_instructions.empty())
+      {
+        data_transfer_instruction* curr = A->send_instructions.front();
+        delete curr; curr=0;
+        A->send_instructions.pop_front();
+      }
+
+      delete A; A=0;
+  }
+}
+
+void mX_matrix_utils::print_vector(std::vector<double>& x)
+{
+  int numprocs = 1, procnum=0;
+
+#ifdef HAVE_MPI
+  /* Find this processor number */
+  MPI_Comm_rank(MPI_COMM_WORLD, &procnum);
+
+  /* Find the number of processors */
+  MPI_Comm_size(MPI_COMM_WORLD, &numprocs);
+#endif
+
+  for (int i=0; i< numprocs; ++i) 
+  {
+    if (i == procnum) 
+    {
+      if (procnum == 0)
+        std::cout << "Proc\tLocal Index\tValue" << std::endl;
+
+      for (int j=0; j<x.size(); ++j) 
+      {
+        std::cout << procnum << "\t" << j << "\t" << x[j] << std::endl;
+      }
+    }
+#ifdef HAVE_MPI
+    MPI_Barrier(MPI_COMM_WORLD);
+#endif
+  }
+}
+
+void mX_matrix_utils::print_matrix(distributed_sparse_matrix &A)
+{
+  int numprocs = 1, procnum=0;
+
+#ifdef HAVE_MPI
+  /* Find this processor number */
+  MPI_Comm_rank(MPI_COMM_WORLD, &procnum);
+
+  /* Find the number of processors */
+  MPI_Comm_size(MPI_COMM_WORLD, &numprocs);
+#endif
+
+  for (int i=0; i< numprocs; ++i)
+  {
+    if (i == procnum)
+    {
+      if (procnum == 0)
+        std::cout << "Matrix\nProc\tRow\tColumn\tValue" << std::endl;
+
+      for (int j=A.start_row, cnt=0; j<=A.end_row; ++j, ++cnt)
+      {
+        distributed_sparse_matrix_entry* curr = A.row_headers[cnt];
+
+        while (curr)
+        {
+          std::cout << procnum << "\t" << j << "\t" << curr->column << "\t" << curr->value << std::endl;
+          curr = curr->next_in_row;
+        }
+      }
+    }
+#ifdef HAVE_MPI
+    MPI_Barrier(MPI_COMM_WORLD);
+#endif
+  }
+}
+

MultiSource/Benchmarks/DOE-ProxyApps-C++/miniXyce/mX_timer.h

+#ifndef __MX_TIMER_H__
+#define __MX_TIMER_H__
+
+//@HEADER
+// ************************************************************************
+// 
+//               miniXyce: A simple circuit simulation benchmark code
+//                 Copyright (2011) Sandia Corporation
+// 
+// Under terms of Contract DE-AC04-94AL85000, there is a non-exclusive
+// license for use of this work by or on behalf of the U.S. Government.
+// 
+// This library is free software; you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as
+// published by the Free Software Foundation; either version 2.1 of the
+// License, or (at your option) any later version.
+//  
+// This library is distributed in the hope that it will be useful, but
+// WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+// Lesser General Public License for more details.
+//  
+// You should have received a copy of the GNU Lesser General Public
+// License along with this library; if not, write to the Free Software
+// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+// USA
+// Questions? Contact Michael A. Heroux ([e-mail]) 
+// 
+// ************************************************************************
+//@HEADER
+
+// Author : Karthik V Aadithya
+// Mentor : Heidi K Thornquist
+// Date : July 2010
+
+double mX_timer(void);
+#endif // __MX_TIMER_H__

MultiSource/Benchmarks/DOE-ProxyApps-C++/miniXyce/mX_timer.cpp

+//@HEADER
+// ************************************************************************
+// 
+//               miniXyce: A simple circuit simulation benchmark code
+//                 Copyright (2011) Sandia Corporation
+// 
+// Under terms of Contract DE-AC04-94AL85000, there is a non-exclusive
+// license for use of this work by or on behalf of the U.S. Government.
+// 
+// This library is free software; you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as
+// published by the Free Software Foundation; either version 2.1 of the
+// License, or (at your option) any later version.
+//  
+// This library is distributed in the hope that it will be useful, but
+// WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+// Lesser General Public License for more details.
+//  
+// You should have received a copy of the GNU Lesser General Public
+// License along with this library; if not, write to the Free Software
+// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+// USA
+// Questions? Contact Michael A. Heroux ([e-mail]) 
+// 
+// ************************************************************************
+//@HEADER
+
+// Function to return time in seconds.
+// If compiled with no flags, return CPU time (user and system).
+// If compiled with -DWALL, returns elapsed time.
+
+/////////////////////////////////////////////////////////////////////////
+#ifdef HAVE_MPI
+#include <mpi.h> // If this routine is compiled with -DHAVE_MPI
+                 // then include mpi.h
+double mX_timer(void)
+{
+   return(MPI_Wtime());
+}
+
+
+#elif defined(UseClock)
+
+#include <time.hpp>
+double mX_timer(void)
+{
+   clock_t t1;
+   static clock_t t0=0;
+   static double CPS = CLOCKS_PER_SEC;
+   double d;
+
+   if (t0 == 0) t0 = clock();
+   t1 = clock() - t0;
+   d = t1 / CPS;
+   return(d);
+}
+
+#elif defined(WALL)
+
+#include <cstdlib>
+#include <sys/time.h>
+#include <sys/resource.h>
+double mX_timer(void)
+{
+   struct timeval tp;
+   static long start=0, startu;
+   if (!start)
+   {
+      gettimeofday(&tp, NULL);
+      start = tp.tv_sec;
+      startu = tp.tv_usec;
+      return(0.0);
+   }
+   gettimeofday(&tp, NULL);
+   return( ((double) (tp.tv_sec - start)) + (tp.tv_usec-startu)/1000000.0 );
+}
+
+#elif defined(UseTimes)
+
+#include <cstdlib>
+#include <sys/times.h>
+#include <unistd.h>
+double mX_timer(void)
+{
+   struct tms ts;
+   static double ClockTick=0.0;
+
+   if (ClockTick == 0.0) ClockTick = (double) sysconf(_SC_CLK_TCK);
+   times(&ts);
+   return( (double) ts.tms_utime / ClockTick );
+}
+
+#else
+
+#include <cstdlib>
+#include <sys/time.h>
+#include <sys/resource.h>
+double mX_timer(void)
+{
+   struct rusage ruse;
+   getrusage(RUSAGE_SELF, &ruse);
+   return( (double)(ruse.ru_utime.tv_sec+ruse.ru_utime.tv_usec / 1000000.0) );
+}
+
+#endif

MultiSource/Benchmarks/DOE-ProxyApps-C++/miniXyce/miniXyce.reference_output

+Processor : 0 has 0 sends: 
+Mini-Application Name: miniXyce
+Mini-Application Version: 1.0
+Circuit_attributes: 
+  Number_of_devices: 6
+  Resistors_(R): 2
+  Inductors_(L): 1
+  Capacitors_(C): 1
+  Voltage_sources_(V): 1
+  Current_sources_(I): 1
+Matrix_attributes: 
+  Global_rows: 5
+  Rows_per_proc_MIN: 5
+  Rows_per_proc_MAX: 5
+  Rows_per_proc_AVG: 5
+  Global_NNZ: 11
+  NNZ_per_proc_MIN: 11
+  NNZ_per_proc_MAX: 11
+  NNZ_per_proc_AVG: 11
+DCOP Calculation: 
+  Init_cond_specified: 0
+  GMRES_tolerance: 1e-06
+  GMRES_subspace_dim: 10
+  GMRES_iterations: 0
+  GMRES_restarts: 0
+  GMRES_native_residual: 8.97932e-10
+Transient Calculation: 
+  Number_of_time_steps: 900
+  Time_start: 1e-06
+  Time_end: 1e-05
+  Time_step: 1e-08
+  GMRES_tolerance: 1e-06
+  GMRES_subspace_dim: 10
+  GMRES_average_iters: 5
+  GMRES_average_res: 5.79223e-16
+exit 0

MultiSource/Benchmarks/Makefile

 # Disable TSVC on Darwin until the tests support SMALL_PROBLEM_SIZE=1.
 ifeq ($(TARGET_OS),Darwin)
 ifeq ($(ARCH),ARM)
 PARALLEL_DIRS := $(filter-out TSVC,$(PARALLEL_DIRS))
 endif
 endif
 
 ifndef DISABLE_CXX
-PARALLEL_DIRS += Prolangs-C++ PAQ8p tramp3d-v4 Bullet 7zip
+PARALLEL_DIRS += Prolangs-C++ PAQ8p tramp3d-v4 Bullet 7zip DOE-ProxyApps-C++
 endif
 
 ifndef SMALL_PROBLEM_SIZE
 PARALLEL_DIRS += \
     mafft
 endif
 
 ifeq ($(ARCH),XCore)
 # XCore does not support pthreads
 PARALLEL_DIRS := $(filter-out 7zip, $(PARALLEL_DIRS))
 # XCore does not support fork().
 # PAQ8p removed pending general method for running tests multiple times.
 PARALLEL_DIRS := $(filter-out PAQ8p, $(PARALLEL_DIRS))
 endif
 
 include $(LEVEL)/MultiSource/Makefile.multisrc