Merge remote-tracking branch 'origin/vc-init-fix' into v9.1.0_beta

src/sst/elements/ember/tests/qos-dragonfly.sh

@@ -0,0 +1,13 @@
+LOADFILE="qos.load"
+
+export PYTHONPATH="../test"
+
+#../sst  \
+/Users/kshemme/work/sst/ember-fix/bin/sst \
+--model-options=" \
+--loadFile=$LOADFILE \
+--platform=default \
+--topo=dragonfly \
+--shape=8:16:16:4 \
+" \
+../test/emberLoad.py

src/sst/elements/ember/tests/qos-fattree.sh

@@ -0,0 +1,13 @@
+LOADFILE="qos.load"
+
+export PYTHONPATH="../test"
+
+#../sst  \
+/Users/kshemme/work/sst/ember-fix/bin/sst \
+--model-options=" \
+--loadFile=$LOADFILE \
+--platform=default \
+--topo=fattree \
+--shape=16,16:32 \
+" \
+../test/emberLoad.py

src/sst/elements/ember/tests/qos-hyperx.sh

@@ -0,0 +1,14 @@
+LOADFILE="qos.load"
+
+export PYTHONPATH="../test"
+
+#../sst  \
+/Users/kshemme/work/sst/ember-fix/bin/sst \
+--model-options=" \
+--loadFile=$LOADFILE \
+--platform=default \
+--topo=hyperx \
+--shape=8x8 \
+--hostsPerRtr=8 \
+" \
+../test/emberLoad.py

src/sst/elements/ember/tests/qos.load

@@ -0,0 +1,29 @@
+[JOB_ID] 1
+[NID_LIST] generateNidList=generateNidListQOS(512,0,random,128,1)
+[PARAM] merlin:portcontrol:output_arb=merlin.arb.output.qos.multi
+[PARAM] merlin:portcontrol:arbitration:qos_settings=[25,-1,25,-1,25,-1,25,-1]
+[MOTIF] Init
+[MOTIF] Sweep3D pex=16 pey=8 nx=16 ny=16 nz=10 kba=10 fields_per_cell=10 iterations=1 computetime=1
+[MOTIF] Allreduce
+[MOTIF] Fini
+
+[JOB_ID] 2
+[NID_LIST] generateNidList=generateNidListQOS(512,1,random,128,1)
+[MOTIF] Init
+[MOTIF] Sweep3D pex=16 pey=8 nx=16 ny=16 nz=10 kba=10 fields_per_cell=10 iterations=1 computetime=1
+[MOTIF] Allreduce
+[MOTIF] Fini
+
+[JOB_ID] 3
+[NID_LIST] generateNidList=generateNidListQOS(512,2,random,128,1)
+[MOTIF] Init
+[MOTIF] Sweep3D pex=16 pey=8 nx=16 ny=16 nz=10 kba=10 fields_per_cell=10 iterations=1 computetime=1
+[MOTIF] Allreduce
+[MOTIF] Fini
+
+[JOB_ID] 4
+[NID_LIST] generateNidList=generateNidListQOS(512,3,random,128,1)
+[MOTIF] Init
+[MOTIF] Sweep3D pex=16 pey=8 nx=16 ny=16 nz=10 kba=10 fields_per_cell=10 iterations=1 computetime=1
+[MOTIF] Allreduce
+[MOTIF] Fini

src/sst/elements/ember/tests/refFiles/test_qos-dragonfly.out

@@ -0,0 +1,32 @@
+EMBER: using param directory: paramFiles
+EMBER: platform: default
+EMBER: network: topology=dragonfly shape=8:16:16:4
+EMBER: numNodes=512 numNics=512
+EMBER: network: BW=4GB/s pktSize=2048B flitSize=8B
+set merlinParams portcontrol:output_arb=merlin.arb.output.qos.multi
+set merlinParams portcontrol:arbitration:qos_settings=[25,-1,25,-1,25,-1,25,-1]
+EMBER: Job=1, nidList='364,299,45,39,423,498,85,283,323,508,126...'
+EMBER: Motif='Init'
+EMBER: Motif='Sweep3D pex=16 pey=8 nx=16 ny=16 nz=10 kba=10 fields_per_cell=10 iterations=1 computetime=1'
+EMBER: Motif='Allreduce'
+EMBER: Motif='Fini'
+EMBER: Job=2, nidList='496,490,398,491,92,33,64,343,70,316,1,36...'
+EMBER: Motif='Init'
+EMBER: Motif='Sweep3D pex=16 pey=8 nx=16 ny=16 nz=10 kba=10 fields_per_cell=10 iterations=1 computetime=1'
+EMBER: Motif='Allreduce'
+EMBER: Motif='Fini'
+EMBER: Job=3, nidList='197,467,218,19,148,149,53,258,411,324,15...'
+EMBER: Motif='Init'
+EMBER: Motif='Sweep3D pex=16 pey=8 nx=16 ny=16 nz=10 kba=10 fields_per_cell=10 iterations=1 computetime=1'
+EMBER: Motif='Allreduce'
+EMBER: Motif='Fini'
+EMBER: Job=4, nidList='306,438,331,140,77,262,3,18,406,353,54,2...'
+EMBER: Motif='Init'
+EMBER: Motif='Sweep3D pex=16 pey=8 nx=16 ny=16 nz=10 kba=10 fields_per_cell=10 iterations=1 computetime=1'
+EMBER: Motif='Allreduce'
+EMBER: Motif='Fini'
+Allreduce: ranks 128, loop 1, 1 double(s), latency 374.762 us
+Allreduce: ranks 128, loop 1, 1 double(s), latency 374.166 us
+Allreduce: ranks 128, loop 1, 1 double(s), latency 372.478 us
+Allreduce: ranks 128, loop 1, 1 double(s), latency 377.456 us
+Simulation is complete, simulated time: 2.87013 ms

src/sst/elements/ember/tests/refFiles/test_qos-fattree.out

@@ -0,0 +1,32 @@
+EMBER: using param directory: paramFiles
+EMBER: platform: default
+EMBER: network: topology=fattree shape=16,16:32
+EMBER: numNodes=512 numNics=512
+EMBER: network: BW=4GB/s pktSize=2048B flitSize=8B
+set merlinParams portcontrol:output_arb=merlin.arb.output.qos.multi
+set merlinParams portcontrol:arbitration:qos_settings=[25,-1,25,-1,25,-1,25,-1]
+EMBER: Job=1, nidList='364,299,45,39,423,498,85,283,323,508,126...'
+EMBER: Motif='Init'
+EMBER: Motif='Sweep3D pex=16 pey=8 nx=16 ny=16 nz=10 kba=10 fields_per_cell=10 iterations=1 computetime=1'
+EMBER: Motif='Allreduce'
+EMBER: Motif='Fini'
+EMBER: Job=2, nidList='496,490,398,491,92,33,64,343,70,316,1,36...'
+EMBER: Motif='Init'
+EMBER: Motif='Sweep3D pex=16 pey=8 nx=16 ny=16 nz=10 kba=10 fields_per_cell=10 iterations=1 computetime=1'
+EMBER: Motif='Allreduce'
+EMBER: Motif='Fini'
+EMBER: Job=3, nidList='197,467,218,19,148,149,53,258,411,324,15...'
+EMBER: Motif='Init'
+EMBER: Motif='Sweep3D pex=16 pey=8 nx=16 ny=16 nz=10 kba=10 fields_per_cell=10 iterations=1 computetime=1'
+EMBER: Motif='Allreduce'
+EMBER: Motif='Fini'
+EMBER: Job=4, nidList='306,438,331,140,77,262,3,18,406,353,54,2...'
+EMBER: Motif='Init'
+EMBER: Motif='Sweep3D pex=16 pey=8 nx=16 ny=16 nz=10 kba=10 fields_per_cell=10 iterations=1 computetime=1'
+EMBER: Motif='Allreduce'
+EMBER: Motif='Fini'
+Allreduce: ranks 128, loop 1, 1 double(s), latency 364.320 us
+Allreduce: ranks 128, loop 1, 1 double(s), latency 364.066 us
+Allreduce: ranks 128, loop 1, 1 double(s), latency 374.834 us
+Allreduce: ranks 128, loop 1, 1 double(s), latency 369.478 us
+Simulation is complete, simulated time: 2.76905 ms

src/sst/elements/ember/tests/refFiles/test_qos-hyperx.out

@@ -0,0 +1,34 @@
+EMBER: using param directory: paramFiles
+EMBER: platform: default
+8x8
+8
+EMBER: network: topology=hyperx shape=8x8
+EMBER: numNodes=512 numNics=512
+EMBER: network: BW=4GB/s pktSize=2048B flitSize=8B
+set merlinParams portcontrol:output_arb=merlin.arb.output.qos.multi
+set merlinParams portcontrol:arbitration:qos_settings=[25,-1,25,-1,25,-1,25,-1]
+EMBER: Job=1, nidList='364,299,45,39,423,498,85,283,323,508,126...'
+EMBER: Motif='Init'
+EMBER: Motif='Sweep3D pex=16 pey=8 nx=16 ny=16 nz=10 kba=10 fields_per_cell=10 iterations=1 computetime=1'
+EMBER: Motif='Allreduce'
+EMBER: Motif='Fini'
+EMBER: Job=2, nidList='496,490,398,491,92,33,64,343,70,316,1,36...'
+EMBER: Motif='Init'
+EMBER: Motif='Sweep3D pex=16 pey=8 nx=16 ny=16 nz=10 kba=10 fields_per_cell=10 iterations=1 computetime=1'
+EMBER: Motif='Allreduce'
+EMBER: Motif='Fini'
+EMBER: Job=3, nidList='197,467,218,19,148,149,53,258,411,324,15...'
+EMBER: Motif='Init'
+EMBER: Motif='Sweep3D pex=16 pey=8 nx=16 ny=16 nz=10 kba=10 fields_per_cell=10 iterations=1 computetime=1'
+EMBER: Motif='Allreduce'
+EMBER: Motif='Fini'
+EMBER: Job=4, nidList='306,438,331,140,77,262,3,18,406,353,54,2...'
+EMBER: Motif='Init'
+EMBER: Motif='Sweep3D pex=16 pey=8 nx=16 ny=16 nz=10 kba=10 fields_per_cell=10 iterations=1 computetime=1'
+EMBER: Motif='Allreduce'
+EMBER: Motif='Fini'
+Allreduce: ranks 128, loop 1, 1 double(s), latency 369.060 us
+Allreduce: ranks 128, loop 1, 1 double(s), latency 372.252 us
+Allreduce: ranks 128, loop 1, 1 double(s), latency 366.944 us
+Allreduce: ranks 128, loop 1, 1 double(s), latency 371.756 us
+Simulation is complete, simulated time: 2.77703 ms

src/sst/elements/merlin/hr_router/hr_router.cc

@@ -125,6 +125,7 @@ hr_router::hr_router(ComponentId_t cid, Params& params) :
     vcs_initialized(false),
     output(Simulation::getSimulation()->getSimulationOutput())
 {
+
     // Get the options for the router
     id = params.find<int>("id",-1);
     if ( id == -1 ) {
@@ -136,8 +137,32 @@ hr_router::hr_router(ComponentId_t cid, Params& params) :
         merlin_abort.fatal(CALL_INFO, -1, "hr_router requires num_ports to be specified\n");
     }
 
+
+    // Get the topology
+    topo = loadUserSubComponent<SST::Merlin::Topology>
+        ("topology", ComponentInfo::SHARE_NONE, num_ports, id);
+
+    if ( !topo ) {
+        // Backward compatibility
+        std::string topology = params.find<std::string>("topology");
+
+        if ( topology == "" ) {
+            merlin_abort.fatal(CALL_INFO, -1, "hr_router requires topology to be specified\n");
+        }
+
+DISABLE_WARN_DEPRECATED_DECLARATION
+        topo = dynamic_cast<Topology*>(loadSubComponent(topology,this,params));
+REENABLE_WARNING
+        if ( !topo ) {
+            merlin_abort.fatal(CALL_INFO, -1, "Unable to find topology '%s'\n", topology.c_str());
+        }
+    }
+
     // Get the number of VNs
     num_vns = params.find<int>("num_vns",-1);
+    if ( num_vns != -1 ) {
+        num_vcs = topo->computeNumVCs(num_vns);
+    }
     // If num VNs is specified, we also need to check to see if remap is on
     vn_remap_shm = params.find<std::string>("vn_remap_shm","");
     if ( vn_remap_shm != "" ) {
@@ -158,26 +183,6 @@ hr_router::hr_router(ComponentId_t cid, Params& params) :
         }
     }
 
-    // Get the topology
-    topo = loadUserSubComponent<SST::Merlin::Topology>
-        ("topology", ComponentInfo::SHARE_NONE, num_ports, id);
-
-    if ( !topo ) {
-        // Backward compatibility
-        std::string topology = params.find<std::string>("topology");
-
-        if ( topology == "" ) {
-            merlin_abort.fatal(CALL_INFO, -1, "hr_router requires topology to be specified\n");
-        }
-
-DISABLE_WARN_DEPRECATED_DECLARATION
-        topo = dynamic_cast<Topology*>(loadSubComponent(topology,this,params));
-REENABLE_WARNING
-        if ( !topo ) {
-            merlin_abort.fatal(CALL_INFO, -1, "Unable to find topology '%s'\n", topology.c_str());
-        }
-    }
-
     // Parse all the timing parameters
 
     // Flit size
@@ -687,7 +692,6 @@ hr_router::init_vcs()
     //     in_buf_sizes[i] = input_buf_size;
     //     out_buf_sizes[i] = output_buf_size;
     // }
-
     vc_heads = new internal_router_event*[num_ports*num_vcs];
     xbar_in_credits = new int[num_ports*num_vcs];
     output_queue_lengths = new int[num_ports*num_vcs];

src/sst/elements/merlin/topology/hyperx.cc

@@ -92,21 +92,27 @@ topo_hyperx::topo_hyperx(Component* comp, Params& params) :
     if ( !route_algo.compare("DOAL") ) {
         // std::cout << "Setting algorithm to DOAL" << std::endl;
         algorithm = DOAL;
+        vcs_per_vn = 2;
     }
     else if ( !route_algo.compare("valiant") ) {
         algorithm = VALIANT;
+        vcs_per_vn = 1;
     }
     else if ( !route_algo.compare("VDAL") ) {
         algorithm = VDAL;
+        vcs_per_vn = 2 * dimensions;
     }
     else if ( !route_algo.compare("DOR-ND") ) {
         algorithm = DORND;
+        vcs_per_vn = 1;
     }
     else if ( !route_algo.compare("DOR") ) {
         algorithm = DOR;
+        vcs_per_vn = 1;
     }
     else if ( !route_algo.compare("MIN-A") ) {
         algorithm = MINA;
+        vcs_per_vn = dimensions;
     }
     else {
         output.fatal(CALL_INFO,-1,"Unknown routing mode specified: %s\n",route_algo.c_str());
@@ -126,7 +132,6 @@ topo_hyperx::topo_hyperx(ComponentId_t cid, Params& params, int num_ports, int r
     Topology(cid),
     router_id(rtr_id)
 {
-    TraceFunction trace(CALL_INFO_LONG);
     // Get the various parameters
     std::string shape;
     shape = params.find<std::string>("shape");
@@ -276,7 +281,7 @@ topo_hyperx::process_input(RtrEvent* ev)
 {
     topo_hyperx_event* tt_ev = new topo_hyperx_event(dimensions);
     tt_ev->setEncapsulatedEvent(ev);
-    tt_ev->setVC(num_vcs * ev->request->vn);
+    tt_ev->setVC(vcs_per_vn * ev->request->vn);
     if ( algorithm == VALIANT ) {
         int mid;
         do {
@@ -435,17 +440,7 @@ topo_hyperx::choose_multipath(int start_port, int num_ports)
 int
 topo_hyperx::computeNumVCs(int vns)
 {
-    switch ( algorithm ) {
-    case VDAL:
-        return 2 * dimensions * vns;
-    case MINA:
-        return dimensions * vns;
-    case DOR:
-    case DORND:
-        return vns;
-    default:
-        return 2 * vns; 
-    }
+    return vcs_per_vn * vns;
 }
 
 int
@@ -736,7 +731,7 @@ topo_hyperx::routeVDAL(int port, int vc, topo_hyperx_event* ev) {
     // trace.getOutput().output("%llu: udims.size = %lu, remaining_vcs = %d\n",ev->id.first,udims.size(),num_vcs - start_vc - 1 );
     // Check to see if there are extra VCs for misroutes.  If not,
     // simply fall back to MIN-A routing
-    if ( udims.size() == num_vcs - start_vc - 1 ) {
+    if ( udims.size() == vcs_per_vn - start_vc - 1 ) {
         // trace.getOutput().output("Falling back to MIN-A, udims.size = %lu, remaining_vcs = %d\n",udims.size(),num_vcs - start_vc - 1 );
         return routeMINA(port,vc,ev);
     }

src/sst/elements/merlin/topology/hyperx.h

@@ -192,6 +192,7 @@ class topo_hyperx: public Topology {
     int const* output_credits;
     int const* output_queue_lengths;
     int num_vcs;
+    int vcs_per_vn;
 
     RouteAlgo algorithm;
     RNG::SSTRandom* rng;