diff --git a/src/sst/elements/ember/tests/qos-dragonfly.sh b/src/sst/elements/ember/tests/qos-dragonfly.sh new file mode 100644 index 0000000000..8dc7dfd3bd --- /dev/null +++ b/src/sst/elements/ember/tests/qos-dragonfly.sh @@ -0,0 +1,13 @@ +LOADFILE="qos.load" + +export PYTHONPATH="../test" + +#../sst \ +/Users/kshemme/work/sst/ember-fix/bin/sst \ +--model-options=" \ +--loadFile=$LOADFILE \ +--platform=default \ +--topo=dragonfly \ +--shape=8:16:16:4 \ +" \ +../test/emberLoad.py diff --git a/src/sst/elements/ember/tests/qos-fattree.sh b/src/sst/elements/ember/tests/qos-fattree.sh new file mode 100644 index 0000000000..9ab5b7cefc --- /dev/null +++ b/src/sst/elements/ember/tests/qos-fattree.sh @@ -0,0 +1,13 @@ +LOADFILE="qos.load" + +export PYTHONPATH="../test" + +#../sst \ +/Users/kshemme/work/sst/ember-fix/bin/sst \ +--model-options=" \ +--loadFile=$LOADFILE \ +--platform=default \ +--topo=fattree \ +--shape=16,16:32 \ +" \ +../test/emberLoad.py diff --git a/src/sst/elements/ember/tests/qos-hyperx.sh b/src/sst/elements/ember/tests/qos-hyperx.sh new file mode 100644 index 0000000000..b43a686122 --- /dev/null +++ b/src/sst/elements/ember/tests/qos-hyperx.sh @@ -0,0 +1,14 @@ +LOADFILE="qos.load" + +export PYTHONPATH="../test" + +#../sst \ +/Users/kshemme/work/sst/ember-fix/bin/sst \ +--model-options=" \ +--loadFile=$LOADFILE \ +--platform=default \ +--topo=hyperx \ +--shape=8x8 \ +--hostsPerRtr=8 \ +" \ +../test/emberLoad.py diff --git a/src/sst/elements/ember/tests/qos.load b/src/sst/elements/ember/tests/qos.load new file mode 100644 index 0000000000..439f8514b6 --- /dev/null +++ b/src/sst/elements/ember/tests/qos.load @@ -0,0 +1,29 @@ +[JOB_ID] 1 +[NID_LIST] generateNidList=generateNidListQOS(512,0,random,128,1) +[PARAM] merlin:portcontrol:output_arb=merlin.arb.output.qos.multi +[PARAM] merlin:portcontrol:arbitration:qos_settings=[25,-1,25,-1,25,-1,25,-1] +[MOTIF] Init +[MOTIF] Sweep3D pex=16 pey=8 nx=16 ny=16 nz=10 kba=10 fields_per_cell=10 iterations=1 computetime=1 +[MOTIF] Allreduce +[MOTIF] Fini + +[JOB_ID] 2 +[NID_LIST] generateNidList=generateNidListQOS(512,1,random,128,1) +[MOTIF] Init +[MOTIF] Sweep3D pex=16 pey=8 nx=16 ny=16 nz=10 kba=10 fields_per_cell=10 iterations=1 computetime=1 +[MOTIF] Allreduce +[MOTIF] Fini + +[JOB_ID] 3 +[NID_LIST] generateNidList=generateNidListQOS(512,2,random,128,1) +[MOTIF] Init +[MOTIF] Sweep3D pex=16 pey=8 nx=16 ny=16 nz=10 kba=10 fields_per_cell=10 iterations=1 computetime=1 +[MOTIF] Allreduce +[MOTIF] Fini + +[JOB_ID] 4 +[NID_LIST] generateNidList=generateNidListQOS(512,3,random,128,1) +[MOTIF] Init +[MOTIF] Sweep3D pex=16 pey=8 nx=16 ny=16 nz=10 kba=10 fields_per_cell=10 iterations=1 computetime=1 +[MOTIF] Allreduce +[MOTIF] Fini diff --git a/src/sst/elements/ember/tests/refFiles/test_qos-dragonfly.out b/src/sst/elements/ember/tests/refFiles/test_qos-dragonfly.out new file mode 100644 index 0000000000..70d7d09069 --- /dev/null +++ b/src/sst/elements/ember/tests/refFiles/test_qos-dragonfly.out @@ -0,0 +1,32 @@ +EMBER: using param directory: paramFiles +EMBER: platform: default +EMBER: network: topology=dragonfly shape=8:16:16:4 +EMBER: numNodes=512 numNics=512 +EMBER: network: BW=4GB/s pktSize=2048B flitSize=8B +set merlinParams portcontrol:output_arb=merlin.arb.output.qos.multi +set merlinParams portcontrol:arbitration:qos_settings=[25,-1,25,-1,25,-1,25,-1] +EMBER: Job=1, nidList='364,299,45,39,423,498,85,283,323,508,126...' +EMBER: Motif='Init' +EMBER: Motif='Sweep3D pex=16 pey=8 nx=16 ny=16 nz=10 kba=10 fields_per_cell=10 iterations=1 computetime=1' +EMBER: Motif='Allreduce' +EMBER: Motif='Fini' +EMBER: Job=2, nidList='496,490,398,491,92,33,64,343,70,316,1,36...' +EMBER: Motif='Init' +EMBER: Motif='Sweep3D pex=16 pey=8 nx=16 ny=16 nz=10 kba=10 fields_per_cell=10 iterations=1 computetime=1' +EMBER: Motif='Allreduce' +EMBER: Motif='Fini' +EMBER: Job=3, nidList='197,467,218,19,148,149,53,258,411,324,15...' +EMBER: Motif='Init' +EMBER: Motif='Sweep3D pex=16 pey=8 nx=16 ny=16 nz=10 kba=10 fields_per_cell=10 iterations=1 computetime=1' +EMBER: Motif='Allreduce' +EMBER: Motif='Fini' +EMBER: Job=4, nidList='306,438,331,140,77,262,3,18,406,353,54,2...' +EMBER: Motif='Init' +EMBER: Motif='Sweep3D pex=16 pey=8 nx=16 ny=16 nz=10 kba=10 fields_per_cell=10 iterations=1 computetime=1' +EMBER: Motif='Allreduce' +EMBER: Motif='Fini' +Allreduce: ranks 128, loop 1, 1 double(s), latency 374.762 us +Allreduce: ranks 128, loop 1, 1 double(s), latency 374.166 us +Allreduce: ranks 128, loop 1, 1 double(s), latency 372.478 us +Allreduce: ranks 128, loop 1, 1 double(s), latency 377.456 us +Simulation is complete, simulated time: 2.87013 ms diff --git a/src/sst/elements/ember/tests/refFiles/test_qos-fattree.out b/src/sst/elements/ember/tests/refFiles/test_qos-fattree.out new file mode 100644 index 0000000000..3b99feadeb --- /dev/null +++ b/src/sst/elements/ember/tests/refFiles/test_qos-fattree.out @@ -0,0 +1,32 @@ +EMBER: using param directory: paramFiles +EMBER: platform: default +EMBER: network: topology=fattree shape=16,16:32 +EMBER: numNodes=512 numNics=512 +EMBER: network: BW=4GB/s pktSize=2048B flitSize=8B +set merlinParams portcontrol:output_arb=merlin.arb.output.qos.multi +set merlinParams portcontrol:arbitration:qos_settings=[25,-1,25,-1,25,-1,25,-1] +EMBER: Job=1, nidList='364,299,45,39,423,498,85,283,323,508,126...' +EMBER: Motif='Init' +EMBER: Motif='Sweep3D pex=16 pey=8 nx=16 ny=16 nz=10 kba=10 fields_per_cell=10 iterations=1 computetime=1' +EMBER: Motif='Allreduce' +EMBER: Motif='Fini' +EMBER: Job=2, nidList='496,490,398,491,92,33,64,343,70,316,1,36...' +EMBER: Motif='Init' +EMBER: Motif='Sweep3D pex=16 pey=8 nx=16 ny=16 nz=10 kba=10 fields_per_cell=10 iterations=1 computetime=1' +EMBER: Motif='Allreduce' +EMBER: Motif='Fini' +EMBER: Job=3, nidList='197,467,218,19,148,149,53,258,411,324,15...' +EMBER: Motif='Init' +EMBER: Motif='Sweep3D pex=16 pey=8 nx=16 ny=16 nz=10 kba=10 fields_per_cell=10 iterations=1 computetime=1' +EMBER: Motif='Allreduce' +EMBER: Motif='Fini' +EMBER: Job=4, nidList='306,438,331,140,77,262,3,18,406,353,54,2...' +EMBER: Motif='Init' +EMBER: Motif='Sweep3D pex=16 pey=8 nx=16 ny=16 nz=10 kba=10 fields_per_cell=10 iterations=1 computetime=1' +EMBER: Motif='Allreduce' +EMBER: Motif='Fini' +Allreduce: ranks 128, loop 1, 1 double(s), latency 364.320 us +Allreduce: ranks 128, loop 1, 1 double(s), latency 364.066 us +Allreduce: ranks 128, loop 1, 1 double(s), latency 374.834 us +Allreduce: ranks 128, loop 1, 1 double(s), latency 369.478 us +Simulation is complete, simulated time: 2.76905 ms diff --git a/src/sst/elements/ember/tests/refFiles/test_qos-hyperx.out b/src/sst/elements/ember/tests/refFiles/test_qos-hyperx.out new file mode 100644 index 0000000000..10a1a9dfbe --- /dev/null +++ b/src/sst/elements/ember/tests/refFiles/test_qos-hyperx.out @@ -0,0 +1,34 @@ +EMBER: using param directory: paramFiles +EMBER: platform: default +8x8 +8 +EMBER: network: topology=hyperx shape=8x8 +EMBER: numNodes=512 numNics=512 +EMBER: network: BW=4GB/s pktSize=2048B flitSize=8B +set merlinParams portcontrol:output_arb=merlin.arb.output.qos.multi +set merlinParams portcontrol:arbitration:qos_settings=[25,-1,25,-1,25,-1,25,-1] +EMBER: Job=1, nidList='364,299,45,39,423,498,85,283,323,508,126...' +EMBER: Motif='Init' +EMBER: Motif='Sweep3D pex=16 pey=8 nx=16 ny=16 nz=10 kba=10 fields_per_cell=10 iterations=1 computetime=1' +EMBER: Motif='Allreduce' +EMBER: Motif='Fini' +EMBER: Job=2, nidList='496,490,398,491,92,33,64,343,70,316,1,36...' +EMBER: Motif='Init' +EMBER: Motif='Sweep3D pex=16 pey=8 nx=16 ny=16 nz=10 kba=10 fields_per_cell=10 iterations=1 computetime=1' +EMBER: Motif='Allreduce' +EMBER: Motif='Fini' +EMBER: Job=3, nidList='197,467,218,19,148,149,53,258,411,324,15...' +EMBER: Motif='Init' +EMBER: Motif='Sweep3D pex=16 pey=8 nx=16 ny=16 nz=10 kba=10 fields_per_cell=10 iterations=1 computetime=1' +EMBER: Motif='Allreduce' +EMBER: Motif='Fini' +EMBER: Job=4, nidList='306,438,331,140,77,262,3,18,406,353,54,2...' +EMBER: Motif='Init' +EMBER: Motif='Sweep3D pex=16 pey=8 nx=16 ny=16 nz=10 kba=10 fields_per_cell=10 iterations=1 computetime=1' +EMBER: Motif='Allreduce' +EMBER: Motif='Fini' +Allreduce: ranks 128, loop 1, 1 double(s), latency 369.060 us +Allreduce: ranks 128, loop 1, 1 double(s), latency 372.252 us +Allreduce: ranks 128, loop 1, 1 double(s), latency 366.944 us +Allreduce: ranks 128, loop 1, 1 double(s), latency 371.756 us +Simulation is complete, simulated time: 2.77703 ms diff --git a/src/sst/elements/merlin/hr_router/hr_router.cc b/src/sst/elements/merlin/hr_router/hr_router.cc index c09fd5fd3c..fc31cc2322 100644 --- a/src/sst/elements/merlin/hr_router/hr_router.cc +++ b/src/sst/elements/merlin/hr_router/hr_router.cc @@ -125,6 +125,7 @@ hr_router::hr_router(ComponentId_t cid, Params& params) : vcs_initialized(false), output(Simulation::getSimulation()->getSimulationOutput()) { + // Get the options for the router id = params.find("id",-1); if ( id == -1 ) { @@ -136,8 +137,32 @@ hr_router::hr_router(ComponentId_t cid, Params& params) : merlin_abort.fatal(CALL_INFO, -1, "hr_router requires num_ports to be specified\n"); } + + // Get the topology + topo = loadUserSubComponent + ("topology", ComponentInfo::SHARE_NONE, num_ports, id); + + if ( !topo ) { + // Backward compatibility + std::string topology = params.find("topology"); + + if ( topology == "" ) { + merlin_abort.fatal(CALL_INFO, -1, "hr_router requires topology to be specified\n"); + } + +DISABLE_WARN_DEPRECATED_DECLARATION + topo = dynamic_cast(loadSubComponent(topology,this,params)); +REENABLE_WARNING + if ( !topo ) { + merlin_abort.fatal(CALL_INFO, -1, "Unable to find topology '%s'\n", topology.c_str()); + } + } + // Get the number of VNs num_vns = params.find("num_vns",-1); + if ( num_vns != -1 ) { + num_vcs = topo->computeNumVCs(num_vns); + } // If num VNs is specified, we also need to check to see if remap is on vn_remap_shm = params.find("vn_remap_shm",""); if ( vn_remap_shm != "" ) { @@ -158,26 +183,6 @@ hr_router::hr_router(ComponentId_t cid, Params& params) : } } - // Get the topology - topo = loadUserSubComponent - ("topology", ComponentInfo::SHARE_NONE, num_ports, id); - - if ( !topo ) { - // Backward compatibility - std::string topology = params.find("topology"); - - if ( topology == "" ) { - merlin_abort.fatal(CALL_INFO, -1, "hr_router requires topology to be specified\n"); - } - -DISABLE_WARN_DEPRECATED_DECLARATION - topo = dynamic_cast(loadSubComponent(topology,this,params)); -REENABLE_WARNING - if ( !topo ) { - merlin_abort.fatal(CALL_INFO, -1, "Unable to find topology '%s'\n", topology.c_str()); - } - } - // Parse all the timing parameters // Flit size @@ -687,7 +692,6 @@ hr_router::init_vcs() // in_buf_sizes[i] = input_buf_size; // out_buf_sizes[i] = output_buf_size; // } - vc_heads = new internal_router_event*[num_ports*num_vcs]; xbar_in_credits = new int[num_ports*num_vcs]; output_queue_lengths = new int[num_ports*num_vcs]; diff --git a/src/sst/elements/merlin/topology/hyperx.cc b/src/sst/elements/merlin/topology/hyperx.cc index 7ad3b4d751..8eb9e199f8 100644 --- a/src/sst/elements/merlin/topology/hyperx.cc +++ b/src/sst/elements/merlin/topology/hyperx.cc @@ -92,21 +92,27 @@ topo_hyperx::topo_hyperx(Component* comp, Params& params) : if ( !route_algo.compare("DOAL") ) { // std::cout << "Setting algorithm to DOAL" << std::endl; algorithm = DOAL; + vcs_per_vn = 2; } else if ( !route_algo.compare("valiant") ) { algorithm = VALIANT; + vcs_per_vn = 1; } else if ( !route_algo.compare("VDAL") ) { algorithm = VDAL; + vcs_per_vn = 2 * dimensions; } else if ( !route_algo.compare("DOR-ND") ) { algorithm = DORND; + vcs_per_vn = 1; } else if ( !route_algo.compare("DOR") ) { algorithm = DOR; + vcs_per_vn = 1; } else if ( !route_algo.compare("MIN-A") ) { algorithm = MINA; + vcs_per_vn = dimensions; } else { output.fatal(CALL_INFO,-1,"Unknown routing mode specified: %s\n",route_algo.c_str()); @@ -126,7 +132,6 @@ topo_hyperx::topo_hyperx(ComponentId_t cid, Params& params, int num_ports, int r Topology(cid), router_id(rtr_id) { - TraceFunction trace(CALL_INFO_LONG); // Get the various parameters std::string shape; shape = params.find("shape"); @@ -276,7 +281,7 @@ topo_hyperx::process_input(RtrEvent* ev) { topo_hyperx_event* tt_ev = new topo_hyperx_event(dimensions); tt_ev->setEncapsulatedEvent(ev); - tt_ev->setVC(num_vcs * ev->request->vn); + tt_ev->setVC(vcs_per_vn * ev->request->vn); if ( algorithm == VALIANT ) { int mid; do { @@ -435,17 +440,7 @@ topo_hyperx::choose_multipath(int start_port, int num_ports) int topo_hyperx::computeNumVCs(int vns) { - switch ( algorithm ) { - case VDAL: - return 2 * dimensions * vns; - case MINA: - return dimensions * vns; - case DOR: - case DORND: - return vns; - default: - return 2 * vns; - } + return vcs_per_vn * vns; } int @@ -736,7 +731,7 @@ topo_hyperx::routeVDAL(int port, int vc, topo_hyperx_event* ev) { // trace.getOutput().output("%llu: udims.size = %lu, remaining_vcs = %d\n",ev->id.first,udims.size(),num_vcs - start_vc - 1 ); // Check to see if there are extra VCs for misroutes. If not, // simply fall back to MIN-A routing - if ( udims.size() == num_vcs - start_vc - 1 ) { + if ( udims.size() == vcs_per_vn - start_vc - 1 ) { // trace.getOutput().output("Falling back to MIN-A, udims.size = %lu, remaining_vcs = %d\n",udims.size(),num_vcs - start_vc - 1 ); return routeMINA(port,vc,ev); } diff --git a/src/sst/elements/merlin/topology/hyperx.h b/src/sst/elements/merlin/topology/hyperx.h index 7779ac9c27..d7ff9a82b1 100644 --- a/src/sst/elements/merlin/topology/hyperx.h +++ b/src/sst/elements/merlin/topology/hyperx.h @@ -192,6 +192,7 @@ class topo_hyperx: public Topology { int const* output_credits; int const* output_queue_lengths; int num_vcs; + int vcs_per_vn; RouteAlgo algorithm; RNG::SSTRandom* rng;