From 68ae584210be1d9df99075c78f5e0e093b12c9bf Mon Sep 17 00:00:00 2001 From: Brad Whitlock Date: Wed, 21 Mar 2018 10:03:23 -0700 Subject: [PATCH 1/5] Use rectilinear grid that uses simulation's coordinate arrays. Make some changes so we don't have empty vtkMultiBlockDataSet blocks. --- Jacobi/Sensei/C/Bridge.cxx | 3 +- Jacobi/Sensei/C/Bridge.h | 3 +- Jacobi/Sensei/C/jacobi_sensei_2.xml | 7 +- Jacobi/Sensei/C/pjacobi.c | 2 +- .../Sensei/C/solution/JacobiDataAdaptor.cxx | 92 ++++++++++++++++++- Jacobi/Sensei/C/solution/JacobiDataAdaptor.h | 5 +- Jacobi/Sensei/C/solvers.c | 2 +- 7 files changed, 102 insertions(+), 12 deletions(-) diff --git a/Jacobi/Sensei/C/Bridge.cxx b/Jacobi/Sensei/C/Bridge.cxx index 527ced9..b7e3e45 100644 --- a/Jacobi/Sensei/C/Bridge.cxx +++ b/Jacobi/Sensei/C/Bridge.cxx @@ -39,10 +39,11 @@ void bridge_initialize(MPI_Comm comm, } //----------------------------------------------------------------------------- -void bridge_update(int tstep, double time, double* temperature) +void bridge_update(int tstep, double time, float *cx, float *cy, double* temperature) { BridgeInternals::GlobalDataAdaptor->SetDataTime(time); BridgeInternals::GlobalDataAdaptor->SetDataTimeStep(tstep); + BridgeInternals::GlobalDataAdaptor->SetCoordinates(cx, cy); BridgeInternals::GlobalDataAdaptor->AddArray("temperature", temperature); if (!BridgeInternals::GlobalAnalysisAdaptor->Execute(BridgeInternals::GlobalDataAdaptor)) { diff --git a/Jacobi/Sensei/C/Bridge.h b/Jacobi/Sensei/C/Bridge.h index 305d636..5f2a5b4 100644 --- a/Jacobi/Sensei/C/Bridge.h +++ b/Jacobi/Sensei/C/Bridge.h @@ -15,7 +15,8 @@ extern "C" { const char* config_file); /// Called per timestep in the simulation loop - void bridge_update(int tstep, double time, double* temperature); + void bridge_update(int tstep, double time, + float *cx, float *cy, double* temperature); /// Called just before simulation terminates. void bridge_finalize(); diff --git a/Jacobi/Sensei/C/jacobi_sensei_2.xml b/Jacobi/Sensei/C/jacobi_sensei_2.xml index 0aa7775..bd54627 100644 --- a/Jacobi/Sensei/C/jacobi_sensei_2.xml +++ b/Jacobi/Sensei/C/jacobi_sensei_2.xml @@ -10,12 +10,13 @@ + filename="jacobi_catalyst_sensei_2.py" enabled="0" /> - + slice-project="1" image-format="png" enabled="1"/> diff --git a/Jacobi/Sensei/C/pjacobi.c b/Jacobi/Sensei/C/pjacobi.c index 3a8f5ab..1a052dd 100644 --- a/Jacobi/Sensei/C/pjacobi.c +++ b/Jacobi/Sensei/C/pjacobi.c @@ -62,7 +62,7 @@ int main(int argc, char *argv[]) { // iterate until error below threshold simulate_one_timestep(&sim); #ifdef ENABLE_SENSEI - bridge_update(sim.iter, sim.iter*1.0, sim.Temp); + bridge_update(sim.iter, sim.iter*1.0, sim.cx, sim.cy, sim.Temp); #endif } diff --git a/Jacobi/Sensei/C/solution/JacobiDataAdaptor.cxx b/Jacobi/Sensei/C/solution/JacobiDataAdaptor.cxx index 9a3204c..4fabd1d 100644 --- a/Jacobi/Sensei/C/solution/JacobiDataAdaptor.cxx +++ b/Jacobi/Sensei/C/solution/JacobiDataAdaptor.cxx @@ -7,6 +7,8 @@ #include #include #include +#include +#include #include #if defined(SENSEI_2) @@ -15,6 +17,8 @@ #include +#define RECTILINEAR_MESH + namespace pjacobi { //----------------------------------------------------------------------------- @@ -23,6 +27,7 @@ senseiNewMacro(JacobiDataAdaptor); //----------------------------------------------------------------------------- JacobiDataAdaptor::JacobiDataAdaptor() { + this->cx = this->cy = NULL; } //----------------------------------------------------------------------------- @@ -31,7 +36,7 @@ JacobiDataAdaptor::~JacobiDataAdaptor() } //----------------------------------------------------------------------------- -void JacobiDataAdaptor::Initialize(int m, int rankx, int ranky, int bx, int by, int ng) +void JacobiDataAdaptor::Initialize(int m, int rankx, int ranky, int bx_, int by_, int ng) { (void)m; // m could be used for whole extents [0 m+2 0 m+2 0 0] @@ -49,6 +54,18 @@ void JacobiDataAdaptor::Initialize(int m, int rankx, int ranky, int bx, int by, this->Extent[3] = this->Extent[2] + by + 2*ng - 1; this->Extent[4] = 0; this->Extent[5] = 0; + +#ifdef RECTILINEAR_MESH + this->bx = bx_; + this->by = by_; +#endif +} + +//----------------------------------------------------------------------------- +void JacobiDataAdaptor::SetCoordinates(float *cx_, float *cy_) +{ + this->cx = cx_; + this->cy = cy_; } //----------------------------------------------------------------------------- @@ -111,16 +128,45 @@ int JacobiDataAdaptor::GetMesh(const std::string &meshName, bool structureOnly, int n_ranks = 1; MPI_Comm_size(MPI_COMM_WORLD, &n_ranks); +#ifdef RECTILINEAR_MESH + vtkRectilinearGrid *block = vtkRectilinearGrid::New(); + int dims[3]; + dims[0] = this->bx+2; + dims[1] = this->by+2; + dims[2] = 1; + block->SetDimensions(dims); + + vtkFloatArray *x = vtkFloatArray::New(); + vtkFloatArray *y = vtkFloatArray::New(); + vtkFloatArray *z = vtkFloatArray::New(); + int dontDelete = 1; + x->SetArray(this->cx, this->bx+2, dontDelete); + y->SetArray(this->cy, this->by+2, dontDelete); + z->SetNumberOfTuples(1); + z->SetTuple1(0,0.); + block->SetXCoordinates(x); + block->SetYCoordinates(y); + block->SetZCoordinates(z); + x->Delete(); + y->Delete(); + z->Delete(); + this->Mesh = vtkSmartPointer::New(); + this->Mesh->SetNumberOfBlocks(1); + this->Mesh->SetBlock(0, block); + + block->Delete(); +#else vtkImageData *block = vtkImageData::New(); block->SetExtent(this->Extent); block->SetOrigin(this->Origin); block->SetSpacing(this->Spacing); this->Mesh = vtkSmartPointer::New(); - this->Mesh->SetNumberOfBlocks(n_ranks); + this->Mesh->SetNumberOfBlocks(n_ranks); // ?? n_ranks here this->Mesh->SetBlock(rank, block); block->Delete(); +#endif } mesh = this->Mesh; @@ -135,14 +181,18 @@ int JacobiDataAdaptor::AddArray(vtkDataObject* mesh, const std::string &meshName vtkMultiBlockDataSet *mb = dynamic_cast(mesh); if (!mb) { - SENSEI_ERROR("Invlaid mesh type " << mesh->GetClassName()) + SENSEI_ERROR("Invalid mesh type " << mesh->GetClassName()) return -1; } int rank = 0; MPI_Comm_rank(MPI_COMM_WORLD, &rank); +#ifdef RECTILINEAR_MESH + vtkRectilinearGrid *block = dynamic_cast(mb->GetBlock(0)); +#else vtkImageData *block = dynamic_cast(mb->GetBlock(rank)); +#endif if (!block) return 0; @@ -173,8 +223,12 @@ int JacobiDataAdaptor::AddArray(vtkDataObject* mesh, const std::string &meshName vtkarray = vtkSmartPointer::New(); vtkarray->SetName(arrayName.c_str()); +#ifdef RECTILINEAR_MESH + vtkIdType size = (this->bx+2) * (this->by+2); +#else vtkIdType size = (this->Extent[1] - this->Extent[0] + 1) * (this->Extent[3] - this->Extent[2] + 1) * (this->Extent[5] - this->Extent[4] + 1); +#endif vtkarray->SetArray(iterV->second, size, 1); @@ -252,17 +306,47 @@ vtkDataObject* JacobiDataAdaptor::GetMesh(bool vtkNotUsed(structure_only)) int n_ranks = 1; MPI_Comm_size(MPI_COMM_WORLD, &n_ranks); +#ifdef RECTILINEAR_MESH + vtkRectilinearGrid *block = vtkRectilinearGrid::New(); + int dims[3]; + dims[0] = this->bx+2; + dims[1] = this->by+2; + dims[2] = 1; + block->SetDimensions(dims); + + vtkFloatArray *x = vtkFloatArray::New(); + vtkFloatArray *y = vtkFloatArray::New(); + vtkFloatArray *z = vtkFloatArray::New(); + int dontDelete = 1; + x->SetArray(this->cx, this->bx+2, dontDelete); + y->SetArray(this->cy, this->by+2, dontDelete); + z->SetNumberOfTuples(1); + z->SetTuple1(0,0.); + block->SetXCoordinates(x); + block->SetYCoordinates(y); + block->SetZCoordinates(z); + x->Delete(); + y->Delete(); + z->Delete(); + this->Mesh = vtkSmartPointer::New(); + this->Mesh->SetNumberOfBlocks(1); + this->Mesh->SetBlock(0, block); + + block->Delete(); +#else vtkImageData *block = vtkImageData::New(); block->SetExtent(this->Extent); block->SetOrigin(this->Origin); block->SetSpacing(this->Spacing); this->Mesh = vtkSmartPointer::New(); - this->Mesh->SetNumberOfBlocks(n_ranks); + this->Mesh->SetNumberOfBlocks(n_ranks); // ?? n_ranks here this->Mesh->SetBlock(rank, block); block->Delete(); +#endif } + return this->Mesh; } diff --git a/Jacobi/Sensei/C/solution/JacobiDataAdaptor.h b/Jacobi/Sensei/C/solution/JacobiDataAdaptor.h index 364f8be..6b9f10a 100644 --- a/Jacobi/Sensei/C/solution/JacobiDataAdaptor.h +++ b/Jacobi/Sensei/C/solution/JacobiDataAdaptor.h @@ -25,6 +25,8 @@ class JacobiDataAdaptor : public sensei::DataAdaptor /// Initialize the data adaptor. void Initialize(int m, int rankx, int ranky, int bx, int by, int ng); + void SetCoordinates(float *cx, float *cy); + /// Set the pointers to simulation memory. void AddArray(const std::string& name, double* data); @@ -68,7 +70,8 @@ class JacobiDataAdaptor : public sensei::DataAdaptor int Extent[6]; double Origin[3]; double Spacing[3]; - + float *cx, *cy; + int bx, by; private: JacobiDataAdaptor(const JacobiDataAdaptor&); // not implemented. void operator=(const JacobiDataAdaptor&); // not implemented. diff --git a/Jacobi/Sensei/C/solvers.c b/Jacobi/Sensei/C/solvers.c index f37b26e..a715573 100644 --- a/Jacobi/Sensei/C/solvers.c +++ b/Jacobi/Sensei/C/solvers.c @@ -1,4 +1,4 @@ -#include +#include #include #include #include From a133e65decd3a14c03c5570457531c7fea76af4a Mon Sep 17 00:00:00 2001 From: Brad Whitlock Date: Wed, 21 Mar 2018 10:42:03 -0700 Subject: [PATCH 2/5] Pass simulation_data structure to the data adaptor. --- Jacobi/Sensei/C/Bridge.cxx | 14 +-- Jacobi/Sensei/C/Bridge.h | 10 +- Jacobi/Sensei/C/pjacobi.c | 6 +- .../Sensei/C/solution/JacobiDataAdaptor.cxx | 97 ++++--------------- Jacobi/Sensei/C/solution/JacobiDataAdaptor.h | 12 +-- Jacobi/Sensei/C/solvers.c | 2 +- Jacobi/Sensei/C/solvers.h | 2 +- 7 files changed, 39 insertions(+), 104 deletions(-) diff --git a/Jacobi/Sensei/C/Bridge.cxx b/Jacobi/Sensei/C/Bridge.cxx index b7e3e45..dbda366 100644 --- a/Jacobi/Sensei/C/Bridge.cxx +++ b/Jacobi/Sensei/C/Bridge.cxx @@ -21,9 +21,7 @@ namespace BridgeInternals } //----------------------------------------------------------------------------- -void bridge_initialize(MPI_Comm comm, - int m, int rankx, int ranky, int bx, int by, int ng, - const char* config_file) +void bridge_initialize(MPI_Comm comm, simulation_data *sim, const char *config_file) { BridgeInternals::comm = comm; @@ -31,7 +29,7 @@ void bridge_initialize(MPI_Comm comm, BridgeInternals::GlobalDataAdaptor = vtkSmartPointer::New(); - BridgeInternals::GlobalDataAdaptor->Initialize(m, rankx, ranky, bx, by, ng); + BridgeInternals::GlobalDataAdaptor->Initialize(sim); // initialize the analysis adaptor BridgeInternals::GlobalAnalysisAdaptor = vtkSmartPointer::New(); @@ -39,12 +37,10 @@ void bridge_initialize(MPI_Comm comm, } //----------------------------------------------------------------------------- -void bridge_update(int tstep, double time, float *cx, float *cy, double* temperature) +void bridge_update(simulation_data *sim) { - BridgeInternals::GlobalDataAdaptor->SetDataTime(time); - BridgeInternals::GlobalDataAdaptor->SetDataTimeStep(tstep); - BridgeInternals::GlobalDataAdaptor->SetCoordinates(cx, cy); - BridgeInternals::GlobalDataAdaptor->AddArray("temperature", temperature); + BridgeInternals::GlobalDataAdaptor->Update(sim); + if (!BridgeInternals::GlobalAnalysisAdaptor->Execute(BridgeInternals::GlobalDataAdaptor)) { cerr << "ERROR: Failed to execute the analysis" << endl; diff --git a/Jacobi/Sensei/C/Bridge.h b/Jacobi/Sensei/C/Bridge.h index 5f2a5b4..4b7e9aa 100644 --- a/Jacobi/Sensei/C/Bridge.h +++ b/Jacobi/Sensei/C/Bridge.h @@ -3,6 +3,7 @@ #include #include +#include #ifdef __cplusplus extern "C" { @@ -10,16 +11,13 @@ extern "C" { /// This defines the analysis bridge for the pjacobi miniapp. /// Called before simulation loop - void bridge_initialize(MPI_Comm comm, - int m, int rankx, int ranky, int bx, int by, int ng, - const char* config_file); + void bridge_initialize(MPI_Comm comm, simulation_data *sim, const char *config); /// Called per timestep in the simulation loop - void bridge_update(int tstep, double time, - float *cx, float *cy, double* temperature); + void bridge_update(simulation_data *sim); /// Called just before simulation terminates. - void bridge_finalize(); + void bridge_finalize(void); #ifdef __cplusplus } // extern "C" diff --git a/Jacobi/Sensei/C/pjacobi.c b/Jacobi/Sensei/C/pjacobi.c index 1a052dd..fb806a9 100644 --- a/Jacobi/Sensei/C/pjacobi.c +++ b/Jacobi/Sensei/C/pjacobi.c @@ -53,8 +53,8 @@ int main(int argc, char *argv[]) const char *config = argv[1]; - bridge_initialize(MPI_COMM_WORLD, sim.m, - sim.rankx, sim.ranky, sim.bx, sim.by, 1, config); + bridge_initialize(MPI_COMM_WORLD, &sim, config);/* sim.m, + sim.rankx, sim.ranky, sim.bx, sim.by, 1, config);*/ #endif @@ -62,7 +62,7 @@ int main(int argc, char *argv[]) { // iterate until error below threshold simulate_one_timestep(&sim); #ifdef ENABLE_SENSEI - bridge_update(sim.iter, sim.iter*1.0, sim.cx, sim.cy, sim.Temp); + bridge_update(&sim); /*sim.iter, sim.iter*1.0, sim.cx, sim.cy, sim.Temp);*/ #endif } diff --git a/Jacobi/Sensei/C/solution/JacobiDataAdaptor.cxx b/Jacobi/Sensei/C/solution/JacobiDataAdaptor.cxx index 4fabd1d..dd2f73b 100644 --- a/Jacobi/Sensei/C/solution/JacobiDataAdaptor.cxx +++ b/Jacobi/Sensei/C/solution/JacobiDataAdaptor.cxx @@ -17,8 +17,6 @@ #include -#define RECTILINEAR_MESH - namespace pjacobi { //----------------------------------------------------------------------------- @@ -27,7 +25,7 @@ senseiNewMacro(JacobiDataAdaptor); //----------------------------------------------------------------------------- JacobiDataAdaptor::JacobiDataAdaptor() { - this->cx = this->cy = NULL; + this->sim = NULL; } //----------------------------------------------------------------------------- @@ -36,36 +34,19 @@ JacobiDataAdaptor::~JacobiDataAdaptor() } //----------------------------------------------------------------------------- -void JacobiDataAdaptor::Initialize(int m, int rankx, int ranky, int bx_, int by_, int ng) +void JacobiDataAdaptor::Initialize(simulation_data *s) //int m, int rankx, int ranky, int bx_, int by_, int ng) { - (void)m; // m could be used for whole extents [0 m+2 0 m+2 0 0] - - this->Origin[0] = -double(ng); - this->Origin[1] = -double(ng); - this->Origin[2] = 0.0; - - this->Spacing[0] = 1.0; - this->Spacing[1] = 1.0; - this->Spacing[2] = 1.0; - - this->Extent[0] = rankx*(bx - 1); - this->Extent[1] = this->Extent[0] + bx + 2*ng - 1; - this->Extent[2] = ranky*by; - this->Extent[3] = this->Extent[2] + by + 2*ng - 1; - this->Extent[4] = 0; - this->Extent[5] = 0; - -#ifdef RECTILINEAR_MESH - this->bx = bx_; - this->by = by_; -#endif + this->sim = s; } //----------------------------------------------------------------------------- -void JacobiDataAdaptor::SetCoordinates(float *cx_, float *cy_) +void JacobiDataAdaptor::Update(simulation_data *s) { - this->cx = cx_; - this->cy = cy_; + this->sim = s; + + this->SetDataTime(sim->iter*1.); + this->SetDataTimeStep(sim->iter); + this->AddArray("temperature", sim->Temp); } //----------------------------------------------------------------------------- @@ -128,11 +109,10 @@ int JacobiDataAdaptor::GetMesh(const std::string &meshName, bool structureOnly, int n_ranks = 1; MPI_Comm_size(MPI_COMM_WORLD, &n_ranks); -#ifdef RECTILINEAR_MESH vtkRectilinearGrid *block = vtkRectilinearGrid::New(); int dims[3]; - dims[0] = this->bx+2; - dims[1] = this->by+2; + dims[0] = this->sim->bx+2; + dims[1] = this->sim->by+2; dims[2] = 1; block->SetDimensions(dims); @@ -140,8 +120,8 @@ int JacobiDataAdaptor::GetMesh(const std::string &meshName, bool structureOnly, vtkFloatArray *y = vtkFloatArray::New(); vtkFloatArray *z = vtkFloatArray::New(); int dontDelete = 1; - x->SetArray(this->cx, this->bx+2, dontDelete); - y->SetArray(this->cy, this->by+2, dontDelete); + x->SetArray(this->sim->cx, this->sim->bx+2, dontDelete); + y->SetArray(this->sim->cy, this->sim->by+2, dontDelete); z->SetNumberOfTuples(1); z->SetTuple1(0,0.); block->SetXCoordinates(x); @@ -155,18 +135,6 @@ int JacobiDataAdaptor::GetMesh(const std::string &meshName, bool structureOnly, this->Mesh->SetBlock(0, block); block->Delete(); -#else - vtkImageData *block = vtkImageData::New(); - block->SetExtent(this->Extent); - block->SetOrigin(this->Origin); - block->SetSpacing(this->Spacing); - - this->Mesh = vtkSmartPointer::New(); - this->Mesh->SetNumberOfBlocks(n_ranks); // ?? n_ranks here - this->Mesh->SetBlock(rank, block); - - block->Delete(); -#endif } mesh = this->Mesh; @@ -188,11 +156,7 @@ int JacobiDataAdaptor::AddArray(vtkDataObject* mesh, const std::string &meshName int rank = 0; MPI_Comm_rank(MPI_COMM_WORLD, &rank); -#ifdef RECTILINEAR_MESH vtkRectilinearGrid *block = dynamic_cast(mb->GetBlock(0)); -#else - vtkImageData *block = dynamic_cast(mb->GetBlock(rank)); -#endif if (!block) return 0; @@ -222,14 +186,7 @@ int JacobiDataAdaptor::AddArray(vtkDataObject* mesh, const std::string &meshName vtkSmartPointer& vtkarray = this->Arrays[iterV->first]; vtkarray = vtkSmartPointer::New(); vtkarray->SetName(arrayName.c_str()); - -#ifdef RECTILINEAR_MESH - vtkIdType size = (this->bx+2) * (this->by+2); -#else - vtkIdType size = (this->Extent[1] - this->Extent[0] + 1) * - (this->Extent[3] - this->Extent[2] + 1) * (this->Extent[5] - this->Extent[4] + 1); -#endif - + vtkIdType size = (this->sim->bx+2) * (this->sim->by+2); vtkarray->SetArray(iterV->second, size, 1); block->GetPointData()->SetScalars(vtkarray); @@ -306,11 +263,10 @@ vtkDataObject* JacobiDataAdaptor::GetMesh(bool vtkNotUsed(structure_only)) int n_ranks = 1; MPI_Comm_size(MPI_COMM_WORLD, &n_ranks); -#ifdef RECTILINEAR_MESH vtkRectilinearGrid *block = vtkRectilinearGrid::New(); int dims[3]; - dims[0] = this->bx+2; - dims[1] = this->by+2; + dims[0] = this->sim->bx+2; + dims[1] = this->sim->by+2; dims[2] = 1; block->SetDimensions(dims); @@ -318,8 +274,8 @@ vtkDataObject* JacobiDataAdaptor::GetMesh(bool vtkNotUsed(structure_only)) vtkFloatArray *y = vtkFloatArray::New(); vtkFloatArray *z = vtkFloatArray::New(); int dontDelete = 1; - x->SetArray(this->cx, this->bx+2, dontDelete); - y->SetArray(this->cy, this->by+2, dontDelete); + x->SetArray(this->sim->cx, this->sim->bx+2, dontDelete); + y->SetArray(this->sim->cy, this->sim->by+2, dontDelete); z->SetNumberOfTuples(1); z->SetTuple1(0,0.); block->SetXCoordinates(x); @@ -333,18 +289,6 @@ vtkDataObject* JacobiDataAdaptor::GetMesh(bool vtkNotUsed(structure_only)) this->Mesh->SetBlock(0, block); block->Delete(); -#else - vtkImageData *block = vtkImageData::New(); - block->SetExtent(this->Extent); - block->SetOrigin(this->Origin); - block->SetSpacing(this->Spacing); - - this->Mesh = vtkSmartPointer::New(); - this->Mesh->SetNumberOfBlocks(n_ranks); // ?? n_ranks here - this->Mesh->SetBlock(rank, block); - - block->Delete(); -#endif } return this->Mesh; @@ -367,7 +311,7 @@ bool JacobiDataAdaptor::AddArray(vtkDataObject* mesh, int association, const std int rank = 0; MPI_Comm_rank(MPI_COMM_WORLD, &rank); vtkMultiBlockDataSet *mb = dynamic_cast(mesh); - vtkImageData *block = mb ? dynamic_cast(mb->GetBlock(rank)) : nullptr; + vtkRectilinearGrid *block = mb ? dynamic_cast(mb->GetBlock(0)) : nullptr; if (!block) return false; @@ -377,8 +321,7 @@ bool JacobiDataAdaptor::AddArray(vtkDataObject* mesh, int association, const std vtkSmartPointer& vtkarray = this->Arrays[iterV->first]; vtkarray = vtkSmartPointer::New(); vtkarray->SetName(name.c_str()); - vtkIdType size = (this->Extent[1] - this->Extent[0] + 1) * - (this->Extent[3] - this->Extent[2] + 1) * (this->Extent[5] - this->Extent[4] + 1); + vtkIdType size = (this->sim->bx+2) * (this->sim->by+2); vtkarray->SetArray(iterV->second, size, 1); block->GetPointData()->SetScalars(vtkarray); return true; diff --git a/Jacobi/Sensei/C/solution/JacobiDataAdaptor.h b/Jacobi/Sensei/C/solution/JacobiDataAdaptor.h index 6b9f10a..2b8035f 100644 --- a/Jacobi/Sensei/C/solution/JacobiDataAdaptor.h +++ b/Jacobi/Sensei/C/solution/JacobiDataAdaptor.h @@ -2,6 +2,8 @@ #define PJACOBI_DATAADAPTOR_H #include +#include +#include "solvers.h" #include "vtkSmartPointer.h" #include #include @@ -23,9 +25,9 @@ class JacobiDataAdaptor : public sensei::DataAdaptor senseiTypeMacro(JacobiDataAdaptor, sensei::DataAdaptor); /// Initialize the data adaptor. - void Initialize(int m, int rankx, int ranky, int bx, int by, int ng); + void Initialize(simulation_data *sim); - void SetCoordinates(float *cx, float *cy); + void Update(simulation_data *sim); /// Set the pointers to simulation memory. void AddArray(const std::string& name, double* data); @@ -67,11 +69,7 @@ class JacobiDataAdaptor : public sensei::DataAdaptor vtkSmartPointer Mesh; - int Extent[6]; - double Origin[3]; - double Spacing[3]; - float *cx, *cy; - int bx, by; + simulation_data *sim; private: JacobiDataAdaptor(const JacobiDataAdaptor&); // not implemented. void operator=(const JacobiDataAdaptor&); // not implemented. diff --git a/Jacobi/Sensei/C/solvers.c b/Jacobi/Sensei/C/solvers.c index a715573..26600d7 100644 --- a/Jacobi/Sensei/C/solvers.c +++ b/Jacobi/Sensei/C/solvers.c @@ -15,7 +15,7 @@ void SimInitialize(simulation_data *sim) sim->savingFiles = 0; sim->saveCounter = 0; sim->batch = 0; - sim->export = 0; + sim->do_export = 0; sim->sessionfile = NULL; sim->par_rank = 0; sim->par_size = 1; diff --git a/Jacobi/Sensei/C/solvers.h b/Jacobi/Sensei/C/solvers.h index 6f707f8..cdbd01e 100644 --- a/Jacobi/Sensei/C/solvers.h +++ b/Jacobi/Sensei/C/solvers.h @@ -19,7 +19,7 @@ typedef struct int savingFiles; int saveCounter; int batch; - int export; + int do_export; char *sessionfile; } simulation_data; From d26e55fa40c3ca647c3ee1db0d0d4b83dd8d63b8 Mon Sep 17 00:00:00 2001 From: Brad Whitlock Date: Wed, 21 Mar 2018 15:00:17 -0700 Subject: [PATCH 3/5] Use simulation_data in the adaptor. Ghost node support. --- Jacobi/Sensei/C/pjacobi.c | 5 +- .../Sensei/C/solution/JacobiDataAdaptor.cxx | 80 ++++++++++++++++++- Jacobi/Sensei/C/solution/JacobiDataAdaptor.h | 3 + 3 files changed, 84 insertions(+), 4 deletions(-) diff --git a/Jacobi/Sensei/C/pjacobi.c b/Jacobi/Sensei/C/pjacobi.c index fb806a9..a1180c1 100644 --- a/Jacobi/Sensei/C/pjacobi.c +++ b/Jacobi/Sensei/C/pjacobi.c @@ -53,8 +53,7 @@ int main(int argc, char *argv[]) const char *config = argv[1]; - bridge_initialize(MPI_COMM_WORLD, &sim, config);/* sim.m, - sim.rankx, sim.ranky, sim.bx, sim.by, 1, config);*/ + bridge_initialize(MPI_COMM_WORLD, &sim, config); #endif @@ -62,7 +61,7 @@ int main(int argc, char *argv[]) { // iterate until error below threshold simulate_one_timestep(&sim); #ifdef ENABLE_SENSEI - bridge_update(&sim); /*sim.iter, sim.iter*1.0, sim.cx, sim.cy, sim.Temp);*/ + bridge_update(&sim); #endif } diff --git a/Jacobi/Sensei/C/solution/JacobiDataAdaptor.cxx b/Jacobi/Sensei/C/solution/JacobiDataAdaptor.cxx index dd2f73b..03fbc4b 100644 --- a/Jacobi/Sensei/C/solution/JacobiDataAdaptor.cxx +++ b/Jacobi/Sensei/C/solution/JacobiDataAdaptor.cxx @@ -9,6 +9,7 @@ #include #include #include +#include #include #if defined(SENSEI_2) @@ -34,7 +35,7 @@ JacobiDataAdaptor::~JacobiDataAdaptor() } //----------------------------------------------------------------------------- -void JacobiDataAdaptor::Initialize(simulation_data *s) //int m, int rankx, int ranky, int bx_, int by_, int ng) +void JacobiDataAdaptor::Initialize(simulation_data *s) { this->sim = s; } @@ -243,6 +244,83 @@ int JacobiDataAdaptor::GetArrayName(const std::string &meshName, int association return 0; } +int JacobiDataAdaptor::GetMeshHasGhostNodes(const std::string &meshName, + bool &hasGhostNodes, int &nLayers) +{ + if (meshName != "mesh") + { + hasGhostNodes = false; + nLayers = 0; + SENSEI_ERROR("No mesh named " << meshName) + return -1; + } + + hasGhostNodes = true; + nLayers = 1; + return 0; +} + +int JacobiDataAdaptor::AddGhostNodesArray(vtkDataObject* mesh, const std::string &meshName) +{ + if (meshName != "mesh") + { + SENSEI_ERROR("No mesh named " << meshName) + return -1; + } + + vtkMultiBlockDataSet *mb = dynamic_cast(mesh); + if (!mb) + { + SENSEI_ERROR("Invalid mesh type " << mesh->GetClassName()) + return -1; + } + + vtkRectilinearGrid *block = dynamic_cast(mb->GetBlock(0)); + if (!block) + return -1; + + int nx = this->sim->bx+2; + int ny = this->sim->by+2; + vtkUnsignedCharArray *gn = vtkUnsignedCharArray::New(); + gn->SetNumberOfTuples(nx*ny); + gn->SetName(GHOST_NODE_ARRAY_NAME().c_str()); + unsigned char *gptr = (unsigned char *)gn->GetVoidPointer(0); + memset(gn->GetVoidPointer(0), 0, nx*nx*sizeof(unsigned char)); + unsigned char VISIT_GHOSTNODE_BLANK = 2; + // Left column + if(this->sim->rankx > 0) + { + int i = 0, j = 0; + for( ; j < ny; ++j) + gptr[j*nx+i] = VISIT_GHOSTNODE_BLANK; + } + // Right column + if(this->sim->rankx < this->sim->cart_dims[0]-1) + { + int i = nx-1, j = 0; + for( ; j < ny; ++j) + gptr[j*nx+i] = VISIT_GHOSTNODE_BLANK; + } + // Bottom row + if(this->sim->ranky > 0) + { + int i = 0, j = 0; + for( ; i < nx; ++i) + gptr[j*nx+i] = VISIT_GHOSTNODE_BLANK; + } + // Top row + if(this->sim->ranky < this->sim->cart_dims[1]-1) + { + int i = 0, j = ny-1; + for( ; i < nx; ++i) + gptr[j*nx+i] = VISIT_GHOSTNODE_BLANK; + } + block->GetPointData()->SetScalars(gn); + gn->Delete(); + + return 0; +} + //----------------------------------------------------------------------------- int JacobiDataAdaptor::ReleaseData() { diff --git a/Jacobi/Sensei/C/solution/JacobiDataAdaptor.h b/Jacobi/Sensei/C/solution/JacobiDataAdaptor.h index 2b8035f..2eed3ce 100644 --- a/Jacobi/Sensei/C/solution/JacobiDataAdaptor.h +++ b/Jacobi/Sensei/C/solution/JacobiDataAdaptor.h @@ -49,6 +49,9 @@ class JacobiDataAdaptor : public sensei::DataAdaptor int GetArrayName(const std::string &meshName, int association, unsigned int index, std::string &arrayName) override; int ReleaseData() override; + + int GetMeshHasGhostNodes(const std::string &meshName, bool &hasGhostNodes, int &nLayers) override; + int AddGhostNodesArray(vtkDataObject* mesh, const std::string &meshName) override; #else vtkDataObject* GetMesh(bool structure_only=false) override; bool AddArray(vtkDataObject* mesh, int association, const std::string& arrayname) override; From 69dc6fb4c40ab59f1b83c3a12a4ef6b4e07e40bf Mon Sep 17 00:00:00 2001 From: Brad Whitlock Date: Wed, 21 Mar 2018 15:01:33 -0700 Subject: [PATCH 4/5] Ghost node support. --- Jacobi/Sensei/C/solution/JacobiDataAdaptor.cxx | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Jacobi/Sensei/C/solution/JacobiDataAdaptor.cxx b/Jacobi/Sensei/C/solution/JacobiDataAdaptor.cxx index 03fbc4b..38a1528 100644 --- a/Jacobi/Sensei/C/solution/JacobiDataAdaptor.cxx +++ b/Jacobi/Sensei/C/solution/JacobiDataAdaptor.cxx @@ -286,34 +286,34 @@ int JacobiDataAdaptor::AddGhostNodesArray(vtkDataObject* mesh, const std::string gn->SetName(GHOST_NODE_ARRAY_NAME().c_str()); unsigned char *gptr = (unsigned char *)gn->GetVoidPointer(0); memset(gn->GetVoidPointer(0), 0, nx*nx*sizeof(unsigned char)); - unsigned char VISIT_GHOSTNODE_BLANK = 2; + unsigned char ghost = 1; // Left column if(this->sim->rankx > 0) { int i = 0, j = 0; for( ; j < ny; ++j) - gptr[j*nx+i] = VISIT_GHOSTNODE_BLANK; + gptr[j*nx+i] = ghost; } // Right column if(this->sim->rankx < this->sim->cart_dims[0]-1) { int i = nx-1, j = 0; for( ; j < ny; ++j) - gptr[j*nx+i] = VISIT_GHOSTNODE_BLANK; + gptr[j*nx+i] = ghost; } // Bottom row if(this->sim->ranky > 0) { int i = 0, j = 0; for( ; i < nx; ++i) - gptr[j*nx+i] = VISIT_GHOSTNODE_BLANK; + gptr[j*nx+i] = ghost; } // Top row if(this->sim->ranky < this->sim->cart_dims[1]-1) { int i = 0, j = ny-1; for( ; i < nx; ++i) - gptr[j*nx+i] = VISIT_GHOSTNODE_BLANK; + gptr[j*nx+i] = ghost; } block->GetPointData()->SetScalars(gn); gn->Delete(); From ae5e05ffeb1800ac65842437f7d4254cdb404fc8 Mon Sep 17 00:00:00 2001 From: Brad Whitlock Date: Tue, 27 Mar 2018 17:32:32 -0700 Subject: [PATCH 5/5] Changed back to vtkImageData by default and changed extents so it makes the same mesh as the rectilinear version. Changed vtkMultiBlockDataSet code so we set the rank dataset and have many empties. Fixed nx*nx typo. --- .../Sensei/C/solution/JacobiDataAdaptor.cxx | 123 ++++++++++++------ Jacobi/Sensei/C/solvers.c | 2 +- 2 files changed, 87 insertions(+), 38 deletions(-) diff --git a/Jacobi/Sensei/C/solution/JacobiDataAdaptor.cxx b/Jacobi/Sensei/C/solution/JacobiDataAdaptor.cxx index 38a1528..c7d3d57 100644 --- a/Jacobi/Sensei/C/solution/JacobiDataAdaptor.cxx +++ b/Jacobi/Sensei/C/solution/JacobiDataAdaptor.cxx @@ -18,6 +18,8 @@ #include +#define USE_IMAGE_DATA + namespace pjacobi { //----------------------------------------------------------------------------- @@ -110,6 +112,30 @@ int JacobiDataAdaptor::GetMesh(const std::string &meshName, bool structureOnly, int n_ranks = 1; MPI_Comm_size(MPI_COMM_WORLD, &n_ranks); +#ifdef USE_IMAGE_DATA + int Extent[6]; + double Origin[3], Spacing[3]; + Extent[0] = this->sim->rankx*this->sim->bx; + Extent[1] = Extent[0] + this->sim->bx + 2 - 1; + Extent[2] = this->sim->ranky*this->sim->by; + Extent[3] = Extent[2] + this->sim->by + 2 - 1; + Extent[4] = 0; + Extent[5] = 0; + + Spacing[0] = this->sim->cx[1] - this->sim->cx[0]; + Spacing[1] = this->sim->cy[1] - this->sim->cy[0]; + Spacing[2] = 0.; + + // Origin on rank 0 using cx[0], cy[0] is zero. + Origin[0] = 0.; + Origin[1] = 0.; + Origin[2] = 0.; + + vtkImageData *block = vtkImageData::New(); + block->SetExtent(Extent); + block->SetOrigin(Origin); + block->SetSpacing(Spacing); +#else vtkRectilinearGrid *block = vtkRectilinearGrid::New(); int dims[3]; dims[0] = this->sim->bx+2; @@ -131,9 +157,10 @@ int JacobiDataAdaptor::GetMesh(const std::string &meshName, bool structureOnly, x->Delete(); y->Delete(); z->Delete(); +#endif this->Mesh = vtkSmartPointer::New(); - this->Mesh->SetNumberOfBlocks(1); - this->Mesh->SetBlock(0, block); + this->Mesh->SetNumberOfBlocks(n_ranks); + this->Mesh->SetBlock(rank, block); block->Delete(); } @@ -157,7 +184,11 @@ int JacobiDataAdaptor::AddArray(vtkDataObject* mesh, const std::string &meshName int rank = 0; MPI_Comm_rank(MPI_COMM_WORLD, &rank); - vtkRectilinearGrid *block = dynamic_cast(mb->GetBlock(0)); +#ifdef USE_IMAGE_DATA + vtkImageData *block = dynamic_cast(mb->GetBlock(rank)); +#else + vtkRectilinearGrid *block = dynamic_cast(mb->GetBlock(rank)); +#endif if (!block) return 0; @@ -275,7 +306,14 @@ int JacobiDataAdaptor::AddGhostNodesArray(vtkDataObject* mesh, const std::string return -1; } - vtkRectilinearGrid *block = dynamic_cast(mb->GetBlock(0)); + int rank = 0; + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + +#ifdef USE_IMAGE_DATA + vtkImageData *block = dynamic_cast(mb->GetBlock(rank)); +#else + vtkRectilinearGrid *block = dynamic_cast(mb->GetBlock(rank)); +#endif if (!block) return -1; @@ -285,37 +323,18 @@ int JacobiDataAdaptor::AddGhostNodesArray(vtkDataObject* mesh, const std::string gn->SetNumberOfTuples(nx*ny); gn->SetName(GHOST_NODE_ARRAY_NAME().c_str()); unsigned char *gptr = (unsigned char *)gn->GetVoidPointer(0); - memset(gn->GetVoidPointer(0), 0, nx*nx*sizeof(unsigned char)); + memset(gn->GetVoidPointer(0), 0, nx*ny*sizeof(unsigned char)); unsigned char ghost = 1; - // Left column - if(this->sim->rankx > 0) - { - int i = 0, j = 0; - for( ; j < ny; ++j) - gptr[j*nx+i] = ghost; - } - // Right column - if(this->sim->rankx < this->sim->cart_dims[0]-1) - { - int i = nx-1, j = 0; - for( ; j < ny; ++j) - gptr[j*nx+i] = ghost; - } - // Bottom row - if(this->sim->ranky > 0) - { - int i = 0, j = 0; - for( ; i < nx; ++i) - gptr[j*nx+i] = ghost; - } - // Top row - if(this->sim->ranky < this->sim->cart_dims[1]-1) - { - int i = 0, j = ny-1; - for( ; i < nx; ++i) - gptr[j*nx+i] = ghost; - } - block->GetPointData()->SetScalars(gn); + for(int i = 0, j = 0 ; j < ny; ++j) + gptr[j*nx+i] = ghost; + for(int i = nx-1, j = 0 ; j < ny; ++j) + gptr[j*nx+i] = ghost; + for(int i = 0, j = 0 ; i < nx; ++i) + gptr[j*nx+i] = ghost; + for(int i = 0, j = ny-1 ; i < nx; ++i) + gptr[j*nx+i] = ghost; + + block->GetPointData()->AddArray(gn); gn->Delete(); return 0; @@ -341,6 +360,30 @@ vtkDataObject* JacobiDataAdaptor::GetMesh(bool vtkNotUsed(structure_only)) int n_ranks = 1; MPI_Comm_size(MPI_COMM_WORLD, &n_ranks); +#ifdef USE_IMAGE_DATA + int Extent[6]; + double Origin[3], Spacing[3]; + Extent[0] = this->sim->rankx*this->sim->bx; + Extent[1] = Extent[0] + this->sim->bx + 2 - 1; + Extent[2] = this->sim->ranky*this->sim->by; + Extent[3] = Extent[2] + this->sim->by + 2 - 1; + Extent[4] = 0; + Extent[5] = 0; + + Spacing[0] = this->sim->cx[1] - this->sim->cx[0]; + Spacing[1] = this->sim->cy[1] - this->sim->cy[0]; + Spacing[2] = 0.; + + // Origin on rank 0 using cx[0], cy[0] is zero. + Origin[0] = 0.; + Origin[1] = 0.; + Origin[2] = 0.; + + vtkImageData *block = vtkImageData::New(); + block->SetExtent(Extent); + block->SetOrigin(Origin); + block->SetSpacing(Spacing); +#else vtkRectilinearGrid *block = vtkRectilinearGrid::New(); int dims[3]; dims[0] = this->sim->bx+2; @@ -362,9 +405,11 @@ vtkDataObject* JacobiDataAdaptor::GetMesh(bool vtkNotUsed(structure_only)) x->Delete(); y->Delete(); z->Delete(); +#endif + this->Mesh = vtkSmartPointer::New(); - this->Mesh->SetNumberOfBlocks(1); - this->Mesh->SetBlock(0, block); + this->Mesh->SetNumberOfBlocks(n_ranks); + this->Mesh->SetBlock(rank, block); block->Delete(); } @@ -389,7 +434,11 @@ bool JacobiDataAdaptor::AddArray(vtkDataObject* mesh, int association, const std int rank = 0; MPI_Comm_rank(MPI_COMM_WORLD, &rank); vtkMultiBlockDataSet *mb = dynamic_cast(mesh); - vtkRectilinearGrid *block = mb ? dynamic_cast(mb->GetBlock(0)) : nullptr; +#ifdef USE_IMAGE_DATA + vtkImageData *block = mb ? dynamic_cast(mb->GetBlock(rank)) : nullptr; +#else + vtkRectilinearGrid *block = mb ? dynamic_cast(mb->GetBlock(rank)) : nullptr; +#endif if (!block) return false; diff --git a/Jacobi/Sensei/C/solvers.c b/Jacobi/Sensei/C/solvers.c index 26600d7..9d674f2 100644 --- a/Jacobi/Sensei/C/solvers.c +++ b/Jacobi/Sensei/C/solvers.c @@ -37,7 +37,7 @@ void MPI_Partition(int PartitioningDimension, simulation_data *sim) sim->cart_dims[1] = 1; MPI_Dims_create(sim->par_size, PartitioningDimension, sim->cart_dims); - fprintf(stdout,"%d: cart_dims[]= %d, %d\n", sim->par_rank, sim->cart_dims[0], sim->cart_dims[1]); + //fprintf(stdout,"%d: cart_dims[]= %d, %d\n", sim->par_rank, sim->cart_dims[0], sim->cart_dims[1]); if(MPI_Cart_create(MPI_COMM_WORLD, 2, sim->cart_dims, periods, 0, &sim->topocomm) != MPI_SUCCESS) sim->topocomm = MPI_COMM_WORLD;