[Debian-astro-commits] [gyoto] 49/221: Better control of when MPI_Initialize() and MPI_Finalize() are called.
Thibaut Jean-Claude Paumard
thibaut at moszumanska.debian.org
Fri May 22 20:52:32 UTC 2015
This is an automated email from the git hooks/post-receive script.
thibaut pushed a commit to branch master
in repository gyoto.
commit 6d902aa3ff440f9543e1133bbacfb4543644707a
Author: Thibaut Paumard <paumard at users.sourceforge.net>
Date: Tue Oct 14 17:59:41 2014 +0200
Better control of when MPI_Initialize() and MPI_Finalize() are called.
---
bin/gyoto-mpi-worker.C | 6 +---
bin/gyoto.C | 10 ++-----
include/GyotoScenery.h | 9 ++++--
lib/Scenery.C | 78 +++++++++++++++++-------------------------------
yorick/gyoto.i | 40 +++++++++++++++++++++++++
yorick/gyoto_namespace.i | 2 ++
yorick/gyoto_utils.C | 23 ++++++++++++++
7 files changed, 102 insertions(+), 66 deletions(-)
diff --git a/bin/gyoto-mpi-worker.C b/bin/gyoto-mpi-worker.C
index 2a1cee1..3a9043f 100644
--- a/bin/gyoto-mpi-worker.C
+++ b/bin/gyoto-mpi-worker.C
@@ -89,8 +89,6 @@ int main(int argc, char** argv) {
int rk=world.rank();
sc = new Scenery();
- sc -> mpi_world_ = &world;
- sc -> mpi_env_ = &env;
sc -> mpi_team_ = &team;
Scenery::mpi_tag task=Scenery::give_task;
@@ -104,8 +102,6 @@ int main(int argc, char** argv) {
curmsg = "In gyoto-mpi-worker.C: Error in Factory creation: ";
curretval = 1;
sc = Factory(const_cast<char*>(parfile.c_str())).getScenery();
- sc -> mpi_world_ = &world;
- sc -> mpi_env_ = &env;
sc -> mpi_team_ = &team;
break;
}
@@ -118,7 +114,7 @@ int main(int argc, char** argv) {
sc = NULL;
break;
default:
- std::cerr << "unknown task" << endl;
+ std::cerr << "unknown task " << task << endl;
}
}
diff --git a/bin/gyoto.C b/bin/gyoto.C
index b3433f9..5055ebf 100644
--- a/bin/gyoto.C
+++ b/bin/gyoto.C
@@ -257,14 +257,8 @@ int main(int argc, char** argv) {
if (xincl) screen -> inclination ( incl );
if (xpaln) screen -> PALN ( paln );
if (xarg) screen -> argument ( arg );
- if (xnthreads) scenery -> nThreads ( nthreads );
-#ifdef HAVE_MPI
- if (xnprocs) {
- cerr << "nprocs="<< nprocs<< endl;
- scenery -> mpiSpawn(nprocs);
- scenery -> mpiClone();
- }
-#endif
+ if (xnthreads) scenery -> nThreads ( nthreads );
+ if (xnprocs) scenery -> nProcesses ( nprocs );
if (ipctfile != "") {
// if (verbose() >= GYOTO_QUIET_VERBOSITY)
diff --git a/include/GyotoScenery.h b/include/GyotoScenery.h
index 1ad05cc..e46225c 100644
--- a/include/GyotoScenery.h
+++ b/include/GyotoScenery.h
@@ -200,6 +200,8 @@ class Gyoto::Scenery : protected Gyoto::SmartPointee {
*/
size_t nthreads_; ///< Number of parallel threads to use in rayTrace()
+ int nprocesses_; ///< Number of parallel processes to use in rayTrace()
+
# ifdef HAVE_UDUNITS
/// See Astrobj::Properties::intensity_converter_
Gyoto::SmartPointer<Gyoto::Units::Converter> intensity_converter_;
@@ -211,12 +213,10 @@ class Gyoto::Scenery : protected Gyoto::SmartPointee {
# ifdef HAVE_MPI
public:
- boost::mpi::environment * mpi_env_;
- boost::mpi::communicator * mpi_world_;
boost::mpi::communicator * mpi_team_;
static bool am_worker;
void mpiSpawn(int nbchildren);
- void mpiTerminate (bool keep_env=false);
+ void mpiTerminate ();
void mpiClone();
enum mpi_tag {give_task, read_scenery, terminate,
raytrace, raytrace_done, ready,
@@ -346,6 +346,9 @@ class Gyoto::Scenery : protected Gyoto::SmartPointee {
void nThreads(size_t); ///< Set nthreads_;
size_t nThreads() const ; ///< Get nthreads_;
+ void nProcesses(int); ///< Set nprocesses_;
+ int nProcesses() const ; ///< Get nprocesses_;
+
/// Set Scenery::intensity_converter_
void intensityConverter(std::string unit);
/// Set Scenery::spectrum_converter_
diff --git a/lib/Scenery.C b/lib/Scenery.C
index 4c2ec17..f2e857d 100644
--- a/lib/Scenery.C
+++ b/lib/Scenery.C
@@ -52,7 +52,7 @@ Scenery::Scenery() :
screen_(NULL), delta_(GYOTO_DEFAULT_DELTA),
quantities_(0), ph_(), nthreads_(0)
#ifdef HAVE_MPI
- , mpi_env_(NULL), mpi_world_(NULL), mpi_team_(NULL)
+ , mpi_team_(NULL)
#endif
{}
@@ -62,7 +62,7 @@ Scenery::Scenery(SmartPointer<Metric::Generic> met,
screen_(scr), delta_(GYOTO_DEFAULT_DELTA),
quantities_(0), ph_(), nthreads_(0)
#ifdef HAVE_MPI
- , mpi_env_(NULL), mpi_world_(NULL), mpi_team_(NULL)
+ , mpi_team_(NULL)
#endif
{
metric(met);
@@ -76,7 +76,7 @@ Scenery::Scenery(const Scenery& o) :
quantities_(o.quantities_), ph_(o.ph_),
nthreads_(o.nthreads_)
#ifdef HAVE_MPI
- , mpi_env_(NULL), mpi_world_(NULL), mpi_team_(NULL)
+ , mpi_team_(NULL)
#endif
{
if (o.screen_()) {
@@ -127,6 +127,9 @@ void Scenery::delta(double d, const string &unit) {
void Scenery::nThreads(size_t n) { nthreads_ = n; }
size_t Scenery::nThreads() const { return nthreads_; }
+void Scenery::nProcesses(int n) { nprocesses_ = n; }
+int Scenery::nProcesses() const { return nprocesses_; }
+
typedef struct SceneryThreadWorkerArg {
#ifdef HAVE_PTHREAD
pthread_mutex_t * mutex;
@@ -240,7 +243,19 @@ void Scenery::rayTrace(size_t imin, size_t imax,
Astrobj::Properties *data,
double * impactcoords) {
-
+#if defined HAVE_MPI
+ bool need_to_terminate=false;
+ if (nprocesses_ && mpi_team_) {
+ if (!MPI::Is_initialized() && !MPI::Is_finalized()) need_to_terminate=true;
+ if (MPI::Is_finalized()) {
+ GYOTO_SEVERE
+ << "MPI_Finalize() has been called already, won't use MPI"<< endl;
+ } else {
+ mpiSpawn(nprocesses_);
+ mpiClone();
+ }
+ }
+#endif
/*
Ray-trace now is multi-threaded. What it does is
@@ -288,17 +303,6 @@ void Scenery::rayTrace(size_t imin, size_t imax,
eol_offset = data->dj*npix - data->di*((imax-imin)/data->di+1);
}
- if (data) {
- int toto=debug();
- debug(1);
- GYOTO_DEBUG_EXPR(data->di);
- GYOTO_DEBUG_EXPR(data->dj);
- GYOTO_DEBUG_EXPR(data->alloc);
- GYOTO_DEBUG_EXPR(curcell);
- GYOTO_DEBUG_EXPR(eol_offset);
- GYOTO_DEBUG_EXPR((imax-imin)%data->di);
- debug(toto);
- }
#ifdef HAVE_MPI
if (mpi_team_) {
// We are in an MPI content, either the manager or a worker.
@@ -439,19 +443,6 @@ void Scenery::rayTrace(size_t imin, size_t imax,
if (ij[0]<=imax) {
cell[w]=curcell; // store curcell
mpi_team_ -> send(w, raytrace, ij, 2);
-
- if (data) {
- int toto=debug();
- debug(1);
- GYOTO_DEBUG_EXPR(data->di);
- GYOTO_DEBUG_EXPR(data->dj);
- GYOTO_DEBUG_EXPR(data->alloc);
- GYOTO_DEBUG_EXPR(curcell);
- GYOTO_DEBUG_EXPR(eol_offset);
- GYOTO_DEBUG_EXPR((imax-imin)%data->di);
- debug(toto);
- }
-
if (impactcoords) {
mpi_team_ -> send(w, Scenery::impactcoords, impactcoords+cell[w]*16, 16);
}
@@ -473,6 +464,10 @@ void Scenery::rayTrace(size_t imin, size_t imax,
}
}
if (verbose()) cout << endl;
+ if (need_to_terminate) {
+ mpiTerminate();
+ MPI_Finalize();
+ }
} else {
// We are a worker, do we need to query for impactcoords?
double ipct[16];
@@ -838,6 +833,7 @@ void Scenery::fillElement(FactoryMessenger *fmp) {
fmp -> setParameter("MinimumTime", tMin());
fmp -> setParameter("NThreads", nthreads_);
+ fmp -> setParameter("NProcesses", nprocesses_);
}
SmartPointer<Scenery> Gyoto::Scenery::Subcontractor(FactoryMessenger* fmp) {
@@ -872,18 +868,9 @@ SmartPointer<Scenery> Gyoto::Scenery::Subcontractor(FactoryMessenger* fmp) {
if (name=="DeltaMaxOverR") sc -> ph_ . deltaMaxOverR (atof(content.c_str()));
if (name=="AbsTol") sc -> ph_ . absTol(atof(content.c_str()));
if (name=="RelTol") sc -> ph_ . relTol(atof(content.c_str()));
- if (name=="NProcesses") mpi=atoi(tc);
-
- }
-#ifdef HAVE_MPI
+ if (name=="NProcesses") sc -> nProcesses(atoi(content.c_str()));
- if (!Scenery::am_worker && mpi) {
- sc -> mpiSpawn(mpi);
- sc -> mpiClone();
}
-
-#endif
-
return sc;
}
#endif
@@ -905,12 +892,11 @@ void Gyoto::Scenery::mpiSpawn(int nbchildren) {
if (mpi_team_) {
if (mpi_team_->size()==nbchildren+1) return;
- mpiTerminate(true);
+ mpiTerminate();
}
if (nbchildren) {
- if (!mpi_env_) mpi_env_ = new mpi::environment();
- if (!mpi_world_) mpi_world_ = new mpi::communicator();
+ if (!MPI::Is_initialized()) MPI::Init();
MPI_Comm children_c;
MPI_Comm_spawn(const_cast<char*>("gyoto-mpi-worker"),
@@ -922,7 +908,7 @@ void Gyoto::Scenery::mpiSpawn(int nbchildren) {
}
}
-void Gyoto::Scenery::mpiTerminate(bool keep_env) {
+void Gyoto::Scenery::mpiTerminate() {
if (mpi_team_) {
mpi_tag tag=terminate;
mpiTask(tag);
@@ -930,14 +916,6 @@ void Gyoto::Scenery::mpiTerminate(bool keep_env) {
delete mpi_team_;
mpi_team_=NULL;
}
- if (mpi_world_ && !keep_env) {
- delete mpi_world_;
- mpi_world_=NULL;
- }
- if (mpi_env_ && !keep_env) {
- delete mpi_env_;
- mpi_env_=NULL;
- }
}
void Gyoto::Scenery::mpiClone()
diff --git a/yorick/gyoto.i b/yorick/gyoto.i
index 5f2f8e8..25973e2 100644
--- a/yorick/gyoto.i
+++ b/yorick/gyoto.i
@@ -65,6 +65,27 @@ extern gyoto_haveMPI;
HAVE_MPI=1 if compiled with MPI, else 0.
*/
+extern gyoto_mpiFinalize;
+/* DOCUMENT gyoto.mpiFinalize;
+ Finalize MPI.
+
+ Unlike the underlying implementation, does not trigger an error if
+ MPI is already finalized.
+
+ This is a noop if MPI is not compiled-in.
+
+ SEE ALSO: gyoto.haveMPI, gyoto.mpiFinalized
+*/
+
+extern gyoto_mpiFinalized;
+/* DOCUMENT is_finalized=gyoto.mpiFinalized();
+ Tell whether some implemention of MPI_Finalize() was already called.
+
+ If MPI support is not present, return 1.
+
+ SEE ALSO: gyoto.haveMPI, gyoto.mpiFinalize
+ */
+
extern __gyoto_setErrorHandler;
/* xDOCUMENT __gyoto_setErrorHandler
Must be called once to attach the GYOTO error handler to Yorick's one
@@ -532,6 +553,25 @@ extern gyoto_Scenery;
The "Spectrum" quantity is a bit peculiar since it take more than
one plane in data.
+
+ PARALLEL COMPUTING:
+
+ Gyoto supports parallel computing using either multi-threading
+ (pthreads) or multi-processing (MPI). Not all classes work well
+ using multi-threading (in particular, Lorene metrics are not
+ thread-safe). On the other hand, all of Gyoto should support
+ multi-processing.
+
+ If you want to use mutli-processing, you should take care of:
+ - calling 'sc, mpispawn=<nprocs>' before sending ray-tracing;
+ - calling 'sc, mpiclone=;' once the scenery is ready, before tracing;
+ - before quitting yorick:
+ * destroy all of your sceneries with sc[];
+ * call gyoto.mpiFinalize().
+
+ libgyoto may automatically initialize MPI, in which case it will
+ also automatically terminate it. To prevent this behaviour, use
+ mpispawn at least once before ray-tracing.
SEE ALSO:
gyoto.Metric, gyoto.Screen, gyoto.Astrobj, gyoto.Photon,
diff --git a/yorick/gyoto_namespace.i b/yorick/gyoto_namespace.i
index 8553209..fe5d564 100644
--- a/yorick/gyoto_namespace.i
+++ b/yorick/gyoto_namespace.i
@@ -5,6 +5,8 @@ gyoto=save(
haveUDUNITS=gyoto_haveUDUNITS,
havePTHREAD=gyoto_havePTHREAD,
haveMPI=gyoto_haveMPI,
+ mpiFinalize=gyoto_mpiFinalize,
+ mpiFinalized=gyoto_mpiFinalized,
loadPlugin=gyoto_loadPlugin,
Scenery=gyoto_Scenery,
diff --git a/yorick/gyoto_utils.C b/yorick/gyoto_utils.C
index 9916112..07a8f47 100644
--- a/yorick/gyoto_utils.C
+++ b/yorick/gyoto_utils.C
@@ -26,6 +26,10 @@
#include <signal.h>
#include <vector>
+#if defined HAVE_MPI
+# include <mpi.h>
+#endif
+
#include "ygyoto.h"
#include "ygyoto_private.h"
@@ -184,6 +188,25 @@ extern "C" {
}
void
+ Y_gyoto_mpiFinalize(int)
+ {
+ ypush_nil();
+#if defined HAVE_MPI
+ if (!MPI::Is_finalized()) MPI::Finalize();
+#endif
+ }
+
+ void
+ Y_gyoto_mpiFinalized(int argc)
+ {
+#if defined HAVE_MPI
+ ypush_long(MPI::Is_finalized());
+#else
+ ypush_long(1);
+#endif
+ }
+
+ void
Y___gyoto_exportSupplier(int)
{
if (!YGyotoGlobalSupplier) {
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/debian-astro/packages/gyoto.git
More information about the Debian-astro-commits
mailing list