From 4bd7ba5550e43e699c3387a9f77fbac9dcfa9d5e Mon Sep 17 00:00:00 2001 From: Joachim Schoeberl Date: Thu, 21 Jul 2011 20:52:45 +0000 Subject: [PATCH] multithreaded mpi --- libsrc/interface/nginterface.cpp | 9 +- ng/ngappinit.cpp | 2 +- ng/ngpkg.cpp | 5 + ng/parallelfunc.cpp | 361 +++++++++++++------------------ 4 files changed, 168 insertions(+), 209 deletions(-) diff --git a/libsrc/interface/nginterface.cpp b/libsrc/interface/nginterface.cpp index 8d76a82c..ac05082f 100644 --- a/libsrc/interface/nginterface.cpp +++ b/libsrc/interface/nginterface.cpp @@ -72,8 +72,13 @@ { bool parthread = netgen::mparam.parthread; - // if (netgen::id > 0) parthread = true; - if (netgen::ntasks > 1) parthread = false; +#ifdef PARALLEL + int provided; + MPI_Query_thread(&provided); + if (provided < 3) + if (netgen::ntasks > 1) parthread = false; + cout << "runparallel = " << parthread << endl; +#endif if (parthread) { diff --git a/ng/ngappinit.cpp b/ng/ngappinit.cpp index ad8f0940..7baca21f 100644 --- a/ng/ngappinit.cpp +++ b/ng/ngappinit.cpp @@ -65,7 +65,7 @@ int main(int argc, char ** argv) #ifdef PARALLEL // MPI_Init(&argc, &argv); - int required = 0; // MPI_THREAD_MULTIPLE; + int required = MPI_THREAD_MULTIPLE; int provided; MPI_Init_thread(&argc, &argv, required, &provided); diff --git a/ng/ngpkg.cpp b/ng/ngpkg.cpp index e8231f0d..7a5d3b3f 100644 --- a/ng/ngpkg.cpp +++ b/ng/ngpkg.cpp @@ -1148,6 +1148,11 @@ namespace netgen mesh->SetMinimalH (mparam.minh); } +#ifdef PARALLEL + MyMPI_SendCmd ("bcastparthread"); + MyMPI_Bcast (mparam.parthread); +#endif + return TCL_OK; } diff --git a/ng/parallelfunc.cpp b/ng/parallelfunc.cpp index fd001ce4..c5e7ca5e 100644 --- a/ng/parallelfunc.cpp +++ b/ng/parallelfunc.cpp @@ -142,253 +142,204 @@ void ParallelRun() } (*NGS_ParallelRun) (message); } - else - - if ( message == "mesh" ) - { - VT_USER_START ("Mesh::ReceiveParallelMesh"); - mesh.Reset( new netgen::Mesh); - // mesh->ReceiveParallelMesh(); - mesh->SendRecvMesh(); - VT_USER_END ("Mesh::ReceiveParallelMesh"); - } + else if ( message == "mesh" ) + { + VT_USER_START ("Mesh::ReceiveParallelMesh"); + mesh.Reset( new netgen::Mesh); + mesh->SendRecvMesh(); + VT_USER_END ("Mesh::ReceiveParallelMesh"); + } - else if ( message == "overlap++" ) - { - PrintMessage (1, "overlap++++++"); - mesh -> UpdateOverlap(); - } + else if ( message == "overlap++" ) + { + PrintMessage (1, "overlap++++++"); + mesh -> UpdateOverlap(); + } - else if ( message == "visualize" ) - { - cout << "p" << id << ": ACHTUNG - alles wieder zumachen, sonst geht nix mehr :)" << endl; - cout << "Tcl-disabled" << endl; - /* - - // initialize application - Tcl_Interp * myinterp = Tcl_CreateInterp (); - if (Tcl_AppInit (myinterp) == TCL_ERROR) - { - cerr << "Exit Netgen due to initialization problem" << endl; - exit (1); - } - - string startfile = ngdir + "/libsrc/parallel/ng_parallel.tcl"; - - if (verbose) - cout << "Load Tcl-script from " << startfile << endl; - - int errcode = Tcl_EvalFile (myinterp, (char*)startfile.c_str()); - - if (errcode) - { - cout << "Error in Tcl-Script:" << endl; - cout << "result = " << myinterp->result << endl; - cout << "in line " << myinterp->errorLine << endl; - - if (myinterp->errorLine == 1) - cout << "\nMake sure to set environment variable NETGENDIR" << endl; - - exit (1); - } - - // lookup user file formats and insert into format list: - ARRAY userformats; - RegisterUserFormats (userformats); - - ostringstream fstr; - for (int i = 1; i <= userformats.Size(); i++) - fstr << ".ngmenu.file.filetype add radio -label \"" - << userformats.Get(i) << "\" -variable exportfiletype\n"; - - - Tcl_Eval (myinterp, (char*)fstr.str().c_str()); - Tcl_SetVar (myinterp, "exportfiletype", "Neutral Format", 0); - - - - Tk_MainLoop(); - - - Tcl_DeleteInterp (myinterp); -*/ - - } + else if ( message == "visualize" ) + { + cout << "parallel message visualize depreciated" << endl; + } + + else if ( message == "bcastparthread" ) + { + MyMPI_Bcast (mparam.parthread); + } #ifdef PARALLELGL - else if ( message == "redraw" ) - { - // draw into the same GLX - drawing context - // works on parallel machine, but - // did not manage to get glXImportContextEXT working on Laptop (JS) + else if ( message == "redraw" ) + { + // draw into the same GLX - drawing context + // works on parallel machine, but + // did not manage to get glXImportContextEXT working on Laptop (JS) - string redraw_cmd; - // MyMPI_Recv (redraw_cmd, 0, MPI_TAG_VIS); - redraw_cmd = MyMPI_RecvCmd(); + string redraw_cmd; + // MyMPI_Recv (redraw_cmd, 0, MPI_TAG_VIS); + redraw_cmd = MyMPI_RecvCmd(); - // PrintMessage (1, "Redraw - ", redraw_cmd); + // PrintMessage (1, "Redraw - ", redraw_cmd); - static string displname; - static int curDrawable, contextid; + static string displname; + static int curDrawable, contextid; - static Display * display = NULL; - static GLXContext context; - static XVisualInfo * visinfo = 0; + static Display * display = NULL; + static GLXContext context; + static XVisualInfo * visinfo = 0; - // if (!display) - if (redraw_cmd == "init") - { - MyMPI_Recv (displname, 0, MPI_TAG_VIS); - MyMPI_Recv (curDrawable, 0, MPI_TAG_VIS); - MyMPI_Recv (contextid, 0, MPI_TAG_VIS); + // if (!display) + if (redraw_cmd == "init") + { + MyMPI_Recv (displname, 0, MPI_TAG_VIS); + MyMPI_Recv (curDrawable, 0, MPI_TAG_VIS); + MyMPI_Recv (contextid, 0, MPI_TAG_VIS); - display = XOpenDisplay (displname.c_str()); + display = XOpenDisplay (displname.c_str()); - /* - PrintMessage (3, "displ - name = ", displname); - PrintMessage (3, "display = ", display, - " display props: ", - " screen w = ", XDisplayWidth (display, 0), - " , h = ", XDisplayHeight (display, 0)); - */ + /* + PrintMessage (3, "displ - name = ", displname); + PrintMessage (3, "display = ", display, + " display props: ", + " screen w = ", XDisplayWidth (display, 0), + " , h = ", XDisplayHeight (display, 0)); + */ - Window win; - int wx, wy; - unsigned int ww, wh, bw, depth; - // cout << "got drawable: " << curDrawable << endl; + Window win; + int wx, wy; + unsigned int ww, wh, bw, depth; + // cout << "got drawable: " << curDrawable << endl; - XGetGeometry(display, curDrawable, &win, - &wx, &wy, &ww, &wh, - &bw, &depth); + XGetGeometry(display, curDrawable, &win, + &wx, &wy, &ww, &wh, + &bw, &depth); - /* - cout << "P" << id << ": window-props: x = " << wx << ", y = " << wy - << ", w = " << ww << ", h = " << wh << ", depth = " << depth << endl; - */ + /* + cout << "P" << id << ": window-props: x = " << wx << ", y = " << wy + << ", w = " << ww << ", h = " << wh << ", depth = " << depth << endl; + */ #define VISUAL #ifdef VISUAL - // make a new GLXContext - // first, generate a visual (copied from togl) + // make a new GLXContext + // first, generate a visual (copied from togl) - int attrib_list[1000]; + int attrib_list[1000]; # define MAX_ATTEMPTS 12 - static int ci_depths[MAX_ATTEMPTS] = { - 8, 4, 2, 1, 12, 16, 8, 4, 2, 1, 12, 16 - }; - static int dbl_flags[MAX_ATTEMPTS] = { - 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1 - }; + static int ci_depths[MAX_ATTEMPTS] = { + 8, 4, 2, 1, 12, 16, 8, 4, 2, 1, 12, 16 + }; + static int dbl_flags[MAX_ATTEMPTS] = { + 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1 + }; - /* It may take a few tries to get a visual */ + /* It may take a few tries to get a visual */ - for (int attempt = 0; attempt < MAX_ATTEMPTS; attempt++) - { - int attrib_count = 0; - attrib_list[attrib_count++] = GLX_USE_GL; + for (int attempt = 0; attempt < MAX_ATTEMPTS; attempt++) + { + int attrib_count = 0; + attrib_list[attrib_count++] = GLX_USE_GL; - /* RGB[A] mode */ - attrib_list[attrib_count++] = GLX_RGBA; - attrib_list[attrib_count++] = GLX_RED_SIZE; - attrib_list[attrib_count++] = 1; - attrib_list[attrib_count++] = GLX_GREEN_SIZE; - attrib_list[attrib_count++] = 1; - attrib_list[attrib_count++] = GLX_BLUE_SIZE; - attrib_list[attrib_count++] = 1; - // attrib_list[attrib_count++] = GLX_ALPHA_SIZE; - // attrib_list[attrib_count++] = 1; + /* RGB[A] mode */ + attrib_list[attrib_count++] = GLX_RGBA; + attrib_list[attrib_count++] = GLX_RED_SIZE; + attrib_list[attrib_count++] = 1; + attrib_list[attrib_count++] = GLX_GREEN_SIZE; + attrib_list[attrib_count++] = 1; + attrib_list[attrib_count++] = GLX_BLUE_SIZE; + attrib_list[attrib_count++] = 1; + // attrib_list[attrib_count++] = GLX_ALPHA_SIZE; + // attrib_list[attrib_count++] = 1; - attrib_list[attrib_count++] = GLX_DEPTH_SIZE; - attrib_list[attrib_count++] = 1; + attrib_list[attrib_count++] = GLX_DEPTH_SIZE; + attrib_list[attrib_count++] = 1; - attrib_list[attrib_count++] = GLX_DOUBLEBUFFER; + attrib_list[attrib_count++] = GLX_DOUBLEBUFFER; - attrib_list[attrib_count++] = None; + attrib_list[attrib_count++] = None; - visinfo = glXChooseVisual(display, 0, - attrib_list); - if (visinfo) { - /* found a GLX visual! */ - // cout << "found VISINFO !!!" << endl; + visinfo = glXChooseVisual(display, 0, + attrib_list); + if (visinfo) { + /* found a GLX visual! */ + // cout << "found VISINFO !!!" << endl; - /* + /* int hi = 0; std::cout << "attribs = "; while (attrib_list[hi] != None) - std::cout << attrib_list[hi++] << " "; + std::cout << attrib_list[hi++] << " "; std::cout << std::endl; - */ + */ - break; - } + break; } - if (!visinfo) - cerr << "no VISINFO found" << endl; + } + if (!visinfo) + cerr << "no VISINFO found" << endl; - // context = glXCreateContext( display, visinfo, 0, /* curContext, */ False ); - context = glXCreateContext( display, visinfo, glXImportContextEXT ( display, contextid ), False); - // cout << "context = " << context << endl; + // context = glXCreateContext( display, visinfo, 0, /* curContext, */ False ); + context = glXCreateContext( display, visinfo, glXImportContextEXT ( display, contextid ), False); + // cout << "context = " << context << endl; - glXMakeCurrent (display, curDrawable, context); + glXMakeCurrent (display, curDrawable, context); #else - // try to get GLXcontext from the master. - // this needs an indirect context (BUT DOES NOT WORK ????) + // try to get GLXcontext from the master. + // this needs an indirect context (BUT DOES NOT WORK ????) - context = glXImportContextEXT ( display, contextid ); + context = glXImportContextEXT ( display, contextid ); - PrintMessage (1, "GLX-contextid = " , contextid, - " imported context ", context); + PrintMessage (1, "GLX-contextid = " , contextid, + " imported context ", context); - glXMakeCurrent (display, curDrawable, context); + glXMakeCurrent (display, curDrawable, context); #endif - // PrintMessage (1, "redraw - init complete"); - } + // PrintMessage (1, "redraw - init complete"); + } - if (redraw_cmd == "broadcast") - { - vsmesh.Broadcast (); - } + if (redraw_cmd == "broadcast") + { + vsmesh.Broadcast (); + } - if (redraw_cmd == "linelist") - { - // glXMakeCurrent (display, curDrawable, context); - vsmesh.BuildLineList(); - // glXMakeCurrent (display, None, NULL); - } + if (redraw_cmd == "linelist") + { + // glXMakeCurrent (display, curDrawable, context); + vsmesh.BuildLineList(); + // glXMakeCurrent (display, None, NULL); + } - if (redraw_cmd == "filledlist") - { - vsmesh.BuildFilledList (false); - } + if (redraw_cmd == "filledlist") + { + vsmesh.BuildFilledList (false); + } - if (redraw_cmd == "solsurfellist") - { - vssolution.DrawSurfaceElements(); - } + if (redraw_cmd == "solsurfellist") + { + vssolution.DrawSurfaceElements(); + } - if (redraw_cmd == "clipplanetrigs") - { - vssolution.DrawClipPlaneTrigs(); - } + if (redraw_cmd == "clipplanetrigs") + { + vssolution.DrawClipPlaneTrigs(); + } - if (redraw_cmd == "getminmax") - { - double hmin, hmax; - vssolution.GetMinMax (-1, -1, hmin, hmax); - } - } + if (redraw_cmd == "getminmax") + { + double hmin, hmax; + vssolution.GetMinMax (-1, -1, hmin, hmax); + } + } #endif @@ -396,26 +347,24 @@ void ParallelRun() - else if ( message == "end" ) - { - // PrintMessage (1, "EXIT"); - test = false; - // end netgen - Ng_Exit(); - } + else if ( message == "end" ) + { + test = false; + Ng_Exit(); + } - - else - { - PrintMessage ( 1, "received unidentified message '" + message + "'\n"); - - test = false; - } - - } - return; + else + { + PrintMessage ( 1, "received unidentified message '" + message + "'\n"); + + test = false; + } + } + + return; +}