parallel fixes

libsrc/general/array.hpp

@@ -170,8 +170,15 @@ namespace netgen
   public:
 
     /// Generate array of logical and physical size asize
-    explicit Array(int asize = 0)
+    explicit Array()
-      : FlatArray<T, BASE> (asize, asize ? new T[asize] : 0)
+      : FlatArray<T, BASE> (0, NULL)
+    {
+      allocsize = 0; 
+      ownmem = 1;
+    }
+
+    explicit Array(int asize)
+      : FlatArray<T, BASE> (asize, new T[asize])
     {
       allocsize = asize; 
       ownmem = 1;

libsrc/include/nginterface.h

@@ -272,59 +272,15 @@ extern "C" {
 					double * p3d, double * jacobian);
 
 #ifdef PARALLEL
-  // Is Element ei an element of this processor ??
-  bool Ng_IsGhostEl (int ei);
-
-  void Ng_SetGhostEl(const int ei, const bool aisghost );
-
-  bool Ng_IsGhostSEl (int ei);
-
-  void Ng_SetGhostSEl(const int ei, const bool aisghost );
-
-  bool Ng_IsGhostVert ( int pnum );
-  bool Ng_IsGhostEdge ( int ednum );
-  bool Ng_IsGhostFace ( int fanum );
-
-  bool Ng_IsExchangeEl ( int elnum );
-  bool Ng_IsExchangeSEl ( int selnr );
-
-  void Ng_UpdateOverlap ();
-  int Ng_Overlap();
-/*   void Ng_SetGhostVert ( const int pnum, const bool aisghost ); */
-/*   void Ng_SetGhostEdge ( const int ednum, const bool aisghost ); */
-/*   void Ng_SetGhostFace ( const int fanum, const bool aisghost ); */
-
-
-
 
   // the folling functions are 0-base  !!
-  int NgPar_GetLoc2Glob_VolEl ( int locnum );
-
-  // int NgPar_GetDistantNodeNums ( int nt, int locnum, int * procs, int * distnum);
 
   // number on distant processor 
-
+  // returns pairs  (dist_proc, num_on_dist_proc)
-  // gibt anzahl an distant pnums zurueck
-  // * pnums entspricht ARRAY<int[2] >
   int NgPar_GetDistantNodeNums ( int nodetype, int locnum, int * pnums );
   int NgPar_GetNDistantNodeNums ( int nodetype, int locnum );
   
-  int NgPar_GetDistantPNum ( int proc, int locnum ) ;
+  int NgPar_GetGlobalNodeNum (int nodetype, int locnum);
-  int NgPar_GetDistantEdgeNum ( int proc, int locnum ) ;
-  int NgPar_GetDistantFaceNum ( int proc, int locnum ) ;
-  int NgPar_GetDistantElNum ( int proc, int locnum );
-
-  bool NgPar_IsExchangeFace ( int fnr ) ;
-  bool NgPar_IsExchangeVert ( int vnum );
-  bool NgPar_IsExchangeEdge ( int ednum );
-  bool NgPar_IsExchangeElement ( int elnum );
-
-  void NgPar_PrintParallelMeshTopology ();
-  bool NgPar_IsElementInPartition ( int elnum, int dest );
-
-  bool NgPar_IsGhostFace ( int facenum );
-  bool NgPar_IsGhostEdge ( int edgenum );
-
 
 #endif
   

libsrc/interface/nginterface.cpp

@@ -1066,6 +1066,20 @@ int Ng_Overlap ()
   return -1;
 }
 
+int NgPar_GetGlobalNodeNum (int nodetype, int locnum)
+{
+  switch (nodetype)
+    {
+    case 0: return mesh->GetParallelTopology().GetDistantPNum (0, locnum+1)-1;
+    case 1: return mesh->GetParallelTopology().GetDistantEdgeNum (0, locnum+1)-1;
+    case 2: return mesh->GetParallelTopology().GetDistantFaceNum (0, locnum+1)-1;
+    case 3: return mesh->GetParallelTopology().GetDistantElNum (0, locnum+1)-1;
+    }
+  return -1;
+}
+
+
+
 int NgPar_GetDistantPNum ( int proc, int locpnum )  
 { 
   return mesh->GetParallelTopology().GetDistantPNum( proc, locpnum+1) - 1; 

libsrc/meshing/hprefinement.cpp

@@ -1446,7 +1446,7 @@ namespace netgen
       { 
 	for(ElementIndex i=0;i<mesh.GetNE(); i++) 
 	  { 
-	    Element el = mesh[i] ;
+	    // Element el = mesh[i] ;
 	    HPRefElement & hpel = hpelements[mesh[i].hp_elnr];
 	    const ELEMENT_EDGE * edges = MeshTopology::GetEdges1 (mesh[i].GetType());
 	    double dist[3] = {0,0,0}; 

libsrc/meshing/meshclass.hpp

@@ -748,9 +748,6 @@ namespace netgen
     /// loads a mesh sent from master processor
     void ReceiveParallelMesh ();
 
-
-    void UpdateOverlap ();
- 
 #endif
 
 

libsrc/meshing/parallelmesh.cpp

@@ -249,7 +249,6 @@ namespace netgen
     for (int dest = 1; dest < ntasks; dest++)
       {
 	FlatArray<PointIndex> verts = verts_of_proc[dest];
-
 	sendrequests.Append (MyMPI_ISend (verts, dest, MPI_TAG_MESH+1));
 
 	MPI_Datatype mptype = MeshPoint::MyGetMPIType();
@@ -270,7 +269,6 @@ namespace netgen
 	sendrequests.Append (request);
       }
 
-
     Array<int> num_distpnums(ntasks);
     num_distpnums = 0;
 
@@ -542,7 +540,8 @@ namespace netgen
       MyMPI_Recv (fddata, 0, MPI_TAG_MESH+3);
       for (int i = 0; i < fddata.Size(); i += 6)
 	{
-	  int faceind = AddFaceDescriptor (FaceDescriptor(int(fddata[i]), int(fddata[i+1]), int(fddata[i+2]), 0));
+	  int faceind = AddFaceDescriptor 
+	    (FaceDescriptor(int(fddata[i]), int(fddata[i+1]), int(fddata[i+2]), 0));
 	  GetFaceDescriptor(faceind).SetBCProperty (int(fddata[i+3]));
 	  GetFaceDescriptor(faceind).domin_singular = fddata[i+4];
 	  GetFaceDescriptor(faceind).domout_singular = fddata[i+5];
@@ -1150,22 +1149,6 @@ namespace netgen
 
 
 
-
-
-
-  
-  void Mesh :: UpdateOverlap()
-  {
-    cout << "UpdateOverlap depreciated" << endl;
-  }
-
-
-
-
-
-
-
-
 }
 
 

libsrc/meshing/paralleltop.cpp

@@ -11,8 +11,9 @@ namespace netgen
   static MPI_Group MPI_HIGHORDER_WORLD;
   static MPI_Comm MPI_HIGHORDER_COMM;
 
-  void MyMPI_ExchangeTable (TABLE<int> & send_verts, 
+  template <typename T>
-			    TABLE<int> & recv_verts, int tag,
+  void MyMPI_ExchangeTable (TABLE<T> & send_verts, 
+			    TABLE<T> & recv_verts, int tag,
 			    MPI_Comm comm = MPI_COMM_WORLD)
   {
     int ntasks, rank;

libsrc/meshing/topology.cpp

@@ -76,8 +76,6 @@ namespace netgen
     ParallelMeshTopology & paralleltop = mesh.GetParallelTopology();
 #endif
 
-    bool isparallel = 0;
-
   
     if (timestamp > mesh.GetTimeStamp()) return;
   
@@ -1151,23 +1149,12 @@ namespace netgen
 
 	    if (cnt_err && ntasks == 1)
 	      cout << cnt_err << " elements are not matching !!!" << endl;
-
-	    if (cnt_err && ntasks > 1)
-	      isparallel = 1;
 	  }
       }
     
 
 #ifdef PARALLEL
-    if (mesh.GetDimension() == 3)
+    if (id != 0)  
-      if (isparallel != (id != 0))
-	{
-	  cerr << " ****************************** " << endl;
-	  cerr << " **** wrong isparallel   ****** " << endl;
-	  cerr << " ****************************** " << endl;
-	}
-    
-    if (id != 0)    // if (isparallel)
       {
 	paralleltop.Update();
 	if ( paralleltop.DoCoarseUpdate() )

libsrc/visualization/mvdraw.cpp

@@ -746,14 +746,14 @@ namespace netgen
 	  {
 	    string displname;
 	    
-	    Display * dpy = glXGetCurrentDisplay();
+	    // Display * dpy = glXGetCurrentDisplay();
 	    GLXDrawable drawable = glXGetCurrentDrawable();
 	    GLXContext ctx = glXGetCurrentContext();
 	    GLXContextID xid = glXGetContextIDEXT (ctx);
 	    
 	    displname = XDisplayName (0);
-	    /*
 
+	    /*
 	    cout << "Init Parallel GL" << endl;
 	    cout << "DisplayName = " << displname << endl;
 	    cout << "current display = " << dpy << endl;

libsrc/visualization/vssolution.cpp

@@ -2223,28 +2223,27 @@ namespace netgen
     MyMPI_Bcast (comp);
 #endif
 
-    const SolData * sol;
     double val;
-    bool considerElem;
+    // bool considerElem;
 
     bool hasit = false;
     minv = 0; maxv = 1;
+
+    if ((ntasks == 1) || (id > 0))
       if (funcnr != -1)
       {
-        sol = soldata[funcnr];
+        const SolData * sol = soldata[funcnr];
 
         if (sol->draw_volume)
           {
             int ne = mesh->GetNE();
             for (int i = 0; i < ne; i++)
               {
-                considerElem = GetValue (sol, i, 0.333, 0.333, 0.333, comp, val);
+                bool considerElem = GetValue (sol, i, 0.333, 0.333, 0.333, comp, val);
                 if (considerElem)
                   {
-                    if (val > maxv || !hasit)
+                    if (val > maxv || !hasit) maxv = val;
-                      maxv = val;
+                    if (val < minv || !hasit) minv = val;
-                    if (val < minv || !hasit)
-                      minv = val;
                     hasit = true;
                   }
               }
@@ -2255,16 +2254,14 @@ namespace netgen
             for (int i = 0; i < nse; i++)
               {
                 ELEMENT_TYPE type = mesh->SurfaceElement(i+1).GetType();
-                if (type == QUAD)
+		bool considerElem = (type == QUAD) 
-                  considerElem = GetSurfValue (sol, i, -1, 0.5, 0.5, comp, val);
+		  ? GetSurfValue (sol, i, -1, 0.5, 0.5, comp, val)
-                else
+		  : GetSurfValue (sol, i, -1, 0.3333333, 0.3333333, comp, val);
-                  considerElem = GetSurfValue (sol, i, -1, 0.3333333, 0.3333333, comp, val);
+
                 if (considerElem)
                   {
-                    if (val > maxv || !hasit)
+                    if (val > maxv || !hasit) maxv = val;
-                      maxv = val;
+                    if (val < minv || !hasit) minv = val;
-                    if (val < minv || !hasit)
-                      minv = val;
                     hasit = true;
                   }
               }

ng/ngappinit.cpp

@@ -68,18 +68,12 @@ int main(int argc, char ** argv)
 {
 
 #ifdef PARALLEL
-  // MPI_Init(&argc, &argv);          
+  int mpi_required = MPI_THREAD_MULTIPLE, mpi_provided;
-
+  MPI_Init_thread(&argc, &argv, mpi_required, &mpi_provided);          
-  int required = MPI_THREAD_MULTIPLE;
-  // int required = 0;
-  int provided;
-  MPI_Init_thread(&argc, &argv, required, &provided);          
 
   MPI_Comm_size(MPI_COMM_WORLD, &netgen::ntasks);
   MPI_Comm_rank(MPI_COMM_WORLD, &netgen::id);
   
-  if (netgen::id == 0 && provided == MPI_THREAD_MULTIPLE)
-    cout << "multithreaded mpi is supported" << endl;
 
   MPI_Comm_dup ( MPI_COMM_WORLD, &netgen::mesh_comm);
 #endif
@@ -114,9 +108,21 @@ int main(int argc, char ** argv)
 
 
 #ifdef PARALLEL
+      if (netgen::ntasks == 1)
+	{
+	  cout << "Run parallel Netgen with 'mpirun -np xy netgen'" << endl;
+	}
+      else
+	{
 	  cout << "Running MPI - parallel using " 
 	       << netgen::ntasks << " processor" 
 	       << ((netgen::ntasks > 1) ? "s " : " ") << endl;
+	  
+	  cout << "MPI-version = " << MPI_VERSION << '.' << MPI_SUBVERSION << endl;
+
+	  if (mpi_provided == MPI_THREAD_MULTIPLE)
+	    cout << "multithreaded MPI is supported" << endl;
+	}
 #endif
     }
 

ng/parallelfunc.cpp

@@ -148,12 +148,6 @@ void ParallelRun()
 	  VT_USER_END ("Mesh::ReceiveParallelMesh");
 	}
 
-      else if ( message == "overlap++" )
-	{
-	  PrintMessage (1, "overlap++++++");
-	  mesh -> UpdateOverlap();
-	}
-
       else if ( message == "visualize" )
 	{
 	  cout << "parallel message visualize depreciated" << endl;