parallel visualization

configure.ac

@@ -87,8 +87,8 @@ AC_ARG_ENABLE([nglib],
 
 AC_ARG_ENABLE([parallel],
         [AC_HELP_STRING([--enable-parallel],[enable mpi parallelization])],
-        [AC_SUBST([MPI_INCLUDES], "-I/usr/lib64/mpi/gcc/openmpi/include -DPARALLEL -I/usr/include/metis -DMETIS")
+        [AC_SUBST([MPI_INCLUDES], "-I/usr/lib64/mpi/gcc/openmpi/include -DPARALLEL -I/usr/include/metis -DMETIS -DVTRACE -I/usr/local/include/vampirtrace")
-        AC_SUBST([MPI_LIBS], "-L/usr/lib64/mpi/gcc/openmpi/lib64 -lmpi_cxx -lmetis")
+        AC_SUBST([MPI_LIBS], "-L/usr/lib64/mpi/gcc/openmpi/lib64 -lmpi_cxx -lmetis -lvt-hyb")
         ]
         )       
 

libsrc/meshing/parallelmesh.cpp

@@ -102,11 +102,9 @@ namespace netgen
 	    MyMPI_Recv (dist_pnums, 0);
 
 	    for (int hi = 0; hi < dist_pnums.Size(); hi += 3)
-	      {
 	      paralleltop ->
 		SetDistantPNum (dist_pnums[hi+1], dist_pnums[hi], dist_pnums[hi+2]);
  	  }
- 	  }
 
 	if (strcmp (st.c_str(), "volumeelements" ) == 0 )
 	  {

libsrc/meshing/paralleltop.cpp

@@ -851,6 +851,7 @@ void ParallelMeshTopology :: UpdateCoarseGridGlobal ()
 #ifdef SCALASCA
 #pragma pomp inst begin(updatecoarsegrid)
 #endif
+
     (*testout) << "UPDATE COARSE GRID PARALLEL TOPOLOGY " << endl;
     PrintMessage (1, "UPDATE COARSE GRID PARALLEL TOPOLOGY ");
 
@@ -865,9 +866,7 @@ void ParallelMeshTopology :: UpdateCoarseGridGlobal ()
     if ( id == 0 ) return;
 
 
-    Array<int> * sendarray, *recvarray;
+    Array<int> sendarray, recvarray;
-    sendarray = new Array<int> (0);
-    recvarray = new Array<int>;
 
     nfa = topology . GetNFaces();
     ned = topology . GetNEdges();
@@ -878,13 +877,131 @@ void ParallelMeshTopology :: UpdateCoarseGridGlobal ()
     nsurfel = mesh.GetNSE();
     
 
-    sendarray->SetSize (0);
+    // exchange vertices
 
-    BitArray recvface(nfa);
+    for (int vertex = 1; vertex <= nv; vertex++)
-    recvface.Clear();
+      if (IsExchangeVert (vertex) ) 
+	{
+	  sendarray.Append (GetLoc2Glob_Vert (vertex));
+	  sendarray.Append (vertex);
+	}
+
+    Array<int,1> glob2loc;
+    glob2loc.SetSize (nvglob);
+    glob2loc = -1;
+    
+    for (int locv = 1; locv <= nv; locv++)
+      if (IsExchangeVert (locv) )
+	glob2loc[GetDistantPNum(0, locv)] = locv;
+
+    for (int sender = 1; sender < ntasks; sender ++)
+      {
+	if (id == sender)
+	  MyMPI_Bcast (sendarray, sender-1, MPI_HIGHORDER_COMM);
+	else
+	  {
+	    MyMPI_Bcast (recvarray, sender-1, MPI_HIGHORDER_COMM);
+	    for (int ii = 0; ii < recvarray.Size(); )
+	      {
+		int globv = recvarray[ii++];
+		int distv = recvarray[ii++];
+		int locv = glob2loc[globv];
+
+		if (locv != -1)
+		  SetDistantPNum (sender, locv, distv);
+	      }
+	  }
+      }
+
+    sendarray.SetSize (0);
+    recvarray.SetSize (0);
+    
+    
+    // exchange edges
+    int maxedge = 0;
+    for (int edge = 1; edge <= ned; edge++)
+      if (IsExchangeEdge (edge) ) 
+	{
+	  sendarray.Append (GetDistantEdgeNum (0, edge));
+	  sendarray.Append (edge);
+	  maxedge = max (maxedge, GetDistantEdgeNum (0, edge));
+	}
+
+    glob2loc.SetSize (maxedge+1);
+    glob2loc = -1;
+    
+    for (int loc = 1; loc <= ned; loc++)
+      if (IsExchangeEdge (loc) )
+	glob2loc[GetDistantEdgeNum(0, loc)] = loc;
+
+    for (int sender = 1; sender < ntasks; sender ++)
+      {
+	if (id == sender)
+	  MyMPI_Bcast (sendarray, sender-1, MPI_HIGHORDER_COMM);
+	else
+	  {
+	    MyMPI_Bcast (recvarray, sender-1, MPI_HIGHORDER_COMM);
+	    for (int ii = 0; ii < recvarray.Size(); )
+	      { 
+		int globe = recvarray[ii++];
+		int diste = recvarray[ii++];
+
+		if (globe > maxedge) continue;
+		int loce = glob2loc[globe];
+
+		if (loce != -1)
+		  SetDistantEdgeNum (sender, loce, diste);
+	      }
+	  }
+      }
+
+    sendarray.SetSize (0);
+    recvarray.SetSize (0);
+    
+
+    // exchange faces
+    for (int face = 1; face <= nfa; face++)
+      if (IsExchangeFace (face) ) 
+	{
+	  sendarray.Append (GetDistantFaceNum (0, face));
+	  sendarray.Append (face);
+	}
+
+    glob2loc.SetSize (nfaglob);
+    glob2loc = -1;
+    
+    for (int loc = 1; loc <= nfa; loc++)
+      if (IsExchangeFace (loc) )
+	glob2loc[GetDistantFaceNum(0, loc)] = loc;
+
+    for (int sender = 1; sender < ntasks; sender ++)
+      {
+	if (id == sender)
+	  MyMPI_Bcast (sendarray, sender-1, MPI_HIGHORDER_COMM);
+	else
+	  {
+	    MyMPI_Bcast (recvarray, sender-1, MPI_HIGHORDER_COMM);
+	    for (int ii = 0; ii < recvarray.Size(); )
+	      { 
+		int globf = recvarray[ii++];
+		int distf = recvarray[ii++];
+		int locf = glob2loc[globf];
+		
+		if (locf != -1)
+		  SetDistantFaceNum (sender, locf, distf);
+	      }
+	  } 
+      }
+
+ 
+
+
+
+#ifdef OLD
+    // BitArray recvface(nfa);
+    // recvface.Clear();
     for (int fa = 1; fa <= nfa; fa++ )
       {
-
 	if ( !IsExchangeFace ( fa ) ) continue;
 	
 	Array<int> edges, pnums;
@@ -893,39 +1010,30 @@ void ParallelMeshTopology :: UpdateCoarseGridGlobal ()
 	topology.GetFaceEdges (fa, edges);
 	topology.GetFaceVertices (fa, pnums);
 	
-	
 	// send : 
-	// localfacenum
+	// localfacenum globalfacenum np ned globalpnums localpnums
-	// globalfacenum
-	// np
-	// ned
-	// globalpnums
-	// localpnums
-	
 	// localedgenums mit globalv1, globalv2
-	// 
 	
-	sendarray -> Append ( fa );
+	sendarray.Append ( fa );
-	sendarray -> Append ( globfa );
+	sendarray.Append ( globfa );
-	sendarray -> Append ( pnums.Size() );
+	sendarray.Append ( pnums.Size() );
-	sendarray -> Append ( edges.Size() );
+	sendarray.Append ( edges.Size() );
+
 	for (int i = 0; i < pnums.Size(); i++ )
-	  {
+	  sendarray.Append( GetLoc2Glob_Vert(pnums[i]) );
-	    sendarray -> Append( GetLoc2Glob_Vert(pnums[i]) );
+
-	  }
 	for ( int i = 0; i < pnums.Size(); i++ )
-	  {
+	  sendarray.Append(pnums[i]);
-	    sendarray -> Append(pnums[i] );
+
-	  }
 	for ( int i = 0; i < edges.Size(); i++ )
 	  {
-	    sendarray -> Append(edges[i] );
+	    sendarray.Append(edges[i]);
 	    int v1, v2;
 	    topology.GetEdgeVertices ( edges[i], v1, v2 );
 	    int dv1 = GetLoc2Glob_Vert ( v1 );
 	    int dv2 = GetLoc2Glob_Vert ( v2 );
-	    sendarray -> Append ( dv1 );
+	    sendarray.Append ( dv1 );
-	    sendarray -> Append ( dv2 );
+	    sendarray.Append ( dv2 );
 	  }
       }
 
@@ -936,49 +1044,38 @@ void ParallelMeshTopology :: UpdateCoarseGridGlobal ()
     // Array for temporary use, to find local from global element fast
     // only for not too big meshes 
     // seems ok, as low-order space is treated on one proc
-    Array<int,1> * glob2locfa;
+    Array<int,1> glob2locfa (nfaglob);
-    glob2locfa = new Array<int,1> ( nfaglob );
+    glob2locfa = -1;
-    (*glob2locfa) = -1;
 
     for (int locfa = 1; locfa <= nfa; locfa++)
-      {
+      if (IsExchangeFace (locfa) )
-	if ( !IsExchangeFace ( locfa ) ) continue;
+	glob2locfa[GetDistantFaceNum(0, locfa)] = locfa;
-	(*glob2locfa)[GetDistantFaceNum(0, locfa) ] = locfa;
+
-      }
 
     for (int sender = 1; sender < ntasks; sender ++)
       {
-		
 	if (id == sender)
-	  MyMPI_Bcast ( *sendarray, sender-1, MPI_HIGHORDER_COMM);
+	  MyMPI_Bcast (sendarray, sender-1, MPI_HIGHORDER_COMM);
-
+	else
-	if ( id != sender )
 	  {
-	    MyMPI_Bcast ( *recvarray, sender-1, MPI_HIGHORDER_COMM);
+	    MyMPI_Bcast ( recvarray, sender-1, MPI_HIGHORDER_COMM);
 	    // compare received vertices with own ones
 	    int ii = 0;
 	    int cntel = 0;
 	    int locfa = 1;
 
-	    while ( ii< recvarray -> Size() )
+	    while (ii < recvarray.Size())
 	      {
-		
 		// receive list : 
-		// distant facenum
+		// distant_facenum global_facenum np ned globalpnums distant_pnums
-		// global facenum
-		// np
-		// ned
-		// globalpnums
-		// distant pnums
-		
 		// distant edgenums mit globalv1, globalv2
 		
-		int distfa = (*recvarray)[ii++];
+		int distfa = recvarray[ii++];
-		int globfa = (*recvarray)[ii++];
+		int globfa = recvarray[ii++];
-		int distnp = (*recvarray)[ii++];
+		int distnp = recvarray[ii++];
-		int distned =(*recvarray)[ii++];
+		int distned =recvarray[ii++];
 		
-		int locfa = (*glob2locfa) [globfa];
+		int locfa = (glob2locfa) [globfa];
 		
 		if ( locfa == -1 ) 
 		  {
@@ -994,7 +1091,6 @@ void ParallelMeshTopology :: UpdateCoarseGridGlobal ()
 		topology.GetFaceEdges ( fa, edges );
 		topology.GetFaceVertices ( fa, pnums );
 		
-		
 		globalpnums.SetSize ( distnp );
 		for ( int i = 0; i < distnp; i++)
 		  globalpnums[i] = GetLoc2Glob_Vert ( pnums[i] );
@@ -1004,18 +1100,17 @@ void ParallelMeshTopology :: UpdateCoarseGridGlobal ()
 		// find exchange points
 		for ( int i = 0;  i < distnp; i++)
 		  {
-		    int distglobalpnum = (*recvarray)[ii+i];
+		    int distglobalpnum = recvarray[ii+i];
 		    for ( int j = 0; j < distnp; j++ )
 		      if ( globalpnums[j] == distglobalpnum )
 			{
 			  // set sender -- distpnum  ---- locpnum
-			  int distpnum = (*recvarray)[ii + i +distnp];
+			  int distpnum = recvarray[ii + i +distnp];
-			  SetDistantPNum ( sender, pnums[j], distpnum );
+			  // SetDistantPNum ( sender, pnums[j], distpnum );
+			}
 		  }
 			
-		  }
+		Array<int> distedgenums(distned);
-			
-		int * distedgenums  = new int [distned];
 		// find exchange edges
 		for ( int i = 0; i  < edges.Size(); i++)
 		  {
@@ -1026,9 +1121,9 @@ void ParallelMeshTopology :: UpdateCoarseGridGlobal ()
 		    if ( dv1 > dv2 ) swap ( dv1, dv2 );
 		    for ( int ed = 0; ed < distned; ed++)
 		      {
-			distedgenums[ed] = (*recvarray)[ii + 2*distnp + 3*ed];
+			distedgenums[ed] = recvarray[ii + 2*distnp + 3*ed];
-			int ddv1 = (*recvarray)[ii + 2*distnp + 3*ed + 1];
+			int ddv1 = recvarray[ii + 2*distnp + 3*ed + 1];
-			int ddv2 = (*recvarray)[ii + 2*distnp + 3*ed + 2];
+			int ddv2 = recvarray[ii + 2*distnp + 3*ed + 2];
 			if ( ddv1 > ddv2 ) swap ( ddv1, ddv2 );
 			if ( dv1 == ddv1 && dv2 == ddv2 )
 			  {
@@ -1036,28 +1131,22 @@ void ParallelMeshTopology :: UpdateCoarseGridGlobal ()
 			    SetDistantEdgeNum ( sender, edges[i], distedgenums[ed] );
 			  }
 		      }
-		    
-		    
 		  }
-		delete [] distedgenums; 
-		
 		ii += 2*distnp + 3*distned;
-		
 	      }
 	  }
       }
+#endif
   
 
     // set which elements are where for the master processor
 
-    delete sendarray; delete recvarray;
-    if ( id > 0 )
-      delete glob2locfa;
     coarseupdate = 1;
 	
 #ifdef SCALASCA
 #pragma pomp inst end(updatecoarsegrid)
 #endif
+
   }
 
   void ParallelMeshTopology :: UpdateCoarseGridOverlap ()

libsrc/visualization/vssolution.cpp

@@ -35,7 +35,8 @@ namespace netgen
   {
     surfellist = 0;
     linelist = 0;
-    clipplanelist = 0;
+    clipplanelist_scal = 0;
+    clipplanelist_vec = 0;
     isolinelist = 0;
     clipplane_isolinelist = 0;
     surface_vector_list = 0;
@@ -417,7 +418,12 @@ namespace netgen
       }
 
     if (showclipsolution)
-      glCallList (clipplanelist);
+      {
+	if (clipsolution == 1)
+	  glCallList (clipplanelist_scal);
+	if (clipsolution == 2)
+	  glCallList (clipplanelist_vec);
+      }
 
 
     if (draw_fieldlines)
@@ -696,31 +702,16 @@ namespace netgen
             // lock->Lock();
           }
 
-        if (clipplanelist)
-          glDeleteLists (clipplanelist, 1);
-      
-
-        clipplanelist = glGenLists (1);
-        glNewList (clipplanelist, GL_COMPILE);
       
         if (vispar.clipenable && clipsolution == 1 && sol)
-          {
+	  DrawClipPlaneTrigs (); // sol, scalcomp);
-            glDisable(GL_CLIP_PLANE0);
 
-            Array<ClipPlaneTrig> cpt;
-            Array<ClipPlanePoint> pts;
-            GetClippingPlaneTrigs (cpt, pts);
 
-            glNormal3d (-clipplane[0], -clipplane[1], -clipplane[2]);
+        if (clipplanelist_vec)
-            glColor3d (1.0, 1.0, 1.0);
+          glDeleteLists (clipplanelist_vec, 1);
-
-            SetTextureMode (usetexture);
-
-            DrawClipPlaneTrigs (sol, scalcomp, cpt, pts);
-
-            glEnable(GL_CLIP_PLANE0);
-          }
       
+        clipplanelist_vec = glGenLists (1);
+        glNewList (clipplanelist_vec, GL_COMPILE);
 
         if (vispar.clipenable && clipsolution == 2 && vsol)
           {
@@ -3719,10 +3710,65 @@ namespace netgen
 
 
   void VisualSceneSolution :: 
-  DrawClipPlaneTrigs (const SolData * sol, int comp,
+  DrawClipPlaneTrigs () // const SolData * sol, int comp)
-                      const Array<ClipPlaneTrig> & trigs, 
-                      const Array<ClipPlanePoint> & points)
   {
+#ifdef PARALLELGL
+
+    if (id == 0 && ntasks > 1)
+      {
+	InitParallelGL();
+
+	Array<int> parlists (ntasks);
+
+	for ( int dest = 1; dest < ntasks; dest++ )
+	  {
+	    MyMPI_Send ("redraw", dest);
+	    MyMPI_Send ("clipplanetrigs", dest);
+	  }
+	for ( int dest = 1; dest < ntasks; dest++ )
+	  MyMPI_Recv (parlists[dest], dest);
+
+	if (clipplanelist_scal)
+	  glDeleteLists (clipplanelist_scal, 1);
+
+	clipplanelist_scal = glGenLists (1);
+	glNewList (clipplanelist_scal, GL_COMPILE);
+	
+	for ( int dest = 1; dest < ntasks; dest++ )
+	  glCallList (parlists[dest]);
+	
+	glEndList();
+	return;
+      }
+#endif
+
+
+
+
+
+    if (clipplanelist_scal)
+      glDeleteLists (clipplanelist_scal, 1);
+    
+    clipplanelist_scal = glGenLists (1);
+    glNewList (clipplanelist_scal, GL_COMPILE);
+
+
+    Array<ClipPlaneTrig> trigs;
+    Array<ClipPlanePoint> points;
+    GetClippingPlaneTrigs (trigs, points);
+	    
+    glNormal3d (-clipplane[0], -clipplane[1], -clipplane[2]);
+    glColor3d (1.0, 1.0, 1.0);
+    
+    SetTextureMode (usetexture);
+
+    SolData * sol = NULL;
+
+    if (scalfunction != -1) 
+      sol = soldata[scalfunction];
+
+
+
     glBegin (GL_TRIANGLES);
 
     int maxlpnr = 0;
@@ -3817,6 +3863,15 @@ namespace netgen
 
       }
     glEnd();
+
+    glEndList ();
+
+
+#ifdef PARALLELGL
+    glFinish();
+    if (id > 0)
+      MyMPI_Send (clipplanelist_scal, 0);
+#endif
   }
 
 
@@ -4058,6 +4113,11 @@ namespace netgen
     MyMPI_Bcast (maxval);
     MyMPI_Bcast (numisolines);
     MyMPI_Bcast (subdivisions);
+
+    MyMPI_Bcast (clipplane[0]);
+    MyMPI_Bcast (clipplane[1]);
+    MyMPI_Bcast (clipplane[2]);
+    MyMPI_Bcast (clipplane[3]);
   }
   
 #endif
@@ -4321,6 +4381,11 @@ namespace netgen
 
       }
 
+    
+    vsmesh.SetClippingPlane ();  // for computing parameters
+    vssolution.SetClippingPlane ();  // for computing parameters
+    glDisable(GL_CLIP_PLANE0);
+
 #ifdef PARALLELGL
     vsmesh.Broadcast ();
 #endif    

libsrc/visualization/vssolution.hpp

@@ -43,7 +43,8 @@ class VisualSceneSolution : public VisualScene
 
   int surfellist;
   int linelist;
-  int clipplanelist;
+  int clipplanelist_scal;
+  int clipplanelist_vec;
   int isolinelist;
   int clipplane_isolinelist;
   int surface_vector_list;
@@ -299,9 +300,7 @@ public:
 		      double val1, double val2, double val3, double val4);
 
 
-  void DrawClipPlaneTrigs (const SolData * sol, int comp,
+  void DrawClipPlaneTrigs (); // const SolData * sol, int comp);
-                           const Array<ClipPlaneTrig> & trigs, 
-                           const Array<ClipPlanePoint> & points);
 		  
   void SetOpenGlColor(double val);
 

ng/drawing.tcl

@@ -72,6 +72,7 @@ proc popupcheckredraw2 { vari boolvar { x 0 } } {
 	set varname 0
     } {
 	Ng_SetVisParameters
+	Ng_Vis_Set parameters
 	if { $boolvar == 1 } { redraw }
 	Ng_SetVisParameters
     }

ng/parallelfunc.cpp

@@ -249,11 +249,13 @@ void ParallelRun()
 		
 		display = XOpenDisplay (displname.c_str());
 		
+		/*
 		  PrintMessage (3, "displ - name = ", displname);
 		  PrintMessage (3, "display = ", display,
 		  " display props: ",
 		  " screen w = ", XDisplayWidth (display, 0),
 		  " , h = ", XDisplayHeight (display, 0));
+		*/
 
 		Window win;
 		int wx, wy;
@@ -328,7 +330,6 @@ void ParallelRun()
 		      break;
 		    }
 		  }
-
 		if (!visinfo)
 		  cerr << "no VISINFO found" << endl;
 
@@ -365,26 +366,26 @@ void ParallelRun()
 
 	    if (redraw_cmd == "linelist")
 	      {
-		glXMakeCurrent (display, curDrawable, context);
+		// glXMakeCurrent (display, curDrawable, context);
 		vsmesh.BuildLineList();
-		glXMakeCurrent (display, None, NULL);
+		// glXMakeCurrent (display, None, NULL);
 	      }
 
 	    if (redraw_cmd == "filledlist")
 	      {
-		glXMakeCurrent (display, curDrawable, context);
 		vsmesh.BuildFilledList (false);
-		glXMakeCurrent (display, None, NULL);
 	      }
 
 
 	    if (redraw_cmd == "solsurfellist")
 	      {
-		glXMakeCurrent (display, curDrawable, context);
 		vssolution.DrawSurfaceElements();
-		glXMakeCurrent (display, None, NULL);
 	      }
 
+	    if (redraw_cmd == "clipplanetrigs")
+	      {
+		vssolution.DrawClipPlaneTrigs();
+	      }
 	    
 	  }
 #endif