[cig-commits] r4865 - in long/3D/Gale/trunk/src/StGermain: . Discretisation/Mesh/src

[walter at geodynamics.org]

Author: walter
Date: 2006-10-11 13:47:46 -0700 (Wed, 11 Oct 2006)
New Revision: 4865

Modified:
   long/3D/Gale/trunk/src/StGermain/
   long/3D/Gale/trunk/src/StGermain/Discretisation/Mesh/src/Decomp_Sync_Negotiate.c
Log:
 r2922 at earth:  boo | 2006-10-11 13:42:44 -0700
  r2838 at earth (orig r3826):  LukeHodkinson | 2006-10-04 18:07:58 -0700
  The 'remote search' routine of this class was not
  very scalable at all; modified it to use range sets.
  
 



Property changes on: long/3D/Gale/trunk/src/StGermain
___________________________________________________________________
Name: svk:merge
   - 1ef209d2-b310-0410-a72d-e20c9eb0015c:/cig:2921
afb6c753-b9d0-0310-b4e7-dbd8d91cdd35:/trunk/StGermain:3825
   + 1ef209d2-b310-0410-a72d-e20c9eb0015c:/cig:2922
afb6c753-b9d0-0310-b4e7-dbd8d91cdd35:/trunk/StGermain:3826

Modified: long/3D/Gale/trunk/src/StGermain/Discretisation/Mesh/src/Decomp_Sync_Negotiate.c
===================================================================
--- long/3D/Gale/trunk/src/StGermain/Discretisation/Mesh/src/Decomp_Sync_Negotiate.c	2006-10-11 20:47:45 UTC (rev 4864)
+++ long/3D/Gale/trunk/src/StGermain/Discretisation/Mesh/src/Decomp_Sync_Negotiate.c	2006-10-11 20:47:46 UTC (rev 4865)
@@ -295,82 +295,72 @@
 void Decomp_Sync_Negotiate_RemoteSearch( Decomp_Sync_Negotiate* self, Decomp_Sync* sync, 
 					 unsigned** nRemFound, unsigned*** remFound )
 {
-	CommTopology*	commTopo;
-	Decomp*		decomp;
-	unsigned	rank;
-	unsigned	nInc;
-	unsigned*	inc;
-	unsigned*	remNReq;
-	unsigned**	remReq;
 	unsigned*	nFound;
 	unsigned**	found;
+	RangeSet*	rSet;
+	unsigned	nBytes;
+	Stg_Byte*	bytes;
+	unsigned*	remNBytes;
+	unsigned**	remBytes;
+	RangeSet**	remSets;
 	unsigned	p_i;
 
 	assert( self );
 	assert( sync );
-	assert( sync->commTopo );
-	assert( sync->decomp );
 	assert( nRemFound );
 	assert( remFound );
 
-	/* Shortcuts. */
-	decomp = sync->decomp;
-	commTopo = sync->commTopo;
-	MPI_Comm_rank( commTopo->comm, (int*)&rank );
+	if( sync->commTopo->nInc ) {
+		/* Build a range set of our remotes and pickle. */
+		rSet = RangeSet_New();
+		RangeSet_SetIndices( rSet, sync->nRemotes, sync->remotes );
+		RangeSet_Pickle( rSet, &nBytes, &bytes );
 
-	/* Retrieve our incident neighbours. */
-	CommTopology_GetIncidence( commTopo, rank, &nInc, &inc );
-
-	if( nInc ) {
 		/* Broadcast our remotes to neighbours. */
-		CommTopology_Allgather( commTopo, 
-					sync->nRemotes, sync->remotes, 
-					&remNReq, (void***)&remReq, 
-					sizeof(unsigned) );
+		CommTopology_Allgather( sync->commTopo, 
+					nBytes, bytes, 
+					&remNBytes, (void***)&remBytes, 
+					sizeof(Stg_Byte) );
 
-		/* Prepare to locate remote requireds. */
-		nFound = Memory_Alloc_Array_Unnamed( unsigned, nInc );
-		found = Memory_Alloc_2DComplex_Unnamed( unsigned, nInc, remNReq );
+		/* Free bytes. */
+		FreeArray( bytes );
 
-		/* Begin searching... */
-		for( p_i = 0; p_i < nInc; p_i++ ) {
-			unsigned	rem_i;
+		/* Unpickle range sets. */
+		remSets = Memory_Alloc_Array_Unnamed( RangeSet*, sync->commTopo->nInc );
+		for( p_i = 0; p_i < sync->commTopo->nInc; p_i++ ) {
+			remSets[p_i] = RangeSet_New();
+			RangeSet_Unpickle( remSets[p_i], remNBytes[p_i], (void*)remBytes[p_i] );
+		}
 
-			/* Initialise found count to zero. */
-			nFound[p_i] = 0;
+		/* Free remote bytes. */
+		FreeArray( remNBytes );
+		FreeArray( remBytes );
 
-			/* Do we have any for this processor? */
-			for( rem_i = 0; rem_i < remNReq[p_i]; rem_i++ ) {
-				unsigned	l_i;
-
-				for( l_i = 0; l_i < decomp->nLocals; l_i++ ) {
-					if( decomp->locals[l_i] == remReq[p_i][rem_i] ) {
-						found[p_i][nFound[p_i]++] = remReq[p_i][rem_i];
-						break;
-					}
-				}
-			}
+		/* Replace our remote range set with a local one, then intersect remote sets. */
+		RangeSet_SetIndices( rSet, sync->decomp->nLocals, sync->decomp->locals );
+		nFound = Memory_Alloc_Array_Unnamed( unsigned, sync->commTopo->nInc );
+		found = Memory_Alloc_Array_Unnamed( unsigned*, sync->commTopo->nInc );
+		for( p_i = 0; p_i < sync->commTopo->nInc; p_i++ ) {
+			RangeSet_Intersection( remSets[p_i], rSet );
+			RangeSet_GetIndices( remSets[p_i], nFound + p_i, found + p_i );
+			FreeObject( remSets[p_i] );
 		}
 
-		/* Release some memory. */
-		FreeArray( remNReq );
-		FreeArray( remReq );
+		/* Free remote set array. */
+		FreeArray( remSets );
 
 		/* Send back all the ones we found and receive from all others all our requireds they found. */
-		CommTopology_Alltoall( commTopo, 
+		CommTopology_Alltoall( sync->commTopo, 
 				       nFound, (void**)found, 
 				       nRemFound, (void***)remFound, 
 				       sizeof(unsigned) );
 
 		/* Release some memory. */
 		FreeArray( nFound );
-		FreeArray( found );
+		FreeArray2D( sync->commTopo->nInc, found );
 	}
 	else {
 		*nRemFound = 0;
 		*remFound = NULL;
 	}
-
-	/* Return the incidence. */
-	CommTopology_ReturnIncidence( commTopo, rank, &nInc, &inc );
 }