[cig-commits] r6198 - in long/3D/Gale/trunk/src/StGermain: .
Discretisation/Mesh/src
walter at geodynamics.org
walter at geodynamics.org
Thu Mar 8 11:38:38 PST 2007
Author: walter
Date: 2007-03-08 11:38:37 -0800 (Thu, 08 Mar 2007)
New Revision: 6198
Modified:
long/3D/Gale/trunk/src/StGermain/
long/3D/Gale/trunk/src/StGermain/Discretisation/Mesh/src/Decomp_Sync.c
long/3D/Gale/trunk/src/StGermain/Discretisation/Mesh/src/Decomp_Sync.h
long/3D/Gale/trunk/src/StGermain/Discretisation/Mesh/src/MeshTopology.c
Log:
r3364 at earth (orig r4023): LukeHodkinson | 2007-03-08 11:05:55 -0800
Walter found that when running Gale in parallel
it would reach deadlock in certain situations. I've
re-ordered the calculation of shared topological
elements to remove the deadlock.
Property changes on: long/3D/Gale/trunk/src/StGermain
___________________________________________________________________
Name: svk:merge
- 1ef209d2-b310-0410-a72d-e20c9eb0015c:/cig:3196
afb6c753-b9d0-0310-b4e7-dbd8d91cdd35:/branches/decomp3d/StGermain:4022
afb6c753-b9d0-0310-b4e7-dbd8d91cdd35:/trunk/StGermain:3899
+ 1ef209d2-b310-0410-a72d-e20c9eb0015c:/cig:3196
afb6c753-b9d0-0310-b4e7-dbd8d91cdd35:/branches/decomp3d/StGermain:4023
afb6c753-b9d0-0310-b4e7-dbd8d91cdd35:/trunk/StGermain:3899
Modified: long/3D/Gale/trunk/src/StGermain/Discretisation/Mesh/src/Decomp_Sync.c
===================================================================
--- long/3D/Gale/trunk/src/StGermain/Discretisation/Mesh/src/Decomp_Sync.c 2007-03-08 19:28:21 UTC (rev 6197)
+++ long/3D/Gale/trunk/src/StGermain/Discretisation/Mesh/src/Decomp_Sync.c 2007-03-08 19:38:37 UTC (rev 6198)
@@ -218,6 +218,7 @@
ranks[p_i] = p_i;
Decomp_Sync_AddSources( self, nIncRanks, ranks, nSrcRemotes, srcRemotes );
Decomp_Sync_AddSinks( self, nIncRanks, ranks, nSnkRemotes, snkRemotes );
+ Decomp_Sync_BuildShared( self );
FreeArray( ranks );
/* Free the space. */
@@ -322,8 +323,6 @@
self->nSrcs[rank] += nSources[p_i];
self->netSrcs += nSources[p_i];
}
-
- Decomp_Sync_BuildShared( self );
}
void Decomp_Sync_AddSinks( void* sync, unsigned nRanks, unsigned* ranks,
@@ -353,8 +352,6 @@
self->nSnks[rank] += nSinks[p_i];
self->netSnks += nSinks[p_i];
}
-
- Decomp_Sync_BuildShared( self );
}
void Decomp_Sync_SetRequired( void* sync, unsigned nRequired, unsigned* required ) {
@@ -494,8 +491,138 @@
FreeObject( reqSet );
FreeObject( remReqSet );
FreeObject( lSet );
+
+ Decomp_Sync_BuildShared( self );
}
+void Decomp_Sync_BuildShared( void* sync ) {
+ Decomp_Sync* self = (Decomp_Sync*)sync;
+ unsigned nIncRanks;
+ RangeSet *dSet, **rSets, *allSet;
+ unsigned nLocals, *locals;
+ unsigned nSendBytes, *nRecvBytes;
+ Stg_Byte *sendBytes, **recvBytes;
+ unsigned **sharers;
+ unsigned nInds, *inds;
+ unsigned r_i, s_i, ind_i;
+
+ assert( self && Stg_CheckType( self, Decomp_Sync ) );
+
+ /* Free shared info. */
+ FreeArray( self->shared );
+ FreeArray( self->nSharers );
+ FreeArray( self->sharers );
+
+ nIncRanks = CommTopology_GetIncidenceSize( self->commTopo );
+
+ /* Communicate a range set of my domain indices to all neighbours. */
+ dSet = RangeSet_New();
+ RangeSet_SetIndices( dSet, self->nRemotes, self->remotes );
+ Decomp_GetLocals( self->decomp, &nLocals, &locals );
+ RangeSet_AddIndices( dSet, nLocals, locals );
+ RangeSet_Pickle( dSet, &nSendBytes, &sendBytes );
+ CommTopology_Allgather( self->commTopo,
+ nSendBytes, sendBytes,
+ &nRecvBytes, &recvBytes,
+ sizeof(Stg_Byte) );
+ FreeArray( sendBytes );
+ rSets = AllocArray( RangeSet*, nIncRanks );
+ allSet = RangeSet_New();
+ for( r_i = 0; r_i < nIncRanks; r_i++ ) {
+ rSets[r_i] = RangeSet_New();
+ RangeSet_Unpickle( rSets[r_i], nRecvBytes[r_i], recvBytes[r_i] );
+ RangeSet_Intersection( rSets[r_i], dSet );
+ RangeSet_Union( allSet, rSets[r_i] );
+ FreeArray( recvBytes[r_i] );
+ }
+ FreeArray( recvBytes );
+ FreeArray( nRecvBytes );
+
+ /* Use the 'allSet' to generate the shared indices. */
+ self->shared = NULL;
+ RangeSet_GetIndices( allSet, &self->nShared, &self->shared );
+ FreeObject( allSet );
+ for( s_i = 0; s_i < self->nShared; s_i++ ) {
+ insist( Decomp_Sync_GlobalToDomain( self, self->shared[s_i], self->shared + s_i ) );
+ UIntMap_Insert( self->dsMap, self->shared[s_i], s_i );
+ }
+
+ /* Now that we have each intersection, convert to shared arrays. */
+ self->nSharers = AllocArray( unsigned, self->nShared );
+ memset( self->nSharers, 0, self->nShared * sizeof(unsigned) );
+ sharers = AllocArray2D( unsigned, self->nShared, nIncRanks );
+ for( r_i = 0; r_i < nIncRanks; r_i++ ) {
+ inds = NULL;
+ RangeSet_GetIndices( rSets[r_i], &nInds, &inds );
+ FreeObject( rSets[r_i] );
+ for( ind_i = 0; ind_i < nInds; ind_i++ ) {
+ insist( Decomp_Sync_GlobalToDomain( self, inds[ind_i], inds + ind_i ) );
+ insist( Decomp_Sync_DomainToShared( self, inds[ind_i], inds + ind_i ) );
+ sharers[inds[ind_i]][self->nSharers[inds[ind_i]]++] = r_i;
+ }
+ FreeArray( inds );
+ }
+ FreeArray( rSets );
+
+ /* Store final array. */
+ self->sharers = AllocComplex2D( unsigned, self->nShared, self->nSharers );
+ for( s_i = 0; s_i < self->nShared; s_i++ )
+ memcpy( self->sharers[s_i], sharers[s_i], self->nSharers[s_i] * sizeof(unsigned) );
+ FreeArray( sharers );
+
+#if 0
+ /* Create a range set of all sinks and sources. */
+ nIncRanks = CommTopology_GetIncidenceSize( self->commTopo );
+ shareSet = RangeSet_New();
+ tmpSet = RangeSet_New();
+ for( p_i = 0; p_i < nIncRanks; p_i++ ) {
+ RangeSet_SetIndices( tmpSet, self->nSnks[p_i], self->snks[p_i] );
+ RangeSet_Union( shareSet, tmpSet );
+ RangeSet_SetIndices( tmpSet, self->nSrcs[p_i], self->srcs[p_i] );
+ RangeSet_Union( shareSet, tmpSet );
+ }
+ FreeObject( tmpSet );
+
+ /* These indices are the shared elements. */
+ self->shared = NULL;
+ RangeSet_GetIndices( shareSet, &self->nShared, &self->shared );
+ FreeObject( shareSet );
+
+ /* If there are no shared indices, exit now. */
+ if( !self->nShared )
+ return;
+
+ /* Build the shared mapping. */
+ for( s_i = 0; s_i < self->nShared; s_i++ )
+ UIntMap_Insert( self->dsMap, self->shared[s_i], s_i );
+
+ /* Create temporary storage for the sharers. */
+ self->nSharers = AllocNamedArray( unsigned, self->nShared, "Decomp_Sync::nSharers" );
+ sharers = Memory_Alloc_2DArray_Unnamed( unsigned, self->nShared, nIncRanks );
+ memset( self->nSharers, 0, self->nShared * sizeof(unsigned) );
+
+ /* Build the sharer lists. */
+ for( p_i = 0; p_i < nIncRanks; p_i++ ) {
+ for( s_i = 0; s_i < self->nSnks[p_i]; s_i++ ) {
+ unsigned sharedInd;
+ unsigned curSharer;
+
+ insist( UIntMap_Map( self->dsMap, self->snks[p_i][s_i], &sharedInd ) );
+ curSharer = self->nSharers[sharedInd]++;
+ sharers[sharedInd][curSharer] = p_i;
+ }
+ }
+
+ /* Transfer to final storage. */
+ self->sharers = Memory_Alloc_2DComplex( unsigned, self->nShared, self->nSharers, "Decomp_Sync::sharers" );
+ for( s_i = 0; s_i < self->nShared; s_i++ )
+ memcpy( self->sharers[s_i], sharers[s_i], self->nSharers[s_i] * sizeof(unsigned) );
+
+ /* Free the old sharers array. */
+ FreeArray( sharers );
+#endif
+}
+
unsigned Decomp_Sync_GetGlobalSize( void* sync ) {
Decomp_Sync* self = (Decomp_Sync*)sync;
@@ -865,133 +992,6 @@
FreeObject( remSet );
}
-void Decomp_Sync_BuildShared( Decomp_Sync* self ) {
- unsigned nIncRanks;
- RangeSet *dSet, **rSets, *allSet;
- unsigned nLocals, *locals;
- unsigned nSendBytes, *nRecvBytes;
- Stg_Byte *sendBytes, **recvBytes;
- unsigned **sharers;
- unsigned nInds, *inds;
- unsigned r_i, s_i, ind_i;
-
- assert( self );
-
- /* Free shared info. */
- FreeArray( self->shared );
- FreeArray( self->nSharers );
- FreeArray( self->sharers );
-
- nIncRanks = CommTopology_GetIncidenceSize( self->commTopo );
-
- /* Communicate a range set of my domain indices to all neighbours. */
- dSet = RangeSet_New();
- RangeSet_SetIndices( dSet, self->nRemotes, self->remotes );
- Decomp_GetLocals( self->decomp, &nLocals, &locals );
- RangeSet_AddIndices( dSet, nLocals, locals );
- RangeSet_Pickle( dSet, &nSendBytes, &sendBytes );
- CommTopology_Allgather( self->commTopo,
- nSendBytes, sendBytes,
- &nRecvBytes, &recvBytes,
- sizeof(Stg_Byte) );
- FreeArray( sendBytes );
- rSets = AllocArray( RangeSet*, nIncRanks );
- allSet = RangeSet_New();
- for( r_i = 0; r_i < nIncRanks; r_i++ ) {
- rSets[r_i] = RangeSet_New();
- RangeSet_Unpickle( rSets[r_i], nRecvBytes[r_i], recvBytes[r_i] );
- RangeSet_Intersection( rSets[r_i], dSet );
- RangeSet_Union( allSet, rSets[r_i] );
- FreeArray( recvBytes[r_i] );
- }
- FreeArray( recvBytes );
- FreeArray( nRecvBytes );
-
- /* Use the 'allSet' to generate the shared indices. */
- self->shared = NULL;
- RangeSet_GetIndices( allSet, &self->nShared, &self->shared );
- FreeObject( allSet );
- for( s_i = 0; s_i < self->nShared; s_i++ ) {
- insist( Decomp_Sync_GlobalToDomain( self, self->shared[s_i], self->shared + s_i ) );
- UIntMap_Insert( self->dsMap, self->shared[s_i], s_i );
- }
-
- /* Now that we have each intersection, convert to shared arrays. */
- self->nSharers = AllocArray( unsigned, self->nShared );
- memset( self->nSharers, 0, self->nShared * sizeof(unsigned) );
- sharers = AllocArray2D( unsigned, self->nShared, nIncRanks );
- for( r_i = 0; r_i < nIncRanks; r_i++ ) {
- inds = NULL;
- RangeSet_GetIndices( rSets[r_i], &nInds, &inds );
- FreeObject( rSets[r_i] );
- for( ind_i = 0; ind_i < nInds; ind_i++ ) {
- insist( Decomp_Sync_GlobalToDomain( self, inds[ind_i], inds + ind_i ) );
- insist( Decomp_Sync_DomainToShared( self, inds[ind_i], inds + ind_i ) );
- sharers[inds[ind_i]][self->nSharers[inds[ind_i]]++] = r_i;
- }
- FreeArray( inds );
- }
- FreeArray( rSets );
-
- /* Store final array. */
- self->sharers = AllocComplex2D( unsigned, self->nShared, self->nSharers );
- for( s_i = 0; s_i < self->nShared; s_i++ )
- memcpy( self->sharers[s_i], sharers[s_i], self->nSharers[s_i] * sizeof(unsigned) );
- FreeArray( sharers );
-
-#if 0
- /* Create a range set of all sinks and sources. */
- nIncRanks = CommTopology_GetIncidenceSize( self->commTopo );
- shareSet = RangeSet_New();
- tmpSet = RangeSet_New();
- for( p_i = 0; p_i < nIncRanks; p_i++ ) {
- RangeSet_SetIndices( tmpSet, self->nSnks[p_i], self->snks[p_i] );
- RangeSet_Union( shareSet, tmpSet );
- RangeSet_SetIndices( tmpSet, self->nSrcs[p_i], self->srcs[p_i] );
- RangeSet_Union( shareSet, tmpSet );
- }
- FreeObject( tmpSet );
-
- /* These indices are the shared elements. */
- self->shared = NULL;
- RangeSet_GetIndices( shareSet, &self->nShared, &self->shared );
- FreeObject( shareSet );
-
- /* If there are no shared indices, exit now. */
- if( !self->nShared )
- return;
-
- /* Build the shared mapping. */
- for( s_i = 0; s_i < self->nShared; s_i++ )
- UIntMap_Insert( self->dsMap, self->shared[s_i], s_i );
-
- /* Create temporary storage for the sharers. */
- self->nSharers = AllocNamedArray( unsigned, self->nShared, "Decomp_Sync::nSharers" );
- sharers = Memory_Alloc_2DArray_Unnamed( unsigned, self->nShared, nIncRanks );
- memset( self->nSharers, 0, self->nShared * sizeof(unsigned) );
-
- /* Build the sharer lists. */
- for( p_i = 0; p_i < nIncRanks; p_i++ ) {
- for( s_i = 0; s_i < self->nSnks[p_i]; s_i++ ) {
- unsigned sharedInd;
- unsigned curSharer;
-
- insist( UIntMap_Map( self->dsMap, self->snks[p_i][s_i], &sharedInd ) );
- curSharer = self->nSharers[sharedInd]++;
- sharers[sharedInd][curSharer] = p_i;
- }
- }
-
- /* Transfer to final storage. */
- self->sharers = Memory_Alloc_2DComplex( unsigned, self->nShared, self->nSharers, "Decomp_Sync::sharers" );
- for( s_i = 0; s_i < self->nShared; s_i++ )
- memcpy( self->sharers[s_i], sharers[s_i], self->nSharers[s_i] * sizeof(unsigned) );
-
- /* Free the old sharers array. */
- FreeArray( sharers );
-#endif
-}
-
void Decomp_Sync_Destruct( Decomp_Sync* self ) {
assert( self );
Modified: long/3D/Gale/trunk/src/StGermain/Discretisation/Mesh/src/Decomp_Sync.h
===================================================================
--- long/3D/Gale/trunk/src/StGermain/Discretisation/Mesh/src/Decomp_Sync.h 2007-03-08 19:28:21 UTC (rev 6197)
+++ long/3D/Gale/trunk/src/StGermain/Discretisation/Mesh/src/Decomp_Sync.h 2007-03-08 19:38:37 UTC (rev 6198)
@@ -120,6 +120,7 @@
unsigned* nSinks, unsigned** sinks );
void Decomp_Sync_SetRequired( void* sync, unsigned nRequired, unsigned* required );
void Decomp_Sync_AddRequired( void* sync, unsigned nRequired, unsigned* required );
+ void Decomp_Sync_BuildShared( void* sync );
unsigned Decomp_Sync_GetGlobalSize( void* sync );
unsigned Decomp_Sync_GetLocalSize( void* sync );
@@ -158,8 +159,6 @@
unsigned* nSrcs, unsigned** srcs,
unsigned* nSnks, unsigned** snks );
- void Decomp_Sync_BuildShared( Decomp_Sync* self );
-
void Decomp_Sync_Destruct( Decomp_Sync* self );
void Decomp_Sync_DestructDecomp( Decomp_Sync* self );
void Decomp_Sync_DestructComm( Decomp_Sync* self );
Modified: long/3D/Gale/trunk/src/StGermain/Discretisation/Mesh/src/MeshTopology.c
===================================================================
--- long/3D/Gale/trunk/src/StGermain/Discretisation/Mesh/src/MeshTopology.c 2007-03-08 19:28:21 UTC (rev 6197)
+++ long/3D/Gale/trunk/src/StGermain/Discretisation/Mesh/src/MeshTopology.c 2007-03-08 19:38:37 UTC (rev 6198)
@@ -1202,6 +1202,7 @@
ranks[p_i] = p_i;
Decomp_Sync_AddSinks( sync, nIncRanks, ranks, nShadowedEls, shadowedEls );
Decomp_Sync_AddSources( sync, nIncRanks, ranks, nShadowEls, shadowEls );
+ Decomp_Sync_BuildShared( sync );
FreeArray( ranks );
FreeArray( nShadowEls );
FreeArray2D( nIncRanks, shadowEls );
@@ -1453,6 +1454,8 @@
for( p_i = 0; p_i < nOldIncRanks; p_i++ )
FreeObject( shdSets[p_i] );
FreeArray( shdSets );
+
+ Decomp_Sync_BuildShared( sync );
}
void MeshTopology_ExchangeShadowedIncidence( MeshTopology* self, MeshTopology_Dim dim,
More information about the cig-commits
mailing list