[cig-commits] r5828 - in long/3D/Gale/trunk/src/StGermain: .
Discretisation/Mesh/src
walter at geodynamics.org
walter at geodynamics.org
Thu Jan 18 14:28:15 PST 2007
Author: walter
Date: 2007-01-18 14:28:15 -0800 (Thu, 18 Jan 2007)
New Revision: 5828
Modified:
long/3D/Gale/trunk/src/StGermain/
long/3D/Gale/trunk/src/StGermain/Discretisation/Mesh/src/Decomp_Sync.c
long/3D/Gale/trunk/src/StGermain/Discretisation/Mesh/src/Decomp_Sync.h
long/3D/Gale/trunk/src/StGermain/Discretisation/Mesh/src/MeshClass.c
long/3D/Gale/trunk/src/StGermain/Discretisation/Mesh/src/MeshClass.h
long/3D/Gale/trunk/src/StGermain/Discretisation/Mesh/src/MeshTopology.c
long/3D/Gale/trunk/src/StGermain/Discretisation/Mesh/src/MeshTopology.h
Log:
r3317 at earth (orig r3978): LukeHodkinson | 2007-01-17 23:54:38 -0800
Changing the way shared topological elements are
detected; the old way was occaisionally skipping
weakly connected processors.
Property changes on: long/3D/Gale/trunk/src/StGermain
___________________________________________________________________
Name: svk:merge
- 1ef209d2-b310-0410-a72d-e20c9eb0015c:/cig:3196
afb6c753-b9d0-0310-b4e7-dbd8d91cdd35:/branches/decomp3d/StGermain:3977
afb6c753-b9d0-0310-b4e7-dbd8d91cdd35:/trunk/StGermain:3899
+ 1ef209d2-b310-0410-a72d-e20c9eb0015c:/cig:3196
afb6c753-b9d0-0310-b4e7-dbd8d91cdd35:/branches/decomp3d/StGermain:3978
afb6c753-b9d0-0310-b4e7-dbd8d91cdd35:/trunk/StGermain:3899
Modified: long/3D/Gale/trunk/src/StGermain/Discretisation/Mesh/src/Decomp_Sync.c
===================================================================
--- long/3D/Gale/trunk/src/StGermain/Discretisation/Mesh/src/Decomp_Sync.c 2007-01-18 22:28:10 UTC (rev 5827)
+++ long/3D/Gale/trunk/src/StGermain/Discretisation/Mesh/src/Decomp_Sync.c 2007-01-18 22:28:15 UTC (rev 5828)
@@ -83,7 +83,7 @@
self->owners = NULL;
self->grMap = UIntMap_New();
- self->lsMap = UIntMap_New();
+ self->dsMap = UIntMap_New();
self->netSrcs = 0;
self->nSrcs = NULL;
@@ -318,6 +318,8 @@
self->nSrcs[rank] += nSources[p_i];
self->netSrcs += nSources[p_i];
}
+
+ Decomp_Sync_BuildShared( self );
}
void Decomp_Sync_AddSinks( void* sync, unsigned nRanks, unsigned* ranks,
@@ -574,19 +576,19 @@
return self->remotes[domain - Decomp_GetLocalSize( self->decomp )];
}
-Bool Decomp_Sync_LocalToShared( void* sync, unsigned local, unsigned* shared ) {
+Bool Decomp_Sync_DomainToShared( void* sync, unsigned domain, unsigned* shared ) {
Decomp_Sync* self = (Decomp_Sync*)sync;
assert( self );
assert( self->decomp );
- assert( local < Decomp_GetLocalSize( self->decomp ) );
- assert( self->lsMap );
+ assert( domain < self->nDomains );
+ assert( self->dsMap );
assert( shared );
- return UIntMap_Map( self->lsMap, local, shared );
+ return UIntMap_Map( self->dsMap, domain, shared );
}
-unsigned Decomp_Sync_SharedToLocal( void* sync, unsigned shared ) {
+unsigned Decomp_Sync_SharedToDomain( void* sync, unsigned shared ) {
Decomp_Sync* self = (Decomp_Sync*)sync;
assert( self );
@@ -857,9 +859,13 @@
void Decomp_Sync_BuildShared( Decomp_Sync* self ) {
unsigned nIncRanks;
- RangeSet *snkSet, *tmpSet;
+ RangeSet *dSet, **rSets, *allSet;
+ unsigned nLocals, *locals;
+ unsigned nSendBytes, *nRecvBytes;
+ Stg_Byte *sendBytes, **recvBytes;
unsigned **sharers;
- unsigned p_i, s_i;
+ unsigned nInds, *inds;
+ unsigned r_i, s_i, ind_i;
assert( self );
@@ -868,19 +874,77 @@
FreeArray( self->nSharers );
FreeArray( self->sharers );
- /* Create a range set of all sinks. */
nIncRanks = CommTopology_GetIncidenceSize( self->commTopo );
- snkSet = RangeSet_New();
+
+ /* Communicate a range set of my domain indices to all neighbours. */
+ dSet = RangeSet_New();
+ RangeSet_SetIndices( dSet, self->nRemotes, self->remotes );
+ Decomp_GetLocals( self->decomp, &nLocals, &locals );
+ RangeSet_AddIndices( dSet, nLocals, locals );
+ RangeSet_Pickle( dSet, &nSendBytes, &sendBytes );
+ CommTopology_Allgather( self->commTopo,
+ nSendBytes, sendBytes,
+ &nRecvBytes, &recvBytes,
+ sizeof(Stg_Byte) );
+ FreeArray( sendBytes );
+ rSets = AllocArray( RangeSet*, nIncRanks );
+ allSet = RangeSet_New();
+ for( r_i = 0; r_i < nIncRanks; r_i++ ) {
+ rSets[r_i] = RangeSet_New();
+ RangeSet_Unpickle( rSets[r_i], nRecvBytes[r_i], recvBytes[r_i] );
+ RangeSet_Intersection( rSets[r_i], dSet );
+ RangeSet_Union( allSet, rSets[r_i] );
+ FreeArray( recvBytes[r_i] );
+ }
+ FreeArray( recvBytes );
+ FreeArray( nRecvBytes );
+
+ /* Use the 'allSet' to generate the shared indices. */
+ RangeSet_GetIndices( allSet, &self->nShared, &self->shared );
+ FreeObject( allSet );
+ for( s_i = 0; s_i < self->nShared; s_i++ ) {
+ insist( Decomp_Sync_GlobalToDomain( self, self->shared[s_i], self->shared + s_i ) );
+ UIntMap_Insert( self->dsMap, self->shared[s_i], s_i );
+ }
+
+ /* Now that we have each intersection, convert to shared arrays. */
+ self->nSharers = AllocArray( unsigned, self->nShared );
+ memset( self->nSharers, 0, self->nShared * sizeof(unsigned) );
+ sharers = AllocArray2D( unsigned, self->nShared, nIncRanks );
+ for( r_i = 0; r_i < nIncRanks; r_i++ ) {
+ RangeSet_GetIndices( rSets[r_i], &nInds, &inds );
+ FreeObject( rSets[r_i] );
+ for( ind_i = 0; ind_i < nInds; ind_i++ ) {
+ insist( Decomp_Sync_GlobalToDomain( self, inds[ind_i], inds + ind_i ) );
+ insist( Decomp_Sync_DomainToShared( self, inds[ind_i], inds + ind_i ) );
+ sharers[inds[ind_i]][self->nSharers[inds[ind_i]]++] = r_i;
+ }
+ FreeArray( inds );
+ }
+ FreeArray( rSets );
+
+ /* Store final array. */
+ self->sharers = AllocComplex2D( unsigned, self->nShared, self->nSharers );
+ for( s_i = 0; s_i < self->nShared; s_i++ )
+ memcpy( self->sharers[s_i], sharers[s_i], self->nSharers[s_i] * sizeof(unsigned) );
+ FreeArray( sharers );
+
+#if 0
+ /* Create a range set of all sinks and sources. */
+ nIncRanks = CommTopology_GetIncidenceSize( self->commTopo );
+ shareSet = RangeSet_New();
tmpSet = RangeSet_New();
for( p_i = 0; p_i < nIncRanks; p_i++ ) {
RangeSet_SetIndices( tmpSet, self->nSnks[p_i], self->snks[p_i] );
- RangeSet_Union( snkSet, tmpSet );
+ RangeSet_Union( shareSet, tmpSet );
+ RangeSet_SetIndices( tmpSet, self->nSrcs[p_i], self->srcs[p_i] );
+ RangeSet_Union( shareSet, tmpSet );
}
FreeObject( tmpSet );
/* These indices are the shared elements. */
- RangeSet_GetIndices( snkSet, &self->nShared, &self->shared );
- FreeObject( snkSet );
+ RangeSet_GetIndices( shareSet, &self->nShared, &self->shared );
+ FreeObject( shareSet );
/* If there are no shared indices, exit now. */
if( !self->nShared )
@@ -888,7 +952,7 @@
/* Build the shared mapping. */
for( s_i = 0; s_i < self->nShared; s_i++ )
- UIntMap_Insert( self->lsMap, self->shared[s_i], s_i );
+ UIntMap_Insert( self->dsMap, self->shared[s_i], s_i );
/* Create temporary storage for the sharers. */
self->nSharers = AllocNamedArray( unsigned, self->nShared, "Decomp_Sync::nSharers" );
@@ -901,7 +965,7 @@
unsigned sharedInd;
unsigned curSharer;
- insist( UIntMap_Map( self->lsMap, self->snks[p_i][s_i], &sharedInd ) );
+ insist( UIntMap_Map( self->dsMap, self->snks[p_i][s_i], &sharedInd ) );
curSharer = self->nSharers[sharedInd]++;
sharers[sharedInd][curSharer] = p_i;
}
@@ -914,6 +978,7 @@
/* Free the old sharers array. */
FreeArray( sharers );
+#endif
}
void Decomp_Sync_Destruct( Decomp_Sync* self ) {
@@ -983,7 +1048,7 @@
self->nShared = 0;
KillArray( self->nSharers );
KillArray( self->sharers );
- UIntMap_Clear( self->lsMap );
+ UIntMap_Clear( self->dsMap );
self->netSnks = 0;
for( p_i = 0; p_i < nIncRanks; p_i++ )
KillArray( self->snks[p_i] );
Modified: long/3D/Gale/trunk/src/StGermain/Discretisation/Mesh/src/Decomp_Sync.h
===================================================================
--- long/3D/Gale/trunk/src/StGermain/Discretisation/Mesh/src/Decomp_Sync.h 2007-01-18 22:28:10 UTC (rev 5827)
+++ long/3D/Gale/trunk/src/StGermain/Discretisation/Mesh/src/Decomp_Sync.h 2007-01-18 22:28:15 UTC (rev 5828)
@@ -67,7 +67,7 @@
unsigned* owners; \
\
UIntMap* grMap; \
- UIntMap* lsMap; \
+ UIntMap* dsMap; \
\
unsigned netSrcs; \
unsigned* nSrcs; \
@@ -137,8 +137,8 @@
Bool Decomp_Sync_GlobalToDomain( void* sync, unsigned global, unsigned* domain );
unsigned Decomp_Sync_DomainToGlobal( void* sync, unsigned domain );
- Bool Decomp_Sync_LocalToShared( void* sync, unsigned local, unsigned* shared );
- unsigned Decomp_Sync_SharedToLocal( void* sync, unsigned shared );
+ Bool Decomp_Sync_DomainToShared( void* sync, unsigned domain, unsigned* shared );
+ unsigned Decomp_Sync_SharedToDomain( void* sync, unsigned shared );
void Decomp_Sync_SyncArray( void* sync, Decomp_Sync_Array* array );
Modified: long/3D/Gale/trunk/src/StGermain/Discretisation/Mesh/src/MeshClass.c
===================================================================
--- long/3D/Gale/trunk/src/StGermain/Discretisation/Mesh/src/MeshClass.c 2007-01-18 22:28:10 UTC (rev 5827)
+++ long/3D/Gale/trunk/src/StGermain/Discretisation/Mesh/src/MeshClass.c 2007-01-18 22:28:15 UTC (rev 5828)
@@ -355,22 +355,22 @@
return MeshTopology_DomainToGlobal( self->topo, dim, domain );
}
-Bool Mesh_LocalToShared( void* mesh, MeshTopology_Dim dim, unsigned local, unsigned* shared ) {
+Bool Mesh_DomainToShared( void* mesh, MeshTopology_Dim dim, unsigned domain, unsigned* shared ) {
Mesh* self = (Mesh*)mesh;
assert( self );
assert( self->topo );
- return MeshTopology_LocalToShared( self->topo, dim, local, shared );
+ return MeshTopology_DomainToShared( self->topo, dim, domain, shared );
}
-unsigned Mesh_SharedToLocal( void* mesh, MeshTopology_Dim dim, unsigned shared ) {
+unsigned Mesh_SharedToDomain( void* mesh, MeshTopology_Dim dim, unsigned shared ) {
Mesh* self = (Mesh*)mesh;
assert( self );
assert( self->topo );
- return MeshTopology_SharedToLocal( self->topo, dim, shared );
+ return MeshTopology_SharedToDomain( self->topo, dim, shared );
}
unsigned Mesh_GetOwner( void* mesh, MeshTopology_Dim dim, unsigned remote ) {
Modified: long/3D/Gale/trunk/src/StGermain/Discretisation/Mesh/src/MeshClass.h
===================================================================
--- long/3D/Gale/trunk/src/StGermain/Discretisation/Mesh/src/MeshClass.h 2007-01-18 22:28:10 UTC (rev 5827)
+++ long/3D/Gale/trunk/src/StGermain/Discretisation/Mesh/src/MeshClass.h 2007-01-18 22:28:15 UTC (rev 5828)
@@ -133,8 +133,8 @@
Bool Mesh_GlobalToDomain( void* mesh, MeshTopology_Dim dim, unsigned global, unsigned* domain );
unsigned Mesh_DomainToGlobal( void* mesh, MeshTopology_Dim dim, unsigned domain );
- Bool Mesh_LocalToShared( void* meshTopology, MeshTopology_Dim dim, unsigned local, unsigned* shared );
- unsigned Mesh_SharedToLocal( void* meshTopology, MeshTopology_Dim dim, unsigned shared );
+ Bool Mesh_DomainToShared( void* meshTopology, MeshTopology_Dim dim, unsigned domain, unsigned* shared );
+ unsigned Mesh_SharedToDomain( void* meshTopology, MeshTopology_Dim dim, unsigned shared );
unsigned Mesh_GetOwner( void* mesh, MeshTopology_Dim dim, unsigned remote );
void Mesh_GetSharers( void* mesh, MeshTopology_Dim dim, unsigned shared,
Modified: long/3D/Gale/trunk/src/StGermain/Discretisation/Mesh/src/MeshTopology.c
===================================================================
--- long/3D/Gale/trunk/src/StGermain/Discretisation/Mesh/src/MeshTopology.c 2007-01-18 22:28:10 UTC (rev 5827)
+++ long/3D/Gale/trunk/src/StGermain/Discretisation/Mesh/src/MeshTopology.c 2007-01-18 22:28:15 UTC (rev 5828)
@@ -626,7 +626,7 @@
return Decomp_Sync_DomainToGlobal( self->domains[dim], domain );
}
-Bool MeshTopology_LocalToShared( void* meshTopology, MeshTopology_Dim dim, unsigned local, unsigned* shared ) {
+Bool MeshTopology_DomainToShared( void* meshTopology, MeshTopology_Dim dim, unsigned domain, unsigned* shared ) {
MeshTopology* self = (MeshTopology*)meshTopology;
assert( self );
@@ -634,10 +634,10 @@
assert( self->domains );
assert( self->domains[dim] );
- return Decomp_Sync_LocalToShared( self->domains[dim], local, shared );
+ return Decomp_Sync_DomainToShared( self->domains[dim], domain, shared );
}
-unsigned MeshTopology_SharedToLocal( void* meshTopology, MeshTopology_Dim dim, unsigned shared ) {
+unsigned MeshTopology_SharedToDomain( void* meshTopology, MeshTopology_Dim dim, unsigned shared ) {
MeshTopology* self = (MeshTopology*)meshTopology;
assert( self );
@@ -645,7 +645,7 @@
assert( self->domains );
assert( self->domains[dim] );
- return Decomp_Sync_SharedToLocal( self->domains[dim], shared );
+ return Decomp_Sync_SharedToDomain( self->domains[dim], shared );
}
unsigned MeshTopology_GetOwner( void* meshTopology, MeshTopology_Dim dim, unsigned remote ) {
@@ -933,12 +933,24 @@
shdSets[p_i] = IndexSet_New( nDomainEls );
/* Store connected vertices. */
+ for( s_i = 0; s_i < Decomp_Sync_GetSharedSize( sync ); s_i++ ) {
+ unsigned domainInd;
+ unsigned nSharers, *sharers;
+ unsigned sharer_i;
+
+ domainInd = Decomp_Sync_SharedToDomain( sync, s_i );
+ Decomp_Sync_GetSharers( sync, s_i, &nSharers, &sharers );
+ for( sharer_i = 0; sharer_i < nSharers; sharer_i++ )
+ IndexSet_Add( shdSets[sharers[sharer_i]], domainInd );
+ }
+#if 0
for( p_i = 0; p_i < nIncRanks; p_i++ ) {
for( s_i = 0; s_i < sync->nSrcs[p_i]; s_i++ )
IndexSet_Add( shdSets[p_i], sync->srcs[p_i][s_i] );
for( s_i = 0; s_i < sync->nSnks[p_i]; s_i++ )
IndexSet_Add( shdSets[p_i], sync->snks[p_i][s_i] );
}
+#endif
/* Build range sets of shadowed elements. */
for( p_i = 0; p_i < nIncRanks; p_i++ ) {
Modified: long/3D/Gale/trunk/src/StGermain/Discretisation/Mesh/src/MeshTopology.h
===================================================================
--- long/3D/Gale/trunk/src/StGermain/Discretisation/Mesh/src/MeshTopology.h 2007-01-18 22:28:10 UTC (rev 5827)
+++ long/3D/Gale/trunk/src/StGermain/Discretisation/Mesh/src/MeshTopology.h 2007-01-18 22:28:15 UTC (rev 5828)
@@ -130,8 +130,8 @@
Bool MeshTopology_GlobalToDomain( void* meshTopology, MeshTopology_Dim dim, unsigned global, unsigned* domain );
unsigned MeshTopology_DomainToGlobal( void* meshTopology, MeshTopology_Dim dim, unsigned domain );
- Bool MeshTopology_LocalToShared( void* meshTopology, MeshTopology_Dim dim, unsigned local, unsigned* shared );
- unsigned MeshTopology_SharedToLocal( void* meshTopology, MeshTopology_Dim dim, unsigned shared );
+ Bool MeshTopology_DomainToShared( void* meshTopology, MeshTopology_Dim dim, unsigned domain, unsigned* shared );
+ unsigned MeshTopology_SharedToDomain( void* meshTopology, MeshTopology_Dim dim, unsigned shared );
Bool MeshTopology_HasIncidence( void* meshTopology, MeshTopology_Dim fromDim, MeshTopology_Dim toDim );
unsigned MeshTopology_GetIncidenceSize( void* meshTopology, MeshTopology_Dim fromDim, unsigned fromInd,
More information about the cig-commits
mailing list