[cig-commits] r4869 - in long/3D/Gale/trunk/src/StGermain: .
Discretisation/Mesh/src
walter at geodynamics.org
walter at geodynamics.org
Wed Oct 11 13:47:55 PDT 2006
Author: walter
Date: 2006-10-11 13:47:54 -0700 (Wed, 11 Oct 2006)
New Revision: 4869
Modified:
long/3D/Gale/trunk/src/StGermain/
long/3D/Gale/trunk/src/StGermain/Discretisation/Mesh/src/MeshTopology.c
long/3D/Gale/trunk/src/StGermain/Discretisation/Mesh/src/MeshTopology.h
Log:
r2926 at earth: boo | 2006-10-11 13:42:45 -0700
r2842 at earth (orig r3830): LukeHodkinson | 2006-10-04 18:14:19 -0700
* Adding a couple of helper routines for extracting
domain sizes and mapping between global and domain
indices.
* When small domain sizes are specified, sometimes the
shadow depth will cause non-neighbouring processes to
become neighbours. This was causing communication
problems. Now, communications are done first then the
domain indices are updated.
Property changes on: long/3D/Gale/trunk/src/StGermain
___________________________________________________________________
Name: svk:merge
- 1ef209d2-b310-0410-a72d-e20c9eb0015c:/cig:2925
afb6c753-b9d0-0310-b4e7-dbd8d91cdd35:/trunk/StGermain:3829
+ 1ef209d2-b310-0410-a72d-e20c9eb0015c:/cig:2926
afb6c753-b9d0-0310-b4e7-dbd8d91cdd35:/trunk/StGermain:3830
Modified: long/3D/Gale/trunk/src/StGermain/Discretisation/Mesh/src/MeshTopology.c
===================================================================
--- long/3D/Gale/trunk/src/StGermain/Discretisation/Mesh/src/MeshTopology.c 2006-10-11 20:47:52 UTC (rev 4868)
+++ long/3D/Gale/trunk/src/StGermain/Discretisation/Mesh/src/MeshTopology.c 2006-10-11 20:47:54 UTC (rev 4869)
@@ -483,7 +483,8 @@
unsigned nbr_i;
/* Don't add current element. */
- if( curNbr == e_i ) continue;
+ if( curNbr == e_i )
+ continue;
/* Ensure is unique. */
for( nbr_i = 0; nbr_i < nNbrs[e_i]; nbr_i++ ) {
@@ -535,7 +536,29 @@
return self->nDomainEls[dim];
}
+unsigned MeshTopology_DomainToGlobal( void* meshTopology, MeshTopology_Dim dim, unsigned domain ) {
+ MeshTopology* self = (MeshTopology*)meshTopology;
+ assert( self );
+ assert( dim < self->nTDims );
+ assert( self->domains );
+ assert( self->domains[dim] );
+
+ return Decomp_Sync_DomainToGlobal( self->domains[dim], domain );
+}
+
+unsigned MeshTopology_GlobalToDomain( void* meshTopology, MeshTopology_Dim dim, unsigned global ) {
+ MeshTopology* self = (MeshTopology*)meshTopology;
+
+ assert( self );
+ assert( dim < self->nTDims );
+ assert( self->domains );
+ assert( self->domains[dim] );
+
+ return Decomp_Sync_GlobalToDomain( self->domains[dim], global );
+}
+
+
/*----------------------------------------------------------------------------------------------------------------------------------
** Private Functions
*/
@@ -689,6 +712,15 @@
unsigned nInds;
unsigned* inds;
+ /* If there are no elements in this dimension or we have no incidence relations between
+ these levels, skip it. */
+ if( MeshTopology_GetLocalSize( self, d_i ) == 0 ||
+ !self->nIncEls[self->nDims][d_i] || !self->incEls[self->nDims][d_i] )
+ {
+ shadows[d_i][p_i] = NULL;
+ continue;
+ }
+
iSet = IndexSet_New( MeshTopology_GetLocalSize( self, d_i ) );
for( e_i = 0; e_i < nElInds; e_i++ ) {
@@ -750,6 +782,13 @@
unsigned offs = 0;
unsigned e_i;
+ /* If we have no shadows for this dimension, skip. */
+ if( !shadows[d_i][p_i] ) {
+ (*incDataSizes)[d_i][p_i] = 0;
+ (*incData)[d_i][p_i] = NULL;
+ continue;
+ }
+
/* Extract the indices. */
RangeSet_GetIndices( shadows[d_i][p_i], &nEls, &els );
for( e_i = 0; e_i < nEls; e_i++ )
@@ -774,7 +813,8 @@
(*incData)[d_i][p_i][offs++] = gElInd;
(*incData)[d_i][p_i][offs++] = nIncEls;
for( inc_i = 0; inc_i < nIncEls; inc_i++ )
- (*incData)[d_i][p_i][offs++] = Decomp_Sync_DomainToGlobal( self->domains[d_i - 1], incEls[inc_i] );
+ (*incData)[d_i][p_i][offs++] = Decomp_Sync_DomainToGlobal( self->domains[d_i - 1],
+ incEls[inc_i] );
}
/* Free the indices. */
@@ -789,11 +829,12 @@
MeshTopology* self = (MeshTopology*)meshTopology;
CommTopology* commTopo;
unsigned nIncRanks;
- unsigned* incRanks;
unsigned* nSrcBytes;
Stg_Byte** srcBytes;
unsigned* nDstBytes;
Stg_Byte** dstBytes;
+ unsigned** dataSize;
+ unsigned*** data;
unsigned p_i, d_i;
assert( self );
@@ -801,9 +842,9 @@
assert( incDataSizes );
assert( incData );
- /* Extract communicator incidence. */
+ /* Shortcuts. */
commTopo = self->domains[MT_VERTEX]->commTopo;
- CommTopology_GetIncidence( commTopo, commTopo->rank, &nIncRanks, &incRanks );
+ nIncRanks = commTopo->nInc;
/* Convert downward incidence relations to global indices. */
for( d_i = 1; d_i < self->nTDims; d_i++ ) {
@@ -823,9 +864,6 @@
nSrcBytes = Memory_Alloc_Array_Unnamed( unsigned, nIncRanks );
srcBytes = Memory_Alloc_Array_Unnamed( Stg_Byte*, nIncRanks );
for( d_i = 0; d_i < self->nTDims; d_i++ ) {
- unsigned nRemotes;
- unsigned* remotes;
-
for( p_i = 0; p_i < nIncRanks; p_i++ )
RangeSet_Pickle( shadows[d_i][p_i], nSrcBytes + p_i, srcBytes + p_i );
CommTopology_Alltoall( commTopo, nSrcBytes, (void**)srcBytes,
@@ -842,12 +880,31 @@
RangeSet_Union( shadows[d_i][0], shadows[d_i][p_i] );
FreeObject( shadows[d_i][p_i] );
}
+ }
+ /* Send and recieve all the incidence relations. */
+ dataSize = Memory_Alloc_Array_Unnamed( unsigned*, self->nTDims );
+ data = Memory_Alloc_Array_Unnamed( unsigned**, self->nTDims );
+ for( d_i = 1; d_i < self->nTDims; d_i++ ) {
+ /* Communicate. */
+ CommTopology_Alltoall( commTopo, incDataSizes[d_i], (void**)incData[d_i],
+ dataSize + d_i, (void***)(data + d_i), sizeof(unsigned) );
+
+ /* Free old incidence relations. */
+ for( p_i = 0; p_i < nIncRanks; p_i++ )
+ FreeArray( incData[d_i][p_i] );
+ }
+ FreeArray( incData );
+ FreeArray( incDataSizes );
+
+ /* Update remote elements. */
+ for( d_i = 0; d_i < self->nTDims; d_i++ ) {
+ unsigned nRemotes;
+ unsigned* remotes;
+
/* Set the remote elements. */
RangeSet_GetIndices( shadows[d_i][0], &nRemotes, &remotes );
Decomp_Sync_SetRemotes( self->domains[d_i], nRemotes, remotes );
- if( d_i == MT_VERTEX )
- commTopo = self->domains[MT_VERTEX]->commTopo;
self->nDomainEls[d_i] = self->domains[d_i]->decomp->nLocals + nRemotes;
FreeArray( remotes );
@@ -858,23 +915,18 @@
FreeArray( srcBytes );
FreeArray( shadows );
- /* Send and recieve all the incidence relations. */
+ /* Update incidence. */
for( d_i = 1; d_i < self->nTDims; d_i++ ) {
- unsigned* dataSize;
- unsigned** data;
unsigned** oldInc;
unsigned e_i;
- CommTopology_Alltoall( commTopo, incDataSizes[d_i], (void**)incData[d_i],
- &dataSize, (void***)&data, sizeof(unsigned) );
-
/* Resize old incidence relations to include new remote sizes. */
self->nIncEls[d_i][d_i - 1] = Memory_Realloc_Array( self->nIncEls[d_i][d_i - 1],
unsigned, self->nDomainEls[d_i] );
/* Insert each other procs' incidence relations. */
for( p_i = 0; p_i < nIncRanks; p_i++ ) {
- unsigned* curData = data[p_i];
+ unsigned* curData = data[d_i][p_i];
unsigned nEls = curData[0];
curData++;
@@ -898,7 +950,7 @@
/* Copy new relations. */
for( p_i = 0; p_i < nIncRanks; p_i++ ) {
- unsigned* curData = data[p_i];
+ unsigned* curData = data[d_i][p_i];
unsigned nEls = curData[0];
curData++;
@@ -911,7 +963,13 @@
curData += curData[1] + 2;
}
}
+
+ /* Free data and sizes. */
+ FreeArray( data[d_i] );
+ FreeArray( dataSize[d_i] );
}
+ FreeArray( data );
+ FreeArray( dataSize );
/* Convert downward incidence relations to back to domain indices. */
for( d_i = 1; d_i < self->nTDims; d_i++ ) {
Modified: long/3D/Gale/trunk/src/StGermain/Discretisation/Mesh/src/MeshTopology.h
===================================================================
--- long/3D/Gale/trunk/src/StGermain/Discretisation/Mesh/src/MeshTopology.h 2006-10-11 20:47:52 UTC (rev 4868)
+++ long/3D/Gale/trunk/src/StGermain/Discretisation/Mesh/src/MeshTopology.h 2006-10-11 20:47:54 UTC (rev 4869)
@@ -125,6 +125,8 @@
unsigned MeshTopology_GetLocalSize( void* meshTopology, MeshTopology_Dim dim );
unsigned MeshTopology_GetShadowSize( void* meshTopology, MeshTopology_Dim dim );
unsigned MeshTopology_GetDomainSize( void* meshTopology, MeshTopology_Dim dim );
+ unsigned MeshTopology_DomainToGlobal( void* meshTopology, MeshTopology_Dim dim, unsigned domain );
+ unsigned MeshTopology_GlobalToDomain( void* meshTopology, MeshTopology_Dim dim, unsigned global );
/*--------------------------------------------------------------------------------------------------------------------------
** Private Member functions
More information about the cig-commits
mailing list