[cig-commits] r4326 - in long/3D/Gale/trunk/src/StGermain: .
Discretisation/Mesh/src
walter at geodynamics.org
walter at geodynamics.org
Thu Aug 17 17:17:12 PDT 2006
Author: walter
Date: 2006-08-17 17:17:12 -0700 (Thu, 17 Aug 2006)
New Revision: 4326
Added:
long/3D/Gale/trunk/src/StGermain/Discretisation/Mesh/src/Decomp.c
long/3D/Gale/trunk/src/StGermain/Discretisation/Mesh/src/Decomp.h
long/3D/Gale/trunk/src/StGermain/Discretisation/Mesh/src/Decomp.meta
Modified:
long/3D/Gale/trunk/src/StGermain/
Log:
r2704 at earth: boo | 2006-08-17 17:14:18 -0700
r2658 at earth (orig r3739): LukeHodkinson | 2006-08-01 22:30:34 -0700
Adding a new component to manage parallel arrays of
data. This is mainly complete, but is still being
tested.
Property changes on: long/3D/Gale/trunk/src/StGermain
___________________________________________________________________
Name: svk:merge
- 1ef209d2-b310-0410-a72d-e20c9eb0015c:/cig:2703
afb6c753-b9d0-0310-b4e7-dbd8d91cdd35:/trunk/StGermain:3738
+ 1ef209d2-b310-0410-a72d-e20c9eb0015c:/cig:2704
afb6c753-b9d0-0310-b4e7-dbd8d91cdd35:/trunk/StGermain:3739
Added: long/3D/Gale/trunk/src/StGermain/Discretisation/Mesh/src/Decomp.c
===================================================================
--- long/3D/Gale/trunk/src/StGermain/Discretisation/Mesh/src/Decomp.c 2006-08-18 00:17:09 UTC (rev 4325)
+++ long/3D/Gale/trunk/src/StGermain/Discretisation/Mesh/src/Decomp.c 2006-08-18 00:17:12 UTC (rev 4326)
@@ -0,0 +1,750 @@
+/*~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+**
+** Copyright (C), 2003, Victorian Partnership for Advanced Computing (VPAC) Ltd, 110 Victoria Street, Melbourne, 3053, Australia.
+**
+** Authors:
+** Stevan M. Quenette, Senior Software Engineer, VPAC. (steve at vpac.org)
+** Patrick D. Sunter, Software Engineer, VPAC. (pds at vpac.org)
+** Luke J. Hodkinson, Computational Engineer, VPAC. (lhodkins at vpac.org)
+** Siew-Ching Tan, Software Engineer, VPAC. (siew at vpac.org)
+** Alan H. Lo, Computational Engineer, VPAC. (alan at vpac.org)
+** Raquibul Hassan, Computational Engineer, VPAC. (raq at vpac.org)
+**
+** This library is free software; you can redistribute it and/or
+** modify it under the terms of the GNU Lesser General Public
+** License as published by the Free Software Foundation; either
+** version 2.1 of the License, or (at your option) any later version.
+**
+** This library is distributed in the hope that it will be useful,
+** but WITHOUT ANY WARRANTY; without even the implied warranty of
+** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+** Lesser General Public License for more details.
+**
+** You should have received a copy of the GNU Lesser General Public
+** License along with this library; if not, write to the Free Software
+** Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
+**
+** $Id: Decomp.c 3584 2006-05-16 11:11:07Z PatrickSunter $
+**
+**~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~*/
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <assert.h>
+#include <mpi.h>
+
+#include "Base/Base.h"
+
+#include "types.h"
+#include "shortcuts.h"
+#include "Decomp.h"
+
+
+/* Textual name of this class */
+const Type Decomp_Type = "Decomp";
+
+/*----------------------------------------------------------------------------------------------------------------------------------
+** Constructors
+*/
+
+Decomp* Decomp_New( Name name ) {
+ return _Decomp_New( sizeof(Decomp),
+ Decomp_Type,
+ _Decomp_Delete,
+ _Decomp_Print,
+ _Decomp_Copy,
+ (void* (*)(Name))_Decomp_New,
+ _Decomp_Construct,
+ _Decomp_Build,
+ _Decomp_Initialise,
+ _Decomp_Execute,
+ _Decomp_Destroy,
+ name,
+ NON_GLOBAL,
+ Decomp_Select );
+}
+
+Decomp* _Decomp_New( DECOMP_DEFARGS ) {
+ Decomp* self;
+
+ /* Allocate memory */
+ assert( sizeOfSelf >= sizeof(Decomp) );
+ self = (Decomp*)_Stg_Component_New( STG_COMPONENT_PASSARGS );
+
+ /* Virtual info */
+ self->selectFunc = selectFunc;
+
+ /* Decomp info */
+ _Decomp_Init( self );
+
+ return self;
+}
+
+void _Decomp_Init( Decomp* self ) {
+ self->comm = MPI_COMM_WORLD;
+ MPI_Comm_size( self->comm, (int*)&self->nProcs );
+ MPI_Comm_rank( self->comm, (int*)&self->rank );
+ self->nGlobals = 0;
+ self->nLocals = 0;
+ self->locals = NULL;
+ self->nLeased = 0;
+ self->leased = NULL;
+ self->nRemotes = 0;
+ self->remotes = NULL;
+ self->nShadows = 0;
+ self->shadows = NULL;
+ self->nDomains = 0;
+ self->domains = NULL;
+ self->nRentals = 0;
+ self->rentals = NULL;
+ self->gdMap = NULL;
+ self->netSrcs = 0;
+ self->nSrcs = NULL;
+ self->srcs = NULL;
+ self->netSnks = 0;
+ self->nSnks = NULL;
+ self->snks = NULL;
+ self->nArrays = 0;
+ self->arrays = NULL;
+}
+
+
+/*----------------------------------------------------------------------------------------------------------------------------------
+** Virtual functions
+*/
+
+void _Decomp_Delete( void* generator ) {
+ Decomp* self = (Decomp*)generator;
+
+ Decomp_Destruct( self );
+
+ /* Delete the parent. */
+ _Stg_Component_Delete( self );
+}
+
+void _Decomp_Print( void* generator, Stream* stream ) {
+ Decomp* self = (Decomp*)generator;
+
+ /* Set the Journal for printing informations */
+ Stream* generatorStream;
+ generatorStream = Journal_Register( InfoStream_Type, "DecompStream" );
+
+ /* Print parent */
+ Journal_Printf( stream, "Decomp (ptr): (%p)\n", self );
+ _Stg_Component_Print( self, stream );
+}
+
+void* _Decomp_Copy( void* generator, void* destProc_I, Bool deep, Name nameExt, PtrMap* ptrMap ) {
+#if 0
+ Decomp* self = (Decomp*)generator;
+ Decomp* newDecomp;
+ PtrMap* map = ptrMap;
+ Bool ownMap = False;
+
+ /* Damn me for making copying so difficult... what was I thinking? */
+
+ /* We need to create a map if it doesn't already exist. */
+ if( !map ) {
+ map = PtrMap_New( 10 );
+ ownMap = True;
+ }
+
+ newDecomp = (Decomp*)_Mesh_Copy( self, destProc_I, deep, nameExt, map );
+
+ /* Copy the virtual methods here. */
+
+ /* Deep or shallow? */
+ if( deep ) {
+ }
+ else {
+ }
+
+ /* If we own the map, get rid of it here. */
+ if( ownMap ) Stg_Class_Delete( map );
+
+ return (void*)newDecomp;
+#endif
+
+ return NULL;
+}
+
+void _Decomp_Construct( void* generator, Stg_ComponentFactory* cf ) {
+}
+
+void _Decomp_Build( void* generator, void* data ) {
+}
+
+void _Decomp_Initialise( void* generator, void* data ) {
+}
+
+void _Decomp_Execute( void* generator, void* data ) {
+}
+
+void _Decomp_Destroy( void* generator, void* data ) {
+}
+
+void Decomp_Select( void* decomp, unsigned* nFound, unsigned** found ) {
+ Decomp* self = (Decomp*)decomp;
+ unsigned* nUniques;
+ unsigned** uniques;
+ unsigned mostUniques;
+ unsigned p_i;
+
+ assert( self );
+ assert( nFound );
+ assert( found );
+
+ /* Set up the source arrays. */
+ self->nSrcs = Memory_Alloc_Array( unsigned, self->nProcs, "Decomp::nSrcs" );
+ self->srcs = Memory_Alloc_Array( unsigned*, self->nProcs, "Decomp::srcs" );
+ memset( self->nSrcs, 0, self->nProcs * sizeof(unsigned) );
+ memset( self->srcs, 0, self->nProcs * sizeof(unsigned*) );
+
+ /* Select source procs from which to retrieve required nodes such that communications are minimized. */
+ nUniques = Memory_Alloc_Array( unsigned, self->nProcs, "" );
+ uniques = Memory_Alloc_2DComplex( unsigned, self->nProcs, nFound, "" );
+ memset( nUniques, 0, self->nProcs * sizeof(unsigned) );
+
+ do {
+ unsigned mostProc = 0;
+
+ /* Loop over procs and find unique nodes per proc. ie. the most nodes not already covered by
+ an existing source. */
+ mostUniques = 0;
+ for( p_i = 0; p_i < self->nProcs; p_i++ ) {
+ unsigned fnd_i;
+
+ /* Clear the number of uniques. */
+ nUniques[p_i] = 0;
+
+ /* If there are no founds for this proc or we've already sourced it, then skip. */
+ if( nFound[p_i] == 0 || self->nSrcs[p_i] > 0 )
+ continue;
+
+ /* Hunt down unique global indices. */
+ for( fnd_i = 0; fnd_i < nFound[p_i]; fnd_i++ ) {
+ unsigned p_j;
+
+ for( p_j = 0; p_j < self->nProcs; p_j++ ) {
+ unsigned src_i;
+
+ for( src_i = 0; src_i < self->nSrcs[p_i]; src_i++ ) {
+ if( self->srcs[p_i][src_i] == found[p_i][fnd_i] )
+ break;
+ }
+ if( src_i < self->nSrcs[p_i] )
+ break;
+ }
+ if( p_j == self->nProcs )
+ uniques[p_i][nUniques[p_i]++] = found[p_i][fnd_i];
+ }
+
+ /* Determine which proc has the most uniques and store. */
+ if( nUniques[p_i] > mostUniques ) {
+ mostUniques = nUniques[p_i];
+ mostProc = p_i;
+ }
+ }
+
+ /* Store result. */
+ if( mostUniques ) {
+ self->nSrcs[mostProc] = mostUniques;
+ if( mostUniques ) {
+ self->srcs[mostProc] = Memory_Alloc_Array( unsigned, mostUniques, "Decomp::srcs[]" );
+ memcpy( self->srcs[mostProc], uniques[mostProc], mostUniques * sizeof(unsigned) );
+ }
+ }
+ }
+ while( mostUniques );
+
+ /* Get rid of all the resources. */
+ FreeArray( nUniques );
+ FreeArray( uniques );
+}
+
+/*--------------------------------------------------------------------------------------------------------------------------
+** Public Functions
+*/
+
+void Decomp_SetComm( void* decomp, MPI_Comm comm ) {
+ Decomp* self = (Decomp*)decomp;
+
+ assert( self );
+
+ Decomp_Destruct( self );
+ self->comm = comm;
+ MPI_Comm_size( comm, (int*)&self->nProcs );
+ MPI_Comm_rank( comm, (int*)&self->rank );
+}
+
+void Decomp_SetNGlobals( void* decomp, unsigned nGlobals ) {
+ Decomp* self = (Decomp*)decomp;
+
+ assert( self );
+
+ Decomp_Destruct( self );
+ self->nGlobals = nGlobals;
+}
+
+void Decomp_SetLocals( void* decomp, unsigned nLocals, unsigned* locals,
+ unsigned nLeased, unsigned* leased )
+{
+ Decomp* self = (Decomp*)decomp;
+
+ assert( self );
+ assert( !nLocals || locals );
+ assert( !nLeased || leased );
+
+ /* Destroy everything. */
+ Decomp_Destruct( self );
+
+ /* Store what we have. */
+ self->nLocals = nLocals;
+ self->nLeased = nLeased;
+ self->nShadows = nLeased;
+ self->nDomains = nLocals + nLeased;
+ if( self->nDomains )
+ self->domains = Memory_Alloc_Array( unsigned, self->nDomains, "Decomp::domains" );
+ if( nLocals ) {
+ self->locals = self->domains;
+ memcpy( self->locals, locals, nLocals * sizeof(unsigned) );
+ }
+ if( nLeased ) {
+ self->leased = self->domains + nLocals;
+ self->shadows = self->leased;
+ memcpy( self->leased, leased, nLeased * sizeof(unsigned) );
+ }
+}
+
+void Decomp_AddRemotes( void* decomp, unsigned nRemotes, unsigned* remotes ) {
+ Decomp* self = (Decomp*)decomp;
+
+ /* Sanity checks. */
+ assert( self );
+ assert( !nRemotes || remotes );
+
+#ifndef NDEBUG
+ /* Ensure we don't already have any of the new remotes. */
+ {
+ unsigned r_i;
+
+ for( r_i = 0; r_i < self->nRemotes; r_i++ ) {
+ unsigned r_j;
+
+ for( r_j = 0; r_j < nRemotes; r_j++ )
+ if( self->remotes[r_i] == remotes[r_j] ) break;
+ assert( r_j < nRemotes );
+ }
+ }
+#endif
+
+ /* Destroy negotiated information. */
+ Decomp_DestructNegotiated( self );
+
+ /* Store remote information. */
+ self->nRemotes += nRemotes;
+ if( nRemotes ) {
+ self->nDomains += nRemotes;
+ if( !self->domains )
+ self->domains = Memory_Alloc_Array( unsigned, self->nDomains, "Decomp::domains" );
+ else
+ self->domains = Memory_Realloc_Array( self->domains, unsigned, self->nDomains );
+ self->remotes = self->domains + self->nLocals + self->nLeased;
+ memcpy( self->remotes, remotes, nRemotes * sizeof(unsigned) );
+ }
+ self->nShadows = self->nLeased + nRemotes;
+ if( self->nShadows )
+ self->shadows = self->domains + self->nLocals;
+ else
+ self->shadows = NULL;
+}
+
+unsigned Decomp_DomainToGlobal( void* decomp, unsigned domain ) {
+ Decomp* self = (Decomp*)decomp;
+
+ assert( self );
+ assert( domain < self->nDomains );
+ assert( self->domains );
+
+ return self->domains[domain];
+}
+
+void Decomp_Negotiate( void* decomp ) {
+ Decomp* self = (Decomp*)decomp;
+ unsigned* nRemFound;
+ unsigned** remFound;
+ unsigned p_i;
+
+ /* Sanity checks. */
+ assert( self );
+
+ /* Clobber existing negotiated information. */
+ Decomp_DestructNegotiated( self );
+
+ /* Build global to domain map. */
+ Decomp_BuildGDMap( self );
+
+ /* Determine sources/sinks, and cleanup afterwards. */
+ Decomp_RemoteSearch( self, &nRemFound, &remFound );
+ Decomp_Select( self, nRemFound, remFound );
+ FreeArray( nRemFound );
+ FreeArray2D( self->nProcs, remFound );
+
+ /* Communicate back to all procs our selection. Note that even though most procs will not need to send
+ anything here, we still need to let them know that. */
+ MPIArray2D_Alltoall( self->nSrcs, (void**)self->srcs,
+ &self->nSnks, (void***)&self->snks,
+ sizeof(unsigned), self->comm );
+
+ /* Calculate net values for source and sink. */
+ for( p_i = 0; p_i < self->nProcs; p_i++ ) {
+ self->netSrcs += self->nSrcs[p_i];
+ self->netSnks += self->nSnks[p_i];
+#if 0
+ for( s_i = 0; s_i < self->nSrcs[p_i]; s_i++ )
+ self->srcs[p_i][s_i] = UIntMap_Map( self->gdMap, self->srcs[p_i][s_i] );
+ for( s_i = 0; s_i < self->nSnks[p_i]; s_i++ )
+ self->snks[p_i][s_i] = UIntMap_Map( self->gdMap, self->snks[p_i][s_i] );
+#endif
+ }
+}
+
+Bool Decomp_IsDomain( void* decomp, unsigned global ) {
+ Decomp* self = (Decomp*)decomp;
+
+ assert( self );
+ assert( global < self->nGlobals );
+ assert( self->gdMap );
+
+ return UIntMap_HasKey( self->gdMap, global );
+}
+
+unsigned Decomp_GlobalToDomain( void* decomp, unsigned global ) {
+ Decomp* self = (Decomp*)decomp;
+
+ assert( self );
+ assert( global < self->nGlobals );
+ assert( self->gdMap );
+
+ return UIntMap_Map( self->gdMap, global );
+}
+
+void Decomp_AddArray( void* decomp, void* localArray, void* shadowArray,
+ size_t localStride, size_t shadowStride, size_t itemSize )
+{
+ Decomp* self = (Decomp*)decomp;
+ Decomp_Array* array;
+
+ /* Sanity checks. */
+ assert( self );
+ assert( !self->nLocals || localArray );
+ assert( !self->nShadows || shadowArray );
+ assert( itemSize );
+
+ /* Resize the array array (?). */
+ if( self->nArrays ) {
+ self->arrays = Memory_Realloc_Array( self->arrays, Decomp_Array, ++self->nArrays );
+ }
+ else {
+ self->arrays = Memory_Alloc_Array( Decomp_Array, ++self->nArrays, "Decomp::Arrays" );
+ }
+ array = self->arrays + self->nArrays - 1;
+
+ /* Store information. */
+ array->snkArray = localArray;
+ array->snkStride = localStride;
+ array->srcArray = shadowArray;
+ array->srcStride = shadowStride;
+ array->itemSize = itemSize;
+
+ /* Build this array. */
+ Decomp_BuildArray( self, array );
+}
+
+void Decomp_Sync( void* decomp ) {
+ Decomp* self = (Decomp*)decomp;
+ unsigned a_i;
+
+ /* Sanity checks. */
+ assert( self );
+ assert( !self->nArrays || self->arrays );
+
+ for( a_i = 0; a_i < self->nArrays; a_i++ )
+ Decomp_SyncArray( self, self->arrays + a_i );
+}
+
+
+/*----------------------------------------------------------------------------------------------------------------------------------
+** Private Functions
+*/
+
+void Decomp_SyncArray( Decomp* self, Decomp_Array* array ) {
+ unsigned char* snkArray;
+ unsigned char* srcArray;
+
+ /* Sanity checks. */
+ assert( array );
+
+ /* Pack from locals to a contiguous array. */
+ if( self->netSnks > 0 ) {
+ unsigned snk_i;
+
+ snkArray = Memory_Alloc_Array_Unnamed( unsigned char, self->netSnks * array->itemSize );
+ for( snk_i = 0; snk_i < self->netSnks; snk_i++ ) {
+ memcpy( snkArray + snk_i * array->itemSize,
+ (unsigned char*)array->snkArray + array->snkOffs[snk_i],
+ array->itemSize );
+ }
+ }
+ else
+ snkArray = NULL;
+
+ /* Allocate for sources. */
+ if( self->netSrcs > 0 )
+ srcArray = Memory_Alloc_Array_Unnamed( unsigned char, self->netSrcs * array->itemSize );
+ else
+ srcArray = NULL;
+
+ /* Transfer. */
+ MPI_Alltoallv( snkArray, (int*)array->snkSizes, (int*)array->snkDisps, MPI_BYTE,
+ srcArray, (int*)array->srcSizes, (int*)array->srcDisps, MPI_BYTE,
+ self->comm );
+
+ /* Free the sink array. */
+ FreeArray( snkArray );
+
+ /* Unpack sources. */
+ if( self->netSnks > 0 ) {
+ unsigned src_i;
+
+ for( src_i = 0; src_i < self->netSrcs; src_i++ ) {
+ memcpy( (unsigned char*)array->srcArray + array->srcOffs[src_i],
+ srcArray + src_i * array->itemSize,
+ array->itemSize );
+ }
+ }
+
+ /* Free source array. */
+ FreeArray( srcArray );
+}
+
+void Decomp_BuildAllArrays( Decomp* self ) {
+ unsigned a_i;
+
+ for( a_i = 0; a_i < self->nArrays; a_i++ )
+ Decomp_BuildArray( self, self->arrays + a_i );
+}
+
+void Decomp_BuildArray( Decomp* self, Decomp_Array* array ) {
+ /* Determine sink (local) information. */
+ if( self->netSnks > 0 ) {
+ unsigned* snkOffs;
+ unsigned* snkSizes;
+ unsigned* snkDisps;
+ unsigned snkInd = 0;
+ unsigned p_i;
+
+ /* Allocate/reallocate memory. */
+ snkDisps = Memory_Alloc_Array( unsigned, self->nProcs, "Decomp_Array::snkDisps" );
+ snkSizes = Memory_Alloc_Array( unsigned, self->nProcs, "Decomp_Array::snkSizes" );
+ snkOffs = Memory_Alloc_Array( unsigned, self->netSnks, "Decomp_Array::snkOffs" );
+
+ /* Calculate offsets and sizes. */
+ for( p_i = 0; p_i < self->nProcs; p_i++ ) {
+ unsigned snk_i;
+
+ snkSizes[p_i] = 0;
+ for( snk_i = 0; snk_i < self->nSnks[p_i]; snk_i++ ) {
+ unsigned dInd = Decomp_GlobalToDomain( self, self->snks[p_i][snk_i] );
+
+ snkOffs[snkInd] = dInd * array->snkStride;
+ snkSizes[p_i] += array->itemSize;
+ snkInd++;
+ }
+ }
+
+ /* Calculate the displacements. */
+ snkDisps[0] = 0;
+ for( p_i = 1; p_i < self->nProcs; p_i++ ) {
+ snkDisps[p_i] = snkDisps[p_i - 1] + snkSizes[p_i - 1];
+ }
+
+ /* Store arrays. */
+ array->snkOffs = snkOffs;
+ array->snkDisps = snkDisps;
+ array->snkSizes = snkSizes;
+ }
+ else {
+ /* Store null information. */
+ array->snkOffs = Memory_Alloc_Array( unsigned, self->nProcs, "Decomp_Array::snkOffs" );
+ array->snkDisps = Memory_Alloc_Array( unsigned, self->nProcs, "Decomp_Array::snkDisps" );
+ array->snkSizes = Memory_Alloc_Array( unsigned, self->nProcs, "Decomp_Array::snkSizes" );
+ memset( array->snkOffs, 0, sizeof(unsigned) * self->nProcs );
+ memset( array->snkDisps, 0, sizeof(unsigned) * self->nProcs );
+ memset( array->snkSizes, 0, sizeof(unsigned) * self->nProcs );
+ }
+
+ /* Determine source (shadow) information. */
+ if( self->netSrcs > 0 ) {
+ unsigned* srcOffs;
+ unsigned* srcSizes;
+ unsigned* srcDisps;
+ unsigned srcInd = 0;
+ unsigned p_i;
+
+ /* Allocate/reallocate memory. */
+ srcDisps = Memory_Alloc_Array( unsigned, self->nProcs, "Decomp_Array::srcDisps" );
+ srcSizes = Memory_Alloc_Array( unsigned, self->nProcs, "Decomp_Array::srcSizes" );
+ srcOffs = Memory_Alloc_Array( unsigned, self->netSrcs, "Decomp_Array::srcOffs" );
+
+ /* Calculate offsets and sizes. */
+ for( p_i = 0; p_i < self->nProcs; p_i++ ) {
+ unsigned src_i;
+
+ srcSizes[p_i] = 0;
+ for( src_i = 0; src_i < self->nSrcs[p_i]; src_i++ ) {
+ unsigned sInd = Decomp_GlobalToDomain( self, self->srcs[p_i][src_i] );
+
+ assert( sInd >= self->nLocals );
+ sInd -= self->nLocals;
+ srcOffs[srcInd] = sInd * array->srcStride;
+ srcSizes[p_i] += array->itemSize;
+ srcInd++;
+ }
+ }
+
+ /* Calculate the displacements. */
+ srcDisps[0] = 0;
+ for( p_i = 1; p_i < self->nProcs; p_i++ ) {
+ srcDisps[p_i] = srcDisps[p_i - 1] + srcSizes[p_i - 1];
+ }
+
+ /* Store arrays. */
+ array->srcOffs = srcOffs;
+ array->srcDisps = srcDisps;
+ array->srcSizes = srcSizes;
+ }
+ else {
+ /* Store null information. */
+ array->srcOffs = Memory_Alloc_Array( unsigned, self->nProcs, "Decomp_Array::srcOffs" );
+ array->srcDisps = Memory_Alloc_Array( unsigned, self->nProcs, "Decomp_Array::srcDisps" );
+ array->srcSizes = Memory_Alloc_Array( unsigned, self->nProcs, "Decomp_Array::srcSizes" );
+ memset( array->srcOffs, 0, sizeof(unsigned) * self->nProcs );
+ memset( array->srcDisps, 0, sizeof(unsigned) * self->nProcs );
+ memset( array->srcSizes, 0, sizeof(unsigned) * self->nProcs );
+ }
+}
+
+void Decomp_RemoteSearch( Decomp* self, unsigned** nRemFound, unsigned*** remFound ) {
+ unsigned* remNReq;
+ unsigned** remReq;
+ unsigned* nFound;
+ unsigned** found;
+ unsigned p_i;
+
+ assert( self );
+ assert( nRemFound );
+ assert( remFound );
+
+ /* Send our requireds to all others and receive from all others. */
+ MPIArray_Allgather( self->nShadows, self->shadows,
+ &remNReq, (void***)&remReq,
+ sizeof(unsigned), self->comm );
+
+ /* Prepare to locate remote requireds. */
+ nFound = Memory_Alloc_Array( unsigned, self->nProcs, "" );
+ found = Memory_Alloc_2DComplex( unsigned, self->nProcs, remNReq, "" );
+
+ /* Begin searching... */
+ for( p_i = 0; p_i < self->nProcs; p_i++ ) {
+ unsigned rem_i;
+
+ /* Initialise found count to zero. */
+ nFound[p_i] = 0;
+
+ /* Skip thyself. */
+ if( p_i == self->rank ) continue;
+
+ /* Do we have any for this processor? */
+ for( rem_i = 0; rem_i < remNReq[p_i]; rem_i++ ) {
+ unsigned l_i;
+
+ for( l_i = 0; l_i < self->nLocals; l_i++ ) {
+ if( self->locals[l_i] == remReq[p_i][rem_i] ) {
+ found[p_i][nFound[p_i]++] = remReq[p_i][rem_i];
+ break;
+ }
+ }
+ }
+ }
+
+ /* Release some memory. */
+ FreeArray( remNReq );
+ FreeArray2D( self->nProcs, remReq );
+
+ /* Send back all the ones we found and receive from all others all our requireds they found. */
+ MPIArray2D_Alltoall( nFound, (void**)found,
+ nRemFound, (void***)remFound,
+ sizeof(unsigned), self->comm );
+
+ /* Release some memory. */
+ FreeArray( nFound );
+ FreeArray( found );
+}
+
+void Decomp_BuildGDMap( Decomp* self ) {
+ UIntMap* map;
+ unsigned d_i;
+
+ assert( self );
+
+ FreeObject( self->gdMap );
+
+ map = UIntMap_New();
+ for( d_i = 0; d_i < self->nDomains; d_i++ )
+ UIntMap_Insert( map, self->domains[d_i], d_i );
+ self->gdMap = map;
+}
+
+void Decomp_Destruct( Decomp* self ) {
+ Decomp_DestructNegotiated( self );
+ KillArray( self->domains );
+ self->domains = NULL;
+ self->locals = NULL;
+ self->leased = NULL;
+ self->remotes = NULL;
+ self->shadows = NULL;
+ self->nLocals = 0;
+ self->nLeased = 0;
+ self->nRemotes = 0;
+ self->nShadows = 0;
+ KillArray( self->rentals );
+ self->rentals = 0;
+}
+
+void Decomp_DestructNegotiated( Decomp* self ) {
+ Decomp_DestructArrays( self );
+ KillObject( self->gdMap );
+ self->netSrcs = 0;
+ KillArray( self->nSrcs );
+ KillArray2D( self->nProcs, self->srcs );
+ self->netSnks = 0;
+ KillArray( self->nSnks );
+ KillArray2D( self->nProcs, self->snks );
+}
+
+void Decomp_DestructArrays( Decomp* self ) {
+ unsigned a_i;
+
+ for( a_i = 0; a_i < self->nArrays; a_i++ ) {
+ FreeArray( self->arrays[a_i].snkDisps );
+ FreeArray( self->arrays[a_i].snkSizes );
+ FreeArray( self->arrays[a_i].snkOffs );
+ FreeArray( self->arrays[a_i].srcDisps );
+ FreeArray( self->arrays[a_i].srcSizes );
+ FreeArray( self->arrays[a_i].srcOffs );
+ }
+ KillArray( self->arrays );
+ self->nArrays = 0;
+}
Added: long/3D/Gale/trunk/src/StGermain/Discretisation/Mesh/src/Decomp.h
===================================================================
--- long/3D/Gale/trunk/src/StGermain/Discretisation/Mesh/src/Decomp.h 2006-08-18 00:17:09 UTC (rev 4325)
+++ long/3D/Gale/trunk/src/StGermain/Discretisation/Mesh/src/Decomp.h 2006-08-18 00:17:12 UTC (rev 4326)
@@ -0,0 +1,179 @@
+/*~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+**
+** Copyright (C), 2003, Victorian Partnership for Advanced Computing (VPAC) Ltd, 110 Victoria Street, Melbourne, 3053, Australia.
+**
+** Authors:
+** Stevan M. Quenette, Senior Software Engineer, VPAC. (steve at vpac.org)
+** Patrick D. Sunter, Software Engineer, VPAC. (pds at vpac.org)
+** Luke J. Hodkinson, Computational Engineer, VPAC. (lhodkins at vpac.org)
+** Siew-Ching Tan, Software Engineer, VPAC. (siew at vpac.org)
+** Alan H. Lo, Computational Engineer, VPAC. (alan at vpac.org)
+** Raquibul Hassan, Computational Engineer, VPAC. (raq at vpac.org)
+**
+** This library is free software; you can redistribute it and/or
+** modify it under the terms of the GNU Lesser General Public
+** License as published by the Free Software Foundation; either
+** version 2.1 of the License, or (at your option) any later version.
+**
+** This library is distributed in the hope that it will be useful,
+** but WITHOUT ANY WARRANTY; without even the implied warranty of
+** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+** Lesser General Public License for more details.
+**
+** You should have received a copy of the GNU Lesser General Public
+** License along with this library; if not, write to the Free Software
+** Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
+**
+*/
+/** \file
+** Role:
+**
+** Assumptions:
+**
+** Invariants:
+**
+** Comments:
+**
+** $Id: Decomp.h 3584 2006-05-16 11:11:07Z PatrickSunter $
+**
+**~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~*/
+
+#ifndef __Discretisaton_Mesh_Decomp_h__
+#define __Discretisaton_Mesh_Decomp_h__
+
+ /** Textual name of this class */
+ extern const Type Decomp_Type;
+
+ /** Virtual function types */
+ typedef void (Decomp_SelectFunc)( void* decomp, unsigned* nFound, unsigned** found );
+
+ /** Mesh class contents */
+ typedef struct {
+ unsigned localInd;
+ unsigned nProcs;
+ unsigned* procs;
+ } Decomp_Rental;
+
+ typedef struct {
+ void* snkArray;
+ size_t snkStride;
+ unsigned* snkDisps;
+ unsigned* snkSizes;
+ unsigned* snkOffs;
+
+ void* srcArray;
+ size_t srcStride;
+ unsigned* srcDisps;
+ unsigned* srcSizes;
+ unsigned* srcOffs;
+
+ size_t itemSize;
+ } Decomp_Array;
+
+ #define __Decomp \
+ /* General info */ \
+ __Stg_Component \
+ \
+ /* Virtual info */ \
+ Decomp_SelectFunc* selectFunc; \
+ \
+ /* Decomp info */ \
+ MPI_Comm comm; \
+ unsigned nProcs; \
+ unsigned rank; \
+ \
+ unsigned nGlobals; \
+ unsigned nLocals; \
+ unsigned* locals; \
+ unsigned nLeased; \
+ unsigned* leased; \
+ unsigned nRemotes; \
+ unsigned* remotes; \
+ unsigned nShadows; \
+ unsigned* shadows; \
+ unsigned nDomains; \
+ unsigned* domains; \
+ unsigned nRentals; \
+ Decomp_Rental* rentals; \
+ \
+ UIntMap* gdMap; \
+ \
+ unsigned netSrcs; \
+ unsigned* nSrcs; \
+ unsigned** srcs; \
+ unsigned netSnks; \
+ unsigned* nSnks; \
+ unsigned** snks; \
+ \
+ unsigned nArrays; \
+ Decomp_Array* arrays;
+
+ struct Decomp { __Decomp };
+
+ /*--------------------------------------------------------------------------------------------------------------------------
+ ** Constructors
+ */
+
+ #define DECOMP_DEFARGS \
+ STG_COMPONENT_DEFARGS, \
+ Decomp_SelectFunc* selectFunc
+
+ #define DECOMP_PASSARGS \
+ STG_COMPONENT_PASSARGS, selectFunc
+
+ Decomp* Decomp_New( Name name );
+ Decomp* _Decomp_New( DECOMP_DEFARGS );
+ void _Decomp_Init( Decomp* self );
+
+ /*--------------------------------------------------------------------------------------------------------------------------
+ ** Virtual functions
+ */
+
+ void _Decomp_Delete( void* generator );
+ void _Decomp_Print( void* generator, Stream* stream );
+
+ #define Decomp_Copy( self ) \
+ (Mesh*)Stg_Class_Copy( self, NULL, False, NULL, NULL )
+ #define Decomp_DeepCopy( self ) \
+ (Mesh*)Stg_Class_Copy( self, NULL, True, NULL, NULL )
+ void* _Decomp_Copy( void* generator, void* dest, Bool deep, Name nameExt, PtrMap* ptrMap );
+
+ void _Decomp_Construct( void* generator, Stg_ComponentFactory* cf );
+ void _Decomp_Build( void* generator, void* data );
+ void _Decomp_Initialise( void* generator, void* data );
+ void _Decomp_Execute( void* generator, void* data );
+ void _Decomp_Destroy( void* generator, void* data );
+
+ void Decomp_Select( void* decomp, unsigned* nFount, unsigned** found );
+
+ /*--------------------------------------------------------------------------------------------------------------------------
+ ** Public functions
+ */
+
+ void Decomp_SetComm( void* decomp, MPI_Comm comm );
+ void Decomp_SetNGlobals( void* decomp, unsigned nGlobals );
+ void Decomp_SetLocals( void* decomp, unsigned nLocals, unsigned* locals,
+ unsigned nLeased, unsigned* leased );
+ void Decomp_AddRemotes( void* decomp, unsigned nRemotes, unsigned* remotes );
+ void Decomp_Negotiate( void* decomp );
+ Bool Decomp_IsDomain( void* decomp, unsigned global );
+ unsigned Decomp_GlobalToDomain( void* decomp, unsigned global );
+ unsigned Decomp_DomainToGlobal( void* decomp, unsigned domain );
+ void Decomp_AddArray( void* decomp, void* localArray, void* shadowArray,
+ size_t localStride, size_t shadowStride, size_t itemSize );
+ void Decomp_Sync( void* decomp );
+
+ /*--------------------------------------------------------------------------------------------------------------------------
+ ** Private Member functions
+ */
+
+ void Decomp_SyncArray( Decomp* self, Decomp_Array* array );
+ void Decomp_BuildAllArrays( Decomp* self );
+ void Decomp_BuildArray( Decomp* self, Decomp_Array* array );
+ void Decomp_RemoteSearch( Decomp* self, unsigned** nRemFound, unsigned*** remFound );
+ void Decomp_BuildGDMap( Decomp* self );
+ void Decomp_Destruct( Decomp* self );
+ void Decomp_DestructNegotiated( Decomp* self );
+ void Decomp_DestructArrays( Decomp* self );
+
+#endif /* __Discretisaton_Mesh_Decomp_h__ */
Added: long/3D/Gale/trunk/src/StGermain/Discretisation/Mesh/src/Decomp.meta
===================================================================
--- long/3D/Gale/trunk/src/StGermain/Discretisation/Mesh/src/Decomp.meta 2006-08-18 00:17:09 UTC (rev 4325)
+++ long/3D/Gale/trunk/src/StGermain/Discretisation/Mesh/src/Decomp.meta 2006-08-18 00:17:12 UTC (rev 4326)
@@ -0,0 +1,15 @@
+<?xml version="1.0"?>
+<!DOCTYPE StGermainData SYSTEM "stgermain.dtd">
+<StGermainData xmlns="http://www.vpac.org/StGermain/XML_IO_Handler/Jun2003">
+
+<param name="Name">Decomp</param>
+<param name="Organisation">VPAC</param>
+<param name="Project">StGermain</param>
+<param name="Location">./StGermain/Discretisation/Utils/src/</param>
+<param name="Project Web">https://csd.vpac.org/twiki/bin/view/Stgermain/WebHome</param>
+<param name="Copyright">StGermain Framework. Copyright (C) 2003-2005 VPAC.</param>
+<param name="License">The Gnu Lesser General Public License http://www.gnu.org/licenses/lgpl.html</param>
+<param name="Parent">Stg_Component</param>
+<param name="Description">...</param>
+
+</StGermainData>
More information about the cig-commits
mailing list