[cig-commits] r4824 - in long/3D/Gale/trunk/src/StGermain: . Discretisation/Mesh/src

[walter at geodynamics.org]

Author: walter
Date: 2006-10-11 13:46:09 -0700 (Wed, 11 Oct 2006)
New Revision: 4824

Added:
   long/3D/Gale/trunk/src/StGermain/Discretisation/Mesh/src/Decomp_Sync.c
   long/3D/Gale/trunk/src/StGermain/Discretisation/Mesh/src/Decomp_Sync.h
   long/3D/Gale/trunk/src/StGermain/Discretisation/Mesh/src/Decomp_Sync.meta
   long/3D/Gale/trunk/src/StGermain/Discretisation/Mesh/src/Decomp_Sync_Claim.c
   long/3D/Gale/trunk/src/StGermain/Discretisation/Mesh/src/Decomp_Sync_Claim.h
   long/3D/Gale/trunk/src/StGermain/Discretisation/Mesh/src/Decomp_Sync_Claim.meta
   long/3D/Gale/trunk/src/StGermain/Discretisation/Mesh/src/Decomp_Sync_Negotiate.c
   long/3D/Gale/trunk/src/StGermain/Discretisation/Mesh/src/Decomp_Sync_Negotiate.h
   long/3D/Gale/trunk/src/StGermain/Discretisation/Mesh/src/Decomp_Sync_Negotiate.meta
Modified:
   long/3D/Gale/trunk/src/StGermain/
   long/3D/Gale/trunk/src/StGermain/Discretisation/Mesh/src/Decomp.c
   long/3D/Gale/trunk/src/StGermain/Discretisation/Mesh/src/Decomp.h
   long/3D/Gale/trunk/src/StGermain/Discretisation/Mesh/src/Mesh.h
   long/3D/Gale/trunk/src/StGermain/Discretisation/Mesh/src/types.h
Log:
 r2881 at earth:  boo | 2006-10-11 13:42:29 -0700
  r2797 at earth (orig r3785):  LukeHodkinson | 2006-09-07 16:56:03 -0700
  Major rebuild of the mesh decomposition system. There are
  three major differences:
  1. More information has been added, such as which processors
     share which topological elements (needed for accurate particle
     ownership tests).
  2. There is a great deal more automation. Now, the system can
     accept an array of requested element indices and negotiate
     which elements it will own and where it will source remote
     elements from.
  3. There are two new interfaces for modifying how the system
     calculates ownership of elements and negotiating where to source
     remote elements from.
  
 



Property changes on: long/3D/Gale/trunk/src/StGermain
___________________________________________________________________
Name: svk:merge
   - 1ef209d2-b310-0410-a72d-e20c9eb0015c:/cig:2880
afb6c753-b9d0-0310-b4e7-dbd8d91cdd35:/trunk/StGermain:3784
   + 1ef209d2-b310-0410-a72d-e20c9eb0015c:/cig:2881
afb6c753-b9d0-0310-b4e7-dbd8d91cdd35:/trunk/StGermain:3785

Modified: long/3D/Gale/trunk/src/StGermain/Discretisation/Mesh/src/Decomp.c
===================================================================
--- long/3D/Gale/trunk/src/StGermain/Discretisation/Mesh/src/Decomp.c	2006-10-11 20:46:07 UTC (rev 4823)
+++ long/3D/Gale/trunk/src/StGermain/Discretisation/Mesh/src/Decomp.c	2006-10-11 20:46:09 UTC (rev 4824)
@@ -38,6 +38,8 @@
 
 #include "types.h"
 #include "shortcuts.h"
+#include "CommTopology.h"
+#include "Decomp_Sync.h"
 #include "Decomp.h"
 
 
@@ -61,8 +63,7 @@
 			    _Decomp_Execute, 
 			    _Decomp_Destroy, 
 			    name, 
-			    NON_GLOBAL, 
-			    Decomp_Select );
+			    NON_GLOBAL );
 }
 
 Decomp* _Decomp_New( DECOMP_DEFARGS ) {
@@ -73,7 +74,6 @@
 	self = (Decomp*)_Stg_Component_New( STG_COMPONENT_PASSARGS );
 
 	/* Virtual info */
-	self->selectFunc = selectFunc;
 
 	/* Decomp info */
 	_Decomp_Init( self );
@@ -83,30 +83,12 @@
 
 void _Decomp_Init( Decomp* self ) {
 	self->comm = MPI_COMM_WORLD;
-	MPI_Comm_size( self->comm, (int*)&self->nProcs );
-	MPI_Comm_rank( self->comm, (int*)&self->rank );
+
 	self->nGlobals = 0;
 	self->nLocals = 0;
 	self->locals = NULL;
-	self->nLeased = 0;
-	self->leased = NULL;
-	self->nRemotes = 0;
-	self->remotes = NULL;
-	self->nShadows = 0;
-	self->shadows = NULL;
-	self->nDomains = 0;
-	self->domains = NULL;
-	self->nRentals = 0;
-	self->rentals = NULL;
-	self->gdMap = NULL;
-	self->netSrcs = 0;
-	self->nSrcs = NULL;
-	self->srcs = NULL;
-	self->netSnks = 0;
-	self->nSnks = NULL;
-	self->snks = NULL;
-	self->nArrays = 0;
-	self->arrays = NULL;
+
+	self->glMap = NULL;
 }
 
 
@@ -114,8 +96,8 @@
 ** Virtual functions
 */
 
-void _Decomp_Delete( void* generator ) {
-	Decomp*	self = (Decomp*)generator;
+void _Decomp_Delete( void* decomp ) {
+	Decomp*	self = (Decomp*)decomp;
 
 	Decomp_Destruct( self );
 
@@ -123,21 +105,21 @@
 	_Stg_Component_Delete( self );
 }
 
-void _Decomp_Print( void* generator, Stream* stream ) {
-	Decomp*	self = (Decomp*)generator;
+void _Decomp_Print( void* decomp, Stream* stream ) {
+	Decomp*	self = (Decomp*)decomp;
 	
 	/* Set the Journal for printing informations */
-	Stream* generatorStream;
-	generatorStream = Journal_Register( InfoStream_Type, "DecompStream" );
+	Stream* decompStream;
+	decompStream = Journal_Register( InfoStream_Type, "DecompStream" );
 
 	/* Print parent */
 	Journal_Printf( stream, "Decomp (ptr): (%p)\n", self );
 	_Stg_Component_Print( self, stream );
 }
 
-void* _Decomp_Copy( void* generator, void* destProc_I, Bool deep, Name nameExt, PtrMap* ptrMap ) {
+void* _Decomp_Copy( void* decomp, void* destProc_I, Bool deep, Name nameExt, PtrMap* ptrMap ) {
 #if 0
-	Decomp*	self = (Decomp*)generator;
+	Decomp*	self = (Decomp*)decomp;
 	Decomp*	newDecomp;
 	PtrMap*	map = ptrMap;
 	Bool	ownMap = False;
@@ -169,586 +151,224 @@
 	return NULL;
 }
 
-void _Decomp_Construct( void* generator, Stg_ComponentFactory* cf ) {
+void _Decomp_Construct( void* decomp, Stg_ComponentFactory* cf ) {
 }
 
-void _Decomp_Build( void* generator, void* data ) {
+void _Decomp_Build( void* decomp, void* data ) {
 }
 
-void _Decomp_Initialise( void* generator, void* data ) {
+void _Decomp_Initialise( void* decomp, void* data ) {
 }
 
-void _Decomp_Execute( void* generator, void* data ) {
+void _Decomp_Execute( void* decomp, void* data ) {
 }
 
-void _Decomp_Destroy( void* generator, void* data ) {
+void _Decomp_Destroy( void* decomp, void* data ) {
 }
 
-void Decomp_Select( void* decomp, unsigned* nFound, unsigned** found ) {
-	Decomp*		self = (Decomp*)decomp;
-	unsigned*	nUniques;
-	unsigned**	uniques;
-	unsigned	mostUniques;
-	unsigned	p_i;
 
-	assert( self );
-	assert( nFound );
-	assert( found );
-
-	/* Set up the source arrays. */
-	self->nSrcs = Memory_Alloc_Array( unsigned, self->nProcs, "Decomp::nSrcs" );
-	self->srcs = Memory_Alloc_Array( unsigned*, self->nProcs, "Decomp::srcs" );
-	memset( self->nSrcs, 0, self->nProcs * sizeof(unsigned) );
-	memset( self->srcs, 0, self->nProcs * sizeof(unsigned*) );
-
-	/* Select source procs from which to retrieve required nodes such that communications are minimized. */
-	nUniques = Memory_Alloc_Array( unsigned, self->nProcs, "" );
-	uniques = Memory_Alloc_2DComplex( unsigned, self->nProcs, nFound, "" );
-	memset( nUniques, 0, self->nProcs * sizeof(unsigned) );
-
-	do {
-		unsigned	mostProc = 0;
-
-		/* Loop over procs and find unique nodes per proc. ie. the most nodes not already covered by
-		   an existing source. */
-		mostUniques = 0;
-		for( p_i = 0; p_i < self->nProcs; p_i++ ) {
-			unsigned	fnd_i;
-
-			/* Clear the number of uniques. */
-			nUniques[p_i] = 0;
-
-			/* If there are no founds for this proc or we've already sourced it, then skip. */
-			if( nFound[p_i] == 0 || self->nSrcs[p_i] > 0 )
-				continue;
-
-			/* Hunt down unique global indices. */
-			for( fnd_i = 0; fnd_i < nFound[p_i]; fnd_i++ ) {
-				unsigned	p_j;
-
-				for( p_j = 0; p_j < self->nProcs; p_j++ ) {
-					unsigned	src_i;
-
-					for( src_i = 0; src_i < self->nSrcs[p_i]; src_i++ ) {
-						if( self->srcs[p_i][src_i] == found[p_i][fnd_i] )
-							break;
-					}
-					if( src_i < self->nSrcs[p_i] )
-						break;
-				}
-				if( p_j == self->nProcs )
-					uniques[p_i][nUniques[p_i]++] = found[p_i][fnd_i];
-			}
-
-			/* Determine which proc has the most uniques and store. */
-			if( nUniques[p_i] > mostUniques ) {
-				mostUniques = nUniques[p_i];
-				mostProc = p_i;
-			}
-		}
-
-		/* Store result. */
-		if( mostUniques ) {
-			self->nSrcs[mostProc] = mostUniques;
-			if( mostUniques ) {
-				self->srcs[mostProc] = Memory_Alloc_Array( unsigned, mostUniques, "Decomp::srcs[]" );
-				memcpy( self->srcs[mostProc], uniques[mostProc], mostUniques * sizeof(unsigned) );
-			}
-		}
-	}
-	while( mostUniques );
-
-	/* Get rid of all the resources. */
-	FreeArray( nUniques );
-	FreeArray( uniques );
-}
-
 /*--------------------------------------------------------------------------------------------------------------------------
 ** Public Functions
 */
 
+#define DECOMP_OK		0x0000
+#define DECOMP_OVERLAPPING	0x0001
+#define DECOMP_INCOMPLETE	0x0002
+
 void Decomp_SetComm( void* decomp, MPI_Comm comm ) {
 	Decomp*	self = (Decomp*)decomp;
 
 	assert( self );
 
 	Decomp_Destruct( self );
+
 	self->comm = comm;
-	MPI_Comm_size( comm, (int*)&self->nProcs );
-	MPI_Comm_rank( comm, (int*)&self->rank );
 }
 
-void Decomp_SetNGlobals( void* decomp, unsigned nGlobals ) {
-	Decomp*	self = (Decomp*)decomp;
+void Decomp_SetLocals( void* decomp, unsigned nLocals, unsigned* locals ) {
+	Decomp*		self = (Decomp*)decomp;
+	unsigned	status;
 
 	assert( self );
-
-	Decomp_Destruct( self );
-	self->nGlobals = nGlobals;
-}
-
-void Decomp_SetLocals( void* decomp, unsigned nLocals, unsigned* locals, 
-		       unsigned nLeased, unsigned* leased )
-{
-	Decomp*	self = (Decomp*)decomp;
-
-	assert( self );
 	assert( !nLocals || locals );
-	assert( !nLeased || leased );
 
-	/* Destroy everything. */
-	Decomp_Destruct( self );
+	Decomp_DestructLocals( self );
 
-	/* Store what we have. */
+	/* Store local information. */
 	self->nLocals = nLocals;
-	self->nLeased = nLeased;
-	self->nShadows = nLeased;
-	self->nDomains = nLocals + nLeased;
-	if( self->nDomains )
-		self->domains = Memory_Alloc_Array( unsigned, self->nDomains, "Decomp::domains" );
-	if( nLocals ) {
-		self->locals = self->domains;
-		memcpy( self->locals, locals, nLocals * sizeof(unsigned) );
-	}
-	if( nLeased ) {
-		self->leased = self->domains + nLocals;
-		self->shadows = self->leased;
-		memcpy( self->leased, leased, nLeased * sizeof(unsigned) );
-	}
-}
+	self->locals = Memory_Alloc_Array( unsigned, nLocals, "Decomp::locals" );
+	memcpy( self->locals, locals, nLocals * sizeof(unsigned) );
 
-void Decomp_SetRemotes( void* decomp, unsigned nRemotes, unsigned* remotes ) {
-	Decomp*	self = (Decomp*)decomp;
+	/* Validate the domain. */
+	Decomp_ValidateDomain( self, &status );
+	assert( status == DECOMP_OK );
 
-	/* Sanity checks. */
-	assert( self );
-	assert( !nRemotes || remotes );
-
-#ifndef NDEBUG
-	/* Ensure we don't already have any of the new remotes. */
-	{
-		unsigned	r_i;
-
-		for( r_i = 0; r_i < self->nRemotes; r_i++ ) {
-			unsigned	r_j;
-
-			for( r_j = 0; r_j < nRemotes; r_j++ )
-				if( self->remotes[r_i] == remotes[r_j] ) break;
-			assert( r_j < nRemotes );
-		}
-	}
-#endif
-
-	/* Destroy negotiated information. */
-	Decomp_DestructNegotiated( self );
-
-	/* Store remote information. */
-	self->nRemotes = nRemotes;
-	self->nDomains = self->nLocals + self->nLeased + nRemotes;
-	if( nRemotes ) {
-		if( !self->domains ) {
-			self->domains = Memory_Alloc_Array( unsigned, self->nDomains, "Decomp::domains" );
-		}
-		else {
-			self->domains = Memory_Realloc_Array( self->domains, unsigned, self->nDomains );
-		}
-		self->locals = self->domains;
-		self->leased = self->locals + self->nLocals;
-		self->remotes = self->leased + self->nLeased;
-		memcpy( self->remotes, remotes, nRemotes * sizeof(unsigned) );
-	}
-	self->nShadows = self->nLeased + nRemotes;
-	if( self->nShadows )
-		self->shadows = self->domains + self->nLocals;
-	else
-		self->shadows = NULL;
+	/* Build global to local map. */
+	Decomp_BuildGLMap( self );
 }
 
-unsigned Decomp_DomainToGlobal( void* decomp, unsigned domain ) {
-	Decomp*	self = (Decomp*)decomp;
+void Decomp_AddSync( void* decomp, Decomp_Sync* sync ) {
+	Decomp*		self = (Decomp*)decomp;
 
 	assert( self );
-	assert( domain < self->nDomains );
-	assert( self->domains );
+	assert( sync );
 
-	return self->domains[domain];
+	if( !self->nSyncs )
+		self->syncs = Memory_Alloc_Array( Decomp_Sync*, ++self->nSyncs, "Decomp::syncs" );
+	else
+		self->syncs = Memory_Realloc_Array( self->syncs, Decomp_Sync*, ++self->nSyncs );
+	self->syncs[self->nSyncs - 1] = sync;
 }
 
-void Decomp_Negotiate( void* decomp ) {
+void Decomp_RemoveSync( void* decomp, Decomp_Sync* sync ) {
 	Decomp*		self = (Decomp*)decomp;
-	unsigned*	nRemFound;
-	unsigned**	remFound;
-	unsigned	p_i;
+	unsigned	s_i = 0;
 
-	/* Sanity checks. */
 	assert( self );
+	assert( sync );
 
-	/* Clobber existing negotiated information. */
-	Decomp_DestructNegotiated( self );
+	while( self->syncs[s_i++] != sync && s_i < self->nSyncs );
+	assert( s_i <= self->nSyncs );
+	for( ; s_i < self->nSyncs; s_i++ )
+		self->syncs[s_i - 1] = self->syncs[s_i];
 
-	/* Build global to domain map. */
-	Decomp_BuildGDMap( self );
-
-	/* Determine sources/sinks, and cleanup afterwards. */
-	Decomp_RemoteSearch( self, &nRemFound, &remFound );
-	Decomp_Select( self, nRemFound, remFound );
-	FreeArray( nRemFound );
-	FreeArray2D( self->nProcs, remFound );
-
-	/* Communicate back to all procs our selection.  Note that even though most procs will not need to send
-	   anything here, we still need to let them know that. */
-	MPIArray2D_Alltoall( self->nSrcs, (void**)self->srcs, 
-			     &self->nSnks, (void***)&self->snks, 
-			     sizeof(unsigned), self->comm );
-
-	/* Calculate net values for source and sink. */
-	for( p_i = 0; p_i < self->nProcs; p_i++ ) {
-		self->netSrcs += self->nSrcs[p_i];
-		self->netSnks += self->nSnks[p_i];
-#if 0
-		for( s_i = 0; s_i < self->nSrcs[p_i]; s_i++ )
-			self->srcs[p_i][s_i] = UIntMap_Map( self->gdMap, self->srcs[p_i][s_i] );
-		for( s_i = 0; s_i < self->nSnks[p_i]; s_i++ )
-			self->snks[p_i][s_i] = UIntMap_Map( self->gdMap, self->snks[p_i][s_i] );
-#endif
+	if( --self->nSyncs == 0 ) {
+		KillArray( self->syncs );
 	}
+	else
+		self->syncs = Memory_Realloc_Array( self->syncs, Decomp_Sync*, self->nSyncs );
 }
 
-Bool Decomp_IsDomain( void* decomp, unsigned global ) {
+Bool Decomp_IsLocal( void* decomp, unsigned global ) {
 	Decomp*	self = (Decomp*)decomp;
 
 	assert( self );
 	assert( global < self->nGlobals );
-	assert( self->gdMap );
+	assert( self->glMap );
 
-	return UIntMap_HasKey( self->gdMap, global );
+	return UIntMap_HasKey( self->glMap, global );
 }
 
-unsigned Decomp_GlobalToDomain( void* decomp, unsigned global ) {
+unsigned Decomp_LocalToGlobal( void* decomp, unsigned local ) {
 	Decomp*	self = (Decomp*)decomp;
 
 	assert( self );
-	assert( global < self->nGlobals );
-	assert( self->gdMap );
+	assert( local < self->nLocals );
+	assert( self->locals );
 
-	return UIntMap_Map( self->gdMap, global );
+	return self->locals[local];
 }
 
-void Decomp_AddArray( void* decomp, void* localArray, void* shadowArray, 
-		      size_t localStride, size_t shadowStride, size_t itemSize )
-{
-	Decomp*		self = (Decomp*)decomp;
-	Decomp_Array*	array;
+unsigned Decomp_GlobalToLocal( void* decomp, unsigned global ) {
+	Decomp*	self = (Decomp*)decomp;
 
-	/* Sanity checks. */
 	assert( self );
-	assert( !self->nLocals || localArray );
-	assert( !self->nShadows || shadowArray );
-	assert( itemSize );
+	assert( global < self->nGlobals );
+	assert( self->glMap );
 
-	/* Resize the array array (?). */
-	if( self->nArrays ) {
-		self->arrays = Memory_Realloc_Array( self->arrays, Decomp_Array, ++self->nArrays );
-	}
-	else {
-		self->arrays = Memory_Alloc_Array( Decomp_Array, ++self->nArrays, "Decomp::Arrays" );
-	}
-	array = self->arrays + self->nArrays - 1;
-
-	/* Store information. */
-	array->snkArray = localArray;
-	array->snkStride = localStride;
-	array->srcArray = shadowArray;
-	array->srcStride = shadowStride;
-	array->itemSize = itemSize;
-
-	/* Build this array. */
-	Decomp_BuildArray( self, array );
+	return UIntMap_Map( self->glMap, global );
 }
 
-void Decomp_Sync( void* decomp ) {
-	Decomp*		self = (Decomp*)decomp;
-	unsigned	a_i;
 
-	/* Sanity checks. */
-	assert( self );
-	assert( !self->nArrays || self->arrays );
-
-	for( a_i = 0; a_i < self->nArrays; a_i++ )
-		Decomp_SyncArray( self, self->arrays + a_i );
-}
-
-
 /*----------------------------------------------------------------------------------------------------------------------------------
 ** Private Functions
 */
 
-void Decomp_SyncArray( Decomp* self, Decomp_Array* array ) {
-	unsigned char*	snkArray;
-	unsigned char*	srcArray;
+void Decomp_BuildGLMap( Decomp* self ) {
+	UIntMap*	map;
+	unsigned	l_i;
 
-	/* Sanity checks. */
-	assert( array );
+	FreeObject( self->glMap );
 
-	/* Pack from locals to a contiguous array. */
-	if( self->netSnks > 0 ) {
-		unsigned	snk_i;
-
-		snkArray = Memory_Alloc_Array_Unnamed( unsigned char, self->netSnks * array->itemSize );
-		for( snk_i = 0; snk_i < self->netSnks; snk_i++ ) {
-			memcpy( snkArray + snk_i * array->itemSize, 
-				(unsigned char*)array->snkArray + array->snkOffs[snk_i], 
-				array->itemSize );
-		}
-	}
-	else
-		snkArray = NULL;
-
-	/* Allocate for sources. */
-	if( self->netSrcs > 0 )
-		srcArray = Memory_Alloc_Array_Unnamed( unsigned char, self->netSrcs * array->itemSize );
-	else
-		srcArray = NULL;
-
-	/* Transfer. */
-	MPI_Alltoallv( snkArray, (int*)array->snkSizes, (int*)array->snkDisps, MPI_BYTE, 
-		       srcArray, (int*)array->srcSizes, (int*)array->srcDisps, MPI_BYTE, 
-		       self->comm );
-
-	/* Free the sink array. */
-	FreeArray( snkArray );
-
-	/* Unpack sources. */
-	if( self->netSnks > 0 ) {
-		unsigned	src_i;
-
-		for( src_i = 0; src_i < self->netSrcs; src_i++ ) {
-			memcpy( (unsigned char*)array->srcArray + array->srcOffs[src_i], 
-				srcArray + src_i * array->itemSize, 
-				array->itemSize );
-		}
-	}
-
-	/* Free source array. */
-	FreeArray( srcArray );
+	map = UIntMap_New();
+	for( l_i = 0; l_i < self->nLocals; l_i++ )
+		UIntMap_Insert( map, self->locals[l_i], l_i );
+	self->glMap = map;
 }
 
-void Decomp_BuildAllArrays( Decomp* self ) {
-	unsigned	a_i;
+void Decomp_ValidateDomain( Decomp* self, unsigned* status ) {
+	unsigned	rank, nProcs;
+	unsigned	nBytes;
+	Byte*		bytes;
+	unsigned	tag = 6669;
+	RangeSet*	lSet;
+	RangeSet*	gSet;
+	unsigned	netInds;
 
-	for( a_i = 0; a_i < self->nArrays; a_i++ )
-		Decomp_BuildArray( self, self->arrays + a_i );
-}
+	assert( self );
 
-void Decomp_BuildArray( Decomp* self, Decomp_Array* array ) {
-	/* Determine sink (local) information. */
-	if( self->netSnks > 0 ) {
-		unsigned*	snkOffs;
-		unsigned*	snkSizes;
-		unsigned*	snkDisps;
-		unsigned	snkInd = 0;
-		unsigned	p_i;
+	/* Clear the status. */
+	*status = DECOMP_OK;
 
-		/* Allocate/reallocate memory. */
-		snkDisps = Memory_Alloc_Array( unsigned, self->nProcs, "Decomp_Array::snkDisps" );
-		snkSizes = Memory_Alloc_Array( unsigned, self->nProcs, "Decomp_Array::snkSizes" );
-		snkOffs = Memory_Alloc_Array( unsigned, self->netSnks, "Decomp_Array::snkOffs" );
+	/* Get basic MPI info. */
+	MPI_Comm_rank( self->comm, (int*)&rank );
+	MPI_Comm_size( self->comm, (int*)&nProcs );
 
-		/* Calculate offsets and sizes. */
-		for( p_i = 0; p_i < self->nProcs; p_i++ ) {
-			unsigned	snk_i;
+	/* Create a local range set. */
+	lSet = RangeSet_New();
+	RangeSet_SetIndices( lSet, self->nLocals, self->locals );
 
-			snkSizes[p_i] = 0;
-			for( snk_i = 0; snk_i < self->nSnks[p_i]; snk_i++ ) {
-				unsigned	dInd = Decomp_GlobalToDomain( self, self->snks[p_i][snk_i] );
+	/* Create a global range set. */
+	gSet = RangeSet_New();
 
-				snkOffs[snkInd] = dInd * array->snkStride;
-				snkSizes[p_i] += array->itemSize;
-				snkInd++;
-			}
-		}
+	/* Receive from previous rank. */
+	if( rank > 0 ) {
+		MPI_Status	mpiStatus;
 
-		/* Calculate the displacements. */
-		snkDisps[0] = 0;
-		for( p_i = 1; p_i < self->nProcs; p_i++ ) {
-			snkDisps[p_i] = snkDisps[p_i - 1] + snkSizes[p_i - 1];
-		}
-
-		/* Store arrays. */
-		array->snkOffs = snkOffs;
-		array->snkDisps = snkDisps;
-		array->snkSizes = snkSizes;
+		MPI_Recv( &nBytes, 1, MPI_UNSIGNED, rank - 1, tag, self->comm, &mpiStatus );
+		bytes = Memory_Alloc_Array_Unnamed( Byte, nBytes );
+		MPI_Recv( bytes, nBytes, MPI_BYTE, rank - 1, tag, self->comm, &mpiStatus );
+		RangeSet_Unpickle( gSet, nBytes, bytes );
+		FreeArray( bytes );
 	}
-	else {
-		/* Store null information. */
-		array->snkOffs = Memory_Alloc_Array( unsigned, self->nProcs, "Decomp_Array::snkOffs" );
-		array->snkDisps = Memory_Alloc_Array( unsigned, self->nProcs, "Decomp_Array::snkDisps" );
-		array->snkSizes = Memory_Alloc_Array( unsigned, self->nProcs, "Decomp_Array::snkSizes" );
-		memset( array->snkOffs, 0, sizeof(unsigned) * self->nProcs );
-		memset( array->snkDisps, 0, sizeof(unsigned) * self->nProcs );
-		memset( array->snkSizes, 0, sizeof(unsigned) * self->nProcs );
-	}
 
-	/* Determine source (shadow) information. */
-	if( self->netSrcs > 0 ) {
-		unsigned*	srcOffs;
-		unsigned*	srcSizes;
-		unsigned*	srcDisps;
-		unsigned	srcInd = 0;
-		unsigned	p_i;
+	/* Combine sets. */
+	RangeSet_Union( gSet, lSet );
 
-		/* Allocate/reallocate memory. */
-		srcDisps = Memory_Alloc_Array( unsigned, self->nProcs, "Decomp_Array::srcDisps" );
-		srcSizes = Memory_Alloc_Array( unsigned, self->nProcs, "Decomp_Array::srcSizes" );
-		srcOffs = Memory_Alloc_Array( unsigned, self->netSrcs, "Decomp_Array::srcOffs" );
-
-		/* Calculate offsets and sizes. */
-		for( p_i = 0; p_i < self->nProcs; p_i++ ) {
-			unsigned	src_i;
-
-			srcSizes[p_i] = 0;
-			for( src_i = 0; src_i < self->nSrcs[p_i]; src_i++ ) {
-				unsigned	sInd = Decomp_GlobalToDomain( self, self->srcs[p_i][src_i] );
-
-				assert( sInd >= self->nLocals );
-				sInd -= self->nLocals;
-				srcOffs[srcInd] = sInd * array->srcStride;
-				srcSizes[p_i] += array->itemSize;
-				srcInd++;
-			}
-		}
-
-		/* Calculate the displacements. */
-		srcDisps[0] = 0;
-		for( p_i = 1; p_i < self->nProcs; p_i++ ) {
-			srcDisps[p_i] = srcDisps[p_i - 1] + srcSizes[p_i - 1];
-		}
-
-		/* Store arrays. */
-		array->srcOffs = srcOffs;
-		array->srcDisps = srcDisps;
-		array->srcSizes = srcSizes;
+	if( rank < nProcs - 1 ) {
+		RangeSet_Pickle( gSet, &nBytes, &bytes );
+		MPI_Send( &nBytes, 1, MPI_UNSIGNED, rank + 1, tag, self->comm );
+		MPI_Send( bytes, nBytes, MPI_BYTE, rank + 1, tag, self->comm );
+		FreeArray( bytes );
 	}
 	else {
-		/* Store null information. */
-		array->srcOffs = Memory_Alloc_Array( unsigned, self->nProcs, "Decomp_Array::srcOffs" );
-		array->srcDisps = Memory_Alloc_Array( unsigned, self->nProcs, "Decomp_Array::srcDisps" );
-		array->srcSizes = Memory_Alloc_Array( unsigned, self->nProcs, "Decomp_Array::srcSizes" );
-		memset( array->srcOffs, 0, sizeof(unsigned) * self->nProcs );
-		memset( array->srcDisps, 0, sizeof(unsigned) * self->nProcs );
-		memset( array->srcSizes, 0, sizeof(unsigned) * self->nProcs );
+		self->nGlobals = RangeSet_GetNIndices( gSet );
+		if( RangeSet_GetNRanges( gSet ) != 1 )
+			*status |= DECOMP_INCOMPLETE;
 	}
-}
 
-void Decomp_RemoteSearch( Decomp* self, unsigned** nRemFound, unsigned*** remFound ) {
-	unsigned*	remNReq;
-	unsigned**	remReq;
-	unsigned*	nFound;
-	unsigned**	found;
-	unsigned	p_i;
+	/* Transfer global count to all. */
+	MPI_Bcast( &self->nGlobals, 1, MPI_UNSIGNED, nProcs - 1, self->comm );
 
-	assert( self );
-	assert( nRemFound );
-	assert( remFound );
+	/* Check for overlap. */
+	MPI_Allreduce( &lSet->nInds, &netInds, 1, MPI_UNSIGNED, MPI_SUM, self->comm );
+	if( netInds != self->nGlobals )
+		*status |= DECOMP_OVERLAPPING;
 
-	/* Send our requireds to all others and receive from all others. */
-	MPIArray_Allgather( self->nShadows, self->shadows, 
-			    &remNReq, (void***)&remReq, 
-			    sizeof(unsigned), self->comm );
-
-	/* Prepare to locate remote requireds. */
-	nFound = Memory_Alloc_Array( unsigned, self->nProcs, "" );
-	found = Memory_Alloc_2DComplex( unsigned, self->nProcs, remNReq, "" );
-
-	/* Begin searching... */
-	for( p_i = 0; p_i < self->nProcs; p_i++ ) {
-		unsigned	rem_i;
-
-		/* Initialise found count to zero. */
-		nFound[p_i] = 0;
-
-		/* Skip thyself. */
-		if( p_i == self->rank ) continue;
-
-		/* Do we have any for this processor? */
-		for( rem_i = 0; rem_i < remNReq[p_i]; rem_i++ ) {
-			unsigned	l_i;
-
-			for( l_i = 0; l_i < self->nLocals; l_i++ ) {
-				if( self->locals[l_i] == remReq[p_i][rem_i] ) {
-					found[p_i][nFound[p_i]++] = remReq[p_i][rem_i];
-					break;
-				}
-			}
-		}
-	}
-
-	/* Release some memory. */
-	FreeArray( remNReq );
-	FreeArray2D( self->nProcs, remReq );
-
-	/* Send back all the ones we found and receive from all others all our requireds they found. */
-	MPIArray2D_Alltoall( nFound, (void**)found, 
-			     nRemFound, (void***)remFound, 
-			     sizeof(unsigned), self->comm );
-
-	/* Release some memory. */
-	FreeArray( nFound );
-	FreeArray( found );
+	/* Free the sets. */
+	FreeObject( lSet );
+	FreeObject( gSet );
 }
 
-void Decomp_BuildGDMap( Decomp* self ) {
-	UIntMap*	map;
-	unsigned	d_i;
-
-	assert( self );
-
-	FreeObject( self->gdMap );
-
-	map = UIntMap_New();
-	for( d_i = 0; d_i < self->nDomains; d_i++ )
-		UIntMap_Insert( map, self->domains[d_i], d_i );
-	self->gdMap = map;
+void Decomp_Destruct( Decomp* self ) {
+	Decomp_DestructLocals( self );
 }
 
-void Decomp_Destruct( Decomp* self ) {
-	Decomp_DestructNegotiated( self );
-	KillArray( self->domains );
-	self->domains = NULL;
-	self->locals = NULL;
-	self->leased = NULL;
-	self->remotes = NULL;
-	self->shadows = NULL;
+void Decomp_DestructLocals( Decomp* self ) {
+	Decomp_DestructSyncs( self );
 	self->nLocals = 0;
-	self->nLeased = 0;
-	self->nRemotes = 0;
-	self->nShadows = 0;
-	KillArray( self->rentals );
-	self->rentals = 0;
+	KillArray( self->locals );
+	KillObject( self->glMap );
 }
 
-void Decomp_DestructNegotiated( Decomp* self ) {
-	Decomp_DestructArrays( self );
-	KillObject( self->gdMap );
-	self->netSrcs = 0;
-	KillArray( self->nSrcs );
-	KillArray2D( self->nProcs, self->srcs );
-	self->netSnks = 0;
-	KillArray( self->nSnks );
-	KillArray2D( self->nProcs, self->snks );
-}
+void Decomp_DestructSyncs( Decomp* self ) {
+	unsigned	s_i;
+	unsigned	tmpNSyncs;
+	Decomp_Sync**	tmpSyncs;
 
-void Decomp_DestructArrays( Decomp* self ) {
-	unsigned	a_i;
-
-	for( a_i = 0; a_i < self->nArrays; a_i++ ) {
-		FreeArray( self->arrays[a_i].snkDisps );
-		FreeArray( self->arrays[a_i].snkSizes );
-		FreeArray( self->arrays[a_i].snkOffs );
-		FreeArray( self->arrays[a_i].srcDisps );
-		FreeArray( self->arrays[a_i].srcSizes );
-		FreeArray( self->arrays[a_i].srcOffs );
-	}
-	KillArray( self->arrays );
-	self->nArrays = 0;
+	tmpNSyncs = self->nSyncs;
+	tmpSyncs = Memory_Alloc_Array_Unnamed( Decomp_Sync*, tmpNSyncs );
+	memcpy( tmpSyncs, self->syncs, tmpNSyncs * sizeof(Decomp_Sync*) );
+	for( s_i = 0; s_i < tmpNSyncs; s_i++ )
+		Decomp_Sync_SetDecomp( tmpSyncs, NULL );
 }

Modified: long/3D/Gale/trunk/src/StGermain/Discretisation/Mesh/src/Decomp.h
===================================================================
--- long/3D/Gale/trunk/src/StGermain/Discretisation/Mesh/src/Decomp.h	2006-10-11 20:46:07 UTC (rev 4823)
+++ long/3D/Gale/trunk/src/StGermain/Discretisation/Mesh/src/Decomp.h	2006-10-11 20:46:09 UTC (rev 4824)
@@ -45,68 +45,25 @@
 	extern const Type Decomp_Type;
 
 	/** Virtual function types */
-	typedef void (Decomp_SelectFunc)( void* decomp, unsigned* nFound, unsigned** found );
 
-	/** Mesh class contents */
-	typedef struct {
-		unsigned	localInd;
-		unsigned	nProcs;
-		unsigned*	procs;
-	} Decomp_Rental;
-
-	typedef struct {
-		void*		snkArray;
-		size_t		snkStride;
-		unsigned*	snkDisps;
-		unsigned*	snkSizes;
-		unsigned*	snkOffs;
-
-		void*		srcArray;
-		size_t		srcStride;
-		unsigned*	srcDisps;
-		unsigned*	srcSizes;
-		unsigned*	srcOffs;
-
-		size_t		itemSize;
-	} Decomp_Array;
-
+	/** Class contents */
 	#define __Decomp				\
 		/* General info */			\
 		__Stg_Component				\
 							\
 		/* Virtual info */			\
-		Decomp_SelectFunc*	selectFunc;	\
 							\
 		/* Decomp info */			\
 		MPI_Comm		comm;		\
-		unsigned		nProcs;		\
-		unsigned		rank; 		\
 							\
 		unsigned		nGlobals;	\
 		unsigned		nLocals;	\
 		unsigned*		locals;		\
-		unsigned		nLeased;	\
-		unsigned*		leased;		\
-		unsigned		nRemotes;	\
-		unsigned*		remotes;	\
-		unsigned		nShadows;	\
-		unsigned*		shadows;	\
-		unsigned		nDomains;	\
-		unsigned*		domains;	\
-		unsigned		nRentals;	\
-		Decomp_Rental*		rentals;	\
 							\
-		UIntMap*		gdMap;		\
+		UIntMap*		glMap;		\
 							\
-		unsigned		netSrcs;	\
-		unsigned*		nSrcs;		\
-		unsigned**		srcs;		\
-		unsigned		netSnks;	\
-		unsigned*		nSnks;		\
-		unsigned**		snks;		\
-							\
-		unsigned		nArrays;	\
-		Decomp_Array*		arrays;
+		unsigned		nSyncs;		\
+		Decomp_Sync**		syncs;
 
 	struct Decomp { __Decomp };
 
@@ -114,12 +71,11 @@
 	** Constructors
 	*/
 
-	#define DECOMP_DEFARGS				\
-		STG_COMPONENT_DEFARGS,			\
-		Decomp_SelectFunc*	selectFunc
+	#define DECOMP_DEFARGS		\
+		STG_COMPONENT_DEFARGS
 
-	#define DECOMP_PASSARGS	\
-		STG_COMPONENT_PASSARGS, selectFunc
+	#define DECOMP_PASSARGS		\
+		STG_COMPONENT_PASSARGS
 
 	Decomp* Decomp_New( Name name );
 	Decomp* _Decomp_New( DECOMP_DEFARGS );
@@ -129,51 +85,42 @@
 	** Virtual functions
 	*/
 
-	void _Decomp_Delete( void* generator );
-	void _Decomp_Print( void* generator, Stream* stream );
+	void _Decomp_Delete( void* decomp );
+	void _Decomp_Print( void* decomp, Stream* stream );
 
 	#define Decomp_Copy( self ) \
 		(Mesh*)Stg_Class_Copy( self, NULL, False, NULL, NULL )
 	#define Decomp_DeepCopy( self ) \
 		(Mesh*)Stg_Class_Copy( self, NULL, True, NULL, NULL )
-	void* _Decomp_Copy( void* generator, void* dest, Bool deep, Name nameExt, PtrMap* ptrMap );
+	void* _Decomp_Copy( void* decomp, void* dest, Bool deep, Name nameExt, PtrMap* ptrMap );
 
-	void _Decomp_Construct( void* generator, Stg_ComponentFactory* cf );
-	void _Decomp_Build( void* generator, void* data );
-	void _Decomp_Initialise( void* generator, void* data );
-	void _Decomp_Execute( void* generator, void* data );
-	void _Decomp_Destroy( void* generator, void* data );
+	void _Decomp_Construct( void* decomp, Stg_ComponentFactory* cf );
+	void _Decomp_Build( void* decomp, void* data );
+	void _Decomp_Initialise( void* decomp, void* data );
+	void _Decomp_Execute( void* decomp, void* data );
+	void _Decomp_Destroy( void* decomp, void* data );
 
-	void Decomp_Select( void* decomp, unsigned* nFount, unsigned** found );
-
 	/*--------------------------------------------------------------------------------------------------------------------------
 	** Public functions
 	*/
 
 	void Decomp_SetComm( void* decomp, MPI_Comm comm );
-	void Decomp_SetNGlobals( void* decomp, unsigned nGlobals );
-	void Decomp_SetLocals( void* decomp, unsigned nLocals, unsigned* locals, 
-			       unsigned nLeased, unsigned* leased );
-	void Decomp_SetRemotes( void* decomp, unsigned nRemotes, unsigned* remotes );
-	void Decomp_Negotiate( void* decomp );
-	Bool Decomp_IsDomain( void* decomp, unsigned global );
-	unsigned Decomp_GlobalToDomain( void* decomp, unsigned global );
-	unsigned Decomp_DomainToGlobal( void* decomp, unsigned domain );
-	void Decomp_AddArray( void* decomp, void* localArray, void* shadowArray, 
-			      size_t localStride, size_t shadowStride, size_t itemSize );
-	void Decomp_Sync( void* decomp );
+	void Decomp_SetLocals( void* decomp, unsigned nLocals, unsigned* locals );
+	void Decomp_AddSync( void* decomp, Decomp_Sync* sync );
+	void Decomp_RemoveSync( void* decomp, Decomp_Sync* sync );
 
+	Bool Decomp_IsLocal( void* decomp, unsigned global );
+	unsigned Decomp_GlobalToLocal( void* decomp, unsigned global );
+	unsigned Decomp_LocalToGlobal( void* decomp, unsigned local );
+
 	/*--------------------------------------------------------------------------------------------------------------------------
 	** Private Member functions
 	*/
 
-	void Decomp_SyncArray( Decomp* self, Decomp_Array* array );
-	void Decomp_BuildAllArrays( Decomp* self );
-	void Decomp_BuildArray( Decomp* self, Decomp_Array* array );
-	void Decomp_RemoteSearch( Decomp* self, unsigned** nRemFound, unsigned*** remFound );
-	void Decomp_BuildGDMap( Decomp* self );
+	void Decomp_BuildGLMap( Decomp* self );
+	void Decomp_ValidateDomain( Decomp* self, unsigned* status );
 	void Decomp_Destruct( Decomp* self );
-	void Decomp_DestructNegotiated( Decomp* self );
-	void Decomp_DestructArrays( Decomp* self );
+	void Decomp_DestructLocals( Decomp* self );
+	void Decomp_DestructSyncs( Decomp* self );
 
 #endif /* __Discretisaton_Mesh_Decomp_h__ */

Added: long/3D/Gale/trunk/src/StGermain/Discretisation/Mesh/src/Decomp_Sync.c
===================================================================
--- long/3D/Gale/trunk/src/StGermain/Discretisation/Mesh/src/Decomp_Sync.c	2006-10-11 20:46:07 UTC (rev 4823)
+++ long/3D/Gale/trunk/src/StGermain/Discretisation/Mesh/src/Decomp_Sync.c	2006-10-11 20:46:09 UTC (rev 4824)
@@ -0,0 +1,911 @@
+/*~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+**
+** Copyright (C), 2003, Victorian Partnership for Advanced Computing (VPAC) Ltd, 110 Victoria Street, Melbourne, 3053, Australia.
+**
+** Authors:
+**	Stevan M. Quenette, Senior Software Engineer, VPAC. (steve at vpac.org)
+**	Patrick D. Sunter, Software Engineer, VPAC. (pds at vpac.org)
+**	Luke J. Hodkinson, Computational Engineer, VPAC. (lhodkins at vpac.org)
+**	Siew-Ching Tan, Software Engineer, VPAC. (siew at vpac.org)
+**	Alan H. Lo, Computational Engineer, VPAC. (alan at vpac.org)
+**	Raquibul Hassan, Computational Engineer, VPAC. (raq at vpac.org)
+**
+**  This library is free software; you can redistribute it and/or
+**  modify it under the terms of the GNU Lesser General Public
+**  License as published by the Free Software Foundation; either
+**  version 2.1 of the License, or (at your option) any later version.
+**
+**  This library is distributed in the hope that it will be useful,
+**  but WITHOUT ANY WARRANTY; without even the implied warranty of
+**  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+**  Lesser General Public License for more details.
+**
+**  You should have received a copy of the GNU Lesser General Public
+**  License along with this library; if not, write to the Free Software
+**  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+**
+** $Id: Decomp_Sync.c 3584 2006-05-16 11:11:07Z PatrickSunter $
+**
+**~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~*/
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <assert.h>
+#include <mpi.h>
+
+#include "Base/Base.h"
+
+#include "types.h"
+#include "shortcuts.h"
+#include "CommTopology.h"
+#include "Decomp.h"
+#include "Decomp_Sync_Claim.h"
+#include "Decomp_Sync_Negotiate.h"
+#include "Decomp_Sync.h"
+
+
+/* Textual name of this class */
+const Type Decomp_Sync_Type = "Decomp_Sync";
+
+/*----------------------------------------------------------------------------------------------------------------------------------
+** Constructors
+*/
+
+Decomp_Sync* Decomp_Sync_New( Name name ) {
+	return _Decomp_Sync_New( sizeof(Decomp_Sync), 
+				 Decomp_Sync_Type, 
+				 _Decomp_Sync_Delete, 
+				 _Decomp_Sync_Print, 
+				 _Decomp_Sync_Copy, 
+				 (void* (*)(Name))_Decomp_Sync_New, 
+				 _Decomp_Sync_Construct, 
+				 _Decomp_Sync_Build, 
+				 _Decomp_Sync_Initialise, 
+				 _Decomp_Sync_Execute, 
+				 _Decomp_Sync_Destroy, 
+				 name, 
+				 NON_GLOBAL );
+}
+
+Decomp_Sync* _Decomp_Sync_New( DECOMP_SYNC_DEFARGS ) {
+	Decomp_Sync* self;
+	
+	/* Allocate memory */
+	assert( sizeOfSelf >= sizeof(Decomp_Sync) );
+	self = (Decomp_Sync*)_Stg_Component_New( STG_COMPONENT_PASSARGS );
+
+	/* Virtual info */
+
+	/* Decomp_Sync info */
+	_Decomp_Sync_Init( self );
+
+	return self;
+}
+
+void _Decomp_Sync_Init( Decomp_Sync* self ) {
+	self->decomp = NULL;
+	self->commTopo = NULL;
+
+	self->claim = Decomp_Sync_Claim_New( "" );
+	self->negotiate = Decomp_Sync_Negotiate_New( "" );
+
+	self->nRemotes = 0;
+	self->remotes = NULL;
+	self->nShared = 0;
+	self->shared = NULL;
+
+	self->netSrcs = 0;
+	self->nSrcs = NULL;
+	self->srcs = NULL;
+	self->netSnks = 0;
+	self->nSnks = NULL;
+	self->snks = NULL;
+
+	self->nArrays = 0;
+	self->arrays = NULL;
+}
+
+
+/*----------------------------------------------------------------------------------------------------------------------------------
+** Virtual functions
+*/
+
+void _Decomp_Sync_Delete( void* sync ) {
+	Decomp_Sync*	self = (Decomp_Sync*)sync;
+
+	Decomp_Sync_Destruct( self );
+	FreeObject( self->claim );
+	FreeObject( self->negotiate );
+
+	/* Delete the parent. */
+	_Stg_Component_Delete( self );
+}
+
+void _Decomp_Sync_Print( void* sync, Stream* stream ) {
+	Decomp_Sync*	self = (Decomp_Sync*)sync;
+	
+	/* Set the Journal for printing informations */
+	Stream* syncStream;
+	syncStream = Journal_Register( InfoStream_Type, "Decomp_SyncStream" );
+
+	/* Print parent */
+	Journal_Printf( stream, "Decomp_Sync (ptr): (%p)\n", self );
+	_Stg_Component_Print( self, stream );
+}
+
+void* _Decomp_Sync_Copy( void* sync, void* destProc_I, Bool deep, Name nameExt, PtrMap* ptrMap ) {
+#if 0
+	Decomp_Sync*	self = (Decomp_Sync*)sync;
+	Decomp_Sync*	newDecomp_Sync;
+	PtrMap*	map = ptrMap;
+	Bool	ownMap = False;
+
+	/* Damn me for making copying so difficult... what was I thinking? */
+	
+	/* We need to create a map if it doesn't already exist. */
+	if( !map ) {
+		map = PtrMap_New( 10 );
+		ownMap = True;
+	}
+	
+	newDecomp_Sync = (Decomp_Sync*)_Mesh_Copy( self, destProc_I, deep, nameExt, map );
+	
+	/* Copy the virtual methods here. */
+
+	/* Deep or shallow? */
+	if( deep ) {
+	}
+	else {
+	}
+	
+	/* If we own the map, get rid of it here. */
+	if( ownMap ) Stg_Class_Delete( map );
+	
+	return (void*)newDecomp_Sync;
+#endif
+
+	return NULL;
+}
+
+void _Decomp_Sync_Construct( void* sync, Stg_ComponentFactory* cf ) {
+}
+
+void _Decomp_Sync_Build( void* sync, void* data ) {
+}
+
+void _Decomp_Sync_Initialise( void* sync, void* data ) {
+}
+
+void _Decomp_Sync_Execute( void* sync, void* data ) {
+}
+
+void _Decomp_Sync_Destroy( void* sync, void* data ) {
+}
+
+
+/*--------------------------------------------------------------------------------------------------------------------------
+** Public Functions
+*/
+
+void Decomp_Sync_SetDecomp( void* sync, Decomp* decomp ) {
+	Decomp_Sync*	self = (Decomp_Sync*)sync;
+
+	assert( self );
+
+	Decomp_Sync_Destruct( self );
+
+	self->decomp = decomp;
+	if( decomp )
+		Decomp_AddSync( decomp, self );
+}
+
+void Decomp_Sync_SetClaim( void* sync, Decomp_Sync_Claim* claim ) {
+	Decomp_Sync*	self = (Decomp_Sync*)sync;
+
+	assert( self );
+
+	Decomp_Sync_DestructRemotes( self );
+	FreeObject( self->claim );
+
+	if( claim )
+		self->claim = claim;
+	else
+		self->claim = Decomp_Sync_Claim_New( "" );
+}
+
+void Decomp_Sync_SetNegotiate( void* sync, Decomp_Sync_Negotiate* negotiate ) {
+	Decomp_Sync*	self = (Decomp_Sync*)sync;
+
+	assert( self );
+
+	Decomp_Sync_DestructRemotes( self );
+	FreeObject( self->negotiate );
+
+	if( negotiate )
+		self->negotiate = negotiate;
+	else
+		self->negotiate = Decomp_Sync_Negotiate_New( "" );
+}
+
+void Decomp_Sync_SetRemotes( void* sync, unsigned nRemotes, unsigned* remotes ) {
+	Decomp_Sync*	self = (Decomp_Sync*)sync;
+	RangeSet*	lSet;
+	RangeSet*	rSet;
+	unsigned	nInds;
+	unsigned*	inds;
+
+	assert( self );
+	assert( self->decomp );
+	assert( !nRemotes || remotes );
+#ifndef NDEBUG
+	{
+		lSet = RangeSet_New();
+		rSet = RangeSet_New();
+		RangeSet_SetIndices( lSet, self->decomp->nLocals, self->decomp->locals );
+		RangeSet_SetIndices( rSet, nRemotes, remotes );
+		RangeSet_Intersection( rSet, lSet );
+		assert( !RangeSet_GetNIndices( rSet ) );
+		FreeObject( lSet );
+		FreeObject( rSet );
+	}
+#endif
+
+	Decomp_Sync_DestructRemotes( self );
+
+	/* Store remote values. */
+	self->nRemotes = nRemotes;
+	if( nRemotes ) {
+		self->remotes = Memory_Alloc_Array( unsigned, nRemotes, "Decomp_Sync::remotes" );
+		memcpy( self->remotes, remotes, nRemotes * sizeof(unsigned) );
+	}
+	else
+		self->remotes = NULL;
+
+	/* Build a communication topology and intersections. */
+	lSet = RangeSet_New();
+	rSet = RangeSet_New();
+	RangeSet_SetIndices( lSet, self->decomp->nLocals, self->decomp->locals );
+	RangeSet_SetIndices( rSet, nRemotes, remotes );
+	RangeSet_Union( lSet, rSet );
+	FreeObject( rSet );
+	RangeSet_GetIndices( lSet, &nInds, &inds );
+	FreeObject( lSet );
+	Decomp_Sync_BuildIntersections( self, nInds, inds );
+	FreeArray( inds );
+
+	/* Negotiate sources and sinks. */
+	Decomp_Sync_Negotiate_Select( self->negotiate, self );
+
+	/* Build the global to remote map. */
+	Decomp_Sync_BuildGRMap( self );
+
+	/* Build shared information. */
+	Decomp_Sync_BuildShared( self );
+}
+
+void Decomp_Sync_Decompose( void* sync, unsigned nRequired, unsigned* required ) {
+	Decomp_Sync*	self = (Decomp_Sync*)sync;
+	Decomp*		decomp;
+	unsigned	nLocals;
+	unsigned*	locals;
+
+	assert( self );
+	assert( self->decomp );
+	assert( !nRequired || required );
+
+	/* Destroy self. */
+	decomp = self->decomp;
+	Decomp_Sync_Destruct( self );
+	self->decomp = decomp;
+
+	/* Build index intersections and communication topology. */
+	Decomp_Sync_BuildIntersections( self, nRequired, required );
+
+	/* Claim ownership. */
+	Decomp_Sync_Claim_Select( self->claim, self, nRequired, required, 
+				  &nLocals, &locals );
+
+	/* Reset the decomposition. */
+	Decomp_SetLocals( decomp, nLocals, locals );
+	self->decomp = decomp;
+	Decomp_AddSync( decomp, self );
+	Decomp_Sync_Negotiate_Select( self->negotiate, self );
+	Decomp_Sync_BuildGRMap( self );
+	Decomp_Sync_BuildShared( self );
+}
+
+Bool Decomp_Sync_IsDomain( void* sync, unsigned global ) {
+	Decomp_Sync*	self = (Decomp_Sync*)sync;
+
+	assert( self );
+	assert( self->decomp );
+	assert( global < self->decomp->nGlobals );
+	assert( self->decomp->glMap );
+	assert( self->grMap );
+
+	return Decomp_IsLocal( self->decomp, global ) || UIntMap_HasKey( self->grMap, global );
+}
+
+Bool Decomp_Sync_IsRemote( void* sync, unsigned domain ) {
+	Decomp_Sync*	self = (Decomp_Sync*)sync;
+
+	assert( self );
+	assert( self->decomp );
+	assert( domain < self->decomp->nLocals + self->nRemotes );
+
+	return domain >= self->decomp->nLocals;
+}
+
+unsigned Decomp_Sync_GlobalToDomain( void* sync, unsigned global ) {
+	Decomp_Sync*	self = (Decomp_Sync*)sync;
+
+	assert( self );
+	assert( self->decomp );
+	assert( global < self->decomp->nGlobals );
+	assert( self->decomp->glMap );
+	assert( self->grMap );
+
+	if( Decomp_IsLocal( self->decomp, global ) )
+		return Decomp_GlobalToLocal( self->decomp, global );
+	else
+		return self->decomp->nLocals + UIntMap_Map( self->grMap, global );
+}
+
+unsigned Decomp_Sync_DomainToGlobal( void* sync, unsigned domain ) {
+	Decomp_Sync*	self = (Decomp_Sync*)sync;
+
+	assert( self );
+	assert( self->decomp );
+	assert( domain < self->decomp->nLocals + self->nRemotes );
+
+	if( domain < self->decomp->nLocals )
+		return Decomp_LocalToGlobal( self->decomp, domain );
+	else
+		return self->remotes[domain - self->decomp->nLocals];
+}
+
+unsigned Decomp_Sync_DomainToShared( void* sync, unsigned domain ) {
+	Decomp_Sync*	self = (Decomp_Sync*)sync;
+
+	assert( self );
+	assert( self->decomp );
+	assert( domain < self->decomp->nLocals + self->nRemotes );
+	assert( self->dsMap );
+
+	return UIntMap_Map( self->dsMap, domain );
+}
+
+unsigned Decomp_Sync_SharedToDomain( void* sync, unsigned shared ) {
+	Decomp_Sync*	self = (Decomp_Sync*)sync;
+
+	assert( self );
+	assert( shared < self->nShared );
+	assert( self->shared );
+
+	return self->shared[shared];
+}
+
+void Decomp_Sync_AddArray( void* sync, void* localArray, void* remoteArray, 
+			   size_t localStride, size_t remoteStride, size_t itemSize )
+{
+	Decomp_Sync*		self = (Decomp_Sync*)sync;
+	Decomp_Sync_Array*	array;
+
+	/* Sanity checks. */
+	assert( self );
+	assert( self->decomp );
+	assert( !self->decomp->nLocals || localArray );
+	assert( !self->nRemotes || remoteArray );
+	assert( itemSize );
+
+	/* Resize the array array (?). */
+	if( self->nArrays ) {
+		self->arrays = Memory_Realloc_Array( self->arrays, Decomp_Sync_Array, ++self->nArrays );
+	}
+	else {
+		self->arrays = Memory_Alloc_Array( Decomp_Sync_Array, ++self->nArrays, "Decomp_Sync::Arrays" );
+	}
+	array = self->arrays + self->nArrays - 1;
+
+	/* Store information. */
+	array->snkArray = localArray;
+	array->snkStride = localStride;
+	array->srcArray = remoteArray;
+	array->srcStride = remoteStride;
+	array->itemSize = itemSize;
+
+	/* Build this array. */
+	Decomp_Sync_BuildArray( self, array );
+}
+
+void Decomp_Sync_Sync( void* sync ) {
+	Decomp_Sync*	self = (Decomp_Sync*)sync;
+	unsigned	a_i;
+
+	/* Sanity checks. */
+	assert( self );
+	assert( !self->nArrays || self->arrays );
+
+	for( a_i = 0; a_i < self->nArrays; a_i++ )
+		Decomp_Sync_SyncArray( self, self->arrays + a_i );
+}
+
+
+/*----------------------------------------------------------------------------------------------------------------------------------
+** Private Functions
+*/
+
+void Decomp_Sync_BuildIntersections( Decomp_Sync* self, unsigned nIndices, unsigned* indices ) {
+	unsigned	rank, nProcs;
+	RangeSet*	lSet;
+	MPI_Group	worldGroup;
+	unsigned*	worldRanks;
+	unsigned*	subRanks;
+	unsigned	nCommInc;
+	unsigned*	commInc;
+	RangeSet**	iSets;
+	unsigned	nInc;
+	unsigned*	inc;
+	unsigned	p_i;
+
+	assert( self );
+	assert( !nIndices || indices );
+
+	/* Get basic MPI info. */
+	MPI_Comm_rank( self->decomp->comm, (int*)&rank );
+	MPI_Comm_size( self->decomp->comm, (int*)&nProcs );
+
+	/* We'll need to modify the world group. */
+	MPI_Comm_group( self->decomp->comm, &worldGroup );
+	worldRanks = Memory_Alloc_Array_Unnamed( unsigned, nProcs );
+	subRanks = Memory_Alloc_Array_Unnamed( unsigned, nProcs );
+
+	/* We need space to store index intersections. */
+	iSets = Memory_Alloc_Array_Unnamed( RangeSet*, nProcs );
+	memset( iSets, 0, nProcs * sizeof(RangeSet*) );
+
+	/* Create a local set of required indices. */
+	lSet = RangeSet_New();
+	RangeSet_SetIndices( lSet, nIndices, indices );
+
+	/* Tackle each processor one at a time. */
+	for( p_i = 0; p_i < nProcs - 1; p_i++ ) {
+		int		groupRange[3];
+		MPI_Group	subGroup;
+		MPI_Comm	subComm;
+		unsigned	p_j;
+
+		/* Set the processor range. */
+		groupRange[0] = p_i;
+		groupRange[1] = nProcs - 1;
+		groupRange[2] = 1;
+
+		/* We'll need a new group, as we only want to communicate using a triangular scheme. */
+		MPI_Group_range_incl( worldGroup, 1, &groupRange, &subGroup );
+		MPI_Comm_create( self->decomp->comm, subGroup, &subComm );
+
+		/* Only continue if we're part of the sub-communicator. */
+		if( rank >= p_i ) {
+			unsigned	nBytes;
+			Byte*		bytes;
+			unsigned*	nFounds;
+			Byte**		founds;
+
+			/* Create a mapping between ranks. */
+			for( p_j = p_i; p_j < nProcs; p_j++ )
+				subRanks[p_j] = p_j - p_i;
+			MPI_Group_translate_ranks( subGroup, nProcs - p_i, (int*)(subRanks + p_i), 
+						   worldGroup, (int*)worldRanks );
+
+			if( p_i == rank )
+				RangeSet_Pickle( lSet, &nBytes, &bytes );
+
+			MPIArray_Bcast( &nBytes, (void**)&bytes, sizeof(Byte), subRanks[p_i], subComm );
+
+			if( p_i != rank ) {
+				/* Create the intersection. */
+				iSets[p_i] = RangeSet_New();
+				RangeSet_Unpickle( iSets[p_i], nBytes, bytes );
+				RangeSet_Intersection( iSets[p_i], lSet );
+
+				/* Pickle the intersection to send back. */
+				FreeArray( bytes );
+				RangeSet_Pickle( iSets[p_i], &nBytes, &bytes );
+			}
+			else {
+				KillArray( bytes );
+				nBytes = 0;
+			}
+
+			/* Retrieve the results and unpickle each of them. */
+			MPIArray_Gather( nBytes, bytes, &nFounds, (void***)&founds, sizeof(unsigned), 
+					 subRanks[p_i], subComm );
+			if( p_i == rank ) {
+				for( p_j = 0; p_j < nProcs - p_i; p_j++ ) {
+					if( !nFounds[p_j] )
+						continue;
+
+					iSets[worldRanks[p_j]] = RangeSet_New();
+					RangeSet_Unpickle( iSets[worldRanks[p_j]], nFounds[p_j], founds[p_j] );
+				}
+			}
+
+			/* Destroy the sub-communicator. */
+			MPI_Comm_free( &subComm );
+		}
+
+		/* Destroy the sub-group. */
+		MPI_Group_free( &subGroup );
+	}
+
+	/* Free rank translation arrays and local range set. */
+	FreeArray( worldRanks );
+	FreeArray( subRanks );
+
+	/* Build a set of communication incidence. */
+	nCommInc = 0;
+	commInc = Memory_Alloc_Array_Unnamed( unsigned, nProcs );
+	for( p_i = 0; p_i < nProcs; p_i++ ) {
+		if( iSets[p_i] && iSets[p_i]->nInds )
+			commInc[nCommInc++] = p_i;
+	}
+
+	/* Create the communication topology. */
+	self->commTopo = CommTopology_New( "Decomp_Sync::commTopo" );
+	CommTopology_SetComm( self->commTopo, self->decomp->comm );
+	CommTopology_SetIncidence( self->commTopo, nCommInc, commInc );
+	FreeArray( commInc );
+
+	/* Build final intersections. */
+	CommTopology_GetIncidence( self->commTopo, rank, &nInc, &inc );
+	if( nInc ) {
+		unsigned	inc_i;
+
+		self->isects = Memory_Alloc_Array_Unnamed( RangeSet*, nInc );
+		for( inc_i = 0; inc_i < nInc; inc_i++ )
+			self->isects[inc_i] = iSets[inc[inc_i]];
+	}
+	else
+		self->isects = NULL;
+	CommTopology_ReturnIncidence( self->commTopo, rank, &nInc, &inc );
+
+	/* Free intersection array. */
+	FreeArray( iSets );
+}
+
+void Decomp_Sync_BuildShared( Decomp_Sync* self ) {
+	unsigned	rank;
+	RangeSet*	sharedSet;
+	unsigned	nInc;
+	unsigned*	inc;
+	unsigned**	sharers;
+	unsigned	s_i, p_i;
+
+	assert( self );
+	assert( self->commTopo );
+
+	/* Get basic MPI info. */
+	MPI_Comm_rank( self->decomp->comm, (int*)&rank );
+
+	/* Get incidence. */
+	CommTopology_GetIncidence( self->commTopo, rank, &nInc, &inc );
+
+	if( nInc ) {
+		assert( self->isects );
+
+		/* Take the union of all intersections to determine how many shared indices we have. */
+		sharedSet = RangeSet_New();
+		for( p_i = 0; p_i < nInc; p_i++ )
+			RangeSet_Union( sharedSet, self->isects[p_i] );
+
+		/* Build table and map. */
+		RangeSet_GetIndices( sharedSet, &self->nShared, &self->shared );
+		self->dsMap = UIntMap_New();
+		for( s_i = 0; s_i < self->nShared; s_i++ ) {
+			self->shared[s_i] = Decomp_Sync_GlobalToDomain( self, self->shared[s_i] );
+			UIntMap_Insert( self->dsMap, self->shared[s_i], s_i );
+		}
+		FreeObject( sharedSet );
+
+		/* Allocate enough space to hold maximum results. */
+		self->nSharers = Memory_Alloc_Array( unsigned, self->nShared, "Decomp_Sync::nSharers" );
+		sharers = Memory_Alloc_2DArray_Unnamed( unsigned, self->nShared, nInc );
+		memset( self->nSharers, 0, self->nShared * sizeof(unsigned) );
+
+		/* Collect processors. */
+		for( p_i = 0; p_i < nInc; p_i++ ) {
+			unsigned	nInds;
+			unsigned*	inds;
+			unsigned	ind_i;
+
+			RangeSet_GetIndices( self->isects[p_i], &nInds, &inds );
+			for( ind_i = 0; ind_i < nInds; ind_i++ ) {
+				unsigned	dInd = Decomp_Sync_GlobalToDomain( self, inds[ind_i] );
+				unsigned       	sInd = Decomp_Sync_DomainToShared( self, dInd );
+
+				sharers[sInd][self->nSharers[sInd]++] = p_i;
+			}
+			FreeArray( inds );
+		}
+
+		/* Transfer results to self. */
+		self->sharers = Memory_Alloc_2DComplex( unsigned, self->nShared, self->nSharers, "Decomp_Sync::sharers" );
+		for( s_i = 0; s_i < self->nShared; s_i++ )
+			memcpy( self->sharers[s_i], sharers[s_i], self->nSharers[s_i] * sizeof(unsigned) );
+
+		/* Cleanup. */
+		FreeArray( sharers );
+	}
+
+	/* Return incidence. */
+	CommTopology_ReturnIncidence( self->commTopo, rank, &nInc, &inc );
+}
+
+void Decomp_Sync_BuildGRMap( Decomp_Sync* self ) {
+	UIntMap*	map;
+	unsigned	r_i;
+
+	FreeObject( self->grMap );
+
+	map = UIntMap_New();
+	for( r_i = 0; r_i < self->nRemotes; r_i++ )
+		UIntMap_Insert( map, self->remotes[r_i], r_i );
+	self->grMap = map;
+}
+
+void Decomp_Sync_SyncArray( Decomp_Sync* self, Decomp_Sync_Array* array ) {
+	CommTopology*	commTopo;
+	unsigned	rank;
+	unsigned	nInc;
+	unsigned*	inc;
+	Byte*		snkArray;
+	Byte*		srcArray;
+	unsigned	p_i;
+
+	/* Sanity checks. */
+	assert( self );
+	assert( self->commTopo );
+	assert( array );
+
+	/* Shortcuts. */
+	commTopo = self->commTopo;
+	MPI_Comm_rank( commTopo->comm, (int*)&rank );
+
+	/* Pack from locals to a contiguous array. */
+	if( self->netSnks > 0 ) {
+		unsigned	snk_i;
+
+		snkArray = Memory_Alloc_Array_Unnamed( Byte, self->netSnks * array->itemSize );
+		for( snk_i = 0; snk_i < self->netSnks; snk_i++ ) {
+			memcpy( snkArray + snk_i * array->itemSize, 
+				(Byte*)array->snkArray + array->snkOffs[snk_i], 
+				array->itemSize );
+		}
+	}
+	else
+		snkArray = NULL;
+
+	/* Allocate for sources. */
+	if( self->netSrcs > 0 )
+		srcArray = Memory_Alloc_Array_Unnamed( Byte, self->netSrcs * array->itemSize );
+	else
+		srcArray = NULL;
+
+	/* Get incidence. */
+	CommTopology_GetIncidence( commTopo, rank, &nInc, &inc );
+
+	/* Transfer. */
+	for( p_i = 0; p_i < nInc; p_i++ ) {
+		int		snkSize = array->snkSizes[p_i];
+		int		snkDisp = array->snkDisps[p_i];
+		int		srcSize = array->srcSizes[p_i];
+		int		srcDisp = array->srcDisps[p_i];
+		MPI_Status	status;
+		unsigned	tag = 6669;
+
+		MPI_Sendrecv( snkArray + snkDisp, snkSize, MPI_BYTE, inc[p_i], tag, 
+			      srcArray + srcDisp, srcSize, MPI_BYTE, inc[p_i], tag, 
+			      commTopo->comm, &status );
+	}
+
+	/* Return incidence. */
+	CommTopology_ReturnIncidence( commTopo, rank, &nInc, &inc );
+
+	/* Free the sink array. */
+	FreeArray( snkArray );
+
+	/* Unpack sources. */
+	if( self->netSnks > 0 ) {
+		unsigned	src_i;
+
+		for( src_i = 0; src_i < self->netSrcs; src_i++ ) {
+			memcpy( (Byte*)array->srcArray + array->srcOffs[src_i], 
+				srcArray + src_i * array->itemSize, 
+				array->itemSize );
+		}
+	}
+
+	/* Free source array. */
+	FreeArray( srcArray );
+}
+
+void Decomp_Sync_BuildArray( Decomp_Sync* self, Decomp_Sync_Array* array ) {
+	CommTopology*	commTopo;
+	Decomp*		decomp;
+	unsigned	rank;
+	unsigned	nInc;
+	unsigned*	inc;
+
+	assert( self );
+	assert( self->commTopo );
+
+	/* Shortcuts. */
+	decomp = self->decomp;
+	commTopo = self->commTopo;
+	MPI_Comm_rank( commTopo->comm, (int*)&rank );
+
+	/* Extract incidence. */
+	CommTopology_GetIncidence( commTopo, rank, &nInc, &inc );
+
+	if( nInc ) {
+		/* Determine sink (local) information. */
+		if( self->netSnks > 0 ) {
+			unsigned*	snkOffs;
+			unsigned*	snkSizes;
+			unsigned*	snkDisps;
+			unsigned	snkInd = 0;
+			unsigned	p_i;
+
+			/* Allocate/reallocate memory. */
+			snkDisps = Memory_Alloc_Array( unsigned, nInc, "Decomp_Sync_Array::snkDisps" );
+			snkSizes = Memory_Alloc_Array( unsigned, nInc, "Decomp_Sync_Array::snkSizes" );
+			snkOffs = Memory_Alloc_Array( unsigned, self->netSnks, "Decomp_Sync_Array::snkOffs" );
+
+			/* Calculate offsets and sizes. */
+			for( p_i = 0; p_i < nInc; p_i++ ) {
+				unsigned	snk_i;
+
+				snkSizes[p_i] = 0;
+				for( snk_i = 0; snk_i < self->nSnks[p_i]; snk_i++ ) {
+					unsigned	dInd = Decomp_Sync_GlobalToDomain( self, self->snks[p_i][snk_i] );
+
+					snkOffs[snkInd] = dInd * array->snkStride;
+					snkSizes[p_i] += array->itemSize;
+					snkInd++;
+				}
+			}
+
+			/* Calculate the displacements. */
+			snkDisps[0] = 0;
+			for( p_i = 1; p_i < nInc; p_i++ )
+				snkDisps[p_i] = snkDisps[p_i - 1] + snkSizes[p_i - 1];
+
+			/* Store arrays. */
+			array->snkOffs = snkOffs;
+			array->snkDisps = snkDisps;
+			array->snkSizes = snkSizes;
+		}
+		else {
+			/* Store null information. */
+			array->snkOffs = NULL;
+			array->snkDisps = Memory_Alloc_Array( unsigned, nInc, "Decomp_Sync_Array::snkDisps" );
+			array->snkSizes = Memory_Alloc_Array( unsigned, nInc, "Decomp_Sync_Array::snkSizes" );
+			memset( array->snkDisps, 0, nInc * sizeof(unsigned) );
+			memset( array->snkSizes, 0, nInc * sizeof(unsigned) );
+		}
+
+		/* Determine source (shadow) information. */
+		if( self->netSrcs > 0 ) {
+			unsigned*	srcOffs;
+			unsigned*	srcSizes;
+			unsigned*	srcDisps;
+			unsigned	srcInd = 0;
+			unsigned	p_i;
+
+			/* Allocate/reallocate memory. */
+			srcDisps = Memory_Alloc_Array( unsigned, nInc, "Decomp_Sync_Array::srcDisps" );
+			srcSizes = Memory_Alloc_Array( unsigned, nInc, "Decomp_Sync_Array::srcSizes" );
+			srcOffs = Memory_Alloc_Array( unsigned, self->netSrcs, "Decomp_Sync_Array::srcOffs" );
+
+			/* Calculate offsets and sizes. */
+			for( p_i = 0; p_i < nInc; p_i++ ) {
+				unsigned	src_i;
+
+				srcSizes[p_i] = 0;
+				for( src_i = 0; src_i < self->nSrcs[p_i]; src_i++ ) {
+					unsigned	sInd = Decomp_Sync_GlobalToDomain( self, self->srcs[p_i][src_i] );
+
+					assert( sInd >= decomp->nLocals );
+					sInd -= decomp->nLocals;
+					srcOffs[srcInd] = sInd * array->srcStride;
+					srcSizes[p_i] += array->itemSize;
+					srcInd++;
+				}
+			}
+
+			/* Calculate the displacements. */
+			srcDisps[0] = 0;
+			for( p_i = 1; p_i < nInc; p_i++ )
+				srcDisps[p_i] = srcDisps[p_i - 1] + srcSizes[p_i - 1];
+
+			/* Store arrays. */
+			array->srcOffs = srcOffs;
+			array->srcDisps = srcDisps;
+			array->srcSizes = srcSizes;
+		}
+		else {
+			/* Store null information. */
+			array->srcOffs = NULL;
+			array->srcDisps = Memory_Alloc_Array( unsigned, nInc, "Decomp_Sync_Array::srcDisps" );
+			array->srcSizes = Memory_Alloc_Array( unsigned, nInc, "Decomp_Sync_Array::srcSizes" );
+			memset( array->srcDisps, 0, nInc * sizeof(unsigned) );
+			memset( array->srcSizes, 0, nInc * sizeof(unsigned) );
+		}
+	}
+	else {
+		array->snkOffs = NULL;
+		array->snkDisps = NULL;
+		array->snkSizes = NULL;
+		array->srcOffs = NULL;
+		array->srcDisps = NULL;
+		array->srcSizes = NULL;
+	}
+
+	/* Return incidence. */
+	CommTopology_ReturnIncidence( commTopo, rank, &nInc, &inc );
+}
+
+void Decomp_Sync_Destruct( Decomp_Sync* self ) {
+	Decomp_Sync_DestructRemotes( self );
+
+	if( self->decomp ) {
+		Decomp_RemoveSync( self->decomp, self );
+		self->decomp = NULL;
+	}
+}
+
+void Decomp_Sync_DestructRemotes( Decomp_Sync* self ) {
+	Decomp_Sync_DestructArrays( self );
+
+	if( self->commTopo ) {
+		unsigned	p_i;
+
+		for( p_i = 0; p_i < self->commTopo->nInc; p_i++ )
+			FreeObject( self->isects[p_i] );
+	}
+	KillArray( self->isects );
+	KillObject( self->commTopo );
+
+	KillArray( self->remotes );
+	self->nRemotes = 0;
+	KillArray( self->shared );
+	self->nShared = 0;
+	KillArray( self->nSharers );
+	KillArray( self->sharers );
+
+	KillObject( self->grMap );
+	KillObject( self->dsMap );
+
+	self->netSrcs = 0;
+	KillArray( self->nSrcs );
+	KillArray( self->srcs );
+	self->netSnks = 0;
+	KillArray( self->nSnks );
+	KillArray( self->snks );
+}
+
+void Decomp_Sync_DestructArrays( Decomp_Sync* self ) {
+	unsigned	a_i;
+
+	for( a_i = 0; a_i < self->nArrays; a_i++ ) {
+		FreeArray( self->arrays[a_i].snkDisps );
+		FreeArray( self->arrays[a_i].snkSizes );
+		FreeArray( self->arrays[a_i].snkOffs );
+		FreeArray( self->arrays[a_i].srcDisps );
+		FreeArray( self->arrays[a_i].srcSizes );
+		FreeArray( self->arrays[a_i].srcOffs );
+	}
+	KillArray( self->arrays );
+	self->nArrays = 0;
+}

Added: long/3D/Gale/trunk/src/StGermain/Discretisation/Mesh/src/Decomp_Sync.h
===================================================================
--- long/3D/Gale/trunk/src/StGermain/Discretisation/Mesh/src/Decomp_Sync.h	2006-10-11 20:46:07 UTC (rev 4823)
+++ long/3D/Gale/trunk/src/StGermain/Discretisation/Mesh/src/Decomp_Sync.h	2006-10-11 20:46:09 UTC (rev 4824)
@@ -0,0 +1,170 @@
+/*~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+**
+** Copyright (C), 2003, Victorian Partnership for Advanced Computing (VPAC) Ltd, 110 Victoria Street, Melbourne, 3053, Australia.
+**
+** Authors:
+**	Stevan M. Quenette, Senior Software Engineer, VPAC. (steve at vpac.org)
+**	Patrick D. Sunter, Software Engineer, VPAC. (pds at vpac.org)
+**	Luke J. Hodkinson, Computational Engineer, VPAC. (lhodkins at vpac.org)
+**	Siew-Ching Tan, Software Engineer, VPAC. (siew at vpac.org)
+**	Alan H. Lo, Computational Engineer, VPAC. (alan at vpac.org)
+**	Raquibul Hassan, Computational Engineer, VPAC. (raq at vpac.org)
+**
+**  This library is free software; you can redistribute it and/or
+**  modify it under the terms of the GNU Lesser General Public
+**  License as published by the Free Software Foundation; either
+**  version 2.1 of the License, or (at your option) any later version.
+**
+**  This library is distributed in the hope that it will be useful,
+**  but WITHOUT ANY WARRANTY; without even the implied warranty of
+**  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+**  Lesser General Public License for more details.
+**
+**  You should have received a copy of the GNU Lesser General Public
+**  License along with this library; if not, write to the Free Software
+**  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+**
+*/
+/** \file
+**  Role:
+**
+** Assumptions:
+**
+** Invariants:
+**
+** Comments:
+**
+** $Id: Decomp_Sync.h 3584 2006-05-16 11:11:07Z PatrickSunter $
+**
+**~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~*/
+
+#ifndef __Discretisaton_Mesh_Decomp_Sync_h__
+#define __Discretisaton_Mesh_Decomp_Sync_h__
+
+	/** Textual name of this class */
+	extern const Type Decomp_Sync_Type;
+
+	/** Virtual function types */
+
+	/** Mesh class contents */
+	typedef struct {
+		void*		snkArray;
+		unsigned	snkStride;
+		unsigned*	snkDisps;
+		unsigned*	snkSizes;
+		unsigned*	snkOffs;
+
+		void*		srcArray;
+		unsigned	srcStride;
+		unsigned*	srcDisps;
+		unsigned*	srcSizes;
+		unsigned*	srcOffs;
+
+		size_t		itemSize;
+	} Decomp_Sync_Array;
+
+	#define __Decomp_Sync				\
+		/* General info */			\
+		__Stg_Component				\
+							\
+		/* Virtual info */			\
+							\
+		/* Decomp_Sync info */			\
+		Decomp*			decomp;		\
+		CommTopology*		commTopo;	\
+		RangeSet**		isects;		\
+							\
+		Decomp_Sync_Claim*	claim;		\
+		Decomp_Sync_Negotiate*	negotiate;	\
+							\
+		unsigned		nRemotes;	\
+		unsigned*		remotes;	\
+		unsigned		nShared;	\
+		unsigned*		shared;		\
+		unsigned*		nSharers;	\
+		unsigned**		sharers;	\
+							\
+		UIntMap*		grMap;		\
+		UIntMap*		dsMap;		\
+							\
+		unsigned		netSrcs;	\
+		unsigned*		nSrcs;		\
+		unsigned**		srcs;		\
+		unsigned		netSnks;	\
+		unsigned*		nSnks;		\
+		unsigned**		snks;		\
+							\
+		unsigned		nArrays;	\
+		Decomp_Sync_Array*	arrays;
+
+	struct Decomp_Sync { __Decomp_Sync };
+
+	/*--------------------------------------------------------------------------------------------------------------------------
+	** Constructors
+	*/
+
+	#define DECOMP_SYNC_DEFARGS	\
+		STG_COMPONENT_DEFARGS
+
+	#define DECOMP_SYNC_PASSARGS	\
+		STG_COMPONENT_PASSARGS
+
+	Decomp_Sync* Decomp_Sync_New( Name name );
+	Decomp_Sync* _Decomp_Sync_New( DECOMP_SYNC_DEFARGS );
+	void _Decomp_Sync_Init( Decomp_Sync* self );
+
+	/*--------------------------------------------------------------------------------------------------------------------------
+	** Virtual functions
+	*/
+
+	void _Decomp_Sync_Delete( void* sync );
+	void _Decomp_Sync_Print( void* sync, Stream* stream );
+
+	#define Decomp_Sync_Copy( self ) \
+		(Mesh*)Stg_Class_Copy( self, NULL, False, NULL, NULL )
+	#define Decomp_Sync_DeepCopy( self ) \
+		(Mesh*)Stg_Class_Copy( self, NULL, True, NULL, NULL )
+	void* _Decomp_Sync_Copy( void* sync, void* dest, Bool deep, Name nameExt, PtrMap* ptrMap );
+
+	void _Decomp_Sync_Construct( void* sync, Stg_ComponentFactory* cf );
+	void _Decomp_Sync_Build( void* sync, void* data );
+	void _Decomp_Sync_Initialise( void* sync, void* data );
+	void _Decomp_Sync_Execute( void* sync, void* data );
+	void _Decomp_Sync_Destroy( void* sync, void* data );
+
+	/*--------------------------------------------------------------------------------------------------------------------------
+	** Public functions
+	*/
+
+	void Decomp_Sync_SetDecomp( void* sync, Decomp* decomp );
+	void Decomp_Sync_SetClaim( void* sync, Decomp_Sync_Claim* claim );
+	void Decomp_Sync_SetNegotiate( void* sync, Decomp_Sync_Negotiate* negotiate );
+	void Decomp_Sync_SetRemotes( void* sync, unsigned nRemotes, unsigned* remotes );
+	void Decomp_Sync_Decompose( void* sync, unsigned nRequired, unsigned* required );
+
+	Bool Decomp_Sync_IsDomain( void* sync, unsigned global );
+	Bool Decomp_Sync_IsRemote( void* sync, unsigned domain );
+	Bool Decomp_Sync_IsShared( void* sync, unsigned domain );
+	unsigned Decomp_Sync_GlobalToDomain( void* sync, unsigned global );
+	unsigned Decomp_Sync_DomainToGlobal( void* sync, unsigned domain );
+	unsigned Decomp_Sync_DomainToShared( void* sync, unsigned domain );
+	unsigned Decomp_Sync_SharedToDomain( void* sync, unsigned shared );
+
+	void Decomp_Sync_AddArray( void* sync, void* localArray, void* remoteArray, 
+				   size_t localStride, size_t remoteStride, size_t itemSize );
+	void Decomp_Sync_Sync( void* sync );
+
+	/*--------------------------------------------------------------------------------------------------------------------------
+	** Private Member functions
+	*/
+
+	void Decomp_Sync_BuildIntersections( Decomp_Sync* self, unsigned nIndices, unsigned* indices );
+	void Decomp_Sync_BuildShared( Decomp_Sync* self );
+	void Decomp_Sync_BuildGRMap( Decomp_Sync* self );
+	void Decomp_Sync_SyncArray( Decomp_Sync* self, Decomp_Sync_Array* array );
+	void Decomp_Sync_BuildArray( Decomp_Sync* self, Decomp_Sync_Array* array );
+	void Decomp_Sync_Destruct( Decomp_Sync* self );
+	void Decomp_Sync_DestructRemotes( Decomp_Sync* self );
+	void Decomp_Sync_DestructArrays( Decomp_Sync* self );
+
+#endif /* __Discretisaton_Mesh_Decomp_Sync_h__ */

Added: long/3D/Gale/trunk/src/StGermain/Discretisation/Mesh/src/Decomp_Sync.meta
===================================================================
--- long/3D/Gale/trunk/src/StGermain/Discretisation/Mesh/src/Decomp_Sync.meta	2006-10-11 20:46:07 UTC (rev 4823)
+++ long/3D/Gale/trunk/src/StGermain/Discretisation/Mesh/src/Decomp_Sync.meta	2006-10-11 20:46:09 UTC (rev 4824)
@@ -0,0 +1,15 @@
+<?xml version="1.0"?>
+<!DOCTYPE StGermainData SYSTEM "stgermain.dtd">
+<StGermainData xmlns="http://www.vpac.org/StGermain/XML_IO_Handler/Jun2003">
+
+<param name="Name">Decomp_Sync</param>
+<param name="Organisation">VPAC</param>
+<param name="Project">StGermain</param>
+<param name="Location">./StGermain/Discretisation/Utils/src/</param>
+<param name="Project Web">https://csd.vpac.org/twiki/bin/view/Stgermain/WebHome</param>
+<param name="Copyright">StGermain Framework. Copyright (C) 2003-2005 VPAC.</param>
+<param name="License">The Gnu Lesser General Public License http://www.gnu.org/licenses/lgpl.html</param>
+<param name="Parent">Stg_Component</param>
+<param name="Description">...</param>
+
+</StGermainData>

Added: long/3D/Gale/trunk/src/StGermain/Discretisation/Mesh/src/Decomp_Sync_Claim.c
===================================================================
--- long/3D/Gale/trunk/src/StGermain/Discretisation/Mesh/src/Decomp_Sync_Claim.c	2006-10-11 20:46:07 UTC (rev 4823)
+++ long/3D/Gale/trunk/src/StGermain/Discretisation/Mesh/src/Decomp_Sync_Claim.c	2006-10-11 20:46:09 UTC (rev 4824)
@@ -0,0 +1,268 @@
+/*~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+**
+** Copyright (C), 2003, Victorian Partnership for Advanced Computing (VPAC) Ltd, 110 Victoria Street, Melbourne, 3053, Australia.
+**
+** Authors:
+**	Stevan M. Quenette, Senior Software Engineer, VPAC. (steve at vpac.org)
+**	Patrick D. Sunter, Software Engineer, VPAC. (pds at vpac.org)
+**	Luke J. Hodkinson, Computational Engineer, VPAC. (lhodkins at vpac.org)
+**	Siew-Ching Tan, Software Engineer, VPAC. (siew at vpac.org)
+**	Alan H. Lo, Computational Engineer, VPAC. (alan at vpac.org)
+**	Raquibul Hassan, Computational Engineer, VPAC. (raq at vpac.org)
+**
+**  This library is free software; you can redistribute it and/or
+**  modify it under the terms of the GNU Lesser General Public
+**  License as published by the Free Software Foundation; either
+**  version 2.1 of the License, or (at your option) any later version.
+**
+**  This library is distributed in the hope that it will be useful,
+**  but WITHOUT ANY WARRANTY; without even the implied warranty of
+**  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+**  Lesser General Public License for more details.
+**
+**  You should have received a copy of the GNU Lesser General Public
+**  License along with this library; if not, write to the Free Software
+**  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+**
+** $Id: Decomp_Sync_Claim.c 3584 2006-05-16 11:11:07Z PatrickSunter $
+**
+**~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~*/
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <assert.h>
+#include <mpi.h>
+
+#include "Base/Base.h"
+
+#include "types.h"
+#include "shortcuts.h"
+#include "CommTopology.h"
+#include "Decomp.h"
+#include "Decomp_Sync.h"
+#include "Decomp_Sync_Claim.h"
+
+
+/* Textual name of this class */
+const Type Decomp_Sync_Claim_Type = "Decomp_Sync_Claim";
+
+/*----------------------------------------------------------------------------------------------------------------------------------
+** Constructors
+*/
+
+Decomp_Sync_Claim* Decomp_Sync_Claim_New( Name name ) {
+	return _Decomp_Sync_Claim_New( sizeof(Decomp_Sync_Claim), 
+				       Decomp_Sync_Claim_Type, 
+				       _Decomp_Sync_Claim_Delete, 
+				       _Decomp_Sync_Claim_Print, 
+				       _Decomp_Sync_Claim_Copy, 
+				       (void* (*)(Name))_Decomp_Sync_Claim_New, 
+				       _Decomp_Sync_Claim_Construct, 
+				       _Decomp_Sync_Claim_Build, 
+				       _Decomp_Sync_Claim_Initialise, 
+				       _Decomp_Sync_Claim_Execute, 
+				       _Decomp_Sync_Claim_Destroy, 
+				       name, 
+				       NON_GLOBAL, 
+				       Decomp_Sync_Claim_Select );
+}
+
+Decomp_Sync_Claim* _Decomp_Sync_Claim_New( DECOMP_SYNC_CLAIM_DEFARGS ) {
+	Decomp_Sync_Claim* self;
+	
+	/* Allocate memory */
+	assert( sizeOfSelf >= sizeof(Decomp_Sync_Claim) );
+	self = (Decomp_Sync_Claim*)_Stg_Component_New( STG_COMPONENT_PASSARGS );
+
+	/* Virtual info */
+	self->selectFunc = selectFunc;
+
+	/* Decomp_Sync_Claim info */
+	_Decomp_Sync_Claim_Init( self );
+
+	return self;
+}
+
+void _Decomp_Sync_Claim_Init( Decomp_Sync_Claim* self ) {
+}
+
+
+/*----------------------------------------------------------------------------------------------------------------------------------
+** Virtual functions
+*/
+
+void _Decomp_Sync_Claim_Delete( void* claim ) {
+	Decomp_Sync_Claim*	self = (Decomp_Sync_Claim*)claim;
+
+	/* Delete the parent. */
+	_Stg_Component_Delete( self );
+}
+
+void _Decomp_Sync_Claim_Print( void* claim, Stream* stream ) {
+	Decomp_Sync_Claim*	self = (Decomp_Sync_Claim*)claim;
+	
+	/* Set the Journal for printing informations */
+	Stream* claimStream;
+	claimStream = Journal_Register( InfoStream_Type, "Decomp_Sync_ClaimStream" );
+
+	/* Print parent */
+	Journal_Printf( stream, "Decomp_Sync_Claim (ptr): (%p)\n", self );
+	_Stg_Component_Print( self, stream );
+}
+
+void* _Decomp_Sync_Claim_Copy( void* claim, void* destProc_I, Bool deep, Name nameExt, PtrMap* ptrMap ) {
+#if 0
+	Decomp_Sync_Claim*	self = (Decomp_Sync_Claim*)claim;
+	Decomp_Sync_Claim*	newDecomp_Sync_Claim;
+	PtrMap*	map = ptrMap;
+	Bool	ownMap = False;
+
+	/* Damn me for making copying so difficult... what was I thinking? */
+	
+	/* We need to create a map if it doesn't already exist. */
+	if( !map ) {
+		map = PtrMap_New( 10 );
+		ownMap = True;
+	}
+	
+	newDecomp_Sync_Claim = (Decomp_Sync_Claim*)_Mesh_Copy( self, destProc_I, deep, nameExt, map );
+	
+	/* Copy the virtual methods here. */
+
+	/* Deep or shallow? */
+	if( deep ) {
+	}
+	else {
+	}
+	
+	/* If we own the map, get rid of it here. */
+	if( ownMap ) Stg_Class_Delete( map );
+	
+	return (void*)newDecomp_Sync_Claim;
+#endif
+
+	return NULL;
+}
+
+void _Decomp_Sync_Claim_Construct( void* claim, Stg_ComponentFactory* cf ) {
+}
+
+void _Decomp_Sync_Claim_Build( void* claim, void* data ) {
+}
+
+void _Decomp_Sync_Claim_Initialise( void* claim, void* data ) {
+}
+
+void _Decomp_Sync_Claim_Execute( void* claim, void* data ) {
+}
+
+void _Decomp_Sync_Claim_Destroy( void* claim, void* data ) {
+}
+
+void Decomp_Sync_Claim_Select( void* claim, Decomp_Sync* sync, unsigned nRequired, unsigned* required, 
+			       unsigned* nLocals, unsigned** locals )
+{
+	Decomp_Sync_Claim*	self = (Decomp_Sync_Claim*)claim;
+	RangeSet*	lSet;
+
+	assert( self );
+	assert( !nRequired || required );
+
+	/* Build a range set of local indices. */
+	lSet = RangeSet_New();
+	RangeSet_SetIndices( lSet, nRequired, required );
+
+	/* Determine ownership. */
+	Decomp_Sync_Claim_ClaimOwnership( self, sync->commTopo, nRequired, required, lSet, sync->isects, 
+					  nLocals, locals, &sync->nRemotes, &sync->remotes );
+
+	/* Free the local set. */
+	FreeObject( lSet );
+}
+
+
+/*--------------------------------------------------------------------------------------------------------------------------
+** Public Functions
+*/
+
+
+/*----------------------------------------------------------------------------------------------------------------------------------
+** Private Functions
+*/
+
+void Decomp_Sync_Claim_ClaimOwnership( Decomp_Sync_Claim* self, CommTopology* topo, unsigned nRequired, unsigned* required, 
+				       RangeSet* lSet, RangeSet** isects, 
+				       unsigned* nLocals, unsigned** locals, unsigned* nRemotes, unsigned** remotes )
+{
+	unsigned	rank, nProcs;
+	unsigned	nInc;
+	unsigned*	inc;
+	unsigned	nBytes;
+	Byte*		bytes;
+	RangeSet*	tmpClaimed;
+	unsigned	tag = 6669;
+	unsigned	p_i, p_j;
+
+	/* Get basic MPI info. */
+	MPI_Comm_rank( topo->comm, (int*)&rank );
+	MPI_Comm_size( topo->comm, (int*)&nProcs );
+
+	/* Extract our neighbouring processors. */
+	CommTopology_GetIncidence( topo, rank, &nInc, &inc );
+
+	/* Figure out where info is coming from and going to. Note that the incidence
+	   is always ordered from lowest to highest rank. */
+	tmpClaimed = RangeSet_New();
+	for( p_i = 0; p_i < nInc; p_i++ ) {
+		MPI_Status	status;
+
+		if( inc[p_i] > rank )
+			break;
+
+		/* Receive from neighbour which indices it has taken. */
+		MPI_Recv( &nBytes, 1, MPI_UNSIGNED, inc[p_i], tag, topo->comm, &status );
+		bytes = Memory_Alloc_Array_Unnamed( Byte, nBytes );
+		MPI_Recv( bytes, nBytes, MPI_BYTE, inc[p_i], tag, topo->comm, &status );
+		RangeSet_Unpickle( tmpClaimed, nBytes, bytes );
+		FreeArray( bytes );
+
+		/* Subtract from our claimed set. */
+		RangeSet_Subtraction( lSet, tmpClaimed );
+	}
+	FreeObject( tmpClaimed );
+
+	/* Extract our ownership. */
+	Decomp_Sync_Claim_BuildIndices( self, nRequired, required, lSet, 
+					nLocals, locals, nRemotes, remotes );
+
+	/* Update remaining neighbours as to which indices we've taken. */
+	for( p_j = p_i; p_j < nInc; p_j++ ) {
+		RangeSet_Intersection( isects[p_j], lSet );
+		RangeSet_Pickle( isects[p_j], &nBytes, &bytes );
+		MPI_Send( &nBytes, 1, MPI_UNSIGNED, inc[p_j], tag, topo->comm );
+		MPI_Send( bytes, nBytes, MPI_BYTE, inc[p_j], tag, topo->comm );
+		FreeArray( bytes );
+	}
+
+	/* Return incidence. */
+	CommTopology_ReturnIncidence( topo, rank, &nInc, &inc );
+}
+
+void Decomp_Sync_Claim_BuildIndices( Decomp_Sync_Claim* self, unsigned nRequired, unsigned* required, RangeSet* claimed, 
+				     unsigned* nLocals, unsigned** locals, unsigned* nRemotes, unsigned** remotes )
+{
+	RangeSet*	rSet;
+
+	/* Extract indices. */
+	RangeSet_GetIndices( claimed, nLocals, locals );
+
+	/* Build a set of remotes. */
+	rSet = RangeSet_New();
+	RangeSet_SetIndices( rSet, nRequired, required );
+	RangeSet_Subtraction( rSet, claimed );
+	RangeSet_GetIndices( rSet, nRemotes, remotes );
+
+	/* Destroy rental set. */
+	FreeObject( rSet );
+}

Added: long/3D/Gale/trunk/src/StGermain/Discretisation/Mesh/src/Decomp_Sync_Claim.h
===================================================================
--- long/3D/Gale/trunk/src/StGermain/Discretisation/Mesh/src/Decomp_Sync_Claim.h	2006-10-11 20:46:07 UTC (rev 4823)
+++ long/3D/Gale/trunk/src/StGermain/Discretisation/Mesh/src/Decomp_Sync_Claim.h	2006-10-11 20:46:09 UTC (rev 4824)
@@ -0,0 +1,114 @@
+/*~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+**
+** Copyright (C), 2003, Victorian Partnership for Advanced Computing (VPAC) Ltd, 110 Victoria Street, Melbourne, 3053, Australia.
+**
+** Authors:
+**	Stevan M. Quenette, Senior Software Engineer, VPAC. (steve at vpac.org)
+**	Patrick D. Sunter, Software Engineer, VPAC. (pds at vpac.org)
+**	Luke J. Hodkinson, Computational Engineer, VPAC. (lhodkins at vpac.org)
+**	Siew-Ching Tan, Software Engineer, VPAC. (siew at vpac.org)
+**	Alan H. Lo, Computational Engineer, VPAC. (alan at vpac.org)
+**	Raquibul Hassan, Computational Engineer, VPAC. (raq at vpac.org)
+**
+**  This library is free software; you can redistribute it and/or
+**  modify it under the terms of the GNU Lesser General Public
+**  License as published by the Free Software Foundation; either
+**  version 2.1 of the License, or (at your option) any later version.
+**
+**  This library is distributed in the hope that it will be useful,
+**  but WITHOUT ANY WARRANTY; without even the implied warranty of
+**  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+**  Lesser General Public License for more details.
+**
+**  You should have received a copy of the GNU Lesser General Public
+**  License along with this library; if not, write to the Free Software
+**  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+**
+*/
+/** \file
+**  Role:
+**
+** Assumptions:
+**
+** Invariants:
+**
+** Comments:
+**
+** $Id: Decomp_Sync_Claim.h 3584 2006-05-16 11:11:07Z PatrickSunter $
+**
+**~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~*/
+
+#ifndef __Discretisaton_Mesh_Decomp_Sync_Claim_h__
+#define __Discretisaton_Mesh_Decomp_Sync_Claim_h__
+
+	/** Textual name of this class */
+	extern const Type Decomp_Sync_Claim_Type;
+
+	/** Virtual function types */
+	typedef void (Decomp_Sync_Claim_SelectFunc)( void* claim, Decomp_Sync* sync, unsigned nRequired, unsigned* required, 
+						     unsigned* nLocals, unsigned** locals );
+
+	/** Mesh class contents */
+	#define __Decomp_Sync_Claim				\
+		/* General info */				\
+		__Stg_Component					\
+								\
+		/* Virtual info */				\
+		Decomp_Sync_Claim_SelectFunc*	selectFunc;	\
+								\
+		/* Decomp_Sync_Claim info */
+
+	struct Decomp_Sync_Claim { __Decomp_Sync_Claim };
+
+	/*--------------------------------------------------------------------------------------------------------------------------
+	** Constructors
+	*/
+
+	#define DECOMP_SYNC_CLAIM_DEFARGS			\
+		STG_COMPONENT_DEFARGS,				\
+		Decomp_Sync_Claim_SelectFunc*	selectFunc
+
+	#define DECOMP_SYNC_CLAIM_PASSARGS		\
+		STG_COMPONENT_PASSARGS, selectFunc
+
+	Decomp_Sync_Claim* Decomp_Sync_Claim_New( Name name );
+	Decomp_Sync_Claim* _Decomp_Sync_Claim_New( DECOMP_SYNC_CLAIM_DEFARGS );
+	void _Decomp_Sync_Claim_Init( Decomp_Sync_Claim* self );
+
+	/*--------------------------------------------------------------------------------------------------------------------------
+	** Virtual functions
+	*/
+
+	void _Decomp_Sync_Claim_Delete( void* claim );
+	void _Decomp_Sync_Claim_Print( void* claim, Stream* stream );
+
+	#define Decomp_Sync_Claim_Copy( self ) \
+		(Mesh*)Stg_Class_Copy( self, NULL, False, NULL, NULL )
+	#define Decomp_Sync_Claim_DeepCopy( self ) \
+		(Mesh*)Stg_Class_Copy( self, NULL, True, NULL, NULL )
+	void* _Decomp_Sync_Claim_Copy( void* claim, void* dest, Bool deep, Name nameExt, PtrMap* ptrMap );
+
+	void _Decomp_Sync_Claim_Construct( void* claim, Stg_ComponentFactory* cf );
+	void _Decomp_Sync_Claim_Build( void* claim, void* data );
+	void _Decomp_Sync_Claim_Initialise( void* claim, void* data );
+	void _Decomp_Sync_Claim_Execute( void* claim, void* data );
+	void _Decomp_Sync_Claim_Destroy( void* claim, void* data );
+
+	void Decomp_Sync_Claim_Select( void* claim, Decomp_Sync* sync, unsigned nRequired, unsigned* required, 
+				       unsigned* nLocals, unsigned** locals );
+
+	/*--------------------------------------------------------------------------------------------------------------------------
+	** Public functions
+	*/
+
+	/*--------------------------------------------------------------------------------------------------------------------------
+	** Private Member functions
+	*/
+
+	void Decomp_Sync_Claim_ClaimOwnership( Decomp_Sync_Claim* self, CommTopology* topo, unsigned nRequired, unsigned* required, 
+					       RangeSet* lSet, RangeSet** isects, 
+					       unsigned* nLocals, unsigned** locals, unsigned* nRemotes, unsigned** remotes );
+	void Decomp_Sync_Claim_BuildIndices( Decomp_Sync_Claim* self, unsigned nRequired, unsigned* required, RangeSet* claimed, 
+					     unsigned* nLocals, unsigned** locals, unsigned* nRemotes, unsigned** remotes );
+
+#endif /* __Discretisaton_Mesh_Decomp_Sync_Claim_h__ */

Added: long/3D/Gale/trunk/src/StGermain/Discretisation/Mesh/src/Decomp_Sync_Claim.meta
===================================================================
--- long/3D/Gale/trunk/src/StGermain/Discretisation/Mesh/src/Decomp_Sync_Claim.meta	2006-10-11 20:46:07 UTC (rev 4823)
+++ long/3D/Gale/trunk/src/StGermain/Discretisation/Mesh/src/Decomp_Sync_Claim.meta	2006-10-11 20:46:09 UTC (rev 4824)
@@ -0,0 +1,15 @@
+<?xml version="1.0"?>
+<!DOCTYPE StGermainData SYSTEM "stgermain.dtd">
+<StGermainData xmlns="http://www.vpac.org/StGermain/XML_IO_Handler/Jun2003">
+
+<param name="Name">Decomp_Sync_Claim</param>
+<param name="Organisation">VPAC</param>
+<param name="Project">StGermain</param>
+<param name="Location">./StGermain/Discretisation/Utils/src/</param>
+<param name="Project Web">https://csd.vpac.org/twiki/bin/view/Stgermain/WebHome</param>
+<param name="Copyright">StGermain Framework. Copyright (C) 2003-2005 VPAC.</param>
+<param name="License">The Gnu Lesser General Public License http://www.gnu.org/licenses/lgpl.html</param>
+<param name="Parent">Stg_Component</param>
+<param name="Description">...</param>
+
+</StGermainData>

Added: long/3D/Gale/trunk/src/StGermain/Discretisation/Mesh/src/Decomp_Sync_Negotiate.c
===================================================================
--- long/3D/Gale/trunk/src/StGermain/Discretisation/Mesh/src/Decomp_Sync_Negotiate.c	2006-10-11 20:46:07 UTC (rev 4823)
+++ long/3D/Gale/trunk/src/StGermain/Discretisation/Mesh/src/Decomp_Sync_Negotiate.c	2006-10-11 20:46:09 UTC (rev 4824)
@@ -0,0 +1,376 @@
+/*~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+**
+** Copyright (C), 2003, Victorian Partnership for Advanced Computing (VPAC) Ltd, 110 Victoria Street, Melbourne, 3053, Australia.
+**
+** Authors:
+**	Stevan M. Quenette, Senior Software Engineer, VPAC. (steve at vpac.org)
+**	Patrick D. Sunter, Software Engineer, VPAC. (pds at vpac.org)
+**	Luke J. Hodkinson, Computational Engineer, VPAC. (lhodkins at vpac.org)
+**	Siew-Ching Tan, Software Engineer, VPAC. (siew at vpac.org)
+**	Alan H. Lo, Computational Engineer, VPAC. (alan at vpac.org)
+**	Raquibul Hassan, Computational Engineer, VPAC. (raq at vpac.org)
+**
+**  This library is free software; you can redistribute it and/or
+**  modify it under the terms of the GNU Lesser General Public
+**  License as published by the Free Software Foundation; either
+**  version 2.1 of the License, or (at your option) any later version.
+**
+**  This library is distributed in the hope that it will be useful,
+**  but WITHOUT ANY WARRANTY; without even the implied warranty of
+**  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+**  Lesser General Public License for more details.
+**
+**  You should have received a copy of the GNU Lesser General Public
+**  License along with this library; if not, write to the Free Software
+**  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+**
+** $Id: Decomp_Sync_Negotiate.c 3584 2006-05-16 11:11:07Z PatrickSunter $
+**
+**~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~*/
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <assert.h>
+#include <mpi.h>
+
+#include "Base/Base.h"
+
+#include "types.h"
+#include "shortcuts.h"
+#include "CommTopology.h"
+#include "Decomp.h"
+#include "Decomp_Sync.h"
+#include "Decomp_Sync_Negotiate.h"
+
+
+/* Textual name of this class */
+const Type Decomp_Sync_Negotiate_Type = "Decomp_Sync_Negotiate";
+
+/*----------------------------------------------------------------------------------------------------------------------------------
+** Constructors
+*/
+
+Decomp_Sync_Negotiate* Decomp_Sync_Negotiate_New( Name name ) {
+	return _Decomp_Sync_Negotiate_New( sizeof(Decomp_Sync_Negotiate), 
+					   Decomp_Sync_Negotiate_Type, 
+					   _Decomp_Sync_Negotiate_Delete, 
+					   _Decomp_Sync_Negotiate_Print, 
+					   _Decomp_Sync_Negotiate_Copy, 
+					   (void* (*)(Name))_Decomp_Sync_Negotiate_New, 
+					   _Decomp_Sync_Negotiate_Construct, 
+					   _Decomp_Sync_Negotiate_Build, 
+					   _Decomp_Sync_Negotiate_Initialise, 
+					   _Decomp_Sync_Negotiate_Execute, 
+					   _Decomp_Sync_Negotiate_Destroy, 
+					   name, 
+					   NON_GLOBAL, 
+					   Decomp_Sync_Negotiate_Select );
+}
+
+Decomp_Sync_Negotiate* _Decomp_Sync_Negotiate_New( DECOMP_SYNC_NEGOTIATE_DEFARGS ) {
+	Decomp_Sync_Negotiate* self;
+	
+	/* Allocate memory */
+	assert( sizeOfSelf >= sizeof(Decomp_Sync_Negotiate) );
+	self = (Decomp_Sync_Negotiate*)_Stg_Component_New( STG_COMPONENT_PASSARGS );
+
+	/* Virtual info */
+	self->selectFunc = selectFunc;
+
+	/* Decomp_Sync_Negotiate info */
+	_Decomp_Sync_Negotiate_Init( self );
+
+	return self;
+}
+
+void _Decomp_Sync_Negotiate_Init( Decomp_Sync_Negotiate* self ) {
+}
+
+
+/*----------------------------------------------------------------------------------------------------------------------------------
+** Virtual functions
+*/
+
+void _Decomp_Sync_Negotiate_Delete( void* negotiate ) {
+	Decomp_Sync_Negotiate*	self = (Decomp_Sync_Negotiate*)negotiate;
+
+	/* Delete the parent. */
+	_Stg_Component_Delete( self );
+}
+
+void _Decomp_Sync_Negotiate_Print( void* negotiate, Stream* stream ) {
+	Decomp_Sync_Negotiate*	self = (Decomp_Sync_Negotiate*)negotiate;
+	
+	/* Set the Journal for printing informations */
+	Stream* negotiateStream;
+	negotiateStream = Journal_Register( InfoStream_Type, "Decomp_Sync_NegotiateStream" );
+
+	/* Print parent */
+	Journal_Printf( stream, "Decomp_Sync_Negotiate (ptr): (%p)\n", self );
+	_Stg_Component_Print( self, stream );
+}
+
+void* _Decomp_Sync_Negotiate_Copy( void* negotiate, void* destProc_I, Bool deep, Name nameExt, PtrMap* ptrMap ) {
+#if 0
+	Decomp_Sync_Negotiate*	self = (Decomp_Sync_Negotiate*)negotiate;
+	Decomp_Sync_Negotiate*	newDecomp_Sync_Negotiate;
+	PtrMap*	map = ptrMap;
+	Bool	ownMap = False;
+
+	/* Damn me for making copying so difficult... what was I thinking? */
+	
+	/* We need to create a map if it doesn't already exist. */
+	if( !map ) {
+		map = PtrMap_New( 10 );
+		ownMap = True;
+	}
+	
+	newDecomp_Sync_Negotiate = (Decomp_Sync_Negotiate*)_Mesh_Copy( self, destProc_I, deep, nameExt, map );
+	
+	/* Copy the virtual methods here. */
+
+	/* Deep or shallow? */
+	if( deep ) {
+	}
+	else {
+	}
+	
+	/* If we own the map, get rid of it here. */
+	if( ownMap ) Stg_Class_Delete( map );
+	
+	return (void*)newDecomp_Sync_Negotiate;
+#endif
+
+	return NULL;
+}
+
+void _Decomp_Sync_Negotiate_Construct( void* negotiate, Stg_ComponentFactory* cf ) {
+}
+
+void _Decomp_Sync_Negotiate_Build( void* negotiate, void* data ) {
+}
+
+void _Decomp_Sync_Negotiate_Initialise( void* negotiate, void* data ) {
+}
+
+void _Decomp_Sync_Negotiate_Execute( void* negotiate, void* data ) {
+}
+
+void _Decomp_Sync_Negotiate_Destroy( void* negotiate, void* data ) {
+}
+
+void Decomp_Sync_Negotiate_Select( void* negotiate, Decomp_Sync* sync ) {
+	Decomp_Sync_Negotiate*	self = (Decomp_Sync_Negotiate*)negotiate;
+	CommTopology*		commTopo;
+	unsigned		rank;
+	unsigned		nInc;
+	unsigned*		inc;
+	unsigned*		nFound;
+	unsigned**		found;
+	unsigned*		nUniques;
+	unsigned**		uniques;
+	unsigned		mostUniques;
+	unsigned**		srcs;
+	unsigned		p_i;
+
+	assert( self );
+	assert( sync );
+	assert( sync->commTopo );
+
+	/* Shortcuts. */
+	commTopo = sync->commTopo;
+	MPI_Comm_rank( commTopo->comm, (int*)&rank );
+
+	/* Locate remote indices. */
+	Decomp_Sync_Negotiate_RemoteSearch( self, sync, &nFound, &found );
+
+	/* Get processor incidence. */
+	CommTopology_GetIncidence( commTopo, rank, &nInc, &inc );
+
+	if( nInc ) {
+		/* Set up the source arrays. */
+		sync->nSrcs = Memory_Alloc_Array( unsigned, nInc, "Decomp_Sync::nSrcs" );
+		srcs = Memory_Alloc_Array_Unnamed( unsigned*, nInc );
+		memset( sync->nSrcs, 0, nInc * sizeof(unsigned) );
+		memset( srcs, 0, nInc * sizeof(unsigned*) );
+
+		/* Select source procs from which to retrieve required nodes such that communications are minimized. */
+		nUniques = Memory_Alloc_Array_Unnamed( unsigned, nInc );
+		uniques = Memory_Alloc_2DComplex_Unnamed( unsigned, nInc, nFound );
+		memset( nUniques, 0, nInc * sizeof(unsigned) );
+
+		do {
+			unsigned	mostProc = 0;
+
+			/* Loop over procs and find unique nodes per proc. ie. the most nodes not already covered by
+			   an existing source. */
+			mostUniques = 0;
+			for( p_i = 0; p_i < nInc; p_i++ ) {
+				unsigned	fnd_i;
+
+				/* Clear the number of uniques. */
+				nUniques[p_i] = 0;
+
+				/* If there are no founds for this proc or we've already sourced it, then skip. */
+				if( nFound[p_i] == 0 || sync->nSrcs[p_i] > 0 )
+					continue;
+
+				/* Hunt down unique global indices. */
+				for( fnd_i = 0; fnd_i < nFound[p_i]; fnd_i++ ) {
+					unsigned	p_j;
+
+					for( p_j = 0; p_j < nInc; p_j++ ) {
+						unsigned	src_i;
+
+						for( src_i = 0; src_i < sync->nSrcs[p_i]; src_i++ ) {
+							if( srcs[p_i][src_i] == found[p_i][fnd_i] )
+								break;
+						}
+						if( src_i < sync->nSrcs[p_i] )
+							break;
+					}
+					if( p_j == nInc )
+						uniques[p_i][nUniques[p_i]++] = found[p_i][fnd_i];
+				}
+
+				/* Determine which proc has the most uniques and store. */
+				if( nUniques[p_i] > mostUniques ) {
+					mostUniques = nUniques[p_i];
+					mostProc = p_i;
+				}
+			}
+
+			/* Store result. */
+			if( mostUniques ) {
+				sync->nSrcs[mostProc] = mostUniques;
+				if( mostUniques ) {
+					srcs[mostProc] = Memory_Alloc_Array_Unnamed( unsigned, mostUniques );
+					memcpy( srcs[mostProc], uniques[mostProc], mostUniques * sizeof(unsigned) );
+				}
+			}
+		}
+		while( mostUniques );
+
+		/* Proper allocation. */
+		sync->srcs = Memory_Alloc_2DComplex( unsigned, nInc, sync->nSrcs, "Decomp_Sync::srcs" );
+		for( p_i = 0; p_i < nInc; p_i++ ) {
+			if( sync->nSrcs[p_i] )
+				memcpy( sync->srcs[p_i], srcs[p_i], sync->nSrcs[p_i] * sizeof(unsigned) );
+		}
+
+		/* Get rid of all the resources so far. */
+		FreeArray2D( nInc, srcs );
+		FreeArray( nUniques );
+		FreeArray( uniques );
+
+		/* Communicate back to all procs our selection.  Note that even though most procs will not need to send
+		   anything here, we still need to let them know that. */
+		CommTopology_Alltoall( commTopo, 
+				       sync->nSrcs, (void**)sync->srcs, 
+				       &sync->nSnks, (void***)&sync->snks, 
+				       sizeof(unsigned) );
+
+		/* Calculate net values for source and sink. */
+		for( p_i = 0; p_i < nInc; p_i++ ) {
+			sync->netSrcs += sync->nSrcs[p_i];
+			sync->netSnks += sync->nSnks[p_i];
+		}
+	}
+
+	/* Return incidence. */
+	CommTopology_ReturnIncidence( commTopo, rank, &nInc, &inc );
+}
+
+
+/*--------------------------------------------------------------------------------------------------------------------------
+** Public Functions
+*/
+
+
+/*----------------------------------------------------------------------------------------------------------------------------------
+** Private Functions
+*/
+
+void Decomp_Sync_Negotiate_RemoteSearch( Decomp_Sync_Negotiate* self, Decomp_Sync* sync, 
+					 unsigned** nRemFound, unsigned*** remFound )
+{
+	CommTopology*	commTopo;
+	Decomp*		decomp;
+	unsigned	rank;
+	unsigned	nInc;
+	unsigned*	inc;
+	unsigned*	remNReq;
+	unsigned**	remReq;
+	unsigned*	nFound;
+	unsigned**	found;
+	unsigned	p_i;
+
+	assert( self );
+	assert( sync );
+	assert( sync->commTopo );
+	assert( sync->decomp );
+	assert( nRemFound );
+	assert( remFound );
+
+	/* Shortcuts. */
+	decomp = sync->decomp;
+	commTopo = sync->commTopo;
+	MPI_Comm_rank( commTopo->comm, (int*)&rank );
+
+	/* Retrieve our incident neighbours. */
+	CommTopology_GetIncidence( commTopo, rank, &nInc, &inc );
+
+	if( nInc ) {
+		/* Broadcast our remotes to neighbours. */
+		CommTopology_Allgather( commTopo, 
+					sync->nRemotes, sync->remotes, 
+					&remNReq, (void***)&remReq, 
+					sizeof(unsigned) );
+
+		/* Prepare to locate remote requireds. */
+		nFound = Memory_Alloc_Array_Unnamed( unsigned, nInc );
+		found = Memory_Alloc_2DComplex_Unnamed( unsigned, nInc, remNReq );
+
+		/* Begin searching... */
+		for( p_i = 0; p_i < nInc; p_i++ ) {
+			unsigned	rem_i;
+
+			/* Initialise found count to zero. */
+			nFound[p_i] = 0;
+
+			/* Do we have any for this processor? */
+			for( rem_i = 0; rem_i < remNReq[p_i]; rem_i++ ) {
+				unsigned	l_i;
+
+				for( l_i = 0; l_i < decomp->nLocals; l_i++ ) {
+					if( decomp->locals[l_i] == remReq[p_i][rem_i] ) {
+						found[p_i][nFound[p_i]++] = remReq[p_i][rem_i];
+						break;
+					}
+				}
+			}
+		}
+
+		/* Release some memory. */
+		FreeArray( remNReq );
+		FreeArray( remReq );
+
+		/* Send back all the ones we found and receive from all others all our requireds they found. */
+		CommTopology_Alltoall( commTopo, 
+				       nFound, (void**)found, 
+				       nRemFound, (void***)remFound, 
+				       sizeof(unsigned) );
+
+		/* Release some memory. */
+		FreeArray( nFound );
+		FreeArray( found );
+	}
+	else {
+		*nRemFound = 0;
+		*remFound = NULL;
+	}
+
+	/* Return the incidence. */
+	CommTopology_ReturnIncidence( commTopo, rank, &nInc, &inc );
+}

Added: long/3D/Gale/trunk/src/StGermain/Discretisation/Mesh/src/Decomp_Sync_Negotiate.h
===================================================================
--- long/3D/Gale/trunk/src/StGermain/Discretisation/Mesh/src/Decomp_Sync_Negotiate.h	2006-10-11 20:46:07 UTC (rev 4823)
+++ long/3D/Gale/trunk/src/StGermain/Discretisation/Mesh/src/Decomp_Sync_Negotiate.h	2006-10-11 20:46:09 UTC (rev 4824)
@@ -0,0 +1,109 @@
+/*~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+**
+** Copyright (C), 2003, Victorian Partnership for Advanced Computing (VPAC) Ltd, 110 Victoria Street, Melbourne, 3053, Australia.
+**
+** Authors:
+**	Stevan M. Quenette, Senior Software Engineer, VPAC. (steve at vpac.org)
+**	Patrick D. Sunter, Software Engineer, VPAC. (pds at vpac.org)
+**	Luke J. Hodkinson, Computational Engineer, VPAC. (lhodkins at vpac.org)
+**	Siew-Ching Tan, Software Engineer, VPAC. (siew at vpac.org)
+**	Alan H. Lo, Computational Engineer, VPAC. (alan at vpac.org)
+**	Raquibul Hassan, Computational Engineer, VPAC. (raq at vpac.org)
+**
+**  This library is free software; you can redistribute it and/or
+**  modify it under the terms of the GNU Lesser General Public
+**  License as published by the Free Software Foundation; either
+**  version 2.1 of the License, or (at your option) any later version.
+**
+**  This library is distributed in the hope that it will be useful,
+**  but WITHOUT ANY WARRANTY; without even the implied warranty of
+**  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+**  Lesser General Public License for more details.
+**
+**  You should have received a copy of the GNU Lesser General Public
+**  License along with this library; if not, write to the Free Software
+**  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+**
+*/
+/** \file
+**  Role:
+**
+** Assumptions:
+**
+** Invariants:
+**
+** Comments:
+**
+** $Id: Decomp_Sync_Negotiate.h 3584 2006-05-16 11:11:07Z PatrickSunter $
+**
+**~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~*/
+
+#ifndef __Discretisaton_Mesh_Decomp_Sync_Negotiate_h__
+#define __Discretisaton_Mesh_Decomp_Sync_Negotiate_h__
+
+	/** Textual name of this class */
+	extern const Type Decomp_Sync_Negotiate_Type;
+
+	/** Virtual function types */
+	typedef void (Decomp_Sync_Negotiate_SelectFunc)( void* negotiate, Decomp_Sync* decomp );
+
+	/** Mesh class contents */
+	#define __Decomp_Sync_Negotiate					\
+		/* General info */					\
+		__Stg_Component						\
+									\
+		/* Virtual info */					\
+		Decomp_Sync_Negotiate_SelectFunc*	selectFunc;	\
+									\
+		/* Decomp_Sync_Negotiate info */
+
+	struct Decomp_Sync_Negotiate { __Decomp_Sync_Negotiate };
+
+	/*--------------------------------------------------------------------------------------------------------------------------
+	** Constructors
+	*/
+
+	#define DECOMP_SYNC_NEGOTIATE_DEFARGS				\
+		STG_COMPONENT_DEFARGS,					\
+		Decomp_Sync_Negotiate_SelectFunc*	selectFunc
+
+	#define DECOMP_SYNC_NEGOTIATE_PASSARGS		\
+		STG_COMPONENT_PASSARGS, selectFunc
+
+	Decomp_Sync_Negotiate* Decomp_Sync_Negotiate_New( Name name );
+	Decomp_Sync_Negotiate* _Decomp_Sync_Negotiate_New( DECOMP_SYNC_NEGOTIATE_DEFARGS );
+	void _Decomp_Sync_Negotiate_Init( Decomp_Sync_Negotiate* self );
+
+	/*--------------------------------------------------------------------------------------------------------------------------
+	** Virtual functions
+	*/
+
+	void _Decomp_Sync_Negotiate_Delete( void* negotiate );
+	void _Decomp_Sync_Negotiate_Print( void* negotiate, Stream* stream );
+
+	#define Decomp_Sync_Negotiate_Copy( self ) \
+		(Mesh*)Stg_Class_Copy( self, NULL, False, NULL, NULL )
+	#define Decomp_Sync_Negotiate_DeepCopy( self ) \
+		(Mesh*)Stg_Class_Copy( self, NULL, True, NULL, NULL )
+	void* _Decomp_Sync_Negotiate_Copy( void* negotiate, void* dest, Bool deep, Name nameExt, PtrMap* ptrMap );
+
+	void _Decomp_Sync_Negotiate_Construct( void* negotiate, Stg_ComponentFactory* cf );
+	void _Decomp_Sync_Negotiate_Build( void* negotiate, void* data );
+	void _Decomp_Sync_Negotiate_Initialise( void* negotiate, void* data );
+	void _Decomp_Sync_Negotiate_Execute( void* negotiate, void* data );
+	void _Decomp_Sync_Negotiate_Destroy( void* negotiate, void* data );
+
+	void Decomp_Sync_Negotiate_Select( void* negotiate, Decomp_Sync* decomp );
+
+	/*--------------------------------------------------------------------------------------------------------------------------
+	** Public functions
+	*/
+
+	/*--------------------------------------------------------------------------------------------------------------------------
+	** Private Member functions
+	*/
+
+	void Decomp_Sync_Negotiate_RemoteSearch( Decomp_Sync_Negotiate* self, Decomp_Sync* decomp, 
+						 unsigned** nRemFound, unsigned*** remFound );
+
+#endif /* __Discretisaton_Mesh_Decomp_Sync_Negotiate_h__ */

Added: long/3D/Gale/trunk/src/StGermain/Discretisation/Mesh/src/Decomp_Sync_Negotiate.meta
===================================================================
--- long/3D/Gale/trunk/src/StGermain/Discretisation/Mesh/src/Decomp_Sync_Negotiate.meta	2006-10-11 20:46:07 UTC (rev 4823)
+++ long/3D/Gale/trunk/src/StGermain/Discretisation/Mesh/src/Decomp_Sync_Negotiate.meta	2006-10-11 20:46:09 UTC (rev 4824)
@@ -0,0 +1,15 @@
+<?xml version="1.0"?>
+<!DOCTYPE StGermainData SYSTEM "stgermain.dtd">
+<StGermainData xmlns="http://www.vpac.org/StGermain/XML_IO_Handler/Jun2003">
+
+<param name="Name">Decomp_Sync_Negotiate</param>
+<param name="Organisation">VPAC</param>
+<param name="Project">StGermain</param>
+<param name="Location">./StGermain/Discretisation/Utils/src/</param>
+<param name="Project Web">https://csd.vpac.org/twiki/bin/view/Stgermain/WebHome</param>
+<param name="Copyright">StGermain Framework. Copyright (C) 2003-2005 VPAC.</param>
+<param name="License">The Gnu Lesser General Public License http://www.gnu.org/licenses/lgpl.html</param>
+<param name="Parent">Stg_Component</param>
+<param name="Description">...</param>
+
+</StGermainData>

Modified: long/3D/Gale/trunk/src/StGermain/Discretisation/Mesh/src/Mesh.h
===================================================================
--- long/3D/Gale/trunk/src/StGermain/Discretisation/Mesh/src/Mesh.h	2006-10-11 20:46:07 UTC (rev 4823)
+++ long/3D/Gale/trunk/src/StGermain/Discretisation/Mesh/src/Mesh.h	2006-10-11 20:46:09 UTC (rev 4824)
@@ -44,7 +44,11 @@
 	
 	#include "types.h"
 	#include "shortcuts.h"
+	#include "CommTopology.h"
 	#include "Decomp.h"
+	#include "Decomp_Sync.h"
+	#include "Decomp_Sync_Claim.h"
+	#include "Decomp_Sync_Negotiate.h"
 	#include "ElementLayout.h"
 	#include "HexaEL.h"
 	#include "ParallelPipedHexaEL.h"

Modified: long/3D/Gale/trunk/src/StGermain/Discretisation/Mesh/src/types.h
===================================================================
--- long/3D/Gale/trunk/src/StGermain/Discretisation/Mesh/src/types.h	2006-10-11 20:46:07 UTC (rev 4823)
+++ long/3D/Gale/trunk/src/StGermain/Discretisation/Mesh/src/types.h	2006-10-11 20:46:09 UTC (rev 4824)
@@ -126,7 +126,11 @@
 	typedef Neighbour_Index			Nbr_Index;
 	
 	/* Classes */
+	typedef struct CommTopology			CommTopology;
 	typedef struct Decomp				Decomp;
+	typedef struct Decomp_Sync			Decomp_Sync;
+	typedef struct Decomp_Sync_Claim		Decomp_Sync_Claim;
+	typedef struct Decomp_Sync_Negotiate		Decomp_Sync_Negotiate;
 	typedef struct ProcNbrInfo			ProcNbrInfo;
 	typedef struct ShadowInfo			ShadowInfo;
 	typedef struct _ElementLayout			ElementLayout;