[cig-commits] commit:
Mercurial
hg at geodynamics.org
Mon Nov 24 11:59:19 PST 2008
changeset: 122:4bc0dd9e4e84
user: BelindaMay
date: Thu Jul 24 00:07:13 2008 +0000
files: Swarm/src/FileParticleLayout.c Swarm/src/FileParticleLayout.h Swarm/src/SwarmDump.c
description:
Changed swarm HDF5 checkpointing to make the files portable
diff -r f9b63e9cc42b -r 4bc0dd9e4e84 Swarm/src/FileParticleLayout.c
--- a/Swarm/src/FileParticleLayout.c Tue Jul 22 02:14:00 2008 +0000
+++ b/Swarm/src/FileParticleLayout.c Thu Jul 24 00:07:13 2008 +0000
@@ -51,6 +51,9 @@
#include "ShadowInfo.h"
#include "CellLayout.h"
#include "ElementCellLayout.h"
+#include "IntegrationPoint.h"
+#include "SwarmVariable.h"
+#include "SwarmVariable_Register.h"
#include <stdio.h>
#include <string.h>
@@ -125,7 +128,6 @@ void _FileParticleLayout_Init( void* par
FileParticleLayout* self = (FileParticleLayout*) particleLayout;
self->filename = StG_Strdup( filename );
- self->file = NULL;
self->errorStream = Journal_MyStream( Error_Type, self );
_GlobalParticleLayout_Init( self, GlobalCoordSystem, False, 0, 0.0 );
}
@@ -152,7 +154,6 @@ void _FileParticleLayout_Print( void* pa
/* FileParticleLayout */
Journal_Printf( stream, "filename: %s\n", self->filename );
- Journal_Printf( stream, "file (ptr): %p\n", self->file );
Stream_UnIndent( stream );
}
@@ -209,46 +210,66 @@ void _FileParticleLayout_Destroy( void*
}
void _FileParticleLayout_SetInitialCounts( void* particleLayout, void* _swarm ) {
- FileParticleLayout* self = (FileParticleLayout*)particleLayout;
- Swarm* swarm = (Swarm*)_swarm;
- Name filename = self->filename;
- MPI_File mpiFile;
- int openResult;
- MPI_Offset bytesCount;
- SizeT particleSize = swarm->particleExtensionMgr->finalSize;
- div_t division;
-#ifdef HAVE_HDF5
- hid_t file, fileData;
- int size[2];
+ FileParticleLayout* self = (FileParticleLayout*)particleLayout;
+ Swarm* swarm = (Swarm*)_swarm;
+ Name filename = self->filename;
+
+#ifdef HAVE_HDF5
+ hid_t file, fileData, fileSpace;
+ hsize_t size[2];
+ char dataSpaceName[1024];
+ SwarmVariable* swarmVar;
+#else
+ MPI_File mpiFile;
+ int openResult;
+ MPI_Offset bytesCount;
+ SizeT particleSize = swarm->particleExtensionMgr->finalSize;
+ div_t division;
#endif
Journal_DPrintf( self->debug, "In %s(): for ParticleLayout \"%s\", of type %s\n",
__func__, self->name, self->type );
Stream_IndentBranch( Swarm_Debug );
- Journal_DPrintf( self->debug, "Finding number of bytes in checkpoint file \"%s\":\n",
+#ifdef HAVE_HDF5
+ /* Open the swarm checkpointing file */
+ file = H5Fopen( filename, H5F_ACC_RDONLY, H5P_DEFAULT );
+ Journal_Firewall(
+ file >= 0,
+ self->errorStream,
+ "Error in %s for %s '%s' - Cannot open file %s.\n",
+ __func__,
+ self->type,
+ self->name,
+ filename );
+
+ /* Open a dataspace */
+ swarmVar = SwarmVariable_Register_GetByIndex( swarm->swarmVariable_Register, 0 );
+ sprintf( dataSpaceName, "/%s", swarmVar->name );
+
+ #if H5_VERS_MAJOR == 1 && H5_VERS_MINOR < 8
+ fileData = H5Dopen( file, dataSpaceName );
+ #else
+ fileData = H5Dopen( file, dataSpaceName, H5P_DEFAULT );
+ #endif
+ fileSpace = H5Dget_space( fileData );
+
+ /* Get the dimensions of the open dataspace */
+ H5Sget_simple_extent_dims( fileSpace, size, NULL );
+
+ self->totalInitialParticles = size[0];
+
+ /* Close the dataspace and file */
+ H5Sclose( fileSpace );
+ H5Dclose( fileData );
+ H5Fclose( file );
+
+#else
+ Journal_DPrintf( self->debug, "Finding number of bytes in checkpoint file \"%s\":\n",
self->filename );
-
-#ifdef HAVE_HDF5
- /* Read in data size. */
- file = H5Fopen( filename, H5F_ACC_RDONLY, H5P_DEFAULT );
-#if H5_VERS_MAJOR == 1 && H5_VERS_MINOR < 8
- fileData = H5Dopen( file, "/size" );
-#else
- fileData = H5Dopen( file, "/size", H5P_DEFAULT );
-#endif
- H5Dread( fileData, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, size );
- H5Dclose( fileData );
- H5Fclose( file );
-
- /* Make sure paricle sizes are the same. */
- assert( size[1] == swarm->particleExtensionMgr->finalSize );
-
- /* Store number of particles. */
- self->totalInitialParticles = size[0];
-#else
+
openResult = MPI_File_open( swarm->comm, filename, MPI_MODE_RDONLY, MPI_INFO_NULL, &mpiFile );
-
+
Journal_Firewall(
openResult == 0,
self->errorStream,
@@ -266,7 +287,7 @@ void _FileParticleLayout_SetInitialCount
/* Divide by particle size to get number of particles */
division = div( bytesCount, particleSize );
self->totalInitialParticles = division.quot;
-
+
Journal_DPrintf( self->debug, "given bytes total %u / particle size %u ->\n"
"\ttotalInitialParticles = %u.\n", bytesCount, (unsigned int)particleSize,
self->totalInitialParticles );
@@ -283,9 +304,10 @@ void _FileParticleLayout_SetInitialCount
bytesCount,
(unsigned int)particleSize,
division.rem );
-#endif
-
- Journal_DPrintf( self->debug, "calling parent func to set cell counts:\n", bytesCount );
+
+ Journal_DPrintf( self->debug, "calling parent func to set cell counts:\n", bytesCount );
+#endif
+
_GlobalParticleLayout_SetInitialCounts( self, swarm );
Stream_UnIndentBranch( Swarm_Debug );
@@ -296,36 +318,59 @@ void _FileParticleLayout_InitialiseParti
void _FileParticleLayout_InitialiseParticles( void* particleLayout, void* _swarm ) {
FileParticleLayout* self = (FileParticleLayout*)particleLayout;
Swarm *swarm = (Swarm*)_swarm;
-#ifdef HAVE_HDF5
- hid_t file;
- hsize_t size[2];
-
- /* Open the file and data set. */
+
+#ifdef HAVE_HDF5
+ SwarmVariable* swarmVar;
+ Index swarmVar_I, dof_I;
+ char dataSpaceName[1024];
+ hid_t file;
+
+ /* Allocate space to store arrays of dataspaces */
+ assert( swarm->swarmVariable_Register );
+ self->fileData = Memory_Alloc_Array( hid_t, swarm->swarmVariable_Register->objects->count, "fileData" );
+ self->fileSpace = Memory_Alloc_Array( hid_t, swarm->swarmVariable_Register->objects->count, "fileSpace" );
+
+ /* Open the file */
file = H5Fopen( self->filename, H5F_ACC_RDONLY, H5P_DEFAULT );
-#if H5_VERS_MAJOR == 1 && H5_VERS_MINOR < 8
- self->fileData = H5Dopen( file, "/data" );
-#else
- self->fileData = H5Dopen( file, "/data", H5P_DEFAULT );
-#endif
- self->fileSpace = H5Dget_space( self->fileData );
-
- /* Need a memory space for extracting to. */
- size[0] = 1;
- size[1] = swarm->particleExtensionMgr->finalSize;
- self->memSpace = H5Screate_simple( 1, size + 1, NULL );
- H5Sselect_all( self->memSpace );
-
- /* Prepare a hyperslab for extracting file data one particle at a time. */
- self->start[0] = 0; self->start[1] = 0;
- self->count[0] = 1; self->count[1] = size[1];
- H5Sselect_hyperslab( self->fileSpace, H5S_SELECT_SET, self->start, NULL, self->count, NULL );
-
+ Journal_Firewall(
+ file >= 0,
+ self->errorStream,
+ "Error in %s for %s '%s' - Cannot open file %s.\n",
+ __func__,
+ self->type,
+ self->name,
+ self->filename );
+
+ /* Open a dataspace for each swarmVariable */
+ for( swarmVar_I = 0; swarmVar_I < swarm->swarmVariable_Register->objects->count; swarmVar_I++ ) {
+ swarmVar = SwarmVariable_Register_GetByIndex( swarm->swarmVariable_Register, swarmVar_I );
+ sprintf( dataSpaceName, "/%s", swarmVar->name );
+
+ #if H5_VERS_MAJOR == 1 && H5_VERS_MINOR < 8
+ self->fileData[swarmVar_I] = H5Dopen( file, dataSpaceName );
+ #else
+ self->fileData[swarmVar_I] = H5Dopen( file, dataSpaceName, H5P_DEFAULT );
+ #endif
+ self->fileSpace[swarmVar_I] = H5Dget_space( self->fileData[swarmVar_I] );
+
+ Variable_Update( swarmVar->variable );
+ }
+
+ self->start[1] = 0;
+ self->count[0] = 1;
+
_GlobalParticleLayout_InitialiseParticles( self, _swarm );
- H5Sclose( self->memSpace );
- H5Sclose( self->fileSpace );
- H5Dclose( self->fileData );
- H5Fclose( file );
+ /* Close dataspaces and the file */
+ for( swarmVar_I = 0; swarmVar_I < swarm->swarmVariable_Register->objects->count; swarmVar_I++ ) {
+ H5Sclose( self->fileSpace[swarmVar_I] );
+ H5Dclose( self->fileData[swarmVar_I] );
+ }
+ H5Fclose( file );
+
+ Memory_Free( self->fileData );
+ Memory_Free( self->fileSpace );
+
#else
self->file = fopen( self->filename, "rb" );
Journal_Firewall(
@@ -340,7 +385,6 @@ void _FileParticleLayout_InitialiseParti
_GlobalParticleLayout_InitialiseParticles( self, _swarm );
fclose( self->file );
- self->file = NULL;
#endif
}
@@ -350,19 +394,76 @@ void _FileParticleLayout_InitialiseParti
Particle_Index newParticle_I,
void* particle )
{
- FileParticleLayout* self = (FileParticleLayout*)particleLayout;
- Swarm* swarm = (Swarm*)_swarm;
- SizeT particleSize = swarm->particleExtensionMgr->finalSize;
+ FileParticleLayout* self = (FileParticleLayout*)particleLayout;
+ Swarm* swarm = (Swarm*)_swarm;
+ SizeT particleSize = swarm->particleExtensionMgr->finalSize;
int result;
-
-#ifdef HAVE_HDF5
- /* Update the hyperslab. */
+ IntegrationPoint* newParticle = (IntegrationPoint*)particle;
+
+#ifdef HAVE_HDF5
+ SwarmVariable* swarmVar;
+ Index swarmVar_I;
+ hid_t memSpace;
+
self->start[0] = newParticle_I;
- H5Sselect_hyperslab( self->fileSpace, H5S_SELECT_SET, self->start, NULL, self->count, NULL );
-
- /* Read particle data. */
- H5Dread( self->fileData, H5T_NATIVE_CHAR, self->memSpace,
- self->fileSpace, H5P_DEFAULT, particle );
+
+ for( swarmVar_I = 0; swarmVar_I < swarm->nSwarmVars; swarmVar_I++ ) {
+ swarmVar = SwarmVariable_Register_GetByIndex( swarm->swarmVariable_Register, swarmVar_I );
+
+ /* Update the hyperslab. */
+ self->count[1] = swarmVar->dofCount;
+ memSpace = H5Screate_simple( 2, self->count, NULL );
+ H5Sselect_hyperslab( self->fileSpace[swarmVar_I], H5S_SELECT_SET, self->start, NULL, self->count, NULL );
+ H5Sselect_all( memSpace );
+
+ /* Treat the data differently depending on its type */
+ if( swarmVar->variable->dataTypes[0] == Variable_DataType_Int ) {
+ int* particleInfo = Memory_Alloc_Array( int, swarmVar->dofCount, "particleCheckpointInfo" );
+
+ /* Read particle data. */
+ H5Dread( self->fileData[swarmVar_I], H5T_NATIVE_INT, memSpace, self->fileSpace[swarmVar_I], H5P_DEFAULT, particleInfo );
+
+ Variable_SetValue( swarmVar->variable, swarm->particleLocalCount, particleInfo );
+
+ Memory_Free( particleInfo );
+ }
+
+ else if( swarmVar->variable->dataTypes[0] == Variable_DataType_Char) {
+ char* particleInfo = Memory_Alloc_Array( char, swarmVar->dofCount, "particleCheckpointInfo" );
+
+ /* Read particle data. */
+ H5Dread( self->fileData[swarmVar_I], H5T_NATIVE_CHAR, memSpace, self->fileSpace[swarmVar_I], H5P_DEFAULT, particleInfo );
+
+ Variable_SetValue( swarmVar->variable, swarm->particleLocalCount, particleInfo );
+
+ Memory_Free( particleInfo );
+ }
+
+ else if( swarmVar->variable->dataTypes[0] == Variable_DataType_Float ) {
+ float* particleInfo = Memory_Alloc_Array( float, swarmVar->dofCount, "particleCheckpointInfo" );
+
+ /* Read particle data. */
+ H5Dread( self->fileData[swarmVar_I], H5T_NATIVE_FLOAT, memSpace, self->fileSpace[swarmVar_I], H5P_DEFAULT, particleInfo );
+
+ Variable_SetValue( swarmVar->variable, swarm->particleLocalCount, particleInfo );
+
+ Memory_Free( particleInfo );
+ }
+
+ else {
+ double* particleInfo = Memory_Alloc_Array( double, swarmVar->dofCount, "particleCheckpointInfo" );
+
+ /* Read particle data. */
+ H5Dread( self->fileData[swarmVar_I], H5T_NATIVE_DOUBLE, memSpace, self->fileSpace[swarmVar_I], H5P_DEFAULT, particleInfo );
+
+ Variable_SetValue( swarmVar->variable, swarm->particleLocalCount, particleInfo );
+
+ Memory_Free( particleInfo );
+ }
+
+ H5Sclose( memSpace );
+ }
+
#else
result = fread( particle, particleSize, 1, self->file );
diff -r f9b63e9cc42b -r 4bc0dd9e4e84 Swarm/src/FileParticleLayout.h
--- a/Swarm/src/FileParticleLayout.h Tue Jul 22 02:14:00 2008 +0000
+++ b/Swarm/src/FileParticleLayout.h Thu Jul 24 00:07:13 2008 +0000
@@ -51,11 +51,9 @@
__GlobalParticleLayout \
\
Name filename; \
- FILE* file; \
Stream* errorStream; \
- hid_t fileData; \
- hid_t fileSpace; \
- hid_t memSpace; \
+ hid_t* fileData; \
+ hid_t* fileSpace; \
hsize_t start[2]; \
hsize_t count[2];
#else
diff -r f9b63e9cc42b -r 4bc0dd9e4e84 Swarm/src/SwarmDump.c
--- a/Swarm/src/SwarmDump.c Tue Jul 22 02:14:00 2008 +0000
+++ b/Swarm/src/SwarmDump.c Thu Jul 24 00:07:13 2008 +0000
@@ -41,6 +41,7 @@
#include "StandardParticle.h"
#include "SwarmDump.h"
#include "SwarmVariable.h"
+#include "SwarmVariable_Register.h"
#include <assert.h>
#include <string.h>
@@ -286,7 +287,7 @@ void _SwarmDump_Execute( void* swarmDump
SwarmDump_DumpToHDF5( self, swarm, filename );
#else
Stg_asprintf( &filename, "%s.dat", filename );
- Stream_RedirectFile( stream, filename );
+ Stream_RedirectFile( stream, filename );
MPIStream_WriteAllProcessors( stream, swarm->particles, particleSize, (SizeT) particleLocalCount, swarm->comm );
Stream_CloseFile( stream );
#endif
@@ -309,51 +310,52 @@ void SwarmDump_Execute( void* swarmDump,
#ifdef HAVE_HDF5
void SwarmDump_DumpToHDF5( SwarmDump* self, Swarm* swarm, const char* filename ) {
- hid_t file, fileSpace, fileData;
- hid_t memSpace;
- hid_t props;
- hsize_t size[2];
- int intSize[2];
- int rank, nRanks, offset;
- hsize_t start[2], count[2];
-
+ hid_t file, fileSpace, fileData;
+ hid_t memSpace;
+ hid_t props;
+ hsize_t size[2];
+ int intSize;
+ int rank, nRanks, offset;
+ hsize_t start[2], count[2], slab[2];
+ Particle_Index lParticle_I = 0;
+ Particle_Index particleGlobalCount;
+ Stream* errorStr = Journal_Register( Error_Type, self->type );
+ SwarmVariable* swarmVar;
+ Index swarmVar_I, dof_I;
+ double* value;
+ unsigned maxDofs = 0;
+ char dataSpaceName[1024];
+
+ /* Caluclate the maximum dofCount of all the swarmVariables */
+ for( swarmVar_I = 0; swarmVar_I < swarm->swarmVariable_Register->objects->count; swarmVar_I++ ) {
+ swarmVar = SwarmVariable_Register_GetByIndex( swarm->swarmVariable_Register, swarmVar_I );
+ if( swarmVar->dofCount > maxDofs )
+ maxDofs = swarmVar->dofCount;
+ }
+
+ /* Allocate space for the values to be written to file */
+ value = Memory_Alloc_Array( double, maxDofs, "swarmVariableValue" );
+
/* Create parallel file property list. */
props = H5Pcreate( H5P_FILE_ACCESS );
H5Pset_fapl_mpio( props, MPI_COMM_WORLD, MPI_INFO_NULL );
/* Open the HDF5 output file. */
file = H5Fcreate( filename, H5F_ACC_TRUNC, H5P_DEFAULT, props );
- assert( file );
+ Journal_Firewall(
+ file >= 0,
+ errorStr,
+ "Error in %s for %s '%s' - Cannot create file %s.\n",
+ __func__,
+ self->type,
+ self->name,
+ filename );
+
H5Pclose( props );
- /* Dump the size so we don't have to do any divisions later on. */
- size[0] = (hsize_t)2;
- fileSpace = H5Screate_simple( 1, size, NULL );
-#if H5_VERS_MAJOR == 1 && H5_VERS_MINOR < 8
- fileData = H5Dcreate( file, "/size", H5T_NATIVE_INT, fileSpace, H5P_DEFAULT );
-#else
- fileData = H5Dcreate( file, "/size", H5T_NATIVE_INT, fileSpace,
- H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT );
-#endif
- MPI_Allreduce( &swarm->particleLocalCount, intSize, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD );
- intSize[1] = swarm->particleExtensionMgr->finalSize;
- props = H5Pcreate( H5P_DATASET_XFER );
- H5Pset_dxpl_mpio( props, H5FD_MPIO_COLLECTIVE );
- H5Dwrite( fileData, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, props, intSize );
- H5Pclose( props );
- H5Dclose( fileData );
- H5Sclose( fileSpace );
-
/* Create our output space and data objects. */
- size[0] = intSize[0];
- size[1] = intSize[1];
- fileSpace = H5Screate_simple( 2, size, NULL );
-#if H5_VERS_MAJOR == 1 && H5_VERS_MINOR < 8
- fileData = H5Dcreate( file, "/data", H5T_NATIVE_CHAR, fileSpace, H5P_DEFAULT );
-#else
- fileData = H5Dcreate( file, "/data", H5T_NATIVE_CHAR, fileSpace,
- H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT );
-#endif
+ MPI_Allreduce( &swarm->particleLocalCount, &particleGlobalCount, 1, MPI_UNSIGNED, MPI_SUM, swarm->comm );
+ size[0] = particleGlobalCount;
/* Calculate our file offset. */
MPI_Comm_rank( MPI_COMM_WORLD, &rank );
@@ -363,26 +365,59 @@ void SwarmDump_DumpToHDF5( SwarmDump* se
MPI_Status status;
MPI_Recv( &offset, 1, MPI_INT, rank - 1, 455, MPI_COMM_WORLD, &status );
}
- start[0] = offset; start[1] = 0;
- count[0] = swarm->particleLocalCount; count[1] = intSize[1];
+ start[0] = offset; start[1] = 0;
+ count[0] = 1; count[1] = maxDofs;
offset += swarm->particleLocalCount;
if( rank < nRanks - 1 )
MPI_Send( &offset, 1, MPI_INT, rank + 1, 455, MPI_COMM_WORLD );
- /* Create our memory space. */
- memSpace = H5Screate_simple( 2, count, NULL );
-
- /* Dump our local data. */
- H5Sselect_hyperslab( fileSpace, H5S_SELECT_SET, start, NULL, count, NULL );
- H5Sselect_all( memSpace );
props = H5Pcreate( H5P_DATASET_XFER );
H5Pset_dxpl_mpio( props, H5FD_MPIO_INDEPENDENT );
- H5Dwrite( fileData, H5T_NATIVE_CHAR, memSpace, fileSpace, props, swarm->particles );
+
+ /* Loop through the swarmVariable_Register */
+ for( swarmVar_I = 0; swarmVar_I < swarm->swarmVariable_Register->objects->count; swarmVar_I++ ) {
+ swarmVar = SwarmVariable_Register_GetByIndex( swarm->swarmVariable_Register, swarmVar_I );
+
+ size[1] = swarmVar->dofCount;
+ fileSpace = H5Screate_simple( 2, size, NULL );
+ count[1] = size[1];
+ slab[0] = start[0];
+ slab[1] = start[1];
+
+ /* Create our memory space. */
+ memSpace = H5Screate_simple( 2, count, NULL );
+
+ /* Create a new dataspace */
+ sprintf( dataSpaceName, "/%s", swarmVar->name );
+
+ #if H5_VERS_MAJOR == 1 && H5_VERS_MINOR < 8
+ fileData = H5Dcreate( file, dataSpaceName, H5T_NATIVE_DOUBLE, fileSpace, H5P_DEFAULT );
+ #else
+ fileData = H5Dcreate( file, dataSpaceName, H5T_NATIVE_DOUBLE, fileSpace,
+ H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT );
+ #endif
+
+ /* Loop through local particles */
+ for( lParticle_I=0; lParticle_I < swarm->particleLocalCount; lParticle_I++ ) {
+ H5Sselect_hyperslab( fileSpace, H5S_SELECT_SET, slab, NULL, count, NULL );
+ H5Sselect_all( memSpace );
+
+ /* Write the value of the current swarmVariable at the current particle to the dataspace */
+ SwarmVariable_ValueAt( swarmVar, lParticle_I, value );
+ H5Dwrite( fileData, H5T_NATIVE_DOUBLE, memSpace, fileSpace, props, value );
+
+ slab[0] += 1;
+ }
+
+ /* Close the dataspace */
+ H5Dclose( fileData );
+ H5Sclose( fileSpace );
+ }
+
+ /* Close off all our handles. */
H5Pclose( props );
-
- /* Close off all our handles. */
- H5Dclose( fileData );
- H5Sclose( fileSpace );
H5Fclose( file );
+
+ Memory_Free( value );
}
#endif
More information about the CIG-COMMITS
mailing list