[cig-commits] [commit] master: Fix parallel Greens output (f24f2e3)

[cig_noreply at geodynamics.org]

Repository : https://github.com/geodynamics/vq

On branch  : master
Link       : https://github.com/geodynamics/vq/compare/0aca7bf57ea151a65788e0106b736313ec48b926...4a71dae87966b80bfee7c194773543359a343617

>---------------------------------------------------------------

commit f24f2e351cbc33ad4ca3cf89ba066fc07334a4d3
Author: Eric Heien <emheien at ucdavis.edu>
Date:   Thu Jan 22 11:24:32 2015 -0800

    Fix parallel Greens output


>---------------------------------------------------------------

f24f2e351cbc33ad4ca3cf89ba066fc07334a4d3
 src/io/GreensFileOutput.cpp | 25 ++++++++++++++++++++-----
 src/io/HDF5Data.cpp         | 13 +++++++++----
 2 files changed, 29 insertions(+), 9 deletions(-)

diff --git a/src/io/GreensFileOutput.cpp b/src/io/GreensFileOutput.cpp
index 3deecf2..e1e0552 100644
--- a/src/io/GreensFileOutput.cpp
+++ b/src/io/GreensFileOutput.cpp
@@ -60,12 +60,27 @@ void GreensFileOutput::init(SimFramework *_sim) {
     shear_vals = new double[green_dim];
     norm_vals = new double[green_dim];
     
-    for (row=0; row<sim->numLocalBlocks(); ++row) {
-        global_row = sim->getGlobalBID(row);
+    // If the number of rows isn't the same on all processes, it will deadlock
+    // Have each process call the writing function an equal number of times,
+    // but use UNDEFINED_ELEMENT_ID if there's nothing new to write
+    int local_num_rows = sim->numLocalBlocks();
+    int global_max_rows;
+#ifdef MPI_C_FOUND
+    MPI_Allreduce(&local_num_rows, &global_max_rows, 1, MPI_INT, MPI_MAX, MPI_COMM_WORLD);
+#else
+    global_max_rows = local_num_rows;
+#endif
+
+    for (row=0; row<global_max_rows; ++row) {
+        if (row >= local_num_rows) {
+            global_row = UNDEFINED_ELEMENT_ID;
+        } else {
+            global_row = sim->getGlobalBID(row);
             
-        for (col=0; col<green_dim; ++col) {
-            shear_vals[col] = sim->getGreenShear(global_row, col);
-            norm_vals[col] = sim->getGreenNormal(global_row, col);
+            for (col=0; col<green_dim; ++col) {
+                shear_vals[col] = sim->getGreenShear(global_row, col);
+                norm_vals[col] = sim->getGreenNormal(global_row, col);
+            }
         }
 
         h5_greens_data->setGreensVals(global_row, shear_vals, norm_vals);
diff --git a/src/io/HDF5Data.cpp b/src/io/HDF5Data.cpp
index f74d770..afd7ffc 100644
--- a/src/io/HDF5Data.cpp
+++ b/src/io/HDF5Data.cpp
@@ -271,11 +271,16 @@ void HDF5GreensDataWriter::setGreensVals(const int &bid, const double *shear_val
     file_select = H5Scopy(green_dataspace);
     mem_select = H5Scopy(green_dataspace);
 
-    file_start[0] = bid;                // start at xth block
-    file_start[1] = 0;
+    if (bid == UNDEFINED_ELEMENT_ID) {
+        file_start[0] = file_start[1] = 0;
+        count[0] = count[1] = 0;
+    } else {
+        file_start[0] = bid;                // start at xth block
+        file_start[1] = 0;
+        count[0] = 1;                       // 1xN set of values
+        count[1] = greens_dim;
+    }
     mem_start[0] = mem_start[1] = 0;    // start at element 0 in memory array
-    count[0] = 1;                       // 1xN set of values
-    count[1] = greens_dim;
 
     // Select the hyperslabs for the memory and file dataspace
     status = H5Sselect_hyperslab(file_select, H5S_SELECT_SET, file_start, NULL, count, NULL);