[cig-commits] [commit] master: Ensure consistent ordering of blocks in MPI (f6702e5)

[cig_noreply at geodynamics.org]

Repository : https://github.com/geodynamics/vc

On branch  : master
Link       : https://github.com/geodynamics/vc/compare/5f229988c0a5e4aeea24d768568f934a3f30e13a...3d22170bd08cd637af7a93121c8b3719dea0bb1d

>---------------------------------------------------------------

commit f6702e5a9c0ea90bbba112bd372aa6df3b022137
Author: Eric Heien <eheien at blast-thickneck.attlocal.net>
Date:   Tue Sep 16 23:40:52 2014 -0700

    Ensure consistent ordering of blocks in MPI


>---------------------------------------------------------------

f6702e5a9c0ea90bbba112bd372aa6df3b022137
 src/core/VCComm.cpp                  | 6 ++++++
 src/simulation/RunEvent.cpp          | 2 ++
 src/simulation/UpdateBlockStress.cpp | 4 ++++
 3 files changed, 12 insertions(+)

diff --git a/src/core/VCComm.cpp b/src/core/VCComm.cpp
index 623a063..e4c20c0 100644
--- a/src/core/VCComm.cpp
+++ b/src/core/VCComm.cpp
@@ -26,6 +26,7 @@
 // Note: We assume these operations are only called for the BlockVal MPI datatype
 /*!
  Calculates the minimum value and its associated block among the specified input array.
+ If two blocks have the same value, order by block ID.
  */
 void BlockValMinimum(void *invec, void *inoutvec, int *len, MPI_Datatype *datatype) {
     BlockVal        *in, *out;
@@ -38,6 +39,8 @@ void BlockValMinimum(void *invec, void *inoutvec, int *len, MPI_Datatype *dataty
         if (in[i].val < out[i].val) {
             out[i].val = in[i].val;
             out[i].block_id = in[i].block_id;
+        } else if (in[i].val == out[i].val) {
+            out[i].block_id = (in[i].block_id < out[i].block_id ? in[i].block_id : out[i].block_id);
         }
 
         // else leave the out array as it is
@@ -46,6 +49,7 @@ void BlockValMinimum(void *invec, void *inoutvec, int *len, MPI_Datatype *dataty
 
 /*!
  Calculates the maximum value and its associated block among the specified input array.
+ If two blocks have the same value, order by block ID.
  */
 void BlockValMaximum(void *invec, void *inoutvec, int *len, MPI_Datatype *datatype) {
     BlockVal        *in, *out;
@@ -58,6 +62,8 @@ void BlockValMaximum(void *invec, void *inoutvec, int *len, MPI_Datatype *dataty
         if (in[i].val > out[i].val) {
             out[i].val = in[i].val;
             out[i].block_id = in[i].block_id;
+        } else if (in[i].val == out[i].val) {
+            out[i].block_id = (in[i].block_id < out[i].block_id ? in[i].block_id : out[i].block_id);
         }
 
         // else leave the out array as it is
diff --git a/src/simulation/RunEvent.cpp b/src/simulation/RunEvent.cpp
index 67af168..673f460 100644
--- a/src/simulation/RunEvent.cpp
+++ b/src/simulation/RunEvent.cpp
@@ -321,6 +321,8 @@ SimRequest RunEvent::run(SimFramework *_sim) {
             sim->setUpdateField(gid, (block.getFailed() ? 0 : block.state.slipDeficit));
         }
 
+        sim->distributeUpdateField();
+        
         // Calculate the new shear stresses and CFFs given the new update field values
         sim->matrixVectorMultiplyAccum(sim->getShearStressPtr(),
                                        sim->greenShear(),
diff --git a/src/simulation/UpdateBlockStress.cpp b/src/simulation/UpdateBlockStress.cpp
index 51c676f..7718f1b 100644
--- a/src/simulation/UpdateBlockStress.cpp
+++ b/src/simulation/UpdateBlockStress.cpp
@@ -167,9 +167,13 @@ void UpdateBlockStress::nextTimeStep(BlockVal &fail_time) {
         if (ts <= 0) continue;
 
         // If the time to slip is less than the current shortest time, record the block
+        // To ensure reproducibility with multiple processes, if multiple blocks fail
+        // at the same time then we choose the block with the lowest ID over all the processes
         if (ts < temp_block_fail.val) {
             temp_block_fail.block_id = gid;
             temp_block_fail.val = ts;
+        } else if (ts == temp_block_fail.val) {
+            temp_block_fail.block_id = (gid < temp_block_fail.block_id ? gid : temp_block_fail.block_id);
         }
     }