[cig-commits] r19602 - in seismo/3D/FAULT_SOURCE/branches/new_fault_db: decompose_mesh_SCOTCH src

[ampuero at geodynamics.org]

Author: ampuero
Date: 2012-02-07 22:56:57 -0800 (Tue, 07 Feb 2012)
New Revision: 19602

Modified:
   seismo/3D/FAULT_SOURCE/branches/new_fault_db/decompose_mesh_SCOTCH/decompose_mesh_SCOTCH.f90
   seismo/3D/FAULT_SOURCE/branches/new_fault_db/decompose_mesh_SCOTCH/fault_scotch.f90
   seismo/3D/FAULT_SOURCE/branches/new_fault_db/src/fault_solver.f90
Log:
started fault parallelization

Modified: seismo/3D/FAULT_SOURCE/branches/new_fault_db/decompose_mesh_SCOTCH/decompose_mesh_SCOTCH.f90
===================================================================
--- seismo/3D/FAULT_SOURCE/branches/new_fault_db/decompose_mesh_SCOTCH/decompose_mesh_SCOTCH.f90	2012-02-08 04:36:19 UTC (rev 19601)
+++ seismo/3D/FAULT_SOURCE/branches/new_fault_db/decompose_mesh_SCOTCH/decompose_mesh_SCOTCH.f90	2012-02-08 06:56:57 UTC (rev 19602)
@@ -392,6 +392,7 @@
     if (allocated(faults)) then 
       call save_nodes_coords(nodes_coords,nnodes)
       call close_faults(nodes_coords,elmnts,nspec,nnodes,esize)    
+      call reorder_fault_elements(nodes_coords,nnodes)
     end if 
 
   end subroutine read_mesh_files

Modified: seismo/3D/FAULT_SOURCE/branches/new_fault_db/decompose_mesh_SCOTCH/fault_scotch.f90
===================================================================
--- seismo/3D/FAULT_SOURCE/branches/new_fault_db/decompose_mesh_SCOTCH/fault_scotch.f90	2012-02-08 04:36:19 UTC (rev 19601)
+++ seismo/3D/FAULT_SOURCE/branches/new_fault_db/decompose_mesh_SCOTCH/fault_scotch.f90	2012-02-08 06:56:57 UTC (rev 19602)
@@ -19,7 +19,7 @@
   double precision, parameter :: FAULT_GAP_TOLERANCE = 1.0d0
 
   public :: read_fault_files, fault_repartition, close_faults, write_fault_database, &
-            save_nodes_coords, nodes_coords_open, faults
+            save_nodes_coords, nodes_coords_open, faults, reorder_fault_elements
 
 CONTAINS 
 !==========================================================================================
@@ -203,7 +203,43 @@
   
   end subroutine close_fault_single
 
+!===================================================================================================
+  subroutine reorder_fault_elements(nodes_coords,nnodes)
+!JPA  ... 
+  do iflt=1,size(faults)
+    call reorder_fault_elements_single(faults(iflt),&
+                            elmnts,nodes_coords,nnodes,esize,nelmnts)
+  enddo
+
+  end subroutine reorder_fault_elements
+
 ! ---------------------------------------------------------------------------------------------------
+  subroutine reorder_fault_elements_single(f)
+
+  ! compute element-face centers for fault side 1
+  do i=1,f%nspec
+    e = f%ispec1(i)
+    do k=1,4
+      iglob = f%inodes1(k,e)
+      xyz(:,k) = nodes_coords(:,iglob)
+    enddo
+    xyz_c(:,k) = ! ... average of xyz
+  enddo
+  ! reorder
+  call lex_order(xyz_c,new_index_list) 
+  f%ispec1 = f%ispec1(new_index_list)
+
+  ! repeat for fault side 2
+  ! ....
+
+  end subroutine reorder_fault_elements_single
+  
+! ---------------------------------------------------------------------------------------------------
+  subroutine lex_order()
+! JPA lexicographic ordering inspired from src/get_global.f90
+  end subroutine lex_order
+
+!===================================================================================================
   !--------------------------------------------------
   ! Repartitioning : two coupled faultside1/side2 elements are transfered to the same partition
   !--------------------------------------------------
@@ -298,6 +334,32 @@
   end subroutine fault_repartition
 
 ! ---------------------------------------------------------------------------------------------------
+! JPA need to call this one now
+  subroutine fault_repartition_parallel (nelmnts, nnodes, elmnts, nsize, &
+                        nproc, part, esize)
+
+!JPA loop over all faults
+!JPA   loop over all fault element pairs
+!JPA     assign the pair of elements to the processor with lowest rank among them
+ 
+  do i=1,2 !JPA repeated to handle triple junctions
+
+  do iflt=1,size(faults)
+    do e=1,faults(iflt)%nspec
+      e1 = faults(iflt)%ispec1(e)
+      e2 = faults(iflt)%ispec2(e)
+      proc1 = part(e1)
+      proc2 = part(e2)
+      part(e1) = min(proc1,proc2)
+      part(e2) = part(e1)
+    end do
+  end do
+
+  end do
+
+  end subroutine fault_repartition_parallel
+
+! ---------------------------------------------------------------------------------------------------
 ! See subroutine write_boundaries_database in part_decompose_mesh_SCOTCH.f90
 !
 ! File format: 

Modified: seismo/3D/FAULT_SOURCE/branches/new_fault_db/src/fault_solver.f90
===================================================================
--- seismo/3D/FAULT_SOURCE/branches/new_fault_db/src/fault_solver.f90	2012-02-08 04:36:19 UTC (rev 19601)
+++ seismo/3D/FAULT_SOURCE/branches/new_fault_db/src/fault_solver.f90	2012-02-08 06:56:57 UTC (rev 19602)
@@ -269,6 +269,12 @@
       bc%B(k) = bc%B(k) + jacobian2Dw(ij,e)
     enddo
   enddo
+
+!JPA assemble bc%B across processors
+!JPA 1. put bc%B in the vector 'accel', which is defined in every GLL node including in the bulk (contains zeros off the fault)
+!JPA 2. call assemble_MPI_vector_* (or maybe assemble_MPI_scalar_*)
+!JPA do the same for bc%n
+
   do k=1,bc%nglob
     norm = sqrt( nx(k)*nx(k) + ny(k)*ny(k) + nz(k)*nz(k) )
     nx(k) = nx(k) / norm