[cig-commits] r20530 - in seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER: EXAMPLES/global_s362ani_shakemovie/DATA UTILS setup src/auxiliaries src/cuda src/meshfem3D src/shared src/specfem3D
danielpeter at geodynamics.org
danielpeter at geodynamics.org
Thu Jul 19 15:05:23 PDT 2012
Author: danielpeter
Date: 2012-07-19 15:05:23 -0700 (Thu, 19 Jul 2012)
New Revision: 20530
Added:
seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/cuda/initialize_cuda.cu
Modified:
seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/global_s362ani_shakemovie/DATA/STATIONS
seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/UTILS/create_specfem3D_gpu_cuda_method_stubs.pl
seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/setup/constants.h.in
seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/auxiliaries/combine_vol_data_vtk.f90
seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/cuda/check_fields_cuda.cu
seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/cuda/compute_coupling_cuda.cu
seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/cuda/compute_forces_crust_mantle_cuda.cu
seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/cuda/compute_forces_inner_core_cuda.cu
seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/cuda/it_update_displacement_cuda.cu
seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/cuda/prepare_mesh_constants_cuda.cu
seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/cuda/specfem3D_gpu_cuda_method_stubs.c
seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/meshfem3D/Makefile.in
seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/meshfem3D/compute_area.f90
seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/meshfem3D/compute_element_properties.f90
seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/meshfem3D/create_MPI_interfaces.f90
seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/meshfem3D/create_addressing.f90
seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/meshfem3D/create_chunk_buffers.f90
seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/meshfem3D/create_mass_matrices.f90
seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/meshfem3D/create_meshes.f90
seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/meshfem3D/create_regions_mesh.f90
seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/meshfem3D/finalize_mesher.f90
seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/meshfem3D/get_global.f90
seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/meshfem3D/get_perm_color.f90
seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/meshfem3D/initialize_mesher.f90
seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/meshfem3D/meshfem3D_models.f90
seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/meshfem3D/meshfem3D_par.f90
seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/meshfem3D/model_attenuation.f90
seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/meshfem3D/save_arrays_solver.f90
seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/meshfem3D/setup_MPI_interfaces.f90
seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/meshfem3D/setup_color_perm.f90
seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/shared/exit_mpi.f90
seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/shared/read_compute_parameters.f90
seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/Makefile.in
seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/check_simulation_stability.f90
seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/compute_coupling.f90
seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/compute_forces_acoustic.F90
seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/compute_forces_crust_mantle.f90
seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/compute_forces_crust_mantle_Dev.F90
seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/compute_forces_elastic.F90
seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/compute_forces_inner_core.f90
seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/compute_forces_inner_core_Dev.F90
seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/finalize_simulation.f90
seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/initialize_simulation.f90
seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/iterate_time.F90
seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/locate_receivers.f90
seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/locate_sources.f90
seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/prepare_timerun.f90
seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/read_arrays_solver.f90
seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/read_mesh_databases.f90
seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/read_topography_bathymetry.f90
seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/specfem3D_par.F90
seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/write_output_SAC.f90
seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/write_seismograms.f90
Log:
updates mesher routines and create_regions_mesh() modules; adds initialize_cuda.cu file
Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/global_s362ani_shakemovie/DATA/STATIONS
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/global_s362ani_shakemovie/DATA/STATIONS 2012-07-19 21:55:39 UTC (rev 20529)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/EXAMPLES/global_s362ani_shakemovie/DATA/STATIONS 2012-07-19 22:05:23 UTC (rev 20530)
@@ -1,6 +1,1838 @@
+DLV AD 11.9520 108.4815 50.0 0.0
+SIM AD 20.1332 92.8852 50.0 0.0
+SLV AD 21.3338 103.9050 50.0 0.0
+SZP AD 17.5520 120.4550 50.0 0.0
+CNG AF -26.2917 32.1883 100.0 0.0
+CVNA AF -31.4821 19.7617 1050.0 0.0
+GRM AF -33.3133 26.5733 610.0 0.0
+GSN AF -22.5667 17.1000 1728.0 0.0
+HVD AF -30.6050 25.4967 1342.0 0.0
+KIG AF -1.9634 30.0633 1545.0 0.0
+MOPA AF -23.5173 31.3977 362.0 0.0
+PKA AF -29.6700 22.7570 290.0 0.0
+POGA AF -28.3233 27.6250 1486.0 0.0
+SEK AF -28.3233 27.6250 1486.0 0.0
+SWZ AF -27.1817 25.3250 1342.0 0.0
+UPI AF -28.3620 21.2530 845.0 0.0
+WIN AF -22.5667 17.1000 1728.0 0.0
+CTAO AS -20.0882 146.2545 357.0 37.0
+KAAO AS 34.5407 69.0428 1920.0 7.0
+KONO AS 59.6491 9.5982 216.0 340.0
+MAJO AS 36.5409 138.2083 431.0 48.0
+ZOBO AS -16.2700 -68.1250 4450.0 300.0
+BVA2 AZ 33.3265 -116.3655 0.0 0.0
+BVDA2 AZ 33.3265 -116.3655 0.0 0.0
+BZN AZ 33.4915 -116.6670 1301.0 0.0
+CPE AZ 32.8889 -117.1051 150.0 0.0
+CRON AZ 33.6118 -116.4597 1268.0 0.0
+CRY AZ 33.5654 -116.7373 1128.0 0.0
+FLV2 AZ 32.9114 -117.0619 278.0 0.0
+FRD AZ 33.4947 -116.6022 1164.0 0.0
+FSE AZ 33.0057 -116.5926 1708.0 0.0
+GLA AZ 33.0512 -114.8270 579.0 0.0
+HWB AZ 33.0262 -116.9596 478.0 0.0
+JOHN AZ 33.6118 -116.4597 1268.0 0.0
+JOSE AZ 33.6118 -116.4597 1268.0 0.0
+KNW AZ 33.7141 -116.7119 1507.0 0.0
+LVA2 AZ 33.3516 -116.5615 1435.0 0.0
+MONP2 AZ 32.8920 -116.4223 1000.0 0.0
+MONP AZ 32.8927 -116.4225 1920.0 0.0
+PFO AZ 33.6117 -116.4594 1259.0 0.0
+RDM AZ 33.6300 -116.8478 1365.0 0.0
+SCV1 AZ 33.6118 -116.4597 1268.0 0.0
+SCV2 AZ 33.6118 -116.4597 1268.0 0.0
+SMER2 AZ 33.4577 -117.1708 355.0 0.0
+SMTC AZ 32.9449 -115.7999 100.0 0.0
+SND AZ 33.5519 -116.6129 1358.0 0.0
+SOL AZ 32.8410 -117.2480 245.0 0.0
+ST2C AZ 33.6118 -116.4597 1268.0 0.0
+STS2 AZ 33.6118 -116.4597 1268.0 0.0
+TRIL AZ 33.6118 -116.4597 1268.0 0.0
+TRO AZ 33.5234 -116.4257 2628.0 0.0
+WMC AZ 33.5736 -116.6747 1271.0 0.0
+YAQ AZ 33.1666 -116.3539 430.0 0.0
+MEM BE 50.6092 6.0067 250.0 0.0
+BFO BF 48.3319 8.3311 589.0 150.0
+ARC BK 40.8777 -124.0774 30.1 0.0
+BDM BK 37.9540 -121.8655 219.8 34.7
+BKS BK 37.8762 -122.2356 243.9 25.6
+BRIB BK 37.9188 -122.1518 218.9 3.3
+BRK BK 37.8735 -122.2610 49.4 2.7
+CMB BK 38.0346 -120.3865 697.0 2.0
+CVS BK 38.3453 -122.4584 295.1 23.2
+ELFS BK 40.6183 -120.7279 1553.8 0.7
+FARB BK 37.6978 -123.0011 -18.5 0.0
+GASB BK 39.6547 -122.7160 1354.8 1.5
+HAST BK 36.3887 -121.5514 539.5 2.5
+HATC BK 40.8161 -121.4612 1008.6 0.7
+HELL BK 36.6801 -119.0228 1137.5 2.5
+HOPS BK 38.9935 -123.0723 299.1 3.0
+HUMO BK 42.6071 -122.9567 554.9 50.0
+JAS1 BK 37.9266 -120.4210 398.1 0.0
+JCC BK 40.8175 -124.0296 27.2 0.0
+JRSC BK 37.4037 -122.2387 70.5 0.0
+KCC BK 37.3236 -119.3187 888.1 87.3
+MCCM BK 38.1448 -122.8802 -7.7 4.0
+MHC BK 37.3416 -121.6426 1250.4 0.0
+MIN BK 40.3460 -121.6066 1470.9 0.0
+MNRC BK 38.8787 -122.4428 709.9 6.5
+MOD BK 41.9025 -120.3029 1554.5 5.0
+ORV BK 39.5545 -121.5004 334.7 3.0
+PACP BK 37.0080 -121.2870 844.0 6.5
+PKD1 BK 35.8894 -120.4261 431.6 0.0
+POTR BK 38.2026 -121.9353 20.0 6.5
+RAMR BK 35.6360 -120.8698 414.3 2.5
+SAO BK 36.7640 -121.4472 317.2 3.0
+STAN BK 37.4039 -122.1751 125.5 0.0
+SUTB BK 39.2291 -121.7861 249.5 2.5
+WDC BK 40.5799 -122.5411 268.3 75.0
+WENL BK 37.6221 -121.7570 138.9 30.3
+YBH BK 41.7319 -122.7104 1059.6 50.0
+BJI CD 40.0403 116.1750 43.0 3.0
+ENH CD 30.2718 109.4868 487.0 0.0
+HIA CD 49.2667 119.7417 610.0 15.0
+KMI CD 25.1233 102.7400 1940.0 35.0
+LSA CD 29.7000 91.1500 3774.0 15.0
+LZH CD 36.0867 103.8444 1440.0 120.0
+MDJ CD 44.6164 129.5919 250.0 50.0
+QIZ CD 19.0294 109.8433 230.0 0.0
+SSE CD 31.0956 121.1867 15.0 0.0
+WMQ CD 43.8211 87.6950 897.0 6.0
+ADO CI 34.5505 -117.4339 908.0 0.0
+AGA CI 33.6384 -116.4011 809.0 0.0
+ALP CI 34.6871 -118.2995 753.0 0.0
+ARV CI 35.1269 -118.8301 258.0 0.0
+BAK CI 35.3444 -119.1044 116.0 0.0
+BAR CI 32.6801 -116.6722 529.0 0.0
+BBR CI 34.2623 -116.9207 2069.0 0.0
+BBS CI 33.9214 -116.9806 785.0 0.0
+BC3 CI 33.6551 -115.4537 1137.0 0.0
+BCC CI 33.5751 -117.2612 391.0 0.0
+BCW CI 34.9401 -119.4131 1101.0 1.6
+BEL CI 34.0006 -115.9982 1388.0 0.0
+BFS CI 34.2388 -117.6585 1312.0 0.0
+BHP CI 33.9905 -118.3617 81.0 1.6
+BKR CI 35.2693 -116.0703 275.0 0.0
+BLA2 CI 34.0693 -116.3899 1247.0 1.4
+BLA CI 34.0695 -116.3890 1243.0 0.0
+BLY CI 33.7503 -114.5238 196.0 0.0
+BOM CI 33.3646 -115.7296 -54.0 1.8
+BOR CI 33.2682 -116.4172 252.0 0.0
+BRE CI 33.8078 -117.9812 24.0 0.0
+BTC CI 33.0121 -115.2199 39.0 0.0
+BTP CI 34.6822 -118.5740 1702.0 0.0
+CAP CI 33.3882 -117.1950 285.0 0.0
+CAR CI 35.3082 -119.8458 765.0 0.0
+CCC CI 35.5249 -117.3645 670.0 0.0
+CGO CI 36.5504 -117.8029 2795.0 0.0
+CHF CI 34.3334 -118.0259 1594.0 0.0
+CHN CI 33.9988 -117.6804 208.0 0.0
+CIA CI 33.4019 -118.4150 477.0 0.0
+CIU CI 33.4458 -118.4830 203.0 0.0
+CLC CI 35.8157 -117.5975 775.0 0.0
+CLT CI 34.0928 -117.3169 325.0 0.0
+CPP CI 34.0602 -117.8090 205.0 0.0
+CRP CI 34.1362 -118.1270 233.0 0.0
+CTC CI 33.6552 -115.9901 538.0 0.0
+CWC CI 36.4399 -118.0802 1595.0 0.0
+DAN CI 34.6375 -115.3812 428.0 0.0
+DEC CI 34.2535 -118.3338 519.0 0.0
+DEV CI 33.9360 -116.5779 337.0 0.0
+DGR CI 33.6500 -117.0095 650.0 0.0
+DJJ CI 34.1062 -118.4550 268.0 0.0
+DLA CI 33.8482 -118.0962 14.0 0.0
+DNR CI 33.5667 -116.6306 1274.0 0.0
+DPP CI 32.9986 -116.9415 463.0 0.0
+DRC CI 32.8054 -115.4465 -15.0 0.0
+DRE CI 32.8053 -115.4468 -13.0 0.0
+DSC CI 35.1425 -116.1039 283.0 0.0
+DVT CI 32.6591 -116.1006 881.0 0.0
+EDW2 CI 34.8811 -117.9939 772.0 0.0
+EDW CI 34.8830 -117.9911 789.0 0.0
+EML CI 32.8908 -116.8457 161.0 0.0
+ERR CI 33.1168 -115.8227 -56.0 0.0
+FIG CI 34.7283 -119.9880 945.0 0.0
+FMP CI 33.7126 -118.2938 89.0 0.0
+FPC CI 35.0820 -117.5827 853.0 0.0
+FUR CI 36.4670 -116.8632 -37.0 0.0
+GATR CI 34.5527 -120.5024 590.0 0.0
+GLA CI 33.0515 -114.8271 610.0 0.0
+GMR CI 34.7846 -115.6599 1326.0 0.0
+GOR CI 33.1537 -117.2292 138.0 0.0
+GPO CI 35.6494 -117.6619 735.0 0.0
+GR2 CI 34.1182 -118.3002 342.0 0.0
+GRA CI 36.9961 -117.3662 689.0 0.0
+GSC CI 35.3018 -116.8057 1000.0 0.0
+HEC CI 34.8294 -116.3350 920.0 0.0
+HLL CI 34.1764 -118.3597 193.0 0.0
+IRM CI 34.1574 -115.1451 567.0 0.0
+ISA CI 35.6628 -118.4740 873.0 0.0
+JCS CI 33.0859 -116.5959 1264.0 0.0
+JEM CI 33.0810 -116.5975 1308.0 1.0
+JRC2 CI 35.9825 -117.8089 1469.0 0.0
+JVA CI 34.3662 -116.6127 904.0 0.0
+LAF CI 33.8689 -118.3314 20.0 0.0
+LCG CI 34.0003 -118.3779 108.0 0.0
+LCP CI 34.7355 -120.2800 199.0 0.0
+LDF CI 35.1307 -115.1842 1239.0 0.0
+LDR CI 34.9906 -118.3416 1121.0 0.0
+LFP CI 34.3053 -118.4881 368.0 0.0
+LGB CI 33.9753 -118.1492 42.0 0.0
+LGU CI 34.1082 -119.0659 416.0 0.0
+LJR CI 34.8076 -118.8678 1443.0 0.0
+LKL CI 34.6159 -117.8249 810.0 0.0
+LLS CI 33.6845 -117.9430 8.0 0.0
+LMR2 CI 34.9344 -117.6963 964.0 0.0
+LPC CI 34.3148 -117.5464 1351.0 0.0
+LRL CI 35.4795 -117.6821 1340.0 0.0
+LTP CI 33.8811 -118.1757 20.0 0.0
+LUG CI 34.3656 -117.3668 1140.0 0.0
+MCT CI 34.2264 -116.0407 661.0 0.0
+MGE CI 33.8184 -116.3687 68.0 0.0
+MLAC CI 37.6302 -118.8361 2162.0 0.0
+MLS CI 34.0046 -117.5616 230.0 0.0
+MOP CI 34.2809 -118.9049 143.0 0.0
+MPI CI 34.8126 -119.1453 2673.0 0.0
+MPM CI 36.0580 -117.4890 1839.0 0.0
+MPP CI 34.8885 -119.8136 1739.0 0.0
+MSC CI 34.0385 -116.6479 938.0 1.8
+MSJ CI 33.8080 -116.9679 466.0 0.0
+MTL CI 34.2688 -118.2374 471.0 0.0
+MTP CI 35.4843 -115.5532 1588.0 0.0
+MUR CI 33.6000 -117.1954 562.0 3.0
+MWC CI 34.2236 -118.0583 1725.0 0.0
+NBS CI 34.7803 -116.5580 590.0 0.0
+NEE2 CI 34.7676 -114.6188 271.0 0.0
+NEE CI 34.8249 -114.5994 170.0 0.0
+NJQ CI 34.5341 -120.1774 219.0 0.0
+NSS2 CI 33.5555 -115.9459 17.0 0.0
+OLI CI 33.9454 -117.9237 162.0 0.0
+OLP CI 32.6078 -116.9304 159.0 0.0
+OSI CI 34.6145 -118.7235 718.0 0.0
+PASA CI 34.1484 -118.1711 257.0 0.0
+PASC CI 34.1714 -118.1852 341.0 0.0
+PAS CI 34.1484 -118.1712 314.0 0.0
+PDE CI 34.4420 -118.5821 328.0 0.0
+PDM CI 34.3034 -114.1415 144.0 0.0
+PDR CI 33.9627 -118.4370 45.0 0.0
+PDU CI 34.1207 -117.6381 445.0 0.0
+PER CI 33.8616 -117.2053 468.0 0.0
+PFO CI 33.6116 -116.4586 1288.0 0.0
+PHL CI 35.4077 -120.5456 355.0 0.0
+PLC CI 33.8244 -116.5119 130.0 0.0
+PLM CI 33.3536 -116.8627 1691.0 0.0
+PLS CI 33.7953 -117.6091 1199.0 0.0
+PMD CI 33.6478 -116.3777 296.0 0.0
+PSD CI 33.8239 -116.5503 144.0 0.0
+PSR CI 34.0918 -117.8071 305.0 1.5
+RCT CI 36.3052 -119.2438 107.0 0.0
+RIN CI 34.2820 -118.4792 304.0 0.0
+RIO CI 34.1047 -117.9796 118.0 0.0
+RPV CI 33.7435 -118.4041 107.0 0.0
+RRX CI 34.8753 -116.9968 439.0 0.0
+RSS CI 33.9733 -117.3267 298.0 0.0
+RUS CI 34.0507 -118.0808 70.0 0.0
+RVR CI 33.9935 -117.3755 266.0 0.0
+RXH CI 33.1831 -115.6226 -59.0 0.0
+SAL CI 33.2801 -115.9858 14.0 0.0
+SBB2 CI 34.6885 -117.8245 825.0 0.0
+SBC CI 34.4408 -119.7149 94.0 0.0
+SBI CI 33.4805 -119.0299 41.0 0.0
+SBPX CI 34.2324 -117.2348 1860.0 0.0
+SCI2 CI 32.9799 -118.5470 199.0 0.0
+SCI CI 32.9799 -118.5470 199.0 0.0
+SCZ2 CI 33.9954 -119.6351 440.0 0.0
+SDD CI 33.5526 -117.6617 120.0 0.0
+SDG CI 32.7840 -117.1381 101.0 0.0
+SDP CI 34.5655 -120.5014 631.0 0.0
+SDR CI 32.7356 -116.9424 113.0 0.0
+SES CI 34.4369 -119.1375 480.0 0.0
+SHO CI 35.8995 -116.2753 451.0 0.0
+SLA CI 35.8909 -117.2833 1174.0 0.0
+SLB CI 33.4852 -115.8664 0.0 1.8
+SLR CI 33.8336 -116.7974 1567.0 0.0
+SMB CI 34.9022 -120.4471 74.0 0.0
+SMM CI 35.3142 -119.9958 599.0 0.0
+SMR CI 35.3770 -120.6125 341.0 2.0
+SMS CI 34.0144 -118.4562 43.0 0.0
+SMW CI 35.0111 -120.4100 169.0 1.0
+SNCC CI 33.2479 -119.5244 275.0 0.0
+SNO CI 34.0352 -116.8078 2339.0 0.0
+SOT CI 34.4160 -118.4490 409.0 0.0
+SPF CI 34.0593 -118.6461 470.0 0.0
+SPG2 CI 36.2006 -118.7662 627.0 1.5
+SPG CI 36.1355 -118.8110 314.0 0.0
+SRN CI 33.8285 -117.7894 208.0 0.0
+SSW CI 33.1775 -115.6156 -64.0 0.0
+STC CI 34.3030 -119.1868 180.0 0.0
+STG CI 33.6640 -117.7686 49.0 0.0
+STS CI 33.7903 -118.1988 4.0 0.0
+SVD CI 34.1065 -117.0982 605.0 0.0
+SWS CI 32.9408 -115.7958 140.0 0.0
+SYP CI 34.5277 -119.9783 1278.0 0.0
+TA2 CI 34.3820 -117.6782 2249.0 0.0
+TEH CI 35.2913 -118.4208 846.0 0.0
+TFT CI 35.1459 -119.4195 233.0 0.0
+THM CI 33.6507 -116.0773 60.0 1.5
+THX CI 33.6350 -116.1640 -31.0 0.0
+TIN CI 37.0542 -118.2301 1197.0 0.0
+TOR CI 33.5753 -116.2258 4.0 0.0
+TOV CI 34.1561 -118.8204 298.0 0.0
+TUQ CI 35.4358 -115.9239 1350.0 0.0
+USC CI 34.0192 -118.2863 58.0 0.0
+VCS CI 34.4836 -118.1178 991.0 0.0
+VES CI 35.8409 -119.0847 154.0 0.0
+VTV CI 34.5607 -117.3296 843.0 0.0
+WBS CI 35.5366 -118.1403 1927.0 0.0
+WER CI 35.0605 -119.0271 130.0 0.0
+WES CI 32.7590 -115.7316 -8.0 0.0
+WGR CI 34.5108 -119.2741 554.0 0.0
+WLT CI 34.0095 -117.9508 107.0 0.0
+WSS CI 34.1717 -118.6497 316.0 0.0
+WTT CI 33.9487 -118.2555 39.0 0.0
+WWC CI 33.9407 -116.4088 619.0 1.7
+BBB CN 52.1847 -128.1133 14.0 0.0
+DRLN CN 49.2560 -57.5042 238.0 0.0
+FRB CN 63.7467 -68.5467 18.0 0.0
+GAC CN 45.7032 -75.4783 62.0 100.0
+INK CN 68.3067 -133.5200 40.0 0.0
+LLLB CN 50.6090 -121.8815 700.0 0.0
+RES CN 74.6870 -94.9000 15.0 0.0
+SADO CN 44.7694 -79.1417 243.0 0.0
+SCHQ CN 54.8319 -66.8336 501.0 0.0
+ULM CN 50.2503 -95.8750 250.9 0.0
+WHY CN 60.6597 -134.8806 1292.0 0.0
+YKW1 CN 62.4822 -114.4843 170.7 0.0
+YKW2 CN 62.4337 -114.6038 143.8 0.0
+YKW3 CN 62.5608 -114.6164 200.0 0.0
+YKW4 CN 62.4925 -114.7432 160.6 0.0
+YKW5 CN 0.0000 0.0000 0.0 0.0
+ANWB CU 17.6685 -61.7856 39.0 0.0
+BBGH CU 13.1434 -59.5588 180.0 0.0
+BCIP CU 9.1665 -79.8373 61.0 0.0
+GRGR CU 12.1324 -61.6540 195.0 0.0
+GRTK CU 21.5115 -71.1327 12.0 0.0
+GTBY CU 19.9268 -75.1108 79.0 0.0
+MTDJ CU 18.2261 -77.5345 925.0 0.0
+SDDR CU 18.9821 -71.2878 589.0 0.0
+TGUH CU 14.0570 -87.2730 0.0 0.0
+DPC CZ 50.3583 16.4111 760.0 0.0
+ILULI DK 69.2121 -51.1048 53.5 0.0
+KULLO DK 74.5805 -57.2201 44.0 0.0
+AFI DW -13.9093 -171.7773 706.0 2.0
+ALQ DW 34.9425 -106.4575 1849.0 10.0
+BDF DW -15.6639 -47.9033 1260.0 2.0
+BER DW 60.3838 5.3339 50.0 64.0
+CMB DW 38.0350 -120.3850 719.0 0.0
+COL DW 64.9000 -147.7933 320.0 0.0
+GDH DW 69.2500 -53.5333 23.0 0.0
+HON DW 21.3217 -158.0083 2.0 0.0
+JAS1 DW 37.9467 -120.4383 457.0 7.0
+KBS DW 78.9175 11.9239 46.0 0.0
+KEV DW 69.7553 27.0067 80.0 15.0
+LEM DW -6.8333 107.6167 1252.0 0.0
+LON DW 46.7500 -121.8100 854.0 0.0
+SCP DW 40.7949 -77.8650 352.0 0.0
+SLR DW -25.7349 28.2816 1348.0 1.0
+TAU DW -42.9099 147.3204 132.0 7.0
+TOL DW 39.8814 -4.0485 480.0 0.0
+AIS G -37.7970 77.5690 35.9 3.0
+ATD G 11.5300 42.8470 610.0 0.0
+BNG G 4.4350 18.5470 378.0 0.0
+CAN G -35.3210 148.9990 650.0 0.0
+CLF G 48.0258 2.2600 145.0 0.0
+CRZF G -46.4300 51.8610 140.0 0.0
+0001 GD 48.3897 -123.4875 32.0 0.0
+0011 GD 48.3897 -123.4875 32.0 0.0
+0012 GD 48.3897 -123.4875 32.0 0.0
+0015 GD 48.3897 -123.4875 32.0 0.0
+0016 GD 48.3897 -123.4875 32.0 0.0
+0019 GD 48.3897 -123.4875 32.0 0.0
+0105 GD 48.3897 -123.4875 32.0 0.0
+0107 GD 48.3897 -123.4875 32.0 0.0
+0109 GD 48.3897 -123.4875 32.0 0.0
+0110 GD 44.0068 142.1517 196.2 0.0
+0111 GD 44.0056 143.3337 233.3 0.0
+0112 GD 42.8945 143.9269 39.7 0.0
+0119 GD 43.3669 145.8011 50.5 0.0
+0120 GD 43.2926 140.5971 47.0 0.0
+0121 GD 43.2895 143.5624 150.1 0.0
+0124 GD 43.1206 144.1265 70.1 0.0
+0125 GD 43.0576 144.8430 62.5 0.0
+0133 GD 42.7272 142.2958 62.5 0.0
+0134 GD 42.6993 143.1033 239.5 0.0
+0135 GD 42.6679 141.0750 425.1 0.0
+0136 GD 42.6555 141.6023 51.3 0.0
+0138 GD 42.5516 143.4611 90.5 0.0
+0141 GD 42.4813 142.0602 41.6 0.0
+0142 GD 42.2477 142.5675 64.4 0.0
+0144 GD 42.1309 142.9354 32.9 0.0
+0147 GD 42.0430 140.8055 44.6 0.0
+0505 GD 43.9888 144.2933 73.7 0.0
+0510 GD 43.7714 142.9030 527.3 0.0
+0516 GD 43.2484 141.8909 79.8 0.0
+0521 GD 42.9389 143.1706 76.0 0.0
+0532 GD 42.1256 143.3157 30.7 0.0
+0533 GD 41.4047 141.4476 67.8 0.0
+ALBH GD 48.3897 -123.4875 32.0 0.0
+AMC2 GD 42.3897 -104.5246 32.0 0.0
+BREW GD 48.1315 -119.6827 249.9 0.0
+DRA2 GD 49.3226 -119.6250 351.0 0.0
+DSLK GD 58.4357 -129.9890 795.0 0.0
+ELDD GD 58.9720 -135.2223 10.0 0.0
+ESQM GD 48.4292 -123.4288 9.0 0.0
+FAIR GD 64.9779 -147.4993 319.0 0.0
+FNEL GD 58.8052 -147.4993 319.0 0.0
+FS32 GD 59.1487 -135.3470 19.0 0.0
+FTSJ GD 56.2549 -120.8487 19.0 0.0
+GODE GD 39.0216 -76.8269 54.9 0.0
+HNSC GD 59.2420 -135.5178 10.0 0.0
+INVM GD 50.5092 -116.0287 859.0 0.0
+MSOL GD 46.9293 -114.1089 975.4 0.0
+PGC4 GD 48.6484 -123.4510 10.0 0.0
+PRDS GD 50.8713 -114.2935 1264.3 0.0
+PTHY GD 50.6855 -127.3754 24.4 0.0
+DRV G -66.6650 140.0020 40.0 4.0
+SMLD GD 49.5654 -119.6443 437.7 0.0
+TEM1 GD 59.4502 -135.3301 13.4 0.0
+TNK1 GD 59.2722 -135.4406 11.3 0.0
+TRRC GD 54.5148 -128.6267 68.9 0.0
+TSEA GD 61.1873 -149.8950 36.0 0.0
+WHIT GD 60.7505 -135.2222 1168.9 0.0
+WLBC GD 52.1305 -122.1369 605.0 0.0
+YELL GD 62.4808 -114.4807 265.5 0.0
+APEZ GE 34.9777 24.8859 435.0 0.0
+BGIO GE 31.7219 35.0877 752.0 0.0
+BKB GE -1.2558 116.9155 0.0 0.0
+BOAB GE 12.4493 -85.6659 381.0 0.0
+BRNL GE 52.4278 13.3580 42.0 0.0
+CART GE 37.5868 -1.0012 65.0 0.0
+CEU GE 35.8987 -5.3731 320.0 0.0
+ECH G 48.2160 7.1580 580.0 0.0
+CSS GE 34.9611 33.3310 396.0 0.0
+DAG GE 76.7713 -18.6550 23.0 0.0
+DSB GE 53.2452 -6.3762 236.0 0.0
+EIL GE 29.6699 34.9512 210.0 0.0
+FODE GE 35.3797 24.9576 50.0 0.0
+GHAR GE 32.1224 13.0885 650.0 0.0
+GSI GE 1.3039 97.5755 107.0 0.0
+GVD GE 34.8392 24.0873 180.0 0.0
+HLG GE 54.1847 7.8839 41.0 0.0
+IBBN GE 52.3072 7.7566 140.0 0.0
+ISP GE 37.8433 30.5093 1100.0 0.0
+JER GE 31.7719 35.1972 770.0 0.0
+KARN GE 35.4019 23.9174 420.0 0.0
+KBS GE 78.9256 11.9417 77.0 0.0
+KBU GE 34.5151 69.1347 1820.0 0.0
+KERA GE 35.3692 23.5577 240.0 0.0
+KMBO GE -1.1268 37.2523 1940.0 0.0
+KRIS GE 35.1780 25.5030 850.0 0.0
+KSDI GE 33.1920 35.6590 123.0 0.0
+KWP GE 49.6305 22.7078 463.0 0.0
+LAST GE 35.1611 25.4786 870.0 0.0
+LID GE 54.5481 13.3664 1.0 0.0
+LVC GE -22.6182 -68.9113 2915.0 0.0
+MAHO GE 39.8959 4.2665 15.0 0.0
+MALT GE 38.3134 38.4273 1120.0 0.0
+MARJ GE 32.5226 20.8776 650.0 0.0
+MATE GE 40.6491 16.7044 494.0 0.0
+MAUI GE 20.7668 -156.2448 2060.0 0.0
+MELI GE 35.2938 -2.9350 5.0 0.0
+MHV GE 54.9595 37.7664 150.0 0.0
+MLR GE 45.4912 25.9456 1378.0 0.0
+MMRI GE -8.6357 122.2376 137.0 0.0
+MNAI GE -4.4500 102.9170 0.0 0.0
+MORC GE 49.7766 17.5428 740.0 0.0
+MRNI GE 33.1178 35.3920 918.0 0.0
+MTE GE 40.3997 -7.5442 815.0 0.0
+NAI GE -1.2739 36.8037 1692.0 0.0
+PMBI GE -2.9270 104.7720 30.0 0.0
+PMG GE -9.4092 147.1539 67.0 0.0
+PSZ GE 47.9184 19.8944 940.0 0.0
+PUL GE 59.7670 30.3170 65.0 0.0
+PVLZ GE 35.1730 -4.3010 90.0 0.0
+RGN GE 54.5477 13.3214 15.0 0.0
+RIOB GE -10.1501 -67.7470 220.0 0.0
+RUE GE 52.4759 13.7800 40.0 0.0
+SANT GE 36.3710 25.4590 540.0 0.0
+SELV GE 37.2383 -3.7277 650.0 0.0
+SFJD GE 66.9960 -50.6215 80.0 0.0
+SFJ GE 66.9967 -50.6156 365.0 0.0
+SFS GE 36.4656 -6.2055 21.0 0.0
+SFUC GE 36.6370 -6.1750 88.0 0.0
+SIVA GE 35.0175 24.8100 95.0 0.0
+SKD GE 35.4120 23.9280 306.0 0.0
+SLIT GE 57.6300 22.2900 15.0 0.0
+SMRI GE -7.0492 110.4407 203.0 0.0
+SNAA GE -71.6707 -2.8379 846.0 0.0
+STU GE 48.7719 9.1950 360.0 0.0
+SUMG GE 72.5763 -38.4539 3240.0 0.0
+SUW GE 54.0125 23.1808 152.0 0.0
+TIRR GE 44.4581 28.4128 77.0 0.0
+TNTI GE 0.7718 127.3667 43.0 0.0
+TRTE GE 58.3786 26.7205 0.0 0.0
+UGM GE -7.9125 110.5231 350.0 0.0
+EVO G 38.5320 -8.0130 0.0 0.0
+VSU GE 58.4620 26.7347 63.0 0.0
+WLF GE 49.6646 6.1526 295.0 0.0
+YOGI GE -7.8166 110.2949 160.0 0.0
+ZKR GE 35.1147 26.2170 270.0 0.0
+FDF G 14.7350 -61.1430 467.0 0.0
+FOMA G -24.9756 46.9788 26.0 0.0
+GRC G 47.2955 3.0736 191.0 0.0
+HDC G 10.0000 -84.1120 1150.0 0.0
+HYB G 17.4170 78.5530 510.0 0.0
+INU G 35.3500 137.0290 132.0 0.0
+KIP G 21.4230 -158.0150 70.0 0.0
+KOG G 5.2070 -52.7320 10.0 0.0
+MBO G 14.3910 -16.9550 3.0 0.0
+MPG G 5.1100 -52.6440 147.0 0.0
+NOC G -22.2840 166.4320 5.0 0.0
+NOUC G -22.1010 166.3030 112.0 0.0
+PAF G -49.3510 70.2130 17.0 2.0
+PEL G -33.1460 -70.6750 660.0 0.0
+PPTF G -17.5896 -149.5652 705.0 0.0
+PVC G -17.7400 168.3120 80.0 0.0
+RER G -21.1710 55.7400 834.0 0.0
+SCZ G 36.5980 -121.4030 261.0 0.0
+SEY G 62.9330 152.3730 206.0 0.0
+SPB G -23.5920 -47.4320 85.0 0.0
+SSB2 G 45.2790 4.5420 700.0 0.0
+SSB3 G 45.2790 4.5420 700.0 0.0
+SSB G 45.2790 4.5420 700.0 0.0
+TAM G 22.7910 5.5270 1377.0 0.0
+BDFB GT -15.6418 -48.0148 1095.0 100.0
+BGCA GT 5.1764 18.4242 576.0 100.0
+BOSA GT -28.6141 25.2555 1180.0 100.0
+CPUP GT -26.3306 -57.3309 5.0 100.0
+DBIC GT 6.6702 -4.8566 25.0 100.0
+LBTB GT -25.0151 25.5967 1028.0 100.0
+LPAZ GT -16.2879 -68.1307 4717.0 100.0
+PLCA GT -40.7328 -70.5508 950.0 100.0
+TRIS G -37.0578 -12.3159 -2.0 2.0
+SBA GT -77.8491 166.7573 20.0 0.0
+VNDA GT -77.5172 161.8528 51.0 100.0
+UNM G 19.3290 -99.1780 2280.0 0.0
+WUS G 41.1990 79.2180 1457.0 0.0
+H2O H2 27.8819 -141.9917 -4947.0 0.4
+ALQ HG 34.9425 -106.4575 1849.0 10.0
+CHG HG 18.7900 98.9769 416.0 0.0
+CTA HG -20.0883 146.2544 357.0 37.0
+EIL HG 29.5500 34.9500 200.0 200.0
+KIP HG 21.4233 -158.0150 37.0 33.0
+KON HG 59.6491 9.5982 216.0 340.0
+MAT HG 36.5431 138.2066 403.0 48.0
+OGD HG 41.0875 -74.5958 -367.0 549.0
+TLO HG 39.8600 -4.0147 465.0 0.0
+ZLP HG -16.2700 -68.1250 4450.0 300.0
+LIT HT 40.1008 22.4900 480.0 0.0
+LKD HT 38.7072 20.6506 1140.0 0.0
+SOH HT 40.8217 23.3539 670.0 0.0
+CCRK HW 46.5585 -119.8548 561.0 0.0
+DDRF HW 46.4911 -119.0595 233.0 0.0
+PHIN HW 45.8951 -119.9278 227.0 0.0
+BJT IC 40.0183 116.1679 137.0 60.0
+ENH IC 30.2762 109.4944 500.0 0.0
+HIA IC 49.2667 119.7417 610.0 0.0
+KMI IC 25.1233 102.7400 1940.0 35.0
+LSA IC 29.7031 91.1270 3645.0 15.0
+MDJ IC 44.6170 129.5908 220.0 50.0
+QIZ IC 19.0291 109.8445 240.0 0.0
+SSE IC 31.0948 121.1908 40.0 0.0
+WMQ IC 43.8138 87.7049 844.0 6.0
+XAN IC 34.0313 108.9237 630.0 0.0
+AAK II 42.6390 74.4940 1645.0 30.0
+ABKT II 37.9304 58.1189 678.0 7.0
+ABPO II -19.0180 47.2290 1528.0 5.3
+ALE II 82.5033 -62.3500 60.0 0.0
+ARU II 56.4302 58.5625 250.0 0.0
+ASCN II -7.9327 -14.3601 173.0 100.0
+BFO II 48.3319 8.3311 589.0 0.0
+BORG II 64.7474 -21.3268 110.0 95.0
+BRVK II 53.0581 70.2828 330.0 15.0
+CMLA II 37.7637 -25.5243 429.0 7.0
+COCO II -12.1901 96.8349 1.0 70.0
+DGAR II -7.4121 72.4525 1.0 2.0
+EFI II -51.6753 -58.0637 110.0 80.0
+ERM II 42.0150 143.1572 40.0 0.0
+ESK II 55.3167 -3.2050 242.0 0.0
+FFC II 54.7250 -101.9783 338.0 0.0
+GAR II 39.0000 70.3167 1300.0 0.0
+HOPE II -54.2836 -36.4879 20.0 0.0
+JTS II 10.2908 -84.9525 340.0 0.0
+KAPI II -5.0142 119.7517 300.0 100.0
+KDAK II 57.7828 -152.5835 152.0 5.5
+KIV II 43.9562 42.6888 1210.0 0.0
+KURK II 50.7154 78.6202 184.0 25.0
+KWAJ II 8.8019 167.6130 0.0 0.0
+LVZ II 67.8979 34.6514 630.0 200.0
+MBAR II -0.6019 30.7382 1390.0 100.0
+MSEY II -4.6737 55.4792 475.0 91.0
+MSVF II -17.7448 178.0528 801.1 100.0
+NIL II 33.6506 73.2686 629.0 68.0
+NNA II -11.9875 -76.8422 575.0 40.0
+NRIL II 69.5049 88.4414 92.0 506.0
+NVS II 54.8404 83.2346 150.0 0.0
+OBN II 55.1146 36.5674 160.0 30.0
+PALK II 7.2728 80.7022 460.0 90.0
+RAYN II 23.5225 45.5032 631.0 2.0
+RPN II -27.1267 -109.3344 110.0 0.0
+SACV II 14.9702 -23.6085 387.0 97.0
+SHEL II -15.9588 -5.7457 537.0 60.0
+SUR II -32.3797 20.8117 1770.0 0.0
+TAU II -42.9099 147.3204 132.0 0.0
+TLY II 51.6807 103.6438 579.0 20.0
+UOSS II 24.9453 56.2042 284.4 0.0
+WRAB II -19.9336 134.3600 366.0 100.0
+XPFO II 33.6107 -116.4555 1280.0 0.0
+VNDA IM -77.5139 161.8456 98.0 0.0
+AAE IU 9.0292 38.7656 2442.0 0.0
+ADK IU 51.8823 -176.6842 130.0 0.0
+AFI IU -13.9093 -171.7773 705.0 1.0
+ANMO IU 34.9459 -106.4572 1720.0 100.0
+ANTO IU 39.8680 32.7934 895.0 195.0
+BBSR IU 32.3713 -64.6963 -1.3 31.4
+BILL IU 68.0653 166.4531 320.0 0.0
+BOCO IU 4.5869 -74.0432 3137.0 23.0
+CASY IU -66.2792 110.5354 5.0 5.0
+CCM IU 38.0557 -91.2446 171.0 51.0
+CHTO IU 18.8141 98.9443 420.0 0.0
+COLA IU 64.8736 -147.8616 200.0 0.0
+COR IU 44.5855 -123.3046 110.0 0.0
+CTAO IU -20.0882 146.2545 320.0 37.0
+DAV IU 7.0697 125.5791 149.0 1.0
+DWPF IU 28.1103 -81.4327 -132.0 162.0
+FUNA IU -8.5259 179.1966 19.0 1.0
+FURI IU 8.8952 38.6798 2565.0 5.0
+GNI IU 40.1480 44.7410 1509.0 100.0
GRFO IU 49.6909 11.2203 384.0 116.0
-BEKI YL 41.31500 34.26300 1415.0 0.0
-BGIO SR 31.72200 35.08780 752.0 100.0
-LIT HT 40.10080 22.49000 480.0 0.0
-ZKR GE 35.11470 26.21700 270.0 0.0
-S001 XS 37.28300 21.71800 156.0 0.0
+GUMO IU 13.5893 144.8684 61.0 109.0
+HKT IU 29.9618 -95.8384 -413.0 450.0
+HNR IU -9.4387 159.9475 0.0 100.0
+HRV IU 42.5064 -71.5583 200.0 0.0
+INCN IU 37.4776 126.6239 79.0 1.0
+JOHN IU 16.7329 -169.5292 -37.0 39.0
+KBS IU 78.9154 11.9385 90.0 3.0
+KEV IU 69.7565 27.0035 85.0 15.0
+KIEV IU 50.7012 29.2242 140.0 40.0
+KIP IU 21.4233 -158.0150 37.0 33.0
+KMBO IU -1.1271 37.2525 1930.0 20.0
+KNTN IU -2.7744 -171.7186 18.0 2.0
+KONO IU 59.6491 9.5982 216.0 340.0
+KOWA IU 14.4967 -4.0140 316.0 5.0
+LCO IU -29.0110 -70.7004 2300.0 0.0
+LSZ IU -15.2779 28.1882 1200.0 0.0
+LVC IU -22.6127 -68.9111 2930.0 30.0
+MA2 IU 59.5756 150.7700 337.0 2.0
+MAJO IU 36.5457 138.2041 405.0 0.0
+MAKZ IU 46.8080 81.9770 590.0 10.0
+MBWA IU -21.1590 119.7313 181.0 9.0
+MIDW IU 28.2155 -177.3697 17.8 1.0
+MSKU IU -1.6557 13.6116 287.0 25.0
+NAI IU -1.2739 36.8037 1692.0 0.0
+NWAO IU -32.9277 117.2390 370.9 9.1
+OTAV IU 0.2376 -78.4508 3495.0 15.0
+PAB IU 39.5446 -4.3499 950.0 0.0
+PAYG IU -0.6742 -90.2861 170.0 100.0
+PET IU 53.0233 158.6499 105.0 5.0
+PMG IU -9.4047 147.1597 90.0 0.0
+PMSA IU -64.7744 -64.0489 40.0 0.0
+POHA IU 19.7573 -155.5326 1910.0 80.0
+PTCN IU -25.0713 -130.0953 218.0 2.0
+PTGA IU -0.7308 -59.9666 141.0 96.0
+QSPA IU -89.9289 144.4382 2847.0 3.0
+RAIO IU 46.0403 -122.8851 11.0 0.0
+RAO IU -29.2450 -177.9290 59.5 0.5
+RAR IU -21.2125 -159.7733 -72.0 100.0
+RCBR IU -5.8274 -35.9014 291.0 109.0
+RSSD IU 44.1212 -104.0359 2022.7 67.3
+SAML IU -8.9489 -63.1831 120.0 0.0
+SBA IU -77.8492 166.7572 48.0 2.0
+SDV IU 8.8839 -70.6340 1588.0 32.0
+SFJD IU 66.9961 -50.6208 329.0 1.0
+SJG IU 18.1091 -66.1500 420.0 0.0
+SLBS IU 23.6858 -109.9443 825.0 0.0
+SNZO IU -41.3087 174.7043 20.0 100.0
+SSPA IU 40.6358 -77.8876 170.0 100.0
+TARA IU 1.3549 172.9229 19.0 1.0
+TATO IU 24.9735 121.4971 77.1 82.9
+TBT IU 28.6794 -17.9145 180.0 40.0
+TEIG IU 20.2263 -88.2763 15.0 25.0
+TIXI IU 71.6341 128.8667 40.0 0.0
+TOL IU 39.8814 -4.0485 480.0 0.0
+TRIS IU -37.0681 -12.3152 58.0 2.0
+TRQA IU -38.0568 -61.9787 439.0 101.0
+TSUM IU -19.2022 17.5838 1260.0 0.0
+TUC IU 32.3098 -110.7847 909.0 1.0
+ULN IU 47.8651 107.0532 1610.0 0.0
+WAKE IU 19.2834 166.6520 19.0 1.0
+WCI IU 38.2289 -86.2939 78.0 132.0
+WVT IU 36.1297 -87.8300 170.0 0.0
+XMAS IU 2.0448 -157.4457 19.0 1.0
+YAK IU 62.0310 129.6805 110.0 14.0
+YSS IU 46.9587 142.7604 148.0 2.0
+AAK KN 42.6333 74.4944 1680.0 0.0
+AML KN 42.1311 73.6941 3400.0 0.0
+CHM KN 42.9986 74.7513 655.0 0.0
+EKS2 KN 42.6615 73.7772 1360.0 0.0
+ERPT KN 42.6011 76.0735 3790.0 0.0
+KBK KN 42.6564 74.9478 1760.0 0.0
+KZA KN 42.0778 75.2496 3520.0 0.0
+NRPT KN 42.6011 76.0735 3790.0 0.0
+TKM2 KN 42.9208 75.5966 2020.0 0.0
+UCH KN 42.2275 74.5134 3850.0 0.0
+ULHL KN 42.2456 76.2417 2040.0 0.0
+USP KN 43.2669 74.4997 740.0 0.0
+BMN LB 40.4315 -117.2218 1500.0 0.0
+DAC LB 36.2770 -117.5937 1813.0 0.0
+LDS LB 37.2425 -113.3514 1101.0 0.0
+MVU LB 38.5037 -112.2123 2239.0 0.0
+TPH LB 38.0750 -117.2225 1883.0 0.0
+MMNY LD 42.7319 -77.9066 241.0 0.0
+NPNY LD 41.7546 -74.1435 216.0 0.0
+PAL LD 41.0057 -73.9071 91.0 0.0
+PANJ LD 40.3769 -74.7029 100.0 0.0
+LTH LI 46.4617 -119.4177 157.0 0.0
+LTL LI 30.5374 -90.7660 21.0 0.0
+AIO MN 37.9712 15.2330 751.4 5.0
+AQU MN 42.3540 13.4050 710.0 15.0
+BGY MN 44.8026 20.5158 250.0 7.0
+BLY MN 44.7488 17.1839 256.0 1.0
+BNI MN 45.0520 6.6780 1395.0 50.0
+CEL MN 38.2603 15.8939 702.0 0.0
+CII MN 41.7230 14.3050 910.0 0.0
+CLTB MN 37.5780 13.2160 949.0 5.0
+CUC MN 39.9938 15.8155 637.0 0.0
+DIVS MN 44.0981 19.9917 1000.0 3.0
+GFA MN 34.3380 9.0730 250.0 200.0
+IDI MN 35.2880 24.8900 750.0 7.0
+ISP MN 37.8430 30.5090 1100.0 0.0
+KEG MN 29.9275 31.8292 460.0 7.0
+MDT MN 32.8170 -4.6140 1200.0 300.0
+MEB MN 36.3030 2.7300 500.0 20.0
+PDG MN 42.4297 19.2608 40.0 5.0
+RTC MN 33.9881 -6.8569 50.0 4.5
+TIP MN 39.1794 16.7583 789.0 0.0
+TIR MN 41.3472 19.8631 247.0 3.0
+TNV MN -74.7000 164.1200 40.0 5.0
+TRI MN 45.7090 13.7640 161.0 100.0
+TTE MN 45.6600 13.7900 92.0 0.0
+TUE MN 46.4722 9.3473 1924.0 0.0
+VAE MN 37.4690 14.3533 735.1 10.0
+VLC MN 44.1594 10.3864 555.0 3.0
+VSL MN 39.4960 9.3780 370.0 10.0
+VTS MN 42.6180 23.2350 1490.0 50.0
+WDD MN 35.8373 14.5242 44.1 20.0
+BTDF MS 1.3617 103.7750 57.0 0.0
+IPM MY 4.4795 101.0255 247.0 0.0
+KKM MY 6.0443 116.2147 830.0 0.0
+KOM MY 1.7922 103.8467 49.0 0.0
+KSM MY 1.4733 110.3083 66.0 0.0
+KUM MY 5.2902 100.6492 74.0 0.0
+LDM MY 5.1777 118.4980 177.0 0.0
+SBM MY 2.4529 112.2140 237.0 0.0
+BLO NM 39.1719 -86.5222 246.0 0.0
+FVM NM 37.9812 -90.4273 137.0 0.0
+MGMO NM 37.1540 -92.2687 453.0 0.0
+MPH NM 35.1230 -89.9320 93.0 0.0
+OLIL NM 38.7338 -88.0991 150.0 0.0
+PBMO NM 36.7785 -90.4297 143.9 0.0
+PLAL NM 34.9824 -88.0755 165.0 0.0
+PVMO NM 36.4137 -89.6997 98.3 0.0
+SIUC NM 37.7148 -89.2174 120.0 0.0
+SLM NM 38.6361 -90.2364 186.0 0.0
+UALR NM 34.7751 -92.3429 138.0 0.0
+USIN NM 37.9650 -87.6660 170.7 0.0
+UTMT NM 36.3424 -88.8654 120.0 0.0
+BEK NN 39.8667 -120.3586 1743.0 0.0
+OMM NN 37.6214 -118.9994 2750.0 0.0
+WCN NN 39.3017 -119.7563 1704.0 0.0
+NE70 NR 32.4209 -115.2608 -23.0 0.0
+NE71 NR 31.6897 -115.9053 1155.0 0.0
+NE72 NR 30.8484 -116.0586 17.0 0.0
+NE73 NR 30.0651 -115.3485 489.0 0.0
+NE74 NR 28.0075 -114.0138 21.0 0.0
+NE75 NR 27.2933 -112.8565 137.0 0.0
+NE76 NR 26.8889 -111.9990 35.0 0.0
+NE77 NR 26.0158 -111.3613 40.0 0.0
+NE78 NR 24.3982 -111.1064 82.0 0.0
+NE79 NR 23.1194 -109.7561 225.0 0.0
+NE80 NR 30.5000 -112.3199 225.0 0.0
+NE81 NR 28.9183 -109.6363 295.0 0.0
+NE82 NR 26.9157 -109.2308 183.0 0.0
+NE83 NR 24.7309 -107.7393 28.0 0.0
+NE84 NR 24.9135 -111.5452 20.0 0.0
+NE85 NR 24.1296 -110.4368 -12.0 0.0
+BFZ NZ -40.6814 176.2461 283.0 0.0
+BKZ NZ -39.1674 176.4923 706.0 0.0
+CTZ NZ -43.7355 -176.6172 99.0 0.0
+HIZ NZ -38.5147 174.8555 237.0 0.0
+KHZ NZ -42.4160 173.5390 64.0 0.0
+ODZ NZ -45.0456 170.6445 274.0 0.0
+OUZ NZ -35.2215 173.5960 63.0 0.0
+QRZ NZ -40.8272 172.5290 263.0 0.0
+RPZ NZ -43.7163 171.0538 453.0 50.0
+URZ NZ -38.2610 177.1107 95.0 50.0
+WPVZ NZ -39.2058 175.5457 1244.0 0.0
+ARSA OE 47.2505 15.5232 577.0 0.0
+DAVA OE 47.2867 9.8803 1602.0 0.0
+KBA OE 47.0784 13.3447 1721.0 0.0
+HDC3 OV 10.0010 -84.0660 1220.0 0.0
+B001 PB 48.0431 -123.1314 237.0 144.4
+B003 PB 48.0624 -124.1409 314.0 159.7
+B004 PB 48.2019 -124.4270 30.0 156.1
+B005 PB 48.0595 -123.5033 302.7 153.0
+B006 PB 48.0588 -123.5008 302.0 150.1
+B007 PB 48.0576 -123.5041 293.0 131.8
+B009 PB 48.6487 -123.4512 15.0 217.6
+B010 PB 48.6502 -123.4513 5.0 199.0
+B011 PB 48.6495 -123.4482 22.0 217.6
+B012 PB 48.9246 -125.5420 13.0 162.7
+B013 PB 47.8130 -122.9108 75.3 162.3
+B014 PB 47.5133 -123.8125 64.7 218.8
+B017 PB 46.9960 -123.5575 33.9 219.6
+B018 PB 46.9795 -123.0203 10.0 217.6
+B019 PB 46.6527 -123.6518 9.0 155.4
+B020 PB 46.3828 -123.8445 0.0 0.0
+B022 PB 45.9546 -123.9310 10.0 211.5
+B023 PB 46.1112 -123.0787 177.4 233.2
+B024 PB 45.6340 -123.8630 226.2 211.5
+B026 PB 45.3094 -123.8230 106.8 223.7
+B027 PB 44.4973 -122.9622 216.0 226.8
+B028 PB 44.4937 -122.9638 140.0 232.3
+B030 PB 43.9713 -122.7717 263.6 160.9
+B031 PB 43.6643 -123.3967 688.5 134.7
+B032 PB 43.6680 -123.3923 63.8 225.6
+B033 PB 43.2917 -123.1245 312.1 0.0
+B035 PB 42.5035 -123.3834 370.0 219.4
+B036 PB 42.5058 -123.3817 315.3 175.0
+B039 PB 41.4667 -122.4847 923.9 202.1
+B040 PB 41.8308 -122.4205 819.5 224.9
+B045 PB 40.4360 -123.9965 29.4 217.6
+B046 PB 40.3220 -124.2843 44.0 187.5
+B047 PB 40.2505 -124.1713 80.0 175.3
+B049 PB 40.2375 -123.8197 117.0 122.0
+B054 PB 37.8602 -122.1995 405.6 0.0
+B057 PB 38.0273 -122.5655 14.4 212.6
+B058 PB 36.7995 -121.5808 114.2 159.4
+B065 PB 36.7437 -121.4742 643.0 185.7
+B066 PB 36.8573 -121.5922 67.0 223.9
+B067 PB 36.7650 -121.5655 126.2 150.6
+B072 PB 35.8310 -120.3450 397.7 154.5
+B073 PB 35.9467 -120.4717 535.3 236.1
+B075 PB 35.9292 -120.5153 583.0 165.4
+B076 PB 35.9398 -120.4248 1362.2 192.7
+B078 PB 35.8376 -120.3443 1474.0 175.1
+B079 PB 35.7160 -120.2047 1362.0 175.1
+B081 PB 33.7112 -116.7142 1467.0 229.8
+B082 PB 33.5982 -116.5960 1374.8 220.7
+B084 PB 33.6116 -116.4564 1271.0 147.6
+B086 PB 33.5575 -116.5300 1392.0 226.8
+B087 PB 33.4955 -116.6027 1139.0 147.5
+B088 PB 33.3749 -116.6205 1403.9 147.4
+B093 PB 33.5937 -116.7641 1229.6 125.0
+B201 PB 46.3033 -122.2648 990.0 237.3
+B202 PB 46.2447 -122.1367 1218.6 236.8
+B203 PB 46.1690 -122.3337 814.4 218.8
+B204 PB 46.1360 -122.1690 784.2 223.0
+B205 PB 44.7135 -110.6783 2360.0 142.2
+B206 PB 44.7177 -110.5117 2399.5 74.1
+B207 PB 44.6190 -110.8485 2182.0 202.1
+B208 PB 44.5602 -110.4015 2405.8 0.0
+B900 PB 35.6300 -120.0150 245.0 181.0
+B901 PB 35.6900 -120.1420 275.3 173.1
+B916 PB 36.1925 -117.6685 1859.9 166.9
+B917 PB 35.3892 -117.2587 1125.9 0.0
+B918 PB 35.9372 -117.5906 1042.6 189.9
+B921 PB 35.5865 -117.4622 694.5 139.9
+B926 PB 48.8202 -124.1312 191.6 233.5
+B927 PB 49.2188 -124.8113 98.7 221.7
+B928 PB 48.8337 -125.1345 9.5 236.5
+B932 PB 40.2750 -124.2212 52.0 0.0
+B933 PB 40.0600 -123.9690 268.1 139.0
+B934 PB 39.8445 -123.6913 331.0 0.0
+B935 PB 40.4787 -123.5732 696.7 152.4
+B941 PB 46.9868 -122.2190 151.0 147.2
+B943 PB 47.8132 -122.9113 84.2 215.6
+B944 PB 44.3897 -110.5438 2364.6 114.3
+B945 PB 44.8940 -110.7441 2241.4 135.6
+P403 PB 48.0624 -124.1409 314.0 159.7
+B1 PI 34.0730 106.9190 1437.0 0.0
+B3 PI 34.0730 106.9190 1437.0 0.0
+B4 PI 34.0736 -106.9189 12.9 0.0
+B5 PI 34.0730 106.9190 1437.0 0.0
+B6 PI 34.0736 -106.9189 12.9 0.0
+B7 PI 34.0730 106.9190 1437.0 0.0
+B8 PI 34.0736 -106.9189 12.9 0.0
+B9 PI 34.0736 -106.9188 1418.0 0.0
+NP00 PI 34.0740 -106.9190 1430.0 0.0
+NP01 PI 34.0740 -106.9191 1430.0 0.0
+NP02 PI 34.0740 -106.9190 1430.0 0.0
+NP03 PI 34.0740 -106.9190 1430.0 0.0
+NP04 PI 34.0740 -106.9191 1430.0 0.0
+NP05 PI 34.0740 -106.9190 1430.0 0.0
+NP06 PI 34.0740 -106.9191 1430.0 0.0
+NP07 PI 34.0740 -106.9190 1430.0 0.0
+NP08 PI 34.0740 -106.9191 1430.0 0.0
+NP09 PI 34.0740 -106.9190 1430.0 0.0
+SP00 PI 34.0740 -106.9190 1430.0 0.0
+SP01 PI 34.0740 -106.9190 1430.0 0.0
+SP02 PI 34.0740 -106.9190 1430.0 0.0
+SP03 PI 34.0740 -106.9190 1430.0 0.0
+SP04 PI 34.0740 -106.9190 1430.0 0.0
+SP05 PI 34.0740 -106.9190 1430.0 0.0
+SP06 PI 34.0740 -106.9190 1430.0 0.0
+SP07 PI 34.0740 -106.9190 1430.0 0.0
+SP08 PI 34.0740 -106.9190 1430.0 0.0
+SP09 PI 34.0740 -106.9190 1430.0 0.0
+PESTR PM 38.8672 -7.5902 410.0 0.0
+PFVI PM 37.1328 -8.8268 189.0 3.0
+PMOZ PM 32.8230 -17.1972 1027.0 0.0
+PVAQ PM 37.4037 -7.7173 200.0 7.0
+ROSA PM 38.7208 -28.2470 310.0 0.0
+BAG PS 16.4108 120.5797 1507.0 0.0
+OGS PS 27.0570 142.2030 20.0 0.0
+PATS PS 6.8367 158.3152 10.0 0.0
+PSI PS 2.6938 98.9237 987.0 0.0
+SYO PS -69.0067 39.5850 20.0 0.0
+RSCP RS 35.6000 -85.5689 485.0 100.0
+RSNT RS 62.4797 -114.5917 -50.0 100.0
+RSNY RS 44.5481 -74.5297 357.0 100.0
+RSON RS 50.8589 -93.7022 221.0 103.0
+RSSD RS 44.1204 -104.0362 1950.0 110.0
+PH008 SF 35.9742 -120.5519 -412.2 1040.0
+ANMO SR 34.9462 -106.4567 1740.0 100.0
+ANTO SR 39.8689 32.7936 883.0 195.0
+BCAO SR 4.4335 18.5354 336.0 64.0
+BGIO SR 31.7220 35.0878 752.0 100.0
+BOCO SR 4.5869 -74.0432 3071.0 100.0
+CHTO SR 18.7900 98.9769 316.0 100.0
+GRFO SR 49.6909 11.2203 384.0 116.0
+GUMO SR 13.5892 144.8684 164.0 0.0
+MAIO SR 36.3000 59.4945 1000.0 100.0
+NWAO SR -32.9266 117.2333 265.0 100.0
+SHIO SR 25.5666 91.8830 1509.0 91.0
+SNZO SR -41.3103 174.7046 62.0 100.0
+TATO SR 24.9735 121.4971 157.0 90.0
+109C TA 32.8889 -117.1051 150.0 0.0
+112A TA 32.5356 -114.5804 87.0 0.0
+113A TA 32.7683 -113.7667 118.0 0.0
+114A TA 32.7513 -112.8830 279.0 0.0
+115A TA 32.7006 -112.2279 606.2 0.0
+116A TA 32.5618 -111.7042 477.3 0.0
+117A TA 32.5716 -110.7393 1544.0 0.0
+118A TA 32.6399 -109.9697 1440.0 0.0
+119A TA 32.7663 -109.3029 1406.0 0.0
+120A TA 32.5466 -108.6330 1528.0 0.0
+121A TA 32.5324 -107.7851 1652.0 0.0
+122A TA 32.6995 -107.0005 1333.0 0.0
+123A TA 32.6349 -106.2622 1206.0 0.0
+124A TA 32.7001 -105.4544 2078.0 0.0
+125A TA 32.6588 -104.6573 1212.0 0.0
+126A TA 32.6462 -104.0204 1032.0 0.0
+127A TA 32.6764 -103.3575 1160.0 0.0
+128A TA 32.6213 -102.4850 966.0 0.0
+129A TA 32.6309 -101.8662 876.0 0.0
+130A TA 32.5961 -100.9652 676.0 0.0
+131A TA 32.6737 -100.3888 622.0 0.0
+214A TA 31.9559 -112.8115 543.0 0.0
+216A TA 32.0022 -111.4574 908.0 0.0
+217A TA 31.7748 -110.8162 1412.0 0.0
+218A TA 31.9737 -110.0464 1486.0 0.0
+219A TA 31.9989 -109.2592 1581.0 0.0
+220A TA 31.9041 -108.5266 1395.0 0.0
+221A TA 32.0094 -107.7782 1277.0 0.0
+222A TA 32.1046 -107.1013 1324.0 0.0
+223A TA 32.0062 -106.4276 1232.0 0.0
+224A TA 32.0760 -105.5226 1487.0 0.0
+225A TA 32.1101 -104.8229 1703.0 0.0
+226A TA 32.0618 -104.1014 929.0 0.0
+226B TA 32.0778 -104.1654 981.0 0.0
+227A TA 32.0120 -103.2924 879.0 0.0
+228A TA 32.1180 -102.5918 954.0 0.0
+229A TA 31.9671 -101.8107 804.0 0.0
+230A TA 31.8878 -101.1124 742.0 0.0
+231A TA 31.9353 -100.3163 574.0 0.0
+232A TA 31.8880 -99.6469 621.0 0.0
+318A TA 31.4390 -109.9907 1576.0 0.0
+319A TA 31.3757 -109.2809 1187.0 0.0
+320A TA 31.3369 -108.5284 1414.0 0.0
+324A TA 31.4425 -105.4828 1441.0 0.0
+325A TA 31.3711 -104.9712 1666.0 0.0
+326A TA 31.3165 -103.9786 982.0 0.0
+327A TA 31.3691 -103.4923 784.0 0.0
+328A TA 31.3818 -102.8097 755.0 0.0
+329A TA 31.4870 -101.9808 842.0 0.0
+330A TA 31.4063 -101.1752 742.0 0.0
+331A TA 31.3085 -100.4266 615.0 0.0
+332A TA 31.3829 -99.7423 513.0 0.0
+425A TA 30.7862 -104.9857 1337.0 0.0
+426A TA 30.6689 -104.0293 1943.0 0.0
+427A TA 30.8498 -103.4018 1031.0 0.0
+428A TA 30.7263 -102.6847 982.0 0.0
+429A TA 30.6248 -101.8938 804.0 0.0
+430A TA 30.7866 -101.2362 768.0 0.0
+431A TA 30.6824 -100.6079 700.0 0.0
+432A TA 30.8755 -99.7937 647.0 0.0
+526A TA 30.0609 -104.0898 1405.0 0.0
+527A TA 30.1456 -103.6119 1419.0 0.0
+528A TA 30.1615 -102.7880 1183.0 0.0
+529A TA 30.1246 -102.2204 764.0 0.0
+530A TA 30.1489 -101.3379 636.0 0.0
+531A TA 30.1645 -100.5464 661.0 0.0
+532A TA 30.1278 -99.9049 703.0 0.0
+626A TA 29.5540 -104.1335 979.0 0.0
+627A TA 29.4528 -103.3887 1163.0 0.0
+628A TA 29.4862 -102.8885 655.0 0.0
+631A TA 29.4123 -100.5759 344.0 0.0
+632A TA 29.5071 -99.7863 420.0 0.0
+732A TA 28.7292 -99.9707 178.0 0.0
+A04A TA 48.7197 -122.7070 23.0 0.0
+A05A TA 48.9983 -122.0852 174.3 0.0
+A06A TA 49.0983 -121.4804 576.0 0.0
+A07A TA 49.0480 -120.3837 1842.0 0.0
+A08A TA 48.9534 -119.2725 1089.0 0.0
+A09A TA 48.9752 -118.5851 931.0 0.0
+A10A TA 48.9813 -117.5586 688.0 0.0
+A11A TA 48.9580 -116.3631 1455.0 0.0
+A12A TA 48.9339 -115.6526 928.0 0.0
+A13A TA 48.9330 -114.4135 1199.0 0.0
+A14A TA 48.9725 -113.4218 1420.0 0.0
+A15A TA 48.9777 -112.7327 1236.0 0.0
+A16A TA 48.9482 -111.5968 1394.0 0.0
+A17A TA 48.9448 -110.6994 973.0 0.0
+A18A TA 48.9198 -109.8459 804.0 0.0
+A19A TA 48.9286 -108.7429 971.0 0.0
+A20A TA 48.8799 -107.9257 868.0 0.0
+A21A TA 48.9912 -106.9254 779.0 0.0
+A22A TA 48.9926 -105.9153 861.0 0.0
+A23A TA 48.9592 -104.8786 665.0 0.0
+A24A TA 48.9239 -104.0018 677.0 0.0
+A25A TA 48.9388 -103.1128 584.0 0.0
+A26A TA 48.8973 -102.0380 570.0 0.0
+A27A TA 48.9533 -101.2407 466.0 0.0
+A28A TA 48.9096 -100.2669 692.0 0.0
+A29A TA 48.9228 -99.2321 480.0 0.0
+A30A TA 48.9402 -98.3009 484.0 0.0
+ABTX TA 32.6238 -99.6431 502.0 0.0
+B04A TA 48.0575 -123.5040 293.8 0.0
+B05A TA 48.2641 -122.0960 154.5 0.0
+B06A TA 48.5183 -121.4845 75.0 0.0
+B07A TA 48.4607 -120.1197 781.0 0.0
+B08A TA 48.3584 -119.3332 696.0 0.0
+B09A TA 48.4226 -118.1488 630.0 0.0
+B10A TA 48.2995 -117.2252 585.0 0.0
+B11A TA 48.4366 -116.3674 870.0 0.0
+B12A TA 48.4694 -115.5889 764.0 0.0
+B13A TA 48.3739 -114.4676 933.0 0.0
+B14A TA 48.3609 -113.2729 1529.0 0.0
+B15A TA 48.3129 -112.5619 1222.0 0.0
+B16A TA 48.4128 -111.7073 1038.0 0.0
+B17A TA 48.2921 -110.7988 894.0 0.0
+B18A TA 48.3943 -109.7775 950.0 0.0
+B19A TA 48.4551 -108.9443 803.0 0.0
+B20A TA 48.4389 -108.0209 827.0 0.0
+B21A TA 48.4278 -107.0166 661.0 0.0
+B22A TA 48.3008 -105.9964 836.0 0.0
+B23A TA 48.4643 -104.9909 716.0 0.0
+B25A TA 48.2716 -103.1613 705.0 0.0
+B26A TA 48.3762 -102.2341 708.0 0.0
+B27A TA 48.4044 -101.2600 492.0 0.0
+B28A TA 48.4492 -100.3616 451.0 0.0
+B29A TA 48.4649 -99.3535 460.0 0.0
+B30A TA 48.4505 -98.3286 480.0 0.0
+BGNE TA 41.4083 -98.1502 573.0 0.0
+BNLO TA 37.1311 -122.1729 794.0 0.0
+BRSD TA 44.4430 -98.9563 519.0 0.0
+C03A TA 47.9475 -124.5662 48.4 0.0
+C04A TA 47.7172 -122.9718 53.0 0.0
+C05A TA 47.6947 -121.6895 541.0 0.0
+C06A TA 47.9230 -120.8944 594.0 0.0
+C07A TA 47.6902 -120.0613 863.0 0.0
+C08A TA 47.7849 -119.0454 783.0 0.0
+C09A TA 47.8006 -118.2741 495.0 0.0
+C10A TA 47.8192 -117.3077 684.0 0.0
+C11A TA 47.8397 -116.2557 914.0 0.0
+C12A TA 47.7039 -115.4811 782.0 0.0
+C12B TA 47.7092 -115.4750 801.0 0.0
+C13A TA 47.6795 -114.5735 976.0 0.0
+C14A TA 47.7687 -113.7468 1173.0 0.0
+C15A TA 47.7750 -112.6072 1522.0 0.0
+C16A TA 47.8012 -111.7458 1240.0 0.0
+C17A TA 47.6327 -110.7626 1080.0 0.0
+C19A TA 47.7343 -109.0489 966.0 0.0
+C20A TA 47.7133 -108.0042 781.0 0.0
+C21A TA 47.8264 -107.0960 804.0 0.0
+C22A TA 47.7456 -105.8841 745.0 0.0
+C23A TA 47.8026 -105.1844 729.0 0.0
+C24A TA 47.5302 -104.4252 713.0 0.0
+C25A TA 47.7124 -103.2558 700.0 0.0
+C26A TA 47.8259 -102.0957 656.0 0.0
+C27A TA 47.7139 -101.5029 602.0 0.0
+C28A TA 47.7157 -100.3871 524.0 0.0
+C30A TA 47.7026 -98.4783 480.0 0.0
+D03A TA 47.1161 -123.7714 31.0 0.0
+D04A TA 47.1109 -122.8035 31.1 0.0
+D05A TA 47.1881 -121.9888 266.4 0.0
+D06A TA 47.1935 -120.8445 575.0 0.0
+D07A TA 47.1936 -119.9732 453.0 0.0
+D08A TA 47.0573 -118.9210 385.0 0.0
+D09A TA 47.0615 -118.3092 540.0 0.0
+D10A TA 47.0547 -117.2759 775.0 0.0
+D11A TA 47.0477 -116.3353 849.0 0.0
+D12A TA 47.0528 -115.3525 1135.0 0.0
+D13A TA 47.0943 -114.4587 1000.0 0.0
+D14A TA 47.0816 -113.5062 1444.0 0.0
+D15A TA 47.0418 -112.5204 1593.0 0.0
+D16A TA 47.0308 -111.5492 1427.0 0.0
+D17A TA 47.1523 -110.6873 1535.0 0.0
+D18A TA 47.1961 -109.8023 1202.0 0.0
+D19A TA 47.1646 -108.8800 1047.0 0.0
+D20A TA 47.0911 -108.1362 902.0 0.0
+D21A TA 47.0658 -106.9939 934.0 0.0
+D22A TA 47.1498 -106.1797 791.0 0.0
+D23A TA 47.1730 -105.2071 895.0 0.0
+D24A TA 47.1152 -104.3270 766.0 0.0
+D25A TA 47.1508 -103.3203 850.0 0.0
+D26A TA 47.0301 -102.4357 693.0 0.0
+D27A TA 47.1081 -101.5254 685.0 0.0
+D28A TA 47.1929 -100.5692 621.0 0.0
+D29A TA 47.0605 -99.5957 551.0 0.0
+D30A TA 47.1082 -98.7576 465.0 0.0
+E03A TA 46.5459 -123.5632 72.8 0.0
+E04A TA 46.5934 -122.7204 215.2 0.0
+E05A TA 46.5605 -121.7608 1481.0 0.0
+E06A TA 46.5433 -120.9791 911.0 0.0
+E07A TA 46.5585 -119.8548 561.0 0.0
+E08A TA 46.4911 -119.0595 233.0 0.0
+E09A TA 46.5139 -118.1455 304.0 0.0
+E10A TA 46.4850 -117.1102 903.0 0.0
+E11A TA 46.3559 -116.2092 831.0 0.0
+E12A TA 46.4151 -115.5705 1555.0 0.0
+E13A TA 46.4423 -114.1880 1257.0 0.0
+E14A TA 46.4162 -113.4926 1675.0 0.0
+E15A TA 46.4246 -112.6412 1520.0 0.0
+E16A TA 46.5336 -111.6764 1398.0 0.0
+E17A TA 46.4620 -110.8576 1835.0 0.0
+E18A TA 46.5663 -109.9137 1459.0 0.0
+E19A TA 46.4611 -108.7860 1059.0 0.0
+E20A TA 46.5038 -108.1301 954.0 0.0
+E21A TA 46.5408 -107.0757 834.0 0.0
+E22A TA 46.4386 -105.9488 784.0 0.0
+E23A TA 46.4978 -105.3062 750.0 0.0
+E24A TA 46.5608 -104.3073 912.0 0.0
+E25A TA 46.5009 -103.4013 885.0 0.0
+E26A TA 46.4712 -102.4621 806.0 0.0
+E27A TA 46.4293 -101.5645 738.0 0.0
+E28A TA 46.5748 -100.6934 579.0 0.0
+E29A TA 46.5682 -99.6420 628.0 0.0
+E30A TA 46.5021 -98.9098 544.0 0.0
+ELFS TA 40.6183 -120.7279 1553.0 0.0
+F03A TA 45.9306 -123.5591 324.6 0.0
+F04A TA 45.9325 -122.4193 211.2 0.0
+F05A TA 45.8839 -121.4588 454.8 0.0
+F06A TA 45.7664 -120.7817 577.0 0.0
+F07A TA 45.8951 -119.9278 227.0 0.0
+F08A TA 45.7971 -118.7768 487.0 0.0
+F09A TA 45.7087 -117.9094 894.0 0.0
+F10A TA 45.9734 -117.2277 1136.0 0.0
+F11A TA 45.8883 -116.1549 1186.0 0.0
+F12A TA 45.7565 -115.2547 1442.0 0.0
+F13A TA 45.7893 -114.3317 1462.0 0.0
+F14A TA 45.8122 -113.3699 1851.0 0.0
+F15A TA 45.8409 -112.4929 1995.0 0.0
+F16A TA 45.7843 -111.6262 1350.0 0.0
+F17A TA 45.9054 -110.6590 1545.0 0.0
+F18A TA 45.9046 -109.7158 1291.0 0.0
+F19A TA 45.8538 -108.9437 1243.0 0.0
+F20A TA 45.8006 -108.1488 1194.0 0.0
+F21A TA 45.8231 -107.1233 1026.0 0.0
+F22A TA 45.7794 -106.2569 882.0 0.0
+F23A TA 45.7212 -105.4056 953.0 0.0
+F24A TA 45.8452 -104.4448 1105.0 0.0
+F25A TA 45.9256 -103.4576 912.0 0.0
+F26A TA 45.8512 -102.6697 852.0 0.0
+F27A TA 45.8727 -102.0203 757.0 0.0
+F28A TA 45.8485 -100.8005 633.0 0.0
+F29A TA 45.8336 -99.8273 575.0 0.0
+F30A TA 45.8356 -99.0250 601.0 0.0
+G03A TA 45.3153 -123.2811 208.0 0.0
+G04A TA 45.2059 -122.4784 272.8 0.0
+G05A TA 45.2422 -121.3167 594.4 0.0
+G06A TA 45.2357 -120.6351 780.0 0.0
+G07A TA 45.2664 -119.6692 801.0 0.0
+G08A TA 45.2904 -118.9595 1318.0 0.0
+G09A TA 45.2781 -117.7802 1027.0 0.0
+G10A TA 45.2925 -117.1204 1527.0 0.0
+G11A TA 45.3997 -116.2680 1343.0 0.0
+G12A TA 45.1285 -115.3257 1780.0 0.0
+G13A TA 45.0931 -114.2329 1538.0 0.0
+G14A TA 45.2432 -113.4604 2140.0 0.0
+G15A TA 45.1660 -112.4887 1857.0 0.0
+G16A TA 45.2285 -111.8046 1769.0 0.0
+G17A TA 45.3212 -110.7398 1574.0 0.0
+G18A TA 45.3171 -109.5631 1699.0 0.0
+G20A TA 45.2202 -108.2141 1431.0 0.0
+G21A TA 45.2250 -107.2070 1227.0 0.0
+G22A TA 45.2189 -106.2924 1203.0 0.0
+G23A TA 45.2012 -105.3907 1064.0 0.0
+G24A TA 45.3280 -104.4163 1147.0 0.0
+G25A TA 45.1935 -103.4173 894.0 0.0
+G26A TA 45.2937 -102.5990 833.0 0.0
+G27A TA 45.3791 -102.9310 761.0 0.0
+G28A TA 45.0749 -100.9161 709.0 0.0
+G29A TA 45.1926 -99.9234 588.0 0.0
+G30A TA 45.1293 -99.1446 496.0 0.0
+H02A TA 44.6764 -123.9997 209.1 0.0
+H03A TA 44.6765 -123.2923 214.3 0.0
+H04A TA 44.6837 -122.1862 652.0 0.0
+H05A TA 44.6473 -121.2271 720.8 0.0
+H06A TA 44.7344 -120.3346 516.0 0.0
+H07A TA 44.5913 -119.5646 1204.0 0.0
+H08A TA 44.5191 -118.6704 1373.0 0.0
+H09A TA 44.6653 -117.6643 1263.0 0.0
+H10A TA 44.5890 -116.7474 882.0 0.0
+H11A TA 44.7035 -116.0127 1525.0 0.0
+H12A TA 44.5494 -114.8554 1777.0 0.0
+H13A TA 44.5642 -114.2545 1563.0 0.0
+H14A TA 44.6165 -113.3674 1933.0 0.0
+H15A TA 44.6173 -112.6439 1957.0 0.0
+H16A TA 44.7038 -111.2478 2080.0 0.0
+H17A TA 44.3951 -110.5762 2400.0 0.0
+H18A TA 44.6769 -109.6641 2221.0 0.0
+H19A TA 44.6704 -108.9857 1479.0 0.0
+H20A TA 44.4868 -107.9990 1236.0 0.0
+H21A TA 44.6277 -107.0423 1411.0 0.0
+H22A TA 44.5943 -106.3521 1276.0 0.0
+H23A TA 44.5620 -105.4007 1213.0 0.0
+H24A TA 44.7473 -104.5524 1166.0 0.0
+H25A TA 44.6215 -103.5981 942.0 0.0
+H26A TA 44.6168 -102.7739 879.0 0.0
+H27A TA 44.6329 -102.0775 748.0 0.0
+H28A TA 44.6751 -101.0206 642.0 0.0
+H29A TA 44.6261 -100.2125 548.0 0.0
+HAST TA 36.3887 -121.5514 542.0 0.0
+HATC TA 40.8161 -121.4612 1013.0 0.0
+HELL TA 36.6801 -119.0228 1145.0 0.0
+I02A TA 44.0035 -123.8299 170.1 0.0
+I03A TA 43.9726 -123.2777 205.7 0.0
+I04A TA 43.7941 -122.4113 731.0 0.0
+I05A TA 44.1626 -121.2675 1011.0 0.0
+I06A TA 43.9437 -120.2107 1296.9 0.0
+I07A TA 44.0820 -119.5042 1293.0 0.0
+I08A TA 43.9104 -118.5692 1189.0 0.0
+I09A TA 43.9733 -117.7407 950.0 0.0
+I10A TA 44.0860 -116.8029 782.0 0.0
+I11A TA 43.9121 -115.9578 1288.0 0.0
+I12A TA 43.7945 -115.1328 1849.0 0.0
+I13A TA 43.9146 -114.1169 2104.0 0.0
+I14A TA 43.9286 -113.4518 1897.0 0.0
+I15A TA 43.9997 -112.4850 1470.0 0.0
+I16A TA 43.8756 -111.4868 1744.0 0.0
+I17A TA 43.9200 -110.5759 2134.0 0.0
+I18A TA 43.7013 -109.8171 2322.0 0.0
+I19A TA 44.0363 -108.9943 1982.0 0.0
+I20A TA 43.9496 -108.1283 1342.0 0.0
+I21A TA 43.8122 -107.2920 1639.0 0.0
+I22A TA 43.8924 -106.4840 1430.0 0.0
+I23A TA 43.9605 -105.4629 1462.0 0.0
+I24A TA 43.8594 -104.6263 1338.0 0.0
+I25A TA 44.0196 -103.7324 1856.0 0.0
+I26A TA 44.0309 -102.8596 971.0 0.0
+I27A TA 44.0690 -102.0525 867.0 0.0
+I28A TA 43.9996 -101.1692 655.0 0.0
+I29A TA 44.0215 -100.3034 592.0 0.0
+I30A TA 43.9143 -99.4262 549.0 0.0
+J02A TA 43.3654 -123.5747 136.2 0.0
+J03A TA 43.3717 -122.9646 292.0 0.0
+J04A TA 43.2406 -122.1093 1952.0 0.0
+J05A TA 43.2843 -121.2357 1540.0 0.0
+J06A TA 43.2515 -120.1528 1407.2 0.0
+J07A TA 43.3736 -119.3105 1273.0 0.0
+J08A TA 43.3580 -118.4743 1229.0 0.0
+J09A TA 43.3471 -117.7542 1303.0 0.0
+J10A TA 43.4275 -116.7670 748.0 0.0
+J11A TA 43.4151 -115.8278 1302.0 0.0
+J12A TA 43.2500 -115.0980 1587.0 0.0
+J13A TA 43.3979 -114.1742 1552.0 0.0
+J14A TA 43.3234 -113.5178 1649.0 0.0
+J15A TA 43.3998 -112.4334 1497.0 0.0
+J16A TA 43.2741 -111.6119 2004.0 0.0
+J17A TA 43.3629 -110.7118 1975.0 0.0
+J18A TA 43.2112 -110.0200 2357.0 0.0
+J19A TA 43.2649 -109.0531 1782.0 0.0
+J20A TA 43.3531 -108.0842 1500.0 0.0
+J21A TA 43.3459 -107.4459 1913.0 0.0
+J22A TA 43.4126 -106.4785 1587.0 0.0
+J23A TA 43.4036 -105.5524 1499.0 0.0
+J24A TA 43.3305 -104.6613 1315.0 0.0
+J25A TA 43.3901 -103.8026 1220.0 0.0
+J26A TA 43.3126 -103.0752 993.0 0.0
+J27A TA 43.2526 -102.0069 1081.0 0.0
+J28A TA 43.4239 -101.2359 835.0 0.0
+J29A TA 43.4061 -100.3253 644.0 0.0
+J30A TA 43.3248 -99.4922 608.0 0.0
+K01A TA 42.8088 -124.4692 175.0 0.0
+K02A TA 42.7667 -123.4898 963.0 0.0
+K04A TA 42.6133 -121.7307 1305.0 0.0
+K05A TA 42.7258 -120.8934 1887.0 0.0
+K06A TA 42.7991 -120.2510 1340.2 0.0
+K07A TA 42.6912 -119.2469 1407.0 0.0
+K08A TA 42.7308 -118.4864 1391.0 0.0
+K09A TA 42.6996 -117.7251 1176.0 0.0
+K10A TA 42.7779 -116.8705 1701.0 0.0
+K11A TA 42.7713 -116.0323 914.0 0.0
+K12A TA 42.6360 -114.9029 1091.0 0.0
+K13A TA 42.6493 -114.0840 1222.0 0.0
+K14A TA 42.5452 -113.1760 1387.0 0.0
+K15A TA 42.6852 -112.5305 1566.0 0.0
+K16A TA 42.8321 -111.5884 1885.0 0.0
+K17A TA 42.7507 -110.9201 1922.0 0.0
+K18A TA 42.6372 -110.0417 2129.0 0.0
+K19A TA 42.8247 -108.8466 1878.0 0.0
+K20A TA 42.6579 -108.3418 1736.0 0.0
+K21A TA 42.6332 -107.2510 1960.0 0.0
+K22A TA 42.6507 -106.5240 1686.0 0.0
+K23A TA 42.7546 -105.6251 1612.0 0.0
+K24A TA 42.7225 -104.8152 1593.0 0.0
+K25A TA 42.6120 -103.8705 1437.0 0.0
+K26A TA 42.7016 -103.1783 1122.0 0.0
+K27A TA 42.7036 -102.2364 1131.0 0.0
+K28A TA 42.7136 -101.3665 986.0 0.0
+K29A TA 42.7612 -100.3182 840.0 0.0
+K30A TA 42.6742 -99.6131 712.0 0.0
+KMSC TA 35.1420 -81.3333 240.0 0.0
+KSCO TA 39.0112 -102.6273 1382.0 0.0
+L02A TA 42.1560 -123.6018 484.6 0.0
+L04A TA 42.1750 -121.8910 1336.0 0.0
+L05A TA 42.0472 -120.8336 1751.0 0.0
+L07A TA 42.0187 -119.3398 1816.0 0.0
+L08A TA 42.1901 -118.3445 1490.0 0.0
+L09A TA 42.0192 -117.6667 1430.5 0.0
+L10A TA 42.0773 -116.4711 1537.0 0.0
+L11A TA 42.1669 -115.7541 1511.0 0.0
+L12A TA 42.1460 -115.0162 1756.0 0.0
+L13A TA 42.0886 -113.9444 1482.0 0.0
+L14A TA 42.0343 -113.2398 1528.0 0.0
+L15A TA 42.0041 -112.3860 1645.6 0.0
+L16A TA 42.0149 -111.4319 2013.0 0.0
+L17A TA 42.0995 -110.8727 1996.0 0.0
+L18A TA 41.9243 -110.0364 2051.0 0.0
+L19A TA 42.1012 -109.3575 2034.0 0.0
+L20A TA 42.0073 -108.3398 2028.0 0.0
+L21A TA 41.9641 -107.3695 2065.0 0.0
+L22A TA 42.0310 -106.4338 2012.0 0.0
+L23A TA 42.1143 -105.7012 2153.0 0.0
+L24A TA 42.0432 -104.9326 1479.0 0.0
+L25A TA 42.1548 -103.9657 1379.0 0.0
+L26A TA 42.0367 -103.0795 1248.0 0.0
+L27A TA 42.0837 -102.3154 1186.0 0.0
+L28A TA 42.0073 -101.5196 1155.0 0.0
+L29A TA 42.0641 -100.4779 901.0 0.0
+L30A TA 41.9492 -99.7196 782.0 0.0
+L31A TA 42.1760 -98.8387 670.0 0.0
+LAVA TA 38.7552 -120.7395 828.0 0.0
+M01C TA 41.8473 -124.1221 38.0 0.0
+M02C TA 41.3920 -122.8538 960.0 0.0
+M03C TA 41.2742 -122.1220 1047.0 0.0
+M04C TA 41.7826 -121.8393 1391.0 0.0
+M05C TA 41.3593 -121.1457 1333.0 0.0
+M06C TA 41.2047 -120.4772 1434.0 0.0
+M07A TA 41.3884 -119.1711 1400.0 0.0
+M08A TA 41.4483 -118.3792 1288.4 0.0
+M09A TA 41.4230 -117.4497 1355.8 0.0
+M10A TA 41.5220 -116.5396 1687.4 0.0
+M11A TA 41.4311 -115.7912 1849.0 0.0
+M12A TA 41.4157 -114.9151 1930.0 0.0
+M13A TA 41.3602 -114.1655 1514.0 0.0
+M14A TA 41.5030 -113.3471 1318.0 0.0
+M15A TA 41.4632 -112.4477 1348.0 0.0
+M16A TA 41.3146 -111.6298 1872.0 0.0
+M17A TA 41.4729 -110.6664 2101.0 0.0
+M18A TA 41.4272 -110.0674 2103.0 0.0
+M19A TA 41.5047 -109.1569 2080.0 0.0
+M20A TA 41.4906 -108.1865 2134.0 0.0
+M21A TA 41.6067 -107.3635 2275.0 0.0
+M22A TA 41.4033 -106.5958 2334.0 0.0
+M23A TA 41.4680 -105.7221 2166.0 0.0
+M24A TA 41.4676 -104.8248 1816.0 0.0
+M25A TA 41.4347 -104.0551 1636.0 0.0
+M26A TA 41.4661 -103.1427 1348.0 0.0
+M27A TA 41.5281 -102.3872 1147.0 0.0
+M28A TA 41.3956 -101.3715 1071.0 0.0
+M29A TA 41.4441 -100.6892 952.0 0.0
+M30A TA 41.5371 -99.8723 853.0 0.0
+M31A TA 41.3440 -98.9766 699.0 0.0
+MDND TA 47.8481 -99.6029 479.0 0.0
+MSTX TA 33.9696 -102.7724 1167.0 0.0
+N02C TA 40.8220 -123.3057 717.0 0.0
+N06A TA 40.7484 -119.8346 1341.0 0.0
+N07A TA 40.7721 -118.9716 1305.5 0.0
+N07B TA 40.7797 -118.9711 1302.0 0.0
+N08A TA 40.7811 -118.1337 1492.9 0.0
+N09A TA 40.8520 -117.5244 1617.3 0.0
+N10A TA 40.7186 -116.5080 1392.0 0.0
+N11A TA 40.8167 -115.7360 1643.0 0.0
+N12A TA 40.8522 -115.0387 1786.0 0.0
+N13A TA 40.8556 -114.2045 1385.0 0.0
+N14A TA 40.8513 -113.1867 1325.0 0.0
+N15A TA 40.8903 -112.5201 1408.0 0.0
+N16A TA 40.8869 -111.4370 2028.0 0.0
+N17A TA 40.9425 -110.8335 2500.0 0.0
+N18A TA 40.9763 -109.6731 1893.0 0.0
+N19A TA 40.8936 -109.1772 1703.0 0.0
+N20A TA 40.8338 -108.2626 2192.0 0.0
+N21A TA 40.7600 -107.5196 2235.0 0.0
+N22A TA 40.8023 -106.4544 2472.0 0.0
+N23A TA 40.8947 -105.9440 2458.0 0.0
+N24A TA 40.8271 -104.8824 1695.0 0.0
+N25A TA 40.8132 -104.0873 1607.0 0.0
+N26A TA 40.8289 -103.2221 1284.0 0.0
+N27A TA 40.8063 -102.5367 1228.0 0.0
+N28A TA 40.6873 -101.5406 1009.0 0.0
+N29A TA 40.8189 -100.6264 897.0 0.0
+N30A TA 40.7957 -100.1001 842.0 0.0
+N31A TA 40.8235 -98.9544 687.0 0.0
+N32A TA 40.7639 -98.2955 587.0 0.0
+O01C TA 40.1398 -123.8195 137.8 0.0
+O02C TA 40.1767 -122.7883 962.0 0.0
+O03C TA 39.9971 -122.0319 85.0 0.0
+O04C TA 40.3200 -121.0860 1513.0 0.0
+O05C TA 39.9620 -120.9176 1032.0 0.0
+O06A TA 40.1650 -119.8275 1228.7 0.0
+O07A TA 40.1614 -118.8772 1203.0 0.0
+O08A TA 40.2903 -118.1550 2137.6 0.0
+O09A TA 40.1697 -117.1899 1491.7 0.0
+O10A TA 40.2917 -116.4999 1470.0 0.0
+O11A TA 40.1313 -115.6570 1948.0 0.0
+O12A TA 40.2679 -114.7454 1743.0 0.0
+O13A TA 40.1314 -113.9814 1573.0 0.0
+O15A TA 40.2807 -112.4694 1611.0 0.0
+O16A TA 40.2067 -111.5024 1789.0 0.0
+O17A TA 40.1948 -110.7378 2079.0 0.0
+O18A TA 40.2655 -110.0082 1582.0 0.0
+O19A TA 40.2982 -109.1236 1649.0 0.0
+O20A TA 40.1348 -108.2416 1915.0 0.0
+O21A TA 40.2149 -107.4705 2343.0 0.0
+O22A TA 40.1618 -106.5470 2747.0 0.0
+O23A TA 40.2109 -105.9178 2635.0 0.0
+O24A TA 40.1227 -105.0729 1553.0 0.0
+O25A TA 40.1129 -104.1201 1444.0 0.0
+O26A TA 40.2422 -103.3128 1413.0 0.0
+O27A TA 40.1900 -102.4794 1206.0 0.0
+O28A TA 40.1262 -101.7735 1032.0 0.0
+O29A TA 40.1253 -100.7977 826.0 0.0
+O30A TA 40.1791 -100.0571 751.0 0.0
+O31A TA 40.1521 -99.3261 631.0 0.0
+O32A TA 40.2610 -98.2764 579.0 0.0
+P01C TA 39.4690 -123.3375 440.9 0.0
+P05C TA 39.3028 -120.6077 1756.5 0.0
+P06A TA 39.6785 -119.8983 1368.6 0.0
+P07A TA 39.5399 -118.8893 1218.6 0.0
+P08A TA 39.6946 -118.0800 1040.2 0.0
+P09A TA 39.5516 -117.1395 1737.7 0.0
+P10A TA 39.6202 -116.4639 1874.0 0.0
+P11A TA 39.5530 -115.7536 1787.0 0.0
+P12A TA 39.4731 -114.9075 1878.0 0.0
+P13A TA 39.4550 -114.0156 1555.0 0.0
+P14A TA 39.5906 -113.0687 1784.0 0.0
+P15A TA 39.5708 -112.2786 1576.0 0.0
+P16A TA 39.6092 -111.6595 1822.0 0.0
+P17A TA 39.4730 -110.7400 1687.0 0.0
+P18A TA 39.6279 -110.2457 2743.0 0.0
+P19A TA 39.6333 -108.9812 2099.0 0.0
+P20A TA 39.4998 -108.3912 1738.0 0.0
+P21A TA 39.5240 -107.4493 2097.0 0.0
+P22A TA 39.5952 -106.7589 2170.0 0.0
+P23A TA 39.3741 -105.8388 3006.0 0.0
+P24A TA 39.5144 -104.9082 1861.0 0.0
+P25A TA 39.5125 -104.1680 1753.0 0.0
+P26A TA 39.5602 -103.3465 1566.0 0.0
+P27A TA 39.5559 -102.5782 1286.0 0.0
+P28A TA 39.5732 -101.7548 1105.0 0.0
+P29A TA 39.6021 -100.9316 929.0 0.0
+P30A TA 39.5442 -100.2468 788.0 0.0
+P31A TA 39.5211 -99.3783 665.0 0.0
+P32A TA 39.6223 -98.6011 524.0 0.0
+P33A TA 39.4363 -97.6946 488.0 0.0
+Q03C TA 38.6333 -122.0145 108.0 0.0
+Q04C TA 38.8390 -121.3769 14.0 0.0
+Q07A TA 38.9383 -118.8078 1275.9 0.0
+Q08A TA 38.8606 -117.9316 1411.5 0.0
+Q09A TA 38.8340 -117.1816 1703.5 0.0
+Q10A TA 38.8247 -116.3999 2027.0 0.0
+Q11A TA 38.8455 -115.6541 1562.0 0.0
+Q12A TA 39.0400 -114.8299 1625.0 0.0
+Q13A TA 38.9551 -114.0202 1583.0 0.0
+Q14A TA 38.9882 -113.2769 1464.0 0.0
+Q15A TA 38.9995 -112.3793 1477.0 0.0
+Q16A TA 38.9176 -111.1716 1912.0 0.0
+Q18A TA 39.1025 -110.1328 1295.0 0.0
+Q19A TA 38.9553 -109.2630 1283.0 0.0
+Q20A TA 38.9541 -108.2956 1716.0 0.0
+Q21A TA 38.8347 -107.5738 1881.0 0.0
+Q22A TA 38.8627 -106.9096 2727.0 0.0
+Q23A TA 38.9045 -105.8337 2774.0 0.0
+Q24A TA 38.9641 -105.1492 2775.0 0.0
+Q25A TA 38.9145 -104.2472 2001.0 0.0
+Q26A TA 38.9234 -103.5195 1546.0 0.0
+Q28A TA 39.0623 -101.8309 1133.0 0.0
+Q29A TA 38.8905 -100.9787 895.0 0.0
+Q30A TA 38.9772 -100.2207 797.0 0.0
+Q31A TA 39.0048 -99.3950 680.0 0.0
+R04C TA 38.2571 -120.9363 115.0 0.0
+R05C TA 38.7028 -120.0755 2366.0 0.0
+R06C TA 38.5226 -119.4509 1698.0 0.0
+R07C TA 38.0890 -119.0469 1996.0 0.0
+R08A TA 38.3489 -118.1064 1419.8 0.0
+R09A TA 38.2397 -117.0718 1759.0 0.0
+R10A TA 38.2886 -116.3021 1600.0 0.0
+R11A TA 38.3489 -115.5854 1756.0 0.0
+R12A TA 38.3281 -114.6076 1845.0 0.0
+R13A TA 38.1802 -113.9691 1969.0 0.0
+R14A TA 38.2987 -113.0213 1542.0 0.0
+R15A TA 38.2108 -112.2767 1861.0 0.0
+R16A TA 38.2843 -111.4827 2202.0 0.0
+R17A TA 38.4187 -110.7109 1357.0 0.0
+R18A TA 38.3862 -109.8942 1828.0 0.0
+R19A TA 38.2916 -109.2607 2035.0 0.0
+R20A TA 38.1888 -108.3792 2003.0 0.0
+R21A TA 38.3685 -107.5501 2260.0 0.0
+R22A TA 38.2289 -106.7561 2773.0 0.0
+R23A TA 38.1907 -105.8264 2547.0 0.0
+R24A TA 38.2315 -105.1075 1978.0 0.0
+R25A TA 38.1474 -104.2834 1461.0 0.0
+R26A TA 38.3009 -103.4470 1304.0 0.0
+R27A TA 38.2859 -102.7980 1201.0 0.0
+R28A TA 38.3423 -101.8106 1108.0 0.0
+R29A TA 38.4243 -101.1383 957.0 0.0
+R30A TA 38.2844 -100.2769 776.0 0.0
+R31A TA 38.2888 -99.5271 667.0 0.0
+S04C TA 37.5049 -121.3278 310.0 0.0
+S05C TA 37.3460 -120.3296 85.0 0.0
+S06C TA 37.8817 -119.8488 1377.0 0.0
+S08C TA 37.4993 -118.1711 3087.0 0.0
+S09A TA 37.7243 -117.2246 1767.8 0.0
+S10A TA 37.9230 -116.5948 1796.0 0.0
+S11A TA 37.6444 -115.7472 1456.0 0.0
+S12A TA 37.6083 -114.8489 1479.0 0.0
+S13A TA 37.5808 -113.8604 1698.0 0.0
+S14A TA 37.7601 -113.1684 1746.0 0.0
+S15A TA 37.6761 -112.3627 2102.0 0.0
+S16A TA 37.7222 -111.5961 1785.0 0.0
+S17A TA 37.6354 -110.8018 1373.0 0.0
+S18A TA 37.6903 -109.9948 2351.0 0.0
+S19A TA 37.7459 -109.1366 2059.0 0.0
+S20A TA 37.8300 -108.3605 2355.0 0.0
+S21A TA 37.6620 -107.7917 2723.0 0.0
+S22A TA 37.7461 -106.8293 2626.0 0.0
+S23A TA 37.7059 -106.0023 2323.0 0.0
+S24A TA 37.6407 -105.2126 2672.0 0.0
+S25A TA 37.6606 -104.4349 1821.0 0.0
+S26A TA 37.6216 -103.4711 1394.0 0.0
+S27A TA 37.6837 -102.8982 1439.0 0.0
+S28A TA 37.5899 -101.8979 1064.0 0.0
+S29A TA 37.6199 -101.1180 931.0 0.0
+S30A TA 37.6402 -100.4640 853.0 0.0
+S31A TA 37.6193 -99.4845 700.0 0.0
+S32A TA 37.7001 -98.9363 558.0 0.0
+SFIN TA 40.3790 -87.0967 165.0 0.0
+SPMN TA 45.2246 -92.8037 311.0 0.0
+SUTB TA 39.2291 -121.7861 252.0 0.0
+T05C TA 36.8953 -120.6737 46.0 0.0
+T06C TA 37.0069 -119.7089 216.0 0.0
+T11A TA 37.2408 -115.2202 1402.0 0.0
+T12A TA 36.7256 -114.7147 538.0 0.0
+T13A TA 37.0195 -113.9073 878.0 0.0
+T14A TA 37.0615 -113.0840 1529.0 0.0
+T15A TA 37.0183 -112.3824 1561.0 0.0
+T16A TA 36.9839 -111.5060 1177.0 0.0
+T17A TA 36.9969 -110.8041 1876.0 0.0
+T18A TA 37.1364 -109.8741 1297.0 0.0
+T19A TA 36.8299 -109.0249 1736.0 0.0
+T21A TA 36.9914 -107.5342 2006.0 0.0
+T22A TA 37.0138 -106.9030 2224.0 0.0
+T23A TA 37.0361 -106.0372 2429.0 0.0
+T24A TA 37.0738 -105.0521 2510.0 0.0
+T24B TA 37.0624 -105.3817 2625.0 0.0
+T25A TA 37.1388 -104.4108 2017.0 0.0
+T26A TA 37.1747 -103.5933 1830.0 0.0
+T27A TA 37.0614 -102.7214 1366.0 0.0
+T28A TA 37.1204 -102.1136 1148.0 0.0
+T29A TA 37.1565 -101.2942 936.0 0.0
+T30A TA 37.0526 -100.5274 778.0 0.0
+T31A TA 37.0977 -99.6771 564.0 0.0
+TIGA TA 31.4389 -83.5898 109.0 0.0
+TUL1 TA 35.9104 -95.7919 256.0 0.0
+U04C TA 36.3627 -120.7828 812.6 0.0
+U05C TA 36.3356 -120.1205 92.0 0.0
+U10A TA 36.4193 -116.3297 668.0 0.0
+U11A TA 36.4230 -115.3835 867.0 0.0
+U12A TA 36.4321 -114.5388 644.0 0.0
+U13A TA 36.4151 -113.9653 712.0 0.0
+U14A TA 36.4182 -113.1805 1916.0 0.0
+U15A TA 36.4280 -112.2915 2489.0 0.0
+U16A TA 36.1428 -111.1297 1544.0 0.0
+U17A TA 36.5996 -110.6624 1976.0 0.0
+U18A TA 36.4199 -109.8696 1878.0 0.0
+U19A TA 36.2925 -109.2076 2166.0 0.0
+U20A TA 36.3758 -108.5204 1682.0 0.0
+U21A TA 36.4277 -107.6578 2124.0 0.0
+U22A TA 36.3773 -106.8546 2119.0 0.0
+U23A TA 36.3289 -106.1919 2093.0 0.0
+U24A TA 36.4093 -105.2784 2573.0 0.0
+U25A TA 36.3998 -104.4081 1845.0 0.0
+U26A TA 36.3945 -103.7439 1861.0 0.0
+U27A TA 36.4213 -102.8246 1342.0 0.0
+U28A TA 36.3793 -102.2243 1165.0 0.0
+U29A TA 36.4519 -101.2702 928.0 0.0
+U30A TA 36.5304 -100.6509 881.0 0.0
+U31A TA 36.3695 -99.8508 725.0 0.0
+V03C TA 36.0214 -121.2356 336.0 0.0
+V04C TA 35.6360 -120.8699 431.0 0.0
+V05C TA 35.8667 -119.9028 118.0 0.0
+V11A TA 35.8384 -115.4305 1117.0 0.0
+V12A TA 35.7266 -114.8511 1098.0 0.0
+V13A TA 35.8522 -113.9840 1402.0 0.0
+V14A TA 35.6339 -113.1053 1731.0 0.0
+V15A TA 35.8191 -112.1731 1891.0 0.0
+V17A TA 35.6218 -110.7938 1596.0 0.0
+V18A TA 35.7113 -109.9327 1977.0 0.0
+V19A TA 35.7147 -109.0456 2082.0 0.0
+V20A TA 35.7992 -108.4662 1889.0 0.0
+V21A TA 35.8054 -107.6378 2046.0 0.0
+V22A TA 35.9138 -106.9094 2164.0 0.0
+V23A TA 35.7452 -106.1833 1991.0 0.0
+V24A TA 35.7259 -105.2716 2187.0 0.0
+V25A TA 35.8383 -104.6164 1861.0 0.0
+V26A TA 35.7964 -103.7860 1362.0 0.0
+V27A TA 35.7569 -102.8361 1228.0 0.0
+V28A TA 35.7505 -102.2243 1163.0 0.0
+V29A TA 35.8734 -101.5209 953.0 0.0
+V30A TA 35.7623 -100.6892 931.0 0.0
+V31A TA 35.7739 -99.8425 687.0 0.0
+W12A TA 35.3010 -114.8701 774.5 0.0
+W13A TA 35.0990 -113.8854 1988.2 0.0
+W14A TA 35.2127 -113.0834 1756.3 0.0
+W15A TA 35.1787 -112.2666 2034.2 0.0
+W16A TA 35.0951 -111.5318 2192.0 0.0
+W17A TA 35.0789 -110.7127 1461.0 0.0
+W18A TA 35.1181 -109.7357 1750.0 0.0
+W19A TA 35.1118 -109.3879 1785.0 0.0
+W20A TA 35.1259 -108.5001 2110.0 0.0
+W21A TA 35.1170 -107.6476 1966.0 0.0
+W22A TA 35.0719 -106.8674 1813.0 0.0
+W23A TA 35.1605 -106.1503 2020.0 0.0
+W24A TA 35.2106 -105.4075 1902.0 0.0
+W25A TA 35.2177 -104.4606 1446.0 0.0
+W26A TA 35.0891 -103.7656 1314.0 0.0
+W27A TA 35.0637 -103.0627 1233.0 0.0
+W28A TA 35.2574 -102.2063 1156.0 0.0
+W29A TA 35.1212 -101.6476 1072.0 0.0
+W30A TA 35.1818 -100.5770 870.0 0.0
+W31A TA 35.1864 -99.9435 644.0 0.0
+WHTX TA 31.9913 -97.4561 190.0 0.0
+X13A TA 34.5935 -113.8302 889.4 0.0
+X14A TA 34.4692 -112.8906 1077.5 0.0
+X15A TA 34.4866 -112.2368 1331.8 0.0
+X16A TA 34.4178 -111.4411 1759.4 0.0
+X17A TA 34.3371 -110.8058 2280.0 0.0
+X18A TA 34.5293 -109.9501 1751.0 0.0
+X19A TA 34.4275 -109.2901 1861.0 0.0
+X20A TA 34.5421 -108.4980 2057.0 0.0
+X21A TA 34.4457 -107.7857 2141.0 0.0
+X22A TA 34.5058 -107.0102 1604.0 0.0
+X23A TA 34.5810 -106.1881 1948.0 0.0
+X24A TA 34.5646 -105.4349 1917.0 0.0
+X25A TA 34.5271 -104.6621 1494.0 0.0
+X26A TA 34.5508 -103.8103 1393.0 0.0
+X27A TA 34.6469 -103.0974 1304.0 0.0
+X28A TA 34.5185 -102.1973 1160.0 0.0
+X29A TA 34.4452 -101.7008 1039.0 0.0
+X30A TA 34.4461 -100.8740 698.0 0.0
+X31A TA 34.6309 -99.9793 514.0 0.0
+Y12C TA 33.7503 -114.5238 196.0 0.0
+Y13A TA 33.8142 -113.8287 356.9 0.0
+Y14A TA 33.9383 -113.0048 730.7 0.0
+Y15A TA 33.9535 -112.3331 572.0 0.0
+Y16A TA 33.8798 -111.4783 1068.0 0.0
+Y17A TA 33.6953 -110.8444 866.0 0.0
+Y18A TA 33.7776 -110.0341 1512.0 0.0
+Y19A TA 33.9571 -109.2541 2448.0 0.0
+Y20A TA 33.9085 -108.3769 2177.0 0.0
+Y21A TA 34.0087 -107.6740 2168.0 0.0
+Y22A TA 33.9370 -106.9652 1674.0 0.0
+Y22C TA 34.0741 -106.9211 1436.0 0.0
+Y22D TA 34.0739 -106.9210 1436.0 0.0
+Y23A TA 33.9315 -106.0549 1789.0 0.0
+Y24A TA 33.9257 -105.4361 1827.0 0.0
+Y25A TA 33.9229 -104.6928 1364.0 0.0
+Y26A TA 33.9232 -103.8246 1371.0 0.0
+Y27A TA 33.8839 -103.1633 1253.0 0.0
+Y28A TA 33.9086 -102.2479 1068.0 0.0
+Y29A TA 33.8602 -101.6712 991.0 0.0
+Y30A TA 33.8766 -100.8978 812.0 0.0
+Y31A TA 33.9629 -100.2615 530.0 0.0
+Y32A TA 34.0036 -99.4420 412.0 0.0
+Z13A TA 33.1999 -113.6568 375.0 0.0
+Z14A TA 33.3627 -112.9458 297.8 0.0
+Z15A TA 33.2893 -112.1581 318.0 0.0
+Z16A TA 33.3410 -111.4273 537.0 0.0
+Z17A TA 33.2969 -110.4723 846.0 0.0
+Z18A TA 33.0851 -110.0362 812.0 0.0
+Z19A TA 33.2921 -109.2657 1397.0 0.0
+Z20A TA 33.1132 -108.5922 1547.0 0.0
+Z21A TA 33.3086 -107.6712 1964.0 0.0
+Z22A TA 33.2555 -106.9639 1497.0 0.0
+Z23A TA 33.2621 -106.2319 1278.0 0.0
+Z24A TA 33.3298 -105.3649 1863.0 0.0
+Z25A TA 33.2797 -104.7171 1233.0 0.0
+Z26A TA 33.2716 -103.9798 1163.0 0.0
+Z27A TA 33.3150 -103.2145 1197.0 0.0
+Z28A TA 33.2884 -102.3866 1045.0 0.0
+Z29A TA 33.2595 -101.7062 938.0 0.0
+Z30A TA 33.2861 -101.1282 729.0 0.0
+Z31A TA 33.3183 -100.1435 547.0 0.0
+Z32A TA 33.3066 -99.4783 427.0 0.0
+BAR TS 32.6800 -116.6720 548.0 0.0
+CALB TS 34.1430 -118.6270 0.0 0.0
+CWC TS 36.4399 -118.0802 1553.0 0.0
+DGR TS 33.6500 -117.0090 700.0 0.0
+GLA TS 33.0520 -114.8270 623.0 4.0
+GPO TS 35.6494 -117.6619 735.0 0.0
+GSC TS 35.3028 -116.8083 990.0 0.0
+ISA TS 35.6630 -118.4733 835.0 0.0
+MLAC TS 37.6310 -118.8340 2170.0 0.0
+NEE TS 34.8230 -114.5960 139.0 0.0
+OSI TS 34.6145 -118.7235 706.0 0.0
+PAS TS 34.1484 -118.1712 314.0 0.0
+PFO TS 33.6092 -116.4553 1280.0 0.0
+RPV TS 33.7438 -118.4035 115.0 0.0
+SBC TS 34.4417 -119.7133 90.0 0.0
+SMTC TS 32.9490 -115.7200 -50.0 0.0
+SNCC TS 33.2480 -119.5240 227.0 0.0
+SVD TS 34.1045 -117.0970 600.0 0.0
+USC TS 34.0210 -118.2870 60.0 0.0
+VTV TS 34.5670 -117.3330 847.0 0.0
+ANPB TW 25.1865 121.5202 825.0 0.0
+HWAB TW 23.9800 121.6000 0.0 0.0
+KMNB TW 24.4638 118.3884 43.0 0.0
+LYUB TW 22.0017 121.5840 40.0 0.0
+MATB TW 26.1515 119.9456 75.1 0.0
+NACB TW 24.1755 121.5867 130.0 0.0
+SSLB TW 23.7909 120.9510 450.0 0.0
+TATO TW 24.9754 121.4881 53.0 90.0
+TDCB TW 24.2574 121.2550 1280.0 0.0
+TPUB TW 23.3038 120.6270 370.0 0.0
+TWGB TW 22.8207 121.0717 195.0 0.0
+TWKB TW 21.9433 120.8053 90.0 0.0
+WFSB TW 25.0710 121.7810 100.0 0.0
+YHNB TW 24.6698 121.3748 775.0 0.0
+YULB TW 23.3964 121.2972 294.7 0.0
+AAM US 42.3012 -83.6567 172.0 0.0
+ACSO US 40.2319 -82.9820 288.0 0.0
+AGMN US 48.2977 -95.8619 351.1 0.0
+AHID US 42.7654 -111.1004 1960.0 0.0
+AMTX US 34.8837 -101.6808 1010.0 0.0
+BINY US 42.1993 -75.9861 498.0 0.0
+BLA US 37.2113 -80.4210 634.0 0.0
+BMN US 40.4315 -117.2218 1500.0 0.0
+BMO US 44.8525 -117.3060 1154.0 0.0
+BOZ US 45.6470 -111.6296 1589.0 0.0
+BRAL US 31.1687 -87.0506 61.0 0.0
+BW06 US 42.7667 -109.5583 2224.0 0.0
+CBKS US 38.8140 -99.7374 677.0 0.0
+CBM US 46.9325 -68.1208 250.0 0.0
+CBN US 38.2046 -77.3732 70.0 0.0
+CCM US 38.0557 -91.2446 171.0 51.0
+CEH US 35.8908 -79.0928 152.0 0.0
+CMB US 38.0350 -120.3850 719.0 0.0
+CNNC US 35.2393 -77.8901 29.0 0.0
+COR US 44.5857 -123.3032 121.0 0.0
+COWI US 46.1003 -89.1369 523.3 0.0
+DGMT US 48.4702 -104.1959 646.0 0.0
+DUG US 40.1950 -112.8133 1477.0 0.0
+DWPF US 28.1103 -81.4328 -142.0 162.0
+ECSD US 43.7337 -96.6141 432.9 45.1
+EGAK US 64.7774 -141.1581 296.6 0.0
+EGMT US 48.0240 -109.7547 1055.0 0.0
+ELK US 40.7448 -115.2388 2210.0 0.0
+ERPA US 42.1175 -79.9891 306.3 0.0
+EYMN US 47.9462 -91.4950 475.0 0.0
+GLMI US 44.8245 -84.6174 387.0 0.0
+GOGA US 33.4112 -83.4666 116.5 0.0
+GOL US 39.7003 -105.3711 2359.0 0.0
+GWDE US 38.8256 -75.6171 19.0 0.0
+HAWA US 46.3925 -119.5326 364.0 0.0
+HDIL US 40.5560 -89.2940 219.4 0.0
+HKT US 29.9500 -95.8333 -415.0 1.0
+HLID US 43.5625 -114.4138 1772.0 0.0
+HRV US 42.5064 -71.5583 180.0 0.0
+HWUT US 41.6069 -111.5652 1830.0 0.0
+ISA US 35.6628 -118.4740 873.0 0.0
+ISCO US 39.7997 -105.6134 2743.0 0.0
+JCT US 30.4794 -99.8022 591.0 0.0
+JFWS US 42.9142 -90.2481 335.0 0.0
+KNB US 37.0166 -112.8224 1715.0 0.0
+KSU1 US 39.1009 -96.6094 286.5 30.5
+KVTX US 27.5459 -97.8929 14.6 0.0
+LAO US 46.6885 -106.2231 902.0 0.0
+LBNH US 44.2401 -71.9259 367.0 0.0
+LKWY US 44.5652 -110.4000 2424.0 0.0
+LONY US 44.6197 -74.5829 440.0 0.0
+LRAL US 33.0348 -86.9978 130.0 0.0
+LSCT US 41.6784 -73.2244 318.0 0.0
+LTX US 29.3339 -103.6669 1013.0 0.0
+MCWV US 39.6581 -79.8456 280.0 0.0
+MIAR US 34.5454 -93.5765 207.0 0.0
+MNTX US 31.6985 -105.3821 404.8 0.0
+MNV US 38.4328 -118.1531 1524.0 0.0
+MSO US 46.8292 -113.9406 1264.0 0.0
+MVCO US 37.2103 -108.4986 2169.9 0.0
+MYNC US 35.0739 -84.1279 550.0 0.0
+NATX US 31.7598 -94.6610 168.0 0.0
+NCB US 43.9708 -74.2236 500.0 0.0
+NEW US 48.2633 -117.1200 760.0 0.0
+NHSC US 33.1067 -80.1778 11.0 0.0
+NLWA US 47.3917 -123.8691 640.0 0.0
+OCWA US 47.7489 -124.1781 671.0 0.0
+OGNE US 40.9451 -102.0330 1054.0 0.0
+OXF US 34.5118 -89.4092 101.0 0.0
+PKME US 45.2644 -69.2917 108.5 0.0
+RLMT US 45.1221 -109.2673 2086.0 0.0
+RSNY US 44.5483 -74.5300 396.0 0.0
+RSSD US 44.1204 -104.0360 2060.0 100.0
+SAO US 36.7650 -121.4450 350.0 0.0
+SCIA US 41.9069 -93.2153 311.5 0.0
+SDCO US 37.7456 -105.5012 2569.0 0.0
+SSPA US 40.6401 -77.8914 252.0 100.0
+TPNV US 36.9488 -116.2495 1600.0 0.0
+TUC US 32.3097 -110.7842 906.0 0.0
+TZTN US 36.5439 -83.5490 394.0 0.0
+VBMS US 32.2185 -90.5177 16.0 0.0
+WCI US 38.2290 -86.2938 500.0 37.0
+WDC US 40.5800 -122.5397 300.0 0.0
+WMOK US 34.7379 -98.7810 486.0 0.0
+WRAK US 56.4191 -132.3466 80.9 0.0
+WUAZ US 35.5169 -111.3739 1592.0 0.0
+WVOR US 42.4339 -118.6367 1344.0 0.0
+WVT US 36.1300 -87.8300 153.0 0.0
+YSNY US 42.4758 -78.5375 628.0 0.0
+BLOW UW 44.6837 -122.1862 652.0 0.0
+BRAN UW 45.9734 -117.2277 1136.0 0.0
+DAVN UW 47.8006 -118.2741 495.0 0.0
+FISH UW 45.9306 -123.5591 324.6 0.0
+IRON UW 43.3580 -118.4743 1229.0 0.0
+IZEE UW 44.0820 -119.5042 1293.0 0.0
+KENT UW 45.2357 -120.6351 780.0 0.0
+LCCR UW 45.2059 -122.4784 272.8 0.0
+LEBA UW 46.5459 -123.5632 72.8 0.0
+MRBL UW 48.5183 -121.4845 75.0 0.0
+OMAK UW 48.3584 -119.3332 696.0 0.0
+PASS UW 48.9983 -122.0852 174.3 0.0
+STOR UW 47.1881 -121.9888 266.4 0.0
+TREE UW 42.7258 -120.8934 1887.0 0.0
+TUCA UW 46.5139 -118.1455 304.0 0.0
+UMAT UW 45.2904 -118.9595 1318.0 0.0
+WOLL UW 47.0573 -118.9210 385.0 0.0
+YACT UW 45.9325 -122.4193 211.2 0.0
Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/UTILS/create_specfem3D_gpu_cuda_method_stubs.pl
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/UTILS/create_specfem3D_gpu_cuda_method_stubs.pl 2012-07-19 21:55:39 UTC (rev 20529)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/UTILS/create_specfem3D_gpu_cuda_method_stubs.pl 2012-07-19 22:05:23 UTC (rev 20530)
@@ -17,11 +17,12 @@
/*
!=====================================================================
!
-! S p e c f e m 3 D V e r s i o n 2 . 0
-! ---------------------------------------
+! S p e c f e m 3 D G l o b e V e r s i o n 5 . 1
+! --------------------------------------------------
!
! Main authors: Dimitri Komatitsch and Jeroen Tromp
-! Princeton University, USA and University of Pau / CNRS / INRIA
+! Princeton University, USA
+! and University of Pau / CNRS / INRIA, France
! (c) Princeton University / California Institute of Technology and University of Pau / CNRS / INRIA
! April 2011
!
@@ -107,7 +108,7 @@
# function declaration
if($line =~ /{/){
# function declaration ends
- if( $line =~ /PREPARE_CUDA_DEVICE/ ){
+ if( $line =~ /INITIALIZE_CUDA_DEVICE/ ){
# adds warning
print IOUT "$line \n$warning\} \n\n";
}else{
Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/setup/constants.h.in
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/setup/constants.h.in 2012-07-19 21:55:39 UTC (rev 20529)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/setup/constants.h.in 2012-07-19 22:05:23 UTC (rev 20530)
@@ -46,16 +46,6 @@
! set to SIZE_DOUBLE to run in double precision (increases memory size by 2)
integer, parameter :: CUSTOM_REAL = @CUSTOM_REAL@
-! this for non blocking assembly
- integer, parameter :: ELEMENTS_NONBLOCKING_CM_IC = 1500
- integer, parameter :: ELEMENTS_NONBLOCKING_OC = 3000
-
-! add mesh coloring for the GPU + MPI implementation
- logical, parameter :: USE_MESH_COLORING_GPU = .false.
- integer, parameter :: MAX_NUMBER_OF_COLORS = 1000
- integer, parameter :: NGNOD_HEXAHEDRA = 8
-
-
! if files on a local path on each node are also seen as global with same path
! set to .true. typically on a machine with a common (shared) file system, e.g. LUSTRE, GPFS or NFS-mounted /home
! set to .false. typically on a cluster of nodes with local disks used to store the mesh (not very common these days)
@@ -241,7 +231,20 @@
! the mesher) and use them for the computation of boundary kernel (in the solver)
logical, parameter :: SAVE_BOUNDARY_MESH = .false.
+! add mesh coloring for the GPU + MPI implementation
+ logical, parameter :: USE_MESH_COLORING_GPU = .false.
+ integer, parameter :: MAX_NUMBER_OF_COLORS = 1000
+! enhanced coloring:
+!
+! using Droux algorithm
+! try several times with one more color before giving up
+ logical, parameter :: USE_DROUX_OPTIMIZATION = .false.
+ integer, parameter :: MAX_NB_TRIES_OF_DROUX_1993 = 15
+!
+! postprocess the colors to balance them if Droux (1993) algorithm is not used
+ logical, parameter :: BALANCE_COLORS_SIMPLE_ALGO = .false.
+
!!-----------------------------------------------------------
!!
!! adjoint kernel outputs
Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/auxiliaries/combine_vol_data_vtk.f90
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/auxiliaries/combine_vol_data_vtk.f90 2012-07-19 21:55:39 UTC (rev 20529)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/auxiliaries/combine_vol_data_vtk.f90 2012-07-19 22:05:23 UTC (rev 20530)
@@ -151,7 +151,7 @@
! resolution
read(arg(6),*) ires
- di = 0; dj = 0; dk = 0
+ di = 0; dj = 0; dk = 0
if (ires == 0) then
HIGH_RESOLUTION_MESH = .false.
di = NGLLX-1; dj = NGLLY-1; dk = NGLLZ-1
Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/cuda/check_fields_cuda.cu
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/cuda/check_fields_cuda.cu 2012-07-19 21:55:39 UTC (rev 20529)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/cuda/check_fields_cuda.cu 2012-07-19 22:05:23 UTC (rev 20530)
@@ -100,26 +100,26 @@
cudaError_t err = cudaGetLastError();
if (err != cudaSuccess){
fprintf(stderr,"Error after %s: %s\n", kernel_name, cudaGetErrorString(err));
-
+
//debugging
//pause_for_debugger(0);
-
+
// outputs error file
FILE* fp;
int myrank;
- char filename[BUFSIZ];
+ char filename[BUFSIZ];
#ifdef WITH_MPI
MPI_Comm_rank(MPI_COMM_WORLD, &myrank);
#else
myrank = 0;
-#endif
+#endif
sprintf(filename,"../in_out_files/OUTPUT_FILES/error_message_%06d.txt",myrank);
fp = fopen(filename,"a+");
if (fp != NULL){
fprintf(fp,"Error after %s: %s\n", kernel_name, cudaGetErrorString(err));
fclose(fp);
}
-
+
// stops program
//free(kernel_name);
#ifdef WITH_MPI
@@ -135,23 +135,23 @@
void exit_on_error(char* info) {
printf("\nERROR: %s\n",info);
fflush(stdout);
-
+
// outputs error file
FILE* fp;
int myrank;
- char filename[BUFSIZ];
+ char filename[BUFSIZ];
#ifdef WITH_MPI
MPI_Comm_rank(MPI_COMM_WORLD, &myrank);
#else
myrank = 0;
-#endif
+#endif
sprintf(filename,"../in_out_files/OUTPUT_FILES/error_message_%06d.txt",myrank);
fp = fopen(filename,"a+");
if (fp != NULL){
fprintf(fp,"ERROR: %s\n",info);
fclose(fp);
}
-
+
// stops program
#ifdef WITH_MPI
MPI_Abort(MPI_COMM_WORLD,1);
@@ -168,23 +168,23 @@
{
printf("\nCUDA error !!!!! <%s> !!!!! \nat CUDA call error code: # %d\n",cudaGetErrorString(err),num);
fflush(stdout);
-
+
// outputs error file
FILE* fp;
int myrank;
- char filename[BUFSIZ];
+ char filename[BUFSIZ];
#ifdef WITH_MPI
MPI_Comm_rank(MPI_COMM_WORLD, &myrank);
#else
myrank = 0;
-#endif
+#endif
sprintf(filename,"../in_out_files/OUTPUT_FILES/error_message_%06d.txt",myrank);
fp = fopen(filename,"a+");
if (fp != NULL){
fprintf(fp,"\nCUDA error !!!!! <%s> !!!!! \nat CUDA call error code: # %d\n",cudaGetErrorString(err),num);
fclose(fp);
}
-
+
// stops program
#ifdef WITH_MPI
MPI_Abort(MPI_COMM_WORLD,1);
@@ -720,20 +720,20 @@
}
*d_max = max;
*/
-
+
// reduction example:
__shared__ realw sdata[BLOCKSIZE_TRANSFER] ;
-
+
// load shared mem
unsigned int tid = threadIdx.x;
unsigned int bx = blockIdx.y*gridDim.x+blockIdx.x;
- unsigned int i = tid + bx*blockDim.x;
-
+ unsigned int i = tid + bx*blockDim.x;
+
// loads absolute values into shared memory
sdata[tid] = (i < size) ? fabs(array[i]) : 0.0 ;
-
+
__syncthreads();
-
+
// do reduction in shared mem
for(unsigned int s=blockDim.x/2; s>0; s>>=1)
{
@@ -745,10 +745,10 @@
}
__syncthreads();
}
-
+
// write result for this block to global mem
if (tid == 0) d_max[bx] = sdata[0];
-
+
}
/* ----------------------------------------------------------------------------------------------- */
@@ -894,19 +894,19 @@
// reduction example:
__shared__ realw sdata[BLOCKSIZE_TRANSFER] ;
-
+
// load shared mem
unsigned int tid = threadIdx.x;
unsigned int bx = blockIdx.y*gridDim.x+blockIdx.x;
- unsigned int i = tid + bx*blockDim.x;
-
+ unsigned int i = tid + bx*blockDim.x;
+
// loads values into shared memory: assume array is a vector array
sdata[tid] = (i < size) ? sqrt(array[i*3]*array[i*3]
+ array[i*3+1]*array[i*3+1]
+ array[i*3+2]*array[i*3+2]) : 0.0 ;
-
+
__syncthreads();
-
+
// do reduction in shared mem
for(unsigned int s=blockDim.x/2; s>0; s>>=1)
{
@@ -918,10 +918,10 @@
}
__syncthreads();
}
-
+
// write result for this block to global mem
if (tid == 0) d_max[bx] = sdata[0];
-
+
}
/* ----------------------------------------------------------------------------------------------- */
@@ -1057,7 +1057,7 @@
// launch simple reduction kernel
realw* h_max;
int blocksize = BLOCKSIZE_TRANSFER;
-
+
// crust_mantle strain arrays
size = NGLL3*(mp->NSPEC_CRUST_MANTLE_STRAIN_ONLY);
Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/cuda/compute_coupling_cuda.cu
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/cuda/compute_coupling_cuda.cu 2012-07-19 21:55:39 UTC (rev 20529)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/cuda/compute_coupling_cuda.cu 2012-07-19 22:05:23 UTC (rev 20530)
@@ -610,7 +610,7 @@
// we divide by rmass() which is 1 / M
// we use the total force which includes the Coriolis term above
force_normal_comp = accel_crust_mantle[iglob*3]*nx / rmassx_crust_mantle[iglob]
- + accel_crust_mantle[iglob*3+1]*ny / rmassy_crust_mantle[iglob]
+ + accel_crust_mantle[iglob*3+1]*ny / rmassy_crust_mantle[iglob]
+ accel_crust_mantle[iglob*3+2]*nz / rmassz_crust_mantle[iglob];
additional_term_x = (rmass_ocean_load[iglob] - rmassx_crust_mantle[iglob]) * force_normal_comp;
@@ -630,7 +630,7 @@
extern "C"
void FC_FUNC_(compute_coupling_ocean_cuda,
COMPUTE_COUPLING_OCEAN_CUDA)(long* Mesh_pointer_f,
- int* NCHUNKS_VAL) {
+ int* NCHUNKS_VAL) {
TRACE("compute_coupling_ocean_cuda");
@@ -656,61 +656,61 @@
if( *NCHUNKS_VAL != 6 && mp->absorbing_conditions){
compute_coupling_ocean_cuda_kernel<<<grid,threads>>>(mp->d_accel_crust_mantle,
- mp->d_rmassx_crust_mantle,
- mp->d_rmassy_crust_mantle,
- mp->d_rmassz_crust_mantle,
- mp->d_rmass_ocean_load,
- mp->d_normal_top_crust_mantle,
- mp->d_ibool_crust_mantle,
- mp->d_ibelm_top_crust_mantle,
- mp->d_updated_dof_ocean_load,
- mp->nspec2D_top_crust_mantle);
+ mp->d_rmassx_crust_mantle,
+ mp->d_rmassy_crust_mantle,
+ mp->d_rmassz_crust_mantle,
+ mp->d_rmass_ocean_load,
+ mp->d_normal_top_crust_mantle,
+ mp->d_ibool_crust_mantle,
+ mp->d_ibelm_top_crust_mantle,
+ mp->d_updated_dof_ocean_load,
+ mp->nspec2D_top_crust_mantle);
// for backward/reconstructed potentials
if( mp->simulation_type == 3 ) {
// re-initializes array
print_CUDA_error_if_any(cudaMemset(mp->d_updated_dof_ocean_load,0,
- sizeof(int)*mp->NGLOB_CRUST_MANTLE_OCEANS),88502);
+ sizeof(int)*mp->NGLOB_CRUST_MANTLE_OCEANS),88502);
compute_coupling_ocean_cuda_kernel<<<grid,threads>>>(mp->d_b_accel_crust_mantle,
- mp->d_rmassx_crust_mantle,
- mp->d_rmassy_crust_mantle,
- mp->d_rmassz_crust_mantle,
- mp->d_rmass_ocean_load,
- mp->d_normal_top_crust_mantle,
- mp->d_ibool_crust_mantle,
- mp->d_ibelm_top_crust_mantle,
- mp->d_updated_dof_ocean_load,
- mp->nspec2D_top_crust_mantle);
+ mp->d_rmassx_crust_mantle,
+ mp->d_rmassy_crust_mantle,
+ mp->d_rmassz_crust_mantle,
+ mp->d_rmass_ocean_load,
+ mp->d_normal_top_crust_mantle,
+ mp->d_ibool_crust_mantle,
+ mp->d_ibelm_top_crust_mantle,
+ mp->d_updated_dof_ocean_load,
+ mp->nspec2D_top_crust_mantle);
}
}else{
compute_coupling_ocean_cuda_kernel<<<grid,threads>>>(mp->d_accel_crust_mantle,
- mp->d_rmassz_crust_mantle,
- mp->d_rmassz_crust_mantle,
- mp->d_rmassz_crust_mantle,
- mp->d_rmass_ocean_load,
- mp->d_normal_top_crust_mantle,
- mp->d_ibool_crust_mantle,
- mp->d_ibelm_top_crust_mantle,
- mp->d_updated_dof_ocean_load,
- mp->nspec2D_top_crust_mantle);
+ mp->d_rmassz_crust_mantle,
+ mp->d_rmassz_crust_mantle,
+ mp->d_rmassz_crust_mantle,
+ mp->d_rmass_ocean_load,
+ mp->d_normal_top_crust_mantle,
+ mp->d_ibool_crust_mantle,
+ mp->d_ibelm_top_crust_mantle,
+ mp->d_updated_dof_ocean_load,
+ mp->nspec2D_top_crust_mantle);
// for backward/reconstructed potentials
if( mp->simulation_type == 3 ) {
// re-initializes array
print_CUDA_error_if_any(cudaMemset(mp->d_updated_dof_ocean_load,0,
- sizeof(int)*mp->NGLOB_CRUST_MANTLE_OCEANS),88502);
+ sizeof(int)*mp->NGLOB_CRUST_MANTLE_OCEANS),88502);
compute_coupling_ocean_cuda_kernel<<<grid,threads>>>(mp->d_b_accel_crust_mantle,
- mp->d_rmassz_crust_mantle,
- mp->d_rmassz_crust_mantle,
- mp->d_rmassz_crust_mantle,
- mp->d_rmass_ocean_load,
- mp->d_normal_top_crust_mantle,
- mp->d_ibool_crust_mantle,
- mp->d_ibelm_top_crust_mantle,
- mp->d_updated_dof_ocean_load,
- mp->nspec2D_top_crust_mantle);
+ mp->d_rmassz_crust_mantle,
+ mp->d_rmassz_crust_mantle,
+ mp->d_rmassz_crust_mantle,
+ mp->d_rmass_ocean_load,
+ mp->d_normal_top_crust_mantle,
+ mp->d_ibool_crust_mantle,
+ mp->d_ibelm_top_crust_mantle,
+ mp->d_updated_dof_ocean_load,
+ mp->nspec2D_top_crust_mantle);
}
}
Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/cuda/compute_forces_crust_mantle_cuda.cu
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/cuda/compute_forces_crust_mantle_cuda.cu 2012-07-19 21:55:39 UTC (rev 20529)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/cuda/compute_forces_crust_mantle_cuda.cu 2012-07-19 22:05:23 UTC (rev 20530)
@@ -689,11 +689,11 @@
//costheta = cosf(theta);
//sintheta = sinf(theta);
sincosf(theta, &sintheta, &costheta);
-
+
//cosphi = cosf(phi);
//sinphi = sinf(phi);
sincosf(phi, &sinphi, &cosphi);
-
+
costwotheta = cosf(2.0f * theta);
sintwotheta = sinf(2.0f * theta);
costwophi = cosf(2.0f * phi);
@@ -1034,26 +1034,26 @@
#endif
if(ATTENUATION){
- if(ATTENUATION_NEW){
- // takes new routines
- // use first order Taylor expansion of displacement for local storage of stresses
- // at this current time step, to fix attenuation in a consistent way
+ if(ATTENUATION_NEW){
+ // takes new routines
+ // use first order Taylor expansion of displacement for local storage of stresses
+ // at this current time step, to fix attenuation in a consistent way
#ifdef USE_TEXTURES
- s_dummyx_loc_att[tx] = s_dummyx_loc[tx] + d_deltat * tex1Dfetch(tex_veloc, iglob);
- s_dummyy_loc_att[tx] = s_dummyy_loc[tx] + d_deltat * tex1Dfetch(tex_veloc, iglob + NGLOB);
- s_dummyz_loc_att[tx] = s_dummyz_loc[tx] + d_deltat * tex1Dfetch(tex_veloc, iglob + 2*NGLOB);
+ s_dummyx_loc_att[tx] = s_dummyx_loc[tx] + d_deltat * tex1Dfetch(tex_veloc, iglob);
+ s_dummyy_loc_att[tx] = s_dummyy_loc[tx] + d_deltat * tex1Dfetch(tex_veloc, iglob + NGLOB);
+ s_dummyz_loc_att[tx] = s_dummyz_loc[tx] + d_deltat * tex1Dfetch(tex_veloc, iglob + 2*NGLOB);
#else
- s_dummyx_loc_att[tx] = s_dummyx_loc[tx] + d_deltat * d_veloc[iglob*3];
- s_dummyy_loc_att[tx] = s_dummyy_loc[tx] + d_deltat * d_veloc[iglob*3 + 1];
- s_dummyz_loc_att[tx] = s_dummyz_loc[tx] + d_deltat * d_veloc[iglob*3 + 2];
+ s_dummyx_loc_att[tx] = s_dummyx_loc[tx] + d_deltat * d_veloc[iglob*3];
+ s_dummyy_loc_att[tx] = s_dummyy_loc[tx] + d_deltat * d_veloc[iglob*3 + 1];
+ s_dummyz_loc_att[tx] = s_dummyz_loc[tx] + d_deltat * d_veloc[iglob*3 + 2];
#endif
- }
- else{
- // takes old routines
- s_dummyx_loc_att[tx] = s_dummyx_loc[tx];
- s_dummyy_loc_att[tx] = s_dummyy_loc[tx];
- s_dummyz_loc_att[tx] = s_dummyz_loc[tx];
- }
+ }
+ else{
+ // takes old routines
+ s_dummyx_loc_att[tx] = s_dummyx_loc[tx];
+ s_dummyy_loc_att[tx] = s_dummyy_loc[tx];
+ s_dummyz_loc_att[tx] = s_dummyz_loc[tx];
+ }
}
}
@@ -1103,21 +1103,21 @@
if( ATTENUATION){
- // temporary variables used for fixing attenuation in a consistent way
+ // temporary variables used for fixing attenuation in a consistent way
- tempx1l_att = 0.f;
- tempx2l_att = 0.f;
- tempx3l_att = 0.f;
+ tempx1l_att = 0.f;
+ tempx2l_att = 0.f;
+ tempx3l_att = 0.f;
- tempy1l_att = 0.f;
- tempy2l_att = 0.f;
- tempy3l_att = 0.f;
+ tempy1l_att = 0.f;
+ tempy2l_att = 0.f;
+ tempy3l_att = 0.f;
- tempz1l_att = 0.f;
- tempz2l_att = 0.f;
- tempz3l_att = 0.f;
+ tempz1l_att = 0.f;
+ tempz2l_att = 0.f;
+ tempz3l_att = 0.f;
- for (l=0;l<NGLLX;l++) {
+ for (l=0;l<NGLLX;l++) {
hp1 = d_hprime_xx[l*NGLLX+I];
offset = K*NGLL2+J*NGLLX+l;
tempx1l_att += s_dummyx_loc_att[offset]*hp1;
@@ -1136,7 +1136,7 @@
tempy3l_att += s_dummyy_loc_att[offset]*hp3;
tempz3l_att += s_dummyz_loc_att[offset]*hp3;
- }
+ }
}
#else
@@ -1195,61 +1195,61 @@
+ s_dummyz_loc[4*NGLL2+J*NGLLX+I]*d_hprime_xx[4*NGLLX+K];
if( ATTENUATION){
- // temporary variables used for fixing attenuation in a consistent way
+ // temporary variables used for fixing attenuation in a consistent way
- tempx1l_att = s_dummyx_loc_att[K*NGLL2+J*NGLLX]*d_hprime_xx[I]
- + s_dummyx_loc_att[K*NGLL2+J*NGLLX+1]*d_hprime_xx[NGLLX+I]
- + s_dummyx_loc_att[K*NGLL2+J*NGLLX+2]*d_hprime_xx[2*NGLLX+I]
- + s_dummyx_loc_att[K*NGLL2+J*NGLLX+3]*d_hprime_xx[3*NGLLX+I]
- + s_dummyx_loc_att[K*NGLL2+J*NGLLX+4]*d_hprime_xx[4*NGLLX+I];
+ tempx1l_att = s_dummyx_loc_att[K*NGLL2+J*NGLLX]*d_hprime_xx[I]
+ + s_dummyx_loc_att[K*NGLL2+J*NGLLX+1]*d_hprime_xx[NGLLX+I]
+ + s_dummyx_loc_att[K*NGLL2+J*NGLLX+2]*d_hprime_xx[2*NGLLX+I]
+ + s_dummyx_loc_att[K*NGLL2+J*NGLLX+3]*d_hprime_xx[3*NGLLX+I]
+ + s_dummyx_loc_att[K*NGLL2+J*NGLLX+4]*d_hprime_xx[4*NGLLX+I];
- tempy1l_att = s_dummyy_loc_att[K*NGLL2+J*NGLLX]*d_hprime_xx[I]
- + s_dummyy_loc_att[K*NGLL2+J*NGLLX+1]*d_hprime_xx[NGLLX+I]
- + s_dummyy_loc_att[K*NGLL2+J*NGLLX+2]*d_hprime_xx[2*NGLLX+I]
- + s_dummyy_loc_att[K*NGLL2+J*NGLLX+3]*d_hprime_xx[3*NGLLX+I]
- + s_dummyy_loc_att[K*NGLL2+J*NGLLX+4]*d_hprime_xx[4*NGLLX+I];
+ tempy1l_att = s_dummyy_loc_att[K*NGLL2+J*NGLLX]*d_hprime_xx[I]
+ + s_dummyy_loc_att[K*NGLL2+J*NGLLX+1]*d_hprime_xx[NGLLX+I]
+ + s_dummyy_loc_att[K*NGLL2+J*NGLLX+2]*d_hprime_xx[2*NGLLX+I]
+ + s_dummyy_loc_att[K*NGLL2+J*NGLLX+3]*d_hprime_xx[3*NGLLX+I]
+ + s_dummyy_loc_att[K*NGLL2+J*NGLLX+4]*d_hprime_xx[4*NGLLX+I];
- tempz1l_att = s_dummyz_loc_att[K*NGLL2+J*NGLLX]*d_hprime_xx[I]
- + s_dummyz_loc_att[K*NGLL2+J*NGLLX+1]*d_hprime_xx[NGLLX+I]
- + s_dummyz_loc_att[K*NGLL2+J*NGLLX+2]*d_hprime_xx[2*NGLLX+I]
- + s_dummyz_loc_att[K*NGLL2+J*NGLLX+3]*d_hprime_xx[3*NGLLX+I]
- + s_dummyz_loc_att[K*NGLL2+J*NGLLX+4]*d_hprime_xx[4*NGLLX+I];
+ tempz1l_att = s_dummyz_loc_att[K*NGLL2+J*NGLLX]*d_hprime_xx[I]
+ + s_dummyz_loc_att[K*NGLL2+J*NGLLX+1]*d_hprime_xx[NGLLX+I]
+ + s_dummyz_loc_att[K*NGLL2+J*NGLLX+2]*d_hprime_xx[2*NGLLX+I]
+ + s_dummyz_loc_att[K*NGLL2+J*NGLLX+3]*d_hprime_xx[3*NGLLX+I]
+ + s_dummyz_loc_att[K*NGLL2+J*NGLLX+4]*d_hprime_xx[4*NGLLX+I];
- tempx2l_att = s_dummyx_loc_att[K*NGLL2+I]*d_hprime_xx[J]
- + s_dummyx_loc_att[K*NGLL2+NGLLX+I]*d_hprime_xx[NGLLX+J]
- + s_dummyx_loc_att[K*NGLL2+2*NGLLX+I]*d_hprime_xx[2*NGLLX+J]
- + s_dummyx_loc_att[K*NGLL2+3*NGLLX+I]*d_hprime_xx[3*NGLLX+J]
- + s_dummyx_loc_att[K*NGLL2+4*NGLLX+I]*d_hprime_xx[4*NGLLX+J];
+ tempx2l_att = s_dummyx_loc_att[K*NGLL2+I]*d_hprime_xx[J]
+ + s_dummyx_loc_att[K*NGLL2+NGLLX+I]*d_hprime_xx[NGLLX+J]
+ + s_dummyx_loc_att[K*NGLL2+2*NGLLX+I]*d_hprime_xx[2*NGLLX+J]
+ + s_dummyx_loc_att[K*NGLL2+3*NGLLX+I]*d_hprime_xx[3*NGLLX+J]
+ + s_dummyx_loc_att[K*NGLL2+4*NGLLX+I]*d_hprime_xx[4*NGLLX+J];
- tempy2l_att = s_dummyy_loc_att[K*NGLL2+I]*d_hprime_xx[J]
- + s_dummyy_loc_att[K*NGLL2+NGLLX+I]*d_hprime_xx[NGLLX+J]
- + s_dummyy_loc_att[K*NGLL2+2*NGLLX+I]*d_hprime_xx[2*NGLLX+J]
- + s_dummyy_loc_att[K*NGLL2+3*NGLLX+I]*d_hprime_xx[3*NGLLX+J]
- + s_dummyy_loc_att[K*NGLL2+4*NGLLX+I]*d_hprime_xx[4*NGLLX+J];
+ tempy2l_att = s_dummyy_loc_att[K*NGLL2+I]*d_hprime_xx[J]
+ + s_dummyy_loc_att[K*NGLL2+NGLLX+I]*d_hprime_xx[NGLLX+J]
+ + s_dummyy_loc_att[K*NGLL2+2*NGLLX+I]*d_hprime_xx[2*NGLLX+J]
+ + s_dummyy_loc_att[K*NGLL2+3*NGLLX+I]*d_hprime_xx[3*NGLLX+J]
+ + s_dummyy_loc_att[K*NGLL2+4*NGLLX+I]*d_hprime_xx[4*NGLLX+J];
- tempz2l_att = s_dummyz_loc_att[K*NGLL2+I]*d_hprime_xx[J]
- + s_dummyz_loc_att[K*NGLL2+NGLLX+I]*d_hprime_xx[NGLLX+J]
- + s_dummyz_loc_att[K*NGLL2+2*NGLLX+I]*d_hprime_xx[2*NGLLX+J]
- + s_dummyz_loc_att[K*NGLL2+3*NGLLX+I]*d_hprime_xx[3*NGLLX+J]
- + s_dummyz_loc_att[K*NGLL2+4*NGLLX+I]*d_hprime_xx[4*NGLLX+J];
+ tempz2l_att = s_dummyz_loc_att[K*NGLL2+I]*d_hprime_xx[J]
+ + s_dummyz_loc_att[K*NGLL2+NGLLX+I]*d_hprime_xx[NGLLX+J]
+ + s_dummyz_loc_att[K*NGLL2+2*NGLLX+I]*d_hprime_xx[2*NGLLX+J]
+ + s_dummyz_loc_att[K*NGLL2+3*NGLLX+I]*d_hprime_xx[3*NGLLX+J]
+ + s_dummyz_loc_att[K*NGLL2+4*NGLLX+I]*d_hprime_xx[4*NGLLX+J];
- tempx3l_att = s_dummyx_loc_att[J*NGLLX+I]*d_hprime_xx[K]
- + s_dummyx_loc_att[NGLL2+J*NGLLX+I]*d_hprime_xx[NGLLX+K]
- + s_dummyx_loc_att[2*NGLL2+J*NGLLX+I]*d_hprime_xx[2*NGLLX+K]
- + s_dummyx_loc_att[3*NGLL2+J*NGLLX+I]*d_hprime_xx[3*NGLLX+K]
- + s_dummyx_loc_att[4*NGLL2+J*NGLLX+I]*d_hprime_xx[4*NGLLX+K];
+ tempx3l_att = s_dummyx_loc_att[J*NGLLX+I]*d_hprime_xx[K]
+ + s_dummyx_loc_att[NGLL2+J*NGLLX+I]*d_hprime_xx[NGLLX+K]
+ + s_dummyx_loc_att[2*NGLL2+J*NGLLX+I]*d_hprime_xx[2*NGLLX+K]
+ + s_dummyx_loc_att[3*NGLL2+J*NGLLX+I]*d_hprime_xx[3*NGLLX+K]
+ + s_dummyx_loc_att[4*NGLL2+J*NGLLX+I]*d_hprime_xx[4*NGLLX+K];
- tempy3l_att = s_dummyy_loc_att[J*NGLLX+I]*d_hprime_xx[K]
- + s_dummyy_loc_att[NGLL2+J*NGLLX+I]*d_hprime_xx[NGLLX+K]
- + s_dummyy_loc_att[2*NGLL2+J*NGLLX+I]*d_hprime_xx[2*NGLLX+K]
- + s_dummyy_loc_att[3*NGLL2+J*NGLLX+I]*d_hprime_xx[3*NGLLX+K]
- + s_dummyy_loc_att[4*NGLL2+J*NGLLX+I]*d_hprime_xx[4*NGLLX+K];
+ tempy3l_att = s_dummyy_loc_att[J*NGLLX+I]*d_hprime_xx[K]
+ + s_dummyy_loc_att[NGLL2+J*NGLLX+I]*d_hprime_xx[NGLLX+K]
+ + s_dummyy_loc_att[2*NGLL2+J*NGLLX+I]*d_hprime_xx[2*NGLLX+K]
+ + s_dummyy_loc_att[3*NGLL2+J*NGLLX+I]*d_hprime_xx[3*NGLLX+K]
+ + s_dummyy_loc_att[4*NGLL2+J*NGLLX+I]*d_hprime_xx[4*NGLLX+K];
- tempz3l_att = s_dummyz_loc_att[J*NGLLX+I]*d_hprime_xx[K]
- + s_dummyz_loc_att[NGLL2+J*NGLLX+I]*d_hprime_xx[NGLLX+K]
- + s_dummyz_loc_att[2*NGLL2+J*NGLLX+I]*d_hprime_xx[2*NGLLX+K]
- + s_dummyz_loc_att[3*NGLL2+J*NGLLX+I]*d_hprime_xx[3*NGLLX+K]
- + s_dummyz_loc_att[4*NGLL2+J*NGLLX+I]*d_hprime_xx[4*NGLLX+K];
+ tempz3l_att = s_dummyz_loc_att[J*NGLLX+I]*d_hprime_xx[K]
+ + s_dummyz_loc_att[NGLL2+J*NGLLX+I]*d_hprime_xx[NGLLX+K]
+ + s_dummyz_loc_att[2*NGLL2+J*NGLLX+I]*d_hprime_xx[2*NGLLX+K]
+ + s_dummyz_loc_att[3*NGLL2+J*NGLLX+I]*d_hprime_xx[3*NGLLX+K]
+ + s_dummyz_loc_att[4*NGLL2+J*NGLLX+I]*d_hprime_xx[4*NGLLX+K];
}
#endif
@@ -1288,60 +1288,60 @@
duzdyl_plus_duydzl = duzdyl + duydzl;
if( ATTENUATION){
- // temporary variables used for fixing attenuation in a consistent way
+ // temporary variables used for fixing attenuation in a consistent way
- duxdxl_att = xixl*tempx1l_att + etaxl*tempx2l_att + gammaxl*tempx3l_att;
- duxdyl_att = xiyl*tempx1l_att + etayl*tempx2l_att + gammayl*tempx3l_att;
- duxdzl_att = xizl*tempx1l_att + etazl*tempx2l_att + gammazl*tempx3l_att;
+ duxdxl_att = xixl*tempx1l_att + etaxl*tempx2l_att + gammaxl*tempx3l_att;
+ duxdyl_att = xiyl*tempx1l_att + etayl*tempx2l_att + gammayl*tempx3l_att;
+ duxdzl_att = xizl*tempx1l_att + etazl*tempx2l_att + gammazl*tempx3l_att;
- duydxl_att = xixl*tempy1l_att + etaxl*tempy2l_att + gammaxl*tempy3l_att;
- duydyl_att = xiyl*tempy1l_att + etayl*tempy2l_att + gammayl*tempy3l_att;
- duydzl_att = xizl*tempy1l_att + etazl*tempy2l_att + gammazl*tempy3l_att;
+ duydxl_att = xixl*tempy1l_att + etaxl*tempy2l_att + gammaxl*tempy3l_att;
+ duydyl_att = xiyl*tempy1l_att + etayl*tempy2l_att + gammayl*tempy3l_att;
+ duydzl_att = xizl*tempy1l_att + etazl*tempy2l_att + gammazl*tempy3l_att;
- duzdxl_att = xixl*tempz1l_att + etaxl*tempz2l_att + gammaxl*tempz3l_att;
- duzdyl_att = xiyl*tempz1l_att + etayl*tempz2l_att + gammayl*tempz3l_att;
- duzdzl_att = xizl*tempz1l_att + etazl*tempz2l_att + gammazl*tempz3l_att;
+ duzdxl_att = xixl*tempz1l_att + etaxl*tempz2l_att + gammaxl*tempz3l_att;
+ duzdyl_att = xiyl*tempz1l_att + etayl*tempz2l_att + gammayl*tempz3l_att;
+ duzdzl_att = xizl*tempz1l_att + etazl*tempz2l_att + gammazl*tempz3l_att;
- // precompute some sums to save CPU time
- duxdyl_plus_duydxl_att = duxdyl_att + duydxl_att;
- duzdxl_plus_duxdzl_att = duzdxl_att + duxdzl_att;
- duzdyl_plus_duydzl_att = duzdyl_att + duydzl_att;
+ // precompute some sums to save CPU time
+ duxdyl_plus_duydxl_att = duxdyl_att + duydxl_att;
+ duzdxl_plus_duxdzl_att = duzdxl_att + duxdzl_att;
+ duzdyl_plus_duydzl_att = duzdyl_att + duydzl_att;
- // computes deviatoric strain attenuation and/or for kernel calculations
- if(COMPUTE_AND_STORE_STRAIN) {
- realw templ = 0.33333333333333333333f * (duxdxl_att + duydyl_att + duzdzl_att); // 1./3. = 0.33333
+ // computes deviatoric strain attenuation and/or for kernel calculations
+ if(COMPUTE_AND_STORE_STRAIN) {
+ realw templ = 0.33333333333333333333f * (duxdxl_att + duydyl_att + duzdzl_att); // 1./3. = 0.33333
- // local storage: stresses at this current time step
- epsilondev_xx_loc = duxdxl_att - templ;
- epsilondev_yy_loc = duydyl_att - templ;
- epsilondev_xy_loc = 0.5f * duxdyl_plus_duydxl_att;
- epsilondev_xz_loc = 0.5f * duzdxl_plus_duxdzl_att;
- epsilondev_yz_loc = 0.5f * duzdyl_plus_duydzl_att;
+ // local storage: stresses at this current time step
+ epsilondev_xx_loc = duxdxl_att - templ;
+ epsilondev_yy_loc = duydyl_att - templ;
+ epsilondev_xy_loc = 0.5f * duxdyl_plus_duydxl_att;
+ epsilondev_xz_loc = 0.5f * duzdxl_plus_duxdzl_att;
+ epsilondev_yz_loc = 0.5f * duzdyl_plus_duydzl_att;
- if(NSPEC_CRUST_MANTLE_STRAIN_ONLY == 1) {
- epsilon_trace_over_3[tx] = templ;
- }else{
- epsilon_trace_over_3[tx + working_element*NGLL3] = templ;
- }
- }
+ if(NSPEC_CRUST_MANTLE_STRAIN_ONLY == 1) {
+ epsilon_trace_over_3[tx] = templ;
+ }else{
+ epsilon_trace_over_3[tx + working_element*NGLL3] = templ;
+ }
+ }
}else{
- // computes deviatoric strain attenuation and/or for kernel calculations
- if(COMPUTE_AND_STORE_STRAIN) {
- realw templ = 0.33333333333333333333f * (duxdxl + duydyl + duzdzl); // 1./3. = 0.33333
+ // computes deviatoric strain attenuation and/or for kernel calculations
+ if(COMPUTE_AND_STORE_STRAIN) {
+ realw templ = 0.33333333333333333333f * (duxdxl + duydyl + duzdzl); // 1./3. = 0.33333
- // local storage: stresses at this current time step
- epsilondev_xx_loc = duxdxl - templ;
- epsilondev_yy_loc = duydyl - templ;
- epsilondev_xy_loc = 0.5f * duxdyl_plus_duydxl;
- epsilondev_xz_loc = 0.5f * duzdxl_plus_duxdzl;
- epsilondev_yz_loc = 0.5f * duzdyl_plus_duydzl;
+ // local storage: stresses at this current time step
+ epsilondev_xx_loc = duxdxl - templ;
+ epsilondev_yy_loc = duydyl - templ;
+ epsilondev_xy_loc = 0.5f * duxdyl_plus_duydxl;
+ epsilondev_xz_loc = 0.5f * duzdxl_plus_duxdzl;
+ epsilondev_yz_loc = 0.5f * duzdyl_plus_duydzl;
- if(NSPEC_CRUST_MANTLE_STRAIN_ONLY == 1) {
- epsilon_trace_over_3[tx] = templ;
- }else{
- epsilon_trace_over_3[tx + working_element*NGLL3] = templ;
- }
- }
+ if(NSPEC_CRUST_MANTLE_STRAIN_ONLY == 1) {
+ epsilon_trace_over_3[tx] = templ;
+ }else{
+ epsilon_trace_over_3[tx + working_element*NGLL3] = templ;
+ }
+ }
}
// attenuation
Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/cuda/compute_forces_inner_core_cuda.cu
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/cuda/compute_forces_inner_core_cuda.cu 2012-07-19 21:55:39 UTC (rev 20529)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/cuda/compute_forces_inner_core_cuda.cu 2012-07-19 22:05:23 UTC (rev 20530)
@@ -434,28 +434,28 @@
s_dummyz_loc[tx] = d_displ[iglob*3 + 2];
#endif
- if(ATTENUATION){
- if(ATTENUATION_NEW){
- // takes new routines
- // use first order Taylor expansion of displacement for local storage of stresses
- // at this current time step, to fix attenuation in a consistent way
+ if(ATTENUATION){
+ if(ATTENUATION_NEW){
+ // takes new routines
+ // use first order Taylor expansion of displacement for local storage of stresses
+ // at this current time step, to fix attenuation in a consistent way
#ifdef USE_TEXTURES
- s_dummyx_loc_att[tx] = s_dummyx_loc[tx] + d_deltat * tex1Dfetch(tex_veloc, iglob);
- s_dummyy_loc_att[tx] = s_dummyy_loc[tx] + d_deltat * tex1Dfetch(tex_veloc, iglob + NGLOB);
- s_dummyz_loc_att[tx] = s_dummyz_loc[tx] + d_deltat * tex1Dfetch(tex_veloc, iglob + 2*NGLOB);
+ s_dummyx_loc_att[tx] = s_dummyx_loc[tx] + d_deltat * tex1Dfetch(tex_veloc, iglob);
+ s_dummyy_loc_att[tx] = s_dummyy_loc[tx] + d_deltat * tex1Dfetch(tex_veloc, iglob + NGLOB);
+ s_dummyz_loc_att[tx] = s_dummyz_loc[tx] + d_deltat * tex1Dfetch(tex_veloc, iglob + 2*NGLOB);
#else
- s_dummyx_loc_att[tx] = s_dummyx_loc[tx] + d_deltat * d_veloc[iglob*3];
- s_dummyy_loc_att[tx] = s_dummyy_loc[tx] + d_deltat * d_veloc[iglob*3 + 1];
- s_dummyz_loc_att[tx] = s_dummyz_loc[tx] + d_deltat * d_veloc[iglob*3 + 2];
+ s_dummyx_loc_att[tx] = s_dummyx_loc[tx] + d_deltat * d_veloc[iglob*3];
+ s_dummyy_loc_att[tx] = s_dummyy_loc[tx] + d_deltat * d_veloc[iglob*3 + 1];
+ s_dummyz_loc_att[tx] = s_dummyz_loc[tx] + d_deltat * d_veloc[iglob*3 + 2];
#endif
- }
- else{
- // takes old routines
- s_dummyx_loc_att[tx] = s_dummyx_loc[tx];
- s_dummyy_loc_att[tx] = s_dummyy_loc[tx];
- s_dummyz_loc_att[tx] = s_dummyz_loc[tx];
- }
- }
+ }
+ else{
+ // takes old routines
+ s_dummyx_loc_att[tx] = s_dummyx_loc[tx];
+ s_dummyy_loc_att[tx] = s_dummyy_loc[tx];
+ s_dummyz_loc_att[tx] = s_dummyz_loc[tx];
+ }
+ }
}
}
@@ -689,40 +689,40 @@
duzdyl_plus_duydzl = duzdyl + duydzl;
if(ATTENUATION){
- // temporary variables used for fixing attenuation in a consistent way
+ // temporary variables used for fixing attenuation in a consistent way
- duxdxl_att = xixl*tempx1l_att + etaxl*tempx2l_att + gammaxl*tempx3l_att;
- duxdyl_att = xiyl*tempx1l_att + etayl*tempx2l_att + gammayl*tempx3l_att;
- duxdzl_att = xizl*tempx1l_att + etazl*tempx2l_att + gammazl*tempx3l_att;
+ duxdxl_att = xixl*tempx1l_att + etaxl*tempx2l_att + gammaxl*tempx3l_att;
+ duxdyl_att = xiyl*tempx1l_att + etayl*tempx2l_att + gammayl*tempx3l_att;
+ duxdzl_att = xizl*tempx1l_att + etazl*tempx2l_att + gammazl*tempx3l_att;
- duydxl_att = xixl*tempy1l_att + etaxl*tempy2l_att + gammaxl*tempy3l_att;
- duydyl_att = xiyl*tempy1l_att + etayl*tempy2l_att + gammayl*tempy3l_att;
- duydzl_att = xizl*tempy1l_att + etazl*tempy2l_att + gammazl*tempy3l_att;
+ duydxl_att = xixl*tempy1l_att + etaxl*tempy2l_att + gammaxl*tempy3l_att;
+ duydyl_att = xiyl*tempy1l_att + etayl*tempy2l_att + gammayl*tempy3l_att;
+ duydzl_att = xizl*tempy1l_att + etazl*tempy2l_att + gammazl*tempy3l_att;
- duzdxl_att = xixl*tempz1l_att + etaxl*tempz2l_att + gammaxl*tempz3l_att;
- duzdyl_att = xiyl*tempz1l_att + etayl*tempz2l_att + gammayl*tempz3l_att;
- duzdzl_att = xizl*tempz1l_att + etazl*tempz2l_att + gammazl*tempz3l_att;
+ duzdxl_att = xixl*tempz1l_att + etaxl*tempz2l_att + gammaxl*tempz3l_att;
+ duzdyl_att = xiyl*tempz1l_att + etayl*tempz2l_att + gammayl*tempz3l_att;
+ duzdzl_att = xizl*tempz1l_att + etazl*tempz2l_att + gammazl*tempz3l_att;
- // precompute some sums to save CPU time
- duxdyl_plus_duydxl_att = duxdyl_att + duydxl_att;
- duzdxl_plus_duxdzl_att = duzdxl_att + duxdzl_att;
- duzdyl_plus_duydzl_att = duzdyl_att + duydzl_att;
+ // precompute some sums to save CPU time
+ duxdyl_plus_duydxl_att = duxdyl_att + duydxl_att;
+ duzdxl_plus_duxdzl_att = duzdxl_att + duxdzl_att;
+ duzdyl_plus_duydzl_att = duzdyl_att + duydzl_att;
- // computes deviatoric strain attenuation and/or for kernel calculations
- if(COMPUTE_AND_STORE_STRAIN) {
- realw templ = 0.33333333333333333333f * (duxdxl_att + duydyl_att + duzdzl_att); // 1./3. = 0.33333
+ // computes deviatoric strain attenuation and/or for kernel calculations
+ if(COMPUTE_AND_STORE_STRAIN) {
+ realw templ = 0.33333333333333333333f * (duxdxl_att + duydyl_att + duzdzl_att); // 1./3. = 0.33333
- // local storage: stresses at this current time step
- epsilondev_xx_loc = duxdxl_att - templ;
- epsilondev_yy_loc = duydyl_att - templ;
- epsilondev_xy_loc = 0.5f * duxdyl_plus_duydxl_att;
- epsilondev_xz_loc = 0.5f * duzdxl_plus_duxdzl_att;
- epsilondev_yz_loc = 0.5f * duzdyl_plus_duydzl_att;
+ // local storage: stresses at this current time step
+ epsilondev_xx_loc = duxdxl_att - templ;
+ epsilondev_yy_loc = duydyl_att - templ;
+ epsilondev_xy_loc = 0.5f * duxdyl_plus_duydxl_att;
+ epsilondev_xz_loc = 0.5f * duzdxl_plus_duxdzl_att;
+ epsilondev_yz_loc = 0.5f * duzdyl_plus_duydzl_att;
- if(SIMULATION_TYPE == 3) {
- epsilon_trace_over_3[tx + working_element*NGLL3] = templ;
- }
- }
+ if(SIMULATION_TYPE == 3) {
+ epsilon_trace_over_3[tx + working_element*NGLL3] = templ;
+ }
+ }
}else{
// computes deviatoric strain attenuation and/or for kernel calculations
if(COMPUTE_AND_STORE_STRAIN) {
Added: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/cuda/initialize_cuda.cu
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/cuda/initialize_cuda.cu (rev 0)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/cuda/initialize_cuda.cu 2012-07-19 22:05:23 UTC (rev 20530)
@@ -0,0 +1,181 @@
+/*
+ !=====================================================================
+ !
+ ! S p e c f e m 3 D V e r s i o n 2 . 1
+ ! ---------------------------------------
+ !
+ ! Main authors: Dimitri Komatitsch and Jeroen Tromp
+ ! Princeton University, USA and CNRS / INRIA / University of Pau
+ ! (c) Princeton University / California Institute of Technology and CNRS / INRIA / University of Pau
+ ! July 2012
+ !
+ ! This program is free software; you can redistribute it and/or modify
+ ! it under the terms of the GNU General Public License as published by
+ ! the Free Software Foundation; either version 2 of the License, or
+ ! (at your option) any later version.
+ !
+ ! This program is distributed in the hope that it will be useful,
+ ! but WITHOUT ANY WARRANTY; without even the implied warranty of
+ ! MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ ! GNU General Public License for more details.
+ !
+ ! You should have received a copy of the GNU General Public License along
+ ! with this program; if not, write to the Free Software Foundation, Inc.,
+ ! 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ !
+ !=====================================================================
+ */
+
+#include <stdio.h>
+#include <cuda.h>
+#include <cublas.h>
+
+#ifdef WITH_MPI
+#include <mpi.h>
+#endif
+
+#include <sys/time.h>
+#include <sys/resource.h>
+
+#include "config.h"
+#include "mesh_constants_cuda.h"
+#include "prepare_constants_cuda.h"
+
+/* ----------------------------------------------------------------------------------------------- */
+
+// GPU initialization
+
+/* ----------------------------------------------------------------------------------------------- */
+
+extern "C"
+void FC_FUNC_(initialize_cuda_device,
+ INITIALIZE_CUDA_DEVICE)(int* myrank_f,int* ncuda_devices) {
+ TRACE("initialize_cuda_device");
+
+ // Gets rank number of MPI process
+ int myrank = *myrank_f;
+
+ /*
+ // cuda initialization (needs -lcuda library)
+ // note: cuInit initializes the driver API.
+ // it is needed for any following CUDA driver API function call (format cuFUNCTION(..) )
+ // however, for the CUDA runtime API functions (format cudaFUNCTION(..) )
+ // the initialization is implicit, thus cuInit() here would not be needed...
+ CUresult status = cuInit(0);
+ if ( CUDA_SUCCESS != status ) exit_on_error("CUDA driver API device initialization failed\n");
+
+ // returns a handle to the first cuda compute device
+ CUdevice dev;
+ status = cuDeviceGet(&dev, 0);
+ if ( CUDA_SUCCESS != status ) exit_on_error("CUDA device not found\n");
+
+ // gets device properties
+ int major,minor;
+ status = cuDeviceComputeCapability(&major,&minor,dev);
+ if ( CUDA_SUCCESS != status ) exit_on_error("CUDA device information not found\n");
+
+ // make sure that the device has compute capability >= 1.3
+ if (major < 1){
+ fprintf(stderr,"Compute capability major number should be at least 1, got: %d \nexiting...\n",major);
+ exit_on_error("CUDA Compute capability major number should be at least 1\n");
+ }
+ if (major == 1 && minor < 3){
+ fprintf(stderr,"Compute capability should be at least 1.3, got: %d.%d \nexiting...\n",major,minor);
+ exit_on_error("CUDA Compute capability major number should be at least 1.3\n");
+ }
+ */
+
+ // note: from here on we use the runtime API ...
+
+ // Gets number of GPU devices
+ int device_count = 0;
+ cudaGetDeviceCount(&device_count);
+ exit_on_cuda_error("CUDA runtime error: cudaGetDeviceCount failed\ncheck if driver and runtime libraries work together\nexiting...\n");
+
+ // returns device count to fortran
+ if (device_count == 0) exit_on_error("CUDA runtime error: there is no device supporting CUDA\n");
+ *ncuda_devices = device_count;
+
+ // Sets the active device
+ if(device_count >= 1) {
+ // generalized for more GPUs per node
+ // note: without previous context release, cudaSetDevice will complain with the cuda error
+ // "setting the device when a process is active is not allowed"
+ // releases previous contexts
+ cudaThreadExit();
+
+ //printf("rank %d: cuda device count = %d sets device = %d \n",myrank,device_count,myrank % device_count);
+ //MPI_Barrier(MPI_COMM_WORLD);
+
+ // sets active device
+ cudaSetDevice( myrank % device_count );
+ exit_on_cuda_error("cudaSetDevice");
+ }
+
+ // returns a handle to the active device
+ int device;
+ cudaGetDevice(&device);
+
+ // get device properties
+ struct cudaDeviceProp deviceProp;
+ cudaGetDeviceProperties(&deviceProp,device);
+
+ // exit if the machine has no CUDA-enabled device
+ if (deviceProp.major == 9999 && deviceProp.minor == 9999){
+ fprintf(stderr,"No CUDA-enabled device found, exiting...\n\n");
+ exit_on_error("CUDA runtime error: there is no CUDA-enabled device found\n");
+ }
+
+ // outputs device infos to file
+ char filename[BUFSIZ];
+ FILE* fp;
+ sprintf(filename,"OUTPUT_FILES/gpu_device_info_proc_%06d.txt",myrank);
+ fp = fopen(filename,"a+");
+ if (fp != NULL){
+ // display device properties
+ fprintf(fp,"Device Name = %s\n",deviceProp.name);
+ fprintf(fp,"multiProcessorCount: %d\n",deviceProp.multiProcessorCount);
+ fprintf(fp,"totalGlobalMem (in MB): %f\n",(unsigned long) deviceProp.totalGlobalMem / (1024.f * 1024.f));
+ fprintf(fp,"totalGlobalMem (in GB): %f\n",(unsigned long) deviceProp.totalGlobalMem / (1024.f * 1024.f * 1024.f));
+ fprintf(fp,"sharedMemPerBlock (in bytes): %lu\n",(unsigned long) deviceProp.sharedMemPerBlock);
+ fprintf(fp,"Maximum number of threads per block: %d\n",deviceProp.maxThreadsPerBlock);
+ fprintf(fp,"Maximum size of each dimension of a block: %d x %d x %d\n",
+ deviceProp.maxThreadsDim[0],deviceProp.maxThreadsDim[1],deviceProp.maxThreadsDim[2]);
+ fprintf(fp,"Maximum sizes of each dimension of a grid: %d x %d x %d\n",
+ deviceProp.maxGridSize[0],deviceProp.maxGridSize[1],deviceProp.maxGridSize[2]);
+ fprintf(fp,"Compute capability of the device = %d.%d\n", deviceProp.major, deviceProp.minor);
+ if(deviceProp.canMapHostMemory){
+ fprintf(fp,"canMapHostMemory: TRUE\n");
+ }else{
+ fprintf(fp,"canMapHostMemory: FALSE\n");
+ }
+ if(deviceProp.deviceOverlap){
+ fprintf(fp,"deviceOverlap: TRUE\n");
+ }else{
+ fprintf(fp,"deviceOverlap: FALSE\n");
+ }
+
+ // outputs initial memory infos via cudaMemGetInfo()
+ double free_db,used_db,total_db;
+ get_free_memory(&free_db,&used_db,&total_db);
+ fprintf(fp,"%d: GPU memory usage: used = %f MB, free = %f MB, total = %f MB\n",myrank,
+ used_db/1024.0/1024.0, free_db/1024.0/1024.0, total_db/1024.0/1024.0);
+
+ fclose(fp);
+ }
+
+ // make sure that the device has compute capability >= 1.3
+ if (deviceProp.major < 1){
+ fprintf(stderr,"Compute capability major number should be at least 1, exiting...\n\n");
+ exit_on_error("CUDA Compute capability major number should be at least 1\n");
+ }
+ if (deviceProp.major == 1 && deviceProp.minor < 3){
+ fprintf(stderr,"Compute capability should be at least 1.3, exiting...\n");
+ exit_on_error("CUDA Compute capability major number should be at least 1.3\n");
+ }
+ // we use pinned memory for asynchronous copy
+ if( ! deviceProp.canMapHostMemory){
+ fprintf(stderr,"Device capability should allow to map host memory, exiting...\n");
+ exit_on_error("CUDA Device capability canMapHostMemory should be TRUE\n");
+ }
+}
Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/cuda/it_update_displacement_cuda.cu
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/cuda/it_update_displacement_cuda.cu 2012-07-19 21:55:39 UTC (rev 20529)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/cuda/it_update_displacement_cuda.cu 2012-07-19 22:05:23 UTC (rev 20530)
@@ -298,8 +298,8 @@
realw* accel, int size,
realw deltatover2,
realw* rmassx,
- realw* rmassy,
- realw* rmassz) {
+ realw* rmassy,
+ realw* rmassz) {
int id = threadIdx.x + blockIdx.x*blockDim.x + blockIdx.y*gridDim.x*blockDim.x;
/* because of block and grid sizing problems, there is a small */
@@ -320,8 +320,8 @@
__global__ void kernel_3_accel_cuda_device(realw* accel,
int size,
realw* rmassx,
- realw* rmassy,
- realw* rmassz) {
+ realw* rmassy,
+ realw* rmassz) {
int id = threadIdx.x + blockIdx.x*blockDim.x + blockIdx.y*gridDim.x*blockDim.x;
/* because of block and grid sizing problems, there is a small */
@@ -359,7 +359,7 @@
int* SIMULATION_TYPE_f,
realw* b_deltatover2_F,
int* OCEANS,
- int* NCHUNKS_VAL) {
+ int* NCHUNKS_VAL) {
TRACE("kernel_3_a_cuda");
Mesh* mp = (Mesh*)(*Mesh_pointer); // get Mesh from fortran integer wrapper
@@ -388,39 +388,39 @@
if( *NCHUNKS_VAL != 6 && mp->absorbing_conditions){
kernel_3_cuda_device<<< grid, threads>>>(mp->d_veloc_crust_mantle,
- mp->d_accel_crust_mantle,
- mp->NGLOB_CRUST_MANTLE,
- deltatover2,
- mp->d_rmassx_crust_mantle,
- mp->d_rmassy_crust_mantle,
- mp->d_rmassz_crust_mantle);
+ mp->d_accel_crust_mantle,
+ mp->NGLOB_CRUST_MANTLE,
+ deltatover2,
+ mp->d_rmassx_crust_mantle,
+ mp->d_rmassy_crust_mantle,
+ mp->d_rmassz_crust_mantle);
if(SIMULATION_TYPE == 3){
- kernel_3_cuda_device<<< grid, threads>>>(mp->d_b_veloc_crust_mantle,
- mp->d_b_accel_crust_mantle,
- mp->NGLOB_CRUST_MANTLE,
- b_deltatover2,
- mp->d_rmassx_crust_mantle,
- mp->d_rmassy_crust_mantle,
- mp->d_rmassz_crust_mantle);
+ kernel_3_cuda_device<<< grid, threads>>>(mp->d_b_veloc_crust_mantle,
+ mp->d_b_accel_crust_mantle,
+ mp->NGLOB_CRUST_MANTLE,
+ b_deltatover2,
+ mp->d_rmassx_crust_mantle,
+ mp->d_rmassy_crust_mantle,
+ mp->d_rmassz_crust_mantle);
}
}else{
kernel_3_cuda_device<<< grid, threads>>>(mp->d_veloc_crust_mantle,
- mp->d_accel_crust_mantle,
- mp->NGLOB_CRUST_MANTLE,
- deltatover2,
- mp->d_rmassz_crust_mantle,
- mp->d_rmassz_crust_mantle,
- mp->d_rmassz_crust_mantle);
+ mp->d_accel_crust_mantle,
+ mp->NGLOB_CRUST_MANTLE,
+ deltatover2,
+ mp->d_rmassz_crust_mantle,
+ mp->d_rmassz_crust_mantle,
+ mp->d_rmassz_crust_mantle);
if(SIMULATION_TYPE == 3){
- kernel_3_cuda_device<<< grid, threads>>>(mp->d_b_veloc_crust_mantle,
- mp->d_b_accel_crust_mantle,
- mp->NGLOB_CRUST_MANTLE,
- b_deltatover2,
- mp->d_rmassz_crust_mantle,
- mp->d_rmassz_crust_mantle,
- mp->d_rmassz_crust_mantle);
+ kernel_3_cuda_device<<< grid, threads>>>(mp->d_b_veloc_crust_mantle,
+ mp->d_b_accel_crust_mantle,
+ mp->NGLOB_CRUST_MANTLE,
+ b_deltatover2,
+ mp->d_rmassz_crust_mantle,
+ mp->d_rmassz_crust_mantle,
+ mp->d_rmassz_crust_mantle);
}
}
@@ -429,31 +429,31 @@
if( *NCHUNKS_VAL != 6 && mp->absorbing_conditions){
kernel_3_accel_cuda_device<<< grid, threads>>>(mp->d_accel_crust_mantle,
- mp->NGLOB_CRUST_MANTLE,
- mp->d_rmassx_crust_mantle,
- mp->d_rmassy_crust_mantle,
- mp->d_rmassz_crust_mantle);
+ mp->NGLOB_CRUST_MANTLE,
+ mp->d_rmassx_crust_mantle,
+ mp->d_rmassy_crust_mantle,
+ mp->d_rmassz_crust_mantle);
if(SIMULATION_TYPE == 3) {
- kernel_3_accel_cuda_device<<< grid, threads>>>(mp->d_b_accel_crust_mantle,
- mp->NGLOB_CRUST_MANTLE,
- mp->d_rmassx_crust_mantle,
- mp->d_rmassy_crust_mantle,
- mp->d_rmassz_crust_mantle);
+ kernel_3_accel_cuda_device<<< grid, threads>>>(mp->d_b_accel_crust_mantle,
+ mp->NGLOB_CRUST_MANTLE,
+ mp->d_rmassx_crust_mantle,
+ mp->d_rmassy_crust_mantle,
+ mp->d_rmassz_crust_mantle);
}
}else{
kernel_3_accel_cuda_device<<< grid, threads>>>(mp->d_accel_crust_mantle,
- mp->NGLOB_CRUST_MANTLE,
- mp->d_rmassz_crust_mantle,
- mp->d_rmassz_crust_mantle,
- mp->d_rmassz_crust_mantle);
+ mp->NGLOB_CRUST_MANTLE,
+ mp->d_rmassz_crust_mantle,
+ mp->d_rmassz_crust_mantle,
+ mp->d_rmassz_crust_mantle);
if(SIMULATION_TYPE == 3) {
- kernel_3_accel_cuda_device<<< grid, threads>>>(mp->d_b_accel_crust_mantle,
- mp->NGLOB_CRUST_MANTLE,
- mp->d_rmassz_crust_mantle,
- mp->d_rmassz_crust_mantle,
- mp->d_rmassz_crust_mantle);
+ kernel_3_accel_cuda_device<<< grid, threads>>>(mp->d_b_accel_crust_mantle,
+ mp->NGLOB_CRUST_MANTLE,
+ mp->d_rmassz_crust_mantle,
+ mp->d_rmassz_crust_mantle,
+ mp->d_rmassz_crust_mantle);
}
}
}
@@ -526,19 +526,19 @@
kernel_3_cuda_device<<< grid, threads>>>(mp->d_veloc_inner_core,
mp->d_accel_inner_core,
mp->NGLOB_INNER_CORE,
- deltatover2,
- mp->d_rmass_inner_core,
- mp->d_rmass_inner_core,
- mp->d_rmass_inner_core);
+ deltatover2,
+ mp->d_rmass_inner_core,
+ mp->d_rmass_inner_core,
+ mp->d_rmass_inner_core);
if(SIMULATION_TYPE == 3) {
kernel_3_cuda_device<<< grid, threads>>>(mp->d_b_veloc_inner_core,
mp->d_b_accel_inner_core,
mp->NGLOB_INNER_CORE,
b_deltatover2,
- mp->d_rmass_inner_core,
- mp->d_rmass_inner_core,
- mp->d_rmass_inner_core);
+ mp->d_rmass_inner_core,
+ mp->d_rmass_inner_core,
+ mp->d_rmass_inner_core);
}
Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/cuda/prepare_mesh_constants_cuda.cu
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/cuda/prepare_mesh_constants_cuda.cu 2012-07-19 21:55:39 UTC (rev 20529)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/cuda/prepare_mesh_constants_cuda.cu 2012-07-19 22:05:23 UTC (rev 20530)
@@ -50,138 +50,6 @@
/* ----------------------------------------------------------------------------------------------- */
extern "C"
-void FC_FUNC_(prepare_cuda_device,
- PREPARE_CUDA_DEVICE)(int* myrank_f,int* ncuda_devices) {
- TRACE("prepare_cuda_device");
-
- // Gets rank number of MPI process
- int myrank = *myrank_f;
-
- // cuda initialization (needs -lcuda library)
- // note: cuInit initializes the driver API.
- // it is needed for any following CUDA driver API function call (format cuFUNCTION(..) )
- // however, for the CUDA runtime API functions (format cudaFUNCTION(..) )
- // the initialization is implicit, thus cuInit() here would not be needed...
- CUresult status = cuInit(0);
- if ( CUDA_SUCCESS != status ) exit_on_error("CUDA driver API device initialization failed\n");
-
- // returns a handle to the first cuda compute device
- CUdevice dev;
- status = cuDeviceGet(&dev, 0);
- if ( CUDA_SUCCESS != status ) exit_on_error("CUDA device not found\n");
-
- // gets device properties
- int major,minor;
- status = cuDeviceComputeCapability(&major,&minor,dev);
- if ( CUDA_SUCCESS != status ) exit_on_error("CUDA device information not found\n");
-
- // make sure that the device has compute capability >= 1.3
- if (major < 1){
- fprintf(stderr,"Compute capability major number should be at least 1, got: %d \nexiting...\n",major);
- exit_on_error("CUDA Compute capability major number should be at least 1\n");
- }
- if (major == 1 && minor < 3){
- fprintf(stderr,"Compute capability should be at least 1.3, got: %d.%d \nexiting...\n",major,minor);
- exit_on_error("CUDA Compute capability major number should be at least 1.3\n");
- }
-
- // note: from here on we use the runtime API ...
- // Gets number of GPU devices
- int device_count = 0;
- cudaGetDeviceCount(&device_count);
- exit_on_cuda_error("CUDA runtime cudaGetDeviceCount: check if driver and runtime libraries work together\nexiting...\n");
-
- // returns device count to fortran
- if (device_count == 0) exit_on_error("CUDA runtime error: there is no device supporting CUDA\n");
- *ncuda_devices = device_count;
-
-
- // Sets the active device
- if(device_count > 1) {
- // generalized for more GPUs per node
- // note: without previous context release, cudaSetDevice will complain with the cuda error
- // "setting the device when a process is active is not allowed"
- // releases previous contexts
- cudaThreadExit();
-
- //printf("rank %d: cuda device count = %d sets device = %d \n",myrank,device_count,myrank % device_count);
- //MPI_Barrier(MPI_COMM_WORLD);
-
- // sets active device
- cudaSetDevice( myrank % device_count );
- exit_on_cuda_error("cudaSetDevice");
- }
-
- // returns a handle to the active device
- int device;
- cudaGetDevice(&device);
-
- // get device properties
- struct cudaDeviceProp deviceProp;
- cudaGetDeviceProperties(&deviceProp,device);
-
- // exit if the machine has no CUDA-enabled device
- if (deviceProp.major == 9999 && deviceProp.minor == 9999){
- fprintf(stderr,"No CUDA-enabled device found, exiting...\n\n");
- exit_on_error("CUDA runtime error: there is no CUDA-enabled device found\n");
- }
-
- // outputs device infos to file
- char filename[BUFSIZ];
- FILE* fp;
- sprintf(filename,"OUTPUT_FILES/gpu_device_info_proc_%06d.txt",myrank);
- fp = fopen(filename,"a+");
- if (fp != NULL){
- // display device properties
- fprintf(fp,"Device Name = %s\n",deviceProp.name);
- fprintf(fp,"multiProcessorCount: %d\n",deviceProp.multiProcessorCount);
- fprintf(fp,"totalGlobalMem (in MB): %f\n",(unsigned long) deviceProp.totalGlobalMem / (1024.f * 1024.f));
- fprintf(fp,"totalGlobalMem (in GB): %f\n",(unsigned long) deviceProp.totalGlobalMem / (1024.f * 1024.f * 1024.f));
- fprintf(fp,"sharedMemPerBlock (in bytes): %lu\n",(unsigned long) deviceProp.sharedMemPerBlock);
- fprintf(fp,"Maximum number of threads per block: %d\n",deviceProp.maxThreadsPerBlock);
- fprintf(fp,"Maximum size of each dimension of a block: %d x %d x %d\n",
- deviceProp.maxThreadsDim[0],deviceProp.maxThreadsDim[1],deviceProp.maxThreadsDim[2]);
- fprintf(fp,"Maximum sizes of each dimension of a grid: %d x %d x %d\n",
- deviceProp.maxGridSize[0],deviceProp.maxGridSize[1],deviceProp.maxGridSize[2]);
- fprintf(fp,"Compute capability of the device = %d.%d\n", deviceProp.major, deviceProp.minor);
- if(deviceProp.canMapHostMemory){
- fprintf(fp,"canMapHostMemory: TRUE\n");
- }else{
- fprintf(fp,"canMapHostMemory: FALSE\n");
- }
- if(deviceProp.deviceOverlap){
- fprintf(fp,"deviceOverlap: TRUE\n");
- }else{
- fprintf(fp,"deviceOverlap: FALSE\n");
- }
-
- // make sure that the device has compute capability >= 1.3
- //if (deviceProp.major < 1){
- // fprintf(stderr,"Compute capability major number should be at least 1, exiting...\n\n");
- // exit_on_error("CUDA Compute capability major number should be at least 1");
- //}
- //if (deviceProp.major == 1 && deviceProp.minor < 3){
- // fprintf(stderr,"Compute capability should be at least 1.3, exiting...\n");
- // exit_on_error("CUDA Compute capability major number should be at least 1.3");
- //}
-
- // outputs initial memory infos via cudaMemGetInfo()
- double free_db,used_db,total_db;
- get_free_memory(&free_db,&used_db,&total_db);
- fprintf(fp,"%d: GPU memory usage: used = %f MB, free = %f MB, total = %f MB\n",myrank,
- used_db/1024.0/1024.0, free_db/1024.0/1024.0, total_db/1024.0/1024.0);
-
- fclose(fp);
- }
-}
-
-/* ----------------------------------------------------------------------------------------------- */
-
-// SIMULATION constants
-
-/* ----------------------------------------------------------------------------------------------- */
-
-extern "C"
void FC_FUNC_(prepare_constants_device,
PREPARE_CONSTANTS_DEVICE)(long* Mesh_pointer,
int* myrank_f,
@@ -1374,8 +1242,8 @@
int* nspec_outer,
int* nspec_inner,
int* NSPEC2D_TOP_CM,
- int* NSPEC2D_BOTTOM_CM,
- int* NCHUNKS_VAL
+ int* NSPEC2D_BOTTOM_CM,
+ int* NCHUNKS_VAL
) {
TRACE("prepare_crust_mantle_device");
@@ -1617,10 +1485,10 @@
if( *NCHUNKS_VAL != 6 && mp->absorbing_conditions){
print_CUDA_error_if_any(cudaMalloc((void**)&(mp->d_rmassx_crust_mantle),sizeof(realw)*size_glob),2005);
print_CUDA_error_if_any(cudaMemcpy(mp->d_rmassx_crust_mantle,h_rmassx,
- sizeof(realw)*size_glob,cudaMemcpyHostToDevice),2100);
+ sizeof(realw)*size_glob,cudaMemcpyHostToDevice),2100);
print_CUDA_error_if_any(cudaMalloc((void**)&(mp->d_rmassy_crust_mantle),sizeof(realw)*size_glob),2005);
print_CUDA_error_if_any(cudaMemcpy(mp->d_rmassy_crust_mantle,h_rmassy,
- sizeof(realw)*size_glob,cudaMemcpyHostToDevice),2100);
+ sizeof(realw)*size_glob,cudaMemcpyHostToDevice),2100);
}
print_CUDA_error_if_any(cudaMalloc((void**)&(mp->d_rmassz_crust_mantle),sizeof(realw)*size_glob),2005);
@@ -1681,20 +1549,20 @@
realw* h_gammax, realw* h_gammay, realw* h_gammaz,
realw* h_rho, realw* h_kappav,
realw* h_rmass,
- realw* h_normal_top_outer_core,
- realw* h_normal_bottom_outer_core,
- realw* h_jacobian2D_top_outer_core,
- realw* h_jacobian2D_bottom_outer_core,
- int* h_ibelm_top_outer_core,
- int* h_ibelm_bottom_outer_core,
+ realw* h_normal_top_outer_core,
+ realw* h_normal_bottom_outer_core,
+ realw* h_jacobian2D_top_outer_core,
+ realw* h_jacobian2D_bottom_outer_core,
+ int* h_ibelm_top_outer_core,
+ int* h_ibelm_bottom_outer_core,
int* h_ibool,
realw* h_xstore, realw* h_ystore, realw* h_zstore,
int* num_phase_ispec,
int* phase_ispec_inner,
int* nspec_outer,
int* nspec_inner,
- int* NSPEC2D_TOP_OC,
- int* NSPEC2D_BOTTOM_OC
+ int* NSPEC2D_TOP_OC,
+ int* NSPEC2D_BOTTOM_OC
) {
TRACE("prepare_outer_core_device");
@@ -1848,23 +1716,23 @@
extern "C"
void FC_FUNC_(prepare_inner_core_device,
PREPARE_INNER_CORE_DEVICE)(long* Mesh_pointer_f,
- realw* h_xix, realw* h_xiy, realw* h_xiz,
- realw* h_etax, realw* h_etay, realw* h_etaz,
- realw* h_gammax, realw* h_gammay, realw* h_gammaz,
- realw* h_rho, realw* h_kappav, realw* h_muv,
- realw* h_rmass,
- int* h_ibelm_top_inner_core,
- int* h_ibool,
- realw* h_xstore, realw* h_ystore, realw* h_zstore,
- realw *c11store,realw *c12store,realw *c13store,
- realw *c33store,realw *c44store,
- int* h_idoubling_inner_core,
- int* num_phase_ispec,
- int* phase_ispec_inner,
- int* nspec_outer,
- int* nspec_inner,
- int* NSPEC2D_TOP_IC) {
-
+ realw* h_xix, realw* h_xiy, realw* h_xiz,
+ realw* h_etax, realw* h_etay, realw* h_etaz,
+ realw* h_gammax, realw* h_gammay, realw* h_gammaz,
+ realw* h_rho, realw* h_kappav, realw* h_muv,
+ realw* h_rmass,
+ int* h_ibelm_top_inner_core,
+ int* h_ibool,
+ realw* h_xstore, realw* h_ystore, realw* h_zstore,
+ realw *c11store,realw *c12store,realw *c13store,
+ realw *c33store,realw *c44store,
+ int* h_idoubling_inner_core,
+ int* num_phase_ispec,
+ int* phase_ispec_inner,
+ int* nspec_outer,
+ int* nspec_inner,
+ int* NSPEC2D_TOP_IC) {
+
TRACE("prepare_inner_core_device");
Mesh* mp = (Mesh*)(*Mesh_pointer_f);
@@ -2432,7 +2300,7 @@
extern "C"
void FC_FUNC_(prepare_cleanup_device,
PREPARE_CLEANUP_DEVICE)(long* Mesh_pointer_f,
- int* NCHUNKS_VAL) {
+ int* NCHUNKS_VAL) {
TRACE("prepare_cleanup_device");
Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/cuda/specfem3D_gpu_cuda_method_stubs.c
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/cuda/specfem3D_gpu_cuda_method_stubs.c 2012-07-19 21:55:39 UTC (rev 20529)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/cuda/specfem3D_gpu_cuda_method_stubs.c 2012-07-19 22:05:23 UTC (rev 20530)
@@ -1,11 +1,12 @@
/*
!=====================================================================
!
-! S p e c f e m 3 D V e r s i o n 2 . 0
-! ---------------------------------------
+! S p e c f e m 3 D G l o b e V e r s i o n 5 . 1
+! --------------------------------------------------
!
! Main authors: Dimitri Komatitsch and Jeroen Tromp
-! Princeton University, USA and University of Pau / CNRS / INRIA
+! Princeton University, USA
+! and University of Pau / CNRS / INRIA, France
! (c) Princeton University / California Institute of Technology and University of Pau / CNRS / INRIA
! April 2011
!
@@ -172,7 +173,8 @@
int GRAVITY_VAL) {}
void FC_FUNC_(compute_coupling_ocean_cuda,
- COMPUTE_COUPLING_OCEAN_CUDA)(long* Mesh_pointer_f) {}
+ COMPUTE_COUPLING_OCEAN_CUDA)(long* Mesh_pointer_f,
+ int* NCHUNKS_VAL) {}
//
@@ -191,7 +193,8 @@
void FC_FUNC_(compute_forces_inner_core_cuda,
COMPUTE_FORCES_INNER_CORE_CUDA)(long* Mesh_pointer_f,
- int* iphase) {}
+ realw* deltat,
+ int* iphase) {}
//
@@ -249,6 +252,17 @@
//
+// src/cuda/initialize_cuda.cu
+//
+
+void FC_FUNC_(initialize_cuda_device,
+ INITIALIZE_CUDA_DEVICE)(int* myrank_f,int* ncuda_devices) {
+ fprintf(stderr,"ERROR: GPU_MODE enabled without GPU/CUDA Support. To enable GPU support, reconfigure with --with-cuda flag.\n");
+ exit(1);
+}
+
+
+//
// src/cuda/it_update_displacement_cuda.cu
//
@@ -284,7 +298,8 @@
realw* deltatover2_F,
int* SIMULATION_TYPE_f,
realw* b_deltatover2_F,
- int* OCEANS) {}
+ int* OCEANS,
+ int* NCHUNKS_VAL) {}
void FC_FUNC_(kernel_3_b_cuda,
KERNEL_3_B_CUDA)(long* Mesh_pointer,
@@ -333,12 +348,6 @@
// src/cuda/prepare_mesh_constants_cuda.cu
//
-void FC_FUNC_(prepare_cuda_device,
- PREPARE_CUDA_DEVICE)(int* myrank_f,int* ncuda_devices) {
- fprintf(stderr,"ERROR: GPU_MODE enabled without GPU/CUDA Support. To enable GPU support, reconfigure with --with-cuda flag.\n");
- exit(1);
-}
-
void FC_FUNC_(prepare_constants_device,
PREPARE_CONSTANTS_DEVICE)(long* Mesh_pointer,
int* myrank_f,
@@ -365,6 +374,7 @@
int* GRAVITY_f,
int* ROTATION_f,
int* ATTENUATION_f,
+ int* ATTENUATION_NEW_f,
int* USE_ATTENUATION_MIMIC_f,
int* COMPUTE_AND_STORE_STRAIN_f,
int* ANISOTROPIC_3D_MANTLE_f,
@@ -507,7 +517,7 @@
realw* jacobian2D_top_crust_mantle) {}
void FC_FUNC_(prepare_crust_mantle_device,
- PREPARE_CRUST_MANTLE_DEVICE)(long* Mesh_pointer_f,
+ PREPARE_CRUST_MANTLE_DEVICE)(long* Mesh_pointer_f,
realw* h_xix, realw* h_xiy, realw* h_xiz,
realw* h_etax, realw* h_etay, realw* h_etaz,
realw* h_gammax, realw* h_gammay, realw* h_gammaz,
@@ -515,7 +525,9 @@
realw* h_kappav, realw* h_muv,
realw* h_kappah, realw* h_muh,
realw* h_eta_aniso,
- realw* h_rmass,
+ realw* h_rmassx,
+ realw* h_rmassy,
+ realw* h_rmassz,
realw* h_normal_top_crust_mantle,
int* h_ibelm_top_crust_mantle,
int* h_ibelm_bottom_crust_mantle,
@@ -534,7 +546,8 @@
int* nspec_outer,
int* nspec_inner,
int* NSPEC2D_TOP_CM,
- int* NSPEC2D_BOTTOM_CM
+ int* NSPEC2D_BOTTOM_CM,
+ int* NCHUNKS_VAL
) {}
void FC_FUNC_(prepare_outer_core_device,
@@ -639,7 +652,8 @@
realw *c66store){}
void FC_FUNC_(prepare_cleanup_device,
- PREPARE_CLEANUP_DEVICE)(long* Mesh_pointer_f) {}
+ PREPARE_CLEANUP_DEVICE)(long* Mesh_pointer_f,
+ int* NCHUNKS_VAL) {}
//
Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/meshfem3D/Makefile.in
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/meshfem3D/Makefile.in 2012-07-19 21:55:39 UTC (rev 20529)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/meshfem3D/Makefile.in 2012-07-19 22:05:23 UTC (rev 20530)
@@ -119,7 +119,7 @@
$O/compute_volumes.o \
$O/create_central_cube.o \
$O/create_central_cube_buffers.mpi.o \
- $O/create_chunk_buffers.mpi.o \
+ $O/create_chunk_buffers.o \
$O/create_doubling_elements.o \
$O/create_mass_matrices.o \
$O/create_regions_mesh.o \
@@ -181,14 +181,14 @@
$(EMPTY_MACRO)
MESHER_ARRAY_OBJECTS = \
- $O/meshfem3D_models.mpi.o \
$O/meshfem3D_par.o \
- $O/compute_area.mpi.o \
+ $O/meshfem3D_models.o \
+ $O/compute_area.o \
$O/create_addressing.o \
- $O/create_meshes.mpi.o \
+ $O/create_meshes.o \
$O/create_MPI_interfaces.mpi.o \
- $O/finalize_mesher.mpi.o \
- $O/initialize_mesher.mpi.o \
+ $O/finalize_mesher.o \
+ $O/initialize_mesher.o \
$O/meshfem3D.mpi.o \
$O/setup_counters.o \
$O/setup_model.o \
Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/meshfem3D/compute_area.f90
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/meshfem3D/compute_area.f90 2012-07-19 21:55:39 UTC (rev 20529)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/meshfem3D/compute_area.f90 2012-07-19 22:05:23 UTC (rev 20530)
@@ -35,8 +35,6 @@
implicit none
- include 'mpif.h'
-
integer :: myrank,NCHUNKS,iregion_code
double precision :: area_local_bottom,area_local_top,volume_local
@@ -45,19 +43,16 @@
! local parameters
double precision :: volume_total_region,area_total_bottom,area_total_top
- integer :: ier
! use MPI reduction to compute total area and volume
volume_total_region = ZERO
area_total_bottom = ZERO
area_total_top = ZERO
- call MPI_REDUCE(area_local_bottom,area_total_bottom,1,MPI_DOUBLE_PRECISION,MPI_SUM,0, &
- MPI_COMM_WORLD,ier)
- call MPI_REDUCE(area_local_top,area_total_top,1,MPI_DOUBLE_PRECISION,MPI_SUM,0, &
- MPI_COMM_WORLD,ier)
- call MPI_REDUCE(volume_local,volume_total_region,1,MPI_DOUBLE_PRECISION,MPI_SUM,0, &
- MPI_COMM_WORLD,ier)
+ call sum_all_dp(area_local_bottom,area_total_bottom)
+ call sum_all_dp(area_local_top,area_total_top)
+ call sum_all_dp(volume_local,volume_total_region)
+
if(myrank == 0) then
! sum volume over all the regions
volume_total = volume_total + volume_total_region
Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/meshfem3D/compute_element_properties.f90
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/meshfem3D/compute_element_properties.f90 2012-07-19 21:55:39 UTC (rev 20529)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/meshfem3D/compute_element_properties.f90 2012-07-19 22:05:23 UTC (rev 20530)
@@ -63,7 +63,7 @@
! meshing phase
integer ipass
-
+
! 3D shape functions and their derivatives
double precision, dimension(NGNOD,NGLLX,NGLLY,NGLLZ) :: shape3D
@@ -212,7 +212,7 @@
ABSORBING_CONDITIONS,elem_in_crust,elem_in_mantle)
endif
-
+
! either use GLL points or anchor points to capture TOPOGRAPHY and ELLIPTICITY
! note: using gll points to capture them results in a slightly more accurate mesh.
! however, it introduces more deformations to the elements which might lead to
@@ -290,14 +290,14 @@
! updates jacobian
! (only needed for second meshing phase)
- if( ipass == 2 ) then
+ if( ipass == 2 ) then
call recalc_jacobian_gll3D(myrank,xstore,ystore,zstore,xigll,yigll,zigll,&
ispec,nspec,ACTUALLY_STORE_ARRAYS,&
xixstore,xiystore,xizstore,&
etaxstore,etaystore,etazstore,&
gammaxstore,gammaystore,gammazstore)
endif
-
+
end subroutine compute_element_properties
!
Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/meshfem3D/create_MPI_interfaces.f90
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/meshfem3D/create_MPI_interfaces.f90 2012-07-19 21:55:39 UTC (rev 20529)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/meshfem3D/create_MPI_interfaces.f90 2012-07-19 22:05:23 UTC (rev 20530)
@@ -29,6 +29,8 @@
use constants,only: CUSTOM_REAL,NUMFACES_SHARED,NB_SQUARE_EDGES_ONEDIR,NDIM,IMAIN
+ implicit none
+
! indirect addressing for each message for faces and corners of the chunks
! a given slice can belong to at most one corner and at most two faces
integer :: NGLOB2DMAX_XY
@@ -42,41 +44,6 @@
! number of message types
integer :: NUM_MSG_TYPES
- integer :: NGLOB1D_RADIAL_CM
- integer :: NGLOB1D_RADIAL_OC
- integer :: NGLOB1D_RADIAL_IC
-
- integer :: NGLOB2DMAX_XMIN_XMAX_CM
- integer :: NGLOB2DMAX_XMIN_XMAX_OC
- integer :: NGLOB2DMAX_XMIN_XMAX_IC
-
- integer :: NGLOB2DMAX_YMIN_YMAX_CM
- integer :: NGLOB2DMAX_YMIN_YMAX_OC
- integer :: NGLOB2DMAX_YMIN_YMAX_IC
-
- integer :: NSPEC2DMAX_XMIN_XMAX_CM
- integer :: NSPEC2DMAX_YMIN_YMAX_CM
- integer :: NSPEC2D_BOTTOM_CM
- integer :: NSPEC2D_TOP_CM
-
- integer :: NSPEC2DMAX_XMIN_XMAX_IC
- integer :: NSPEC2DMAX_YMIN_YMAX_IC
- integer :: NSPEC2D_BOTTOM_IC
- integer :: NSPEC2D_TOP_IC
-
- integer :: NSPEC2DMAX_XMIN_XMAX_OC
- integer :: NSPEC2DMAX_YMIN_YMAX_OC
- integer :: NSPEC2D_BOTTOM_OC
- integer :: NSPEC2D_TOP_OC
-
- integer :: NSPEC_CRUST_MANTLE
- integer :: NSPEC_INNER_CORE
- integer :: NSPEC_OUTER_CORE
-
- integer :: NGLOB_CRUST_MANTLE
- integer :: NGLOB_INNER_CORE
- integer :: NGLOB_OUTER_CORE
-
!-----------------------------------------------------------------
! assembly
!-----------------------------------------------------------------
@@ -106,7 +73,9 @@
! collected MPI interfaces
+ !--------------------------------------
! MPI crust/mantle mesh
+ !--------------------------------------
integer :: num_interfaces_crust_mantle
integer :: max_nibool_interfaces_crust_mantle
integer, dimension(:), allocatable :: my_neighbours_crust_mantle,nibool_interfaces_crust_mantle
@@ -116,7 +85,9 @@
integer, dimension(:), allocatable :: request_send_vector_crust_mantle,request_recv_vector_crust_mantle
+ !--------------------------------------
! MPI inner core mesh
+ !--------------------------------------
integer :: num_interfaces_inner_core
integer :: max_nibool_interfaces_inner_core
integer, dimension(:), allocatable :: my_neighbours_inner_core,nibool_interfaces_inner_core
@@ -126,7 +97,9 @@
integer, dimension(:), allocatable :: request_send_vector_inner_core,request_recv_vector_inner_core
+ !--------------------------------------
! MPI outer core mesh
+ !--------------------------------------
integer :: num_interfaces_outer_core
integer :: max_nibool_interfaces_outer_core
integer, dimension(:), allocatable :: my_neighbours_outer_core,nibool_interfaces_outer_core
@@ -145,6 +118,17 @@
!--------------------------------------
! crust mantle
!--------------------------------------
+ integer :: NSPEC_CRUST_MANTLE
+ integer :: NGLOB_CRUST_MANTLE
+
+ integer :: NGLOB1D_RADIAL_CM
+ integer :: NGLOB2DMAX_XMIN_XMAX_CM
+ integer :: NGLOB2DMAX_YMIN_YMAX_CM
+ integer :: NSPEC2DMAX_XMIN_XMAX_CM
+ integer :: NSPEC2DMAX_YMIN_YMAX_CM
+ integer :: NSPEC2D_BOTTOM_CM
+ integer :: NSPEC2D_TOP_CM
+
real(kind=CUSTOM_REAL), dimension(:),allocatable :: &
xstore_crust_mantle,ystore_crust_mantle,zstore_crust_mantle
integer, dimension(:),allocatable :: idoubling_crust_mantle
@@ -175,6 +159,17 @@
!--------------------------------------
! outer core
!--------------------------------------
+ integer :: NSPEC_OUTER_CORE
+ integer :: NGLOB_OUTER_CORE
+
+ integer :: NGLOB1D_RADIAL_OC
+ integer :: NGLOB2DMAX_XMIN_XMAX_OC
+ integer :: NGLOB2DMAX_YMIN_YMAX_OC
+ integer :: NSPEC2DMAX_XMIN_XMAX_OC
+ integer :: NSPEC2DMAX_YMIN_YMAX_OC
+ integer :: NSPEC2D_BOTTOM_OC
+ integer :: NSPEC2D_TOP_OC
+
real(kind=CUSTOM_REAL), dimension(:),allocatable :: &
xstore_outer_core,ystore_outer_core,zstore_outer_core
integer, dimension(:),allocatable :: idoubling_outer_core
@@ -206,7 +201,17 @@
!--------------------------------------
! inner core
!--------------------------------------
+ integer :: NSPEC_INNER_CORE
+ integer :: NGLOB_INNER_CORE
+ integer :: NGLOB1D_RADIAL_IC
+ integer :: NGLOB2DMAX_XMIN_XMAX_IC
+ integer :: NGLOB2DMAX_YMIN_YMAX_IC
+ integer :: NSPEC2DMAX_XMIN_XMAX_IC
+ integer :: NSPEC2DMAX_YMIN_YMAX_IC
+ integer :: NSPEC2D_BOTTOM_IC
+ integer :: NSPEC2D_TOP_IC
+
real(kind=CUSTOM_REAL), dimension(:),allocatable :: &
xstore_inner_core,ystore_inner_core,zstore_inner_core
integer, dimension(:),allocatable :: idoubling_inner_core
@@ -257,8 +262,6 @@
subroutine create_MPI_interfaces()
- use meshfem3D_par
- use create_MPI_interfaces_par
implicit none
! sets up arrays
@@ -323,7 +326,7 @@
else
call exit_MPI(myrank,'number of chunks must be either 1, 2, 3 or 6')
endif
-
+
! if more than one chunk then same number of processors in each direction
NPROC_ONE_DIRECTION = NPROC_XI
! total number of messages corresponding to these common faces
Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/meshfem3D/create_addressing.f90
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/meshfem3D/create_addressing.f90 2012-07-19 21:55:39 UTC (rev 20529)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/meshfem3D/create_addressing.f90 2012-07-19 22:05:23 UTC (rev 20530)
@@ -74,7 +74,7 @@
if(myrank == 0) close(IOUT)
! output a topology map of slices - fix 20x by nproc
- if (myrank == 0 ) then
+ if (myrank == 0 ) then
if( NCHUNKS == 6 .and. NPROC < 1000 ) then
write(IMAIN,*) 'Spatial distribution of the slices'
do iproc_xi = NPROC_XI-1, 0, -1
Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/meshfem3D/create_chunk_buffers.f90
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/meshfem3D/create_chunk_buffers.f90 2012-07-19 21:55:39 UTC (rev 20529)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/meshfem3D/create_chunk_buffers.f90 2012-07-19 22:05:23 UTC (rev 20530)
@@ -41,10 +41,6 @@
include "constants.h"
- ! standard include of the MPI library
- include 'mpif.h'
- include "precision.h"
-
integer iregion_code
integer nspec
@@ -135,7 +131,7 @@
integer npoin2D,npoin2D_send_local,npoin2D_receive_local
- integer i,j,k,ispec,ispec2D,ipoin2D,ier
+ integer i,j,k,ispec,ispec2D,ipoin2D
! current message number
integer imsg
@@ -770,19 +766,23 @@
!
! synchronize all the processes to make sure all the buffers are ready
- call MPI_BARRIER(MPI_COMM_WORLD,ier)
+ call sync_all()
! gather information about all the messages on all processes
do imsg = 1,NUMMSGS_FACES
! gather number of points for sender
npoin2D_send_local = npoin2D_send(imsg)
- call MPI_BCAST(npoin2D_send_local,1,MPI_INTEGER,iproc_sender(imsg),MPI_COMM_WORLD,ier)
+
+ call bcast_iproc_i(npoin2D_send_local,iproc_sender(imsg))
+
if(myrank /= iproc_sender(imsg)) npoin2D_send(imsg) = npoin2D_send_local
! gather number of points for receiver
npoin2D_receive_local = npoin2D_receive(imsg)
- call MPI_BCAST(npoin2D_receive_local,1,MPI_INTEGER,iproc_receiver(imsg),MPI_COMM_WORLD,ier)
+
+ call bcast_iproc_i(npoin2D_receive_local,iproc_receiver(imsg))
+
if(myrank /= iproc_receiver(imsg)) npoin2D_receive(imsg) = npoin2D_receive_local
enddo
@@ -824,68 +824,68 @@
! this only if more than 3 chunks
if(NCHUNKS > 3) then
- ichunk = 2
- iprocscorners(1,ichunk) = addressing_big(CHUNK_AB,NPROC_XI-1,0)
- iprocscorners(2,ichunk) = addressing_big(CHUNK_AC_ANTIPODE,0,0)
- iprocscorners(3,ichunk) = addressing_big(CHUNK_BC_ANTIPODE,NPROC_XI-1,0)
+ ichunk = 2
+ iprocscorners(1,ichunk) = addressing_big(CHUNK_AB,NPROC_XI-1,0)
+ iprocscorners(2,ichunk) = addressing_big(CHUNK_AC_ANTIPODE,0,0)
+ iprocscorners(3,ichunk) = addressing_big(CHUNK_BC_ANTIPODE,NPROC_XI-1,0)
- itypecorner(1,ichunk) = IUPPERLOWER
- itypecorner(2,ichunk) = ILOWERLOWER
- itypecorner(3,ichunk) = IUPPERLOWER
+ itypecorner(1,ichunk) = IUPPERLOWER
+ itypecorner(2,ichunk) = ILOWERLOWER
+ itypecorner(3,ichunk) = IUPPERLOWER
- ichunk = 3
- iprocscorners(1,ichunk) = addressing_big(CHUNK_AB,0,0)
- iprocscorners(2,ichunk) = addressing_big(CHUNK_AC,NPROC_XI-1,0)
- iprocscorners(3,ichunk) = addressing_big(CHUNK_BC_ANTIPODE,NPROC_XI-1,NPROC_ETA-1)
+ ichunk = 3
+ iprocscorners(1,ichunk) = addressing_big(CHUNK_AB,0,0)
+ iprocscorners(2,ichunk) = addressing_big(CHUNK_AC,NPROC_XI-1,0)
+ iprocscorners(3,ichunk) = addressing_big(CHUNK_BC_ANTIPODE,NPROC_XI-1,NPROC_ETA-1)
- itypecorner(1,ichunk) = ILOWERLOWER
- itypecorner(2,ichunk) = IUPPERLOWER
- itypecorner(3,ichunk) = IUPPERUPPER
+ itypecorner(1,ichunk) = ILOWERLOWER
+ itypecorner(2,ichunk) = IUPPERLOWER
+ itypecorner(3,ichunk) = IUPPERUPPER
- ichunk = 4
- iprocscorners(1,ichunk) = addressing_big(CHUNK_AB,NPROC_XI-1,NPROC_ETA-1)
- iprocscorners(2,ichunk) = addressing_big(CHUNK_BC,NPROC_XI-1,NPROC_ETA-1)
- iprocscorners(3,ichunk) = addressing_big(CHUNK_AC_ANTIPODE,0,NPROC_ETA-1)
+ ichunk = 4
+ iprocscorners(1,ichunk) = addressing_big(CHUNK_AB,NPROC_XI-1,NPROC_ETA-1)
+ iprocscorners(2,ichunk) = addressing_big(CHUNK_BC,NPROC_XI-1,NPROC_ETA-1)
+ iprocscorners(3,ichunk) = addressing_big(CHUNK_AC_ANTIPODE,0,NPROC_ETA-1)
- itypecorner(1,ichunk) = IUPPERUPPER
- itypecorner(2,ichunk) = IUPPERUPPER
- itypecorner(3,ichunk) = ILOWERUPPER
+ itypecorner(1,ichunk) = IUPPERUPPER
+ itypecorner(2,ichunk) = IUPPERUPPER
+ itypecorner(3,ichunk) = ILOWERUPPER
- ichunk = 5
- iprocscorners(1,ichunk) = addressing_big(CHUNK_AC,0,0)
- iprocscorners(2,ichunk) = addressing_big(CHUNK_BC_ANTIPODE,0,NPROC_ETA-1)
- iprocscorners(3,ichunk) = addressing_big(CHUNK_AB_ANTIPODE,NPROC_XI-1,0)
+ ichunk = 5
+ iprocscorners(1,ichunk) = addressing_big(CHUNK_AC,0,0)
+ iprocscorners(2,ichunk) = addressing_big(CHUNK_BC_ANTIPODE,0,NPROC_ETA-1)
+ iprocscorners(3,ichunk) = addressing_big(CHUNK_AB_ANTIPODE,NPROC_XI-1,0)
- itypecorner(1,ichunk) = ILOWERLOWER
- itypecorner(2,ichunk) = ILOWERUPPER
- itypecorner(3,ichunk) = IUPPERLOWER
+ itypecorner(1,ichunk) = ILOWERLOWER
+ itypecorner(2,ichunk) = ILOWERUPPER
+ itypecorner(3,ichunk) = IUPPERLOWER
- ichunk = 6
- iprocscorners(1,ichunk) = addressing_big(CHUNK_AC_ANTIPODE,NPROC_XI-1,0)
- iprocscorners(2,ichunk) = addressing_big(CHUNK_BC_ANTIPODE,0,0)
- iprocscorners(3,ichunk) = addressing_big(CHUNK_AB_ANTIPODE,0,0)
+ ichunk = 6
+ iprocscorners(1,ichunk) = addressing_big(CHUNK_AC_ANTIPODE,NPROC_XI-1,0)
+ iprocscorners(2,ichunk) = addressing_big(CHUNK_BC_ANTIPODE,0,0)
+ iprocscorners(3,ichunk) = addressing_big(CHUNK_AB_ANTIPODE,0,0)
- itypecorner(1,ichunk) = IUPPERLOWER
- itypecorner(2,ichunk) = ILOWERLOWER
- itypecorner(3,ichunk) = ILOWERLOWER
+ itypecorner(1,ichunk) = IUPPERLOWER
+ itypecorner(2,ichunk) = ILOWERLOWER
+ itypecorner(3,ichunk) = ILOWERLOWER
- ichunk = 7
- iprocscorners(1,ichunk) = addressing_big(CHUNK_AC,0,NPROC_ETA-1)
- iprocscorners(2,ichunk) = addressing_big(CHUNK_BC,0,0)
- iprocscorners(3,ichunk) = addressing_big(CHUNK_AB_ANTIPODE,NPROC_XI-1,NPROC_ETA-1)
+ ichunk = 7
+ iprocscorners(1,ichunk) = addressing_big(CHUNK_AC,0,NPROC_ETA-1)
+ iprocscorners(2,ichunk) = addressing_big(CHUNK_BC,0,0)
+ iprocscorners(3,ichunk) = addressing_big(CHUNK_AB_ANTIPODE,NPROC_XI-1,NPROC_ETA-1)
- itypecorner(1,ichunk) = ILOWERUPPER
- itypecorner(2,ichunk) = ILOWERLOWER
- itypecorner(3,ichunk) = IUPPERUPPER
+ itypecorner(1,ichunk) = ILOWERUPPER
+ itypecorner(2,ichunk) = ILOWERLOWER
+ itypecorner(3,ichunk) = IUPPERUPPER
- ichunk = 8
- iprocscorners(1,ichunk) = addressing_big(CHUNK_BC,0,NPROC_ETA-1)
- iprocscorners(2,ichunk) = addressing_big(CHUNK_AC_ANTIPODE,NPROC_XI-1,NPROC_ETA-1)
- iprocscorners(3,ichunk) = addressing_big(CHUNK_AB_ANTIPODE,0,NPROC_ETA-1)
+ ichunk = 8
+ iprocscorners(1,ichunk) = addressing_big(CHUNK_BC,0,NPROC_ETA-1)
+ iprocscorners(2,ichunk) = addressing_big(CHUNK_AC_ANTIPODE,NPROC_XI-1,NPROC_ETA-1)
+ iprocscorners(3,ichunk) = addressing_big(CHUNK_AB_ANTIPODE,0,NPROC_ETA-1)
- itypecorner(1,ichunk) = ILOWERUPPER
- itypecorner(2,ichunk) = IUPPERUPPER
- itypecorner(3,ichunk) = ILOWERUPPER
+ itypecorner(1,ichunk) = ILOWERUPPER
+ itypecorner(2,ichunk) = IUPPERUPPER
+ itypecorner(3,ichunk) = ILOWERUPPER
endif
@@ -895,71 +895,71 @@
! loop over all the messages to create the addressing
do imsg = 1,NCORNERSCHUNKS
- if(myrank == 0) write(IMAIN,*) 'Generating message ',imsg,' for corners out of ',NCORNERSCHUNKS
+ if(myrank == 0) write(IMAIN,*) 'Generating message ',imsg,' for corners out of ',NCORNERSCHUNKS
! save triplet of processors in list of messages
- if(myrank == 0) write(IOUT,*) iprocscorners(1,imsg),iprocscorners(2,imsg),iprocscorners(3,imsg)
+ if(myrank == 0) write(IOUT,*) iprocscorners(1,imsg),iprocscorners(2,imsg),iprocscorners(3,imsg)
! loop on the three processors of a given corner
- do imember_corner = 1,3
+ do imember_corner = 1,3
- if(imember_corner == 1) then
- write(filename_out,"('buffer_corners_chunks_master_msg',i6.6,'.txt')") imsg
- else if(imember_corner == 2) then
- write(filename_out,"('buffer_corners_chunks_worker1_msg',i6.6,'.txt')") imsg
- else
- write(filename_out,"('buffer_corners_chunks_worker2_msg',i6.6,'.txt')") imsg
- endif
+ if(imember_corner == 1) then
+ write(filename_out,"('buffer_corners_chunks_master_msg',i6.6,'.txt')") imsg
+ else if(imember_corner == 2) then
+ write(filename_out,"('buffer_corners_chunks_worker1_msg',i6.6,'.txt')") imsg
+ else
+ write(filename_out,"('buffer_corners_chunks_worker2_msg',i6.6,'.txt')") imsg
+ endif
! only do this if current processor is the right one for MPI version
! this line is ok even for NCHUNKS = 2
- if(iprocscorners(imember_corner,imsg) == myrank) then
+ if(iprocscorners(imember_corner,imsg) == myrank) then
! pick the correct 1D buffer
! this scheme works fine even if NPROC_XI = NPROC_ETA = 1
- if(itypecorner(imember_corner,imsg) == ILOWERLOWER) then
- filename_in = prname(1:len_trim(prname))//'ibool1D_leftxi_lefteta.txt'
- NGLOB1D_RADIAL = NGLOB1D_RADIAL_CORNER(iregion_code,1)
- else if(itypecorner(imember_corner,imsg) == ILOWERUPPER) then
- filename_in = prname(1:len_trim(prname))//'ibool1D_leftxi_righteta.txt'
- NGLOB1D_RADIAL = NGLOB1D_RADIAL_CORNER(iregion_code,4)
- else if(itypecorner(imember_corner,imsg) == IUPPERLOWER) then
- filename_in = prname(1:len_trim(prname))//'ibool1D_rightxi_lefteta.txt'
- NGLOB1D_RADIAL = NGLOB1D_RADIAL_CORNER(iregion_code,2)
- else if(itypecorner(imember_corner,imsg) == IUPPERUPPER) then
- filename_in = prname(1:len_trim(prname))//'ibool1D_rightxi_righteta.txt'
- NGLOB1D_RADIAL = NGLOB1D_RADIAL_CORNER(iregion_code,3)
- else
- call exit_MPI(myrank,'incorrect corner coordinates')
- endif
+ if(itypecorner(imember_corner,imsg) == ILOWERLOWER) then
+ filename_in = prname(1:len_trim(prname))//'ibool1D_leftxi_lefteta.txt'
+ NGLOB1D_RADIAL = NGLOB1D_RADIAL_CORNER(iregion_code,1)
+ else if(itypecorner(imember_corner,imsg) == ILOWERUPPER) then
+ filename_in = prname(1:len_trim(prname))//'ibool1D_leftxi_righteta.txt'
+ NGLOB1D_RADIAL = NGLOB1D_RADIAL_CORNER(iregion_code,4)
+ else if(itypecorner(imember_corner,imsg) == IUPPERLOWER) then
+ filename_in = prname(1:len_trim(prname))//'ibool1D_rightxi_lefteta.txt'
+ NGLOB1D_RADIAL = NGLOB1D_RADIAL_CORNER(iregion_code,2)
+ else if(itypecorner(imember_corner,imsg) == IUPPERUPPER) then
+ filename_in = prname(1:len_trim(prname))//'ibool1D_rightxi_righteta.txt'
+ NGLOB1D_RADIAL = NGLOB1D_RADIAL_CORNER(iregion_code,3)
+ else
+ call exit_MPI(myrank,'incorrect corner coordinates')
+ endif
! read 1D buffer for corner
- open(unit=IIN,file=filename_in,status='old',action='read')
- do ipoin1D = 1,NGLOB1D_RADIAL
- read(IIN,*) ibool1D(ipoin1D), &
- xread1D(ipoin1D),yread1D(ipoin1D),zread1D(ipoin1D)
- enddo
- close(IIN)
+ open(unit=IIN,file=filename_in,status='old',action='read')
+ do ipoin1D = 1,NGLOB1D_RADIAL
+ read(IIN,*) ibool1D(ipoin1D), &
+ xread1D(ipoin1D),yread1D(ipoin1D),zread1D(ipoin1D)
+ enddo
+ close(IIN)
! sort array read based upon the coordinates of the points
! to ensure conforming matching with other buffers from neighbors
- call sort_array_coordinates(NGLOB1D_RADIAL,xread1D,yread1D,zread1D, &
- ibool1D,iglob,locval,ifseg,nglob,ind,ninseg,iwork,work)
+ call sort_array_coordinates(NGLOB1D_RADIAL,xread1D,yread1D,zread1D, &
+ ibool1D,iglob,locval,ifseg,nglob,ind,ninseg,iwork,work)
! check that no duplicates have been found
- if(nglob /= NGLOB1D_RADIAL) call exit_MPI(myrank,'duplicates found for corners')
+ if(nglob /= NGLOB1D_RADIAL) call exit_MPI(myrank,'duplicates found for corners')
! write file with 1D buffer for corner
- open(unit=IOUT_BUFFERS,file=prname(1:len_trim(prname))//filename_out,status='unknown')
- write(IOUT_BUFFERS,*) NGLOB1D_RADIAL
- do ipoin1D = 1,NGLOB1D_RADIAL
- write(IOUT_BUFFERS,*) ibool1D(ipoin1D), &
- xread1D(ipoin1D),yread1D(ipoin1D),zread1D(ipoin1D)
- enddo
- close(IOUT_BUFFERS)
+ open(unit=IOUT_BUFFERS,file=prname(1:len_trim(prname))//filename_out,status='unknown')
+ write(IOUT_BUFFERS,*) NGLOB1D_RADIAL
+ do ipoin1D = 1,NGLOB1D_RADIAL
+ write(IOUT_BUFFERS,*) ibool1D(ipoin1D), &
+ xread1D(ipoin1D),yread1D(ipoin1D),zread1D(ipoin1D)
+ enddo
+ close(IOUT_BUFFERS)
! end of section done only if right processor for MPI
- endif
+ endif
enddo
Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/meshfem3D/create_mass_matrices.f90
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/meshfem3D/create_mass_matrices.f90 2012-07-19 21:55:39 UTC (rev 20529)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/meshfem3D/create_mass_matrices.f90 2012-07-19 22:05:23 UTC (rev 20530)
@@ -86,7 +86,7 @@
! Stacey conditions put back
integer :: NSPEC2D_TOP,NSPEC2D_BOTTOM,NSPEC2DMAX_XMIN_XMAX,NSPEC2DMAX_YMIN_YMAX
integer :: nspec_stacey
-
+
double precision, dimension(NGLLX) :: wxgll
double precision, dimension(NGLLY) :: wygll
double precision, dimension(NGLLZ) :: wzgll
@@ -128,11 +128,11 @@
integer :: ix_oceans,iy_oceans,iz_oceans,ispec_oceans,ispec2D_top_crust
! initializes matrices
- !
- ! in the case of stacey boundary conditions, add C*deltat/2 contribution to the mass matrix
+ !
+ ! in the case of stacey boundary conditions, add C*deltat/2 contribution to the mass matrix
! on Stacey edges for the crust_mantle and outer_core regions but not for the inner_core region
! thus the mass matrix must be replaced by three mass matrices including the "C" damping matrix
- !
+ !
! if absorbing_conditions are not set or if NCHUNKS=6, only one mass matrix is needed
! for the sake of performance, only "rmassz" array will be filled and "rmassx" & "rmassy" will be obsolete
@@ -349,7 +349,7 @@
rmassx(:) = rmassz(:)
rmassy(:) = rmassz(:)
-
+
! xmin
! if two chunks exclude this face for one of them
if(NCHUNKS == 1 .or. ichunk == CHUNK_AC) then
Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/meshfem3D/create_meshes.f90
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/meshfem3D/create_meshes.f90 2012-07-19 21:55:39 UTC (rev 20529)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/meshfem3D/create_meshes.f90 2012-07-19 22:05:23 UTC (rev 20530)
@@ -38,6 +38,9 @@
integer, dimension(:,:,:,:), allocatable :: ibool
! arrays with the mesh in double precision
double precision, dimension(:,:,:,:), allocatable :: xstore,ystore,zstore
+ ! this for non blocking MPI
+ logical, dimension(:), allocatable :: is_on_a_slice_edge
+ integer :: ipass
integer :: ier
! get addressing for this process
@@ -98,60 +101,17 @@
! perform two passes in this part to be able to save memory
do ipass = 1,2
call create_regions_mesh(iregion_code,ibool,idoubling,is_on_a_slice_edge, &
- xstore,ystore,zstore,rmins,rmaxs, &
- iproc_xi,iproc_eta,ichunk,NSPEC(iregion_code),nspec_tiso, &
- volume_local,area_local_bottom,area_local_top, &
+ xstore,ystore,zstore, &
+ NSPEC(iregion_code), &
NGLOB(iregion_code),npointot, &
- NEX_XI,NEX_PER_PROC_XI,NEX_PER_PROC_ETA, &
+ NEX_PER_PROC_XI,NEX_PER_PROC_ETA, &
NSPEC2DMAX_XMIN_XMAX(iregion_code),NSPEC2DMAX_YMIN_YMAX(iregion_code), &
+ NGLOB2DMAX_XMIN_XMAX(iregion_code),NGLOB2DMAX_YMIN_YMAX(iregion_code), &
NSPEC2D_BOTTOM(iregion_code),NSPEC2D_TOP(iregion_code), &
- NPROC_XI,NPROC_ETA, &
- NSPEC2D_XI_FACE,NSPEC2D_ETA_FACE,NSPEC1D_RADIAL_CORNER,NGLOB1D_RADIAL_CORNER, &
- myrank,LOCAL_PATH,rotation_matrix,ANGULAR_WIDTH_XI_RAD,ANGULAR_WIDTH_ETA_RAD, &
- SAVE_MESH_FILES,NCHUNKS,INCLUDE_CENTRAL_CUBE,ABSORBING_CONDITIONS, &
- R_CENTRAL_CUBE,RICB,RHO_OCEANS,RCMB,R670,RMOHO,RMOHO_FICTITIOUS_IN_MESHER,&
- RTOPDDOUBLEPRIME,R600,R220,R771,R400,R120,R80,RMIDDLE_CRUST,ROCEAN, &
- ner,ratio_sampling_array,doubling_index,r_bottom,r_top,&
- this_region_has_a_doubling,ratio_divide_central_cube, &
- CUT_SUPERBRICK_XI,CUT_SUPERBRICK_ETA, &
mod(iproc_xi_slice(myrank),2),mod(iproc_eta_slice(myrank),2), &
ipass)
enddo
- ! checks number of anisotropic elements found in the mantle
- if(iregion_code /= IREGION_CRUST_MANTLE .and. nspec_tiso /= 0 ) &
- call exit_MPI(myrank,'found anisotropic elements outside of the mantle')
-
- if( TRANSVERSE_ISOTROPY ) then
- if(iregion_code == IREGION_CRUST_MANTLE .and. nspec_tiso == 0) &
- call exit_MPI(myrank,'found no anisotropic elements in the mantle')
- endif
-
- ! computes total area and volume
- call compute_area(myrank,NCHUNKS,iregion_code, &
- area_local_bottom,area_local_top,&
- volume_local,volume_total, &
- RCMB,RICB,R_CENTRAL_CUBE)
-
- ! create chunk buffers if more than one chunk
- if(NCHUNKS > 1) then
- call create_chunk_buffers(iregion_code,NSPEC(iregion_code),ibool,idoubling, &
- xstore,ystore,zstore, &
- NGLOB(iregion_code), &
- NSPEC2DMAX_XMIN_XMAX(iregion_code),NSPEC2DMAX_YMIN_YMAX(iregion_code), &
- NPROC_XI,NPROC_ETA,NPROC,NPROCTOT, &
- NGLOB1D_RADIAL_CORNER,maxval(NGLOB1D_RADIAL_CORNER(iregion_code,:)), &
- NGLOB2DMAX_XMIN_XMAX(iregion_code),NGLOB2DMAX_YMIN_YMAX(iregion_code), &
- myrank,LOCAL_PATH,addressing, &
- ichunk_slice,iproc_xi_slice,iproc_eta_slice,NCHUNKS)
- else
- if(myrank == 0) then
- write(IMAIN,*)
- write(IMAIN,*) 'only one chunk, no need to create chunk buffers'
- write(IMAIN,*)
- endif
- endif
-
! deallocate arrays used for that region
deallocate(idoubling)
deallocate(ibool)
Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/meshfem3D/create_regions_mesh.f90
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/meshfem3D/create_regions_mesh.f90 2012-07-19 21:55:39 UTC (rev 20529)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/meshfem3D/create_regions_mesh.f90 2012-07-19 22:05:23 UTC (rev 20530)
@@ -29,6 +29,8 @@
use constants,only: NGLLX,NGLLY,NGLLZ,NGNOD,NGNOD2D,NDIM,NDIM2D
+ implicit none
+
! topology of the elements
integer, dimension(NGNOD) :: iaddx,iaddy,iaddz
@@ -51,121 +53,23 @@
end module create_regions_mesh_par
-
!
!-------------------------------------------------------------------------------------------------
!
- subroutine create_regions_mesh(iregion_code,ibool,idoubling,is_on_a_slice_edge, &
- xstore,ystore,zstore,rmins,rmaxs, &
- iproc_xi,iproc_eta,ichunk, &
- nspec,nspec_tiso, &
- volume_local,area_local_bottom,area_local_top, &
- nglob_theor,npointot, &
- NEX_XI,NEX_PER_PROC_XI,NEX_PER_PROC_ETA, &
- NSPEC2DMAX_XMIN_XMAX,NSPEC2DMAX_YMIN_YMAX, &
- NSPEC2D_BOTTOM,NSPEC2D_TOP, &
- NPROC_XI,NPROC_ETA, &
- NSPEC2D_XI_FACE,NSPEC2D_ETA_FACE,NSPEC1D_RADIAL_CORNER,NGLOB1D_RADIAL_CORNER, &
- myrank,LOCAL_PATH,rotation_matrix,ANGULAR_WIDTH_XI_RAD,ANGULAR_WIDTH_ETA_RAD,&
- SAVE_MESH_FILES,NCHUNKS,INCLUDE_CENTRAL_CUBE,ABSORBING_CONDITIONS, &
- R_CENTRAL_CUBE,RICB,RHO_OCEANS,RCMB,R670,RMOHO,RMOHO_FICTITIOUS_IN_MESHER,&
- RTOPDDOUBLEPRIME,R600,R220,R771,R400,R120,R80,RMIDDLE_CRUST,ROCEAN, &
- ner,ratio_sampling_array,doubling_index,r_bottom,r_top, &
- this_region_has_a_doubling,ratio_divide_central_cube, &
- CUT_SUPERBRICK_XI,CUT_SUPERBRICK_ETA, &
- offset_proc_xi,offset_proc_eta, &
- ipass)
+ module create_regions_mesh_par2
-! creates the different regions of the mesh
+ use constants,only: CUSTOM_REAL,N_SLS
- use meshfem3D_models_par
- use create_regions_mesh_par
implicit none
- ! code for the four regions of the mesh
- integer :: iregion_code
-
- ! correct number of spectral elements in each block depending on chunk type
- integer :: nspec,nspec_tiso
integer :: nspec_stacey,nspec_actually,nspec_att
- integer, dimension(NGLLX,NGLLY,NGLLZ,nspec) :: ibool
- integer, dimension(nspec) :: idoubling
-
- ! this for non blocking MPI
- logical, dimension(nspec) :: is_on_a_slice_edge
-
- ! arrays with the mesh in double precision
- double precision xstore(NGLLX,NGLLY,NGLLZ,nspec)
- double precision ystore(NGLLX,NGLLY,NGLLZ,nspec)
- double precision zstore(NGLLX,NGLLY,NGLLZ,nspec)
-
- ! meshing parameters
- double precision, dimension(MAX_NUMBER_OF_MESH_LAYERS) :: rmins,rmaxs
-
- integer :: iproc_xi,iproc_eta,ichunk
-
- ! check area and volume of the final mesh
- double precision :: area_local_bottom,area_local_top
- double precision :: volume_local
-
- integer :: nglob_theor,npointot
-
- integer :: NEX_XI,NEX_PER_PROC_XI,NEX_PER_PROC_ETA
- integer :: NSPEC2DMAX_XMIN_XMAX,NSPEC2DMAX_YMIN_YMAX,NSPEC2D_BOTTOM,NSPEC2D_TOP
- integer :: NPROC_XI,NPROC_ETA
-
- ! this to cut the doubling brick
- integer, dimension(MAX_NUM_REGIONS,NB_SQUARE_EDGES_ONEDIR) :: NSPEC2D_XI_FACE,NSPEC2D_ETA_FACE
- integer, dimension(MAX_NUM_REGIONS,NB_SQUARE_CORNERS) :: NSPEC1D_RADIAL_CORNER,NGLOB1D_RADIAL_CORNER
-
- ! proc numbers for MPI
- integer myrank
-
- character(len=150) :: LOCAL_PATH
-
- ! rotation matrix from Euler angles
- double precision, dimension(NDIM,NDIM) :: rotation_matrix
-
- double precision :: ANGULAR_WIDTH_XI_RAD,ANGULAR_WIDTH_ETA_RAD
-
- logical :: SAVE_MESH_FILES
-
- integer :: NCHUNKS
-
- logical :: INCLUDE_CENTRAL_CUBE,ABSORBING_CONDITIONS
-
- double precision :: R_CENTRAL_CUBE,RICB
- double precision :: RHO_OCEANS
- double precision :: RCMB,R670,RMOHO,RMOHO_FICTITIOUS_IN_MESHER, &
- RTOPDDOUBLEPRIME,R600,R220,R771,R400,R120,R80,RMIDDLE_CRUST,ROCEAN
-
- integer, dimension(MAX_NUMBER_OF_MESH_LAYERS) :: ner,ratio_sampling_array
- integer, dimension(MAX_NUMBER_OF_MESH_LAYERS) :: doubling_index
-
- double precision, dimension(MAX_NUMBER_OF_MESH_LAYERS) :: r_bottom,r_top
- logical, dimension(MAX_NUMBER_OF_MESH_LAYERS) :: this_region_has_a_doubling
-
- integer :: ratio_divide_central_cube
-
- logical :: CUT_SUPERBRICK_XI,CUT_SUPERBRICK_ETA
- integer :: offset_proc_xi,offset_proc_eta
-
- ! now perform two passes in this part to be able to save memory
- integer :: ipass
-
- ! local parameters
-
- integer :: ner_without_doubling,ilayer,ilayer_loop, &
- ifirst_region,ilast_region
+ integer :: ifirst_region,ilast_region
integer, dimension(:), allocatable :: perm_layer
- ! parameters needed to store the radii of the grid points in the spherically symmetric Earth
- double precision rmin,rmax
-
! for model density and anisotropy
- integer nspec_ani
+ integer :: nspec_ani
real(kind=CUSTOM_REAL), dimension(:,:,:,:), allocatable :: rhostore,dvpstore, &
kappavstore,kappahstore,muvstore,muhstore,eta_anisostore
@@ -182,19 +86,8 @@
real(kind=CUSTOM_REAL), dimension(:,:,:,:), allocatable :: xixstore,xiystore,xizstore, &
etaxstore,etaystore,etazstore,gammaxstore,gammaystore,gammazstore
- ! variables for creating array ibool (some arrays also used for AVS or DX files)
- integer, dimension(:), allocatable :: locval
- logical, dimension(:), allocatable :: ifseg
-
- integer, dimension(:), allocatable :: num_ibool_AVS_DX
- logical, dimension(:), allocatable :: mask_ibool
-
- double precision, dimension(:), allocatable :: xp,yp,zp
- integer :: nglob
- integer :: ieoff,ilocnum,ier
-
! mass matrices
- real(kind=CUSTOM_REAL), dimension(:), allocatable :: rmassx,rmassy,rmassz
+ real(kind=CUSTOM_REAL), dimension(:), allocatable :: rmassx,rmassy,rmassz
integer :: nglob_xy
! mass matrix and bathymetry for ocean load
@@ -241,8 +134,6 @@
logical :: ACTUALLY_STORE_ARRAYS
- integer, save :: npoin2D_xi,npoin2D_eta
-
! Boundary Mesh
integer NSPEC2D_MOHO,NSPEC2D_400,NSPEC2D_670,nex_eta_moho
integer, dimension(:), allocatable :: ibelm_moho_top,ibelm_moho_bot,ibelm_400_top,ibelm_400_bot, &
@@ -256,16 +147,95 @@
! flags for transverse isotropic elements
logical, dimension(:), allocatable :: ispec_is_tiso
- integer i,j,k,ispec
-
- ! timing
- double precision, external :: wtime
- double precision :: time_start,tCPU
-
! name of the database file
character(len=150) :: prname
- character(len=150) :: errmsg
+ end module create_regions_mesh_par2
+
+
+!
+!-------------------------------------------------------------------------------------------------
+!
+
+ subroutine create_regions_mesh(iregion_code,ibool,idoubling,is_on_a_slice_edge, &
+ xstore,ystore,zstore, &
+ nspec, &
+ nglob_theor,npointot, &
+ NEX_PER_PROC_XI,NEX_PER_PROC_ETA, &
+ NSPEC2DMAX_XMIN_XMAX,NSPEC2DMAX_YMIN_YMAX, &
+ NGLOB2DMAX_XMIN_XMAX,NGLOB2DMAX_YMIN_YMAX, &
+ NSPEC2D_BOTTOM,NSPEC2D_TOP, &
+ offset_proc_xi,offset_proc_eta, &
+ ipass)
+
+! creates the different regions of the mesh
+
+ use meshfem3D_par,only: &
+ IMAIN,volume_total,addressing,ichunk_slice,iproc_xi_slice,iproc_eta_slice, &
+ myrank,LOCAL_PATH, &
+ IREGION_CRUST_MANTLE,IREGION_OUTER_CORE,IREGION_INNER_CORE,IFLAG_IN_FICTITIOUS_CUBE, &
+ NPROC,NPROCTOT,NPROC_XI,NPROC_ETA,NCHUNKS, &
+ SAVE_MESH_FILES,INCLUDE_CENTRAL_CUBE,ABSORBING_CONDITIONS, &
+ R_CENTRAL_CUBE,RICB,RHO_OCEANS,RCMB,R670,RMOHO,RMOHO_FICTITIOUS_IN_MESHER,&
+ RTOPDDOUBLEPRIME,R600,R220,R771,R400,R120,R80,RMIDDLE_CRUST,ROCEAN, &
+ MAX_NUMBER_OF_MESH_LAYERS,MAX_NUM_REGIONS,NB_SQUARE_EDGES_ONEDIR,NB_SQUARE_CORNERS, &
+ rmins,rmaxs,iproc_xi,iproc_eta,ichunk,NEX_XI, &
+ rotation_matrix,ANGULAR_WIDTH_XI_RAD,ANGULAR_WIDTH_ETA_RAD, &
+ ratio_sampling_array,doubling_index,this_region_has_a_doubling, &
+ ratio_divide_central_cube,CUT_SUPERBRICK_XI,CUT_SUPERBRICK_ETA, &
+ NSPEC1D_RADIAL_CORNER,NGLOB1D_RADIAL_CORNER, &
+ ner,r_bottom,r_top
+
+ use meshfem3D_models_par,only: &
+ ATTENUATION,ANISOTROPIC_INNER_CORE,ANISOTROPIC_3D_MANTLE, &
+ SAVE_BOUNDARY_MESH,ONE_CRUST,CASE_3D,SUPPRESS_CRUSTAL_MESH,REGIONAL_MOHO_MESH, &
+ OCEANS,TRANSVERSE_ISOTROPY,HETEROGEN_3D_MANTLE,HONOR_1D_SPHERICAL_MOHO, &
+ ELLIPTICITY,ISOTROPIC_3D_MANTLE, &
+ AM_V,nspl,rspl,espl,espl2
+
+ use create_regions_mesh_par
+ use create_regions_mesh_par2
+
+ implicit none
+
+ ! code for the four regions of the mesh
+ integer :: iregion_code
+
+ ! correct number of spectral elements in each block depending on chunk type
+ integer :: nspec
+
+ integer, dimension(NGLLX,NGLLY,NGLLZ,nspec) :: ibool
+ integer, dimension(nspec) :: idoubling
+
+ ! this for non blocking MPI
+ logical, dimension(nspec) :: is_on_a_slice_edge
+
+ ! arrays with the mesh in double precision
+ double precision xstore(NGLLX,NGLLY,NGLLZ,nspec)
+ double precision ystore(NGLLX,NGLLY,NGLLZ,nspec)
+ double precision zstore(NGLLX,NGLLY,NGLLZ,nspec)
+
+ integer :: nglob_theor,npointot
+
+ integer :: NEX_PER_PROC_XI,NEX_PER_PROC_ETA
+ integer :: NSPEC2DMAX_XMIN_XMAX,NSPEC2DMAX_YMIN_YMAX
+ integer :: NGLOB2DMAX_XMIN_XMAX,NGLOB2DMAX_YMIN_YMAX
+ integer :: NSPEC2D_BOTTOM,NSPEC2D_TOP
+
+ integer :: offset_proc_xi,offset_proc_eta
+
+ ! now perform two passes in this part to be able to save memory
+ integer,intent(in) :: ipass
+
+ ! local parameters
+ integer :: ier
+ integer :: nglob
+ ! saved for second call
+ integer, save :: npoin2D_xi,npoin2D_eta
+ ! check area and volume of the final mesh
+ double precision :: area_local_bottom,area_local_top
+ double precision :: volume_local
+
! user output
if(myrank == 0 ) then
select case(ipass)
@@ -282,11 +252,324 @@
call create_name_database(prname,myrank,iregion_code,LOCAL_PATH)
! initializes arrays
+ call sync_all()
if( myrank == 0) then
- write(IMAIN,*)
write(IMAIN,*) ' ...allocating arrays '
endif
+ call crm_allocate_arrays(iregion_code,nspec, &
+ NSPEC2DMAX_XMIN_XMAX,NSPEC2DMAX_YMIN_YMAX, &
+ NSPEC2D_BOTTOM,NSPEC2D_TOP, &
+ ipass)
+
+! initialize number of layers
+ call sync_all()
+ if( myrank == 0) then
+ write(IMAIN,*) ' ...setting up layers '
+ endif
+ call crm_setup_layers(iregion_code,nspec,ipass, &
+ xstore,ystore,zstore,ibool,idoubling, &
+ NEX_PER_PROC_ETA,is_on_a_slice_edge)
+
+! creates mesh elements
+ call sync_all()
+ if( myrank == 0) then
+ write(IMAIN,*) ' ...creating mesh elements '
+ endif
+ call crm_create_elements(iregion_code,nspec,ipass, &
+ xstore,ystore,zstore,idoubling, &
+ NEX_PER_PROC_XI,NEX_PER_PROC_ETA, &
+ is_on_a_slice_edge, &
+ offset_proc_xi,offset_proc_eta)
+
+
+ ! only create global addressing and the MPI buffers in the first pass
+ select case(ipass)
+ case( 1 )
+ ! creates ibool index array for projection from local to global points
+ call sync_all()
+ if( myrank == 0) then
+ write(IMAIN,*) ' ...creating global addressing'
+ endif
+ call crm_setup_indexing(ibool,xstore,ystore,zstore, &
+ nspec,nglob_theor,npointot)
+
+
+ ! create MPI buffers
+ call sync_all()
+ if( myrank == 0) then
+ write(IMAIN,*) ' ...creating MPI buffers'
+ endif
+ call crm_setup_mpi_buffers(npointot,nspec,ibool,idoubling, &
+ xstore,ystore,zstore,iregion_code, &
+ npoin2D_xi,npoin2D_eta)
+
+
+ ! Stacey
+ if(NCHUNKS /= 6) then
+ call get_absorb(myrank,prname,iboun,nspec,nimin,nimax,njmin,njmax,nkmin_xi,nkmin_eta, &
+ NSPEC2DMAX_XMIN_XMAX,NSPEC2DMAX_YMIN_YMAX,NSPEC2D_BOTTOM)
+ endif
+
+ ! only create mass matrix and save all the final arrays in the second pass
+ case( 2 )
+ ! precomputes jacobian for 2d absorbing boundary surfaces
+ call sync_all()
+ if( myrank == 0) then
+ write(IMAIN,*) ' ...precomputing jacobian'
+ endif
+ call get_jacobian_boundaries(myrank,iboun,nspec,xstore,ystore,zstore, &
+ dershape2D_x,dershape2D_y,dershape2D_bottom,dershape2D_top, &
+ ibelm_xmin,ibelm_xmax,ibelm_ymin,ibelm_ymax,ibelm_bottom,ibelm_top, &
+ nspec2D_xmin,nspec2D_xmax,nspec2D_ymin,nspec2D_ymax, &
+ jacobian2D_xmin,jacobian2D_xmax, &
+ jacobian2D_ymin,jacobian2D_ymax, &
+ jacobian2D_bottom,jacobian2D_top, &
+ normal_xmin,normal_xmax, &
+ normal_ymin,normal_ymax, &
+ normal_bottom,normal_top, &
+ NSPEC2D_BOTTOM,NSPEC2D_TOP, &
+ NSPEC2DMAX_XMIN_XMAX,NSPEC2DMAX_YMIN_YMAX,&
+ xigll,yigll,zigll)
+
+ !uncomment: adds model smoothing for point profile models
+ ! if( THREE_D_MODEL == THREE_D_MODEL_PPM ) then
+ ! call smooth_model(myrank, nproc_xi,nproc_eta,&
+ ! rho_vp,rho_vs,nspec_stacey, &
+ ! iregion_code,xixstore,xiystore,xizstore, &
+ ! etaxstore,etaystore,etazstore, &
+ ! gammaxstore,gammaystore,gammazstore, &
+ ! xstore,ystore,zstore,rhostore,dvpstore, &
+ ! kappavstore,kappahstore,muvstore,muhstore,eta_anisostore,&
+ ! nspec,HETEROGEN_3D_MANTLE, &
+ ! NEX_XI,NCHUNKS,ABSORBING_CONDITIONS,PPM_V )
+
+ ! creates mass matrix
+ call sync_all()
+ if( myrank == 0) then
+ write(IMAIN,*) ' ...creating mass matrix'
+ endif
+
+ ! allocates mass matrices in this slice (will be fully assembled in the solver)
+ !
+ ! in the case of stacey boundary conditions, add C*deltat/2 contribution to the mass matrix
+ ! on Stacey edges for the crust_mantle and outer_core regions but not for the inner_core region
+ ! thus the mass matrix must be replaced by three mass matrices including the "C" damping matrix
+ !
+ ! if absorbing_conditions are not set or if NCHUNKS=6, only one mass matrix is needed
+ ! for the sake of performance, only "rmassz" array will be filled and "rmassx" & "rmassy" will be obsolete
+
+ ! copy the theoretical number of points for the second pass
+ nglob = nglob_theor
+
+ if(NCHUNKS /= 6 .and. ABSORBING_CONDITIONS) then
+ select case(iregion_code)
+ case( IREGION_CRUST_MANTLE )
+ nglob_xy = nglob
+ case( IREGION_INNER_CORE, IREGION_OUTER_CORE )
+ nglob_xy = 1
+ endselect
+ else
+ nglob_xy = 1
+ endif
+
+ allocate(rmassx(nglob_xy),stat=ier)
+ if(ier /= 0) stop 'error in allocate 21'
+ allocate(rmassy(nglob_xy),stat=ier)
+ if(ier /= 0) stop 'error in allocate 21'
+ allocate(rmassz(nglob),stat=ier)
+ if(ier /= 0) stop 'error in allocate 21'
+
+ ! allocates ocean load mass matrix as well if oceans
+ if(OCEANS .and. iregion_code == IREGION_CRUST_MANTLE) then
+ nglob_oceans = nglob
+ else
+ ! allocate dummy array if no oceans
+ nglob_oceans = 1
+ endif
+ allocate(rmass_ocean_load(nglob_oceans),stat=ier)
+ if(ier /= 0) stop 'error in allocate 22'
+
+ ! creating mass matrices in this slice (will be fully assembled in the solver)
+ call create_mass_matrices(myrank,nspec,idoubling,wxgll,wygll,wzgll,ibool, &
+ nspec_actually,xixstore,xiystore,xizstore, &
+ etaxstore,etaystore,etazstore, &
+ gammaxstore,gammaystore,gammazstore, &
+ iregion_code,rhostore,kappavstore, &
+ nglob_xy,nglob,prname, &
+ rmassx,rmassy,rmassz, &
+ nglob_oceans,rmass_ocean_load, &
+ xstore,ystore,zstore,RHO_OCEANS, &
+ NSPEC2D_TOP,NSPEC2D_BOTTOM,NSPEC2DMAX_XMIN_XMAX,NSPEC2DMAX_YMIN_YMAX, &
+ ibelm_xmin,ibelm_xmax,ibelm_ymin,ibelm_ymax,ibelm_bottom,ibelm_top, &
+ nspec2D_xmin,nspec2D_xmax,nspec2D_ymin,nspec2D_ymax, &
+ normal_xmin,normal_xmax,normal_ymin,normal_ymax, &
+ rho_vp,rho_vs,nspec_stacey, &
+ jacobian2D_xmin,jacobian2D_xmax,jacobian2D_ymin,jacobian2D_ymax, &
+ jacobian2D_bottom,jacobian2D_top)
+
+ ! save the binary files
+ call sync_all()
+ if( myrank == 0) then
+ write(IMAIN,*) ' ...saving binary files'
+ endif
+ call save_arrays_solver(myrank,rho_vp,rho_vs,nspec_stacey, &
+ prname,iregion_code,xixstore,xiystore,xizstore, &
+ etaxstore,etaystore,etazstore,gammaxstore,gammaystore,gammazstore, &
+ xstore,ystore,zstore,rhostore,dvpstore, &
+ kappavstore,kappahstore,muvstore,muhstore,eta_anisostore, &
+ nspec_ani,c11store,c12store,c13store,c14store,c15store,c16store,c22store, &
+ c23store,c24store,c25store,c26store,c33store,c34store,c35store, &
+ c36store,c44store,c45store,c46store,c55store,c56store,c66store, &
+ ibool,idoubling,is_on_a_slice_edge,nglob_xy,nglob, &
+ rmassx,rmassy,rmassz,rmass_ocean_load,nglob_oceans, &
+ ibelm_xmin,ibelm_xmax,ibelm_ymin,ibelm_ymax,ibelm_bottom,ibelm_top, &
+ nspec2D_xmin,nspec2D_xmax,nspec2D_ymin,nspec2D_ymax, &
+ normal_xmin,normal_xmax,normal_ymin,normal_ymax,normal_bottom,normal_top, &
+ jacobian2D_xmin,jacobian2D_xmax,jacobian2D_ymin,jacobian2D_ymax, &
+ jacobian2D_bottom,jacobian2D_top,nspec, &
+ NSPEC2DMAX_XMIN_XMAX,NSPEC2DMAX_YMIN_YMAX,NSPEC2D_BOTTOM,NSPEC2D_TOP, &
+ TRANSVERSE_ISOTROPY,HETEROGEN_3D_MANTLE,ANISOTROPIC_3D_MANTLE, &
+ ANISOTROPIC_INNER_CORE,OCEANS, &
+ tau_s,tau_e_store,Qmu_store,T_c_source,ATTENUATION, &
+ size(tau_e_store,2),size(tau_e_store,3),size(tau_e_store,4),size(tau_e_store,5),&
+ ABSORBING_CONDITIONS,SAVE_MESH_FILES,ispec_is_tiso)
+
+ deallocate(rmassx,rmassy,rmassz)
+ deallocate(rmass_ocean_load)
+
+ ! create chunk buffers if more than one chunk
+ call sync_all()
+ if( myrank == 0) then
+ write(IMAIN,*) ' ...creating chunk buffers'
+ endif
+ if(NCHUNKS > 1) then
+ call create_chunk_buffers(iregion_code,nspec,ibool,idoubling, &
+ xstore,ystore,zstore, &
+ nglob_theor, &
+ NSPEC2DMAX_XMIN_XMAX,NSPEC2DMAX_YMIN_YMAX, &
+ NPROC_XI,NPROC_ETA, &
+ NPROC,NPROCTOT, &
+ NGLOB1D_RADIAL_CORNER,maxval(NGLOB1D_RADIAL_CORNER(iregion_code,:)), &
+ NGLOB2DMAX_XMIN_XMAX,NGLOB2DMAX_YMIN_YMAX, &
+ myrank,LOCAL_PATH,addressing, &
+ ichunk_slice,iproc_xi_slice,iproc_eta_slice,NCHUNKS)
+ else
+ if(myrank == 0) then
+ write(IMAIN,*)
+ write(IMAIN,*) 'only one chunk, no need to create chunk buffers'
+ write(IMAIN,*)
+ endif
+ endif
+
+ ! boundary mesh
+ if (SAVE_BOUNDARY_MESH .and. iregion_code == IREGION_CRUST_MANTLE) then
+ ! user output
+ call sync_all()
+ if( myrank == 0) then
+ write(IMAIN,*) ' ...saving boundary mesh files'
+ endif
+ call crm_save_boundary_mesh()
+ endif
+
+ ! compute volume, bottom and top area of that part of the slice
+ call compute_volumes(volume_local,area_local_bottom,area_local_top, &
+ nspec,wxgll,wygll,wzgll,xixstore,xiystore,xizstore, &
+ etaxstore,etaystore,etazstore,gammaxstore,gammaystore,gammazstore, &
+ NSPEC2D_BOTTOM,jacobian2D_bottom,NSPEC2D_TOP,jacobian2D_top)
+
+ ! computes total area and volume
+ call compute_area(myrank,NCHUNKS,iregion_code, &
+ area_local_bottom,area_local_top,&
+ volume_local,volume_total, &
+ RCMB,RICB,R_CENTRAL_CUBE)
+
+ ! create AVS or DX mesh data for the slices
+ if(SAVE_MESH_FILES) then
+ ! user output
+ call sync_all()
+ if( myrank == 0) then
+ write(IMAIN,*) ' ...saving AVS mesh files'
+ endif
+ call crm_save_mesh_files(nspec,npointot,iregion_code,ibool,idoubling, &
+ xstore,ystore,zstore)
+ endif
+
+ case default
+ stop 'there cannot be more than two passes in mesh creation'
+
+ end select ! end of test if first or second pass
+
+ deallocate(stretch_tab)
+ deallocate(perm_layer)
+
+ ! deallocate these arrays after each pass
+ ! because they have a different size in each pass to save memory
+ deallocate(xixstore,xiystore,xizstore)
+ deallocate(etaxstore,etaystore,etazstore)
+ deallocate(gammaxstore,gammaystore,gammazstore)
+
+ ! deallocate arrays
+ deallocate(rhostore,dvpstore,kappavstore,kappahstore)
+ deallocate(muvstore,muhstore)
+ deallocate(eta_anisostore)
+ deallocate(ispec_is_tiso)
+ deallocate(c11store,c12store,c13store,c14store,c15store,c16store,c22store, &
+ c23store,c24store,c25store,c26store,c33store,c34store,c35store, &
+ c36store,c44store,c45store,c46store,c55store,c56store,c66store)
+ deallocate(iboun)
+ deallocate(ibelm_xmin,ibelm_xmax,ibelm_ymin,ibelm_ymax)
+ deallocate(ibelm_bottom,ibelm_top)
+ deallocate(jacobian2D_xmin,jacobian2D_xmax,jacobian2D_ymin,jacobian2D_ymax)
+ deallocate(jacobian2D_bottom,jacobian2D_top)
+ deallocate(normal_xmin,normal_xmax,normal_ymin,normal_ymax)
+ deallocate(normal_bottom,normal_top)
+ deallocate(iMPIcut_xi,iMPIcut_eta)
+ deallocate(nimin,nimax,njmin,njmax,nkmin_xi,nkmin_eta)
+ deallocate(rho_vp,rho_vs)
+ deallocate(Qmu_store)
+ deallocate(tau_e_store)
+ deallocate(ibelm_moho_top,ibelm_moho_bot)
+ deallocate(ibelm_400_top,ibelm_400_bot)
+ deallocate(ibelm_670_top,ibelm_670_bot)
+ deallocate(normal_moho,normal_400,normal_670)
+ deallocate(jacobian2D_moho,jacobian2D_400,jacobian2D_670)
+
+ ! user output
+ if(myrank == 0 ) write(IMAIN,*)
+
+ end subroutine create_regions_mesh
+
+!
+!-------------------------------------------------------------------------------------------------
+!
+
+ subroutine crm_allocate_arrays(iregion_code,nspec, &
+ NSPEC2DMAX_XMIN_XMAX,NSPEC2DMAX_YMIN_YMAX, &
+ NSPEC2D_BOTTOM,NSPEC2D_TOP, &
+ ipass)
+
+ use meshfem3D_par,only: &
+ myrank,NGLLX,NGLLY,NGLLZ,NDIM, &
+ IREGION_CRUST_MANTLE,IREGION_OUTER_CORE,IREGION_INNER_CORE, &
+ NCHUNKS
+
+ use meshfem3D_models_par,only: &
+ ATTENUATION,ANISOTROPIC_INNER_CORE,ANISOTROPIC_3D_MANTLE,SAVE_BOUNDARY_MESH, &
+ AM_V
+
+ use create_regions_mesh_par2
+
+ implicit none
+
+ integer,intent(in) :: iregion_code,nspec
+ integer,intent(in) :: NSPEC2DMAX_XMIN_XMAX,NSPEC2DMAX_YMIN_YMAX
+ integer,intent(in) :: NSPEC2D_BOTTOM,NSPEC2D_TOP
+ integer,intent(in) :: ipass
+
+ ! local parameters
+ integer :: ier
+
! New Attenuation definition on all GLL points
! Attenuation
if (ATTENUATION) then
@@ -314,9 +597,6 @@
ispec_is_tiso(nspec),stat=ier)
if(ier /= 0) stop 'error in allocate 7'
- ! initializes flags for transverse isotropic elements
- ispec_is_tiso(:) = .false.
-
! Stacey absorbing boundaries
if(NCHUNKS /= 6) then
nspec_stacey = nspec
@@ -442,7 +722,50 @@
jacobian2D_670(NGLLX,NGLLY,NSPEC2D_670),stat=ier)
if(ier /= 0) stop 'error in allocate 17'
- ! initialize number of layers
+ end subroutine crm_allocate_arrays
+
+!
+!-------------------------------------------------------------------------------------------------
+!
+
+ subroutine crm_setup_layers(iregion_code,nspec,ipass, &
+ xstore,ystore,zstore,ibool,idoubling, &
+ NEX_PER_PROC_ETA,is_on_a_slice_edge)
+
+ use meshfem3D_par,only: &
+ myrank,NGLLX,NGLLY,NGLLZ, &
+ IREGION_CRUST_MANTLE,IREGION_OUTER_CORE,IREGION_INNER_CORE, &
+ R670,RMOHO,R400,RMIDDLE_CRUST,MAX_NUMBER_OF_MESH_LAYERS, &
+ ner,r_top,r_bottom
+
+ use meshfem3D_models_par,only: &
+ CASE_3D,SUPPRESS_CRUSTAL_MESH,ONE_CRUST,REGIONAL_MOHO_MESH
+
+ use create_regions_mesh_par
+ use create_regions_mesh_par2
+
+ implicit none
+
+ integer,intent(in) :: iregion_code,nspec
+ integer,intent(in) :: ipass
+
+ ! arrays with the mesh in double precision
+ double precision xstore(NGLLX,NGLLY,NGLLZ,nspec)
+ double precision ystore(NGLLX,NGLLY,NGLLZ,nspec)
+ double precision zstore(NGLLX,NGLLY,NGLLZ,nspec)
+
+ integer, dimension(NGLLX,NGLLY,NGLLZ,nspec) :: ibool
+ integer, dimension(nspec) :: idoubling
+
+ ! this for non blocking MPI
+ logical, dimension(nspec) :: is_on_a_slice_edge
+
+ integer :: NEX_PER_PROC_ETA
+
+ ! local parameters
+ integer :: i,ier
+
+ ! initializes element layers
call initialize_layers(myrank,ipass,xigll,yigll,zigll,wxgll,wygll,wzgll, &
shape3D,dershape3D,shape2D_x,shape2D_y,shape2D_bottom,shape2D_top, &
dershape2D_x,dershape2D_y,dershape2D_bottom,dershape2D_top, &
@@ -498,13 +821,79 @@
! note: stretch_tab uses (dimensionalized) radii from r_top and r_bottom
!(with stretch_tab( index_radius(1=top,2=bottom), index_layer( 1=first layer, 2=second layer, 3= ...) )
RMIDDLE_CRUST = stretch_tab(2,1)
-
endif
-!----
-!---- creates mesh elements
-!----
+ end subroutine crm_setup_layers
+!
+!-------------------------------------------------------------------------------------------------
+!
+
+ subroutine crm_create_elements(iregion_code,nspec,ipass, &
+ xstore,ystore,zstore,idoubling, &
+ NEX_PER_PROC_XI,NEX_PER_PROC_ETA, &
+ is_on_a_slice_edge, &
+ offset_proc_xi,offset_proc_eta)
+
+! creates the different regions of the mesh
+
+ use meshfem3D_par,only: &
+ IMAIN,volume_total,addressing,ichunk_slice,iproc_xi_slice,iproc_eta_slice, &
+ myrank,LOCAL_PATH, &
+ IREGION_CRUST_MANTLE,IREGION_OUTER_CORE,IREGION_INNER_CORE,IFLAG_IN_FICTITIOUS_CUBE, &
+ NPROC,NPROCTOT,NPROC_XI,NPROC_ETA,NCHUNKS, &
+ SAVE_MESH_FILES,INCLUDE_CENTRAL_CUBE,ABSORBING_CONDITIONS, &
+ R_CENTRAL_CUBE,RICB,RHO_OCEANS,RCMB,R670,RMOHO,RMOHO_FICTITIOUS_IN_MESHER,&
+ RTOPDDOUBLEPRIME,R600,R220,R771,R400,R120,R80,RMIDDLE_CRUST,ROCEAN, &
+ MAX_NUMBER_OF_MESH_LAYERS,MAX_NUM_REGIONS,NB_SQUARE_EDGES_ONEDIR,NB_SQUARE_CORNERS, &
+ rmins,rmaxs,iproc_xi,iproc_eta,ichunk,NEX_XI, &
+ rotation_matrix,ANGULAR_WIDTH_XI_RAD,ANGULAR_WIDTH_ETA_RAD, &
+ ratio_sampling_array,doubling_index,this_region_has_a_doubling, &
+ ratio_divide_central_cube,CUT_SUPERBRICK_XI,CUT_SUPERBRICK_ETA, &
+ ner,r_top,r_bottom
+
+ use meshfem3D_models_par,only: &
+ ATTENUATION,ANISOTROPIC_INNER_CORE,ANISOTROPIC_3D_MANTLE, &
+ SAVE_BOUNDARY_MESH,ONE_CRUST,CASE_3D,SUPPRESS_CRUSTAL_MESH,REGIONAL_MOHO_MESH, &
+ OCEANS,TRANSVERSE_ISOTROPY,HETEROGEN_3D_MANTLE,HONOR_1D_SPHERICAL_MOHO, &
+ ELLIPTICITY,ISOTROPIC_3D_MANTLE, &
+ AM_V,nspl,rspl,espl,espl2
+
+ use create_regions_mesh_par
+ use create_regions_mesh_par2
+
+ implicit none
+
+ integer,intent(in) :: iregion_code,nspec
+ integer,intent(in) :: ipass
+
+ ! arrays with the mesh in double precision
+ double precision xstore(NGLLX,NGLLY,NGLLZ,nspec)
+ double precision ystore(NGLLX,NGLLY,NGLLZ,nspec)
+ double precision zstore(NGLLX,NGLLY,NGLLZ,nspec)
+
+ integer, dimension(nspec) :: idoubling
+
+ ! this for non blocking MPI
+ logical, dimension(nspec) :: is_on_a_slice_edge
+
+ integer :: NEX_PER_PROC_XI,NEX_PER_PROC_ETA
+
+ integer :: offset_proc_xi,offset_proc_eta
+
+ ! local parameters
+ integer :: ispec,nspec_tiso
+ ! parameters needed to store the radii of the grid points in the spherically symmetric Earth
+ double precision :: rmin,rmax
+ integer :: ner_without_doubling,ilayer,ilayer_loop
+ ! timing
+ double precision, external :: wtime
+ double precision :: time_start,tCPU
+ integer,dimension(8) :: tval
+
+ ! initializes flags for transverse isotropic elements
+ ispec_is_tiso(:) = .false.
+
! get MPI starting time
time_start = wtime()
@@ -519,7 +908,7 @@
write(IMAIN,*) ' creating layer ',ilayer_loop-ifirst_region+1, &
'out of ',ilast_region-ifirst_region+1
endif
-
+
! determine the radii that define the shell
rmin = rmins(ilayer)
rmax = rmaxs(ilayer)
@@ -616,14 +1005,17 @@
tCPU = wtime() - time_start
! remaining
tCPU = (1.0-(ilayer_loop-ifirst_region+1.0)/(ilast_region-ifirst_region+1.0)) &
- /(ilayer_loop-ifirst_region+1.0)/(ilast_region-ifirst_region+1.0)*tCPU
+ /(ilayer_loop-ifirst_region+1.0)/(ilast_region-ifirst_region+1.0)*tCPU*10.0
write(IMAIN,*) " ",(ilayer_loop-ifirst_region+1.0)/(ilast_region-ifirst_region+1.0) * 100.0," %", &
" time remaining:", tCPU,"s"
+ call date_and_time(VALUES=tval)
+ write(IMAIN,*) " ",tval(5),"h",tval(6),"min",tval(7),".",tval(8),"sec"
endif
enddo !ilayer_loop
+
if(myrank == 0 ) write(IMAIN,*)
-
+
! define central cube in inner core
if(INCLUDE_CENTRAL_CUBE .and. iregion_code == IREGION_INNER_CORE) then
! user output
@@ -665,348 +1057,283 @@
! for which communications cannot be overlapped with calculations
where(idoubling == IFLAG_IN_FICTITIOUS_CUBE) is_on_a_slice_edge = .false.
- ! only create global addressing and the MPI buffers in the first pass
- select case(ipass)
- case( 1 )
- ! user output
- if(myrank == 0 ) write(IMAIN,*) ' creating global addressing'
+ ! checks transverse isotropic elements
+ if( ipass == 2 ) then
+ ! count number of anisotropic elements in current region
+ ! should be zero in all the regions except in the mantle
+ nspec_tiso = count(ispec_is_tiso(:))
- ! allocate memory for arrays
- allocate(locval(npointot), &
- ifseg(npointot), &
- xp(npointot), &
- yp(npointot), &
- zp(npointot),stat=ier)
- if(ier /= 0) stop 'error in allocate 20'
+ ! checks number of anisotropic elements found in the mantle
+ if(iregion_code /= IREGION_CRUST_MANTLE .and. nspec_tiso /= 0 ) &
+ call exit_MPI(myrank,'found anisotropic elements outside of the mantle')
+ if( TRANSVERSE_ISOTROPY ) then
+ if(iregion_code == IREGION_CRUST_MANTLE .and. nspec_tiso == 0) &
+ call exit_MPI(myrank,'found no anisotropic elements in the mantle')
+ endif
+ endif
- locval = 0
- ifseg = .false.
- xp = 0.d0
- yp = 0.d0
- zp = 0.d0
+ end subroutine crm_create_elements
- ! we need to create a copy of the x, y and z arrays because sorting in get_global will swap
- ! these arrays and therefore destroy them
- do ispec=1,nspec
- ieoff = NGLLX * NGLLY * NGLLZ * (ispec-1)
- ilocnum = 0
- do k=1,NGLLZ
- do j=1,NGLLY
- do i=1,NGLLX
- ilocnum = ilocnum + 1
- xp(ilocnum+ieoff) = xstore(i,j,k,ispec)
- yp(ilocnum+ieoff) = ystore(i,j,k,ispec)
- zp(ilocnum+ieoff) = zstore(i,j,k,ispec)
- enddo
+!
+!-------------------------------------------------------------------------------------------------
+!
+
+ subroutine crm_setup_indexing(ibool,xstore,ystore,zstore, &
+ nspec,nglob_theor,npointot)
+
+! creates global indexing array ibool
+
+ use meshfem3d_par,only: &
+ myrank,NGLLX,NGLLY,NGLLZ
+
+ use create_regions_mesh_par2
+ implicit none
+
+ ! number of spectral elements in each block
+ integer,intent(in) :: nspec,npointot,nglob_theor
+
+ ! arrays with the mesh
+ integer, dimension(NGLLX,NGLLY,NGLLZ,nspec) :: ibool
+ double precision, dimension(NGLLX,NGLLY,NGLLZ,nspec) :: xstore,ystore,zstore
+
+ ! local parameters
+ ! variables for creating array ibool
+ double precision, dimension(:), allocatable :: xp,yp,zp
+ integer, dimension(:), allocatable :: locval
+ logical, dimension(:), allocatable :: ifseg
+
+ integer :: nglob
+ integer :: ieoff,ilocnum,ier
+ integer :: i,j,k,ispec
+ character(len=150) :: errmsg
+
+ ! allocate memory for arrays
+ allocate(locval(npointot), &
+ ifseg(npointot), &
+ xp(npointot), &
+ yp(npointot), &
+ zp(npointot),stat=ier)
+ if(ier /= 0) stop 'error in allocate 20'
+
+ locval = 0
+ ifseg = .false.
+ xp = 0.d0
+ yp = 0.d0
+ zp = 0.d0
+
+ ! we need to create a copy of the x, y and z arrays because sorting in get_global will swap
+ ! these arrays and therefore destroy them
+ do ispec=1,nspec
+ ieoff = NGLLX * NGLLY * NGLLZ * (ispec-1)
+ ilocnum = 0
+ do k=1,NGLLZ
+ do j=1,NGLLY
+ do i=1,NGLLX
+ ilocnum = ilocnum + 1
+ xp(ilocnum+ieoff) = xstore(i,j,k,ispec)
+ yp(ilocnum+ieoff) = ystore(i,j,k,ispec)
+ zp(ilocnum+ieoff) = zstore(i,j,k,ispec)
enddo
enddo
enddo
+ enddo
- call get_global(nspec,xp,yp,zp,ibool,locval,ifseg,nglob,npointot)
+ call get_global(nspec,xp,yp,zp,ibool,locval,ifseg,nglob,npointot)
- deallocate(xp,yp,zp)
- deallocate(locval,ifseg)
+ deallocate(xp,yp,zp)
+ deallocate(locval,ifseg)
- ! check that number of points found equals theoretical value
- if(nglob /= nglob_theor) then
- write(errmsg,*) 'incorrect total number of points found: myrank,nglob,nglob_theor,ipass,iregion_code = ',&
- myrank,nglob,nglob_theor,ipass,iregion_code
- call exit_MPI(myrank,errmsg)
- endif
- if(minval(ibool) /= 1 .or. maxval(ibool) /= nglob_theor) call exit_MPI(myrank,'incorrect global numbering')
+ ! check that number of points found equals theoretical value
+ if(nglob /= nglob_theor) then
+ write(errmsg,*) 'incorrect total number of points found: myrank,nglob,nglob_theor = ',&
+ myrank,nglob,nglob_theor
+ call exit_MPI(myrank,errmsg)
+ endif
+ if(minval(ibool) /= 1 .or. maxval(ibool) /= nglob_theor) call exit_MPI(myrank,'incorrect global numbering')
- ! creates a new indirect addressing to reduce cache misses in memory access in the solver
- ! this is *critical* to improve performance in the solver
- call get_global_indirect_addressing(nspec,nglob,ibool)
+ ! creates a new indirect addressing to reduce cache misses in memory access in the solver
+ ! this is *critical* to improve performance in the solver
+ call get_global_indirect_addressing(nspec,nglob_theor,ibool)
- ! checks again
- if(minval(ibool) /= 1 .or. maxval(ibool) /= nglob_theor) call exit_MPI(myrank,'incorrect global numbering after sorting')
+ ! checks again
+ if(minval(ibool) /= 1 .or. maxval(ibool) /= nglob_theor) call exit_MPI(myrank,'incorrect global numbering after sorting')
- ! create MPI buffers
- ! arrays mask_ibool(npointot) used to save memory
- ! allocate memory for arrays
- allocate(mask_ibool(npointot), &
- stat=ier)
- if(ier /= 0) stop 'error in allocate 20b'
-
- call get_MPI_cutplanes_xi(myrank,prname,nspec,iMPIcut_xi,ibool, &
- xstore,ystore,zstore,mask_ibool,npointot, &
- NSPEC2D_ETA_FACE,iregion_code,npoin2D_xi)
+ end subroutine crm_setup_indexing
- call get_MPI_cutplanes_eta(myrank,prname,nspec,iMPIcut_eta,ibool, &
- xstore,ystore,zstore,mask_ibool,npointot, &
- NSPEC2D_XI_FACE,iregion_code,npoin2D_eta)
+!
+!-------------------------------------------------------------------------------------------------
+!
- call get_MPI_1D_buffers(myrank,prname,nspec,iMPIcut_xi,iMPIcut_eta,ibool,idoubling, &
- xstore,ystore,zstore,mask_ibool,npointot, &
- NSPEC1D_RADIAL_CORNER,NGLOB1D_RADIAL_CORNER,iregion_code)
+ subroutine crm_setup_mpi_buffers(npointot,nspec,ibool,idoubling, &
+ xstore,ystore,zstore,iregion_code, &
+ npoin2D_xi,npoin2D_eta)
- deallocate(mask_ibool)
+! creates global indexing array ibool
- ! Stacey
- if(NCHUNKS /= 6) &
- call get_absorb(myrank,prname,iboun,nspec,nimin,nimax,njmin,njmax,nkmin_xi,nkmin_eta, &
- NSPEC2DMAX_XMIN_XMAX,NSPEC2DMAX_YMIN_YMAX,NSPEC2D_BOTTOM)
-
- ! only create mass matrix and save all the final arrays in the second pass
- case( 2 )
- ! user output
- if(myrank == 0 ) write(IMAIN,*) ' creating mass matrix'
+ use meshfem3d_par,only: &
+ myrank,NGLLX,NGLLY,NGLLZ, &
+ NSPEC1D_RADIAL_CORNER,NGLOB1D_RADIAL_CORNER, &
+ NSPEC2D_XI_FACE,NSPEC2D_ETA_FACE
- ! copy the theoretical number of points for the second pass
- nglob = nglob_theor
+ use create_regions_mesh_par2
+ implicit none
- ! count number of anisotropic elements in current region
- ! should be zero in all the regions except in the mantle
- ! (used only for checks in meshfem3D() routine)
- !nspec_tiso = count(idoubling(1:nspec) == IFLAG_220_80) + count(idoubling(1:nspec) == IFLAG_80_MOHO)
- nspec_tiso = count(ispec_is_tiso(:))
+ ! number of spectral elements in each block
+ integer,intent(in) :: nspec,npointot
- ! daniel: deprecated - mesh sorting and coloring
- !call setup_color_perm(myrank,iregion_code,nspec,nglob, &
- ! ibool,is_on_a_slice_edge,prname, &
- ! npoin2D_xi,npoin2D_eta)
+ ! arrays with the mesh
+ integer, dimension(NGLLX,NGLLY,NGLLZ,nspec) :: ibool
+ integer, dimension(nspec) :: idoubling
- ! precomputes jacobian for 2d absorbing boundary surfaces
- call get_jacobian_boundaries(myrank,iboun,nspec,xstore,ystore,zstore, &
- dershape2D_x,dershape2D_y,dershape2D_bottom,dershape2D_top, &
- ibelm_xmin,ibelm_xmax,ibelm_ymin,ibelm_ymax,ibelm_bottom,ibelm_top, &
- nspec2D_xmin,nspec2D_xmax,nspec2D_ymin,nspec2D_ymax, &
- jacobian2D_xmin,jacobian2D_xmax, &
- jacobian2D_ymin,jacobian2D_ymax, &
- jacobian2D_bottom,jacobian2D_top, &
- normal_xmin,normal_xmax, &
- normal_ymin,normal_ymax, &
- normal_bottom,normal_top, &
- NSPEC2D_BOTTOM,NSPEC2D_TOP, &
- NSPEC2DMAX_XMIN_XMAX,NSPEC2DMAX_YMIN_YMAX,&
- xigll,yigll,zigll)
+ double precision, dimension(NGLLX,NGLLY,NGLLZ,nspec) :: xstore,ystore,zstore
- !uncomment: adds model smoothing for point profile models
- ! if( THREE_D_MODEL == THREE_D_MODEL_PPM ) then
- ! call smooth_model(myrank, nproc_xi,nproc_eta,&
- ! rho_vp,rho_vs,nspec_stacey, &
- ! iregion_code,xixstore,xiystore,xizstore, &
- ! etaxstore,etaystore,etazstore, &
- ! gammaxstore,gammaystore,gammazstore, &
- ! xstore,ystore,zstore,rhostore,dvpstore, &
- ! kappavstore,kappahstore,muvstore,muhstore,eta_anisostore,&
- ! nspec,HETEROGEN_3D_MANTLE, &
- ! NEX_XI,NCHUNKS,ABSORBING_CONDITIONS,PPM_V )
+ integer,intent(in) :: iregion_code
+ integer :: npoin2D_xi,npoin2D_eta
- ! allocates mass matrices in this slice (will be fully assembled in the solver)
- !
- ! in the case of stacey boundary conditions, add C*deltat/2 contribution to the mass matrix
- ! on Stacey edges for the crust_mantle and outer_core regions but not for the inner_core region
- ! thus the mass matrix must be replaced by three mass matrices including the "C" damping matrix
- !
- ! if absorbing_conditions are not set or if NCHUNKS=6, only one mass matrix is needed
- ! for the sake of performance, only "rmassz" array will be filled and "rmassx" & "rmassy" will be obsolete
-
- if(NCHUNKS /= 6 .and. ABSORBING_CONDITIONS) then
- select case(iregion_code)
- case( IREGION_CRUST_MANTLE )
- nglob_xy = nglob
- case( IREGION_INNER_CORE, IREGION_OUTER_CORE )
- nglob_xy = 1
- endselect
- else
- nglob_xy = 1
- endif
+ ! local parameters
+ logical, dimension(:), allocatable :: mask_ibool
+ integer :: ier
- allocate(rmassx(nglob_xy),stat=ier)
- if(ier /= 0) stop 'error in allocate 21'
- allocate(rmassy(nglob_xy),stat=ier)
- if(ier /= 0) stop 'error in allocate 21'
- allocate(rmassz(nglob),stat=ier)
- if(ier /= 0) stop 'error in allocate 21'
+ ! arrays mask_ibool(npointot) used to save memory
+ ! allocate memory for arrays
+ allocate(mask_ibool(npointot), &
+ stat=ier)
+ if(ier /= 0) stop 'error in allocate 20b'
- ! allocates ocean load mass matrix as well if oceans
- if(OCEANS .and. iregion_code == IREGION_CRUST_MANTLE) then
- nglob_oceans = nglob
- else
- ! allocate dummy array if no oceans
- nglob_oceans = 1
- endif
- allocate(rmass_ocean_load(nglob_oceans),stat=ier)
- if(ier /= 0) stop 'error in allocate 22'
+ call get_MPI_cutplanes_xi(myrank,prname,nspec,iMPIcut_xi,ibool, &
+ xstore,ystore,zstore,mask_ibool,npointot, &
+ NSPEC2D_ETA_FACE,iregion_code,npoin2D_xi)
- ! creating mass matrices in this slice (will be fully assembled in the solver)
- call create_mass_matrices(myrank,nspec,idoubling,wxgll,wygll,wzgll,ibool, &
- nspec_actually,xixstore,xiystore,xizstore, &
- etaxstore,etaystore,etazstore, &
- gammaxstore,gammaystore,gammazstore, &
- iregion_code,rhostore,kappavstore, &
- nglob_xy,nglob,prname, &
- rmassx,rmassy,rmassz, &
- nglob_oceans,rmass_ocean_load, &
- xstore,ystore,zstore,RHO_OCEANS, &
- NSPEC2D_TOP,NSPEC2D_BOTTOM,NSPEC2DMAX_XMIN_XMAX,NSPEC2DMAX_YMIN_YMAX, &
- ibelm_xmin,ibelm_xmax,ibelm_ymin,ibelm_ymax,ibelm_bottom,ibelm_top, &
- nspec2D_xmin,nspec2D_xmax,nspec2D_ymin,nspec2D_ymax, &
- normal_xmin,normal_xmax,normal_ymin,normal_ymax, &
- rho_vp,rho_vs,nspec_stacey, &
- jacobian2D_xmin,jacobian2D_xmax,jacobian2D_ymin,jacobian2D_ymax, &
- jacobian2D_bottom,jacobian2D_top)
+ call get_MPI_cutplanes_eta(myrank,prname,nspec,iMPIcut_eta,ibool, &
+ xstore,ystore,zstore,mask_ibool,npointot, &
+ NSPEC2D_XI_FACE,iregion_code,npoin2D_eta)
- ! user output
- if(myrank == 0 ) write(IMAIN,*) ' saving binary files'
+ call get_MPI_1D_buffers(myrank,prname,nspec,iMPIcut_xi,iMPIcut_eta,ibool,idoubling, &
+ xstore,ystore,zstore,mask_ibool,npointot, &
+ NSPEC1D_RADIAL_CORNER,NGLOB1D_RADIAL_CORNER,iregion_code)
- ! save the binary files
- call save_arrays_solver(myrank,rho_vp,rho_vs,nspec_stacey, &
- prname,iregion_code,xixstore,xiystore,xizstore, &
- etaxstore,etaystore,etazstore,gammaxstore,gammaystore,gammazstore, &
- xstore,ystore,zstore,rhostore,dvpstore, &
- kappavstore,kappahstore,muvstore,muhstore,eta_anisostore, &
- nspec_ani,c11store,c12store,c13store,c14store,c15store,c16store,c22store, &
- c23store,c24store,c25store,c26store,c33store,c34store,c35store, &
- c36store,c44store,c45store,c46store,c55store,c56store,c66store, &
- ibool,idoubling,is_on_a_slice_edge,nglob_xy,nglob, &
- rmassx,rmassy,rmassz,rmass_ocean_load,nglob_oceans, &
- ibelm_xmin,ibelm_xmax,ibelm_ymin,ibelm_ymax,ibelm_bottom,ibelm_top, &
- nspec2D_xmin,nspec2D_xmax,nspec2D_ymin,nspec2D_ymax, &
- normal_xmin,normal_xmax,normal_ymin,normal_ymax,normal_bottom,normal_top, &
- jacobian2D_xmin,jacobian2D_xmax,jacobian2D_ymin,jacobian2D_ymax, &
- jacobian2D_bottom,jacobian2D_top,nspec, &
- NSPEC2DMAX_XMIN_XMAX,NSPEC2DMAX_YMIN_YMAX,NSPEC2D_BOTTOM,NSPEC2D_TOP, &
- TRANSVERSE_ISOTROPY,HETEROGEN_3D_MANTLE,ANISOTROPIC_3D_MANTLE, &
- ANISOTROPIC_INNER_CORE,OCEANS, &
- tau_s,tau_e_store,Qmu_store,T_c_source,ATTENUATION, &
- size(tau_e_store,2),size(tau_e_store,3),size(tau_e_store,4),size(tau_e_store,5),&
- ABSORBING_CONDITIONS,SAVE_MESH_FILES,ispec_is_tiso)
+ deallocate(mask_ibool)
- deallocate(rmassx,rmassy,rmassz)
- deallocate(rmass_ocean_load)
+ end subroutine crm_setup_mpi_buffers
- ! compute volume, bottom and top area of that part of the slice
- call compute_volumes(volume_local,area_local_bottom,area_local_top, &
- nspec,wxgll,wygll,wzgll,xixstore,xiystore,xizstore, &
- etaxstore,etaystore,etazstore,gammaxstore,gammaystore,gammazstore, &
- NSPEC2D_BOTTOM,jacobian2D_bottom,NSPEC2D_TOP,jacobian2D_top)
+!
+!-------------------------------------------------------------------------------------------------
+!
- ! boundary mesh
- if (SAVE_BOUNDARY_MESH .and. iregion_code == IREGION_CRUST_MANTLE) then
- ! user output
- if(myrank == 0 ) write(IMAIN,*) ' saving boundary mesh files'
+ subroutine crm_save_boundary_mesh()
- ! first check the number of surface elements are the same for Moho, 400, 670
- if (.not. SUPPRESS_CRUSTAL_MESH .and. HONOR_1D_SPHERICAL_MOHO) then
- if (ispec2D_moho_top /= NSPEC2D_MOHO .or. ispec2D_moho_bot /= NSPEC2D_MOHO) &
- call exit_mpi(myrank, 'Not the same number of Moho surface elements')
- endif
- if (ispec2D_400_top /= NSPEC2D_400 .or. ispec2D_400_bot /= NSPEC2D_400) &
- call exit_mpi(myrank,'Not the same number of 400 surface elements')
- if (ispec2D_670_top /= NSPEC2D_670 .or. ispec2D_670_bot /= NSPEC2D_670) &
- call exit_mpi(myrank,'Not the same number of 670 surface elements')
+! creates global indexing array ibool
- ! writing surface topology databases
- open(unit=27,file=prname(1:len_trim(prname))//'boundary_disc.bin', &
- status='unknown',form='unformatted',iostat=ier)
- if( ier /= 0 ) call exit_mpi(myrank,'error opening boundary_disc.bin file')
+ use meshfem3d_par,only: &
+ myrank
- write(27) NSPEC2D_MOHO, NSPEC2D_400, NSPEC2D_670
- write(27) ibelm_moho_top
- write(27) ibelm_moho_bot
- write(27) ibelm_400_top
- write(27) ibelm_400_bot
- write(27) ibelm_670_top
- write(27) ibelm_670_bot
- write(27) normal_moho
- write(27) normal_400
- write(27) normal_670
- close(27)
- endif
+ use meshfem3D_models_par,only: &
+ SAVE_BOUNDARY_MESH,HONOR_1D_SPHERICAL_MOHO,SUPPRESS_CRUSTAL_MESH
- ! create AVS or DX mesh data for the slices
- if(SAVE_MESH_FILES) then
- ! user output
- if(myrank == 0 ) write(IMAIN,*) ' saving AVS mesh files'
-
- ! arrays num_ibool_AVS_DX and mask_ibool used to save memory
- ! allocate memory for arrays
- allocate(num_ibool_AVS_DX(npointot), &
- mask_ibool(npointot), &
- stat=ier)
- if(ier /= 0) stop 'error in allocate 21'
-
- call write_AVS_DX_global_data(myrank,prname,nspec,ibool,idoubling,xstore,ystore,zstore, &
- num_ibool_AVS_DX,mask_ibool,npointot)
+ use create_regions_mesh_par2
+ implicit none
- call write_AVS_DX_global_faces_data(myrank,prname,nspec,iMPIcut_xi,iMPIcut_eta,ibool, &
- idoubling,xstore,ystore,zstore,num_ibool_AVS_DX,mask_ibool,npointot, &
- rhostore,kappavstore,muvstore,nspl,rspl,espl,espl2, &
- ELLIPTICITY,ISOTROPIC_3D_MANTLE, &
- RICB,RCMB,RTOPDDOUBLEPRIME,R600,R670,R220,R771,R400,R120,R80,RMOHO, &
- RMIDDLE_CRUST,ROCEAN,iregion_code)
+ ! local parameters
+ integer :: ier
- call write_AVS_DX_global_chunks_data(myrank,prname,nspec,iboun,ibool, &
- idoubling,xstore,ystore,zstore,num_ibool_AVS_DX,mask_ibool,npointot, &
- rhostore,kappavstore,muvstore,nspl,rspl,espl,espl2, &
- ELLIPTICITY,ISOTROPIC_3D_MANTLE, &
- RICB,RCMB,RTOPDDOUBLEPRIME,R600,R670,R220,R771,R400,R120,R80,RMOHO, &
- RMIDDLE_CRUST,ROCEAN,iregion_code)
+ ! first check the number of surface elements are the same for Moho, 400, 670
+ if (.not. SUPPRESS_CRUSTAL_MESH .and. HONOR_1D_SPHERICAL_MOHO) then
+ if (ispec2D_moho_top /= NSPEC2D_MOHO .or. ispec2D_moho_bot /= NSPEC2D_MOHO) &
+ call exit_mpi(myrank, 'Not the same number of Moho surface elements')
+ endif
+ if (ispec2D_400_top /= NSPEC2D_400 .or. ispec2D_400_bot /= NSPEC2D_400) &
+ call exit_mpi(myrank,'Not the same number of 400 surface elements')
+ if (ispec2D_670_top /= NSPEC2D_670 .or. ispec2D_670_bot /= NSPEC2D_670) &
+ call exit_mpi(myrank,'Not the same number of 670 surface elements')
- call write_AVS_DX_surface_data(myrank,prname,nspec,iboun,ibool, &
- idoubling,xstore,ystore,zstore,num_ibool_AVS_DX,mask_ibool,npointot, &
- rhostore,kappavstore,muvstore,nspl,rspl,espl,espl2, &
- ELLIPTICITY,ISOTROPIC_3D_MANTLE, &
- RICB,RCMB,RTOPDDOUBLEPRIME,R600,R670,R220,R771,R400,R120,R80,RMOHO, &
- RMIDDLE_CRUST,ROCEAN,iregion_code)
-
- !> Hejun
- ! Output material information for all GLL points
- ! Can be use to check the mesh
- ! call write_AVS_DX_global_data_gll(prname,nspec,xstore,ystore,zstore,&
- ! rhostore,kappavstore,muvstore,Qmu_store,ATTENUATION)
+ ! writing surface topology databases
+ open(unit=27,file=prname(1:len_trim(prname))//'boundary_disc.bin', &
+ status='unknown',form='unformatted',iostat=ier)
+ if( ier /= 0 ) call exit_mpi(myrank,'error opening boundary_disc.bin file')
- deallocate(num_ibool_AVS_DX,mask_ibool)
- endif
+ write(27) NSPEC2D_MOHO, NSPEC2D_400, NSPEC2D_670
+ write(27) ibelm_moho_top
+ write(27) ibelm_moho_bot
+ write(27) ibelm_400_top
+ write(27) ibelm_400_bot
+ write(27) ibelm_670_top
+ write(27) ibelm_670_bot
+ write(27) normal_moho
+ write(27) normal_400
+ write(27) normal_670
+ close(27)
- case default
- stop 'there cannot be more than two passes in mesh creation'
+ end subroutine crm_save_boundary_mesh
- end select ! end of test if first or second pass
+!
+!-------------------------------------------------------------------------------------------------
+!
- deallocate(stretch_tab)
- deallocate(perm_layer)
+ subroutine crm_save_mesh_files(nspec,npointot,iregion_code,ibool,idoubling, &
+ xstore,ystore,zstore)
- ! deallocate these arrays after each pass
- ! because they have a different size in each pass to save memory
- deallocate(xixstore,xiystore,xizstore)
- deallocate(etaxstore,etaystore,etazstore)
- deallocate(gammaxstore,gammaystore,gammazstore)
+ use meshfem3d_par,only: &
+ myrank,NGLLX,NGLLY,NGLLZ, &
+ RICB,RCMB,RTOPDDOUBLEPRIME,R600,R670,R220,R771,R400,R120,R80,RMOHO, &
+ RMIDDLE_CRUST,ROCEAN
- ! deallocate arrays
- deallocate(rhostore,dvpstore,kappavstore,kappahstore)
- deallocate(muvstore,muhstore)
- deallocate(eta_anisostore)
- deallocate(ispec_is_tiso)
- deallocate(c11store,c12store,c13store,c14store,c15store,c16store,c22store, &
- c23store,c24store,c25store,c26store,c33store,c34store,c35store, &
- c36store,c44store,c45store,c46store,c55store,c56store,c66store)
- deallocate(iboun)
- deallocate(ibelm_xmin,ibelm_xmax,ibelm_ymin,ibelm_ymax)
- deallocate(ibelm_bottom,ibelm_top)
- deallocate(jacobian2D_xmin,jacobian2D_xmax,jacobian2D_ymin,jacobian2D_ymax)
- deallocate(jacobian2D_bottom,jacobian2D_top)
- deallocate(normal_xmin,normal_xmax,normal_ymin,normal_ymax)
- deallocate(normal_bottom,normal_top)
- deallocate(iMPIcut_xi,iMPIcut_eta)
- deallocate(nimin,nimax,njmin,njmax,nkmin_xi,nkmin_eta)
- deallocate(rho_vp,rho_vs)
- deallocate(Qmu_store)
- deallocate(tau_e_store)
- deallocate(ibelm_moho_top,ibelm_moho_bot)
- deallocate(ibelm_400_top,ibelm_400_bot)
- deallocate(ibelm_670_top,ibelm_670_bot)
- deallocate(normal_moho,normal_400,normal_670)
- deallocate(jacobian2D_moho,jacobian2D_400,jacobian2D_670)
+ use meshfem3D_models_par,only: &
+ ELLIPTICITY,ISOTROPIC_3D_MANTLE, &
+ nspl,rspl,espl,espl2
- ! user output
- if(myrank == 0 ) write(IMAIN,*)
+ use create_regions_mesh_par2
+ implicit none
- ! synchronizes processes
- !call sync_all()
+ ! number of spectral elements in each block
+ integer,intent(in) :: nspec,npointot,iregion_code
- end subroutine create_regions_mesh
+ ! arrays with the mesh
+ integer, dimension(NGLLX,NGLLY,NGLLZ,nspec) :: ibool
+ integer, dimension(nspec) :: idoubling
+ double precision, dimension(NGLLX,NGLLY,NGLLZ,nspec) :: xstore,ystore,zstore
+
+ ! local parameters
+ ! arrays used for AVS or DX files
+ integer, dimension(:), allocatable :: num_ibool_AVS_DX
+ logical, dimension(:), allocatable :: mask_ibool
+ integer :: ier
+
+ ! arrays num_ibool_AVS_DX and mask_ibool used to save memory
+ ! allocate memory for arrays
+ allocate(num_ibool_AVS_DX(npointot), &
+ mask_ibool(npointot), &
+ stat=ier)
+ if(ier /= 0) stop 'error in allocate 21'
+
+ call write_AVS_DX_global_data(myrank,prname,nspec,ibool,idoubling,xstore,ystore,zstore, &
+ num_ibool_AVS_DX,mask_ibool,npointot)
+
+ call write_AVS_DX_global_faces_data(myrank,prname,nspec,iMPIcut_xi,iMPIcut_eta,ibool, &
+ idoubling,xstore,ystore,zstore,num_ibool_AVS_DX,mask_ibool,npointot, &
+ rhostore,kappavstore,muvstore,nspl,rspl,espl,espl2, &
+ ELLIPTICITY,ISOTROPIC_3D_MANTLE, &
+ RICB,RCMB,RTOPDDOUBLEPRIME,R600,R670,R220,R771,R400,R120,R80,RMOHO, &
+ RMIDDLE_CRUST,ROCEAN,iregion_code)
+
+ call write_AVS_DX_global_chunks_data(myrank,prname,nspec,iboun,ibool, &
+ idoubling,xstore,ystore,zstore,num_ibool_AVS_DX,mask_ibool,npointot, &
+ rhostore,kappavstore,muvstore,nspl,rspl,espl,espl2, &
+ ELLIPTICITY,ISOTROPIC_3D_MANTLE, &
+ RICB,RCMB,RTOPDDOUBLEPRIME,R600,R670,R220,R771,R400,R120,R80,RMOHO, &
+ RMIDDLE_CRUST,ROCEAN,iregion_code)
+
+ call write_AVS_DX_surface_data(myrank,prname,nspec,iboun,ibool, &
+ idoubling,xstore,ystore,zstore,num_ibool_AVS_DX,mask_ibool,npointot, &
+ rhostore,kappavstore,muvstore,nspl,rspl,espl,espl2, &
+ ELLIPTICITY,ISOTROPIC_3D_MANTLE, &
+ RICB,RCMB,RTOPDDOUBLEPRIME,R600,R670,R220,R771,R400,R120,R80,RMOHO, &
+ RMIDDLE_CRUST,ROCEAN,iregion_code)
+
+ ! Output material information for all GLL points
+ ! Can be use to check the mesh
+ ! call write_AVS_DX_global_data_gll(prname,nspec,xstore,ystore,zstore,&
+ ! rhostore,kappavstore,muvstore,Qmu_store,ATTENUATION)
+ deallocate(num_ibool_AVS_DX,mask_ibool)
+
+ end subroutine crm_save_mesh_files
Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/meshfem3D/finalize_mesher.f90
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/meshfem3D/finalize_mesher.f90 2012-07-19 21:55:39 UTC (rev 20529)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/meshfem3D/finalize_mesher.f90 2012-07-19 22:05:23 UTC (rev 20530)
@@ -29,12 +29,14 @@
subroutine finalize_mesher()
use meshfem3D_par
+ use meshfem3D_models_par
+
implicit none
! local parameters
! timing
double precision, external :: wtime
-
+
!--- print number of points and elements in the mesh for each region
if(myrank == 0) then
Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/meshfem3D/get_global.f90
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/meshfem3D/get_global.f90 2012-07-19 21:55:39 UTC (rev 20529)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/meshfem3D/get_global.f90 2012-07-19 22:05:23 UTC (rev 20530)
@@ -149,7 +149,7 @@
include "constants.h"
- integer :: nspec,nglob
+ integer,intent(in) :: nspec,nglob
integer, dimension(NGLLX,NGLLY,NGLLZ,nspec) :: ibool
! mask to sort ibool
Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/meshfem3D/get_perm_color.f90
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/meshfem3D/get_perm_color.f90 2012-07-19 21:55:39 UTC (rev 20529)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/meshfem3D/get_perm_color.f90 2012-07-19 22:05:23 UTC (rev 20530)
@@ -1,160 +1,382 @@
+!=====================================================================
+!
+! S p e c f e m 3 D G l o b e V e r s i o n 5 . 1
+! --------------------------------------------------
+!
+! Main authors: Dimitri Komatitsch and Jeroen Tromp
+! Princeton University, USA
+! and University of Pau / CNRS / INRIA, France
+! (c) Princeton University / California Institute of Technology and University of Pau / CNRS / INRIA
+! April 2011
+!
+! This program is free software; you can redistribute it and/or modify
+! it under the terms of the GNU General Public License as published by
+! the Free Software Foundation; either version 2 of the License, or
+! (at your option) any later version.
+!
+! This program is distributed in the hope that it will be useful,
+! but WITHOUT ANY WARRANTY; without even the implied warranty of
+! MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+! GNU General Public License for more details.
+!
+! You should have received a copy of the GNU General Public License along
+! with this program; if not, write to the Free Software Foundation, Inc.,
+! 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+!
+!=====================================================================
+
! define sets of colors that contain disconnected elements for the CUDA solver.
! also split the elements into two subsets: inner and outer elements, in order
! to be able to compute the outer elements first in the solver and then
! start non-blocking MPI calls and overlap them with the calculation of the inner elements
! (which works fine because there are always far more inner elements than outer elements)
-!*********************************************************************************************************
-! Mila
-subroutine get_perm_color_faster(is_on_a_slice_edge,ibool,perm,nspec,nglob, &
- nb_colors_outer_elements,nb_colors_inner_elements,nspec_outer,first_elem_number_in_this_color,myrank)
+! note: these are modified routines to use element domain flags given in ispec_is_d, thus
+! coloring only acoustic or elastic (or..) elements in one run, then repeat run for other domains.
+! also, the permutation re-starts at 1 for outer and for inner elements,
+! making it usable for the phase_ispec_inner_** arrays for acoustic and elastic elements.
+ subroutine get_perm_color_faster(is_on_a_slice_edge,ispec_is_d, &
+ ibool,perm,color, &
+ nspec,nglob, &
+ nb_colors_outer_elements,nb_colors_inner_elements, &
+ nspec_outer,nspec_inner,nspec_domain, &
+ first_elem_number_in_this_color, &
+ myrank)
+
implicit none
include "constants.h"
-! local variables
- integer nspec, nglob
+ integer, intent(in) :: nspec, nglob
+ logical, dimension(nspec), intent(in) :: is_on_a_slice_edge
+ logical, dimension(nspec), intent(in) :: ispec_is_d
- logical, dimension(nspec) :: is_on_a_slice_edge
+ integer, dimension(NGLLX,NGLLY,NGLLZ,nspec), intent(in) :: ibool
+ integer, dimension(nspec),intent(inout) :: perm
- integer, dimension(NGLLX,NGLLY,NGLLZ,nspec) :: ibool
- integer, dimension(nspec) :: perm
- integer, dimension(nspec) :: color
- integer, dimension(MAX_NUMBER_OF_COLORS) :: first_elem_number_in_this_color
- integer :: nb_colors_outer_elements,nb_colors_inner_elements,nspec_outer,myrank
+ integer, dimension(nspec),intent(inout) :: color
+ integer, dimension(MAX_NUMBER_OF_COLORS+1),intent(inout) :: first_elem_number_in_this_color
+ integer, intent(out) :: nb_colors_outer_elements,nb_colors_inner_elements
- call get_color_faster(ibool, is_on_a_slice_edge, myrank, nspec, nglob, &
- color, nb_colors_outer_elements, nb_colors_inner_elements, nspec_outer)
+ integer, intent(out) :: nspec_outer,nspec_inner,nspec_domain
+ integer, intent(in) :: myrank
- if(myrank == 0) then
- write(IMAIN,*) 'number of colors of the graph for inner elements = ',nb_colors_inner_elements
- write(IMAIN,*) 'number of colors of the graph for outer elements = ',nb_colors_outer_elements
- write(IMAIN,*) 'total number of colors of the graph (sum of both) = ', nb_colors_inner_elements + nb_colors_outer_elements
- write(IMAIN,*) 'number of elements of the graph for outer elements = ',nspec_outer
- endif
+ ! local variables
+ integer :: nb_colors
- if(myrank == 0) write(IMAIN,*) 'generating the final colors'
- first_elem_number_in_this_color(:) = -1
- call get_final_perm(color,perm,first_elem_number_in_this_color,nspec,nb_colors_inner_elements+nb_colors_outer_elements)
+ ! coloring algorithm w/ Droux
+ call get_color_faster(ibool, is_on_a_slice_edge, ispec_is_d, &
+ myrank, nspec, nglob, &
+ color, nb_colors_outer_elements, nb_colors_inner_elements, &
+ nspec_outer,nspec_inner,nspec_domain)
- if(myrank == 0) write(IMAIN,*) 'done with mesh coloring and inner/outer element splitting'
+ !debug output
+ if(myrank == 0) then
+ write(IMAIN,*) ' colors:'
+ write(IMAIN,*) ' number of colors for inner elements = ',nb_colors_inner_elements
+ write(IMAIN,*) ' number of colors for outer elements = ',nb_colors_outer_elements
+ write(IMAIN,*) ' total number of colors (sum of both) = ', nb_colors_inner_elements + nb_colors_outer_elements
+ write(IMAIN,*) ' elements:'
+ write(IMAIN,*) ' number of elements for outer elements = ',nspec_outer
+ write(IMAIN,*) ' number of elements for inner elements = ',nspec_inner
+ write(IMAIN,*) ' total number of elements for domain elements = ',nspec_domain
+ endif
+ ! total number of colors used
+ nb_colors = nb_colors_inner_elements+nb_colors_outer_elements
+ first_elem_number_in_this_color(:) = 0
+
+ ! gets element permutation depending on colors
+ call get_final_perm(color,perm,first_elem_number_in_this_color(1:nb_colors), &
+ nspec,nb_colors,nb_colors_outer_elements, &
+ ispec_is_d,nspec_domain)
+
+
end subroutine get_perm_color_faster
-!-----------------------------------------------------------------------
+!
+!-------------------------------------------------------------------------------------------------
+!
-subroutine get_color_faster(ibool, is_on_a_slice_edge, myrank, nspec, nglob, &
- color, nb_colors_outer_elements, nb_colors_inner_elements, nspec_outer)
+ subroutine get_color_faster(ibool, is_on_a_slice_edge, ispec_is_d, &
+ myrank, nspec, nglob, &
+ color, nb_colors_outer_elements, nb_colors_inner_elements, &
+ nspec_outer,nspec_inner,nspec_domain)
implicit none
include "constants.h"
-! local variables
integer nspec,nglob
+ logical, dimension(nspec) :: is_on_a_slice_edge,ispec_is_d
- logical, dimension(nspec) :: is_on_a_slice_edge
-
integer, dimension(NGLLX,NGLLY,NGLLZ,nspec) :: ibool
integer, dimension(nspec) :: color
- integer :: nb_colors_outer_elements,nb_colors_inner_elements,myrank,nspec_outer
+ integer :: nb_colors_outer_elements,nb_colors_inner_elements,myrank
+ integer :: nspec_outer,nspec_inner,nspec_domain
+
+ ! local variables
integer :: ispec
-
-!! DK DK for mesh coloring GPU Joseph Fourier
logical, dimension(:), allocatable :: mask_ibool
-
integer :: icolor, nb_already_colored
integer :: iglob1,iglob2,iglob3,iglob4,iglob5,iglob6,iglob7,iglob8
-
+ integer :: ier
logical :: conflict_found_need_new_color
+ ! Droux
+ logical :: try_Droux_coloring
+ logical :: fail_safe
+ ! valence
+ integer :: maxval_count_ibool_outer,maxval_count_ibool_inner
+ ! display absolute minimum possible number of colors, i.e., maximum valence (for information only)
+ ! beware: this wastes memory (needs an additional array called "count_ibool")
+ logical, parameter :: DISPLAY_MIN_POSSIBLE_COLORS = .false.
+
+ ! user output
+ if( myrank == 0 ) then
+ if( USE_DROUX_OPTIMIZATION ) then
+ write(IMAIN,*) ' fast coloring mesh algorithm w/ Droux optimization'
+ else if( BALANCE_COLORS_SIMPLE_ALGO ) then
+ write(IMAIN,*) ' fast coloring mesh algorithm w/ color balancing'
+ else
+ write(IMAIN,*) ' fast coloring mesh algorithm'
+ endif
+ endif
+
+ ! counts number of elements for inner, outer and total domain
nspec_outer = 0
+ nspec_inner = 0
+ nspec_domain = 0
do ispec=1,nspec
- if(is_on_a_slice_edge(ispec)) nspec_outer=nspec_outer+1
+ ! domain elements
+ if(ispec_is_d(ispec)) then
+ ! outer/inner elements
+ if(is_on_a_slice_edge(ispec)) then
+ nspec_outer=nspec_outer+1
+ else
+ nspec_inner=nspec_inner+1
+ endif
+ nspec_domain=nspec_domain+1
+ endif
enddo
-!! DK DK start mesh coloring (new Apr 2010 version by DK for GPU Joseph Fourier)
- allocate(mask_ibool(nglob))
+ ! debug
+ !if(myrank == 0) then
+ ! print *
+ ! print *,'----------------------------------'
+ ! print *,'coloring the mesh'
+ ! print *,'----------------------------------'
+ ! print *
+ !endif
-!! DK DK ----------------------------------
-!! DK DK color the mesh in the crust_mantle
-!! DK DK ----------------------------------
+ ! Droux optimization
+ try_Droux_coloring = USE_DROUX_OPTIMIZATION
- if(myrank == 0) then
- print *
- print *,'----------------------------------'
- print *,'coloring the mesh'
- print *,'----------------------------------'
- print *
- endif
+ if(BALANCE_COLORS_SIMPLE_ALGO .and. USE_DROUX_OPTIMIZATION ) then
+ if( myrank == 0 ) then
+ print *,'noticed a problem with mesh coloring options: '
+ print *,' cannot set both USE_DROUX_OPTIMAL_ALGO and BALANCE_COLORS_SIMPLE_ALGO'
+ print *,' -> this run will use only BALANCE_COLORS_SIMPLE_ALGO'
+ print *,'please check parameter settings in constants.h...'
+ endif
+ try_Droux_coloring = .false.
+ endif
+ ! gives a lower bound for the number of colors needed
+ if(DISPLAY_MIN_POSSIBLE_COLORS .or. try_Droux_coloring) then
+ ! gets maximum values of valence for inner and outer element points
+ call count_mesh_valence(ibool,is_on_a_slice_edge,ispec_is_d, &
+ myrank, nspec, nglob, &
+ maxval_count_ibool_outer,maxval_count_ibool_inner)
+ endif
-! first set color of all elements to 1
+ ! allocates mask
+ allocate(mask_ibool(nglob),stat=ier)
+ if( ier /= 0 ) stop 'error allocating mask_ibool array'
+
+ ! entry point for fail-safe mechanism when Droux 1993 fails
+ 999 continue
+
+ ! first set color of all elements to 0,
+ ! to use it as a flag to detect elements not yet colored
color(:) = 0
icolor = 0
nb_already_colored = 0
- do while(nb_already_colored<nspec_outer)
+ ! colors outer elements
+ do while( nb_already_colored < nspec_outer )
+
+ 333 continue
icolor = icolor + 1
- 333 continue
- !if(myrank == 0) print *,'analyzing color ',icolor,' for all the elements of the mesh'
+
+ ! debug: user output
+ !if(myrank == 0) then
+ ! print *,' analyzing color ',icolor,' - outer elements'
+ !endif
+
+ ! resets flags
mask_ibool(:) = .false.
conflict_found_need_new_color = .false.
+ ! finds un-colored elements
do ispec = 1,nspec
- if(is_on_a_slice_edge(ispec)) then
- if(color(ispec) == 0) then
- ! the eight corners of the current element
- iglob1=ibool(1,1,1,ispec)
- iglob2=ibool(NGLLX,1,1,ispec)
- iglob3=ibool(NGLLX,NGLLY,1,ispec)
- iglob4=ibool(1,NGLLY,1,ispec)
- iglob5=ibool(1,1,NGLLZ,ispec)
- iglob6=ibool(NGLLX,1,NGLLZ,ispec)
- iglob7=ibool(NGLLX,NGLLY,NGLLZ,ispec)
- iglob8=ibool(1,NGLLY,NGLLZ,ispec)
+ ! domain elements only
+ if( ispec_is_d(ispec) ) then
+ ! outer elements
+ if( is_on_a_slice_edge(ispec) ) then
+ if(color(ispec) == 0) then
+ ! the eight corners of the current element
+ iglob1=ibool(1,1,1,ispec)
+ iglob2=ibool(NGLLX,1,1,ispec)
+ iglob3=ibool(NGLLX,NGLLY,1,ispec)
+ iglob4=ibool(1,NGLLY,1,ispec)
+ iglob5=ibool(1,1,NGLLZ,ispec)
+ iglob6=ibool(NGLLX,1,NGLLZ,ispec)
+ iglob7=ibool(NGLLX,NGLLY,NGLLZ,ispec)
+ iglob8=ibool(1,NGLLY,NGLLZ,ispec)
- if(mask_ibool(iglob1) .or. mask_ibool(iglob2) .or. mask_ibool(iglob3) .or. mask_ibool(iglob4) .or. &
- mask_ibool(iglob5) .or. mask_ibool(iglob6) .or. mask_ibool(iglob7) .or. mask_ibool(iglob8)) then
- ! if element of this color has a common point with another element of that same color
- ! then we need to create a new color, i.e., increment the color of the current element
- conflict_found_need_new_color = .true.
- else
- color(ispec) = icolor
- nb_already_colored = nb_already_colored + 1
- mask_ibool(iglob1) = .true.
- mask_ibool(iglob2) = .true.
- mask_ibool(iglob3) = .true.
- mask_ibool(iglob4) = .true.
- mask_ibool(iglob5) = .true.
- mask_ibool(iglob6) = .true.
- mask_ibool(iglob7) = .true.
- mask_ibool(iglob8) = .true.
+ if(mask_ibool(iglob1) .or. mask_ibool(iglob2) .or. mask_ibool(iglob3) .or. mask_ibool(iglob4) .or. &
+ mask_ibool(iglob5) .or. mask_ibool(iglob6) .or. mask_ibool(iglob7) .or. mask_ibool(iglob8)) then
+ ! if element of this color has a common point with another element of that same color
+ ! then we need to create a new color, i.e., increment the color of the current element
+ conflict_found_need_new_color = .true.
+ else
+ color(ispec) = icolor
+ nb_already_colored = nb_already_colored + 1
+ mask_ibool(iglob1) = .true.
+ mask_ibool(iglob2) = .true.
+ mask_ibool(iglob3) = .true.
+ mask_ibool(iglob4) = .true.
+ mask_ibool(iglob5) = .true.
+ mask_ibool(iglob6) = .true.
+ mask_ibool(iglob7) = .true.
+ mask_ibool(iglob8) = .true.
+ endif
endif
endif
endif
enddo
+ ! debug: user output
+ !if(myrank == 0) then
+ ! print *,' done ',(100.0*nb_already_colored)/nspec_domain,'% of ',nspec_domain,'elements'
+ !endif
+
if(conflict_found_need_new_color) then
- icolor = icolor + 1
+ if( icolor >= MAX_NUMBER_OF_COLORS ) stop 'error MAX_NUMBER_OF_COLORS too small'
goto 333
endif
enddo
nb_colors_outer_elements = icolor
- do while(nb_already_colored<nspec)
+ ! colors inner elements
+ do while(nb_already_colored < nspec_domain)
+
+ 334 continue
icolor = icolor + 1
- 334 continue
- !if(myrank == 0) print *,'analyzing color ',icolor,' for all the elements of the mesh'
+
+ ! debug: user output
+ !if(myrank == 0) then
+ ! print *,' analyzing color ',icolor,' - inner elements'
+ !endif
+
+ ! resets flags
mask_ibool(:) = .false.
conflict_found_need_new_color = .false.
+
do ispec = 1,nspec
- if (.not. is_on_a_slice_edge(ispec)) then
- if(color(ispec) == 0) then
+ ! domain elements only
+ if(ispec_is_d(ispec)) then
+ ! inner elements
+ if (.not. is_on_a_slice_edge(ispec)) then
+ if(color(ispec) == 0) then
+ ! the eight corners of the current element
+ iglob1=ibool(1,1,1,ispec)
+ iglob2=ibool(NGLLX,1,1,ispec)
+ iglob3=ibool(NGLLX,NGLLY,1,ispec)
+ iglob4=ibool(1,NGLLY,1,ispec)
+ iglob5=ibool(1,1,NGLLZ,ispec)
+ iglob6=ibool(NGLLX,1,NGLLZ,ispec)
+ iglob7=ibool(NGLLX,NGLLY,NGLLZ,ispec)
+ iglob8=ibool(1,NGLLY,NGLLZ,ispec)
+
+ if(mask_ibool(iglob1) .or. mask_ibool(iglob2) .or. mask_ibool(iglob3) .or. mask_ibool(iglob4) .or. &
+ mask_ibool(iglob5) .or. mask_ibool(iglob6) .or. mask_ibool(iglob7) .or. mask_ibool(iglob8)) then
+ ! if element of this color has a common point with another element of that same color
+ ! then we need to create a new color, i.e., increment the color of the current element
+ conflict_found_need_new_color = .true.
+ else
+ color(ispec) = icolor
+ nb_already_colored = nb_already_colored + 1
+ mask_ibool(iglob1) = .true.
+ mask_ibool(iglob2) = .true.
+ mask_ibool(iglob3) = .true.
+ mask_ibool(iglob4) = .true.
+ mask_ibool(iglob5) = .true.
+ mask_ibool(iglob6) = .true.
+ mask_ibool(iglob7) = .true.
+ mask_ibool(iglob8) = .true.
+ endif
+ endif
+ endif
+ endif
+ enddo
+
+ ! debug user output
+ !if(myrank == 0) then
+ ! print *,' done ',(100.0*nb_already_colored)/nspec_domain,'% of ',nspec_domain,'elements'
+ !endif
+
+ if(conflict_found_need_new_color) then
+ if( icolor >= MAX_NUMBER_OF_COLORS ) stop 'error MAX_NUMBER_OF_COLORS too small'
+ goto 334
+ endif
+ enddo
+
+ nb_colors_inner_elements = icolor - nb_colors_outer_elements
+
+ ! Droux optimization:
+ ! added this to create more balanced colors according to JJ Droux (1993)
+ ! note: this might not find an optimial solution.
+ ! we will probably have to try a few times with increasing colors
+ if( try_Droux_coloring ) then
+ ! initializes fail-safe mechanism
+ fail_safe = .false.
+
+ ! tries to find a balanced coloring
+ call balance_colors_Droux(ibool,is_on_a_slice_edge,ispec_is_d, &
+ myrank, nspec, nglob, &
+ color,nb_colors_outer_elements,nb_colors_inner_elements, &
+ nspec_outer,nspec_inner,maxval_count_ibool_inner, &
+ mask_ibool,fail_safe)
+
+ ! in case it fails go back to simple coloring algorithm
+ if( fail_safe ) then
+ try_Droux_coloring = .false.
+ if(myrank == 0) write(IMAIN,*) ' giving up on Droux 1993 algorithm, calling fail-safe mechanism'
+ goto 999
+ endif
+ endif ! of if(try_Droux_coloring)
+
+ ! balances colors using a simple algorithm (if Droux was not used)
+ if( BALANCE_COLORS_SIMPLE_ALGO ) then
+ call balance_colors_simple(ibool,is_on_a_slice_edge,ispec_is_d, &
+ myrank, nspec, nglob, &
+ color,nb_colors_outer_elements,nb_colors_inner_elements, &
+ nspec_outer,nspec_inner,mask_ibool)
+ endif
+
+ ! checks that all the color sets are independent
+ do icolor = 1,maxval(color)
+ mask_ibool(:) = .false.
+ do ispec = 1,nspec
+ ! domain elements only
+ if(ispec_is_d(ispec)) then
+ if(color(ispec) == icolor ) then
! the eight corners of the current element
iglob1=ibool(1,1,1,ispec)
iglob2=ibool(NGLLX,1,1,ispec)
@@ -168,11 +390,10 @@
if(mask_ibool(iglob1) .or. mask_ibool(iglob2) .or. mask_ibool(iglob3) .or. mask_ibool(iglob4) .or. &
mask_ibool(iglob5) .or. mask_ibool(iglob6) .or. mask_ibool(iglob7) .or. mask_ibool(iglob8)) then
! if element of this color has a common point with another element of that same color
- ! then we need to create a new color, i.e., increment the color of the current element
- conflict_found_need_new_color = .true.
+ ! then there is a problem, the color set is not correct
+ print*,'error check color:',icolor
+ stop 'error detected: found a common point inside a color set'
else
- color(ispec) = icolor
- nb_already_colored = nb_already_colored + 1
mask_ibool(iglob1) = .true.
mask_ibool(iglob2) = .true.
mask_ibool(iglob3) = .true.
@@ -186,21 +407,61 @@
endif
enddo
- if(conflict_found_need_new_color) then
- icolor = icolor + 1
- goto 334
- endif
+ !debug output
+ !if(myrank == 0) print *,' color ',icolor,' has disjoint elements only and is therefore OK'
+ !if(myrank == 0) print *,' color ',icolor,' contains ',count(color == icolor),' elements'
enddo
+ ! debug output
+ !if(myrank == 0) then
+ ! print*, ' the ',maxval(color),' color sets are OK'
+ !endif
- nb_colors_inner_elements = icolor - nb_colors_outer_elements
+ deallocate(mask_ibool)
- if(myrank == 0) print *,'created a total of ',maxval(color),' colors for all the elements of the mesh'
+ end subroutine get_color_faster
- !!!!!!!! DK DK now check that all the color sets are independent
- do icolor = 1,maxval(color)
- mask_ibool(:) = .false.
- do ispec = 1,nspec
- if(color(ispec) == icolor) then
+!
+!-------------------------------------------------------------------------------------------------
+!
+
+ subroutine count_mesh_valence(ibool,is_on_a_slice_edge,ispec_is_d, &
+ myrank, nspec, nglob, &
+ maxval_count_ibool_outer,maxval_count_ibool_inner)
+
+ implicit none
+
+ include "constants.h"
+
+ integer :: nspec,nglob
+
+ integer, dimension(NGLLX,NGLLY,NGLLZ,nspec) :: ibool
+
+ logical, dimension(nspec) :: is_on_a_slice_edge,ispec_is_d
+
+ integer :: myrank
+ integer :: maxval_count_ibool_outer,maxval_count_ibool_inner
+
+ ! local parameters
+ integer, dimension(:), allocatable :: count_ibool
+ integer :: ispec
+ integer :: iglob1,iglob2,iglob3,iglob4,iglob5,iglob6,iglob7,iglob8
+ integer :: ier
+
+ ! allocates count array
+ allocate(count_ibool(nglob),stat=ier)
+ if( ier /= 0 ) stop 'error allocating count_ibool array'
+
+ ! valence numbers of the mesh
+ maxval_count_ibool_outer = 0
+ maxval_count_ibool_inner = 0
+
+ ! valence for outer elements
+ count_ibool(:) = 0
+ do ispec = 1,nspec
+ ! domain elements only
+ if(ispec_is_d(ispec)) then
+ ! outer elements
+ if (is_on_a_slice_edge(ispec)) then
! the eight corners of the current element
iglob1=ibool(1,1,1,ispec)
iglob2=ibool(NGLLX,1,1,ispec)
@@ -211,526 +472,631 @@
iglob7=ibool(NGLLX,NGLLY,NGLLZ,ispec)
iglob8=ibool(1,NGLLY,NGLLZ,ispec)
- if(mask_ibool(iglob1) .or. mask_ibool(iglob2) .or. mask_ibool(iglob3) .or. mask_ibool(iglob4) .or. &
- mask_ibool(iglob5) .or. mask_ibool(iglob6) .or. mask_ibool(iglob7) .or. mask_ibool(iglob8)) then
- ! if element of this color has a common point with another element of that same color
- ! then there is a problem, the color set is not correct
- stop 'error detected: found a common point inside a color set'
- else
- mask_ibool(iglob1) = .true.
- mask_ibool(iglob2) = .true.
- mask_ibool(iglob3) = .true.
- mask_ibool(iglob4) = .true.
- mask_ibool(iglob5) = .true.
- mask_ibool(iglob6) = .true.
- mask_ibool(iglob7) = .true.
- mask_ibool(iglob8) = .true.
- endif
+ count_ibool(iglob1) = count_ibool(iglob1) + 1
+ count_ibool(iglob2) = count_ibool(iglob2) + 1
+ count_ibool(iglob3) = count_ibool(iglob3) + 1
+ count_ibool(iglob4) = count_ibool(iglob4) + 1
+ count_ibool(iglob5) = count_ibool(iglob5) + 1
+ count_ibool(iglob6) = count_ibool(iglob6) + 1
+ count_ibool(iglob7) = count_ibool(iglob7) + 1
+ count_ibool(iglob8) = count_ibool(iglob8) + 1
endif
- enddo
+ endif
+ enddo
+ maxval_count_ibool_outer = maxval(count_ibool)
- if(myrank == 0) print *,'color ',icolor,' has disjoint elements only and is therefore OK'
- if(myrank == 0) print *,'it contains ',count(color == icolor),' elements'
+ ! valence for inner elements
+ count_ibool(:) = 0
+ do ispec = 1,nspec
+ ! domain elements only
+ if(ispec_is_d(ispec)) then
+ ! inner elements
+ if (.not. is_on_a_slice_edge(ispec)) then
+ ! the eight corners of the current element
+ iglob1=ibool(1,1,1,ispec)
+ iglob2=ibool(NGLLX,1,1,ispec)
+ iglob3=ibool(NGLLX,NGLLY,1,ispec)
+ iglob4=ibool(1,NGLLY,1,ispec)
+ iglob5=ibool(1,1,NGLLZ,ispec)
+ iglob6=ibool(NGLLX,1,NGLLZ,ispec)
+ iglob7=ibool(NGLLX,NGLLY,NGLLZ,ispec)
+ iglob8=ibool(1,NGLLY,NGLLZ,ispec)
+
+ count_ibool(iglob1) = count_ibool(iglob1) + 1
+ count_ibool(iglob2) = count_ibool(iglob2) + 1
+ count_ibool(iglob3) = count_ibool(iglob3) + 1
+ count_ibool(iglob4) = count_ibool(iglob4) + 1
+ count_ibool(iglob5) = count_ibool(iglob5) + 1
+ count_ibool(iglob6) = count_ibool(iglob6) + 1
+ count_ibool(iglob7) = count_ibool(iglob7) + 1
+ count_ibool(iglob8) = count_ibool(iglob8) + 1
+ endif
+ endif
enddo
+ maxval_count_ibool_inner = maxval(count_ibool)
- if(myrank == 0) print *,'the ',maxval(color),' color sets are OK'
+ ! debug outupt
+ if( myrank == 0 ) then
+ write(IMAIN,*) ' maximum valence (i.e. minimum possible nb of colors) for outer = ',maxval_count_ibool_outer
+ write(IMAIN,*) ' maximum valence (i.e. minimum possible nb of colors) for inner = ',maxval_count_ibool_inner
+ endif
- deallocate(mask_ibool)
+ deallocate(count_ibool)
-end subroutine get_color_faster
+ end subroutine count_mesh_valence
-!*********************************************************************************************************
+!
+!-------------------------------------------------------------------------------------------------
+!
- subroutine get_perm_color(is_on_a_slice_edge,ibool,perm,nspec,nglob, &
- nb_colors_outer_elements,nb_colors_inner_elements,nspec_outer,first_elem_number_in_this_color,myrank)
+ subroutine balance_colors_Droux(ibool,is_on_a_slice_edge,ispec_is_d, &
+ myrank, nspec, nglob, &
+ color, nb_colors_outer_elements, nb_colors_inner_elements, &
+ nspec_outer,nspec_inner,maxval_count_ibool_inner, &
+ mask_ibool,fail_safe)
implicit none
include "constants.h"
-! local variables
- integer nspec,nglob_GLL_full
+ integer :: nspec,nglob
- logical, dimension(nspec) :: is_on_a_slice_edge
-
integer, dimension(NGLLX,NGLLY,NGLLZ,nspec) :: ibool
- integer, dimension(nspec) :: perm
- integer, dimension(nspec) :: color
- integer, dimension(MAX_NUMBER_OF_COLORS) :: first_elem_number_in_this_color
- integer :: nb_colors_outer_elements,nb_colors_inner_elements,nspec_outer,myrank
-! a neighbor of a hexahedral node is a hexahedron that shares a face with it -> max degree of a node = 6
- integer, parameter :: MAX_NUMBER_OF_NEIGHBORS = 100
+ logical, dimension(nspec) :: is_on_a_slice_edge,ispec_is_d
+ logical, dimension(nglob) :: mask_ibool
- integer nglob_eight_corners_only,nglob
+ integer, dimension(nspec) :: color
-! global corner numbers that need to be created
- integer, dimension(nglob) :: global_corner_number
+ integer :: myrank
+ integer :: nb_colors_outer_elements,nb_colors_inner_elements
- integer mn(nspec*NGNOD_HEXAHEDRA),mp(nspec+1)
- integer, dimension(:), allocatable :: ne,np,adj
- integer xadj(nspec+1)
+ integer :: nspec_outer,nspec_inner
+ integer :: maxval_count_ibool_inner
- integer i,istart,istop,number_of_neighbors
+ logical :: fail_safe
-! only count the total size of the array that will be created, or actually create it
- logical count_only
- integer total_size_ne,total_size_adj
+ ! local parameters
+ logical, dimension(:), allocatable :: icolor_conflict_found
+ integer, dimension(:), allocatable :: nb_elems_in_this_color
+ integer :: ispec,ispec2,icolor,ncolors,icolormin,icolormax,icolor_chosen,nb_elems_in_color_chosen
+ integer :: nb_tries_of_Droux_1993,last_ispec_studied
+ integer :: ier
-!
-!-----------------------------------------------------------------------
-!
+ ! debug outupt
+ if( myrank == 0 ) then
+ write(IMAIN,*) ' balancing colors: Droux algorithm'
+ write(IMAIN,*) ' initial number of outer element colors = ',nb_colors_outer_elements
+ write(IMAIN,*) ' initial number of inner element colors = ',nb_colors_inner_elements
+ write(IMAIN,*) ' initial number of total colors = ',nb_colors_outer_elements + nb_colors_inner_elements
+ endif
-! total number of points in the mesh
- nglob_GLL_full = nglob
+ ! initial guess of number of colors needed
+ if( maxval_count_ibool_inner > 0 .and. maxval_count_ibool_inner < nb_colors_inner_elements ) then
+ ! uses maximum valence to estimate number of colors for Droux
+ nb_colors_inner_elements = maxval_count_ibool_inner
+ endif
-!---- call Charbel Farhat's routines
- if(myrank == 0) &
- write(IMAIN,*) 'calling form_elt_connectivity_foelco to perform mesh coloring and inner/outer element splitting'
- call form_elt_connectivity_foelco(mn,mp,nspec,global_corner_number,nglob_GLL_full,ibool,nglob_eight_corners_only)
- do i=1,nspec
- istart = mp(i)
- istop = mp(i+1) - 1
- enddo
+ !! DK DK do it for inner elements only for now
+ ! Droux optimization run
+ nb_tries_of_Droux_1993 = 1
-! count only, to determine the size needed for the array
- allocate(np(nglob_eight_corners_only+1))
- count_only = .true.
- total_size_ne = 1
- if(myrank == 0) write(IMAIN,*) 'calling form_node_connectivity_fonoco to determine the size of the table'
- allocate(ne(total_size_ne))
- call form_node_connectivity_fonoco(mn,mp,ne,np,nglob_eight_corners_only,nspec,count_only,total_size_ne)
- deallocate(ne)
+ ! entry point to re-try Droux
+ 765 continue
-!print *, 'nglob_eight_corners_only'
-!print *, nglob_eight_corners_only
+ ! initial guess of number of colors needed
+ ncolors = nb_colors_outer_elements + nb_colors_inner_elements
-! allocate the array with the right size
- allocate(ne(total_size_ne))
+ ! debug output
+ if( myrank == 0 ) then
+ write(IMAIN,*) ' Droux optimization: try = ',nb_tries_of_Droux_1993,'colors = ',ncolors
+ endif
-! now actually generate the array
- count_only = .false.
- if(myrank == 0) write(IMAIN,*) 'calling form_node_connectivity_fonoco to actually create the table'
- call form_node_connectivity_fonoco(mn,mp,ne,np,nglob_eight_corners_only,nspec,count_only,total_size_ne)
- do i=1,nglob_eight_corners_only
- istart = np(i)
- istop = np(i+1) - 1
- enddo
+ icolormin = nb_colors_outer_elements + 1
+ icolormax = ncolors
-!print *, 'total_size_ne'
-!print *, total_size_ne
+ ! allocates temporary arrays
+ allocate(nb_elems_in_this_color(ncolors), &
+ icolor_conflict_found(ncolors),stat=ier)
+ if( ier /= 0 ) stop 'error allocating nb_elems_in_this_color arrays'
-! count only, to determine the size needed for the array
- count_only = .true.
- total_size_adj = 1
- if(myrank == 0) write(IMAIN,*) 'calling create_adjacency_table_adjncy to determine the size of the table'
- allocate(adj(total_size_adj))
- !call create_adjacency_table_adjncy(mn,mp,ne,np,adj,xadj,maskel,nspec,nglob_eight_corners_only,&
- !count_only,total_size_ne,total_size_adj,.false.)
- call create_adjacency_table_adjncy(mn,mp,ne,np,adj,xadj,nspec,nglob_eight_corners_only,&
- count_only,total_size_ne,total_size_adj,.false.)
- deallocate(adj)
+ nb_elems_in_this_color(:) = 0
+ mask_ibool(:) = .false.
+ last_ispec_studied = -1
-! allocate the array with the right size
- allocate(adj(total_size_adj))
+ do ispec = 1,nspec
+ ! domain elements only
+ if(ispec_is_d(ispec)) then
-! now actually generate the array
- count_only = .false.
- if(myrank == 0) write(IMAIN,*) 'calling create_adjacency_table_adjncy again to actually create the table'
- !call create_adjacency_table_adjncy(mn,mp,ne,np,adj,xadj,maskel,nspec,nglob_eight_corners_only,&
- !count_only,total_size_ne,total_size_adj,.false.)
- call create_adjacency_table_adjncy(mn,mp,ne,np,adj,xadj,nspec,nglob_eight_corners_only,&
- count_only,total_size_ne,total_size_adj,.false.)
+ ! only inner elements
+ if (is_on_a_slice_edge(ispec)) cycle
- do i=1,nspec
- istart = xadj(i)
- istop = xadj(i+1) - 1
- number_of_neighbors = istop-istart+1
- if(number_of_neighbors < 1 .or. number_of_neighbors > MAX_NUMBER_OF_NEIGHBORS) stop 'incorrect number of neighbors'
- enddo
+ ! unmark the eight corners of the previously marked element
+ if(last_ispec_studied > 0) then
+ mask_ibool(ibool(1,1,1,last_ispec_studied)) = .false.
+ mask_ibool(ibool(NGLLX,1,1,last_ispec_studied)) = .false.
+ mask_ibool(ibool(NGLLX,NGLLY,1,last_ispec_studied)) = .false.
+ mask_ibool(ibool(1,NGLLY,1,last_ispec_studied)) = .false.
+ mask_ibool(ibool(1,1,NGLLZ,last_ispec_studied)) = .false.
+ mask_ibool(ibool(NGLLX,1,NGLLZ,last_ispec_studied)) = .false.
+ mask_ibool(ibool(NGLLX,NGLLY,NGLLZ,last_ispec_studied)) = .false.
+ mask_ibool(ibool(1,NGLLY,NGLLZ,last_ispec_studied)) = .false.
+ endif
+ icolor_conflict_found(icolormin:icolormax) = .false.
- deallocate(ne,np)
+ ! mark the eight corners of the current element
+ mask_ibool(ibool(1,1,1,ispec)) = .true.
+ mask_ibool(ibool(NGLLX,1,1,ispec)) = .true.
+ mask_ibool(ibool(NGLLX,NGLLY,1,ispec)) = .true.
+ mask_ibool(ibool(1,NGLLY,1,ispec)) = .true.
+ mask_ibool(ibool(1,1,NGLLZ,ispec)) = .true.
+ mask_ibool(ibool(NGLLX,1,NGLLZ,ispec)) = .true.
+ mask_ibool(ibool(NGLLX,NGLLY,NGLLZ,ispec)) = .true.
+ mask_ibool(ibool(1,NGLLY,NGLLZ,ispec)) = .true.
+ last_ispec_studied = ispec
- call get_color(adj,xadj,color,nspec,total_size_adj,is_on_a_slice_edge, &
- nb_colors_outer_elements,nb_colors_inner_elements,nspec_outer)
+ if(ispec > 1) then
+ do ispec2 = 1,ispec - 1
+ ! domain elements only
+ if(ispec_is_d(ispec2)) then
- if(myrank == 0) then
- write(IMAIN,*) 'number of colors of the graph for inner elements = ',nb_colors_inner_elements
- write(IMAIN,*) 'number of colors of the graph for outer elements = ',nb_colors_outer_elements
- write(IMAIN,*) 'total number of colors of the graph (sum of both) = ', nb_colors_inner_elements + nb_colors_outer_elements
- write(IMAIN,*) 'number of elements of the graph for outer elements = ',nspec_outer
- endif
+ ! only inner elements
+ if (is_on_a_slice_edge(ispec2)) cycle
- deallocate(adj)
+ ! if conflict already found previously with this color, no need to test again
+ if (icolor_conflict_found(color(ispec2))) cycle
- if(myrank == 0) write(IMAIN,*) 'generating the final colors'
- first_elem_number_in_this_color(:) = -1
- call get_final_perm(color,perm,first_elem_number_in_this_color,nspec,nb_colors_inner_elements+nb_colors_outer_elements)
+ ! test the eight corners of the current element for a common point with element under study
+ if (mask_ibool(ibool(1,1,1,ispec2)) .or. &
+ mask_ibool(ibool(NGLLX,1,1,ispec2)) .or. &
+ mask_ibool(ibool(NGLLX,NGLLY,1,ispec2)) .or. &
+ mask_ibool(ibool(1,NGLLY,1,ispec2)) .or. &
+ mask_ibool(ibool(1,1,NGLLZ,ispec2)) .or. &
+ mask_ibool(ibool(NGLLX,1,NGLLZ,ispec2)) .or. &
+ mask_ibool(ibool(NGLLX,NGLLY,NGLLZ,ispec2)) .or. &
+ mask_ibool(ibool(1,NGLLY,NGLLZ,ispec2))) &
+ icolor_conflict_found(color(ispec2)) = .true.
- if(myrank == 0) write(IMAIN,*) 'done with mesh coloring and inner/outer element splitting'
+ endif ! domain elements
+ enddo
+ endif
- end subroutine get_perm_color
+ ! check if the Droux 1993 algorithm found a solution
+ if (all(icolor_conflict_found(icolormin:icolormax))) then
+ ! user output
+ !if(myrank == 0) write(IMAIN,*) ' Droux 1993 algorithm did not find any solution for ncolors = ',ncolors
-!------------------------------------------------------------------
+ ! try with one more color
+ if(nb_tries_of_Droux_1993 < MAX_NB_TRIES_OF_DROUX_1993) then
+ nb_colors_inner_elements = nb_colors_inner_elements + 1
+ deallocate(nb_elems_in_this_color)
+ deallocate(icolor_conflict_found)
+ nb_tries_of_Droux_1993 = nb_tries_of_Droux_1993 + 1
+ goto 765
+ else
+ ! fail-safe mechanism: if Droux 1993 still fails after all the tries with one more color,
+ ! then go back to my original simple and fast coloring algorithm
+ fail_safe = .true.
+ return
+ endif
+ endif
-subroutine get_final_perm(color,perm,first_elem_number_in_this_color,nspec,nb_color)
+ ! loop on all the colors to determine the color with the smallest number
+ ! of elements and for which there is no conflict
+ nb_elems_in_color_chosen = 2147000000 ! start with extremely large unrealistic value
+ icolor_chosen = 0
+ do icolor = icolormin,icolormax
+ if (.not. icolor_conflict_found(icolor) .and. nb_elems_in_this_color(icolor) < nb_elems_in_color_chosen) then
+ icolor_chosen = icolor
+ nb_elems_in_color_chosen = nb_elems_in_this_color(icolor)
+ endif
+ enddo
- integer, intent(in) :: nspec,nb_color
- integer, intent(in) :: color(nspec)
- integer, intent(inout) :: perm(nspec)
- integer, intent(inout) :: first_elem_number_in_this_color(nb_color)
- integer :: ielem,icolor,counter
+ ! store the color finally chosen
+ color(ispec) = icolor_chosen
+ nb_elems_in_this_color(icolor_chosen) = nb_elems_in_this_color(icolor_chosen) + 1
- counter = 1
- do icolor = 1, nb_color
- first_elem_number_in_this_color(icolor) = counter
- do ielem = 1, nspec
- if(color(ielem) == icolor) then
- perm(ielem) = counter
- counter = counter + 1
- endif
- enddo
+ endif ! domain elements
enddo
-end subroutine get_final_perm
+ ! debug output
+ if(myrank == 0) then
+ write(IMAIN,*) ' created a total of ',maxval(color),' colors in this domain' ! 'for all the domain elements of the mesh'
+ if( nb_colors_outer_elements > 0 ) &
+ write(IMAIN,*) ' typical nb of elements per color for outer elements should be ', &
+ nspec_outer / nb_colors_outer_elements
+ if( nb_colors_inner_elements > 0 ) &
+ write(IMAIN,*) ' typical nb of elements per color for inner elements should be ', &
+ nspec_inner / nb_colors_inner_elements
+ endif
-!------------------------------------------------------------------
-subroutine get_color(adj,xadj,color,nspec,total_size_adj,is_on_a_slice_edge, &
- nb_colors_outer_elements,nb_colors_inner_elements,nspec_outer)
+ end subroutine balance_colors_Droux
- integer, intent(in) :: nspec,total_size_adj
- integer, intent(in) :: adj(total_size_adj),xadj(nspec+1)
- integer :: color(nspec)
- integer :: this_color,nb_already_colored,ispec,ixadj,ok
- logical, dimension(nspec) :: is_on_a_slice_edge
- integer :: nb_colors_outer_elements,nb_colors_inner_elements,nspec_outer
- logical :: is_outer_element(nspec)
+!
+!-------------------------------------------------------------------------------------------------
+!
- nspec_outer = 0
+ subroutine balance_colors_simple(ibool,is_on_a_slice_edge,ispec_is_d, &
+ myrank, nspec, nglob, &
+ color, nb_colors_outer_elements, nb_colors_inner_elements, &
+ nspec_outer,nspec_inner,mask_ibool)
- is_outer_element(:) = .false.
+ implicit none
- do ispec=1,nspec
- if (is_on_a_slice_edge(ispec)) then
- is_outer_element(ispec) = .true.
- nspec_outer=nspec_outer+1
- endif
- enddo
+ include "constants.h"
-! outer elements
- color(:) = 0
- this_color = 0
- nb_already_colored = 0
- do while(nb_already_colored<nspec_outer)
- this_color = this_color + 1
- do ispec = 1, nspec
- if (is_outer_element(ispec)) then
- if (color(ispec) == 0) then
- ok = 1
- do ixadj = xadj(ispec), (xadj(ispec+1)-1)
- if (is_outer_element(adj(ixadj)) .and. color(adj(ixadj)) == this_color) ok = 0
- enddo
- if (ok /= 0) then
- color(ispec) = this_color
- nb_already_colored = nb_already_colored + 1
- endif
- endif
- endif
- enddo
- enddo
- nb_colors_outer_elements = this_color
+ integer :: nspec,nglob
-! inner elements
- do while(nb_already_colored<nspec)
- this_color = this_color + 1
- do ispec = 1, nspec
- if (.not. is_outer_element(ispec)) then
- if (color(ispec) == 0) then
- ok = 1
- do ixadj = xadj(ispec), (xadj(ispec+1)-1)
- if (.not. is_outer_element(adj(ixadj)) .and. color(adj(ixadj)) == this_color) ok = 0
- enddo
- if (ok /= 0) then
- color(ispec) = this_color
- nb_already_colored = nb_already_colored + 1
- endif
- endif
- endif
- enddo
- enddo
+ integer, dimension(NGLLX,NGLLY,NGLLZ,nspec) :: ibool
- nb_colors_inner_elements = this_color - nb_colors_outer_elements
+ logical, dimension(nspec) :: is_on_a_slice_edge,ispec_is_d
+ logical, dimension(nglob) :: mask_ibool
-end subroutine get_color
+ integer, dimension(nspec) :: color
-!------------------------------------------------------------------
+ integer :: myrank
+ integer :: nb_colors_outer_elements,nb_colors_inner_elements
-!=======================================================================
-!
-! Charbel Farhat's FEM topology routines
-!
-! Dimitri Komatitsch, February 1996 - Code based on Farhat's original version from 1987
-!
-! modified and adapted by Dimitri Komatitsch, May 2006
-!
-!=======================================================================
+ integer :: nspec_outer,nspec_inner
- subroutine form_elt_connectivity_foelco(mn,mp,nspec,global_corner_number,&
-nglob_GLL_full,ibool,nglob_eight_corners_only)
+ ! local parameters
+ logical, dimension(:), allocatable :: icolor_conflict_found
+ integer, dimension(:), allocatable :: nb_elems_in_this_color
+ integer :: ispec,ispec2,icolor,ncolors,icolormin,icolormax,icolor_chosen,nb_elems_in_color_chosen
+ integer :: last_ispec_studied
+ integer :: target_nb_elems_per_color,icolor_target
+ integer :: ier
-!-----------------------------------------------------------------------
-!
-! Forms the MN and MP arrays
-!
-! Input :
-! -------
-! ibool Array needed to build the element connectivity table
-! nspec Number of elements in the domain
-! NGNOD_HEXAHEDRA number of nodes per hexahedron (brick with 8 corners)
-!
-! Output :
-! --------
-! MN, MP This is the element connectivity array pair.
-! Array MN contains the list of the element
-! connectivity, that is, the nodes contained in each
-! element. They are stored in a stacked fashion.
-!
-! Pointer array MP stores the location of each
-! element list. Its length is equal to the number
-! of elements plus one.
-!
-!-----------------------------------------------------------------------
+ ! debug outupt
+ if( myrank == 0 ) then
+ write(IMAIN,*) ' balancing colors: simple algorithm'
+ write(IMAIN,*) ' number of outer element colors = ',nb_colors_outer_elements
+ write(IMAIN,*) ' number of inner element colors = ',nb_colors_inner_elements
+ write(IMAIN,*) ' number of total colors = ',nb_colors_outer_elements + nb_colors_inner_elements
+ endif
- implicit none
+ ! balances colors in postprocess if Droux (1993) is not used
+ ncolors = nb_colors_outer_elements + nb_colors_inner_elements
- include "constants.h"
+ ! allocates temporary arrays
+ allocate(nb_elems_in_this_color(ncolors), &
+ icolor_conflict_found(ncolors),stat=ier)
+ if( ier /= 0 ) stop 'error allocating nb_elems_in_this_color arrays'
- integer nspec,nglob_GLL_full
+ !! DK DK do it for outer elements
+ icolormin = 1
+ icolormax = nb_colors_outer_elements
-! arrays with mesh parameters per slice
- integer, intent(in), dimension(NGLLX,NGLLY,NGLLZ,nspec) :: ibool
+ ! ideal value if all colors are perfectly balanced
+ if( nb_colors_outer_elements > 0 ) then
+ target_nb_elems_per_color = nspec_outer / nb_colors_outer_elements + 1
+ else
+ target_nb_elems_per_color = 1
+ endif
-! global corner numbers that need to be created
- integer, intent(out), dimension(nglob_GLL_full) :: global_corner_number
- integer, intent(out) :: mn(nspec*NGNOD_HEXAHEDRA),mp(nspec+1)
- integer, intent(out) :: nglob_eight_corners_only
+ ! print *,'nspec_outer,target_nb_elems_per_color = ',nspec_outer,target_nb_elems_per_color
- integer ninter,nsum,ispec,node,k,inumcorner,ix,iy,iz
+ ! count the initial number of elements in each color
+ nb_elems_in_this_color(:) = 0
+ do icolor = icolormin,icolormax
+ nb_elems_in_this_color(icolor) = count(color == icolor)
+ enddo
- ninter = 1
- nsum = 1
- mp(1) = 1
+ ! do not balance the last one, because it will be balanced automatically by the others
+ do icolor = icolormin,icolormax-1
-!---- define topology of the elements in the mesh
-!---- we need to define adjacent numbers from the sub-mesh consisting of the corners only
- nglob_eight_corners_only = 0
- global_corner_number(:) = -1
+ ! if color is already balanced, do nothing
+ ! (this works because in the initial set the number of elements per color decreases when the color number increases)
+ if(nb_elems_in_this_color(icolor) <= target_nb_elems_per_color) cycle
- do ispec=1,nspec
+ mask_ibool(:) = .false.
+ last_ispec_studied = -1
- inumcorner = 0
- do iz = 1,NGLLZ,NGLLZ-1
- do iy = 1,NGLLY,NGLLY-1
- do ix = 1,NGLLX,NGLLX-1
+ do ispec = 1,nspec
+ ! domain elements only
+ if(ispec_is_d(ispec)) then
- inumcorner = inumcorner + 1
- if(inumcorner > NGNOD_HEXAHEDRA) stop 'corner number too large'
+ ! only outer elements
+ if (.not. is_on_a_slice_edge(ispec)) cycle
-! check if this point was already assigned a number previously, otherwise create one and store it
- if(global_corner_number(ibool(ix,iy,iz,ispec)) == -1) then
- nglob_eight_corners_only = nglob_eight_corners_only + 1
- global_corner_number(ibool(ix,iy,iz,ispec)) = nglob_eight_corners_only
- endif
+ ! only elements of this color
+ if (color(ispec) /= icolor) cycle
- node = global_corner_number(ibool(ix,iy,iz,ispec))
- do k=nsum,ninter-1
- if(node == mn(k)) goto 200
- enddo
+ ! if color is now balanced because we have moved enough elements then stop searching
+ if(nb_elems_in_this_color(icolor) <= target_nb_elems_per_color) exit
- mn(ninter) = node
- ninter = ninter + 1
- 200 continue
+ ! unmark the eight corners of the previously marked element
+ if(last_ispec_studied > 0) then
+ mask_ibool(ibool(1,1,1,last_ispec_studied)) = .false.
+ mask_ibool(ibool(NGLLX,1,1,last_ispec_studied)) = .false.
+ mask_ibool(ibool(NGLLX,NGLLY,1,last_ispec_studied)) = .false.
+ mask_ibool(ibool(1,NGLLY,1,last_ispec_studied)) = .false.
+ mask_ibool(ibool(1,1,NGLLZ,last_ispec_studied)) = .false.
+ mask_ibool(ibool(NGLLX,1,NGLLZ,last_ispec_studied)) = .false.
+ mask_ibool(ibool(NGLLX,NGLLY,NGLLZ,last_ispec_studied)) = .false.
+ mask_ibool(ibool(1,NGLLY,NGLLZ,last_ispec_studied)) = .false.
+ endif
+ icolor_conflict_found(icolormin:icolormax) = .false.
+ icolor_conflict_found(icolor) = .true. ! cannot move element to the color it already has
+ ! mark the eight corners of the current element
+ mask_ibool(ibool(1,1,1,ispec)) = .true.
+ mask_ibool(ibool(NGLLX,1,1,ispec)) = .true.
+ mask_ibool(ibool(NGLLX,NGLLY,1,ispec)) = .true.
+ mask_ibool(ibool(1,NGLLY,1,ispec)) = .true.
+ mask_ibool(ibool(1,1,NGLLZ,ispec)) = .true.
+ mask_ibool(ibool(NGLLX,1,NGLLZ,ispec)) = .true.
+ mask_ibool(ibool(NGLLX,NGLLY,NGLLZ,ispec)) = .true.
+ mask_ibool(ibool(1,NGLLY,NGLLZ,ispec)) = .true.
+ last_ispec_studied = ispec
+
+ ! test if we can move this element to another color
+ do ispec2 = 1,nspec
+ ! domain elements only
+ if(ispec_is_d(ispec2)) then
+
+ ! do not test that element itself
+ if (ispec2 == ispec) cycle
+
+ ! only outer elements
+ if (.not. is_on_a_slice_edge(ispec2)) cycle
+
+ ! if conflict already found previously with this color, no need to test again
+ if (icolor_conflict_found(color(ispec2))) cycle
+
+ ! test the eight corners of the current element for a common point with element under study
+ if (mask_ibool(ibool(1,1,1,ispec2)) .or. &
+ mask_ibool(ibool(NGLLX,1,1,ispec2)) .or. &
+ mask_ibool(ibool(NGLLX,NGLLY,1,ispec2)) .or. &
+ mask_ibool(ibool(1,NGLLY,1,ispec2)) .or. &
+ mask_ibool(ibool(1,1,NGLLZ,ispec2)) .or. &
+ mask_ibool(ibool(NGLLX,1,NGLLZ,ispec2)) .or. &
+ mask_ibool(ibool(NGLLX,NGLLY,NGLLZ,ispec2)) .or. &
+ mask_ibool(ibool(1,NGLLY,NGLLZ,ispec2))) &
+ icolor_conflict_found(color(ispec2)) = .true.
+ endif ! domain elements
enddo
- enddo
+
+ ! if color is already above target size for a balanced set, do not move to it
+ do icolor_target = icolormin,icolormax
+ if(nb_elems_in_this_color(icolor_target) >= target_nb_elems_per_color) &
+ icolor_conflict_found(icolor_target) = .true.
+ enddo
+
+ ! if cannot find any other color to move this element to
+ if (all(icolor_conflict_found(icolormin:icolormax))) cycle
+
+ ! loop on all the colors to determine the color with the smallest number of elements
+ ! and for which there is no conflict
+ nb_elems_in_color_chosen = 2147000000 ! start with extremely large unrealistic value
+ icolor_chosen = 0
+ do icolor_target = icolormin,icolormax
+ if (.not. icolor_conflict_found(icolor_target) .and. &
+ nb_elems_in_this_color(icolor_target) < nb_elems_in_color_chosen) then
+ icolor_chosen = icolor_target
+ nb_elems_in_color_chosen = nb_elems_in_this_color(icolor_target)
+ endif
+ enddo
+
+ ! move the element to that new color
+ ! remove element from its current color
+ nb_elems_in_this_color(color(ispec)) = nb_elems_in_this_color(color(ispec)) - 1
+ color(ispec) = icolor_chosen
+ ! and add it to the new color
+ nb_elems_in_this_color(icolor_chosen) = nb_elems_in_this_color(icolor_chosen) + 1
+
+ endif ! domain elements
enddo
- nsum = ninter
- mp(ispec + 1) = nsum
+ enddo ! icolor
+!! DK DK do it for inner elements
+
+ icolormin = nb_colors_outer_elements + 1
+ icolormax = ncolors
+
+ ! ideal value if all colors are perfectly balanced
+ if( nb_colors_inner_elements > 0 ) then
+ target_nb_elems_per_color = nspec_inner / nb_colors_inner_elements + 1
+ else
+ target_nb_elems_per_color = 1
+ endif
+ ! print *,'nspec_inner,target_nb_elems_per_color = ',nspec_inner,target_nb_elems_per_color
+
+ ! count the initial number of elements in each color
+ nb_elems_in_this_color(:) = 0
+ do icolor = icolormin,icolormax
+ nb_elems_in_this_color(icolor) = count(color == icolor)
enddo
- end subroutine form_elt_connectivity_foelco
+ ! do not balance the last one, because it will be balanced automatically by the others
+ do icolor = icolormin,icolormax-1
-!
-!----------------------------------------------------
-!
+ ! if color is already balanced, do nothing
+ ! (this works because in the initial set the number of elements per color decreases when the color number increases)
+ if(nb_elems_in_this_color(icolor) <= target_nb_elems_per_color) cycle
- subroutine form_node_connectivity_fonoco(mn,mp,ne,np,nglob_eight_corners_only,&
-nspec,count_only,total_size_ne)
+ mask_ibool(:) = .false.
+ last_ispec_studied = -1
-!-----------------------------------------------------------------------
-!
-! Forms the NE and NP arrays
-!
-! Input :
-! -------
-! MN, MP, nspec
-! nglob_eight_corners_only Number of nodes in the domain
-!
-! Output :
-! --------
-! NE, NP This is the node-connected element array pair.
-! Integer array NE contains a list of the
-! elements connected to each node, stored in stacked fashion.
-!
-! Array NP is the pointer array for the
-! location of a node's element list in the NE array.
-! Its length is equal to the number of points plus one.
-!
-!-----------------------------------------------------------------------
+ do ispec = 1,nspec
+ ! domain elements only
+ if(ispec_is_d(ispec)) then
- implicit none
+ ! only inner elements
+ if (is_on_a_slice_edge(ispec)) cycle
- include "constants.h"
+ ! only elements of this color
+ if (color(ispec) /= icolor) cycle
-! only count the total size of the array that will be created, or actually create it
- logical count_only
- integer total_size_ne
+ ! if color is now balanced because we have moved enough elements then stop searching
+ if(nb_elems_in_this_color(icolor) <= target_nb_elems_per_color) exit
- integer nglob_eight_corners_only,nspec
+ ! unmark the eight corners of the previously marked element
+ if(last_ispec_studied > 0) then
+ mask_ibool(ibool(1,1,1,last_ispec_studied)) = .false.
+ mask_ibool(ibool(NGLLX,1,1,last_ispec_studied)) = .false.
+ mask_ibool(ibool(NGLLX,NGLLY,1,last_ispec_studied)) = .false.
+ mask_ibool(ibool(1,NGLLY,1,last_ispec_studied)) = .false.
+ mask_ibool(ibool(1,1,NGLLZ,last_ispec_studied)) = .false.
+ mask_ibool(ibool(NGLLX,1,NGLLZ,last_ispec_studied)) = .false.
+ mask_ibool(ibool(NGLLX,NGLLY,NGLLZ,last_ispec_studied)) = .false.
+ mask_ibool(ibool(1,NGLLY,NGLLZ,last_ispec_studied)) = .false.
+ endif
+ icolor_conflict_found(icolormin:icolormax) = .false.
+ icolor_conflict_found(icolor) = .true. ! cannot move element to the color it already has
- integer, intent(in) :: mn(nspec*NGNOD_HEXAHEDRA),mp(nspec+1)
+ ! mark the eight corners of the current element
+ mask_ibool(ibool(1,1,1,ispec)) = .true.
+ mask_ibool(ibool(NGLLX,1,1,ispec)) = .true.
+ mask_ibool(ibool(NGLLX,NGLLY,1,ispec)) = .true.
+ mask_ibool(ibool(1,NGLLY,1,ispec)) = .true.
+ mask_ibool(ibool(1,1,NGLLZ,ispec)) = .true.
+ mask_ibool(ibool(NGLLX,1,NGLLZ,ispec)) = .true.
+ mask_ibool(ibool(NGLLX,NGLLY,NGLLZ,ispec)) = .true.
+ mask_ibool(ibool(1,NGLLY,NGLLZ,ispec)) = .true.
+ last_ispec_studied = ispec
- integer, intent(out) :: ne(total_size_ne),np(nglob_eight_corners_only+1)
+ ! test if we can move this element to another color
+ do ispec2 = 1,nspec
+ ! domain elements only
+ if(ispec_is_d(ispec2)) then
- integer nsum,inode,ispec,j
+ ! do not test that element itself
+ if (ispec2 == ispec) cycle
- nsum = 1
- np(1) = 1
+ ! only inner elements
+ if (is_on_a_slice_edge(ispec2)) cycle
- do inode=1,nglob_eight_corners_only
- do 200 ispec=1,nspec
+ ! if conflict already found previously with this color, no need to test again
+ if (icolor_conflict_found(color(ispec2))) cycle
- do j=mp(ispec),mp(ispec + 1) - 1
- if (mn(j) == inode) then
- if(count_only) then
- total_size_ne = nsum
- else
- ne(nsum) = ispec
- endif
- nsum = nsum + 1
- goto 200
- endif
- enddo
- 200 continue
+ ! test the eight corners of the current element for a common point with element under study
+ if (mask_ibool(ibool(1,1,1,ispec2)) .or. &
+ mask_ibool(ibool(NGLLX,1,1,ispec2)) .or. &
+ mask_ibool(ibool(NGLLX,NGLLY,1,ispec2)) .or. &
+ mask_ibool(ibool(1,NGLLY,1,ispec2)) .or. &
+ mask_ibool(ibool(1,1,NGLLZ,ispec2)) .or. &
+ mask_ibool(ibool(NGLLX,1,NGLLZ,ispec2)) .or. &
+ mask_ibool(ibool(NGLLX,NGLLY,NGLLZ,ispec2)) .or. &
+ mask_ibool(ibool(1,NGLLY,NGLLZ,ispec2))) &
+ icolor_conflict_found(color(ispec2)) = .true.
- np(inode + 1) = nsum
+ endif ! domain elements
+ enddo
- enddo
+ ! if color is already above target size for a balanced set, do not move to it
+ do icolor_target = icolormin,icolormax
+ if(nb_elems_in_this_color(icolor_target) >= target_nb_elems_per_color) &
+ icolor_conflict_found(icolor_target) = .true.
+ enddo
- end subroutine form_node_connectivity_fonoco
+ ! if cannot find any other color to move this element to
+ if (all(icolor_conflict_found(icolormin:icolormax))) cycle
-!
-!----------------------------------------------------
-!
+ ! loops on all the colors to determine the color with the smallest number of elements
+ ! and for which there is no conflict
+ nb_elems_in_color_chosen = 2147000000 ! start with extremely large unrealistic value
+ icolor_chosen = 0
+ do icolor_target = icolormin,icolormax
+ if (.not. icolor_conflict_found(icolor_target) .and. &
+ nb_elems_in_this_color(icolor_target) < nb_elems_in_color_chosen) then
+ icolor_chosen = icolor_target
+ nb_elems_in_color_chosen = nb_elems_in_this_color(icolor_target)
+ endif
+ enddo
- !subroutine create_adjacency_table_adjncy(mn,mp,ne,np,adj,xadj,maskel,nspec,nglob_eight_corners_only,&
- ! count_only,total_size_ne,total_size_adj,face)
- subroutine create_adjacency_table_adjncy(mn,mp,ne,np,adj,xadj,nspec,nglob_eight_corners_only,&
- count_only,total_size_ne,total_size_adj,face)
+ ! moves the element to that new color
+ ! remove element from its current color
+ nb_elems_in_this_color(color(ispec)) = nb_elems_in_this_color(color(ispec)) - 1
+ color(ispec) = icolor_chosen
+ ! and add it to the new color
+ nb_elems_in_this_color(icolor_chosen) = nb_elems_in_this_color(icolor_chosen) + 1
-!-----------------------------------------------------------------------
-!
-! Establishes the element adjacency information of the mesh
-! Two elements are considered adjacent if they share a face.
-!
-! Input :
-! -------
-! MN, MP, NE, NP, nspec
-! MASKEL logical mask (length = nspec)
-!
-! Output :
-! --------
-! ADJ, XADJ This is the element adjacency array pair. Array
-! ADJ contains the list of the elements adjacent to
-! element i. They are stored in a stacked fashion.
-! Pointer array XADJ stores the location of each element list.
-!
-!-----------------------------------------------------------------------
+ endif ! domain elements
+ enddo
- implicit none
+ enddo ! icolor
- include "constants.h"
+ end subroutine balance_colors_simple
-! only count the total size of the array that will be created, or actually create it
- logical count_only,face
- integer total_size_ne,total_size_adj
+!
+!-------------------------------------------------------------------------------------------------
+!
- integer nglob_eight_corners_only
+ subroutine get_final_perm(color,perm,first_elem_number_in_this_color, &
+ nspec,nb_colors,nb_colors_outer_elements, &
+ ispec_is_d,nspec_domain)
- integer nspec,iad,ispec,istart,istop,ino,node,jstart,jstop,nelem,jel
+ integer, intent(in) :: nspec,nb_colors
- integer, intent(in) :: mn(nspec*NGNOD_HEXAHEDRA),mp(nspec+1),ne(total_size_ne),np(nglob_eight_corners_only+1)
+ integer,dimension(nspec), intent(in) :: color
+ integer,dimension(nspec), intent(inout) :: perm
- integer, intent(out) :: adj(total_size_adj),xadj(nspec+1)
+ integer, intent(inout) :: first_elem_number_in_this_color(nb_colors)
- logical maskel(nspec)
- integer countel(nspec)
+ logical,dimension(nspec),intent(in) :: ispec_is_d
- xadj(1) = 1
- iad = 1
+ integer,intent(in) :: nb_colors_outer_elements,nspec_domain
- do ispec=1,nspec
+ ! local parameters
+ integer :: ispec,icolor,icounter,counter_outer
-! reset mask
- maskel(:) = .false.
+ ! note: permutations are only valid within each domain
+ ! also, the counters start at 1 for each inner/outer element range
-! mask current element
- maskel(ispec) = .true.
- if (face) countel(:) = 0
+ ! outer elements first ( note: inner / outer order sensitive)
+ icounter = 1
+ do icolor = 1, nb_colors_outer_elements
+ first_elem_number_in_this_color(icolor) = icounter
+ do ispec = 1, nspec
+ ! elements in this domain only
+ if( ispec_is_d(ispec) ) then
+ if(color(ispec) == icolor) then
+ perm(ispec) = icounter
+ icounter = icounter + 1
+ endif
+ endif
+ enddo
+ enddo
+ counter_outer = icounter - 1
- istart = mp(ispec)
- istop = mp(ispec+1) - 1
- do ino=istart,istop
- node = mn(ino)
- jstart = np(node)
- jstop = np(node + 1) - 1
- do 120 jel=jstart,jstop
- nelem = ne(jel)
- if(maskel(nelem)) goto 120
- if (face) then
- ! if 2 elements share at least 3 corners, therefore they share a face
- countel(nelem) = countel(nelem) + 1
- if (countel(nelem)>=3) then
- if(count_only) then
- total_size_adj = iad
- else
- adj(iad) = nelem
- endif
- maskel(nelem) = .true.
- iad = iad + 1
- endif
- else
- if(count_only) then
- total_size_adj = iad
- else
- adj(iad) = nelem
- endif
- maskel(nelem) = .true.
- iad = iad + 1
- endif
- 120 continue
+ ! inner elements second
+ icounter = 1
+ do icolor = nb_colors_outer_elements+1, nb_colors
+ first_elem_number_in_this_color(icolor) = icounter + counter_outer
+ do ispec = 1, nspec
+ ! elements in this domain only
+ if( ispec_is_d(ispec) ) then
+ ! outer elements
+ if(color(ispec) == icolor) then
+ perm(ispec) = icounter
+ icounter = icounter + 1
+ endif
+ endif
enddo
+ enddo
- xadj(ispec+1) = iad
+ ! checks
+ if( counter_outer + icounter -1 /= nspec_domain ) then
+ print*,'error: perm: ',nspec_domain,counter_outer,icounter,counter_outer+icounter-1
+ stop 'error get_final_perm: counter incomplete'
+ endif
- enddo
+ end subroutine get_final_perm
- end subroutine create_adjacency_table_adjncy
+!
+!-------------------------------------------------------------------------------------------------
+!
+! PERMUTATIONS
+!
+!-------------------------------------------------------------------------------------------------
+!
+! implement permutation of elements for arrays of real (CUSTOM_REAL) type
+
subroutine permute_elements_real(array_to_permute,temp_array,perm,nspec)
implicit none
@@ -740,11 +1106,12 @@
integer, intent(in) :: nspec
integer, intent(in), dimension(nspec) :: perm
- real(kind=CUSTOM_REAL), intent(inout), dimension(NGLLX,NGLLY,NGLLZ,nspec) :: array_to_permute,temp_array
+ real(kind=CUSTOM_REAL), intent(inout), dimension(NGLLX,NGLLY,NGLLZ,nspec) :: &
+ array_to_permute,temp_array
integer old_ispec,new_ispec
-! copy the original array
+ ! copy the original array
temp_array(:,:,:,:) = array_to_permute(:,:,:,:)
do old_ispec = 1,nspec
@@ -755,10 +1122,11 @@
end subroutine permute_elements_real
!
-!-----------------------------------------------------------------------
+!-------------------------------------------------------------------------------------------------
!
! implement permutation of elements for arrays of integer type
+
subroutine permute_elements_integer(array_to_permute,temp_array,perm,nspec)
implicit none
@@ -768,11 +1136,12 @@
integer, intent(in) :: nspec
integer, intent(in), dimension(nspec) :: perm
- integer, intent(inout), dimension(NGLLX,NGLLY,NGLLZ,nspec) :: array_to_permute,temp_array
+ integer, intent(inout), dimension(NGLLX,NGLLY,NGLLZ,nspec) :: &
+ array_to_permute,temp_array
integer old_ispec,new_ispec
-! copy the original array
+ ! copy the original array
temp_array(:,:,:,:) = array_to_permute(:,:,:,:)
do old_ispec = 1,nspec
@@ -783,10 +1152,11 @@
end subroutine permute_elements_integer
!
-!-----------------------------------------------------------------------
+!-------------------------------------------------------------------------------------------------
!
! implement permutation of elements for arrays of double precision type
+
subroutine permute_elements_dble(array_to_permute,temp_array,perm,nspec)
implicit none
@@ -796,11 +1166,12 @@
integer, intent(in) :: nspec
integer, intent(in), dimension(nspec) :: perm
- double precision, intent(inout), dimension(NGLLX,NGLLY,NGLLZ,nspec) :: array_to_permute,temp_array
+ double precision, intent(inout), dimension(NGLLX,NGLLY,NGLLZ,nspec) :: &
+ array_to_permute,temp_array
integer old_ispec,new_ispec
-! copy the original array
+ ! copy the original array
temp_array(:,:,:,:) = array_to_permute(:,:,:,:)
do old_ispec = 1,nspec
@@ -809,3 +1180,32 @@
enddo
end subroutine permute_elements_dble
+
+!
+!-------------------------------------------------------------------------------------------------
+!
+
+! implement permutation of elements for arrays of double precision type
+
+ subroutine permute_elements_logical1D(array_to_permute,temp_array,perm,nspec)
+
+ implicit none
+
+ include "constants.h"
+
+ integer, intent(in) :: nspec
+ integer, intent(in), dimension(nspec) :: perm
+
+ logical, intent(inout), dimension(nspec) :: array_to_permute,temp_array
+
+ integer old_ispec,new_ispec
+
+ ! copy the original array
+ temp_array(:) = array_to_permute(:)
+
+ do old_ispec = 1,nspec
+ new_ispec = perm(old_ispec)
+ array_to_permute(new_ispec) = temp_array(old_ispec)
+ enddo
+
+ end subroutine permute_elements_logical1D
Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/meshfem3D/initialize_mesher.f90
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/meshfem3D/initialize_mesher.f90 2012-07-19 21:55:39 UTC (rev 20529)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/meshfem3D/initialize_mesher.f90 2012-07-19 22:05:23 UTC (rev 20530)
@@ -28,23 +28,22 @@
subroutine initialize_mesher()
use meshfem3D_par
+ use meshfem3D_models_par
+
implicit none
- ! standard include of the MPI library
- include 'mpif.h'
-
! local parameters
- integer :: ier
+ integer, external :: err_occurred
! timing
double precision, external :: wtime
-
+
! sizeprocs returns number of processes started (should be equal to NPROCTOT).
! myrank is the rank of each process, between 0 and NPROCTOT-1.
! as usual in MPI, process 0 is in charge of coordinating everything
! and also takes care of the main output
! do not create anything for the inner core here, will be done in solver
- call MPI_COMM_SIZE(MPI_COMM_WORLD,sizeprocs,ier)
- call MPI_COMM_RANK(MPI_COMM_WORLD,myrank,ier)
+ call world_size(sizeprocs)
+ call world_rank(myrank)
! get the base pathname for output files
call get_value_string(OUTPUT_FILES, 'OUTPUT_FILES', 'OUTPUT_FILES')
Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/meshfem3D/meshfem3D_models.f90
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/meshfem3D/meshfem3D_models.f90 2012-07-19 21:55:39 UTC (rev 20529)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/meshfem3D/meshfem3D_models.f90 2012-07-19 22:05:23 UTC (rev 20530)
@@ -25,431 +25,7 @@
!
!=====================================================================
-module constants
- include "constants.h"
-
-end module constants
-
-!
-!-------------------------------------------------------------------------------------------------
-!
-
-module meshfem3D_models_par
-
-!---
-!
-! ADD YOUR MODEL HERE
-!
-!---
-
- use constants
-
- implicit none
-
-
-! model_aniso_mantle_variables
- type model_aniso_mantle_variables
- sequence
- double precision beta(14,34,37,73)
- double precision pro(47)
- integer npar1
- integer dummy_pad ! padding 4 bytes to align the structure
- end type model_aniso_mantle_variables
- type (model_aniso_mantle_variables) AMM_V
-! model_aniso_mantle_variables
-
-! model_attenuation_variables
- type model_attenuation_variables
- sequence
- double precision min_period, max_period
- double precision :: QT_c_source ! Source Frequency
- double precision, dimension(:), pointer :: Qtau_s ! tau_sigma
- double precision, dimension(:), pointer :: QrDisc ! Discontinutitues Defined
- double precision, dimension(:), pointer :: Qr ! Radius
- double precision, dimension(:), pointer :: Qmu ! Shear Attenuation
- double precision, dimension(:,:), pointer :: Qtau_e ! tau_epsilon
- double precision, dimension(:), pointer :: Qomsb, Qomsb2 ! one_minus_sum_beta
- double precision, dimension(:,:), pointer :: Qfc, Qfc2 ! factor_common
- double precision, dimension(:), pointer :: Qsf, Qsf2 ! scale_factor
- integer, dimension(:), pointer :: Qrmin ! Max and Mins of idoubling
- integer, dimension(:), pointer :: Qrmax ! Max and Mins of idoubling
- integer, dimension(:), pointer :: interval_Q ! Steps
- integer :: Qn ! Number of points
- integer dummy_pad ! padding 4 bytes to align the structure
- end type model_attenuation_variables
- type (model_attenuation_variables) AM_V
-! model_attenuation_variables
-
-! model_atten3D_QRFSI12_variables
- type model_atten3D_QRFSI12_variables
- sequence
- double precision dqmu(NKQ,NSQ)
- double precision spknt(NKQ)
- double precision refdepth(NDEPTHS_REFQ)
- double precision refqmu(NDEPTHS_REFQ)
- end type model_atten3D_QRFSI12_variables
- type (model_atten3D_QRFSI12_variables) QRFSI12_Q
-! model_atten3D_QRFSI12_variables
-
-! model_1066a_variables
- type model_1066a_variables
- sequence
- double precision, dimension(NR_1066A) :: radius_1066a
- double precision, dimension(NR_1066A) :: density_1066a
- double precision, dimension(NR_1066A) :: vp_1066a
- double precision, dimension(NR_1066A) :: vs_1066a
- double precision, dimension(NR_1066A) :: Qkappa_1066a
- double precision, dimension(NR_1066A) :: Qmu_1066a
- end type model_1066a_variables
- type (model_1066a_variables) M1066a_V
-! model_1066a_variables
-
-! model_ak135_variables
- type model_ak135_variables
- sequence
- double precision, dimension(NR_AK135) :: radius_ak135
- double precision, dimension(NR_AK135) :: density_ak135
- double precision, dimension(NR_AK135) :: vp_ak135
- double precision, dimension(NR_AK135) :: vs_ak135
- double precision, dimension(NR_AK135) :: Qkappa_ak135
- double precision, dimension(NR_AK135) :: Qmu_ak135
- end type model_ak135_variables
- type (model_ak135_variables) Mak135_V
-! model_ak135_variables
-
-! model_1dref_variables
- type model_1dref_variables
- sequence
- double precision, dimension(NR_REF) :: radius_ref
- double precision, dimension(NR_REF) :: density_ref
- double precision, dimension(NR_REF) :: vpv_ref
- double precision, dimension(NR_REF) :: vph_ref
- double precision, dimension(NR_REF) :: vsv_ref
- double precision, dimension(NR_REF) :: vsh_ref
- double precision, dimension(NR_REF) :: eta_ref
- double precision, dimension(NR_REF) :: Qkappa_ref
- double precision, dimension(NR_REF) :: Qmu_ref
- end type model_1dref_variables
- type (model_1dref_variables) Mref_V
-! model_1dref_variables
-
-! model_sea1d_variables
- type model_sea1d_variables
- sequence
- double precision, dimension(NR_SEA1D) :: radius_sea1d
- double precision, dimension(NR_SEA1D) :: density_sea1d
- double precision, dimension(NR_SEA1D) :: vp_sea1d
- double precision, dimension(NR_SEA1D) :: vs_sea1d
- double precision, dimension(NR_SEA1D) :: Qkappa_sea1d
- double precision, dimension(NR_SEA1D) :: Qmu_sea1d
- end type model_sea1d_variables
- type (model_sea1d_variables) SEA1DM_V
-! model_sea1d_variables
-
-! model_s20rts_variables
- type model_s20rts_variables
- sequence
- double precision dvs_a(0:NK_20,0:NS_20,0:NS_20) !a = positive m (radial, theta, phi) --> (k,l,m) (maybe other way around??)
- double precision dvs_b(0:NK_20,0:NS_20,0:NS_20) !b = negative m (radial, theta, phi) --> (k,l,-m)
- double precision dvp_a(0:NK_20,0:NS_20,0:NS_20)
- double precision dvp_b(0:NK_20,0:NS_20,0:NS_20)
- double precision spknt(NK_20+1)
- double precision qq0(NK_20+1,NK_20+1)
- double precision qq(3,NK_20+1,NK_20+1)
- end type model_s20rts_variables
- type (model_s20rts_variables) S20RTS_V
-! model_s20rts_variables
-
-! model_s40rts_variables
- type model_s40rts_variables
- sequence
- double precision dvs_a(0:NK_20,0:NS_40,0:NS_40)
- double precision dvs_b(0:NK_20,0:NS_40,0:NS_40)
- double precision dvp_a(0:NK_20,0:NS_40,0:NS_40)
- double precision dvp_b(0:NK_20,0:NS_40,0:NS_40)
- double precision spknt(NK_20+1)
- double precision qq0(NK_20+1,NK_20+1)
- double precision qq(3,NK_20+1,NK_20+1)
- end type model_s40rts_variables
- type (model_s40rts_variables) S40RTS_V
-! model_s40rts_variables
-
-! model_heterogen_m_variables
- type model_heterogen_m_variables
- sequence
- double precision rho_in(N_R*N_THETA*N_PHI)
- end type model_heterogen_m_variables
- type (model_heterogen_m_variables) HMM
-! model_heterogen_m_variables
-
-! model_jp3d_variables
- type model_jp3d_variables
- sequence
- ! vmod3d
- double precision :: PNA(MPA)
- double precision :: RNA(MRA)
- double precision :: HNA(MHA)
- double precision :: PNB(MPB)
- double precision :: RNB(MRB)
- double precision :: HNB(MHB)
- double precision :: VELAP(MPA,MRA,MHA)
- double precision :: VELBP(MPB,MRB,MHB)
- ! discon
- double precision :: PN(51)
- double precision :: RRN(63)
- double precision :: DEPA(51,63)
- double precision :: DEPB(51,63)
- double precision :: DEPC(51,63)
- ! locate
- double precision :: PLA
- double precision :: RLA
- double precision :: HLA
- double precision :: PLB
- double precision :: RLB
- double precision :: HLB
- ! weight
- double precision :: WV(8)
- ! prhfd
- double precision :: P
- double precision :: R
- double precision :: H
- double precision :: PF
- double precision :: RF
- double precision :: HF
- double precision :: PF1
- double precision :: RF1
- double precision :: HF1
- double precision :: PD
- double precision :: RD
- double precision :: HD
- ! jpmodv
- double precision :: VP(29)
- double precision :: VS(29)
- double precision :: RA(29)
- double precision :: DEPJ(29)
- ! locate integers
- integer :: IPLOCA(MKA)
- integer :: IRLOCA(MKA)
- integer :: IHLOCA(MKA)
- integer :: IPLOCB(MKB)
- integer :: IRLOCB(MKB)
- integer :: IHLOCB(MKB)
- ! vmod3D integers
- integer :: NPA
- integer :: NRA
- integer :: NHA
- integer :: NPB
- integer :: NRB
- integer :: NHB
- ! weight integers
- integer :: IP
- integer :: JP
- integer :: KP
- integer :: IP1
- integer :: JP1
- integer :: KP1
- end type model_jp3d_variables
- type (model_jp3d_variables) JP3DM_V
-! model_jp3d_variables
-
-! model_sea99_s_variables
- type model_sea99_s_variables
- sequence
- double precision :: sea99_vs(100,100,100)
- double precision :: sea99_depth(100)
- double precision :: sea99_ddeg
- double precision :: alatmin
- double precision :: alatmax
- double precision :: alonmin
- double precision :: alonmax
- integer :: sea99_ndep
- integer :: sea99_nlat
- integer :: sea99_nlon
- integer :: dummy_pad ! padding 4 bytes to align the structure
- end type model_sea99_s_variables
- type (model_sea99_s_variables) SEA99M_V
-! model_sea99_s_variables
-
-! crust 2.0 model_crust_variables
- type model_crust_variables
- sequence
- double precision, dimension(NKEYS_CRUST,NLAYERS_CRUST) :: thlr
- double precision, dimension(NKEYS_CRUST,NLAYERS_CRUST) :: velocp
- double precision, dimension(NKEYS_CRUST,NLAYERS_CRUST) :: velocs
- double precision, dimension(NKEYS_CRUST,NLAYERS_CRUST) :: dens
- character(len=2) abbreviation(NCAP_CRUST/2,NCAP_CRUST)
- character(len=2) code(NKEYS_CRUST)
- character(len=2) dummy_pad ! padding 2 bytes to align the structure
- end type model_crust_variables
- type (model_crust_variables) CM_V
-! model_crust_variables
-
-! EUcrust
- type model_eucrust_variables
- sequence
- double precision, dimension(:),pointer :: eucrust_lat,eucrust_lon,&
- eucrust_vp_uppercrust,eucrust_vp_lowercrust,eucrust_mohodepth,&
- eucrust_basement,eucrust_ucdepth
- integer :: num_eucrust
- integer :: dummy_pad ! padding 4 bytes to align the structure
- end type model_eucrust_variables
- type (model_eucrust_variables) EUCM_V
-
-! type for EPCRUST 1.0
- type model_epcrust_variables
- sequence
- double precision,dimension(EPCRUST_NLON,EPCRUST_NLAT):: lon_ep,lat_ep,topo_ep
- double precision,dimension(EPCRUST_NLON,EPCRUST_NLAT,EPCRUST_NLAYER):: thickness_ep
- double precision,dimension(EPCRUST_NLON,EPCRUST_NLAT,EPCRUST_NLAYER):: vp_ep
- double precision,dimension(EPCRUST_NLON,EPCRUST_NLAT,EPCRUST_NLAYER):: vs_ep
- double precision,dimension(EPCRUST_NLON,EPCRUST_NLAT,EPCRUST_NLAYER):: rho_ep
- end type model_epcrust_variables
- type (model_epcrust_variables) EPCRUST
-
-! model_crustmaps_variables combined crustal maps
- type model_crustmaps_variables
- sequence
- double precision, dimension(180*CRUSTMAP_RESOLUTION,360*CRUSTMAP_RESOLUTION,NLAYERS_CRUSTMAP) :: thickness
- double precision, dimension(180*CRUSTMAP_RESOLUTION,360*CRUSTMAP_RESOLUTION,NLAYERS_CRUSTMAP) :: density
- double precision, dimension(180*CRUSTMAP_RESOLUTION,360*CRUSTMAP_RESOLUTION,NLAYERS_CRUSTMAP) :: velocp
- double precision, dimension(180*CRUSTMAP_RESOLUTION,360*CRUSTMAP_RESOLUTION,NLAYERS_CRUSTMAP) :: velocs
- double precision thicknessnp(NLAYERS_CRUSTMAP)
- double precision densitynp(NLAYERS_CRUSTMAP)
- double precision velocpnp(NLAYERS_CRUSTMAP)
- double precision velocsnp(NLAYERS_CRUSTMAP)
- double precision thicknesssp(NLAYERS_CRUSTMAP)
- double precision densitysp(NLAYERS_CRUSTMAP)
- double precision velocpsp(NLAYERS_CRUSTMAP)
- double precision velocssp(NLAYERS_CRUSTMAP)
- end type model_crustmaps_variables
- type (model_crustmaps_variables) GC_V
-!model_crustmaps_variables
-
-! model_attenuation_storage_var
- type model_attenuation_storage_var
- sequence
- double precision, dimension(:,:), pointer :: tau_e_storage
- double precision, dimension(:), pointer :: Qmu_storage
- integer Q_resolution
- integer Q_max
- end type model_attenuation_storage_var
- type (model_attenuation_storage_var) AM_S
-! model_attenuation_storage_var
-
-! attenuation_simplex_variables
- type attenuation_simplex_variables
- sequence
- double precision Q ! Q = Desired Value of Attenuation or Q
- double precision iQ ! iQ = 1/Q
- double precision, dimension(:), pointer :: f
- ! f = Frequencies at which to evaluate the solution
- double precision, dimension(:), pointer :: tau_s
- ! tau_s = Tau_sigma defined by the frequency range and
- ! number of standard linear solids
- integer nf ! nf = Number of Frequencies
- integer nsls ! nsls = Number of Standard Linear Solids
- end type attenuation_simplex_variables
- type(attenuation_simplex_variables) AS_V
-! attenuation_simplex_variables
-
-! point profile model_variables
- type model_ppm_variables
- sequence
- double precision,dimension(:),pointer :: dvs,lat,lon,depth
- double precision :: maxlat,maxlon,minlat,minlon,maxdepth,mindepth
- double precision :: dlat,dlon,ddepth,max_dvs,min_dvs
- integer :: num_v,num_latperlon,num_lonperdepth
- integer :: dummy_pad ! padding 4 bytes to align the structure
- end type model_ppm_variables
- type (model_ppm_variables) PPM_V
-
-! GLL model_variables
- type model_gll_variables
- sequence
- ! tomographic iteration model on GLL points
- double precision :: scale_velocity,scale_density
- ! isotropic model
- real(kind=CUSTOM_REAL),dimension(:,:,:,:),pointer :: vs_new,vp_new,rho_new
- ! transverse isotropic model
- real(kind=CUSTOM_REAL),dimension(:,:,:,:),pointer :: vsv_new,vpv_new, &
- vsh_new,vph_new,eta_new
- logical :: MODEL_GLL
- logical,dimension(3) :: dummy_pad ! padding 3 bytes to align the structure
- end type model_gll_variables
- type (model_gll_variables) MGLL_V
-
-! bathymetry and topography: use integer array to store values
- integer, dimension(NX_BATHY,NY_BATHY) :: ibathy_topo
-
-! used for 3D Harvard models s362ani, s362wmani, s362ani_prem and s2.9ea
- integer, parameter :: maxker=200
- integer, parameter :: maxl=72
- integer, parameter :: maxcoe=2000
- integer, parameter :: maxver=1000
- integer, parameter :: maxhpa=2
-
- real(kind=4) conpt(maxver,maxhpa)
- real(kind=4) xlaspl(maxcoe,maxhpa)
- real(kind=4) xlospl(maxcoe,maxhpa)
- real(kind=4) radspl(maxcoe,maxhpa)
- real(kind=4) coe(maxcoe,maxker)
- real(kind=4) vercof(maxker)
- real(kind=4) vercofd(maxker)
-
- real(kind=4) ylmcof((maxl+1)**2,maxhpa)
- real(kind=4) wk1(maxl+1)
- real(kind=4) wk2(maxl+1)
- real(kind=4) wk3(maxl+1)
-
- integer lmxhpa(maxhpa)
- integer itypehpa(maxhpa)
- integer ihpakern(maxker)
- integer numcoe(maxhpa)
- integer ivarkern(maxker)
- integer itpspl(maxcoe,maxhpa)
-
- integer nconpt(maxhpa),iver
- integer iconpt(maxver,maxhpa)
- integer numker
- integer numhpa,numcof
- integer ihpa,lmax,nylm
-
- character(len=80) kerstr
- character(len=80) refmdl
- character(len=40) varstr(maxker)
- character(len=80) hsplfl(maxhpa)
- character(len=40) dskker(maxker)
-
-
-! for ellipticity
- double precision rspl(NR),espl(NR),espl2(NR)
- integer nspl
-
-! model parameter and flags
- integer REFERENCE_1D_MODEL,THREE_D_MODEL
-
- logical ELLIPTICITY,GRAVITY,ROTATION,TOPOGRAPHY,OCEANS
-
- logical HONOR_1D_SPHERICAL_MOHO,CRUSTAL,ONE_CRUST,CASE_3D,TRANSVERSE_ISOTROPY
-
- logical ISOTROPIC_3D_MANTLE,ANISOTROPIC_3D_MANTLE,HETEROGEN_3D_MANTLE
-
- logical ATTENUATION,ATTENUATION_NEW,ATTENUATION_3D
-
- logical ANISOTROPIC_INNER_CORE
-
-! to create a reference model based on 1D_REF but with 3D crust and 410/660 topography
- logical,parameter :: USE_1D_REFERENCE = .false.
-
-end module meshfem3D_models_par
-
-
-!
-!-------------------------------------------------------------------------------------------------
-!
-
-
subroutine meshfem3D_models_broadcast(myrank,NSPEC, &
MIN_ATTENUATION_PERIOD,MAX_ATTENUATION_PERIOD,&
R80,R220,R670,RCMB,RICB, &
@@ -461,9 +37,6 @@
implicit none
-! standard include of the MPI library
- include 'mpif.h'
-
integer myrank
integer, dimension(MAX_NUM_REGIONS) :: NSPEC
@@ -612,9 +185,6 @@
implicit none
-! standard include of the MPI library
- include 'mpif.h'
-
integer myrank
@@ -1181,9 +751,6 @@
implicit none
-! standard include of the MPI library
- include 'mpif.h'
-
!integer myrank
double precision,intent(in) :: lat,lon,r
double precision,intent(out) :: vpc,vsc,rhoc
Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/meshfem3D/meshfem3D_par.f90
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/meshfem3D/meshfem3D_par.f90 2012-07-19 21:55:39 UTC (rev 20529)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/meshfem3D/meshfem3D_par.f90 2012-07-19 22:05:23 UTC (rev 20530)
@@ -27,24 +27,447 @@
!
! United States and French Government Sponsorship Acknowledged.
+module constants
+
+ include "constants.h"
+
+end module constants
+
+!
+!-------------------------------------------------------------------------------------------------
+!
+
+module meshfem3D_models_par
+
+!---
+!
+! ADD YOUR MODEL HERE
+!
+!---
+
+ use constants
+
+ implicit none
+
+
+! model_aniso_mantle_variables
+ type model_aniso_mantle_variables
+ sequence
+ double precision beta(14,34,37,73)
+ double precision pro(47)
+ integer npar1
+ integer dummy_pad ! padding 4 bytes to align the structure
+ end type model_aniso_mantle_variables
+ type (model_aniso_mantle_variables) AMM_V
+! model_aniso_mantle_variables
+
+! model_attenuation_variables
+ type model_attenuation_variables
+ sequence
+ double precision min_period, max_period
+ double precision :: QT_c_source ! Source Frequency
+ double precision, dimension(:), pointer :: Qtau_s ! tau_sigma
+ double precision, dimension(:), pointer :: QrDisc ! Discontinutitues Defined
+ double precision, dimension(:), pointer :: Qr ! Radius
+ double precision, dimension(:), pointer :: Qmu ! Shear Attenuation
+ double precision, dimension(:,:), pointer :: Qtau_e ! tau_epsilon
+ double precision, dimension(:), pointer :: Qomsb, Qomsb2 ! one_minus_sum_beta
+ double precision, dimension(:,:), pointer :: Qfc, Qfc2 ! factor_common
+ double precision, dimension(:), pointer :: Qsf, Qsf2 ! scale_factor
+ integer, dimension(:), pointer :: Qrmin ! Max and Mins of idoubling
+ integer, dimension(:), pointer :: Qrmax ! Max and Mins of idoubling
+ integer, dimension(:), pointer :: interval_Q ! Steps
+ integer :: Qn ! Number of points
+ integer dummy_pad ! padding 4 bytes to align the structure
+ end type model_attenuation_variables
+ type (model_attenuation_variables) AM_V
+! model_attenuation_variables
+
+! model_atten3D_QRFSI12_variables
+ type model_atten3D_QRFSI12_variables
+ sequence
+ double precision dqmu(NKQ,NSQ)
+ double precision spknt(NKQ)
+ double precision refdepth(NDEPTHS_REFQ)
+ double precision refqmu(NDEPTHS_REFQ)
+ end type model_atten3D_QRFSI12_variables
+ type (model_atten3D_QRFSI12_variables) QRFSI12_Q
+! model_atten3D_QRFSI12_variables
+
+! model_1066a_variables
+ type model_1066a_variables
+ sequence
+ double precision, dimension(NR_1066A) :: radius_1066a
+ double precision, dimension(NR_1066A) :: density_1066a
+ double precision, dimension(NR_1066A) :: vp_1066a
+ double precision, dimension(NR_1066A) :: vs_1066a
+ double precision, dimension(NR_1066A) :: Qkappa_1066a
+ double precision, dimension(NR_1066A) :: Qmu_1066a
+ end type model_1066a_variables
+ type (model_1066a_variables) M1066a_V
+! model_1066a_variables
+
+! model_ak135_variables
+ type model_ak135_variables
+ sequence
+ double precision, dimension(NR_AK135) :: radius_ak135
+ double precision, dimension(NR_AK135) :: density_ak135
+ double precision, dimension(NR_AK135) :: vp_ak135
+ double precision, dimension(NR_AK135) :: vs_ak135
+ double precision, dimension(NR_AK135) :: Qkappa_ak135
+ double precision, dimension(NR_AK135) :: Qmu_ak135
+ end type model_ak135_variables
+ type (model_ak135_variables) Mak135_V
+! model_ak135_variables
+
+! model_1dref_variables
+ type model_1dref_variables
+ sequence
+ double precision, dimension(NR_REF) :: radius_ref
+ double precision, dimension(NR_REF) :: density_ref
+ double precision, dimension(NR_REF) :: vpv_ref
+ double precision, dimension(NR_REF) :: vph_ref
+ double precision, dimension(NR_REF) :: vsv_ref
+ double precision, dimension(NR_REF) :: vsh_ref
+ double precision, dimension(NR_REF) :: eta_ref
+ double precision, dimension(NR_REF) :: Qkappa_ref
+ double precision, dimension(NR_REF) :: Qmu_ref
+ end type model_1dref_variables
+ type (model_1dref_variables) Mref_V
+! model_1dref_variables
+
+! model_sea1d_variables
+ type model_sea1d_variables
+ sequence
+ double precision, dimension(NR_SEA1D) :: radius_sea1d
+ double precision, dimension(NR_SEA1D) :: density_sea1d
+ double precision, dimension(NR_SEA1D) :: vp_sea1d
+ double precision, dimension(NR_SEA1D) :: vs_sea1d
+ double precision, dimension(NR_SEA1D) :: Qkappa_sea1d
+ double precision, dimension(NR_SEA1D) :: Qmu_sea1d
+ end type model_sea1d_variables
+ type (model_sea1d_variables) SEA1DM_V
+! model_sea1d_variables
+
+! model_s20rts_variables
+ type model_s20rts_variables
+ sequence
+ double precision dvs_a(0:NK_20,0:NS_20,0:NS_20) !a = positive m (radial, theta, phi) --> (k,l,m) (maybe other way around??)
+ double precision dvs_b(0:NK_20,0:NS_20,0:NS_20) !b = negative m (radial, theta, phi) --> (k,l,-m)
+ double precision dvp_a(0:NK_20,0:NS_20,0:NS_20)
+ double precision dvp_b(0:NK_20,0:NS_20,0:NS_20)
+ double precision spknt(NK_20+1)
+ double precision qq0(NK_20+1,NK_20+1)
+ double precision qq(3,NK_20+1,NK_20+1)
+ end type model_s20rts_variables
+ type (model_s20rts_variables) S20RTS_V
+! model_s20rts_variables
+
+! model_s40rts_variables
+ type model_s40rts_variables
+ sequence
+ double precision dvs_a(0:NK_20,0:NS_40,0:NS_40)
+ double precision dvs_b(0:NK_20,0:NS_40,0:NS_40)
+ double precision dvp_a(0:NK_20,0:NS_40,0:NS_40)
+ double precision dvp_b(0:NK_20,0:NS_40,0:NS_40)
+ double precision spknt(NK_20+1)
+ double precision qq0(NK_20+1,NK_20+1)
+ double precision qq(3,NK_20+1,NK_20+1)
+ end type model_s40rts_variables
+ type (model_s40rts_variables) S40RTS_V
+! model_s40rts_variables
+
+! model_heterogen_m_variables
+ type model_heterogen_m_variables
+ sequence
+ double precision rho_in(N_R*N_THETA*N_PHI)
+ end type model_heterogen_m_variables
+ type (model_heterogen_m_variables) HMM
+! model_heterogen_m_variables
+
+! model_jp3d_variables
+ type model_jp3d_variables
+ sequence
+ ! vmod3d
+ double precision :: PNA(MPA)
+ double precision :: RNA(MRA)
+ double precision :: HNA(MHA)
+ double precision :: PNB(MPB)
+ double precision :: RNB(MRB)
+ double precision :: HNB(MHB)
+ double precision :: VELAP(MPA,MRA,MHA)
+ double precision :: VELBP(MPB,MRB,MHB)
+ ! discon
+ double precision :: PN(51)
+ double precision :: RRN(63)
+ double precision :: DEPA(51,63)
+ double precision :: DEPB(51,63)
+ double precision :: DEPC(51,63)
+ ! locate
+ double precision :: PLA
+ double precision :: RLA
+ double precision :: HLA
+ double precision :: PLB
+ double precision :: RLB
+ double precision :: HLB
+ ! weight
+ double precision :: WV(8)
+ ! prhfd
+ double precision :: P
+ double precision :: R
+ double precision :: H
+ double precision :: PF
+ double precision :: RF
+ double precision :: HF
+ double precision :: PF1
+ double precision :: RF1
+ double precision :: HF1
+ double precision :: PD
+ double precision :: RD
+ double precision :: HD
+ ! jpmodv
+ double precision :: VP(29)
+ double precision :: VS(29)
+ double precision :: RA(29)
+ double precision :: DEPJ(29)
+ ! locate integers
+ integer :: IPLOCA(MKA)
+ integer :: IRLOCA(MKA)
+ integer :: IHLOCA(MKA)
+ integer :: IPLOCB(MKB)
+ integer :: IRLOCB(MKB)
+ integer :: IHLOCB(MKB)
+ ! vmod3D integers
+ integer :: NPA
+ integer :: NRA
+ integer :: NHA
+ integer :: NPB
+ integer :: NRB
+ integer :: NHB
+ ! weight integers
+ integer :: IP
+ integer :: JP
+ integer :: KP
+ integer :: IP1
+ integer :: JP1
+ integer :: KP1
+ end type model_jp3d_variables
+ type (model_jp3d_variables) JP3DM_V
+! model_jp3d_variables
+
+! model_sea99_s_variables
+ type model_sea99_s_variables
+ sequence
+ double precision :: sea99_vs(100,100,100)
+ double precision :: sea99_depth(100)
+ double precision :: sea99_ddeg
+ double precision :: alatmin
+ double precision :: alatmax
+ double precision :: alonmin
+ double precision :: alonmax
+ integer :: sea99_ndep
+ integer :: sea99_nlat
+ integer :: sea99_nlon
+ integer :: dummy_pad ! padding 4 bytes to align the structure
+ end type model_sea99_s_variables
+ type (model_sea99_s_variables) SEA99M_V
+! model_sea99_s_variables
+
+! crust 2.0 model_crust_variables
+ type model_crust_variables
+ sequence
+ double precision, dimension(NKEYS_CRUST,NLAYERS_CRUST) :: thlr
+ double precision, dimension(NKEYS_CRUST,NLAYERS_CRUST) :: velocp
+ double precision, dimension(NKEYS_CRUST,NLAYERS_CRUST) :: velocs
+ double precision, dimension(NKEYS_CRUST,NLAYERS_CRUST) :: dens
+ character(len=2) abbreviation(NCAP_CRUST/2,NCAP_CRUST)
+ character(len=2) code(NKEYS_CRUST)
+ character(len=2) dummy_pad ! padding 2 bytes to align the structure
+ end type model_crust_variables
+ type (model_crust_variables) CM_V
+! model_crust_variables
+
+! EUcrust
+ type model_eucrust_variables
+ sequence
+ double precision, dimension(:),pointer :: eucrust_lat,eucrust_lon,&
+ eucrust_vp_uppercrust,eucrust_vp_lowercrust,eucrust_mohodepth,&
+ eucrust_basement,eucrust_ucdepth
+ integer :: num_eucrust
+ integer :: dummy_pad ! padding 4 bytes to align the structure
+ end type model_eucrust_variables
+ type (model_eucrust_variables) EUCM_V
+
+! type for EPCRUST 1.0
+ type model_epcrust_variables
+ sequence
+ double precision,dimension(EPCRUST_NLON,EPCRUST_NLAT):: lon_ep,lat_ep,topo_ep
+ double precision,dimension(EPCRUST_NLON,EPCRUST_NLAT,EPCRUST_NLAYER):: thickness_ep
+ double precision,dimension(EPCRUST_NLON,EPCRUST_NLAT,EPCRUST_NLAYER):: vp_ep
+ double precision,dimension(EPCRUST_NLON,EPCRUST_NLAT,EPCRUST_NLAYER):: vs_ep
+ double precision,dimension(EPCRUST_NLON,EPCRUST_NLAT,EPCRUST_NLAYER):: rho_ep
+ end type model_epcrust_variables
+ type (model_epcrust_variables) EPCRUST
+
+! model_crustmaps_variables combined crustal maps
+ type model_crustmaps_variables
+ sequence
+ double precision, dimension(180*CRUSTMAP_RESOLUTION,360*CRUSTMAP_RESOLUTION,NLAYERS_CRUSTMAP) :: thickness
+ double precision, dimension(180*CRUSTMAP_RESOLUTION,360*CRUSTMAP_RESOLUTION,NLAYERS_CRUSTMAP) :: density
+ double precision, dimension(180*CRUSTMAP_RESOLUTION,360*CRUSTMAP_RESOLUTION,NLAYERS_CRUSTMAP) :: velocp
+ double precision, dimension(180*CRUSTMAP_RESOLUTION,360*CRUSTMAP_RESOLUTION,NLAYERS_CRUSTMAP) :: velocs
+ double precision thicknessnp(NLAYERS_CRUSTMAP)
+ double precision densitynp(NLAYERS_CRUSTMAP)
+ double precision velocpnp(NLAYERS_CRUSTMAP)
+ double precision velocsnp(NLAYERS_CRUSTMAP)
+ double precision thicknesssp(NLAYERS_CRUSTMAP)
+ double precision densitysp(NLAYERS_CRUSTMAP)
+ double precision velocpsp(NLAYERS_CRUSTMAP)
+ double precision velocssp(NLAYERS_CRUSTMAP)
+ end type model_crustmaps_variables
+ type (model_crustmaps_variables) GC_V
+!model_crustmaps_variables
+
+! model_attenuation_storage_var
+ type model_attenuation_storage_var
+ sequence
+ double precision, dimension(:,:), pointer :: tau_e_storage
+ double precision, dimension(:), pointer :: Qmu_storage
+ integer Q_resolution
+ integer Q_max
+ end type model_attenuation_storage_var
+ type (model_attenuation_storage_var) AM_S
+! model_attenuation_storage_var
+
+! attenuation_simplex_variables
+ type attenuation_simplex_variables
+ sequence
+ double precision Q ! Q = Desired Value of Attenuation or Q
+ double precision iQ ! iQ = 1/Q
+ double precision, dimension(:), pointer :: f
+ ! f = Frequencies at which to evaluate the solution
+ double precision, dimension(:), pointer :: tau_s
+ ! tau_s = Tau_sigma defined by the frequency range and
+ ! number of standard linear solids
+ integer nf ! nf = Number of Frequencies
+ integer nsls ! nsls = Number of Standard Linear Solids
+ end type attenuation_simplex_variables
+ type(attenuation_simplex_variables) AS_V
+! attenuation_simplex_variables
+
+! point profile model_variables
+ type model_ppm_variables
+ sequence
+ double precision,dimension(:),pointer :: dvs,lat,lon,depth
+ double precision :: maxlat,maxlon,minlat,minlon,maxdepth,mindepth
+ double precision :: dlat,dlon,ddepth,max_dvs,min_dvs
+ integer :: num_v,num_latperlon,num_lonperdepth
+ integer :: dummy_pad ! padding 4 bytes to align the structure
+ end type model_ppm_variables
+ type (model_ppm_variables) PPM_V
+
+! GLL model_variables
+ type model_gll_variables
+ sequence
+ ! tomographic iteration model on GLL points
+ double precision :: scale_velocity,scale_density
+ ! isotropic model
+ real(kind=CUSTOM_REAL),dimension(:,:,:,:),pointer :: vs_new,vp_new,rho_new
+ ! transverse isotropic model
+ real(kind=CUSTOM_REAL),dimension(:,:,:,:),pointer :: vsv_new,vpv_new, &
+ vsh_new,vph_new,eta_new
+ logical :: MODEL_GLL
+ logical,dimension(3) :: dummy_pad ! padding 3 bytes to align the structure
+ end type model_gll_variables
+ type (model_gll_variables) MGLL_V
+
+! bathymetry and topography: use integer array to store values
+ integer, dimension(NX_BATHY,NY_BATHY) :: ibathy_topo
+
+! used for 3D Harvard models s362ani, s362wmani, s362ani_prem and s2.9ea
+ integer, parameter :: maxker=200
+ integer, parameter :: maxl=72
+ integer, parameter :: maxcoe=2000
+ integer, parameter :: maxver=1000
+ integer, parameter :: maxhpa=2
+
+ real(kind=4) conpt(maxver,maxhpa)
+ real(kind=4) xlaspl(maxcoe,maxhpa)
+ real(kind=4) xlospl(maxcoe,maxhpa)
+ real(kind=4) radspl(maxcoe,maxhpa)
+ real(kind=4) coe(maxcoe,maxker)
+ real(kind=4) vercof(maxker)
+ real(kind=4) vercofd(maxker)
+
+ real(kind=4) ylmcof((maxl+1)**2,maxhpa)
+ real(kind=4) wk1(maxl+1)
+ real(kind=4) wk2(maxl+1)
+ real(kind=4) wk3(maxl+1)
+
+ integer lmxhpa(maxhpa)
+ integer itypehpa(maxhpa)
+ integer ihpakern(maxker)
+ integer numcoe(maxhpa)
+ integer ivarkern(maxker)
+ integer itpspl(maxcoe,maxhpa)
+
+ integer nconpt(maxhpa),iver
+ integer iconpt(maxver,maxhpa)
+ integer numker
+ integer numhpa,numcof
+ integer ihpa,lmax,nylm
+
+ character(len=80) kerstr
+ character(len=80) refmdl
+ character(len=40) varstr(maxker)
+ character(len=80) hsplfl(maxhpa)
+ character(len=40) dskker(maxker)
+
+
+! for ellipticity
+ double precision rspl(NR),espl(NR),espl2(NR)
+ integer nspl
+
+! model parameter and flags
+ integer REFERENCE_1D_MODEL,THREE_D_MODEL
+
+ logical ELLIPTICITY,GRAVITY,ROTATION,TOPOGRAPHY,OCEANS
+
+ logical HONOR_1D_SPHERICAL_MOHO,CRUSTAL,ONE_CRUST,CASE_3D,TRANSVERSE_ISOTROPY
+
+ logical ISOTROPIC_3D_MANTLE,ANISOTROPIC_3D_MANTLE,HETEROGEN_3D_MANTLE
+
+ logical ATTENUATION,ATTENUATION_NEW,ATTENUATION_3D
+
+ logical ANISOTROPIC_INNER_CORE
+
+! to create a reference model based on 1D_REF but with 3D crust and 410/660 topography
+ logical,parameter :: USE_1D_REFERENCE = .false.
+
+end module meshfem3D_models_par
+
+
+!
+!-------------------------------------------------------------------------------------------------
+!
+
+
module meshfem3D_par
! main parameter module for specfem simulations
- use meshfem3D_models_par
+ use constants
implicit none
! correct number of spectral elements in each block depending on chunk type
- integer nspec_tiso,npointot
+ integer npointot
! proc numbers for MPI
integer myrank,sizeprocs
! check area and volume of the final mesh
- double precision area_local_bottom
- double precision area_local_top
- double precision volume_local,volume_total
+ double precision volume_total
! for loop on all the slices
integer iregion_code
@@ -73,7 +496,8 @@
NER_TOP_CENTRAL_CUBE_ICB,NEX_XI,NEX_ETA, &
NPROC_XI,NPROC_ETA,NTSTEP_BETWEEN_OUTPUT_SEISMOS, &
NTSTEP_BETWEEN_READ_ADJSRC,NSTEP,NSOURCES,NTSTEP_BETWEEN_FRAMES, &
- NTSTEP_BETWEEN_OUTPUT_INFO,NUMBER_OF_RUNS,NUMBER_OF_THIS_RUN,NCHUNKS,SIMULATION_TYPE, &
+ NTSTEP_BETWEEN_OUTPUT_INFO,NUMBER_OF_RUNS,NUMBER_OF_THIS_RUN, &
+ NCHUNKS,SIMULATION_TYPE, &
MOVIE_VOLUME_TYPE,MOVIE_START,MOVIE_STOP,NOISE_TOMOGRAPHY
double precision DT,ANGULAR_WIDTH_XI_IN_DEGREES,ANGULAR_WIDTH_ETA_IN_DEGREES,CENTER_LONGITUDE_IN_DEGREES, &
@@ -95,8 +519,6 @@
! parameters deduced from parameters read from file
integer NPROC,NPROCTOT,NEX_PER_PROC_XI,NEX_PER_PROC_ETA,ratio_divide_central_cube
- integer, external :: err_occurred
-
! this for all the regions
integer, dimension(MAX_NUM_REGIONS) :: NSPEC,NSPEC2D_XI,NSPEC2D_ETA, &
NSPEC2DMAX_XMIN_XMAX,NSPEC2DMAX_YMIN_YMAX, &
@@ -114,8 +536,6 @@
! memory size of all the static arrays
double precision :: static_memory_size
- integer :: ipass
-
integer :: NSPECMAX_ANISO_IC,NSPECMAX_ISO_MANTLE,NSPECMAX_TISO_MANTLE, &
NSPECMAX_ANISO_MANTLE,NSPEC_CRUST_MANTLE_ATTENUAT, &
NSPEC_INNER_CORE_ATTENUATION, &
@@ -144,7 +564,4 @@
logical :: CUT_SUPERBRICK_XI,CUT_SUPERBRICK_ETA
integer, dimension(MAX_NUM_REGIONS) :: NGLOB1D_RADIAL_TEMP
- ! this for non blocking MPI
- logical, dimension(:), allocatable :: is_on_a_slice_edge
-
end module meshfem3D_par
Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/meshfem3D/model_attenuation.f90
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/meshfem3D/model_attenuation.f90 2012-07-19 21:55:39 UTC (rev 20529)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/meshfem3D/model_attenuation.f90 2012-07-19 22:05:23 UTC (rev 20530)
@@ -162,7 +162,6 @@
implicit none
- include 'mpif.h'
include 'constants.h'
! model_attenuation_variables
@@ -281,14 +280,14 @@
double precision RICB, RCMB, R670, R220, R80
double precision tau_e(N_SLS)
- integer i,ier
+ integer i
double precision Qb
double precision R120
Qb = 57287.0d0
R120 = 6251.d3 ! as defined by IASP91
- call MPI_COMM_RANK(MPI_COMM_WORLD, myrank, ier)
+ call world_rank(myrank)
if(myrank > 0) return
@@ -440,7 +439,7 @@
subroutine model_attenuation_storage(Qmu, tau_e, rw, AM_S)
implicit none
- include 'mpif.h'
+
include 'constants.h'
! model_attenuation_storage_var
@@ -455,7 +454,7 @@
type (model_attenuation_storage_var) AM_S
! model_attenuation_storage_var
- integer myrank, ier
+ integer myrank
double precision Qmu, Qmu_new
double precision, dimension(N_SLS) :: tau_e
integer rw
@@ -477,7 +476,7 @@
write(IMAIN,*) 'Error attenuation_storage()'
write(IMAIN,*) 'Attenuation Value out of Range: ', Qmu
write(IMAIN,*) 'Attenuation Value out of Range: Min, Max ', 0, AM_S%Q_max
- call MPI_COMM_RANK(MPI_COMM_WORLD, myrank, ier)
+ call world_rank(myrank)
call exit_MPI(myrank, 'Attenuation Value out of Range')
endif
@@ -582,8 +581,6 @@
implicit none
- include 'mpif.h'
-
! attenuation_simplex_variables
type attenuation_simplex_variables
sequence
@@ -602,7 +599,7 @@
! attenuation_simplex_variables
! Input / Output
- integer myrank, ier
+ integer myrank
double precision t1, t2
double precision Q_real
double precision omega_not
@@ -633,8 +630,8 @@
exp2 = log10(f2)
if(f2 < f1 .OR. Q_real < 0.0d0 .OR. n < 1) then
- call MPI_COMM_RANK(MPI_COMM_WORLD, myrank, ier)
- call exit_MPI(myrank, 'frequencies flipped or Q less than zero or N_SLS < 0')
+ call world_rank(myrank)
+ call exit_MPI(myrank, 'frequencies flipped or Q less than zero or N_SLS < 0')
endif
! Determine the Source frequency
@@ -670,7 +667,7 @@
write(*,*)' Iterations: ', iterations
write(*,*)' Min Value: ', min_value
write(*,*)' Aborting program'
- call MPI_COMM_RANK(MPI_COMM_WORLD, myrank, ier)
+ call world_rank(myrank)
call exit_MPI(myrank,'attenuation_simplex: Search for Strain relaxation times did not converge')
endif
deallocate(f)
Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/meshfem3D/save_arrays_solver.f90
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/meshfem3D/save_arrays_solver.f90 2012-07-19 21:55:39 UTC (rev 20529)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/meshfem3D/save_arrays_solver.f90 2012-07-19 22:05:23 UTC (rev 20530)
@@ -144,7 +144,7 @@
! save nspec and nglob, to be used in combine_paraview_data
open(unit=27,file=prname(1:len_trim(prname))//'array_dims.txt', &
- status='unknown',action='write',iostat=ier)
+ status='unknown',action='write',iostat=ier)
if( ier /= 0 ) call exit_mpi(myrank,'error opening array_dims file')
nspec1 = nspec
@@ -154,6 +154,7 @@
write(27,*) nglob1
close(27)
+ ! mesh parameters
open(unit=27,file=prname(1:len_trim(prname))//'solver_data_1.bin', &
status='unknown',form='unformatted',action='write',iostat=ier)
if( ier /= 0 ) call exit_mpi(myrank,'error opening solver_data_1.bin file')
@@ -243,17 +244,17 @@
! mass matrices
!
- ! in the case of stacey boundary conditions, add C*deltat/2 contribution to the mass matrix
+ ! in the case of stacey boundary conditions, add C*deltat/2 contribution to the mass matrix
! on Stacey edges for the crust_mantle and outer_core regions but not for the inner_core region
! thus the mass matrix must be replaced by three mass matrices including the "C" damping matrix
- !
+ !
! if absorbing_conditions are not set or if NCHUNKS=6, only one mass matrix is needed
! for the sake of performance, only "rmassz" array will be filled and "rmassx" & "rmassy" will be obsolete
if(NCHUNKS /= 6 .and. ABSORBING_CONDITIONS .and. iregion_code == IREGION_CRUST_MANTLE) then
write(27) rmassx
write(27) rmassy
endif
-
+
write(27) rmassz
! additional ocean load mass matrix if oceans and if we are in the crust
@@ -261,6 +262,7 @@
close(27)
+ ! mesh topology
open(unit=27,file=prname(1:len_trim(prname))//'solver_data_2.bin', &
status='unknown',form='unformatted',action='write',iostat=ier)
if( ier /= 0 ) call exit_mpi(myrank,'error opening solver_data_2.bin file')
@@ -327,11 +329,8 @@
write(27) rmassz
write(27) ibool
-
write(27) idoubling
-
write(27) is_on_a_slice_edge
-
write(27) ispec_is_tiso
close(27)
Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/meshfem3D/setup_MPI_interfaces.f90
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/meshfem3D/setup_MPI_interfaces.f90 2012-07-19 21:55:39 UTC (rev 20529)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/meshfem3D/setup_MPI_interfaces.f90 2012-07-19 22:05:23 UTC (rev 20530)
@@ -28,32 +28,19 @@
subroutine setup_MPI_interfaces()
- use meshfem3D_par
+ use meshfem3D_par,only: INCLUDE_CENTRAL_CUBE,myrank
use create_MPI_interfaces_par
implicit none
-! include 'mpif.h'
-
! local parameters
- integer :: ier,ndim_assemble
-
- ! temporary buffers for send and receive between faces of the slices and the chunks
- real(kind=CUSTOM_REAL), dimension(npoin2D_max_all_CM_IC) :: &
- buffer_send_faces_scalar,buffer_received_faces_scalar
-
! assigns initial maximum arrays
! for global slices, maximum number of neighbor is around 17 ( 8 horizontal, max of 8 on bottom )
- integer :: MAX_NEIGHBOURS
+ integer :: MAX_NEIGHBOURS,max_nibool
integer, dimension(:),allocatable :: my_neighbours,nibool_neighbours
integer, dimension(:,:),allocatable :: ibool_neighbours
- integer :: max_nibool
- real(kind=CUSTOM_REAL),dimension(:),allocatable :: test_flag
- integer,dimension(:),allocatable :: dummy_i
- integer :: i,j,k,ispec,iglob
- ! debug
- character(len=150) :: filename
- logical,parameter :: DEBUG_INTERFACES = .false.
+ integer :: ier
+ ! allocates temporary arrays for setup routines
! estimates a maximum size of needed arrays
MAX_NEIGHBOURS = 8 + NCORNERSCHUNKS
if( INCLUDE_CENTRAL_CUBE ) MAX_NEIGHBOURS = MAX_NEIGHBOURS + NUMMSGS_FACES
@@ -69,9 +56,56 @@
allocate(ibool_neighbours(max_nibool,MAX_NEIGHBOURS), stat=ier)
if( ier /= 0 ) call exit_mpi(myrank,'error allocating ibool_neighbours')
+ ! sets up MPI interfaces between different processes
+ ! crust/mantle
+ call setup_MPI_interfaces_cm(MAX_NEIGHBOURS,my_neighbours,nibool_neighbours, &
+ max_nibool,ibool_neighbours)
-! sets up MPI interfaces
-! crust mantle region
+ ! outer core
+ call setup_MPI_interfaces_oc(MAX_NEIGHBOURS,my_neighbours,nibool_neighbours, &
+ max_nibool,ibool_neighbours)
+
+ ! inner core
+ call setup_MPI_interfaces_ic(MAX_NEIGHBOURS,my_neighbours,nibool_neighbours, &
+ max_nibool,ibool_neighbours)
+
+ ! frees temporary array
+ deallocate(ibool_neighbours)
+ deallocate(my_neighbours,nibool_neighbours)
+
+ ! synchronizes MPI processes
+ call sync_all()
+
+ end subroutine setup_MPI_interfaces
+
+!
+!-------------------------------------------------------------------------------------------------
+!
+
+ subroutine setup_MPI_interfaces_cm(MAX_NEIGHBOURS,my_neighbours,nibool_neighbours, &
+ max_nibool,ibool_neighbours)
+
+ use meshfem3D_par
+ use create_MPI_interfaces_par
+ implicit none
+
+ integer :: MAX_NEIGHBOURS,max_nibool
+ integer, dimension(MAX_NEIGHBOURS) :: my_neighbours,nibool_neighbours
+ integer, dimension(max_nibool,MAX_NEIGHBOURS) :: ibool_neighbours
+
+ ! local parameters
+ ! temporary buffers for send and receive between faces of the slices and the chunks
+ real(kind=CUSTOM_REAL), dimension(npoin2D_max_all_CM_IC) :: &
+ buffer_send_faces_scalar,buffer_received_faces_scalar
+ real(kind=CUSTOM_REAL),dimension(:),allocatable :: test_flag
+ integer,dimension(:),allocatable :: dummy_i
+ integer :: i,ier
+ ! debug
+ character(len=150) :: filename
+ logical,parameter :: DEBUG_INTERFACES = .false.
+
+ ! sets up MPI interfaces
+ ! crust mantle region
if( myrank == 0 ) write(IMAIN,*) 'crust mantle mpi:'
allocate(test_flag(NGLOB_CRUST_MANTLE), &
stat=ier)
@@ -98,12 +132,6 @@
! removes own myrank id (+1)
test_flag(:) = test_flag(:) - ( myrank + 1.0)
- ! debug: saves array
- !write(filename,'(a,i6.6)') trim(OUTPUT_FILES)//'/MPI_test_flag_crust_mantle_proc',myrank
- !call write_VTK_glob_points(NGLOB_CRUST_MANTLE, &
- ! xstore_crust_mantle,ystore_crust_mantle,zstore_crust_mantle, &
- ! test_flag,filename)
-
allocate(dummy_i(NSPEC_CRUST_MANTLE),stat=ier)
if( ier /= 0 ) call exit_mpi(myrank,'error allocating dummy_i')
@@ -149,15 +177,15 @@
! debug: outputs MPI interface
if( DEBUG_INTERFACES ) then
- do i=1,num_interfaces_crust_mantle
- write(filename,'(a,i6.6,a,i2.2)') trim(OUTPUT_FILES)//'/MPI_points_crust_mantle_proc',myrank, &
- '_',my_neighbours_crust_mantle(i)
- call write_VTK_data_points(NGLOB_crust_mantle, &
- xstore_crust_mantle,ystore_crust_mantle,zstore_crust_mantle, &
- ibool_interfaces_crust_mantle(1:nibool_interfaces_crust_mantle(i),i), &
- nibool_interfaces_crust_mantle(i),filename)
- enddo
- call sync_all()
+ do i=1,num_interfaces_crust_mantle
+ write(filename,'(a,i6.6,a,i2.2)') trim(OUTPUT_FILES)//'/MPI_points_crust_mantle_proc',myrank, &
+ '_',my_neighbours_crust_mantle(i)
+ call write_VTK_data_points(NGLOB_crust_mantle, &
+ xstore_crust_mantle,ystore_crust_mantle,zstore_crust_mantle, &
+ ibool_interfaces_crust_mantle(1:nibool_interfaces_crust_mantle(i),i), &
+ nibool_interfaces_crust_mantle(i),filename)
+ enddo
+ call sync_all()
endif
! checks addressing
@@ -178,8 +206,36 @@
! checks with assembly of test fields
call test_MPI_cm()
+ end subroutine setup_MPI_interfaces_cm
-! outer core region
+!
+!-------------------------------------------------------------------------------------------------
+!
+
+ subroutine setup_MPI_interfaces_oc(MAX_NEIGHBOURS,my_neighbours,nibool_neighbours, &
+ max_nibool,ibool_neighbours)
+
+ use meshfem3D_par
+ use create_MPI_interfaces_par
+ implicit none
+
+ integer :: MAX_NEIGHBOURS,max_nibool
+ integer, dimension(MAX_NEIGHBOURS) :: my_neighbours,nibool_neighbours
+ integer, dimension(max_nibool,MAX_NEIGHBOURS) :: ibool_neighbours
+
+ ! local parameters
+ ! temporary buffers for send and receive between faces of the slices and the chunks
+ real(kind=CUSTOM_REAL), dimension(npoin2D_max_all_CM_IC) :: &
+ buffer_send_faces_scalar,buffer_received_faces_scalar
+ real(kind=CUSTOM_REAL),dimension(:),allocatable :: test_flag
+ integer,dimension(:),allocatable :: dummy_i
+ integer :: i,ier
+ ! debug
+ character(len=150) :: filename
+ logical,parameter :: DEBUG_INTERFACES = .false.
+
+ ! sets up MPI interfaces
+ ! outer core region
if( myrank == 0 ) write(IMAIN,*) 'outer core mpi:'
allocate(test_flag(NGLOB_OUTER_CORE), &
@@ -208,12 +264,6 @@
! removes own myrank id (+1)
test_flag(:) = test_flag(:) - ( myrank + 1.0)
- ! debug: saves array
- !write(filename,'(a,i6.6)') trim(OUTPUT_FILES)//'/MPI_test_flag_outer_core_proc',myrank
- !call write_VTK_glob_points(NGLOB_OUTER_CORE, &
- ! xstore_outer_core,ystore_outer_core,zstore_outer_core, &
- ! test_flag,filename)
-
allocate(dummy_i(NSPEC_OUTER_CORE),stat=ier)
if( ier /= 0 ) call exit_mpi(myrank,'error allocating dummy_i')
@@ -259,15 +309,15 @@
! debug: outputs MPI interface
if( DEBUG_INTERFACES ) then
- do i=1,num_interfaces_outer_core
- write(filename,'(a,i6.6,a,i2.2)') trim(OUTPUT_FILES)//'/MPI_points_outer_core_proc',myrank, &
- '_',my_neighbours_outer_core(i)
- call write_VTK_data_points(NGLOB_OUTER_CORE, &
- xstore_outer_core,ystore_outer_core,zstore_outer_core, &
- ibool_interfaces_outer_core(1:nibool_interfaces_outer_core(i),i), &
- nibool_interfaces_outer_core(i),filename)
- enddo
- call sync_all()
+ do i=1,num_interfaces_outer_core
+ write(filename,'(a,i6.6,a,i2.2)') trim(OUTPUT_FILES)//'/MPI_points_outer_core_proc',myrank, &
+ '_',my_neighbours_outer_core(i)
+ call write_VTK_data_points(NGLOB_OUTER_CORE, &
+ xstore_outer_core,ystore_outer_core,zstore_outer_core, &
+ ibool_interfaces_outer_core(1:nibool_interfaces_outer_core(i),i), &
+ nibool_interfaces_outer_core(i),filename)
+ enddo
+ call sync_all()
endif
! checks addressing
@@ -288,8 +338,36 @@
! checks with assembly of test fields
call test_MPI_oc()
+ end subroutine setup_MPI_interfaces_oc
-! inner core
+!
+!-------------------------------------------------------------------------------------------------
+!
+
+ subroutine setup_MPI_interfaces_ic(MAX_NEIGHBOURS,my_neighbours,nibool_neighbours, &
+ max_nibool,ibool_neighbours)
+
+ use meshfem3D_par
+ use create_MPI_interfaces_par
+ implicit none
+
+ integer :: MAX_NEIGHBOURS,max_nibool
+ integer, dimension(MAX_NEIGHBOURS) :: my_neighbours,nibool_neighbours
+ integer, dimension(max_nibool,MAX_NEIGHBOURS) :: ibool_neighbours
+
+ ! local parameters
+ ! temporary buffers for send and receive between faces of the slices and the chunks
+ real(kind=CUSTOM_REAL), dimension(npoin2D_max_all_CM_IC) :: &
+ buffer_send_faces_scalar,buffer_received_faces_scalar
+ real(kind=CUSTOM_REAL),dimension(:),allocatable :: test_flag
+ integer :: i,j,k,ispec,iglob,ier
+ integer :: ndim_assemble
+ ! debug
+ character(len=150) :: filename
+ logical,parameter :: DEBUG_INTERFACES = .false.
+
+ ! sets up MPI interfaces
+ ! inner core
if( myrank == 0 ) write(IMAIN,*) 'inner core mpi:'
allocate(test_flag(NGLOB_INNER_CORE), &
@@ -327,12 +405,6 @@
NPROC_XI,NPROC_ETA,NGLOB1D_RADIAL(IREGION_INNER_CORE), &
NGLOB2DMAX_XMIN_XMAX(IREGION_INNER_CORE),NGLOB2DMAX_YMIN_YMAX(IREGION_INNER_CORE),NGLOB2DMAX_XY,NCHUNKS)
- ! debug: saves array
- !write(filename,'(a,i6.6)') trim(OUTPUT_FILES)//'/MPI_test_flag_inner_core_A_proc',myrank
- !call write_VTK_glob_points(NGLOB_INNER_CORE, &
- ! xstore_inner_core,ystore_inner_core,zstore_inner_core, &
- ! test_flag,filename)
-
! debug: idoubling inner core
if( DEBUG_INTERFACES ) then
write(filename,'(a,i6.6)') trim(OUTPUT_FILES)//'/MPI_idoubling_inner_core_proc',myrank
@@ -367,34 +439,35 @@
test_flag = test_flag - ( myrank + 1.0)
where( test_flag < 0.0 ) test_flag = 0.0
- ! debug: saves array
- !write(filename,'(a,i6.6)') trim(OUTPUT_FILES)//'/MPI_test_flag_inner_core_B_proc',myrank
- !call write_VTK_glob_points(NGLOB_INNER_CORE, &
- ! xstore_inner_core,ystore_inner_core,zstore_inner_core, &
- ! test_flag,filename)
- !call sync_all()
+ ! debug: in sequential order, for testing purpose
+ !do i=0,NPROCTOT - 1
+ ! if( myrank == i ) then
+ ! ! gets new interfaces for inner_core without central cube yet
+ ! ! determines neighbor rank for shared faces
+ ! call get_MPI_interfaces(myrank,NGLOB_INNER_CORE,NSPEC_INNER_CORE, &
+ ! test_flag,my_neighbours,nibool_neighbours,ibool_neighbours, &
+ ! num_interfaces_inner_core,max_nibool_interfaces_inner_core, &
+ ! max_nibool,MAX_NEIGHBOURS, &
+ ! ibool_inner_core,&
+ ! is_on_a_slice_edge_inner_core, &
+ ! IREGION_INNER_CORE,.false.,idoubling_inner_core,INCLUDE_CENTRAL_CUBE, &
+ ! xstore_inner_core,ystore_inner_core,zstore_inner_core,NPROCTOT)
+ ! endif
+ ! call sync_all()
+ !enddo
- ! in sequential order, for testing purpose
- do i=0,NPROCTOT - 1
- if( myrank == i ) then
- ! gets new interfaces for inner_core without central cube yet
- ! determines neighbor rank for shared faces
- call get_MPI_interfaces(myrank,NGLOB_INNER_CORE,NSPEC_INNER_CORE, &
- test_flag,my_neighbours,nibool_neighbours,ibool_neighbours, &
- num_interfaces_inner_core,max_nibool_interfaces_inner_core, &
- max_nibool,MAX_NEIGHBOURS, &
- ibool_inner_core,&
- is_on_a_slice_edge_inner_core, &
- IREGION_INNER_CORE,.false.,idoubling_inner_core,INCLUDE_CENTRAL_CUBE, &
- xstore_inner_core,ystore_inner_core,zstore_inner_core,NPROCTOT)
+ ! gets new interfaces for inner_core without central cube yet
+ ! determines neighbor rank for shared faces
+ call get_MPI_interfaces(myrank,NGLOB_INNER_CORE,NSPEC_INNER_CORE, &
+ test_flag,my_neighbours,nibool_neighbours,ibool_neighbours, &
+ num_interfaces_inner_core,max_nibool_interfaces_inner_core, &
+ max_nibool,MAX_NEIGHBOURS, &
+ ibool_inner_core,&
+ is_on_a_slice_edge_inner_core, &
+ IREGION_INNER_CORE,.false.,idoubling_inner_core,INCLUDE_CENTRAL_CUBE, &
+ xstore_inner_core,ystore_inner_core,zstore_inner_core,NPROCTOT)
- endif
- call sync_all()
- enddo
-
-
deallocate(test_flag)
- call sync_all()
! stores MPI interfaces informations
allocate(my_neighbours_inner_core(num_interfaces_inner_core), &
@@ -425,15 +498,15 @@
! debug: saves MPI interfaces
if( DEBUG_INTERFACES ) then
- do i=1,num_interfaces_inner_core
- write(filename,'(a,i6.6,a,i2.2)') trim(OUTPUT_FILES)//'/MPI_points_inner_core_proc',myrank, &
- '_',my_neighbours_inner_core(i)
- call write_VTK_data_points(NGLOB_INNER_CORE, &
- xstore_inner_core,ystore_inner_core,zstore_inner_core, &
- ibool_interfaces_inner_core(1:nibool_interfaces_inner_core(i),i), &
- nibool_interfaces_inner_core(i),filename)
- enddo
- call sync_all()
+ do i=1,num_interfaces_inner_core
+ write(filename,'(a,i6.6,a,i2.2)') trim(OUTPUT_FILES)//'/MPI_points_inner_core_proc',myrank, &
+ '_',my_neighbours_inner_core(i)
+ call write_VTK_data_points(NGLOB_INNER_CORE, &
+ xstore_inner_core,ystore_inner_core,zstore_inner_core, &
+ ibool_interfaces_inner_core(1:nibool_interfaces_inner_core(i),i), &
+ nibool_interfaces_inner_core(i),filename)
+ enddo
+ call sync_all()
endif
! checks addressing
@@ -454,11 +527,5 @@
! checks with assembly of test fields
call test_MPI_ic()
- ! synchronizes MPI processes
- call sync_all()
+ end subroutine setup_MPI_interfaces_ic
- ! frees temporary array
- deallocate(ibool_neighbours)
- deallocate(my_neighbours,nibool_neighbours)
-
- end subroutine setup_MPI_interfaces
Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/meshfem3D/setup_color_perm.f90
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/meshfem3D/setup_color_perm.f90 2012-07-19 21:55:39 UTC (rev 20529)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/meshfem3D/setup_color_perm.f90 2012-07-19 22:05:23 UTC (rev 20530)
@@ -29,13 +29,19 @@
subroutine setup_color_perm()
- use meshfem3D_par
+ use meshfem3D_par,only: &
+ myrank,IMAIN,USE_MESH_COLORING_GPU,SAVE_MESH_FILES, &
+ IREGION_CRUST_MANTLE,IREGION_OUTER_CORE,IREGION_INNER_CORE
+
use create_MPI_interfaces_par
+
implicit none
! local parameters
integer, dimension(:), allocatable :: perm
integer :: ier
+ integer :: nspec,nglob
+ integer :: idomain
! user output
if(myrank == 0) then
@@ -50,17 +56,45 @@
! mesh coloring
if( USE_MESH_COLORING_GPU ) then
+ !daniel: safety stop...
+ call exit_mpi(myrank,'MESH COLORING not fully implemented yet, please recompile...')
+
! user output
if(myrank == 0) write(IMAIN,*) ' coloring crust mantle... '
+ ! crust/mantle region
+ nspec = NSPEC_CRUST_MANTLE
+ nglob = NGLOB_CRUST_MANTLE
+ idomain = IREGION_CRUST_MANTLE
+
! creates coloring of elements
- allocate(perm(NSPEC_CRUST_MANTLE),stat=ier)
+ allocate(perm(nspec),stat=ier)
if( ier /= 0 ) call exit_mpi(myrank,'error allocating temporary perm crust mantle array')
perm(:) = 0
- allocate(num_elem_colors_crust_mantle(num_colors_outer_crust_mantle+num_colors_inner_crust_mantle),stat=ier)
- if( ier /= 0 ) call exit_mpi(myrank,'error allocating num_elem_colors_crust_mantle array')
+ call setup_color(myrank,nspec,nglob,ibool_crust_mantle,perm, &
+ idomain,is_on_a_slice_edge_crust_mantle, &
+ num_phase_ispec_crust_mantle,phase_ispec_inner_crust_mantle, &
+ SAVE_MESH_FILES)
+ ! checks
+ if(minval(perm) /= 1) &
+ call exit_MPI(myrank, 'minval(perm) should be 1')
+ if(maxval(perm) /= num_phase_ispec_crust_mantle) &
+ call exit_MPI(myrank, 'maxval(perm) should be num_phase_ispec_crust_mantle')
+
+ ! sorts array according to permutation
+ call sync_all()
+ if(myrank == 0) then
+ write(IMAIN,*) ' mesh permutation:'
+ endif
+ call setup_permutation(myrank,nspec,nglob,ibool_crust_mantle, &
+ idomain,perm, &
+ num_colors_outer_crust_mantle,num_colors_inner_crust_mantle, &
+ num_elem_colors_crust_mantle, &
+ num_phase_ispec_crust_mantle,phase_ispec_inner_crust_mantle, &
+ SAVE_MESH_FILES)
+
deallocate(perm)
else
! dummy array
@@ -79,14 +113,39 @@
! user output
if(myrank == 0) write(IMAIN,*) ' coloring outer core... '
+ ! outer core region
+ nspec = NSPEC_OUTER_CORE
+ nglob = NGLOB_OUTER_CORE
+ idomain = IREGION_OUTER_CORE
+
! creates coloring of elements
- allocate(perm(NSPEC_OUTER_CORE),stat=ier)
- if( ier /= 0 ) call exit_mpi(myrank,'error allocating temporary perm outer core array')
+ allocate(perm(nspec),stat=ier)
+ if( ier /= 0 ) call exit_mpi(myrank,'error allocating temporary perm outer_core array')
perm(:) = 0
- allocate(num_elem_colors_outer_core(num_colors_outer_outer_core+num_colors_inner_outer_core),stat=ier)
- if( ier /= 0 ) call exit_mpi(myrank,'error allocating num_elem_colors_outer_core array')
+ call setup_color(myrank,nspec,nglob,ibool_outer_core,perm, &
+ idomain,is_on_a_slice_edge_outer_core, &
+ num_phase_ispec_outer_core,phase_ispec_inner_outer_core, &
+ SAVE_MESH_FILES)
+ ! checks
+ if(minval(perm) /= 1) &
+ call exit_MPI(myrank, 'minval(perm) should be 1')
+ if(maxval(perm) /= num_phase_ispec_outer_core) &
+ call exit_MPI(myrank, 'maxval(perm) should be num_phase_ispec_outer_core')
+
+ ! sorts array according to permutation
+ call sync_all()
+ if(myrank == 0) then
+ write(IMAIN,*) ' mesh permutation:'
+ endif
+ call setup_permutation(myrank,nspec,nglob,ibool_outer_core, &
+ idomain,perm, &
+ num_colors_outer_outer_core,num_colors_inner_outer_core, &
+ num_elem_colors_outer_core, &
+ num_phase_ispec_outer_core,phase_ispec_inner_outer_core, &
+ SAVE_MESH_FILES)
+
deallocate(perm)
else
! dummy array
@@ -105,14 +164,42 @@
! user output
if(myrank == 0) write(IMAIN,*) ' coloring inner core... '
+ ! inner core region
+ nspec = NSPEC_INNER_CORE
+ nglob = NGLOB_INNER_CORE
+ idomain = IREGION_INNER_CORE
+
! creates coloring of elements
- allocate(perm(NSPEC_INNER_CORE),stat=ier)
- if( ier /= 0 ) call exit_mpi(myrank,'error allocating temporary perm inner core array')
+ allocate(perm(nspec),stat=ier)
+ if( ier /= 0 ) call exit_mpi(myrank,'error allocating temporary perm inner_core array')
perm(:) = 0
- allocate(num_elem_colors_inner_core(num_colors_outer_inner_core+num_colors_inner_inner_core),stat=ier)
- if( ier /= 0 ) call exit_mpi(myrank,'error allocating num_elem_colors_inner_core array')
+ call setup_color(myrank,nspec,nglob,ibool_inner_core,perm, &
+ idomain,is_on_a_slice_edge_inner_core, &
+ num_phase_ispec_inner_core,phase_ispec_inner_inner_core, &
+ SAVE_MESH_FILES)
+ ! checks
+ ! inner core contains ficticious elements not counted for
+ if(minval(perm) < 0) &
+ call exit_MPI(myrank, 'minval(perm) should be at least 0')
+ if(maxval(perm) > num_phase_ispec_inner_core) then
+ print*,'error perm inner core:',minval(perm),maxval(perm),num_phase_ispec_inner_core
+ call exit_MPI(myrank, 'maxval(perm) should be num_phase_ispec_inner_core')
+ endif
+
+ ! sorts array according to permutation
+ call sync_all()
+ if(myrank == 0) then
+ write(IMAIN,*) ' mesh permutation:'
+ endif
+ call setup_permutation(myrank,nspec,nglob,ibool_inner_core, &
+ idomain,perm, &
+ num_colors_outer_inner_core,num_colors_inner_inner_core, &
+ num_elem_colors_inner_core, &
+ num_phase_ispec_inner_core,phase_ispec_inner_inner_core, &
+ SAVE_MESH_FILES)
+
deallocate(perm)
else
! dummy array
@@ -122,11 +209,557 @@
end subroutine setup_color_perm
+!
+!-------------------------------------------------------------------------------------------------
+!
+ subroutine setup_color(myrank,nspec,nglob,ibool,perm, &
+ idomain,is_on_a_slice_edge, &
+ num_phase_ispec_d,phase_ispec_inner_d, &
+ SAVE_MESH_FILES)
+
+! sets up mesh coloring
+ use meshfem3D_par,only: &
+ LOCAL_PATH,MAX_NUMBER_OF_COLORS,IMAIN,NGLLX,NGLLY,NGLLZ,IFLAG_IN_FICTITIOUS_CUBE, &
+ IREGION_CRUST_MANTLE,IREGION_OUTER_CORE,IREGION_INNER_CORE
+
+ use create_MPI_interfaces_par,only: &
+ num_colors_outer_crust_mantle,num_colors_inner_crust_mantle,num_elem_colors_crust_mantle, &
+ num_colors_outer_outer_core,num_colors_inner_outer_core,num_elem_colors_outer_core, &
+ num_colors_outer_inner_core,num_colors_inner_inner_core,num_elem_colors_inner_core, &
+ idoubling_inner_core, &
+ xstore => xstore_crust_mantle,ystore => ystore_crust_mantle,zstore => zstore_crust_mantle
+
+ implicit none
+
+ integer :: myrank,nspec,nglob
+ integer, dimension(NGLLX,NGLLY,NGLLZ,nspec) :: ibool
+
+ integer, dimension(nspec) :: perm
+
+ ! wrapper array for ispec is in domain:
+ ! idomain: 1 == crust/mantle, 2 == outer core, 3 == inner core
+ integer :: idomain
+ logical, dimension(nspec) :: is_on_a_slice_edge
+ integer :: num_phase_ispec_d
+ integer, dimension(num_phase_ispec_d,2) :: phase_ispec_inner_d
+
+ logical :: SAVE_MESH_FILES
+
+ ! local parameters
+ ! added for color permutation
+ integer :: nb_colors_outer_elements,nb_colors_inner_elements
+ integer, dimension(:), allocatable :: num_of_elems_in_this_color
+ integer, dimension(:), allocatable :: color
+ integer, dimension(:), allocatable :: first_elem_number_in_this_color
+ logical, dimension(:), allocatable :: ispec_is_d
+
+ integer :: nspec_outer,nspec_inner,nspec_domain
+ integer :: nspec_outer_min_global,nspec_outer_max_global
+ integer :: nb_colors,nb_colors_min,nb_colors_max
+
+ integer :: icolor,ispec,ispec_counter
+ integer :: ispec_inner,ispec_outer
+ integer :: ier
+
+ character(len=2),dimension(3) :: str_domain = (/ "cm", "oc", "ic" /)
+ character(len=256) :: filename
+
+ logical, parameter :: DEBUG = .true.
+ character(len=150) :: prname
+
+ !!!! David Michea: detection of the edges, coloring and permutation separately
+
+ ! implement mesh coloring for GPUs if needed, to create subsets of disconnected elements
+ ! to remove dependencies and the need for atomic operations in the sum of
+ ! elemental contributions in the solver
+
+ ! allocates temporary array with colors
+ allocate(color(nspec),stat=ier)
+ if( ier /= 0 ) stop 'error allocating temporary color array'
+ allocate(first_elem_number_in_this_color(MAX_NUMBER_OF_COLORS + 1),stat=ier)
+ if( ier /= 0 ) stop 'error allocating first_elem_number_in_this_color array'
+
+ ! flags for elements in this domain
+ ! for compatiblity with SPECFEM3D mesh coloring routine
+ allocate(ispec_is_d(nspec),stat=ier)
+ if( ier /= 0 ) stop 'error allocating ispec_is_d array'
+
+ ! sets up domain coloring arrays
+ select case(idomain)
+ case( IREGION_CRUST_MANTLE,IREGION_OUTER_CORE )
+ ! crust/mantle and outer core region meshes use all elements
+ ispec_is_d(:) = .true.
+ case( IREGION_INNER_CORE )
+ ! initializes
+ ispec_is_d(:) = .true.
+ ! excludes ficticious elements from coloring
+ where(idoubling_inner_core == IFLAG_IN_FICTITIOUS_CUBE) ispec_is_d = .false.
+ ! checks
+ if( count(ispec_is_d) == 0 ) then
+ stop 'error no inner core elements'
+ endif
+ case default
+ stop 'error idomain in setup_color'
+ end select
+
+ ! fast element coloring scheme
+ call get_perm_color_faster(is_on_a_slice_edge,ispec_is_d, &
+ ibool,perm,color, &
+ nspec,nglob, &
+ nb_colors_outer_elements,nb_colors_inner_elements, &
+ nspec_outer,nspec_inner,nspec_domain, &
+ first_elem_number_in_this_color, &
+ myrank)
+
+ ! debug: file output
+ if( SAVE_MESH_FILES .and. DEBUG .and. idomain == 1 ) then
+ call create_name_database(prname,myrank,idomain,LOCAL_PATH)
+ filename = prname(1:len_trim(prname))//'color_'//str_domain(idomain)
+ call write_VTK_data_elem_i(nspec,nglob, &
+ xstore,ystore,zstore,ibool, &
+ color,filename)
+ endif
+ deallocate(color)
+
+ ! for the last color, the next color is fictitious and its first (fictitious) element number is nspec + 1
+ first_elem_number_in_this_color(nb_colors_outer_elements + nb_colors_inner_elements + 1) &
+ = nspec_domain + 1
+
+ allocate(num_of_elems_in_this_color(nb_colors_outer_elements + nb_colors_inner_elements),stat=ier)
+ if( ier /= 0 ) then
+ print*,'error',myrank,' allocating num_of_elems_in_this_color:',nb_colors_outer_elements,nb_colors_inner_elements, &
+ nb_colors_outer_elements + nb_colors_inner_elements
+ call exit_MPI(myrank,'error allocating num_of_elems_in_this_color array')
+ endif
+
+ num_of_elems_in_this_color(:) = 0
+ do icolor = 1, nb_colors_outer_elements + nb_colors_inner_elements
+ num_of_elems_in_this_color(icolor) = first_elem_number_in_this_color(icolor+1) - first_elem_number_in_this_color(icolor)
+ enddo
+ deallocate(first_elem_number_in_this_color)
+
+ ! check that the sum of all the numbers of elements found in each color is equal
+ ! to the total number of elements in the mesh
+ if(sum(num_of_elems_in_this_color) /= nspec_domain) then
+ print *,'error number of elements in this color:',idomain
+ print *,'rank: ',myrank,' nspec = ',nspec_domain
+ print *,' total number of elements in all the colors of the mesh = ', &
+ sum(num_of_elems_in_this_color)
+ call exit_MPI(myrank, 'incorrect total number of elements in all the colors of the mesh')
+ endif
+
+ ! check that the sum of all the numbers of elements found in each color for the outer elements is equal
+ ! to the total number of outer elements found in the mesh
+ if(sum(num_of_elems_in_this_color(1:nb_colors_outer_elements)) /= nspec_outer) then
+ print *,'error number of outer elements in this color:',idomain
+ print *,'rank: ',myrank,' nspec_outer = ',nspec_outer
+ print*,'nb_colors_outer_elements = ',nb_colors_outer_elements
+ print *,'total number of elements in all the colors of the mesh for outer elements = ', &
+ sum(num_of_elems_in_this_color(1:nb_colors_outer_elements))
+ call exit_MPI(myrank, 'incorrect total number of elements in all the colors of the mesh for outer elements')
+ endif
+
+
+
+ ! debug: no mesh coloring, only creates dummy coloring arrays
+ if( DEBUG ) then
+ nb_colors_outer_elements = 0
+ nb_colors_inner_elements = 0
+ ispec_counter = 0
+
+ ! first generate all the outer elements
+ do ispec = 1,nspec
+ if( ispec_is_d(ispec) ) then
+ if( is_on_a_slice_edge(ispec) .eqv. .true. ) then
+ ispec_counter = ispec_counter + 1
+ perm(ispec) = ispec_counter
+ endif
+ endif
+ enddo
+
+ ! store total number of outer elements
+ nspec_outer = ispec_counter
+
+ ! only single color
+ if(nspec_outer > 0 ) nb_colors_outer_elements = 1
+
+ ! then generate all the inner elements
+ do ispec = 1,nspec
+ if( ispec_is_d(ispec) ) then
+ if( is_on_a_slice_edge(ispec) .eqv. .false. ) then
+ ispec_counter = ispec_counter + 1
+ perm(ispec) = ispec_counter - nspec_outer ! starts again at 1
+ endif
+ endif
+ enddo
+ nspec_inner = ispec_counter - nspec_outer
+
+ ! only single color
+ if(nspec_inner > 0 ) nb_colors_inner_elements = 1
+
+ ! user output
+ if(myrank == 0 ) then
+ write(IMAIN,*) 'debugging mesh coloring:'
+ write(IMAIN,*) 'nb_colors inner / outer: ',nb_colors_inner_elements,nb_colors_outer_elements
+ endif
+
+ ! re-allocate
+ if(allocated(num_of_elems_in_this_color) ) deallocate(num_of_elems_in_this_color)
+ allocate(num_of_elems_in_this_color(nb_colors_outer_elements + nb_colors_inner_elements),stat=ier)
+ if( ier /= 0 ) then
+ print*,'error',myrank,' allocating num_of_elems_in_this_color:',nb_colors_outer_elements,nb_colors_inner_elements, &
+ nb_colors_outer_elements + nb_colors_inner_elements
+ call exit_MPI(myrank,'error allocating num_of_elems_in_this_color array')
+ endif
+
+ if( nspec_outer > 0 ) num_of_elems_in_this_color(1) = nspec_outer
+ if( nspec_inner > 0 ) num_of_elems_in_this_color(2) = nspec_inner
+ endif ! debug
+
+ ! debug: saves mesh coloring numbers into files
+ if( DEBUG ) then
+ ! debug file output
+ call create_name_database(prname,myrank,idomain,LOCAL_PATH)
+ filename = prname(1:len_trim(prname))//'num_of_elems_in_this_color_'//str_domain(idomain)//'.dat'
+ open(unit=99,file=trim(filename),status='unknown',iostat=ier)
+ if( ier /= 0 ) stop 'error opening num_of_elems_in_this_color file'
+ ! number of colors for outer elements
+ write(99,*) nb_colors_outer_elements
+ ! number of colors for inner elements
+ write(99,*) nb_colors_inner_elements
+ ! number of elements in each color
+ ! outer elements
+ do icolor = 1, nb_colors_outer_elements + nb_colors_inner_elements
+ write(99,*) num_of_elems_in_this_color(icolor)
+ enddo
+ close(99)
+ endif
+
+ ! sets up domain coloring arrays
+ select case(idomain)
+ case( IREGION_CRUST_MANTLE )
+ ! crust/mantle domains
+ num_colors_outer_crust_mantle = nb_colors_outer_elements
+ num_colors_inner_crust_mantle = nb_colors_inner_elements
+
+ allocate(num_elem_colors_crust_mantle(num_colors_outer_crust_mantle + num_colors_inner_crust_mantle),stat=ier)
+ if( ier /= 0 ) stop 'error allocating num_elem_colors_crust_mantle array'
+
+ num_elem_colors_crust_mantle(:) = num_of_elems_in_this_color(:)
+
+ case( IREGION_OUTER_CORE )
+ ! outer core domains
+ num_colors_outer_outer_core = nb_colors_outer_elements
+ num_colors_inner_outer_core = nb_colors_inner_elements
+
+ allocate(num_elem_colors_outer_core(num_colors_outer_outer_core + num_colors_inner_outer_core),stat=ier)
+ if( ier /= 0 ) stop 'error allocating num_elem_colors_outer_core array'
+
+ num_elem_colors_outer_core(:) = num_of_elems_in_this_color(:)
+
+ case( IREGION_INNER_CORE )
+ ! inner core domains
+ num_colors_outer_inner_core = nb_colors_outer_elements
+ num_colors_inner_inner_core = nb_colors_inner_elements
+
+ allocate(num_elem_colors_inner_core(num_colors_outer_inner_core + num_colors_inner_inner_core),stat=ier)
+ if( ier /= 0 ) stop 'error allocating num_elem_colors_inner_core array'
+
+ num_elem_colors_inner_core(:) = num_of_elems_in_this_color(:)
+
+ case default
+ stop 'error idomain not recognized'
+ end select
+
+ ! sets up elements for loops in simulations
+ ispec_inner = 0
+ ispec_outer = 0
+ do ispec = 1, nspec
+ ! only elements in this domain
+ if( ispec_is_d(ispec) ) then
+
+ ! sets phase_ispec arrays with ordering of elements
+ if( is_on_a_slice_edge(ispec) .eqv. .true. ) then
+ ! outer elements
+ ispec_outer = perm(ispec)
+
+ ! checks
+ if( ispec_outer < 1 .or. ispec_outer > num_phase_ispec_d ) then
+ print*,'error outer permutation:',idomain
+ print*,'rank:',myrank,' ispec_inner = ',ispec_outer
+ print*,'num_phase_ispec_d = ',num_phase_ispec_d
+ call exit_MPI(myrank,'error outer permutation')
+ endif
+
+ phase_ispec_inner_d(ispec_outer,1) = ispec
+
+ else
+ ! inner elements
+ ispec_inner = perm(ispec)
+
+ ! checks
+ if( ispec_inner < 1 .or. ispec_inner > num_phase_ispec_d ) then
+ print*,'error inner permutation:',idomain
+ print*,'rank:',myrank,' ispec_inner = ',ispec_inner
+ print*,'num_phase_ispec_d = ',num_phase_ispec_d
+ call exit_MPI(myrank,'error inner permutation')
+ endif
+
+ phase_ispec_inner_d(ispec_inner,2) = ispec
+
+ endif
+ endif
+ enddo
+
+ ! total number of colors
+ nb_colors = nb_colors_inner_elements + nb_colors_outer_elements
+ call min_all_i(nb_colors,nb_colors_min)
+ call max_all_i(nb_colors,nb_colors_max)
+
+ ! user output
+ call min_all_i(nspec_outer,nspec_outer_min_global)
+ call max_all_i(nspec_outer,nspec_outer_max_global)
+ call min_all_i(nspec_outer,nspec_outer_min_global)
+ call max_all_i(nspec_outer,nspec_outer_max_global)
+ if(myrank == 0) then
+ write(IMAIN,*) ' colors min = ',nb_colors_min
+ write(IMAIN,*) ' colors max = ',nb_colors_max
+ write(IMAIN,*) ' outer elements: min = ',nspec_outer_min_global
+ write(IMAIN,*) ' outer elements: max = ',nspec_outer_max_global
+ endif
+
+ ! debug: outputs permutation array as vtk file
+ if( DEBUG .and. idomain == IREGION_CRUST_MANTLE ) then
+ call create_name_database(prname,myrank,idomain,LOCAL_PATH)
+ filename = prname(1:len_trim(prname))//'perm_'//str_domain(idomain)
+ call write_VTK_data_elem_i(nspec,nglob, &
+ xstore,ystore,zstore,ibool, &
+ perm,filename)
+ endif
+
+ deallocate(ispec_is_d)
+ deallocate(num_of_elems_in_this_color)
+
+ end subroutine setup_color
+
!
!-------------------------------------------------------------------------------------------------
!
+ subroutine setup_permutation(myrank,nspec,nglob,ibool, &
+ idomain,perm, &
+ num_colors_outer,num_colors_inner, &
+ num_elem_colors, &
+ num_phase_ispec_d,phase_ispec_inner_d, &
+ SAVE_MESH_FILES)
+
+ use meshfem3D_par,only: &
+ CUSTOM_REAL,LOCAL_PATH,NGLLX,NGLLY,NGLLZ,IMAIN, &
+ IREGION_CRUST_MANTLE,IREGION_OUTER_CORE,IREGION_INNER_CORE
+
+ use create_MPI_interfaces_par,only: &
+ NSPEC_CRUST_MANTLE,ibool_crust_mantle,is_on_a_slice_edge_crust_mantle, &
+ NSPEC_OUTER_CORE,ibool_outer_core,is_on_a_slice_edge_outer_core, &
+ NSPEC_INNER_CORE,ibool_inner_core,is_on_a_slice_edge_inner_core, &
+ xstore => xstore_crust_mantle,ystore => ystore_crust_mantle,zstore => zstore_crust_mantle
+
+ implicit none
+
+ integer :: myrank,nspec,nglob
+ integer, dimension(NGLLX,NGLLY,NGLLZ,nspec) :: ibool
+
+ integer :: idomain
+ integer, dimension(nspec),intent(inout) :: perm
+
+ integer :: num_colors_outer,num_colors_inner
+ integer, dimension(num_colors_outer + num_colors_inner) :: num_elem_colors
+ integer :: num_phase_ispec_d
+ integer, dimension(num_phase_ispec_d,2) :: phase_ispec_inner_d
+
+ logical :: SAVE_MESH_FILES
+
+ ! local parameters
+ ! added for sorting
+ integer, dimension(:,:,:,:), allocatable :: temp_array_int
+ real(kind=CUSTOM_REAL), dimension(:,:,:,:), allocatable :: temp_array_real
+ logical, dimension(:), allocatable :: temp_array_logical_1D
+
+ integer, dimension(:), allocatable :: temp_perm_global
+ logical, dimension(:), allocatable :: mask_global
+
+ integer :: icolor,icounter,ispec,ielem,ier,i
+ integer :: iface,old_ispec,new_ispec
+
+ character(len=256) :: filename
+
+ logical,parameter :: DEBUG = .true.
+ character(len=150) :: prname
+
+ ! sorts array according to permutation
+ allocate(temp_perm_global(nspec),stat=ier)
+ if( ier /= 0 ) stop 'error temp_perm_global array'
+
+ ! global ordering
+ temp_perm_global(:) = 0
+ icounter = 0
+
+ ! fills global permutation array
+
+ ! first outer elements coloring
+ ! phase element counter
+ ielem = 0
+ do icolor = 1,num_colors_outer
+ ! loops through elements
+ do i = 1,num_elem_colors(icolor)
+ ielem = ielem + 1
+ ispec = phase_ispec_inner_d(ielem,1) ! 1 <-- first phase, outer elements
+ ! reorders elements
+ icounter = icounter + 1
+ temp_perm_global(ispec) = icounter
+ ! resets to new order
+ phase_ispec_inner_d(ielem,1) = icounter
+ enddo
+ enddo
+ ! inner elements coloring
+ ielem = 0
+ do icolor = num_colors_outer+1,num_colors_outer+num_colors_inner
+ ! loops through elements
+ do i = 1,num_elem_colors(icolor)
+ ielem = ielem + 1
+ ispec = phase_ispec_inner_d(ielem,2) ! 2 <-- second phase, inner elements
+ ! reorders elements
+ icounter = icounter + 1
+ temp_perm_global(ispec) = icounter
+ ! resets to new order
+ phase_ispec_inner_d(ielem,2) = icounter
+ enddo
+ enddo
+
+ ! checks
+ if( icounter /= nspec ) then
+ print*,'error temp perm: ',icounter,nspec
+ stop 'error temporary global permutation incomplete'
+ endif
+
+ ! checks perm entries
+ if( idomain /= IREGION_INNER_CORE ) then
+ ! exclude inner core: contains ficticious elements not counted for
+ if(minval(temp_perm_global) /= 1) call exit_MPI(myrank, 'minval(temp_perm_global) should be 1')
+ if(maxval(temp_perm_global) /= nspec) call exit_MPI(myrank, 'maxval(temp_perm_global) should be nspec')
+ endif
+
+ ! checks if every element was uniquely set
+ allocate(mask_global(nspec),stat=ier)
+ if( ier /= 0 ) stop 'error allocating temporary mask_global'
+ mask_global(:) = .false.
+ icounter = 0 ! counts permutations
+ do ispec = 1, nspec
+ new_ispec = temp_perm_global(ispec)
+ ! checks bounds
+ if( new_ispec < 1 .or. new_ispec > nspec ) call exit_MPI(myrank,'error temp_perm_global ispec bounds')
+ ! checks if already set
+ if( mask_global(new_ispec) ) then
+ print*,'error temp_perm_global:',ispec,new_ispec,'element already set'
+ call exit_MPI(myrank,'error global permutation')
+ else
+ mask_global(new_ispec) = .true.
+ endif
+ ! counts permutations
+ if( new_ispec /= ispec ) icounter = icounter + 1
+ enddo
+
+ ! checks number of set elements
+ if( count(mask_global(:)) /= nspec ) then
+ print*,'error temp_perm_global:',count(mask_global(:)),nspec,'permutation incomplete'
+ call exit_MPI(myrank,'error global permutation incomplete')
+ endif
+ deallocate(mask_global)
+
+ ! user output
+ if(myrank == 0) then
+ write(IMAIN,*) ' number of permutations = ',icounter
+ endif
+
+ ! outputs permutation array as vtk file
+ if( SAVE_MESH_FILES .and. DEBUG .and. idomain == IREGION_CRUST_MANTLE ) then
+ call create_name_database(prname,myrank,idomain,LOCAL_PATH)
+ filename = prname(1:len_trim(prname))//'perm_global'
+ call write_VTK_data_elem_i(nspec,nglob, &
+ xstore,ystore,zstore,ibool, &
+ temp_perm_global,filename)
+ endif
+
+ ! store as new permutation
+ perm(:) = temp_perm_global(:)
+ deallocate(temp_perm_global)
+
+ ! permutes all required mesh arrays according to new ordering
+ select case( idomain )
+ case( IREGION_CRUST_MANTLE )
+ ! region
+ nspec = NSPEC_CRUST_MANTLE
+
+ ! permutation of ibool
+ allocate(temp_array_int(NGLLX,NGLLY,NGLLZ,nspec))
+ call permute_elements_integer(ibool_crust_mantle,temp_array_int,perm,nspec)
+ deallocate(temp_array_int)
+
+ ! element domain flags
+ allocate(temp_array_logical_1D(nspec))
+ call permute_elements_logical1D(is_on_a_slice_edge_crust_mantle,temp_array_logical_1D,perm,nspec)
+ deallocate(temp_array_logical_1D)
+
+ ! mesh arrays
+
+ ! material parameters
+ ! crust/mantle arrays
+ ! outer core arrays
+ ! inner core arrays
+
+ ! boundary surface
+ ! free surface
+ ! coupling surface
+ ! moho surface
+
+ case( IREGION_OUTER_CORE )
+ ! region
+ nspec = NSPEC_OUTER_CORE
+
+ ! permutation of ibool
+ allocate(temp_array_int(NGLLX,NGLLY,NGLLZ,nspec))
+ call permute_elements_integer(ibool_outer_core,temp_array_int,perm,nspec)
+ deallocate(temp_array_int)
+
+ ! element domain flags
+ allocate(temp_array_logical_1D(nspec))
+ call permute_elements_logical1D(is_on_a_slice_edge_outer_core,temp_array_logical_1D,perm,nspec)
+ deallocate(temp_array_logical_1D)
+
+ case( IREGION_INNER_CORE )
+ ! region
+ nspec = NSPEC_INNER_CORE
+
+ ! permutation of ibool
+ allocate(temp_array_int(NGLLX,NGLLY,NGLLZ,nspec))
+ call permute_elements_integer(ibool_inner_core,temp_array_int,perm,nspec)
+ deallocate(temp_array_int)
+
+ ! element domain flags
+ allocate(temp_array_logical_1D(nspec))
+ call permute_elements_logical1D(is_on_a_slice_edge_inner_core,temp_array_logical_1D,perm,nspec)
+ deallocate(temp_array_logical_1D)
+
+ case default
+ stop 'error idomain in setup_permutation'
+ end select
+
+ end subroutine setup_permutation
+
+!
+!-------------------------------------------------------------------------------------------------
+!
+
! daniel: deprecated ...
!
! subroutine setup_color_perm(myrank,iregion_code,nspec,nglob, &
Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/shared/exit_mpi.f90
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/shared/exit_mpi.f90 2012-07-19 21:55:39 UTC (rev 20529)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/shared/exit_mpi.f90 2012-07-19 22:05:23 UTC (rev 20530)
@@ -105,10 +105,14 @@
end subroutine exit_MPI_without_rank
+
+!-------------------------------------------------------------------------------------------------
!
+! MPI wrapper functions
+!
!-------------------------------------------------------------------------------------------------
-!
+
subroutine sync_all()
implicit none
@@ -210,3 +214,142 @@
wtime = MPI_WTIME()
end function wtime
+
+!
+!-------------------------------------------------------------------------------------------------
+!
+
+ subroutine min_all_i(sendbuf, recvbuf)
+
+ implicit none
+
+! standard include of the MPI library
+ include 'mpif.h'
+
+ include "constants.h"
+ include "precision.h"
+
+ integer:: sendbuf, recvbuf
+ integer ier
+
+ call MPI_REDUCE(sendbuf,recvbuf,1,MPI_INTEGER, &
+ MPI_MIN,0,MPI_COMM_WORLD,ier)
+
+ end subroutine min_all_i
+
+!
+!-------------------------------------------------------------------------------------------------
+!
+
+ subroutine max_all_i(sendbuf, recvbuf)
+
+ implicit none
+
+! standard include of the MPI library
+ include 'mpif.h'
+
+ include "constants.h"
+ include "precision.h"
+
+ integer :: sendbuf, recvbuf
+ integer :: ier
+
+ call MPI_REDUCE(sendbuf,recvbuf,1,MPI_INTEGER, &
+ MPI_MAX,0,MPI_COMM_WORLD,ier)
+
+ end subroutine max_all_i
+
+!
+!-------------------------------------------------------------------------------------------------
+!
+
+ subroutine sum_all_dp(sendbuf, recvbuf)
+
+ implicit none
+
+! standard include of the MPI library
+ include 'mpif.h'
+
+ double precision sendbuf, recvbuf
+ integer ier
+
+ call MPI_REDUCE(sendbuf,recvbuf,1,MPI_DOUBLE_PRECISION, &
+ MPI_SUM,0,MPI_COMM_WORLD,ier)
+
+ end subroutine sum_all_dp
+
+!
+!-------------------------------------------------------------------------------------------------
+!
+
+ subroutine bcast_iproc_i(buffer,iproc)
+
+ implicit none
+
+! standard include of the MPI library
+ include 'mpif.h'
+
+ integer :: iproc
+ integer :: buffer
+
+ integer ier
+
+ call MPI_BCAST(buffer,1,MPI_INTEGER,iproc,MPI_COMM_WORLD,ier)
+
+ end subroutine bcast_iproc_i
+
+!
+!-------------------------------------------------------------------------------------------------
+!
+
+ subroutine bcast_all_singlei(buffer)
+
+ implicit none
+
+! standard include of the MPI library
+ include 'mpif.h'
+
+ integer :: buffer
+
+ integer ier
+
+ call MPI_BCAST(buffer,1,MPI_INTEGER,0,MPI_COMM_WORLD,ier)
+
+ end subroutine bcast_all_singlei
+
+!
+!-------------------------------------------------------------------------------------------------
+!
+
+ subroutine world_size(size)
+
+ implicit none
+
+! standard include of the MPI library
+ include 'mpif.h'
+
+ integer size
+ integer ier
+
+ call MPI_COMM_SIZE(MPI_COMM_WORLD,size,ier)
+
+ end subroutine world_size
+
+!
+!-------------------------------------------------------------------------------------------------
+!
+
+ subroutine world_rank(rank)
+
+ implicit none
+
+! standard include of the MPI library
+ include 'mpif.h'
+
+ integer rank
+ integer ier
+
+ call MPI_COMM_RANK(MPI_COMM_WORLD,rank,ier)
+
+ end subroutine world_rank
+
Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/shared/read_compute_parameters.f90
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/shared/read_compute_parameters.f90 2012-07-19 21:55:39 UTC (rev 20529)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/shared/read_compute_parameters.f90 2012-07-19 22:05:23 UTC (rev 20530)
@@ -232,7 +232,7 @@
! if doing benchmark runs to measure scaling of the code for a limited number of time steps only
if (DO_BENCHMARK_RUN_ONLY) NSTEP = NSTEP_FOR_BENCHMARK
- ! noise tomography:
+ ! noise tomography:
! time steps needs to be doubled, due to +/- branches
if ( NOISE_TOMOGRAPHY /= 0 ) NSTEP = 2*NSTEP-1
Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/Makefile.in
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/Makefile.in 2012-07-19 21:55:39 UTC (rev 20529)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/Makefile.in 2012-07-19 22:05:23 UTC (rev 20530)
@@ -28,10 +28,13 @@
# @configure_input@
# CUDA
- at COND_CUDA_TRUE@CUDA_LIBS = -lcuda -lcudart -lcublas
+# with configure: ./configure --with-cuda CUDA_LIB=.. CUDA_INC=.. MPI_INC=..
+
+# default cuda libraries
+# runtime library -lcudart needed, others are optional -lcuda -lcublas
+ at COND_CUDA_TRUE@CUDA_LIBS = -lcudart
@COND_CUDA_FALSE at CUDA_LIBS =
-# with configure: ./configure CUDA_LIB=.. CUDA_INC=.. MPI_INC=..
CUDA_LIB_LOCATION = @CUDA_LIB@
CUDA_LINK = $(CUDA_LIB_LOCATION) $(CUDA_LIBS)
CUDA_INC = @CUDA_INC@ -I../../setup -I../../
@@ -165,7 +168,7 @@
$O/compute_stacey_crust_mantle.solver.o \
$O/compute_stacey_outer_core.solver.o \
$O/finalize_simulation.mpisolver.o \
- $O/initialize_simulation.mpisolver.o \
+ $O/initialize_simulation.solver.o \
$O/iterate_time.mpisolver.o \
$O/noise_tomography.mpisolver.o \
$O/prepare_timerun.mpisolver.o \
@@ -195,6 +198,7 @@
$O/compute_kernels_cuda.cuda.o \
$O/compute_stacey_acoustic_cuda.cuda.o \
$O/compute_stacey_elastic_cuda.cuda.o \
+ $O/initialize_cuda.cuda.o \
$O/it_update_displacement_cuda.cuda.o \
$O/noise_tomography_cuda.cuda.o \
$O/prepare_mesh_constants_cuda.cuda.o \
Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/check_simulation_stability.f90
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/check_simulation_stability.f90 2012-07-19 21:55:39 UTC (rev 20529)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/check_simulation_stability.f90 2012-07-19 22:05:23 UTC (rev 20530)
@@ -96,7 +96,7 @@
integer, external :: idaywk
! timing
double precision, external :: wtime
-
+
double precision,parameter :: scale_displ = R_EARTH
Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/compute_coupling.f90
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/compute_coupling.f90 2012-07-19 21:55:39 UTC (rev 20529)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/compute_coupling.f90 2012-07-19 22:05:23 UTC (rev 20530)
@@ -458,11 +458,11 @@
b_accel_crust_mantle
! mass matrices
- !
- ! in the case of stacey boundary conditions, add C*deltat/2 contribution to the mass matrix
+ !
+ ! in the case of stacey boundary conditions, add C*deltat/2 contribution to the mass matrix
! on Stacey edges for the crust_mantle and outer_core regions but not for the inner_core region
! thus the mass matrix must be replaced by three mass matrices including the "C" damping matrix
- !
+ !
! if absorbing_conditions are not set or if NCHUNKS=6, only one mass matrix is needed
! for the sake of performance, only "rmassz" array will be filled and "rmassx" & "rmassy" will be obsolete
real(kind=CUSTOM_REAL), dimension(NGLOB_XY) :: rmassx_crust_mantle
@@ -476,7 +476,7 @@
integer, dimension(NSPEC2D_TOP_CM) :: ibelm_top_crust_mantle
logical, dimension(NGLOB_CRUST_MANTLE_OCEANS) :: updated_dof_ocean_load
- logical :: ABSORBING_CONDITIONS
+ logical :: ABSORBING_CONDITIONS
integer SIMULATION_TYPE
integer nspec_top
@@ -553,7 +553,7 @@
enddo
enddo
enddo
-
+
else
! for surface elements exactly at the top of the crust (ocean bottom)
Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/compute_forces_acoustic.F90
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/compute_forces_acoustic.F90 2012-07-19 21:55:39 UTC (rev 20529)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/compute_forces_acoustic.F90 2012-07-19 22:05:23 UTC (rev 20530)
@@ -273,7 +273,7 @@
! on CPU
call compute_forces_ac_update_veloc(NGLOB_OUTER_CORE,veloc_outer_core,accel_outer_core, &
deltatover2,rmass_outer_core)
-
+
! adjoint / kernel runs
if (SIMULATION_TYPE == 3) &
call compute_forces_ac_update_veloc(NGLOB_OUTER_CORE_ADJOINT,b_veloc_outer_core,b_accel_outer_core, &
Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/compute_forces_crust_mantle.f90
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/compute_forces_crust_mantle.f90 2012-07-19 21:55:39 UTC (rev 20529)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/compute_forces_crust_mantle.f90 2012-07-19 22:05:23 UTC (rev 20530)
@@ -241,14 +241,14 @@
tempy1l_att = tempy1l
tempy2l_att = tempy2l
tempy3l_att = tempy3l
-
+
tempz1l_att = tempz1l
tempz2l_att = tempz2l
tempz3l_att = tempz3l
if(ATTENUATION_NEW_VAL) then
! takes new routines
- ! use first order Taylor expansion of displacement for local storage of stresses
+ ! use first order Taylor expansion of displacement for local storage of stresses
! at this current time step, to fix attenuation in a consistent way
do l=1,NGLLX
hp1 = hprime_xx(i,l)
Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/compute_forces_crust_mantle_Dev.F90
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/compute_forces_crust_mantle_Dev.F90 2012-07-19 21:55:39 UTC (rev 20529)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/compute_forces_crust_mantle_Dev.F90 2012-07-19 22:05:23 UTC (rev 20530)
@@ -259,7 +259,7 @@
if(ATTENUATION_VAL .and. COMPUTE_AND_STORE_STRAIN) then
if(ATTENUATION_NEW_VAL) then
! takes new routines
- ! use first order Taylor expansion of displacement for local storage of stresses
+ ! use first order Taylor expansion of displacement for local storage of stresses
! at this current time step, to fix attenuation in a consistent way
do k=1,NGLLZ
@@ -302,7 +302,7 @@
#endif
enddo
enddo
- else
+ else
! takes old routines
do k=1,NGLLZ
do j=1,NGLLY
@@ -310,29 +310,29 @@
dummyx_loc_att(1,j,k) = 0._CUSTOM_REAL
dummyy_loc_att(1,j,k) = 0._CUSTOM_REAL
dummyz_loc_att(1,j,k) = 0._CUSTOM_REAL
-
+
dummyx_loc_att(2,j,k) = 0._CUSTOM_REAL
dummyy_loc_att(2,j,k) = 0._CUSTOM_REAL
dummyz_loc_att(2,j,k) = 0._CUSTOM_REAL
-
+
dummyx_loc_att(3,j,k) = 0._CUSTOM_REAL
dummyy_loc_att(3,j,k) = 0._CUSTOM_REAL
dummyz_loc_att(3,j,k) = 0._CUSTOM_REAL
-
+
dummyx_loc_att(4,j,k) = 0._CUSTOM_REAL
dummyy_loc_att(4,j,k) = 0._CUSTOM_REAL
dummyz_loc_att(4,j,k) = 0._CUSTOM_REAL
-
+
dummyx_loc_att(5,j,k) = 0._CUSTOM_REAL
dummyy_loc_att(5,j,k) = 0._CUSTOM_REAL
dummyz_loc_att(5,j,k) = 0._CUSTOM_REAL
-#else
+#else
do i=1,NGLLX
dummyx_loc_att(i,j,k) = 0._CUSTOM_REAL
dummyy_loc_att(i,j,k) = 0._CUSTOM_REAL
dummyz_loc_att(i,j,k) = 0._CUSTOM_REAL
enddo
-#endif
+#endif
enddo
enddo
endif
@@ -364,7 +364,7 @@
! temporary variables used for fixing attenuation in a consistent way
do j=1,m2
do i=1,m1
- C1_m1_m2_5points_att(i,j) = C1_m1_m2_5points(i,j) + &
+ C1_m1_m2_5points_att(i,j) = C1_m1_m2_5points(i,j) + &
hprime_xx(i,1)*B1_m1_m2_5points_att(1,j) + &
hprime_xx(i,2)*B1_m1_m2_5points_att(2,j) + &
hprime_xx(i,3)*B1_m1_m2_5points_att(3,j) + &
Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/compute_forces_elastic.F90
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/compute_forces_elastic.F90 2012-07-19 21:55:39 UTC (rev 20529)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/compute_forces_elastic.F90 2012-07-19 22:05:23 UTC (rev 20530)
@@ -578,11 +578,11 @@
real(kind=CUSTOM_REAL), dimension(NDIM,NGLOB) :: veloc_crust_mantle,accel_crust_mantle
! mass matrices
- !
- ! in the case of stacey boundary conditions, add C*deltat/2 contribution to the mass matrix
+ !
+ ! in the case of stacey boundary conditions, add C*deltat/2 contribution to the mass matrix
! on Stacey edges for the crust_mantle and outer_core regions but not for the inner_core region
! thus the mass matrix must be replaced by three mass matrices including the "C" damping matrix
- !
+ !
! if absorbing_conditions are not set or if NCHUNKS=6, only one mass matrix is needed
! for the sake of performance, only "rmassz" array will be filled and "rmassx" & "rmassy" will be obsolete
real(kind=CUSTOM_REAL), dimension(NGLOB_XY) :: rmassx_crust_mantle
@@ -646,7 +646,7 @@
accel_crust_mantle(3,i) = accel_crust_mantle(3,i)*rmassz_crust_mantle(i)
enddo
#endif
-
+
else
#ifdef _HANDOPT_NEWMARK
Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/compute_forces_inner_core.f90
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/compute_forces_inner_core.f90 2012-07-19 21:55:39 UTC (rev 20529)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/compute_forces_inner_core.f90 2012-07-19 22:05:23 UTC (rev 20530)
@@ -222,14 +222,14 @@
tempy1l_att = tempy1l
tempy2l_att = tempy2l
tempy3l_att = tempy3l
-
+
tempz1l_att = tempz1l
tempz2l_att = tempz2l
tempz3l_att = tempz3l
if(ATTENUATION_NEW_VAL) then
! takes new routines
- ! use first order Taylor expansion of displacement for local storage of stresses
+ ! use first order Taylor expansion of displacement for local storage of stresses
! at this current time step, to fix attenuation in a consistent way
do l=1,NGLLX
hp1 = hprime_xx(i,l)
Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/compute_forces_inner_core_Dev.F90
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/compute_forces_inner_core_Dev.F90 2012-07-19 21:55:39 UTC (rev 20529)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/compute_forces_inner_core_Dev.F90 2012-07-19 22:05:23 UTC (rev 20530)
@@ -275,11 +275,11 @@
#endif
enddo
enddo
-
+
if(ATTENUATION_VAL .and. COMPUTE_AND_STORE_STRAIN) then
if(ATTENUATION_NEW_VAL) then
! takes new routines
- ! use first order Taylor expansion of displacement for local storage of stresses
+ ! use first order Taylor expansion of displacement for local storage of stresses
! at this current time step, to fix attenuation in a consistent way
do k=1,NGLLZ
@@ -322,7 +322,7 @@
#endif
enddo
enddo
- else
+ else
! takes old routines
do k=1,NGLLZ
do j=1,NGLLY
@@ -346,13 +346,13 @@
dummyx_loc_att(5,j,k) = 0._CUSTOM_REAL
dummyy_loc_att(5,j,k) = 0._CUSTOM_REAL
dummyz_loc_att(5,j,k) = 0._CUSTOM_REAL
-#else
+#else
do i=1,NGLLX
dummyx_loc_att(i,j,k) = 0._CUSTOM_REAL
dummyy_loc_att(i,j,k) = 0._CUSTOM_REAL
dummyz_loc_att(i,j,k) = 0._CUSTOM_REAL
enddo
-#endif
+#endif
enddo
enddo
endif
Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/finalize_simulation.f90
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/finalize_simulation.f90 2012-07-19 21:55:39 UTC (rev 20529)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/finalize_simulation.f90 2012-07-19 22:05:23 UTC (rev 20530)
@@ -141,7 +141,7 @@
! frees dynamically allocated memory
! mass matrices
-
+
deallocate(rmassx_crust_mantle)
deallocate(rmassy_crust_mantle)
deallocate(rmassz_crust_mantle)
Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/initialize_simulation.f90
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/initialize_simulation.f90 2012-07-19 21:55:39 UTC (rev 20529)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/initialize_simulation.f90 2012-07-19 22:05:23 UTC (rev 20530)
@@ -29,10 +29,9 @@
subroutine initialize_simulation()
use specfem_par
+
implicit none
- include 'mpif.h'
-
! local parameters
integer, dimension(MAX_NUM_REGIONS) :: NSPEC_computed,NGLOB_computed, &
NSPEC2D_XI,NSPEC2D_ETA,NSPEC1D_RADIAL
@@ -41,7 +40,7 @@
integer, dimension(NB_SQUARE_EDGES_ONEDIR,NB_CUT_CASE) :: DIFF_NSPEC2D_XI,DIFF_NSPEC2D_ETA
integer :: ratio_divide_central_cube
integer :: sizeprocs
- integer :: ier,ios
+ integer :: ios
integer :: NPROC,NPROCTOT,NEX_PER_PROC_XI,NEX_PER_PROC_ETA,NCHUNKS,NPROC_XI,NPROC_ETA
double precision :: RMOHO_FICTITIOUS_IN_MESHER,R120,R_CENTRAL_CUBE,CENTER_LONGITUDE_IN_DEGREES,&
CENTER_LATITUDE_IN_DEGREES,ANGULAR_WIDTH_ETA_IN_DEGREES,&
@@ -57,8 +56,8 @@
! myrank is the rank of each process, between 0 and sizeprocs-1.
! as usual in MPI, process 0 is in charge of coordinating everything
! and also takes care of the main output
- call MPI_COMM_SIZE(MPI_COMM_WORLD,sizeprocs,ier)
- call MPI_COMM_RANK(MPI_COMM_WORLD,myrank,ier)
+ call world_size(sizeprocs)
+ call world_rank(myrank)
if (myrank == 0) then
@@ -266,9 +265,14 @@
endif
! broadcast the information read on the master to the nodes
- call MPI_BCAST(nrec,1,MPI_INTEGER,0,MPI_COMM_WORLD,ier)
+ call bcast_all_singlei(nrec)
+
+ ! checks number of total receivers
if(nrec < 1) call exit_MPI(myrank,trim(STATIONS)//': need at least one receiver')
+ ! initializes GPU cards
+ if( GPU_MODE ) call initialize_GPU()
+
end subroutine initialize_simulation
!
@@ -452,3 +456,48 @@
endif
end subroutine initialize_simulation_check
+
+!
+!-------------------------------------------------------------------------------------------------
+!
+
+ subroutine initialize_GPU()
+
+! initialization for GPU cards
+
+ use specfem_par
+ implicit none
+ ! local parameters
+ integer :: ncuda_devices,ncuda_devices_min,ncuda_devices_max
+
+ ! GPU_MODE now defined in Par_file
+ if(myrank == 0 ) then
+ write(IMAIN,*)
+ write(IMAIN,*) "GPU_MODE Active."
+ endif
+
+ ! check for GPU runs
+ if( NGLLX /= 5 .or. NGLLY /= 5 .or. NGLLZ /= 5 ) &
+ stop 'GPU mode can only be used if NGLLX == NGLLY == NGLLZ == 5'
+ if( CUSTOM_REAL /= 4 ) &
+ stop 'GPU mode runs only with CUSTOM_REAL == 4'
+ if( ATTENUATION_VAL ) then
+ if( N_SLS /= 3 ) &
+ stop 'GPU mode does not support N_SLS /= 3 yet'
+ endif
+
+ ! initializes GPU and outputs info to files for all processes
+ call initialize_cuda_device(myrank,ncuda_devices)
+
+ ! collects min/max of local devices found for statistics
+ call sync_all()
+ call min_all_i(ncuda_devices,ncuda_devices_min)
+ call max_all_i(ncuda_devices,ncuda_devices_max)
+
+ if( myrank == 0 ) then
+ write(IMAIN,*) "GPU number of devices per node: min =",ncuda_devices_min
+ write(IMAIN,*) " max =",ncuda_devices_max
+ write(IMAIN,*)
+ endif
+
+ end subroutine initialize_GPU
Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/iterate_time.F90
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/iterate_time.F90 2012-07-19 21:55:39 UTC (rev 20529)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/iterate_time.F90 2012-07-19 22:05:23 UTC (rev 20530)
@@ -439,7 +439,7 @@
if(myrank == 0) then
! elapsed time since beginning of the simulation
tCPU = wtime() - time_start
-
+
int_tCPU = int(tCPU)
ihours = int_tCPU / 3600
iminutes = (int_tCPU - 3600*ihours) / 60
Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/locate_receivers.f90
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/locate_receivers.f90 2012-07-19 21:55:39 UTC (rev 20529)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/locate_receivers.f90 2012-07-19 22:05:23 UTC (rev 20530)
@@ -145,7 +145,7 @@
! timing
double precision, external :: wtime
-
+
! get MPI starting time
time_start = wtime()
@@ -202,7 +202,7 @@
! initializes
final_distance(:) = HUGEVAL
final_distance_all(:,:) = HUGEVAL
-
+
! read that STATIONS file on the master
if(myrank == 0) then
call get_value_string(STATIONS, 'solver.STATIONS', trim(rec_filename))
@@ -348,7 +348,7 @@
+ (z_target_rec - dble(zstore(iglob)))**2)
if(dist > typical_size) cycle
endif
-
+
! loop only on points inside the element
! exclude edges to ensure this point is not shared with other elements
do k=2,NGLLZ-1
@@ -403,7 +403,7 @@
open(unit=1,file=trim(OUTPUT_FILES)//'/RECORDHEADERS', &
status='unknown',iostat=ier)
if( ier /= 0 ) call exit_MPI(myrank,'error opening file RECORDHEADERS')
-
+
nsamp = nint(dble(NSTEP-1)*DT)
do irec = 1,nrec
@@ -475,7 +475,7 @@
call exit_MPI(myrank,'incorrect value of iaddy')
endif
- iaz = 0
+ iaz = 0
if(iaddr(ia) == 0) then
iaz = 1
else if(iaddr(ia) == 1) then
@@ -616,7 +616,7 @@
if( ier /= 0 ) call exit_MPI(myrank,'Error opening and appending receivers to file sr_tmp.vtk')
islice_selected_rec_found(:) = -1
- nrec_found = 0
+ nrec_found = 0
do irec=1,nrec
if(final_distance(irec) == HUGEVAL) call exit_MPI(myrank,'error locating receiver')
@@ -664,7 +664,7 @@
write(IOVTK,*) sngl(x_found(irec)), sngl(y_found(irec)), sngl(z_found(irec))
endif
enddo
-
+
! finishes sr_tmp.vtk file
write(IOVTK,*)
close(IOVTK)
@@ -734,7 +734,7 @@
endif ! end of section executed by main process only
call sync_all()
-
+
! deallocate arrays
deallocate(epidist)
deallocate(ix_initial_guess,iy_initial_guess,iz_initial_guess)
Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/locate_sources.f90
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/locate_sources.f90 2012-07-19 21:55:39 UTC (rev 20529)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/locate_sources.f90 2012-07-19 22:05:23 UTC (rev 20530)
@@ -143,7 +143,7 @@
! timing
double precision, external :: wtime
-
+
! get MPI starting time for all sources
time_start = wtime()
@@ -190,7 +190,7 @@
position='append',status='old',iostat=ier)
if( ier /= 0 ) call exit_MPI(myrank,'Error opening and appending sources to file sr_tmp.vtk')
endif
-
+
! loop on all the sources
! gather source information in subsets to reduce memory requirements
@@ -219,12 +219,12 @@
y_found_source(NSOURCES_SUBSET_current_size), &
z_found_source(NSOURCES_SUBSET_current_size),stat=ier)
if( ier /= 0 ) call exit_MPI(myrank,'error allocating temporary source arrays')
-
+
! make sure we clean the subset array before the gather
ispec_selected_source_subset(:) = 0
final_distance_source_subset(:) = HUGEVAL
- final_distance_source_all(:,:) = HUGEVAL
-
+ final_distance_source_all(:,:) = HUGEVAL
+
! loop over sources within this subset
do isource_in_this_subset = 1,NSOURCES_SUBSET_current_size
@@ -353,7 +353,7 @@
+ (z_target_source - dble(zstore(iglob)))**2)
if( dist > typical_size ) cycle
endif
-
+
! define the interval in which we look for points
if(USE_FORCE_POINT_SOURCE) then
! force sources will be put on an exact GLL point
@@ -463,7 +463,7 @@
else
call exit_MPI(myrank,'incorrect value of iaddx')
endif
-
+
iay = 0
if(iaddy(ia) == 0) then
iay = 1
@@ -714,7 +714,7 @@
enddo ! end of loop on all the sources within current source subset
endif ! end of section executed by main process only
-
+
! deallocate arrays specific to each subset
deallocate(final_distance_source_subset)
deallocate(ispec_selected_source_subset)
@@ -731,12 +731,12 @@
write(IMAIN,*)
write(IMAIN,*) 'maximum error in location of the sources: ',sngl(maxval(final_distance_source)),' km'
write(IMAIN,*)
-
+
! closing sr_tmp.vtk
close(IOVTK)
endif
call sync_all()
-
+
! main process broadcasts the results to all the slices
call MPI_BCAST(islice_selected_source,NSOURCES,MPI_INTEGER,0,MPI_COMM_WORLD,ier)
call MPI_BCAST(ispec_selected_source,NSOURCES,MPI_INTEGER,0,MPI_COMM_WORLD,ier)
Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/prepare_timerun.f90
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/prepare_timerun.f90 2012-07-19 21:55:39 UTC (rev 20529)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/prepare_timerun.f90 2012-07-19 22:05:23 UTC (rev 20530)
@@ -35,7 +35,7 @@
! local parameters
! timing
double precision, external :: wtime
-
+
! get MPI starting time
time_start = wtime()
@@ -1091,7 +1091,6 @@
! local parameters
integer :: ier
real :: free_mb,used_mb,total_mb
- integer :: ncuda_devices,ncuda_devices_min,ncuda_devices_max
! dummy custom_real variables to convert from double precision
real(kind=CUSTOM_REAL),dimension(:,:,:),allocatable:: cr_wgll_cube
real(kind=CUSTOM_REAL),dimension(:),allocatable:: &
@@ -1099,21 +1098,12 @@
cr_minus_gravity_table,cr_minus_deriv_gravity_table, &
cr_density_table
- ! GPU_MODE now defined in Par_file
+ ! user output
if(myrank == 0 ) then
- write(IMAIN,*) "GPU_MODE Active."
- write(IMAIN,*)
write(IMAIN,*) "preparing Fields and Constants on GPU Device."
write(IMAIN,*)
endif
- ! initializes GPU and outputs info to files for all processes
- call prepare_cuda_device(myrank,ncuda_devices)
-
- ! collects min/max of local devices found for statistics
- call MPI_REDUCE(ncuda_devices,ncuda_devices_min,1,MPI_INTEGER,MPI_MIN,0,MPI_COMM_WORLD,ier)
- call MPI_REDUCE(ncuda_devices,ncuda_devices_max,1,MPI_INTEGER,MPI_MAX,0,MPI_COMM_WORLD,ier)
-
! prepares general fields on GPU
call prepare_constants_device(Mesh_pointer,myrank,NGLLX, &
hprime_xx, hprime_yy, hprime_zz, &
@@ -1298,7 +1288,7 @@
! crust/mantle region
if(myrank == 0 ) write(IMAIN,*) " loading crust/mantle region"
-
+
call prepare_crust_mantle_device(Mesh_pointer, &
xix_crust_mantle,xiy_crust_mantle,xiz_crust_mantle, &
etax_crust_mantle,etay_crust_mantle,etaz_crust_mantle, &
@@ -1404,17 +1394,14 @@
! outputs usage for main process
if( myrank == 0 ) then
- write(IMAIN,*)" GPU number of devices per node: min =",ncuda_devices_min
- write(IMAIN,*)" max =",ncuda_devices_max
- write(IMAIN,*)
-
+ ! gets memory usage for main process
call get_free_device_memory(free_mb,used_mb,total_mb)
+ ! outputs info
+ write(IMAIN,*)
write(IMAIN,*)" GPU usage: free =",free_mb," MB",nint(free_mb/total_mb*100.0),"%"
write(IMAIN,*)" used =",used_mb," MB",nint(used_mb/total_mb*100.0),"%"
write(IMAIN,*)" total =",total_mb," MB",nint(total_mb/total_mb*100.0),"%"
write(IMAIN,*)
endif
- call sync_all()
-
end subroutine prepare_timerun_GPU
Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/read_arrays_solver.f90
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/read_arrays_solver.f90 2012-07-19 21:55:39 UTC (rev 20529)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/read_arrays_solver.f90 2012-07-19 22:05:23 UTC (rev 20530)
@@ -178,10 +178,10 @@
! mass matrices
!
- ! in the case of stacey boundary conditions, add C*deltat/2 contribution to the mass matrix
+ ! in the case of stacey boundary conditions, add C*deltat/2 contribution to the mass matrix
! on Stacey edges for the crust_mantle and outer_core regions but not for the inner_core region
! thus the mass matrix must be replaced by three mass matrices including the "C" damping matrix
- !
+ !
! if absorbing_conditions are not set or if NCHUNKS=6, only one mass matrix is needed
! for the sake of performance, only "rmassz" array will be filled and "rmassx" & "rmassy" will be obsolete
if(NCHUNKS_VAL /= 6 .and. ABSORBING_CONDITIONS .and. iregion_code == IREGION_CRUST_MANTLE) then
Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/read_mesh_databases.f90
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/read_mesh_databases.f90 2012-07-19 21:55:39 UTC (rev 20529)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/read_mesh_databases.f90 2012-07-19 22:05:23 UTC (rev 20530)
@@ -37,7 +37,7 @@
! local parameters
! timing
double precision, external :: wtime
-
+
! get MPI starting time
time_start = wtime()
@@ -125,12 +125,12 @@
! sets number of top elements for surface movies & noise tomography
NSPEC_TOP = NSPEC2D_TOP(IREGION_CRUST_MANTLE)
- ! allocates mass matrices in this slice (will be fully assembled in the solver)
+ ! allocates mass matrices in this slice (will be fully assembled in the solver)
!
- ! in the case of stacey boundary conditions, add C*deltat/2 contribution to the mass matrix
+ ! in the case of stacey boundary conditions, add C*deltat/2 contribution to the mass matrix
! on Stacey edges for the crust_mantle and outer_core regions but not for the inner_core region
! thus the mass matrix must be replaced by three mass matrices including the "C" damping matrix
- !
+ !
! if absorbing_conditions are not set or if NCHUNKS=6, only one mass matrix is needed
! for the sake of performance, only "rmassz" array will be filled and "rmassx" & "rmassy" will be obsolete
if(NCHUNKS_VAL /= 6 .and. ABSORBING_CONDITIONS) then
@@ -216,12 +216,12 @@
allocate(dummy_ispec_is_tiso(NSPEC_OUTER_CORE))
allocate(dummy_idoubling_outer_core(NSPEC_OUTER_CORE))
- ! allocates mass matrices in this slice (will be fully assembled in the solver)
+ ! allocates mass matrices in this slice (will be fully assembled in the solver)
!
- ! in the case of stacey boundary conditions, add C*deltat/2 contribution to the mass matrix
+ ! in the case of stacey boundary conditions, add C*deltat/2 contribution to the mass matrix
! on Stacey edges for the crust_mantle and outer_core regions but not for the inner_core region
! thus the mass matrix must be replaced by three mass matrices including the "C" damping matrix
- !
+ !
! if absorbing_conditions are not set or if NCHUNKS=6, only one mass matrix is needed
! for the sake of performance, only "rmassz" array will be filled and "rmassx" & "rmassy" will be obsolete
allocate(rmass_outer_core(NGLOB_OUTER_CORE),stat=ier)
@@ -299,12 +299,12 @@
allocate(dummy_rmass(NGLOB_XY))
allocate(dummy_ispec_is_tiso(NSPEC_INNER_CORE))
- ! allocates mass matrices in this slice (will be fully assembled in the solver)
+ ! allocates mass matrices in this slice (will be fully assembled in the solver)
!
- ! in the case of stacey boundary conditions, add C*deltat/2 contribution to the mass matrix
+ ! in the case of stacey boundary conditions, add C*deltat/2 contribution to the mass matrix
! on Stacey edges for the crust_mantle and outer_core regions but not for the inner_core region
! thus the mass matrix must be replaced by three mass matrices including the "C" damping matrix
- !
+ !
! if absorbing_conditions are not set or if NCHUNKS=6, only one mass matrix is needed
! for the sake of performance, only "rmassz" array will be filled and "rmassx" & "rmassy" will be obsolete
allocate(rmass_inner_core(NGLOB_INNER_CORE),stat=ier)
@@ -606,7 +606,7 @@
! determine chunk number and local slice coordinates using addressing
! (needed for stacey conditions)
ichunk = ichunk_slice(myrank)
-
+
!iproc_xi = iproc_xi_slice(myrank)
!iproc_eta = iproc_eta_slice(myrank)
Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/read_topography_bathymetry.f90
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/read_topography_bathymetry.f90 2012-07-19 21:55:39 UTC (rev 20529)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/read_topography_bathymetry.f90 2012-07-19 22:05:23 UTC (rev 20530)
@@ -39,7 +39,7 @@
integer :: ier
! timing
double precision, external :: wtime
-
+
! get MPI starting time
time_start = wtime()
Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/specfem3D_par.F90
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/specfem3D_par.F90 2012-07-19 21:55:39 UTC (rev 20529)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/specfem3D_par.F90 2012-07-19 22:05:23 UTC (rev 20530)
@@ -360,11 +360,11 @@
logical, dimension(NSPEC_CRUST_MANTLE) :: ispec_is_tiso_crust_mantle
! mass matrices
- !
- ! in the case of stacey boundary conditions, add C*deltat/2 contribution to the mass matrix
+ !
+ ! in the case of stacey boundary conditions, add C*deltat/2 contribution to the mass matrix
! on Stacey edges for the crust_mantle and outer_core regions but not for the inner_core region
! thus the mass matrix must be replaced by three mass matrices including the "C" damping matrix
- !
+ !
! if absorbing_conditions are not set or if NCHUNKS=6, only one mass matrix is needed
! for the sake of performance, only "rmassz" array will be filled and "rmassx" & "rmassy" will be obsolete
real(kind=CUSTOM_REAL), dimension(:), allocatable :: rmassx_crust_mantle
Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/write_output_SAC.f90
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/write_output_SAC.f90 2012-07-19 21:55:39 UTC (rev 20529)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/write_output_SAC.f90 2012-07-19 22:05:23 UTC (rev 20530)
@@ -257,7 +257,7 @@
CMPAZ = sngl(modulo(phi_dble,360.d0)) ! phi is calculated above (see call distaz())
CMPINC =90.00
else if(iorientation == 5) then !T
- phi_dble = phi
+ phi_dble = phi
CMPAZ = sngl(modulo(phi_dble+90.d0,360.d0)) ! phi is calculated above (see call distaz())
CMPINC =90.00
endif
Modified: seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/write_seismograms.f90
===================================================================
--- seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/write_seismograms.f90 2012-07-19 21:55:39 UTC (rev 20529)
+++ seismo/3D/SPECFEM3D_GLOBE/branches/SPECFEM3D_GLOBE_SUNFLOWER/src/specfem3D/write_seismograms.f90 2012-07-19 22:05:23 UTC (rev 20530)
@@ -155,7 +155,7 @@
character(len=256) :: sisname
! timing
double precision, external :: wtime
-
+
! allocates single station seismogram
allocate(one_seismogram(NDIM,NTSTEP_BETWEEN_OUTPUT_SEISMOS),stat=ier)
if(ier /= 0) call exit_mpi(myrank,'error while allocating one temporary seismogram')
More information about the CIG-COMMITS
mailing list