[cig-commits] r20726 - in short/3D/PyLith/trunk/libsrc/pylith: faults problems topology
knepley at geodynamics.org
knepley at geodynamics.org
Tue Sep 18 15:40:41 PDT 2012
Author: knepley
Date: 2012-09-18 15:40:40 -0700 (Tue, 18 Sep 2012)
New Revision: 20726
Modified:
short/3D/PyLith/trunk/libsrc/pylith/faults/FaultCohesiveDyn.cc
short/3D/PyLith/trunk/libsrc/pylith/problems/SolverNonlinear.cc
short/3D/PyLith/trunk/libsrc/pylith/topology/Field.cc
short/3D/PyLith/trunk/libsrc/pylith/topology/Field.hh
Log:
Use DM to encapsulate scatter for Field
Modified: short/3D/PyLith/trunk/libsrc/pylith/faults/FaultCohesiveDyn.cc
===================================================================
--- short/3D/PyLith/trunk/libsrc/pylith/faults/FaultCohesiveDyn.cc 2012-09-18 17:16:13 UTC (rev 20725)
+++ short/3D/PyLith/trunk/libsrc/pylith/faults/FaultCohesiveDyn.cc 2012-09-18 22:40:40 UTC (rev 20726)
@@ -2167,8 +2167,8 @@
err = KSPSetOperators(_ksp, jacobianMat, jacobianMat,
DIFFERENT_NONZERO_PATTERN); CHECK_PETSC_ERROR(err);
- const PetscVec residualVec = residual.vector();
- const PetscVec solutionVec = solution.vector();
+ const PetscVec residualVec = residual.globalVector();
+ const PetscVec solutionVec = solution.globalVector();
err = KSPSolve(_ksp, residualVec, solutionVec); CHECK_PETSC_ERROR(err);
// Update section view of field.
Modified: short/3D/PyLith/trunk/libsrc/pylith/problems/SolverNonlinear.cc
===================================================================
--- short/3D/PyLith/trunk/libsrc/pylith/problems/SolverNonlinear.cc 2012-09-18 17:16:13 UTC (rev 20725)
+++ short/3D/PyLith/trunk/libsrc/pylith/problems/SolverNonlinear.cc 2012-09-18 22:40:40 UTC (rev 20726)
@@ -101,7 +101,7 @@
err = SNESCreate(fields.mesh().comm(), &_snes); CHECK_PETSC_ERROR(err);
const topology::Field<topology::Mesh>& residual = fields.get("residual");
- const PetscVec residualVec = residual.vector();
+ const PetscVec residualVec = residual.globalVector();
err = SNESSetFunction(_snes, residualVec, reformResidual,
(void*) formulation);
CHECK_PETSC_ERROR(err);
@@ -141,7 +141,7 @@
_logger->eventBegin(solveEvent);
PetscErrorCode err = 0;
- const PetscVec solutionVec = solution->vector();
+ const PetscVec solutionVec = solution->globalVector();
err = SNESSolve(_snes, PETSC_NULL, solutionVec); CHECK_PETSC_ERROR(err);
Modified: short/3D/PyLith/trunk/libsrc/pylith/topology/Field.cc
===================================================================
--- short/3D/PyLith/trunk/libsrc/pylith/topology/Field.cc 2012-09-18 17:16:13 UTC (rev 20725)
+++ short/3D/PyLith/trunk/libsrc/pylith/topology/Field.cc 2012-09-18 22:40:40 UTC (rev 20726)
@@ -122,7 +122,9 @@
s_iter != scattersEnd;
++s_iter) {
+ err = DMDestroy(&s_iter->second.dm);CHECK_PETSC_ERROR(err);
err = VecDestroy(&s_iter->second.vector);CHECK_PETSC_ERROR(err);
+
err = VecScatterDestroy(&s_iter->second.scatter);CHECK_PETSC_ERROR(err);
err = VecDestroy(&s_iter->second.scatterVec);CHECK_PETSC_ERROR(err);
} // for
@@ -461,6 +463,11 @@
sinfo.vector = 0;
sinfo.scatterVec = 0;
+ // Copy DM
+ sinfo.dm = s_iter->second.dm;
+ err = PetscObjectReference((PetscObject) sinfo.dm);
+ CHECK_PETSC_ERROR(err);
+
// Copy scatter
sinfo.scatter = s_iter->second.scatter;
err = PetscObjectReference((PetscObject) sinfo.scatter);
@@ -485,15 +492,22 @@
// Create vector using sizes from source section
int vecLocalSize = 0;
- int vecGlobalSize = 0;
+ int vecGlobalSize = 0, vecGlobalSize2 = 0;
err = VecGetLocalSize(s_iter->second.vector, &vecLocalSize);CHECK_PETSC_ERROR(err);
err = VecGetSize(s_iter->second.vector, &vecGlobalSize);CHECK_PETSC_ERROR(err);
+ err = VecGetSize(_globalVec, &vecGlobalSize2);CHECK_PETSC_ERROR(err);
- err = VecCreate(_mesh.comm(), &sinfo.vector);CHECK_PETSC_ERROR(err);
- err = PetscObjectSetName((PetscObject)sinfo.vector, _metadata["default"].label.c_str());CHECK_PETSC_ERROR(err);
- err = VecSetSizes(sinfo.vector, vecLocalSize, vecGlobalSize);CHECK_PETSC_ERROR(err);
- err = VecSetBlockSize(sinfo.vector, blockSize);CHECK_PETSC_ERROR(err);
- err = VecSetFromOptions(sinfo.vector); CHECK_PETSC_ERROR(err);
+ if (vecGlobalSize != vecGlobalSize2) {
+ err = VecCreate(_mesh.comm(), &sinfo.vector);CHECK_PETSC_ERROR(err);
+ err = VecSetSizes(sinfo.vector, vecLocalSize, vecGlobalSize);CHECK_PETSC_ERROR(err);
+ err = VecSetBlockSize(sinfo.vector, blockSize);CHECK_PETSC_ERROR(err);
+ err = VecSetFromOptions(sinfo.vector); CHECK_PETSC_ERROR(err);
+ } else {
+ sinfo.vector = _globalVec;
+ err = PetscObjectReference((PetscObject) sinfo.dm);
+ CHECK_PETSC_ERROR(err);
+ }
+ err = PetscObjectSetName((PetscObject)sinfo.vector, _metadata["default"].label.c_str());CHECK_PETSC_ERROR(err);
_scatters[s_iter->first] = sinfo;
} // for
@@ -1065,6 +1079,7 @@
} // else
// Create vector
+#if 0
err = VecCreate(_mesh.comm(), &sinfo.vector);
CHECK_PETSC_ERROR(err);
err = PetscObjectSetName((PetscObject)sinfo.vector,
@@ -1072,7 +1087,19 @@
err = VecSetSizes(sinfo.vector,
order->getLocalSize(), order->getGlobalSize());CHECK_PETSC_ERROR(err);
err = VecSetBlockSize(sinfo.vector, blockSize);CHECK_PETSC_ERROR(err);
- err = VecSetFromOptions(sinfo.vector);CHECK_PETSC_ERROR(err);
+ err = VecSetFromOptions(sinfo.vector);CHECK_PETSC_ERROR(err);
+#endif
+ PetscInt localSize, globalSize;
+
+ err = PetscObjectReference((PetscObject) _dm);CHECK_PETSC_ERROR(err);
+ err = PetscObjectReference((PetscObject) _globalVec);CHECK_PETSC_ERROR(err);
+ err = PetscObjectSetName((PetscObject) _globalVec, _metadata["default"].label.c_str());CHECK_PETSC_ERROR(err);
+ err = VecGetSize(_localVec, &localSize);CHECK_PETSC_ERROR(err);
+ err = VecGetSize(_globalVec, &globalSize);CHECK_PETSC_ERROR(err);
+ assert(order->getLocalSize() == localSize);
+ assert(order->getGlobalSize() == globalSize);
+ sinfo.vector = _globalVec;
+ sinfo.dm = _dm;
logger.stagePop();
} // createScatter
@@ -1144,6 +1171,7 @@
} // else
// Create vector
+#if 0
err = VecCreate(mesh.comm(), &sinfo.vector);CHECK_PETSC_ERROR(err);
err = PetscObjectSetName((PetscObject)sinfo.vector,
_metadata["default"].label.c_str());CHECK_PETSC_ERROR(err);
@@ -1151,7 +1179,19 @@
order->getLocalSize(), order->getGlobalSize());CHECK_PETSC_ERROR(err);
err = VecSetBlockSize(sinfo.vector, blockSize);CHECK_PETSC_ERROR(err);
err = VecSetFromOptions(sinfo.vector); CHECK_PETSC_ERROR(err);
+#endif
+ PetscInt localSize, globalSize;
+ err = PetscObjectReference((PetscObject) _dm);CHECK_PETSC_ERROR(err);
+ err = PetscObjectReference((PetscObject) _globalVec);CHECK_PETSC_ERROR(err);
+ err = PetscObjectSetName((PetscObject) _globalVec, _metadata["default"].label.c_str());CHECK_PETSC_ERROR(err);
+ err = VecGetSize(_localVec, &localSize);CHECK_PETSC_ERROR(err);
+ err = VecGetSize(_globalVec, &globalSize);CHECK_PETSC_ERROR(err);
+ assert(order->getLocalSize() == localSize);
+ assert(order->getGlobalSize() == globalSize);
+ sinfo.vector = _globalVec;
+ sinfo.dm = _dm;
+
#if 0
std::cout << "CONTEXT: " << context
<< ", orderLabel: " << orderLabel
@@ -1224,12 +1264,32 @@
} // else
// Create vector
+#if 0
err = VecCreate(mesh.comm(), &sinfo.vector);CHECK_PETSC_ERROR(err);
err = PetscObjectSetName((PetscObject)sinfo.vector, _metadata["default"].label.c_str());CHECK_PETSC_ERROR(err);
err = VecSetSizes(sinfo.vector, order->getLocalSize(), order->getGlobalSize());CHECK_PETSC_ERROR(err);
err = VecSetBlockSize(sinfo.vector, blockSize);CHECK_PETSC_ERROR(err);
- err = VecSetFromOptions(sinfo.vector); CHECK_PETSC_ERROR(err);
-
+ err = VecSetFromOptions(sinfo.vector);CHECK_PETSC_ERROR(err);
+#endif
+
+ PetscSection section, gsection;
+ PetscSF sf;
+
+ err = DMComplexClone(_dm, &sinfo.dm);CHECK_PETSC_ERROR(err);
+ err = DMGetDefaultSection(_dm, §ion);CHECK_PETSC_ERROR(err);
+ err = DMSetDefaultSection(sinfo.dm, section);CHECK_PETSC_ERROR(err);
+ err = DMGetPointSF(sinfo.dm, &sf);CHECK_PETSC_ERROR(err);
+ err = PetscSectionCreateGlobalSection(section, sf, PETSC_TRUE, &gsection);CHECK_PETSC_ERROR(err);
+ err = DMSetDefaultGlobalSection(sinfo.dm, gsection);CHECK_PETSC_ERROR(err);
+ err = DMCreateGlobalVector(sinfo.dm, &sinfo.vector);CHECK_PETSC_ERROR(err);
+ err = PetscObjectSetName((PetscObject) sinfo.vector, _metadata["default"].label.c_str());CHECK_PETSC_ERROR(err);
+ PetscInt localSize, globalSize;
+
+ err = PetscSectionGetStorageSize(section, &localSize);CHECK_PETSC_ERROR(err);
+ err = VecGetSize(sinfo.vector, &globalSize);CHECK_PETSC_ERROR(err);
+ assert(order->getLocalSize() == localSize);
+ assert(order->getGlobalSize() == globalSize);
+
logger.stagePop();
} // createScatterWithBC
@@ -1300,12 +1360,32 @@
} // else
// Create vector
+#if 0
err = VecCreate(mesh.comm(), &sinfo.vector);CHECK_PETSC_ERROR(err);
err = PetscObjectSetName((PetscObject)sinfo.vector, _metadata["default"].label.c_str());CHECK_PETSC_ERROR(err);
err = VecSetSizes(sinfo.vector,order->getLocalSize(), order->getGlobalSize());CHECK_PETSC_ERROR(err);
err = VecSetBlockSize(sinfo.vector, blockSize);CHECK_PETSC_ERROR(err);
err = VecSetFromOptions(sinfo.vector); CHECK_PETSC_ERROR(err);
+#endif
+ PetscSection section, gsection;
+ PetscSF sf;
+
+ err = DMComplexClone(_dm, &sinfo.dm);CHECK_PETSC_ERROR(err);
+ err = DMGetDefaultSection(_dm, §ion);CHECK_PETSC_ERROR(err);
+ err = DMSetDefaultSection(sinfo.dm, section);CHECK_PETSC_ERROR(err);
+ err = DMGetPointSF(sinfo.dm, &sf);CHECK_PETSC_ERROR(err);
+ err = PetscSectionCreateGlobalSection(section, sf, PETSC_TRUE, &gsection);CHECK_PETSC_ERROR(err);
+ err = DMSetDefaultGlobalSection(sinfo.dm, gsection);CHECK_PETSC_ERROR(err);
+ err = DMCreateGlobalVector(sinfo.dm, &sinfo.vector);CHECK_PETSC_ERROR(err);
+ err = PetscObjectSetName((PetscObject) sinfo.vector, _metadata["default"].label.c_str());CHECK_PETSC_ERROR(err);
+ PetscInt localSize, globalSize;
+
+ err = PetscSectionGetStorageSize(section, &localSize);CHECK_PETSC_ERROR(err);
+ err = VecGetSize(sinfo.vector, &globalSize);CHECK_PETSC_ERROR(err);
+ assert(order->getLocalSize() == localSize);
+ assert(order->getGlobalSize() == globalSize);
+
#if 0
std::cout << "["<<sieveMesh->commRank()<<"] CONTEXT: " << context
<< ", orderLabel: " << orderLabel
@@ -1330,6 +1410,8 @@
PetscVec
pylith::topology::Field<mesh_type, section_type>::vector(const char* context)
{ // vector
+ std::ostringstream msg;
+
ScatterInfo& sinfo = _getScatter(context);
return sinfo.vector;
} // vector
@@ -1340,6 +1422,8 @@
const PetscVec
pylith::topology::Field<mesh_type, section_type>::vector(const char* context) const
{ // vector
+ std::ostringstream msg;
+
const ScatterInfo& sinfo = _getScatter(context);
return sinfo.vector;
} // vector
@@ -1376,9 +1460,9 @@
err = VecScatterEnd(sinfo.scatter, sinfo.scatterVec, vector,
INSERT_VALUES, SCATTER_FORWARD); CHECK_PETSC_ERROR(err);
- if (_dm) {
- err = DMLocalToGlobalBegin(_dm, _localVec, ADD_VALUES, _globalVec);CHECK_PETSC_ERROR(err);
- err = DMLocalToGlobalEnd(_dm, _localVec, ADD_VALUES, _globalVec);CHECK_PETSC_ERROR(err);
+ if (sinfo.dm) {
+ err = DMLocalToGlobalBegin(sinfo.dm, _localVec, INSERT_VALUES, vector);CHECK_PETSC_ERROR(err);
+ err = DMLocalToGlobalEnd(sinfo.dm, _localVec, INSERT_VALUES, vector);CHECK_PETSC_ERROR(err);
}
} // scatterSectionToVector
@@ -1414,9 +1498,9 @@
err = VecScatterEnd(sinfo.scatter, vector, sinfo.scatterVec,
INSERT_VALUES, SCATTER_REVERSE); CHECK_PETSC_ERROR(err);
- if (_dm) {
- err = DMGlobalToLocalBegin(_dm, _globalVec, INSERT_VALUES, _localVec);CHECK_PETSC_ERROR(err);
- err = DMGlobalToLocalEnd(_dm, _globalVec, INSERT_VALUES, _localVec);CHECK_PETSC_ERROR(err);
+ if (sinfo.dm) {
+ err = DMGlobalToLocalBegin(sinfo.dm, vector, INSERT_VALUES, _localVec);CHECK_PETSC_ERROR(err);
+ err = DMGlobalToLocalEnd(sinfo.dm, vector, INSERT_VALUES, _localVec);CHECK_PETSC_ERROR(err);
}
} // scatterVectorToSection
@@ -1506,6 +1590,7 @@
ScatterInfo& sinfo = _scatters[context];
if (isNewScatter) {
+ sinfo.dm = 0;
sinfo.vector = 0;
sinfo.scatter = 0;
sinfo.scatterVec = 0;
Modified: short/3D/PyLith/trunk/libsrc/pylith/topology/Field.hh
===================================================================
--- short/3D/PyLith/trunk/libsrc/pylith/topology/Field.hh 2012-09-18 17:16:13 UTC (rev 20725)
+++ short/3D/PyLith/trunk/libsrc/pylith/topology/Field.hh 2012-09-18 22:40:40 UTC (rev 20726)
@@ -431,7 +431,9 @@
/// Data structures used in scattering to/from PETSc Vecs.
struct ScatterInfo {
+ DM dm; ///< PETSc DM defining the communication pattern
PetscVec vector; ///< PETSc vector associated with field.
+ // Deprecated
PetscVecScatter scatter; ///< PETSc scatter associated with field.
PetscVec scatterVec; ///< PETSC vector used in scattering.
}; // ScatterInfo
More information about the CIG-COMMITS
mailing list