My Project
Loading...
Searching...
No Matches
ISTLSolverEbos.hpp
1/*
2 Copyright 2016 IRIS AS
3 Copyright 2019, 2020 Equinor ASA
4 Copyright 2020 SINTEF Digital, Mathematics and Cybernetics
5
6 This file is part of the Open Porous Media project (OPM).
7
8 OPM is free software: you can redistribute it and/or modify
9 it under the terms of the GNU General Public License as published by
10 the Free Software Foundation, either version 3 of the License, or
11 (at your option) any later version.
12
13 OPM is distributed in the hope that it will be useful,
14 but WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 GNU General Public License for more details.
17
18 You should have received a copy of the GNU General Public License
19 along with OPM. If not, see <http://www.gnu.org/licenses/>.
20*/
21
22#ifndef OPM_ISTLSOLVER_EBOS_HEADER_INCLUDED
23#define OPM_ISTLSOLVER_EBOS_HEADER_INCLUDED
24
25#include <dune/istl/owneroverlapcopy.hh>
26#include <dune/istl/solver.hh>
27
28#include <ebos/eclbasevanguard.hh>
29
30#include <opm/common/ErrorMacros.hpp>
31#include <opm/common/Exceptions.hpp>
32#include <opm/common/TimingMacros.hpp>
33
34#include <opm/models/discretization/common/fvbaseproperties.hh>
35#include <opm/models/common/multiphasebaseproperties.hh>
36#include <opm/models/utils/parametersystem.hh>
37#include <opm/models/utils/propertysystem.hh>
38#include <opm/simulators/flow/BlackoilModelParametersEbos.hpp>
39#include <opm/simulators/linalg/ExtractParallelGridInformationToISTL.hpp>
40#include <opm/simulators/linalg/FlowLinearSolverParameters.hpp>
41#include <opm/simulators/linalg/matrixblock.hh>
42#include <opm/simulators/linalg/istlsparsematrixadapter.hh>
43#include <opm/simulators/linalg/PreconditionerWithUpdate.hpp>
44#include <opm/simulators/linalg/WellOperators.hpp>
45#include <opm/simulators/linalg/WriteSystemMatrixHelper.hpp>
46#include <opm/simulators/linalg/findOverlapRowsAndColumns.hpp>
47#include <opm/simulators/linalg/getQuasiImpesWeights.hpp>
48#include <opm/simulators/linalg/setupPropertyTree.hpp>
49
50#include <any>
51#include <cstddef>
52#include <functional>
53#include <memory>
54#include <set>
55#include <sstream>
56#include <string>
57#include <tuple>
58#include <vector>
59
60namespace Opm::Properties {
61
62namespace TTag {
64 using InheritsFrom = std::tuple<FlowIstlSolverParams>;
65};
66}
67
68template <class TypeTag, class MyTypeTag>
70
73template<class TypeTag>
74struct SparseMatrixAdapter<TypeTag, TTag::FlowIstlSolver>
75{
76private:
80
81public:
82 using type = typename Linear::IstlSparseMatrixAdapter<Block>;
83};
84
85} // namespace Opm::Properties
86
87namespace Opm
88{
89
90
91namespace detail
92{
93
94template<class Matrix, class Vector, class Comm>
95struct FlexibleSolverInfo
96{
97 using AbstractSolverType = Dune::InverseOperator<Vector, Vector>;
98 using AbstractOperatorType = Dune::AssembledLinearOperator<Matrix, Vector, Vector>;
99 using AbstractPreconditionerType = Dune::PreconditionerWithUpdate<Vector, Vector>;
100
101 void create(const Matrix& matrix,
102 bool parallel,
103 const PropertyTree& prm,
104 std::size_t pressureIndex,
105 std::function<Vector()> trueFunc,
106 const bool forceSerial,
107 Comm& comm);
108
109 std::unique_ptr<AbstractSolverType> solver_;
110 std::unique_ptr<AbstractOperatorType> op_;
111 std::unique_ptr<LinearOperatorExtra<Vector,Vector>> wellOperator_;
112 AbstractPreconditionerType* pre_ = nullptr;
113 std::size_t interiorCellNum_ = 0;
114};
115
116
117#ifdef HAVE_MPI
119void copyParValues(std::any& parallelInformation, std::size_t size,
120 Dune::OwnerOverlapCopyCommunication<int,int>& comm);
121#endif
122
125template<class Matrix>
126void makeOverlapRowsInvalid(Matrix& matrix,
127 const std::vector<int>& overlapRows);
128
131template<class Matrix, class Grid>
132std::unique_ptr<Matrix> blockJacobiAdjacency(const Grid& grid,
133 const std::vector<int>& cell_part,
134 std::size_t nonzeroes,
135 const std::vector<std::set<int>>& wellConnectionsGraph);
136}
137
142 template <class TypeTag>
144 {
145 protected:
153 using Matrix = typename SparseMatrixAdapter::IstlMatrix;
156 using AbstractSolverType = Dune::InverseOperator<Vector, Vector>;
157 using AbstractOperatorType = Dune::AssembledLinearOperator<Matrix, Vector, Vector>;
161 constexpr static std::size_t pressureIndex = GetPropType<TypeTag, Properties::Indices>::pressureSwitchIdx;
162
163#if HAVE_MPI
164 using CommunicationType = Dune::OwnerOverlapCopyCommunication<int,int>;
165#else
166 using CommunicationType = Dune::CollectiveCommunication<int>;
167#endif
168
169 public:
170 using AssembledLinearOperatorType = Dune::AssembledLinearOperator< Matrix, Vector, Vector >;
171
172 static void registerParameters()
173 {
174 FlowLinearSolverParameters::registerParameters<TypeTag>();
175 }
176
185 : simulator_(simulator),
186 iterations_( 0 ),
187 converged_(false),
188 matrix_(nullptr),
189 parameters_{parameters},
190 forceSerial_(forceSerial)
191 {
192 initialize();
193 }
194
197 explicit ISTLSolverEbos(const Simulator& simulator)
198 : simulator_(simulator),
199 iterations_( 0 ),
200 solveCount_(0),
201 converged_(false),
202 matrix_(nullptr)
203 {
204 parameters_.resize(1);
205 parameters_[0].template init<TypeTag>(simulator_.vanguard().eclState().getSimulationConfig().useCPR());
206 initialize();
207 }
208
209 void initialize()
210 {
212
213 if (parameters_[0].linsolver_ == "hybrid") {
214 // Experimental hybrid configuration.
215 // When chosen, will set up two solvers, one with CPRW
216 // and the other with ILU0 preconditioner. More general
217 // options may be added later.
218 prm_.clear();
219 parameters_.clear();
220 {
222 para.init<TypeTag>(false);
223 para.linsolver_ = "cprw";
224 parameters_.push_back(para);
225 prm_.push_back(setupPropertyTree(parameters_[0],
226 EWOMS_PARAM_IS_SET(TypeTag, int, LinearSolverMaxIter),
227 EWOMS_PARAM_IS_SET(TypeTag, double, LinearSolverReduction)));
228 }
229 {
230 FlowLinearSolverParameters para;
231 para.init<TypeTag>(false);
232 para.linsolver_ = "ilu0";
233 parameters_.push_back(para);
234 prm_.push_back(setupPropertyTree(parameters_[1],
235 EWOMS_PARAM_IS_SET(TypeTag, int, LinearSolverMaxIter),
236 EWOMS_PARAM_IS_SET(TypeTag, double, LinearSolverReduction)));
237 }
238 // ------------
239 } else {
240 // Do a normal linear solver setup.
241 assert(parameters_.size() == 1);
242 assert(prm_.empty());
243 prm_.push_back(setupPropertyTree(parameters_[0],
244 EWOMS_PARAM_IS_SET(TypeTag, int, LinearSolverMaxIter),
245 EWOMS_PARAM_IS_SET(TypeTag, double, LinearSolverReduction)));
246 }
247 flexibleSolver_.resize(prm_.size());
248
249 const bool on_io_rank = (simulator_.gridView().comm().rank() == 0);
250#if HAVE_MPI
251 comm_.reset( new CommunicationType( simulator_.vanguard().grid().comm() ) );
252#endif
253 extractParallelGridInformationToISTL(simulator_.vanguard().grid(), parallelInformation_);
254
255 // For some reason simulator_.model().elementMapper() is not initialized at this stage
256 //const auto& elemMapper = simulator_.model().elementMapper(); //does not work.
257 // Set it up manually
258 ElementMapper elemMapper(simulator_.vanguard().gridView(), Dune::mcmgElementLayout());
259 detail::findOverlapAndInterior(simulator_.vanguard().grid(), elemMapper, overlapRows_, interiorRows_);
260 useWellConn_ = EWOMS_GET_PARAM(TypeTag, bool, MatrixAddWellContributions);
261 const bool ownersFirst = EWOMS_GET_PARAM(TypeTag, bool, OwnerCellsFirst);
262 if (!ownersFirst) {
263 const std::string msg = "The linear solver no longer supports --owner-cells-first=false.";
264 if (on_io_rank) {
265 OpmLog::error(msg);
266 }
267 OPM_THROW_NOLOG(std::runtime_error, msg);
268 }
269
270 const int interiorCellNum_ = detail::numMatrixRowsToUseInSolver(simulator_.vanguard().grid(), true);
271 for (auto& f : flexibleSolver_) {
272 f.interiorCellNum_ = interiorCellNum_;
273 }
274
275 // Print parameters to PRT/DBG logs.
276 if (on_io_rank && parameters_[activeSolverNum_].linear_solver_print_json_definition_) {
277 std::ostringstream os;
278 os << "Property tree for linear solvers:\n";
279 for (std::size_t i = 0; i<prm_.size(); i++) {
280 prm_[i].write_json(os, true);
281 }
282 OpmLog::note(os.str());
283 }
284 }
285
286 // nothing to clean here
287 void eraseMatrix()
288 {
289 }
290
291 void setActiveSolver(const int num)
292 {
293 if (num > static_cast<int>(prm_.size()) - 1) {
294 OPM_THROW(std::logic_error, "Solver number " + std::to_string(num) + " not available.");
295 }
296 activeSolverNum_ = num;
297 if (simulator_.gridView().comm().rank() == 0) {
298 OpmLog::debug("Active solver = " + std::to_string(activeSolverNum_)
299 + " (" + parameters_[activeSolverNum_].linsolver_ + ")");
300 }
301 }
302
303 int numAvailableSolvers()
304 {
305 return flexibleSolver_.size();
306 }
307
308 void prepare(const SparseMatrixAdapter& M, Vector& b)
309 {
310 prepare(M.istlMatrix(), b);
311 }
312
313 void prepare(const Matrix& M, Vector& b)
314 {
315 OPM_TIMEBLOCK(istlSolverEbosPrepare);
316 const bool firstcall = (matrix_ == nullptr);
317#if HAVE_MPI
318 if (firstcall && isParallel()) {
319 const std::size_t size = M.N();
320 detail::copyParValues(parallelInformation_, size, *comm_);
321 }
322#endif
323
324 // update matrix entries for solvers.
325 if (firstcall) {
326 // ebos will not change the matrix object. Hence simply store a pointer
327 // to the original one with a deleter that does nothing.
328 // Outch! We need to be able to scale the linear system! Hence const_cast
329 matrix_ = const_cast<Matrix*>(&M);
330
331 useWellConn_ = EWOMS_GET_PARAM(TypeTag, bool, MatrixAddWellContributions);
332 // setup sparsity pattern for jacobi matrix for preconditioner (only used for openclSolver)
333 } else {
334 // Pointers should not change
335 if ( &M != matrix_ ) {
336 OPM_THROW(std::logic_error,
337 "Matrix objects are expected to be reused when reassembling!");
338 }
339 }
340 rhs_ = &b;
341
342 // TODO: check all solvers, not just one.
343 if (isParallel() && prm_[activeSolverNum_].template get<std::string>("preconditioner.type") != "ParOverILU0") {
344 detail::makeOverlapRowsInvalid(getMatrix(), overlapRows_);
345 }
346 prepareFlexibleSolver();
347 }
348
349
350 void setResidual(Vector& /* b */)
351 {
352 // rhs_ = &b; // Must be handled in prepare() instead.
353 }
354
355 void getResidual(Vector& b) const
356 {
357 b = *rhs_;
358 }
359
360 void setMatrix(const SparseMatrixAdapter& /* M */)
361 {
362 // matrix_ = &M.istlMatrix(); // Must be handled in prepare() instead.
363 }
364
365 int getSolveCount() const {
366 return solveCount_;
367 }
368
369 void resetSolveCount() {
370 solveCount_ = 0;
371 }
372
373 bool solve(Vector& x)
374 {
375 OPM_TIMEBLOCK(istlSolverEbosSolve);
376 ++solveCount_;
377 // Write linear system if asked for.
378 const int verbosity = prm_[activeSolverNum_].get("verbosity", 0);
379 const bool write_matrix = verbosity > 10;
380 if (write_matrix) {
381 Helper::writeSystem(simulator_, //simulator is only used to get names
382 getMatrix(),
383 *rhs_,
384 comm_.get());
385 }
386
387 // Solve system.
388 Dune::InverseOperatorResult result;
389 {
390 OPM_TIMEBLOCK(flexibleSolverApply);
391 assert(flexibleSolver_[activeSolverNum_].solver_);
392 flexibleSolver_[activeSolverNum_].solver_->apply(x, *rhs_, result);
393 }
394
395 // Check convergence, iterations etc.
396 checkConvergence(result);
397
398 return converged_;
399 }
400
401
407
409 int iterations () const { return iterations_; }
410
412 const std::any& parallelInformation() const { return parallelInformation_; }
413
414 const CommunicationType* comm() const { return comm_.get(); }
415
416 protected:
417#if HAVE_MPI
418 using Comm = Dune::OwnerOverlapCopyCommunication<int, int>;
419#endif
420
421 void checkConvergence( const Dune::InverseOperatorResult& result ) const
422 {
423 // store number of iterations
424 iterations_ = result.iterations;
425 converged_ = result.converged;
426 if(!converged_){
427 if(result.reduction < parameters_[activeSolverNum_].relaxed_linear_solver_reduction_){
428 std::stringstream ss;
429 ss<< "Full linear solver tolerance not achieved. The reduction is:" << result.reduction
430 << " after " << result.iterations << " iterations ";
431 OpmLog::warning(ss.str());
432 converged_ = true;
433 }
434 }
435 // Check for failure of linear solver.
436 if (!parameters_[activeSolverNum_].ignoreConvergenceFailure_ && !converged_) {
437 const std::string msg("Convergence failure for linear solver.");
438 OPM_THROW_NOLOG(NumericalProblem, msg);
439 }
440 }
441 protected:
442
443 bool isParallel() const {
444#if HAVE_MPI
445 return !forceSerial_ && comm_->communicator().size() > 1;
446#else
447 return false;
448#endif
449 }
450
451 void prepareFlexibleSolver()
452 {
453 OPM_TIMEBLOCK(flexibleSolverPrepare);
454 if (shouldCreateSolver()) {
455 std::function<Vector()> trueFunc =
456 [this]
457 {
458 return this->getTrueImpesWeights(pressureIndex);
459 };
460
461 if (!useWellConn_) {
462 auto wellOp = std::make_unique<WellModelOperator>(simulator_.problem().wellModel());
463 flexibleSolver_[activeSolverNum_].wellOperator_ = std::move(wellOp);
464 }
465 OPM_TIMEBLOCK(flexibleSolverCreate);
466 flexibleSolver_[activeSolverNum_].create(getMatrix(),
467 isParallel(),
468 prm_[activeSolverNum_],
469 pressureIndex,
470 trueFunc,
471 forceSerial_,
472 *comm_);
473 }
474 else
475 {
476 OPM_TIMEBLOCK(flexibleSolverUpdate);
477 flexibleSolver_[activeSolverNum_].pre_->update();
478 }
479 }
480
481
485 {
486 // Decide if we should recreate the solver or just do
487 // a minimal preconditioner update.
488 if (flexibleSolver_.empty()) {
489 return true;
490 }
491 if (!flexibleSolver_[activeSolverNum_].solver_) {
492 return true;
493 }
494 if (this->parameters_[activeSolverNum_].cpr_reuse_setup_ == 0) {
495 // Always recreate solver.
496 return true;
497 }
498 if (this->parameters_[activeSolverNum_].cpr_reuse_setup_ == 1) {
499 // Recreate solver on the first iteration of every timestep.
500 const int newton_iteration = this->simulator_.model().newtonMethod().numIterations();
501 return newton_iteration == 0;
502 }
503 if (this->parameters_[activeSolverNum_].cpr_reuse_setup_ == 2) {
504 // Recreate solver if the last solve used more than 10 iterations.
505 return this->iterations() > 10;
506 }
507 if (this->parameters_[activeSolverNum_].cpr_reuse_setup_ == 3) {
508 // Recreate solver if the last solve used more than 10 iterations.
509 return false;
510 }
511 if (this->parameters_[activeSolverNum_].cpr_reuse_setup_ == 4) {
512 // Recreate solver every 'step' solve calls.
513 const int step = this->parameters_[activeSolverNum_].cpr_reuse_interval_;
514 const bool create = ((solveCount_ % step) == 0);
515 return create;
516 }
517
518 // If here, we have an invalid parameter.
519 const bool on_io_rank = (simulator_.gridView().comm().rank() == 0);
520 std::string msg = "Invalid value: " + std::to_string(this->parameters_[activeSolverNum_].cpr_reuse_setup_)
521 + " for --cpr-reuse-setup parameter, run with --help to see allowed values.";
522 if (on_io_rank) {
523 OpmLog::error(msg);
524 }
525 throw std::runtime_error(msg);
526
527 // Never reached.
528 return false;
529 }
530
531
532 // Weights to make approximate pressure equations.
533 // Calculated from the storage terms (only) of the
534 // conservation equations, ignoring all other terms.
535 Vector getTrueImpesWeights(int pressureVarIndex) const
536 {
537 OPM_TIMEBLOCK(getTrueImpesWeights);
538 Vector weights(rhs_->size());
539 ElementContext elemCtx(simulator_);
540 Amg::getTrueImpesWeights(pressureVarIndex, weights,
541 simulator_.vanguard().gridView(),
542 elemCtx, simulator_.model(),
543 ThreadManager::threadId());
544 return weights;
545 }
546
547 Matrix& getMatrix()
548 {
549 return *matrix_;
550 }
551
552 const Matrix& getMatrix() const
553 {
554 return *matrix_;
555 }
556
557 const Simulator& simulator_;
558 mutable int iterations_;
559 mutable int solveCount_;
560 mutable bool converged_;
561 std::any parallelInformation_;
562
563 // non-const to be able to scale the linear system
564 Matrix* matrix_;
565 Vector *rhs_;
566
567 int activeSolverNum_ = 0;
568 std::vector<detail::FlexibleSolverInfo<Matrix,Vector,CommunicationType>> flexibleSolver_;
569 std::vector<int> overlapRows_;
570 std::vector<int> interiorRows_;
571
572 bool useWellConn_;
573
574 std::vector<FlowLinearSolverParameters> parameters_;
575 bool forceSerial_ = false;
576 std::vector<PropertyTree> prm_;
577
578 std::shared_ptr< CommunicationType > comm_;
579 }; // end ISTLSolver
580
581} // namespace Opm
582#endif
Interface class adding the update() method to the preconditioner interface.
Definition PreconditionerWithUpdate.hpp:32
Definition AquiferInterface.hpp:35
This class solves the fully implicit black-oil system by solving the reduced system (after eliminatin...
Definition ISTLSolverEbos.hpp:144
ISTLSolverEbos(const Simulator &simulator, const FlowLinearSolverParameters &parameters, bool forceSerial=false)
Construct a system solver.
Definition ISTLSolverEbos.hpp:184
int iterations() const
Solve the system of linear equations Ax = b, with A being the combined derivative matrix of the resid...
Definition ISTLSolverEbos.hpp:409
const std::any & parallelInformation() const
Definition ISTLSolverEbos.hpp:412
ISTLSolverEbos(const Simulator &simulator)
Construct a system solver.
Definition ISTLSolverEbos.hpp:197
bool shouldCreateSolver() const
Return true if we should (re)create the whole solver, instead of just calling update() on the precond...
Definition ISTLSolverEbos.hpp:484
Definition PropertyTree.hpp:37
This file contains a set of helper functions used by VFPProd / VFPInj.
Definition BlackoilPhases.hpp:27
PropertyTree setupPropertyTree(FlowLinearSolverParameters p, bool linearSolverMaxIterSet, bool linearSolverReductionSet)
Set up a property tree intended for FlexibleSolver by either reading the tree from a JSON file or cre...
Definition setupPropertyTree.cpp:40
This class carries all parameters for the NewtonIterationBlackoilInterleaved class.
Definition FlowLinearSolverParameters.hpp:237
Definition ISTLSolverEbos.hpp:69
Definition ISTLSolverEbos.hpp:63