My Project
FlowMainEbos.hpp
1/*
2 Copyright 2013, 2014, 2015 SINTEF ICT, Applied Mathematics.
3 Copyright 2014 Dr. Blatt - HPC-Simulation-Software & Services
4 Copyright 2015 IRIS AS
5 Copyright 2014 STATOIL ASA.
6
7 This file is part of the Open Porous Media project (OPM).
8
9 OPM is free software: you can redistribute it and/or modify
10 it under the terms of the GNU General Public License as published by
11 the Free Software Foundation, either version 3 of the License, or
12 (at your option) any later version.
13
14 OPM is distributed in the hope that it will be useful,
15 but WITHOUT ANY WARRANTY; without even the implied warranty of
16 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 GNU General Public License for more details.
18
19 You should have received a copy of the GNU General Public License
20 along with OPM. If not, see <http://www.gnu.org/licenses/>.
21*/
22#ifndef OPM_FLOW_MAIN_EBOS_HEADER_INCLUDED
23#define OPM_FLOW_MAIN_EBOS_HEADER_INCLUDED
24
25#include <opm/simulators/flow/Banners.hpp>
26#include <opm/simulators/flow/SimulatorFullyImplicitBlackoilEbos.hpp>
27
28#include <opm/input/eclipse/EclipseState/EclipseState.hpp>
29#include <opm/input/eclipse/EclipseState/IOConfig/IOConfig.hpp>
30#include <opm/input/eclipse/EclipseState/InitConfig/InitConfig.hpp>
31
32#if HAVE_DUNE_FEM
33#include <dune/fem/misc/mpimanager.hh>
34#else
35#include <dune/common/parallel/mpihelper.hh>
36#endif
37
38#include <memory>
39#include <string_view>
40
41namespace Opm::Properties {
42
43template<class TypeTag, class MyTypeTag>
45 using type = UndefinedProperty;
46};
47template<class TypeTag, class MyTypeTag>
49 using type = UndefinedProperty;
50};
51template<class TypeTag, class MyTypeTag>
53 using type = UndefinedProperty;
54};
55
56// TODO: enumeration parameters. we use strings for now.
57template<class TypeTag>
58struct EnableDryRun<TypeTag, TTag::EclFlowProblem> {
59 static constexpr auto value = "auto";
60};
61// Do not merge parallel output files or warn about them
62template<class TypeTag>
63struct EnableLoggingFalloutWarning<TypeTag, TTag::EclFlowProblem> {
64 static constexpr bool value = false;
65};
66template<class TypeTag>
67struct OutputInterval<TypeTag, TTag::EclFlowProblem> {
68 static constexpr int value = 1;
69};
70
71} // namespace Opm::Properties
72
73namespace Opm {
74namespace detail {
75
76void mergeParallelLogFiles(std::string_view output_dir,
77 std::string_view deck_filename,
78 bool enableLoggingFalloutWarning);
79
80void handleExtraConvergenceOutput(SimulatorReport& report,
81 std::string_view option,
82 std::string_view optionName,
83 std::string_view output_dir,
84 std::string_view base_name);
85
86}
87
88 class Deck;
89
90 // The FlowMain class is the ebos based black-oil simulator.
91 template <class TypeTag>
93 {
94 public:
95 using MaterialLawManager = typename GetProp<TypeTag, Properties::MaterialLaw>::EclMaterialLawManager;
96 using EbosSimulator = GetPropType<TypeTag, Properties::Simulator>;
97 using Grid = GetPropType<TypeTag, Properties::Grid>;
98 using GridView = GetPropType<TypeTag, Properties::GridView>;
99 using Problem = GetPropType<TypeTag, Properties::Problem>;
100 using Scalar = GetPropType<TypeTag, Properties::Scalar>;
101 using FluidSystem = GetPropType<TypeTag, Properties::FluidSystem>;
102
104
105 FlowMainEbos(int argc, char **argv, bool output_cout, bool output_files )
106 : argc_{argc}, argv_{argv},
107 output_cout_{output_cout}, output_files_{output_files}
108 {
109
110 }
111
112 // Read the command line parameters. Throws an exception if something goes wrong.
113 static int setupParameters_(int argc, char** argv, Parallel::Communication comm)
114 {
115 using ParamsMeta = GetProp<TypeTag, Properties::ParameterMetaData>;
116 if (!ParamsMeta::registrationOpen()) {
117 // We have already successfully run setupParameters_().
118 // For the dynamically chosen runs (as from the main flow
119 // executable) we must run this function again with the
120 // real typetag to be used, as the first time was with the
121 // "FlowEarlyBird" typetag. However, for the static ones (such
122 // as 'flow_onephase_energy') it has already been run with the
123 // correct typetag.
124 return EXIT_SUCCESS;
125 }
126 // register the flow specific parameters
127 EWOMS_REGISTER_PARAM(TypeTag, std::string, EnableDryRun,
128 "Specify if the simulation ought to be actually run, or just pretended to be");
129 EWOMS_REGISTER_PARAM(TypeTag, int, OutputInterval,
130 "Specify the number of report steps between two consecutive writes of restart data");
131 EWOMS_REGISTER_PARAM(TypeTag, bool, EnableLoggingFalloutWarning,
132 "Developer option to see whether logging was on non-root processors. In that case it will be appended to the *.DBG or *.PRT files");
133
134 Simulator::registerParameters();
135
136 // register the parameters inherited from ebos
137 registerAllParameters_<TypeTag>(/*finalizeRegistration=*/false);
138
139 // hide the parameters unused by flow. TODO: this is a pain to maintain
140 EWOMS_HIDE_PARAM(TypeTag, EnableGravity);
141 EWOMS_HIDE_PARAM(TypeTag, EnableGridAdaptation);
142
143 // this parameter is actually used in eWoms, but the flow well model
144 // hard-codes the assumption that the intensive quantities cache is enabled,
145 // so flow crashes. Let's hide the parameter for that reason.
146 EWOMS_HIDE_PARAM(TypeTag, EnableIntensiveQuantityCache);
147
148 // thermodynamic hints are not implemented/required by the eWoms blackoil
149 // model
150 EWOMS_HIDE_PARAM(TypeTag, EnableThermodynamicHints);
151
152 // in flow only the deck file determines the end time of the simulation
153 EWOMS_HIDE_PARAM(TypeTag, EndTime);
154
155 // time stepping is not done by the eWoms code in flow
156 EWOMS_HIDE_PARAM(TypeTag, InitialTimeStepSize);
157 EWOMS_HIDE_PARAM(TypeTag, MaxTimeStepDivisions);
158 EWOMS_HIDE_PARAM(TypeTag, MaxTimeStepSize);
159 EWOMS_HIDE_PARAM(TypeTag, MinTimeStepSize);
160 EWOMS_HIDE_PARAM(TypeTag, PredeterminedTimeStepsFile);
161
162 EWOMS_HIDE_PARAM(TypeTag, EclMaxTimeStepSizeAfterWellEvent);
163 EWOMS_HIDE_PARAM(TypeTag, EclRestartShrinkFactor);
164 EWOMS_HIDE_PARAM(TypeTag, EclEnableTuning);
165
166 // flow also does not use the eWoms Newton method
167 EWOMS_HIDE_PARAM(TypeTag, NewtonMaxError);
168 EWOMS_HIDE_PARAM(TypeTag, NewtonTolerance);
169 EWOMS_HIDE_PARAM(TypeTag, NewtonTargetIterations);
170 EWOMS_HIDE_PARAM(TypeTag, NewtonVerbose);
171 EWOMS_HIDE_PARAM(TypeTag, NewtonWriteConvergence);
172 EWOMS_HIDE_PARAM(TypeTag, EclNewtonSumTolerance);
173 EWOMS_HIDE_PARAM(TypeTag, EclNewtonSumToleranceExponent);
174 EWOMS_HIDE_PARAM(TypeTag, EclNewtonStrictIterations);
175 EWOMS_HIDE_PARAM(TypeTag, EclNewtonRelaxedVolumeFraction);
176 EWOMS_HIDE_PARAM(TypeTag, EclNewtonRelaxedTolerance);
177
178 // the default eWoms checkpoint/restart mechanism does not work with flow
179 EWOMS_HIDE_PARAM(TypeTag, RestartTime);
180 EWOMS_HIDE_PARAM(TypeTag, RestartWritingInterval);
181 // hide all vtk related it is not currently possible to do this dependet on if the vtk writing is used
182 //if(not(EWOMS_GET_PARAM(TypeTag,bool,EnableVtkOutput))){
183 EWOMS_HIDE_PARAM(TypeTag, VtkWriteOilFormationVolumeFactor);
184 EWOMS_HIDE_PARAM(TypeTag, VtkWriteOilSaturationPressure);
185 EWOMS_HIDE_PARAM(TypeTag, VtkWriteOilVaporizationFactor);
186 EWOMS_HIDE_PARAM(TypeTag, VtkWritePorosity);
187 EWOMS_HIDE_PARAM(TypeTag, VtkWritePotentialGradients);
188 EWOMS_HIDE_PARAM(TypeTag, VtkWritePressures);
189 EWOMS_HIDE_PARAM(TypeTag, VtkWritePrimaryVars);
190 EWOMS_HIDE_PARAM(TypeTag, VtkWritePrimaryVarsMeaning);
191 EWOMS_HIDE_PARAM(TypeTag, VtkWriteProcessRank);
192 EWOMS_HIDE_PARAM(TypeTag, VtkWriteRelativePermeabilities);
193 EWOMS_HIDE_PARAM(TypeTag, VtkWriteSaturatedGasOilVaporizationFactor);
194 EWOMS_HIDE_PARAM(TypeTag, VtkWriteSaturatedOilGasDissolutionFactor);
195 EWOMS_HIDE_PARAM(TypeTag, VtkWriteSaturationRatios);
196 EWOMS_HIDE_PARAM(TypeTag, VtkWriteSaturations);
197 EWOMS_HIDE_PARAM(TypeTag, VtkWriteTemperature);
198 EWOMS_HIDE_PARAM(TypeTag, VtkWriteViscosities);
199 EWOMS_HIDE_PARAM(TypeTag, VtkWriteWaterFormationVolumeFactor);
200 EWOMS_HIDE_PARAM(TypeTag, VtkWriteGasDissolutionFactor);
201 EWOMS_HIDE_PARAM(TypeTag, VtkWriteGasFormationVolumeFactor);
202 EWOMS_HIDE_PARAM(TypeTag, VtkWriteGasSaturationPressure);
203 EWOMS_HIDE_PARAM(TypeTag, VtkWriteIntrinsicPermeabilities);
204 EWOMS_HIDE_PARAM(TypeTag, VtkWriteEclTracerConcentration);
205 EWOMS_HIDE_PARAM(TypeTag, VtkWriteExtrusionFactor);
206 EWOMS_HIDE_PARAM(TypeTag, VtkWriteFilterVelocities);
207 EWOMS_HIDE_PARAM(TypeTag, VtkWriteDensities);
208 EWOMS_HIDE_PARAM(TypeTag, VtkWriteDofIndex);
209 EWOMS_HIDE_PARAM(TypeTag, VtkWriteMobilities);
210 //}
211 EWOMS_HIDE_PARAM(TypeTag, VtkWriteAverageMolarMasses);
212 EWOMS_HIDE_PARAM(TypeTag, VtkWriteFugacities);
213 EWOMS_HIDE_PARAM(TypeTag, VtkWriteFugacityCoeffs);
214 EWOMS_HIDE_PARAM(TypeTag, VtkWriteMassFractions);
215 EWOMS_HIDE_PARAM(TypeTag, VtkWriteMolarities);
216 EWOMS_HIDE_PARAM(TypeTag, VtkWriteMoleFractions);
217 EWOMS_HIDE_PARAM(TypeTag, VtkWriteTotalMassFractions);
218 EWOMS_HIDE_PARAM(TypeTag, VtkWriteTotalMoleFractions);
219
220 EWOMS_HIDE_PARAM(TypeTag, VtkWriteTortuosities);
221 EWOMS_HIDE_PARAM(TypeTag, VtkWriteDiffusionCoefficients);
222 EWOMS_HIDE_PARAM(TypeTag, VtkWriteEffectiveDiffusionCoefficients);
223
224 EWOMS_END_PARAM_REGISTRATION(TypeTag);
225
226 int mpiRank = comm.rank();
227
228 // read in the command line parameters
229 int status = ::Opm::setupParameters_<TypeTag>(argc, const_cast<const char**>(argv), /*doRegistration=*/false, /*allowUnused=*/true, /*handleHelp=*/(mpiRank==0));
230 if (status == 0) {
231
232 // deal with unknown parameters.
233
234 int unknownKeyWords = 0;
235 if (mpiRank == 0) {
236 unknownKeyWords = Parameters::printUnused<TypeTag>(std::cerr);
237 }
238 int globalUnknownKeyWords = comm.sum(unknownKeyWords);
239 unknownKeyWords = globalUnknownKeyWords;
240 if ( unknownKeyWords )
241 {
242 if ( mpiRank == 0 )
243 {
244 std::string msg = "Aborting simulation due to unknown "
245 "parameters. Please query \"flow --help\" for "
246 "supported command line parameters.";
247 if (OpmLog::hasBackend("STREAMLOG"))
248 {
249 OpmLog::error(msg);
250 }
251 else {
252 std::cerr << msg << std::endl;
253 }
254 }
255 return EXIT_FAILURE;
256 }
257
258 // deal with --print-properties and --print-parameters and unknown parameters.
259
260 bool doExit = false;
261
262 if (EWOMS_GET_PARAM(TypeTag, int, PrintProperties) == 1) {
263 doExit = true;
264 if (mpiRank == 0)
265 Properties::printValues<TypeTag>(std::cout);
266 }
267
268 if (EWOMS_GET_PARAM(TypeTag, int, PrintParameters) == 1) {
269 doExit = true;
270 if (mpiRank == 0)
271 Parameters::printValues<TypeTag>();
272 }
273
274 if (doExit)
275 return -1;
276 }
277
278 return status;
279 }
280
285 {
286 return execute_(&FlowMainEbos::runSimulator, /*cleanup=*/true);
287 }
288
289 int executeInitStep()
290 {
291 return execute_(&FlowMainEbos::runSimulatorInit, /*cleanup=*/false);
292 }
293
294 // Returns true unless "EXIT" was encountered in the schedule
295 // section of the input datafile.
296 int executeStep()
297 {
298 return simulator_->runStep(*simtimer_);
299 }
300
301 // Called from Python to cleanup after having executed the last
302 // executeStep()
303 int executeStepsCleanup()
304 {
305 SimulatorReport report = simulator_->finalize();
306 runSimulatorAfterSim_(report);
307 return report.success.exit_status;
308 }
309
310 EbosSimulator *getSimulatorPtr() {
311 return ebosSimulator_.get();
312 }
313
314 SimulatorTimer* getSimTimer() {
315 return simtimer_.get();
316 }
317
318 private:
319 // called by execute() or executeInitStep()
320 int execute_(int (FlowMainEbos::* runOrInitFunc)(), bool cleanup)
321 {
322 try {
323 // deal with some administrative boilerplate
324
325 int status = setupParameters_(this->argc_, this->argv_, EclGenericVanguard::comm());
326 if (status)
327 return status;
328
329 setupParallelism();
330 setupEbosSimulator();
332
333 // if run, do the actual work, else just initialize
334 int exitCode = (this->*runOrInitFunc)();
335 if (cleanup) {
336 executeCleanup_();
337 }
338 return exitCode;
339 }
340 catch (const std::exception& e) {
341 std::ostringstream message;
342 message << "Program threw an exception: " << e.what();
343
344 if (this->output_cout_) {
345 // in some cases exceptions are thrown before the logging system is set
346 // up.
347 if (OpmLog::hasBackend("STREAMLOG")) {
348 OpmLog::error(message.str());
349 }
350 else {
351 std::cout << message.str() << "\n";
352 }
353 }
354#if HAVE_MPI
355 if (this->mpi_size_ > 1)
356 MPI_Abort(MPI_COMM_WORLD, EXIT_FAILURE);
357#endif
358 return EXIT_FAILURE;
359 }
360 }
361
362 void executeCleanup_() {
363 // clean up
364 mergeParallelLogFiles();
365 }
366
367 protected:
368 void setupParallelism()
369 {
370 // determine the rank of the current process and the number of processes
371 // involved in the simulation. MPI must have already been initialized
372 // here. (yes, the name of this method is misleading.)
373 auto comm = EclGenericVanguard::comm();
374 mpi_rank_ = comm.rank();
375 mpi_size_ = comm.size();
376
377#if _OPENMP
378 // if openMP is available, default to 2 threads per process.
379 if (!getenv("OMP_NUM_THREADS"))
380 omp_set_num_threads(std::min(2, omp_get_num_procs()));
381#endif
382
383 using ThreadManager = GetPropType<TypeTag, Properties::ThreadManager>;
384 ThreadManager::init();
385 }
386
387 void mergeParallelLogFiles()
388 {
389 // force closing of all log files.
390 OpmLog::removeAllBackends();
391
392 if (mpi_rank_ != 0 || mpi_size_ < 2 || !this->output_files_) {
393 return;
394 }
395
396 detail::mergeParallelLogFiles(eclState().getIOConfig().getOutputDir(),
397 EWOMS_GET_PARAM(TypeTag, std::string, EclDeckFileName),
398 EWOMS_GET_PARAM(TypeTag, bool, EnableLoggingFalloutWarning));
399 }
400
401 void setupEbosSimulator()
402 {
403 ebosSimulator_ = std::make_unique<EbosSimulator>(EclGenericVanguard::comm(), /*verbose=*/false);
404 ebosSimulator_->executionTimer().start();
405 ebosSimulator_->model().applyInitialSolution();
406
407 try {
408 // Possible to force initialization only behavior (NOSIM).
409 const std::string& dryRunString = EWOMS_GET_PARAM(TypeTag, std::string, EnableDryRun);
410 if (dryRunString != "" && dryRunString != "auto") {
411 bool yesno;
412 if (dryRunString == "true"
413 || dryRunString == "t"
414 || dryRunString == "1")
415 yesno = true;
416 else if (dryRunString == "false"
417 || dryRunString == "f"
418 || dryRunString == "0")
419 yesno = false;
420 else
421 throw std::invalid_argument("Invalid value for parameter EnableDryRun: '"
422 +dryRunString+"'");
423 auto& ioConfig = eclState().getIOConfig();
424 ioConfig.overrideNOSIM(yesno);
425 }
426 }
427 catch (const std::invalid_argument& e) {
428 std::cerr << "Failed to create valid EclipseState object" << std::endl;
429 std::cerr << "Exception caught: " << e.what() << std::endl;
430 throw;
431 }
432 }
433
434 const Deck& deck() const
435 { return ebosSimulator_->vanguard().deck(); }
436
437 Deck& deck()
438 { return ebosSimulator_->vanguard().deck(); }
439
440 const EclipseState& eclState() const
441 { return ebosSimulator_->vanguard().eclState(); }
442
443 EclipseState& eclState()
444 { return ebosSimulator_->vanguard().eclState(); }
445
446 const Schedule& schedule() const
447 { return ebosSimulator_->vanguard().schedule(); }
448
449 // Run the simulator.
450 int runSimulator()
451 {
452 return runSimulatorInitOrRun_(&FlowMainEbos::runSimulatorRunCallback_);
453 }
454
455 int runSimulatorInit()
456 {
457 return runSimulatorInitOrRun_(&FlowMainEbos::runSimulatorInitCallback_);
458 }
459
460 private:
461 // Callback that will be called from runSimulatorInitOrRun_().
462 int runSimulatorRunCallback_()
463 {
464 SimulatorReport report = simulator_->run(*simtimer_);
465 runSimulatorAfterSim_(report);
466 return report.success.exit_status;
467 }
468
469 // Callback that will be called from runSimulatorInitOrRun_().
470 int runSimulatorInitCallback_()
471 {
472 simulator_->init(*simtimer_);
473 return EXIT_SUCCESS;
474 }
475
476 // Output summary after simulation has completed
477 void runSimulatorAfterSim_(SimulatorReport &report)
478 {
479 if (! this->output_cout_) {
480 return;
481 }
482
483 const int threads
484#if !defined(_OPENMP) || !_OPENMP
485 = 1;
486#else
487 = omp_get_max_threads();
488#endif
489
490 printFlowTrailer(mpi_size_, threads, report);
491
492 detail::handleExtraConvergenceOutput(report,
493 EWOMS_GET_PARAM(TypeTag, std::string, OutputExtraConvergenceInfo),
494 R"(OutputExtraConvergenceInfo (--output-extra-convergence-info))",
495 eclState().getIOConfig().getOutputDir(),
496 eclState().getIOConfig().getBaseName());
497 }
498
499 // Run the simulator.
500 int runSimulatorInitOrRun_(int (FlowMainEbos::* initOrRunFunc)())
501 {
502
503 const auto& schedule = this->schedule();
504 auto& ioConfig = eclState().getIOConfig();
505 simtimer_ = std::make_unique<SimulatorTimer>();
506
507 // initialize variables
508 const auto& initConfig = eclState().getInitConfig();
509 simtimer_->init(schedule, (size_t)initConfig.getRestartStep());
510
511 if (this->output_cout_) {
512 std::ostringstream oss;
513
514 // This allows a user to catch typos and misunderstandings in the
515 // use of simulator parameters.
516 if (Parameters::printUnused<TypeTag>(oss)) {
517 std::cout << "----------------- Unrecognized parameters: -----------------\n";
518 std::cout << oss.str();
519 std::cout << "----------------------------------------------------------------" << std::endl;
520 }
521 }
522
523 if (!ioConfig.initOnly()) {
524 if (this->output_cout_) {
525 std::string msg;
526 msg = "\n\n================ Starting main simulation loop ===============\n";
527 OpmLog::info(msg);
528 }
529
530 return (this->*initOrRunFunc)();
531 }
532 else {
533 if (this->output_cout_) {
534 std::cout << "\n\n================ Simulation turned off ===============\n" << std::flush;
535 }
536 return EXIT_SUCCESS;
537 }
538 }
539
540 protected:
541
543 // Create simulator instance.
544 // Writes to:
545 // simulator_
547 {
548 // Create the simulator instance.
549 simulator_ = std::make_unique<Simulator>(*ebosSimulator_);
550 }
551
552 Grid& grid()
553 { return ebosSimulator_->vanguard().grid(); }
554
555 private:
556 std::unique_ptr<EbosSimulator> ebosSimulator_;
557 int mpi_rank_ = 0;
558 int mpi_size_ = 1;
559 std::any parallel_information_;
560 std::unique_ptr<Simulator> simulator_;
561 std::unique_ptr<SimulatorTimer> simtimer_;
562 int argc_;
563 char **argv_;
564 bool output_cout_;
565 bool output_files_;
566 };
567
568} // namespace Opm
569
570#endif // OPM_FLOW_MAIN_EBOS_HEADER_INCLUDED
Definition: FlowMainEbos.hpp:93
int execute()
This is the main function of Flow.
Definition: FlowMainEbos.hpp:284
void createSimulator()
This is the main function of Flow.
Definition: FlowMainEbos.hpp:546
a simulator for the blackoil model
Definition: SimulatorFullyImplicitBlackoilEbos.hpp:144
This file contains a set of helper functions used by VFPProd / VFPInj.
Definition: BlackoilPhases.hpp:27
Definition: FlowMainEbos.hpp:44
Definition: FlowMainEbos.hpp:52
Definition: FlowMainEbos.hpp:48
Definition: SimulatorReport.hpp:100