22#ifndef OPM_SIMULATORFULLYIMPLICITBLACKOILEBOS_HEADER_INCLUDED
23#define OPM_SIMULATORFULLYIMPLICITBLACKOILEBOS_HEADER_INCLUDED
25#include <dune/common/hash.hh>
27#include <opm/simulators/flow/BlackoilModelEbos.hpp>
28#include <opm/simulators/flow/BlackoilModelParametersEbos.hpp>
29#include <opm/simulators/flow/ConvergenceOutputConfiguration.hpp>
30#include <opm/simulators/flow/ExtraConvergenceOutputThread.hpp>
31#include <opm/simulators/flow/NonlinearSolverEbos.hpp>
32#include <opm/simulators/aquifers/BlackoilAquiferModel.hpp>
33#include <opm/simulators/timestepping/AdaptiveTimeSteppingEbos.hpp>
34#include <opm/simulators/timestepping/ConvergenceReport.hpp>
35#include <opm/simulators/utils/moduleVersion.hpp>
36#include <opm/simulators/wells/WellState.hpp>
38#include <opm/grid/utility/StopWatch.hpp>
40#include <opm/input/eclipse/Units/UnitSystem.hpp>
42#include <opm/common/ErrorMacros.hpp>
44#include <boost/date_time/gregorian/gregorian.hpp>
55#include <ebos/hdf5serializer.hh>
58namespace Opm::Properties {
60template<
class TypeTag,
class MyTypeTag>
62 using type = UndefinedProperty;
64template<
class TypeTag,
class MyTypeTag>
66 using type = UndefinedProperty;
69template <
class TypeTag,
class MyTypeTag>
72 using type = UndefinedProperty;
75template <
class TypeTag,
class MyTypeTag>
78 using type = UndefinedProperty;
81template <
class TypeTag,
class MyTypeTag>
84 using type = UndefinedProperty;
87template <
class TypeTag,
class MyTypeTag>
90 using type = UndefinedProperty;
93template<
class TypeTag>
95 static constexpr bool value =
true;
97template<
class TypeTag>
99 static constexpr bool value =
true;
101template<
class TypeTag>
103 static constexpr bool value =
false;
106template <
class TypeTag>
109 static constexpr auto* value =
"none";
112template <
class TypeTag>
115 static constexpr auto* value =
"";
118template <
class TypeTag>
121 static constexpr auto* value =
"";
124template <
class TypeTag>
127 static constexpr int value = -1;
136 const std::string& title,
137 const std::string& version);
138void checkSerializedCmdLine(
const std::string& current,
139 const std::string& stored);
142template<
class TypeTag>
146 using Simulator = GetPropType<TypeTag, Properties::Simulator>;
147 using Grid = GetPropType<TypeTag, Properties::Grid>;
148 using FluidSystem = GetPropType<TypeTag, Properties::FluidSystem>;
149 using ElementContext = GetPropType<TypeTag, Properties::ElementContext>;
150 using BlackoilIndices = GetPropType<TypeTag, Properties::Indices>;
151 using PrimaryVariables = GetPropType<TypeTag, Properties::PrimaryVariables>;
152 using MaterialLaw = GetPropType<TypeTag, Properties::MaterialLaw>;
153 using SolutionVector = GetPropType<TypeTag, Properties::SolutionVector>;
154 using MaterialLawParams = GetPropType<TypeTag, Properties::MaterialLawParams>;
155 using AquiferModel = GetPropType<TypeTag, Properties::EclAquiferModel>;
158 typedef BlackOilPolymerModule<TypeTag> PolymerModule;
159 typedef BlackOilMICPModule<TypeTag> MICPModule;
190 : ebosSimulator_(ebosSimulator)
195 this->terminalOutput_ =
false;
196 if (this->grid().comm().rank() == 0) {
197 this->terminalOutput_ = EWOMS_GET_PARAM(TypeTag,
bool, EnableTerminalOutput);
199 this->startConvergenceOutputThread(EWOMS_GET_PARAM(TypeTag, std::string,
200 OutputExtraConvergenceInfo),
201 R
"(OutputExtraConvergenceInfo (--output-extra-convergence-info))");
204 const std::string saveSpec = EWOMS_GET_PARAM(TypeTag, std::string, SaveStep);
205 if (saveSpec ==
"all") {
207 }
else if (!saveSpec.empty() && saveSpec[0] ==
':') {
209 }
else if (!saveSpec.empty()) {
213 loadStep_ = EWOMS_GET_PARAM(TypeTag,
int, LoadStep);
215 saveFile_ = EWOMS_GET_PARAM(TypeTag, std::string, SaveFile);
217 const auto& ioconfig = ebosSimulator_.vanguard().eclState().getIOConfig();
218 saveFile_ = ioconfig.fullBasePath() +
".OPMRST";
220 std::filesystem::path path(ioconfig.getInputDir() +
"/");
221 path.replace_filename(ioconfig.getBaseName() +
".OPMRST");
223 if (!std::filesystem::exists(
saveFile_)) {
224 OPM_THROW(std::runtime_error,
"Error locating serialized restart file");
233 this->endConvergenceOutputThread();
236 static void registerParameters()
238 ModelParameters::registerParameters();
239 SolverParameters::registerParameters();
240 TimeStepper::registerParameters();
242 EWOMS_REGISTER_PARAM(TypeTag,
bool, EnableTerminalOutput,
243 "Print high-level information about the simulation's progress to the terminal");
244 EWOMS_REGISTER_PARAM(TypeTag,
bool, EnableAdaptiveTimeStepping,
245 "Use adaptive time stepping between report steps");
246 EWOMS_REGISTER_PARAM(TypeTag,
bool, EnableTuning,
247 "Honor some aspects of the TUNING keyword.");
248 EWOMS_REGISTER_PARAM(TypeTag, std::string, OutputExtraConvergenceInfo,
249 "Provide additional convergence output "
250 "files for diagnostic purposes. "
251 "\"none\" gives no extra output and "
252 "overrides all other options, "
253 "\"steps\" generates an INFOSTEP file, "
254 "\"iterations\" generates an INFOITER file. "
255 "Combine options with commas, e.g., "
256 "\"steps,iterations\" for multiple outputs.");
257 EWOMS_REGISTER_PARAM(TypeTag, std::string, SaveStep,
258 "Save serialized state to .OPMRST file. "
259 "Either a specific report step, \"all\" to save "
260 "all report steps or \":x\" to save every x'th step.");
261 EWOMS_REGISTER_PARAM(TypeTag,
int, LoadStep,
262 "Load serialized state from .OPMRST file. "
263 "Either a specific report step, or 0 to load last "
264 "stored report step.");
265 EWOMS_REGISTER_PARAM(TypeTag, std::string, SaveFile,
266 "FileName for .OPMRST file used for serialized state. "
267 "If empty, CASENAME.OPMRST is used.");
268 EWOMS_HIDE_PARAM(TypeTag, SaveFile);
281 ebosSimulator_.model().invalidateAndUpdateIntensiveQuantities(0);
283 while (!timer.
done()) {
284 bool continue_looping = runStep(timer);
285 if (!continue_looping)
break;
292 ebosSimulator_.setEpisodeIndex(-1);
295 solverTimer_ = std::make_unique<time::StopWatch>();
296 totalTimer_ = std::make_unique<time::StopWatch>();
297 totalTimer_->start();
300 bool enableAdaptive = EWOMS_GET_PARAM(TypeTag,
bool, EnableAdaptiveTimeStepping);
301 bool enableTUNING = EWOMS_GET_PARAM(TypeTag,
bool, EnableTuning);
302 if (enableAdaptive) {
303 const UnitSystem& unitSystem = this->ebosSimulator_.vanguard().eclState().getUnits();
306 auto max_next_tstep = sched_state.max_next_tstep();
307 adaptiveTimeStepping_ = std::make_unique<TimeStepper>(max_next_tstep,
308 sched_state.tuning(),
309 unitSystem, terminalOutput_);
312 adaptiveTimeStepping_ = std::make_unique<TimeStepper>(unitSystem, terminalOutput_);
318 adaptiveTimeStepping_->setSuggestedNextStep(ebosSimulator_.timeStepSize());
323 bool runStep(SimulatorTimer& timer)
325 if (schedule().exitStatus().has_value()) {
326 if (terminalOutput_) {
327 OpmLog::info(
"Stopping simulation since EXIT was triggered by an action keyword.");
329 report_.success.exit_status = schedule().exitStatus().value();
338 if (terminalOutput_) {
339 std::ostringstream ss;
341 OpmLog::debug(ss.str());
344 if (terminalOutput_) {
345 outputReportStep(timer);
349 if (timer.initialStep()) {
350 Dune::Timer perfTimer;
353 ebosSimulator_.setEpisodeIndex(-1);
354 ebosSimulator_.setEpisodeLength(0.0);
355 ebosSimulator_.setTimeStepSize(0.0);
356 wellModel_().beginReportStep(timer.currentStepNum());
357 ebosSimulator_.problem().writeOutput();
359 report_.success.output_write_time += perfTimer.stop();
363 solverTimer_->start();
365 auto solver = createSolver(wellModel_());
367 ebosSimulator_.startNextEpisode(
368 ebosSimulator_.startTime()
369 + schedule().seconds(timer.currentStepNum()),
370 timer.currentStepLength());
371 ebosSimulator_.setEpisodeIndex(timer.currentStepNum());
373 wellModel_().prepareDeserialize(
loadStep_ - 1);
376 ebosSimulator_.model().invalidateAndUpdateIntensiveQuantities(0);
378 solver->model().beginReportStep();
380 bool enableTUNING = EWOMS_GET_PARAM(TypeTag,
bool, EnableTuning);
387 if (adaptiveTimeStepping_) {
388 const auto& events = schedule()[timer.currentStepNum()].events();
390 if (events.hasEvent(ScheduleEvents::TUNING_CHANGE)) {
391 const auto& sched_state = schedule()[timer.currentStepNum()];
392 const auto& tuning = sched_state.tuning();
393 const auto& max_next_tstep = sched_state.max_next_tstep();
394 adaptiveTimeStepping_->updateTUNING(max_next_tstep, tuning);
397 bool event = events.hasEvent(ScheduleEvents::NEW_WELL) ||
398 events.hasEvent(ScheduleEvents::INJECTION_TYPE_CHANGED) ||
399 events.hasEvent(ScheduleEvents::WELL_SWITCHED_INJECTOR_PRODUCER) ||
400 events.hasEvent(ScheduleEvents::WELL_STATUS_CHANGE);
401 auto stepReport = adaptiveTimeStepping_->step(timer, *solver, event,
nullptr);
402 report_ += stepReport;
404 ebosSimulator_.problem().setSimulationReport(report_);
407 auto stepReport = solver->step(timer);
408 report_ += stepReport;
409 if (terminalOutput_) {
410 std::ostringstream ss;
411 stepReport.reportStep(ss);
412 OpmLog::info(ss.str());
417 Dune::Timer perfTimer;
419 const double nextstep = adaptiveTimeStepping_ ? adaptiveTimeStepping_->suggestedNextStep() : -1.0;
420 ebosSimulator_.problem().setNextTimeStepSize(nextstep);
421 ebosSimulator_.problem().writeOutput();
422 report_.success.output_write_time += perfTimer.stop();
424 solver->model().endReportStep();
427 solverTimer_->stop();
430 report_.success.solver_time += solverTimer_->secsSinceStart();
432 if (this->grid().comm().rank() == 0) {
436 this->writeConvergenceOutput(solver->model().getStepReportsDestructively());
442 if (terminalOutput_) {
443 if (!timer.initialStep()) {
445 outputTimestampFIP(timer, eclState().getTitle(), version);
449 "Time step took " + std::to_string(solverTimer_->secsSinceStart()) +
" seconds; "
450 "total solver time " + std::to_string(report_.success.solver_time) +
" seconds.";
459 SimulatorReport finalize()
463 Dune::Timer finalOutputTimer;
464 finalOutputTimer.start();
466 ebosSimulator_.problem().finalizeOutput();
467 report_.success.output_write_time += finalOutputTimer.stop();
472 report_.success.total_time = totalTimer_->secsSinceStart();
473 report_.success.converged =
true;
478 const Grid& grid()
const
479 {
return ebosSimulator_.vanguard().grid(); }
481 template<
class Serializer>
482 void serializeOp(Serializer& serializer)
484 serializer(ebosSimulator_);
486 serializer(adaptiveTimeStepping_);
491 std::unique_ptr<Solver> createSolver(WellModel& wellModel)
493 auto model = std::make_unique<Model>(ebosSimulator_,
498 return std::make_unique<Solver>(solverParam_, std::move(model));
501 const EclipseState& eclState()
const
502 {
return ebosSimulator_.vanguard().eclState(); }
505 const Schedule& schedule()
const
506 {
return ebosSimulator_.vanguard().schedule(); }
508 bool isRestart()
const
510 const auto& initconfig = eclState().getInitConfig();
511 return initconfig.restartRequested();
514 WellModel& wellModel_()
515 {
return ebosSimulator_.problem().wellModel(); }
517 const WellModel& wellModel_()
const
518 {
return ebosSimulator_.problem().wellModel(); }
520 void startConvergenceOutputThread(std::string_view convOutputOptions,
521 std::string_view optionName)
523 const auto config = ConvergenceOutputConfiguration {
524 convOutputOptions, optionName
526 if (! config.want(ConvergenceOutputConfiguration::Option::Iterations)) {
531 [compNames =
typename Model::ComponentName{}](
const int compIdx)
532 {
return std::string_view { compNames.name(compIdx) }; }
536 [usys = this->eclState().getUnits()](
const double time)
537 {
return usys.from_si(UnitSystem::measure::time, time); }
540 this->convergenceOutputQueue_.emplace();
541 this->convergenceOutputObject_.emplace
542 (this->eclState().getIOConfig().getOutputDir(),
543 this->eclState().getIOConfig().getBaseName(),
544 std::move(getPhaseName),
545 std::move(convertTime),
546 config, *this->convergenceOutputQueue_);
548 this->convergenceOutputThread_
550 &this->convergenceOutputObject_.value());
553 void writeConvergenceOutput(std::vector<StepReport>&& reports)
555 if (! this->convergenceOutputThread_.has_value()) {
559 auto requests = std::vector<ConvergenceReportQueue::OutputRequest>{};
560 requests.reserve(reports.size());
562 for (
auto&& report : reports) {
563 requests.push_back({ report.report_step, report.current_step, std::move(report.report) });
566 this->convergenceOutputQueue_->enqueue(std::move(requests));
569 void endConvergenceOutputThread()
571 if (! this->convergenceOutputThread_.has_value()) {
575 this->convergenceOutputQueue_->signalLastOutputRequest();
576 this->convergenceOutputThread_->join();
586 OPM_BEGIN_PARALLEL_TRY_CATCH();
592 OpmLog::error(
"Saving of serialized state requested, but no HDF5 support available.");
594 const std::string groupName =
"/report_step/" + std::to_string(nextStep);
600 EclGenericVanguard::comm());
602 std::ostringstream str;
603 Parameters::printValues<TypeTag>(str);
604 writer.writeHeader(
"OPM Flow",
607 ebosSimulator_.vanguard().caseName(),
609 EclGenericVanguard::comm().size());
611 if (EclGenericVanguard::comm().size() > 1) {
612 const auto& cellMapping = ebosSimulator_.vanguard().globalCell();
613 std::size_t hash = Dune::hash_range(cellMapping.begin(), cellMapping.end());
614 writer.write(hash,
"/",
"grid_checksum");
617 writer.write(*
this, groupName,
"simulator_data");
618 writer.write(timer, groupName,
"simulator_timer",
620 OpmLog::info(
"Serialized state written for report step " + std::to_string(nextStep));
624 OPM_END_PARALLEL_TRY_CATCH(
"Error saving serialized state: ",
625 EclGenericVanguard::comm());
632 OpmLog::error(
"Loading of serialized state requested, but no HDF5 support available.");
635 OPM_BEGIN_PARALLEL_TRY_CATCH();
639 EclGenericVanguard::comm());
645 OpmLog::info(
"Loading serialized state for report step " + std::to_string(
loadStep_));
646 const std::string groupName =
"/report_step/" + std::to_string(
loadStep_);
649 std::tuple<std::array<std::string,5>,
int> header;
651 const auto& [strings, procs] = header;
653 if (EclGenericVanguard::comm().size() != procs) {
654 throw std::runtime_error(
"Number of processes (procs=" +
655 std::to_string(EclGenericVanguard::comm().size()) +
656 ") does not match .OPMRST file (procs=" +
657 std::to_string(procs) +
")");
660 if (EclGenericVanguard::comm().size() > 1) {
661 std::size_t stored_hash;
662 reader.read(stored_hash,
"/",
"grid_checksum");
663 const auto& cellMapping = ebosSimulator_.vanguard().globalCell();
664 std::size_t hash = Dune::hash_range(cellMapping.begin(), cellMapping.end());
665 if (hash != stored_hash) {
666 throw std::runtime_error(
"Grid hash mismatch, .OPMRST file cannot be used");
670 if (EclGenericVanguard::comm().rank() == 0) {
671 std::ostringstream str;
672 Parameters::printValues<TypeTag>(str);
673 checkSerializedCmdLine(str.str(), strings[4]);
676 OPM_END_PARALLEL_TRY_CATCH(
"Error loading serialized state: ",
677 EclGenericVanguard::comm());
685 OPM_BEGIN_PARALLEL_TRY_CATCH();
689 EclGenericVanguard::comm());
690 const std::string groupName =
"/report_step/" + std::to_string(
loadStep_);
691 reader.read(*
this, groupName,
"simulator_data");
693 OPM_END_PARALLEL_TRY_CATCH(
"Error loading serialized state: ",
694 EclGenericVanguard::comm());
699 Simulator& ebosSimulator_;
700 std::unique_ptr<WellConnectionAuxiliaryModule<TypeTag>> wellAuxMod_;
702 ModelParameters modelParam_;
703 SolverParameters solverParam_;
708 bool terminalOutput_;
711 std::unique_ptr<time::StopWatch> solverTimer_;
712 std::unique_ptr<time::StopWatch> totalTimer_;
713 std::unique_ptr<TimeStepper> adaptiveTimeStepping_;
715 std::optional<ConvergenceReportQueue> convergenceOutputQueue_{};
716 std::optional<ConvergenceOutputThread> convergenceOutputObject_{};
717 std::optional<std::thread> convergenceOutputThread_{};
Definition: AdaptiveTimeSteppingEbos.hpp:271
A model implementation for three-phase black oil.
Definition: BlackoilModelEbos.hpp:155
Class for handling the blackoil well model.
Definition: BlackoilWellModel.hpp:92
std::function< std::string_view(int)> ComponentToPhaseName
Protocol for converting a phase/component ID into a human readable phase/component name.
Definition: ExtraConvergenceOutputThread.hpp:109
void writeASynchronous()
Output thread worker function.
Definition: ExtraConvergenceOutputThread.cpp:297
std::function< double(double)> ConvertToTimeUnits
Protocol for converting an SI elapsed time value into an equivalent time value in the run's output co...
Definition: ExtraConvergenceOutputThread.hpp:115
@ READ
Open existing file for reading.
@ APPEND
Append to an existing file (creates new if not)
@ ROOT_ONLY
A single dataset created at the root process.
A nonlinear solver class suitable for general fully-implicit models, as well as pressure,...
Definition: NonlinearSolverEbos.hpp:110
a simulator for the blackoil model
Definition: SimulatorFullyImplicitBlackoilEbos.hpp:144
void handleSave(SimulatorTimer &timer)
Serialization of simulator data to .OPMRST files at end of report steps.
Definition: SimulatorFullyImplicitBlackoilEbos.hpp:580
std::string saveFile_
File to load/save serialized state from/to.
Definition: SimulatorFullyImplicitBlackoilEbos.hpp:722
SimulatorReport run(SimulatorTimer &timer)
Run the simulation.
Definition: SimulatorFullyImplicitBlackoilEbos.hpp:277
SimulatorFullyImplicitBlackoilEbos(Simulator &ebosSimulator)
Initialise from parameters and objects to observe.
Definition: SimulatorFullyImplicitBlackoilEbos.hpp:189
int loadStep_
Step to load serialized state from.
Definition: SimulatorFullyImplicitBlackoilEbos.hpp:721
int saveStep_
Specific step to save serialized state at.
Definition: SimulatorFullyImplicitBlackoilEbos.hpp:720
void loadSimulatorState()
Load simulator state from serialized state.
Definition: SimulatorFullyImplicitBlackoilEbos.hpp:682
int saveStride_
Stride to save serialized state at.
Definition: SimulatorFullyImplicitBlackoilEbos.hpp:719
void loadTimerInfo(SimulatorTimer &timer)
Load timer info from serialized state.
Definition: SimulatorFullyImplicitBlackoilEbos.hpp:629
Definition: SimulatorTimer.hpp:38
int currentStepNum() const override
Current step number.
Definition: SimulatorTimer.cpp:93
bool done() const override
Return true if op++() has been called numSteps() times.
Definition: SimulatorTimer.cpp:166
This file contains a set of helper functions used by VFPProd / VFPInj.
Definition: BlackoilPhases.hpp:27
std::string moduleVersionName()
Return the version name of the module, for example "2015.10" (for a release branch) or "2016....
Definition: moduleVersion.cpp:34
std::string compileTimestamp()
Return a string "dd-mm-yyyy at HH::MM::SS hrs" which is the time the binary was compiled.
Definition: moduleVersion.cpp:57
std::string moduleVersion()
Return a string containing both the name and hash, if N is the name and H is the hash it will be "N (...
Definition: moduleVersion.cpp:50
PhaseUsage phaseUsageFromDeck(const EclipseState &eclipseState)
Looks at presence of WATER, OIL and GAS keywords in state object to determine active phases.
Definition: phaseUsageFromDeck.cpp:145
Solver parameters for the BlackoilModel.
Definition: BlackoilModelParametersEbos.hpp:327
Definition: NonlinearSolverEbos.hpp:116
Definition: BlackoilPhases.hpp:46
Definition: SimulatorFullyImplicitBlackoilEbos.hpp:61
Definition: BlackoilWellModel.hpp:80
Definition: SimulatorFullyImplicitBlackoilEbos.hpp:65
Definition: SimulatorFullyImplicitBlackoilEbos.hpp:83
Definition: SimulatorFullyImplicitBlackoilEbos.hpp:89
Definition: SimulatorFullyImplicitBlackoilEbos.hpp:77
Definition: SimulatorReport.hpp:100