Enable saving and restoring full simulation state (#777)

This closes #768. Some changes may warrant a bit of extra explanation:

**`execution` and `algorithm`:** I have taken one step towards prohibiting adding/removing sub-simulators after the co-simulation has begun, as decided in #771.  In particular, I've added the `execution::initialize()` function, which marks the point where such changes are no longer allowed.  (This is a backwards-compatible change, because it gets automatically called by `execution::step()` if it hasn't been called manually.)

**`slave_simulator`**: The changes in `slave_simulator.cpp` really ought to have been included in #769. Unfortunately, I didn't realise they were necessary before it was all put into the context of saving algorithm and execution state.  Basically, I thought I could get away with just saving each FMU's internal state, but it turns out that we also need to save the `slave_simulator` "get" and "set" caches.

(For those interested in the nitty-gritties, this is due to some subtleties regarding exactly when the cache values are set in the course of a co-simulation, relative to when the values are passed to the FMU. At the end of an "algorithm step", the "set cache" is out of sync with the FMUs input variables, and won't be synced before the next step. Simply performing an additional sync prior to saving the state is not sufficient, because that could have an effect on the FMUs output variables, thus invalidating the "get cache". That could in principle be updated too, but then the `slave_simulator` is in a whole different state from where it was when we started to save the state.)

Author: kyllingstad

include/cosim/algorithm/algorithm.hpp

@@ -15,6 +15,7 @@
 #include <cosim/function/function.hpp>
 #include <cosim/model_description.hpp>
 #include <cosim/observer/observer.hpp>
+#include <cosim/serialization.hpp>
 #include <cosim/time.hpp>
 
 #include <functional>
@@ -157,7 +158,9 @@ class algorithm
      *  values for some of them.
      *
      *  This function is guaranteed to be called after `setup()` and before
-     *  the first `do_step()` call.
+     *  the first `do_step()` call. Furthermore, no more subsimulators and
+     *  functions will be added or removed after `initialize()` has been called;
+     *  that is, `{add,remove}_{simulator,function}()` will not be called again.
      */
     virtual void initialize() = 0;
 
@@ -180,6 +183,36 @@ class algorithm
      */
     virtual std::pair<duration, std::unordered_set<simulator_index>> do_step(time_point currentT) = 0;
 
+    /**
+     *  Exports the current state of the algorithm.
+     *
+     *  Note that system-structural information should not be included in the
+     *  data exported by this function, only internal, algorithm-specific data.
+     *  This is because it will be assumed that the system structure is
+     *  unchanged or has already been restored when the state is imported
+     *  again, as explained in the `import_state()` function documentation.
+     */
+    virtual serialization::node export_current_state() const = 0;
+
+    /**
+     *  Imports a previously-exported algorithm state.
+     *
+     *  When this function is called, it should be assumed that the system
+     *  structure is the same as when the state was exported. That is, either
+     *
+     *  1. this is the algorithm instance from which the state was exported,
+     *     and the system structure actually hasn't changed
+     *  2. this is a new instance, but the original system structure has been
+     *     restored prior to calling this function
+     *
+     *  By "system structure", we here mean the subsimulator indexes, the
+     *  function indexes, and the variable connections.
+     *
+     *  It is guaranteed that this function is never called before
+     *  `initialize()`.
+     */
+    virtual void import_state(const serialization::node& exportedState) = 0;
+
     virtual ~algorithm() noexcept = default;
 };
 

include/cosim/algorithm/fixed_step_algorithm.hpp

@@ -56,6 +56,8 @@ class fixed_step_algorithm : public algorithm
     void setup(time_point startTime, std::optional<time_point> stopTime) override;
     void initialize() override;
     std::pair<duration, std::unordered_set<simulator_index>> do_step(time_point currentT) override;
+    serialization::node export_current_state() const override;
+    void import_state(const serialization::node& exportedState) override;
 
     /**
      * Sets step size decimation factor for a simulator.

include/cosim/execution.hpp

@@ -19,6 +19,7 @@
 
 #include <boost/functional/hash.hpp>
 
+#include <cstdint>
 #include <future>
 #include <memory>
 #include <optional>
@@ -36,7 +37,7 @@ using simulator_index = int;
 using function_index = int;
 
 /// An number which identifies a specific time step in an execution.
-using step_number = long long;
+using step_number = std::int64_t;
 
 /// An object which uniquely identifies a simulator variable in a simulation.
 struct variable_id
@@ -145,6 +146,13 @@ class simulator;
  *  The `execution` class manages all the entities involved in an execution
  *  and provides a high-level API for driving the co-simulation algorithm
  *  forward.
+ *
+ *  \warning
+ *      In general, the member functions of this class are not exception safe.
+ *      This means that if any of them throw an exception, one must assume that
+ *      the `execution` object is in an invalid state and can no longer be used.
+ *      The same holds for its associated algorithm and any simulators or
+ *      functions that are part of the execution.
  */
 class execution
 {
@@ -179,13 +187,19 @@ class execution
      *      The recommended co-simulation step size for this slave.
      *      Whether and how this is taken into account is algorithm dependent.
      *      If zero, the algorithm will attempt to choose a sensible default.
+     *
+     *  \pre `initialize()` has not been called.
      */
     simulator_index add_slave(
         std::shared_ptr<slave> slave,
         std::string_view name,
         duration stepSizeHint = duration::zero());
 
-    /// Adds a function to the execution.
+    /**
+     *  Adds a function to the execution.
+     *
+     *  \pre `initialize()` has not been called.
+     */
     function_index add_function(std::shared_ptr<function> fun);
 
     /// Adds an observer to the execution.
@@ -242,6 +256,14 @@ class execution
     /// Returns the current logical time.
     time_point current_time() const noexcept;
 
+    /**
+     *  Initialize the co-simulation (in an algorithm-dependent manner).
+     *
+     *  After this function is called, it is no longer possible to add more
+     *  subsimulators or functions.
+     */
+    void initialize();
+
     /**
      *  Advance the co-simulation forward to the given logical time (blocks the current thread).
      *
@@ -255,6 +277,12 @@ class execution
      *      `true` if the co-simulation was advanced to the given time,
      *      or `false` if it was stopped before this. In the latter case,
      *      `current_time()` may be called to determine the actual end time.
+     *
+     *  \note
+     *      For backwards compatibility, this function automatically calls
+     *      `initialize()` if this hasn't already been done. However, new code
+     *      should always call `initialize()` before any of the
+     *      simulation/stepping functions.
      */
     bool simulate_until(std::optional<time_point> targetTime);
 
@@ -271,6 +299,12 @@ class execution
      *      `true` if the co-simulation was advanced to the given time,
      *      or `false` if it was stopped before this. In the latter case,
      *      `current_time()` may be called to determine the actual end time.
+     *
+     *  \note
+     *      For backwards compatibility, this function automatically calls
+     *      `initialize()` if this hasn't already been done. However, new code
+     *      should always call `initialize()` before any of the
+     *      simulation/stepping functions.
      */
     std::future<bool> simulate_until_async(std::optional<time_point> targetTime);
 
@@ -281,6 +315,12 @@ class execution
      *      The actual duration of the step.
      *      `current_time()` may be called to determine the actual time after
      *      the step completed.
+     *
+     *  \note
+     *      For backwards compatibility, this function automatically calls
+     *      `initialize()` if this hasn't already been done. However, new code
+     *      should always call `initialize()` before any of the
+     *      simulation/stepping functions.
      */
     duration step();
 
@@ -314,6 +354,28 @@ class execution
     /// Set initial value for a variable of type string. Must be called before simulation is started.
     void set_string_initial_value(simulator_index sim, value_reference var, const std::string& value);
 
+    /**
+     *  Exports the current state of the co-simulation.
+     *
+     *  \pre `initialize()` has been called.
+     *  \pre `!is_running()`
+     */
+    serialization::node export_current_state() const;
+
+    /**
+     *  Imports a previously-exported co-simulation state.
+     *
+     *  Note that the data returned by `export_current_state()` only describe
+     *  the *state* of the system, not its structure.  This means that it's the
+     *  caller's responsibility to either
+     *
+     *  1. not modify the system structure between state export and state import
+     *  2. restore the pre-export system structure prior to state import
+     *
+     *  \pre `initialize()` has been called.
+     *  \pre `!is_running()`
+     */
+    void import_state(const serialization::node& exportedState);
 
 private:
     class impl;

include/cosim/observer/file_observer.hpp

@@ -170,6 +170,8 @@ class file_observer : public observer
         duration lastStepSize,
         time_point currentTime) override;
 
+    void state_restored(step_number currentStep, time_point currentTime) override;
+
     cosim::filesystem::path get_log_path();
 
     ~file_observer() override;

include/cosim/observer/last_value_observer.hpp

@@ -56,6 +56,8 @@ class last_value_observer : public last_value_provider
         duration lastStepSize,
         time_point currentTime) override;
 
+    void state_restored(step_number currentStep, time_point currentTime) override;
+
     void get_real(
         simulator_index sim,
         gsl::span<const value_reference> variables,

include/cosim/observer/observer.hpp

@@ -115,6 +115,17 @@ class observer
         duration lastStepSize,
         time_point currentTime) = 0;
 
+    /**
+     *  The simulation was restored to a previously saved state.
+     *
+     *  Note that observers which support this feature must be able to
+     *  reconstruct their internal state using information which is available
+     *  through the `observable` objects they have been given access to.  For
+     *  observers where this is not the case, this function should throw
+     *  `cosim::error` with error code `cosim::errc::unsupported_feature`.
+     */
+    virtual void state_restored(step_number currentStep, time_point currentTime) = 0;
+
     virtual ~observer() noexcept { }
 };
 

include/cosim/observer/time_series_observer.hpp

@@ -65,6 +65,8 @@ class time_series_observer : public time_series_provider
         duration lastStepSize,
         time_point currentTime) override;
 
+    void state_restored(step_number currentStep, time_point currentTime) override;
+
     /**
      * Start observing a variable.
      *

src/cosim/algorithm/fixed_step_algorithm.cpp

@@ -193,9 +193,9 @@ class fixed_step_algorithm::impl
             if (stepCounter_ % info.decimationFactor == 0) {
                 pool_.submit([&] {
                     try {
-                        info.stepResult = info.sim->do_step(currentT, baseStepSize_ * info.decimationFactor);
+                        const auto stepResult = info.sim->do_step(currentT, baseStepSize_ * info.decimationFactor);
 
-                        if (info.stepResult != step_result::complete) {
+                        if (stepResult != step_result::complete) {
                             std::lock_guard<std::mutex> lck(m);
                             errMessages
                                 << info.sim->name() << ": "
@@ -231,6 +231,28 @@ class fixed_step_algorithm::impl
         return {baseStepSize_, std::move(finished)};
     }
 
+    serialization::node export_current_state() const
+    {
+        auto exportedState = serialization::node();
+        exportedState.put("type", std::string("fixed_step_algorithm"));
+        exportedState.put("step_counter", stepCounter_);
+        return exportedState;
+    }
+
+    void import_state(const serialization::node& exportedState)
+    {
+        try {
+            if (exportedState.get<std::string>("type") != "fixed_step_algorithm") {
+                throw std::exception();
+            }
+            stepCounter_ = exportedState.get<std::int64_t>("step_counter");
+        } catch (...) {
+            throw error(
+                make_error_code(errc::bad_file),
+                "The serialized algorithm state is invalid or corrupt");
+        }
+    }
+
     void set_stepsize_decimation_factor(cosim::simulator_index i, int factor)
     {
         COSIM_INPUT_CHECK(factor > 0);
@@ -261,7 +283,6 @@ class fixed_step_algorithm::impl
     {
         simulator* sim;
         int decimationFactor = 1;
-        step_result stepResult;
         std::vector<connection_ss> outgoingSimConnections;
         std::vector<connection_sf> outgoingFunConnections;
     };
@@ -421,13 +442,20 @@ class fixed_step_algorithm::impl
         }
     }
 
+    // Algorithm parameters
     const duration baseStepSize_;
     time_point startTime_;
     std::optional<time_point> stopTime_;
+    unsigned int max_threads_ = std::thread::hardware_concurrency() - 1;
+
+    // System structure
     std::unordered_map<simulator_index, simulator_info> simulators_;
     std::unordered_map<function_index, function_info> functions_;
-    int64_t stepCounter_ = 0;
-    unsigned int max_threads_ = std::thread::hardware_concurrency() - 1;
+
+    // Simulation state
+    std::int64_t stepCounter_ = 0;
+
+    // Other
     utility::thread_pool pool_;
 };
 
@@ -521,6 +549,16 @@ std::pair<duration, std::unordered_set<simulator_index>> fixed_step_algorithm::d
     return pimpl_->do_step(currentT);
 }
 
+serialization::node fixed_step_algorithm::export_current_state() const
+{
+    return pimpl_->export_current_state();
+}
+
+void fixed_step_algorithm::import_state(const serialization::node& exportedState)
+{
+    pimpl_->import_state(exportedState);
+}
+
 void fixed_step_algorithm::set_stepsize_decimation_factor(cosim::simulator_index simulator, int factor)
 {
     pimpl_->set_stepsize_decimation_factor(simulator, factor);