Feature/779 state serialization (#780)

* Enable saving and restoring full simulation state

Closes #768

* Added tinycbor based serialization method

* Debug test

* Replaced tinycbor -> libcbor (ver 0.1)

* deserialization using cbor_reader

* Separated ctx to own struct

* Tag as std::optional in ctx

* State serialization test

* Removed comments

* Addressed Linux build issue

* Added BouncingBall reference model from modelica for adding byte vector serialization test

* Added BouncingBall license in README

* Added comments

* Deleted state_serialization_test.cpp

* Added libcbor as PRIVATE for `target_link_libraries`

* Controlling fixed precision for logging float point values (#775)

* Added an option to file_observer_config to set
fixed precision value.

* Fixed build failure

* Reading canGetAndSetFMUstate and canSerializeFMUstate from model description file

* Added transitive_libs for libcbor to avoid cmake error

* linking libcbor statically.

* Resetting time series observer upon state restore

* State saving support with proxyfmu

* Split proxyfmu from the save_state_test.

* Uses upgraded thrift/boost

* Fixed boost header issue

* Fixed boost header issue

* Fixed boost header issue

* Update dependency

* Updated action

* Upgraded libzip

* Upgraded libzip

* libzip deprecated func fix

* Observer fix

* Addressing comments
1. Defining exported sourced explicitly
2. Try to use shared libcbor.

* made libcbor PUBLIC in cmake (may require the consumer also declare libcbor for linking libcosim)

* Addressed comments

* Addressed comments

* Addressed comments - 02

* Addressed comments - 03

* Updated cosim_capabilities -> simulator_capabilities. Using cosim::serialization::format::cbor to choose which serialization/deserialization to use

* update

* pretty_print format added

* Updated error message

* Updated error message

* Updated error message

* Using fmu::model_description::capabilities in copy_current_state() and export_state()

* Updated error message

* Don't require importing def_types.h

* Using updated proxyfmu that hides thrift related libs

* Updated field names for the model description

* Updated proxyfmu dependency

---------

Co-authored-by: Lars T. Kyllingstad <lars.kyllingstad@sintef.no>

Author: davidhjp01

.github/workflows/ci-conan.yml

@@ -48,6 +48,7 @@ jobs:
             --settings="build_type=${{ matrix.build_type }}" \
             --options="${{ matrix.option_proxyfmu }}" \
             --options="${{ matrix.option_shared }}" \
+            --update \
             --build=missing \
             --user=osp \
             --channel="${CHANNEL}" \
@@ -103,6 +104,7 @@ jobs:
             --settings="build_type=${{ matrix.build_type }}" \
             --options="${{ matrix.option_proxyfmu }}" \
             --options="${{ matrix.option_shared }}" \
+            --update \
             --build=missing \
             --user=osp \
             --channel="${CHANNEL}" \

CMakeLists.txt

@@ -113,6 +113,7 @@ find_package(libzip REQUIRED)
 find_package(Microsoft.GSL REQUIRED)
 find_package(yaml-cpp REQUIRED)
 find_package(XercesC MODULE REQUIRED)
+find_package(libcbor REQUIRED)
 if(LIBCOSIM_WITH_PROXYFMU)
     find_package(PROXYFMU CONFIG REQUIRED)
 endif()

conanfile.py

@@ -39,19 +39,21 @@ def set_version(self):
     def requirements(self):
         self.tool_requires("cmake/[>=3.19]")
         self.requires("fmilibrary/[~2.3]")
-        self.requires("libzip/[>=1.7 <1.10]") # 1.10 deprecates some functions we use
+        self.requires("libcbor/0.11.0")
+        self.requires("libzip/[~1.11]")
         self.requires("ms-gsl/[>=3 <5]", transitive_headers=True)
+        self.requires("boost/[~1.85]", transitive_headers=True, transitive_libs=True)  # Required by Thrift
         if self.options.proxyfmu:
-            self.requires("proxyfmu/0.3.2@osp/stable")
-            self.requires("boost/[~1.81]", transitive_headers=True, transitive_libs=True) # Required by Thrift
-        else:
-            self.requires("boost/[>=1.71]", transitive_headers=True, transitive_libs=True)
+            self.requires("proxyfmu/0.3.3@osp/testing",
+                          transitive_headers=True,
+                          transitive_libs=True)
         self.requires("yaml-cpp/[~0.8]")
         self.requires("xerces-c/[~3.2]")
 
     # Exports
     exports = "version.txt"
-    exports_sources = "*"
+    exports_sources = ("src/*", "include/*", "cmake/*", "data/*", "docs/*", "tests/*", "CHANGELOG.md", "CMakeLists.txt",
+                       "CONTRIBUTING.md", "LICENSE", "README.md", "version.txt")
 
     # Build steps
 

include/cosim/model_description.hpp

@@ -59,6 +59,13 @@ enum variable_variability
     continuous
 };
 
+/// A list of simulator capabilities
+struct simulator_capabilities
+{
+    bool can_save_state = false;
+    bool can_export_state = false;
+};
+
 
 /// Returns a textual representation of `v`.
 constexpr const char* to_text(variable_type v)
@@ -195,6 +202,9 @@ struct model_description
 
     /// Variable descriptions.
     std::vector<variable_description> variables;
+
+    /// Simulator capabilities
+    simulator_capabilities capabilities;
 };
 
 /// Getter for returning a variable description.

include/cosim/observer/file_observer.hpp

@@ -23,6 +23,7 @@ namespace cosim
 
 class file_observer;
 
+
 /**
  * Configuration options for file_observer.
  */
@@ -88,9 +89,22 @@ class file_observer_config
      */
     static file_observer_config parse(const filesystem::path& configPath);
 
+    /**
+     * Sets fixed precision value for floating point numbers
+     *
+     * \param precision the number of digits after the decimal point. The observer uses the default precision
+     *  value of 6 if the argument is less than zero.
+     */
+    void fixed_precision(const int precision)
+    {
+        precision_ = precision;
+    }
+
 private:
     bool timeStampedFileNames_{true};
     size_t defaultDecimationFactor_{1};
+    int precision_{-1};
+
     std::unordered_map<std::string, std::pair<size_t, std::vector<std::string>>> variablesToLog_;
 
     [[nodiscard]] bool should_log_simulator(const std::string& name) const

include/cosim/serialization.hpp

@@ -98,13 +98,30 @@ struct node_data_translator
     node_data put_value(const T& value) { return node_data(value); }
 };
 
-}} // namespace cosim::serialization
+namespace format
+{
+const auto format_xalloc = std::ios_base::xalloc();
+const int FORMAT_CBOR = 1;
+const int FORMAT_PRETTY_PRINT = 2;
+
+std::ios_base& cbor(std::ios_base& os);
+std::ios_base& pretty_print(std::ios_base& os);
+
+} // namespace format
 
+} // namespace serialization
+} // namespace cosim
+
+/**
+ *  Serializes `data` into a format specified via cosim::serialization::format and writes it to the output stream `out`.
+ */
+std::ostream& operator<<(std::ostream& out, const cosim::serialization::node& data);
+
+/**
+ *  Reads a stream from `in` and converts it to `data` format specified via cosim::serialization::format.
+ */
+std::istream& operator>>(std::istream& in, cosim::serialization::node& data);
 
-// Ordinarily, the following function would be in the `cosim::serialization`
-// namespace and found by the compiler via ADL.  This doesn't work here,
-// because `cosim::serialization::node` is just an alias for a type in the
-// `boost::property_tree` namespace.
 /**
  *  Writes the contents of `data` to the output stream `out` in a human-readable
  *  format.
@@ -113,7 +130,7 @@ struct node_data_translator
  *  corresponding "read" function, nor is the output format designed to support
  *  round-trip information or type preservation.
  */
-std::ostream& operator<<(std::ostream& out, const cosim::serialization::node& data);
+void print_ptree(std::ostream& out, const cosim::serialization::node& data);
 
 
 // Make node_translator the default translator for property trees whose data

src/CMakeLists.txt

@@ -180,6 +180,7 @@ target_link_libraries(cosim
         libzip::zip
         XercesC::XercesC
         yaml-cpp
+        libcbor::libcbor
     )
 
 if(LIBCOSIM_WITH_PROXYFMU)

src/cosim/fmi/v2/fmu.cpp

@@ -63,6 +63,12 @@ fmu::fmu(
     modelDescription_.description = fmi2_import_get_description(handle_);
     modelDescription_.author = fmi2_import_get_author(handle_);
     modelDescription_.version = fmi2_import_get_model_version(handle_);
+    modelDescription_.capabilities.can_save_state = !!fmi2_import_get_capability(
+        handle_,
+        fmi2_cs_canGetAndSetFMUstate);
+    modelDescription_.capabilities.can_export_state = !!fmi2_import_get_capability(
+        handle_,
+        fmi2_cs_canSerializeFMUstate);
     const auto varList = fmi2_import_get_variable_list(handle_, 0);
     const auto _ = gsl::finally([&]() {
         fmi2_import_free_variable_list(varList);
@@ -624,7 +630,7 @@ serialization::node slave_instance::export_state(state_index stateIndex) const
     const auto& savedState = savedStates_.at(stateIndex);
 
     // Check that the FMU supports state serialization
-    if (!fmi2_import_get_capability(handle_, fmi2_cs_canSerializeFMUstate))
+    if (!fmu_->model_description()->capabilities.can_export_state)
     {
         throw error(
             make_error_code(errc::unsupported_feature),
@@ -737,7 +743,7 @@ fmi2_import_t* slave_instance::fmilib_handle() const
 
 void slave_instance::copy_current_state(saved_state& state)
 {
-    if (!fmi2_import_get_capability(handle_, fmi2_cs_canGetAndSetFMUstate)) {
+    if (!fmu_->model_description()->capabilities.can_save_state) {
         throw error(
             make_error_code(errc::unsupported_feature),
             instanceName_ + ": FMU does not support state saving");

src/cosim/observer/file_observer.cpp

@@ -46,6 +46,7 @@ void clear_file_contents_if_exists(const cosim::filesystem::path& filePath, std:
         fsw.close();
     }
 }
+
 } // namespace
 
 
@@ -60,12 +61,18 @@ class file_observer::slave_value_writer
         initialize_default();
     }
 
-    slave_value_writer(observable* observable, cosim::filesystem::path& logDir, size_t decimationFactor,
-        const std::vector<variable_description>& variables, bool timeStampedFileNames = true)
+    slave_value_writer(
+        observable* observable, 
+        cosim::filesystem::path& logDir, 
+        size_t decimationFactor,
+        const std::vector<variable_description>& variables, 
+        bool timeStampedFileNames = true,
+        const int precision = -1)
         : observable_(observable)
         , logDir_(logDir)
         , decimationFactor_(decimationFactor)
         , timeStampedFileNames_(timeStampedFileNames)
+        , precision_(precision)
     {
         initialize_config(variables);
     }
@@ -301,10 +308,19 @@ class file_observer::slave_value_writer
     void persist()
     {
         std::stringstream ss;
+        const auto defaultPrecision = ss.precision();
+
         if (fsw_.is_open()) {
 
             for (const auto& [stepCount, times] : timeSamples_) {
-                ss << times << "," << stepCount;
+                if (precision_ > -1) {
+                    ss.precision(defaultPrecision);
+                    ss << times << "," << stepCount;
+                    ss.precision(precision_);
+                    ss << std::fixed;
+                } else {
+                    ss << times << "," << stepCount;
+                }
 
                 if (realSamples_.count(stepCount)) write<double>(realSamples_[stepCount], ss);
                 if (intSamples_.count(stepCount)) write<int>(intSamples_[stepCount], ss);
@@ -340,6 +356,7 @@ class file_observer::slave_value_writer
     std::atomic<bool> recording_ = true;
     std::mutex mutex_;
     bool timeStampedFileNames_ = true;
+    int precision_ = -1;
 };
 
 file_observer::file_observer(const cosim::filesystem::path& logDir, std::optional<file_observer_config> config)
@@ -387,7 +404,8 @@ void file_observer::simulator_added(
                 logDir_,
                 config.decimationFactor,
                 config.variables,
-                config.timeStampedFileNames);
+                config.timeStampedFileNames,
+                config_->precision_);
         } else {
             return;
         }

src/cosim/observer/time_series_observer.cpp

@@ -93,6 +93,17 @@ class time_series_observer::single_slave_observer
         adjustIfFull(timeSamples_, bufSize_);
     }
 
+    void reset()
+    {
+        timeSamples_.clear();
+        for (auto& [_, step_numbers] : realSamples_) {
+            step_numbers.clear();
+        }
+        for (auto& [_, step_numbers] : intSamples_) {
+            step_numbers.clear();
+        }
+    }
+
     void start_observing(variable_type type, value_reference reference)
     {
         std::lock_guard<std::mutex> lock(lock_);
@@ -266,6 +277,7 @@ void time_series_observer::state_restored(
     step_number currentStep, time_point currentTime)
 {
     for (const auto& entry : slaveObservers_) {
+        entry.second->reset();
         entry.second->observe(currentStep, currentTime);
     }
 }

src/cosim/proxy/remote_fmu.cpp

@@ -71,6 +71,8 @@ std::unique_ptr<cosim::model_description> parse_model_description(const proxyfmu
     _md->author = md.author;
     _md->name = md.modelName;
     _md->description = md.description;
+    _md->capabilities.can_save_state = md.canGetAndSetFMUstate;
+    _md->capabilities.can_export_state = md.canSerializeFMUstate;
 
     for (auto& var : md.modelVariables) {
         cosim::variable_description vd;

src/cosim/proxy/remote_slave.cpp

@@ -6,6 +6,7 @@
 #include <cosim/error.hpp>
 #include <cosim/exception.hpp>
 #include <cosim/proxy/remote_slave.hpp>
+#include <proxyfmu/state.hpp>
 
 #include <utility>
 
@@ -204,45 +205,60 @@ void cosim::proxy::remote_slave::set_string_variables(gsl::span<const cosim::val
 
 cosim::slave::state_index cosim::proxy::remote_slave::save_state()
 {
-    throw error(
-        make_error_code(errc::unsupported_feature),
-        "Proxyfmu does not support state saving");
+    return slave_->save_state();
 }
 
-void cosim::proxy::remote_slave::save_state(state_index)
+void cosim::proxy::remote_slave::save_state(state_index idx)
 {
-    throw error(
-        make_error_code(errc::unsupported_feature),
-        "Proxyfmu does not support state saving");
+    slave_->save_state(idx);
 }
 
-void cosim::proxy::remote_slave::restore_state(state_index)
+void cosim::proxy::remote_slave::restore_state(state_index idx)
 {
-    throw error(
-        make_error_code(errc::unsupported_feature),
-        "Proxyfmu does not support state saving");
+    slave_->restore_state(idx);
 }
 
-void cosim::proxy::remote_slave::release_state(state_index)
+void cosim::proxy::remote_slave::release_state(state_index idx)
 {
-    throw error(
-        make_error_code(errc::unsupported_feature),
-        "Proxyfmu does not support state saving");
+    slave_->release_state(idx);
 }
 
-cosim::serialization::node cosim::proxy::remote_slave::export_state(state_index) const
+cosim::serialization::node cosim::proxy::remote_slave::export_state(state_index idx) const
 {
-    throw error(
-        make_error_code(errc::unsupported_feature),
-        "Proxyfmu does not support state serialization");
+    proxyfmu::state::exported_state state;
+    slave_->export_state(idx, state);
+
+    // Create the exported state
+    serialization::node exportedState;
+    exportedState.put("scheme_version", state.schemeVersion);
+    exportedState.put("fmu_uuid", state.uuid);
+
+    std::vector<std::byte> vec;
+    std::transform(state.fmuState.begin(), state.fmuState.end(), std::back_inserter(vec),
+        [](unsigned char c) { return std::byte(c); });
+
+    exportedState.put("serialized_fmu_state", vec);
+    exportedState.put("setup_complete", state.setupComplete);
+    exportedState.put("simulation_started", state.simStarted);
+    return exportedState;
 }
 
 cosim::slave::state_index cosim::proxy::remote_slave::import_state(
-    const cosim::serialization::node&)
+    const cosim::serialization::node& node)
 {
-    throw error(
-        make_error_code(errc::unsupported_feature),
-        "Proxyfmu does not support state serialization");
+    proxyfmu::state::exported_state state;
+    state.schemeVersion = node.get<std::int32_t>("scheme_version");
+    state.uuid = node.get<std::string>("fmu_uuid");
+
+    const auto& serializedFMUState = std::get<std::vector<std::byte>>(
+        node.get_child("serialized_fmu_state").data());
+    state.fmuState.reserve(serializedFMUState.size());
+    std::transform(serializedFMUState.begin(), serializedFMUState.end(), std::back_inserter(state.fmuState),
+        [](std::byte b) { return static_cast<unsigned char>(b); });
+
+    state.setupComplete = node.get<bool>("setup_complete");
+    state.simStarted = node.get<bool>("simulation_started");
+    return slave_->import_state(state);
 }
 
 cosim::proxy::remote_slave::~remote_slave()