Merge branch 'main' into feature/only-important-integration-tests

Author: mgovers

.github/workflows/reuse-compliance.yml

@@ -29,4 +29,4 @@ jobs:
     - name: checkout
       uses: actions/checkout@v4
     - name: REUSE Compliance Check
-      uses: fsfe/reuse-action@v4
+      uses: fsfe/reuse-action@v5

code_generation/meta_data.py

@@ -5,7 +5,6 @@
 # define dataclass for meta data
 
 from dataclasses import dataclass
-from typing import Optional
 
 from dataclasses_json import DataClassJsonMixin
 
@@ -15,19 +14,19 @@ class Attribute(DataClassJsonMixin):
     data_type: str
     names: str | list[str]
     description: str
-    nan_value: Optional[str] = None
+    nan_value: str | None = None
 
 
 @dataclass
 class AttributeClass(DataClassJsonMixin):
     name: str
     attributes: list[Attribute]
-    full_attributes: Optional[list[Attribute]] = None
-    base: Optional[str] = None
+    full_attributes: list[Attribute] | None = None
+    base: str | None = None
     is_template: bool = False
-    full_name: Optional[str] = None
-    specification_names: Optional[list[str]] = None
-    base_attributes: Optional[dict[str, list[Attribute]]] = None
+    full_name: str | None = None
+    specification_names: list[str] | None = None
+    base_attributes: dict[str, list[Attribute]] | None = None
 
 
 @dataclass

docs/api_reference/python-api-reference.md

@@ -14,6 +14,8 @@ This is the Python API reference for the `power-grid-model` library
 .. autoclass:: PowerGridModel
    :show-inheritance:
 .. autofunction:: initialize_array
+.. autofunction:: attribute_dtype
+.. autofunction:: attribute_empty_value
 
 .. py:data:: power_grid_meta_data
    :type: typing.PowerGridMetaData

docs/examples/Power Flow Example.ipynb

@@ -49,7 +49,7 @@
     "\n",
     "from power_grid_model import LoadGenType, ComponentType, DatasetType, ComponentAttributeFilterOptions\n",
     "from power_grid_model import PowerGridModel, CalculationMethod, CalculationType\n",
-    "from power_grid_model import initialize_array, power_grid_meta_data"
+    "from power_grid_model import initialize_array, attribute_dtype"
    ]
   },
   {
@@ -127,7 +127,7 @@
     "\n",
     "A columnar data format better integrates with most databases. In addition, it may bring memory and, in some cases, even computational performance improvements, because unused attribute columns can be omitted.\n",
     "\n",
-    "Make sure to provide the correct `dtype` to the numpy arrays, exposed for each dataset type, component and attribute via the `power_grid_meta_data` object."
+    "Make sure to provide the correct `dtype` to the numpy arrays, exposed for each dataset type, component and attribute via the helper function `attribute_dtype` function."
    ]
   },
   {
@@ -137,12 +137,11 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "source_attribute_dtypes = power_grid_meta_data[DatasetType.input][ComponentType.source].dtype\n",
     "source_columns = {\n",
-    "    \"id\": np.array([10], dtype=source_attribute_dtypes[\"id\"]),\n",
-    "    \"node\": np.array([1], dtype=source_attribute_dtypes[\"node\"]),\n",
-    "    \"status\": np.array([1], dtype=source_attribute_dtypes[\"status\"]),\n",
-    "    \"u_ref\": np.array([1.0], dtype=source_attribute_dtypes[\"u_ref\"]),\n",
+    "    \"id\": np.array([10], dtype=attribute_dtype(DatasetType.input, ComponentType.source, \"id\")),\n",
+    "    \"node\": np.array([1], dtype=attribute_dtype(DatasetType.input, ComponentType.source, \"node\")),\n",
+    "    \"status\": np.array([1], dtype=attribute_dtype(DatasetType.input, ComponentType.source, \"status\")),\n",
+    "    \"u_ref\": np.array([1.0], dtype=attribute_dtype(DatasetType.input, ComponentType.source, \"u_ref\")),\n",
     "    # We're not creating columns for u_ref_angle, sk, ... Instead, the default values are used. This saves us memory.\n",
     "}\n",
     "\n",
@@ -674,10 +673,11 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "line_update_dtype = power_grid_meta_data[DatasetType.update][ComponentType.line].dtype\n",
     "columnar_update_line = {\n",
-    "    \"id\": np.array([3], dtype=line_update_dtype[\"id\"]),  # change line ID 3\n",
-    "    \"from_status\": np.array([0], dtype=line_update_dtype[\"from_status\"]),  # switch off at from side\n",
+    "    \"id\": np.array([3], dtype=attribute_dtype(DatasetType.update, ComponentType.line, \"id\")),  # change line ID 3\n",
+    "    \"from_status\": np.array(\n",
+    "        [0], dtype=attribute_dtype(DatasetType.update, ComponentType.line, \"from_status\")\n",
+    "    ),  # switch off at from side\n",
     "}\n",
     "# leave to-side swiching status the same, no need to specify\n",
     "\n",
@@ -699,11 +699,10 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "line_update_dtype = power_grid_meta_data[DatasetType.update][ComponentType.line].dtype\n",
     "columnar_no_ID_update_line = {\n",
     "    # Update IDs are not specified\n",
     "    \"from_status\": np.array(\n",
-    "        [0, 1, 1], dtype=line_update_dtype[\"from_status\"]\n",
+    "        [0, 1, 1], dtype=attribute_dtype(DatasetType.update, ComponentType.line, \"from_status\")\n",
     "    ),  # The update for the whole column needs to be specified\n",
     "}\n",
     "# leave to-side swiching status the same, no need to specify\n",
@@ -1404,16 +1403,16 @@
      "output_type": "stream",
      "text": [
       "Node data with invalid results\n",
-      "[[0.99940117 0.99268579 0.99452137]\n",
-      " [0.99934769 0.98622639 0.98935286]\n",
-      " [0.99928838 0.97965401 0.98409554]\n",
-      " [0.         0.         0.        ]\n",
-      " [0.99915138 0.96614948 0.97329879]\n",
-      " [0.99907317 0.95920586 0.96775071]\n",
-      " [0.9989881  0.95212621 0.96209647]\n",
-      " [0.         0.         0.        ]\n",
-      " [0.99879613 0.93753005 0.95044796]\n",
-      " [0.9986885  0.92999747 0.94444167]]\n",
+      "[[9.99401170e-001 9.92685785e-001 9.94521366e-001]\n",
+      " [9.99347687e-001 9.86226389e-001 9.89352855e-001]\n",
+      " [9.99288384e-001 9.79654011e-001 9.84095542e-001]\n",
+      " [3.94357132e+180 2.87518198e+161 2.04418455e+214]\n",
+      " [9.99151380e-001 9.66149483e-001 9.73298790e-001]\n",
+      " [9.99073166e-001 9.59205860e-001 9.67750710e-001]\n",
+      " [9.98988099e-001 9.52126208e-001 9.62096474e-001]\n",
+      " [0.00000000e+000 0.00000000e+000 0.00000000e+000]\n",
+      " [9.98796126e-001 9.37530046e-001 9.50447962e-001]\n",
+      " [9.98688504e-001 9.29997471e-001 9.44441670e-001]]\n",
       "Node data with only valid results\n",
       "[[0.99940117 0.99268579 0.99452137]\n",
       " [0.99934769 0.98622639 0.98935286]\n",
@@ -1456,7 +1455,7 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "venv",
+   "display_name": ".venv",
    "language": "python",
    "name": "python3"
   },

power_grid_model_c/power_grid_model/include/power_grid_model/main_model_impl.hpp

@@ -316,12 +316,9 @@ class MainModelImpl<ExtraRetrievableTypes<ExtraRetrievableType...>, ComponentLis
 
     // update all components
     template <cache_type_c CacheType>
-    void
-    update_component(ConstDataset const& update_data, Idx pos,
-                     std::array<std::reference_wrapper<std::vector<Idx2D> const>, n_types> const& sequence_idx_map) {
+    void update_component(ConstDataset const& update_data, Idx pos, SequenceIdxView const& sequence_idx_map) {
         run_functor_with_all_types_return_void([this, pos, &update_data, &sequence_idx_map]<typename CT>() {
-            this->update_component<CT, CacheType>(update_data, pos,
-                                                  std::get<index_of_component<CT>>(sequence_idx_map).get());
+            this->update_component<CT, CacheType>(update_data, pos, std::get<index_of_component<CT>>(sequence_idx_map));
         });
     }
     template <cache_type_c CacheType>
@@ -432,36 +429,28 @@ class MainModelImpl<ExtraRetrievableTypes<ExtraRetrievableType...>, ComponentLis
         return get_sequence(buffer_span);
     }
     SequenceIdx get_sequence_idx_map(ConstDataset const& update_data, Idx scenario_idx,
-                                     ComponentFlags const& to_store) const {
-        // TODO: (jguo) this function could be encapsulated in UpdateCompIndependence in update.hpp
-        return run_functor_with_all_types_return_array([this, scenario_idx, &update_data, &to_store]<typename CT>() {
-            if (!to_store[index_of_component<CT>]) {
-                return std::vector<Idx2D>{};
-            }
-            auto const independence = check_components_independence<CT>(update_data);
-            validate_update_data_independence(independence);
-            return get_component_sequence<CT>(update_data, scenario_idx, independence);
-        });
-    }
-    // get sequence idx map of an entire batch for fast caching of component sequences
-    SequenceIdx get_sequence_idx_map(ConstDataset const& update_data, ComponentFlags const& to_store) const {
+                                     ComponentFlags const& components_to_store) const {
         // TODO: (jguo) this function could be encapsulated in UpdateCompIndependence in update.hpp
-        return run_functor_with_all_types_return_array([this, &update_data, &to_store]<typename CT>() {
-            if (!to_store[index_of_component<CT>]) {
-                return std::vector<Idx2D>{};
-            }
-            auto const independence = check_components_independence<CT>(update_data);
-            validate_update_data_independence(independence);
-            return get_component_sequence<CT>(update_data, 0, independence);
-        });
+        return run_functor_with_all_types_return_array(
+            [this, scenario_idx, &update_data, &components_to_store]<typename CT>() {
+                if (!std::get<index_of_component<CT>>(components_to_store)) {
+                    return std::vector<Idx2D>{};
+                }
+                auto const independence = check_components_independence<CT>(update_data);
+                validate_update_data_independence(independence);
+                return get_component_sequence<CT>(update_data, scenario_idx, independence);
+            });
     }
+    // Get sequence idx map of an entire batch for fast caching of component sequences.
+    // The sequence idx map of the batch is the same as that of the first scenario in the batch (assuming homogeneity)
+    // This is the entry point for permanent updates.
     SequenceIdx get_sequence_idx_map(ConstDataset const& update_data) const {
         constexpr ComponentFlags all_true = [] {
             ComponentFlags result{};
             std::ranges::fill(result, true);
             return result;
         }();
-        return get_sequence_idx_map(update_data, all_true);
+        return get_sequence_idx_map(update_data, 0, all_true);
     }
 
   private:
@@ -621,9 +610,10 @@ class MainModelImpl<ExtraRetrievableTypes<ExtraRetrievableType...>, ComponentLis
         // const ref of current instance
         MainModelImpl const& base_model = *this;
 
-        // cache component update order if possible
+        // cache component update order where possible.
+        // the order for a cacheable (independent) component by definition is the same across all scenarios
         auto const is_independent = is_update_independent(update_data);
-        all_scenarios_sequence = get_sequence_idx_map(update_data, is_independent);
+        all_scenarios_sequence = get_sequence_idx_map(update_data, 0, is_independent);
 
         return [&base_model, &exceptions, &infos, &calculation_fn, &result_data, &update_data,
                 &all_scenarios_sequence = std::as_const(all_scenarios_sequence),
@@ -640,15 +630,8 @@ class MainModelImpl<ExtraRetrievableTypes<ExtraRetrievableType...>, ComponentLis
             auto model = copy_model_functor(start);
 
             SequenceIdx current_scenario_sequence_cache = SequenceIdx{};
-            std::array<std::reference_wrapper<std::vector<Idx2D> const>, n_types> const current_scenario_sequence =
-                run_functor_with_all_types_return_array(
-                    [&is_independent, &all_scenarios_sequence, &current_scenario_sequence_cache]<typename CT>() {
-                        constexpr auto comp_idx = index_of_component<CT>;
-                        return is_independent[comp_idx] ? std::cref(all_scenarios_sequence[comp_idx])
-                                                        : std::cref(current_scenario_sequence_cache[comp_idx]);
-                    });
-            auto [setup, winddown] = scenario_update_restore(model, update_data, current_scenario_sequence,
-                                                             current_scenario_sequence_cache, is_independent, infos);
+            auto [setup, winddown] = scenario_update_restore(model, update_data, is_independent, all_scenarios_sequence,
+                                                             current_scenario_sequence_cache, infos);
 
             auto calculate_scenario = MainModelImpl::call_with<Idx>(
                 [&model, &calculation_fn, &result_data, &infos](Idx scenario_idx) {
@@ -720,28 +703,41 @@ class MainModelImpl<ExtraRetrievableTypes<ExtraRetrievableType...>, ComponentLis
         };
     }
 
-    static auto scenario_update_restore(
-        MainModelImpl& model, ConstDataset const& update_data,
-        std::array<std::reference_wrapper<std::vector<Idx2D> const>, n_types> const& scenario_sequence,
-        SequenceIdx& current_scenario_sequence_cache, ComponentFlags const& is_independent,
-        std::vector<CalculationInfo>& infos) noexcept {
+    static auto scenario_update_restore(MainModelImpl& model, ConstDataset const& update_data,
+                                        ComponentFlags const& is_independent, SequenceIdx const& all_scenario_sequence,
+                                        SequenceIdx& current_scenario_sequence_cache,
+                                        std::vector<CalculationInfo>& infos) noexcept {
         auto do_update_cache = [&is_independent] {
             ComponentFlags result;
             std::ranges::transform(is_independent, result.begin(), std::logical_not<>{});
             return result;
         }();
 
+        auto const scenario_sequence = [&all_scenario_sequence, &current_scenario_sequence_cache,
+                                        &is_independent]() -> SequenceIdxView {
+            return run_functor_with_all_types_return_array(
+                [&all_scenario_sequence, &current_scenario_sequence_cache, &is_independent]<typename CT>() {
+                    constexpr auto comp_idx = index_of_component<CT>;
+                    if (std::get<comp_idx>(is_independent)) {
+                        return std::span<Idx2D const>{std::get<comp_idx>(all_scenario_sequence)};
+                    }
+                    return std::span<Idx2D const>{std::get<comp_idx>(current_scenario_sequence_cache)};
+                });
+        };
+
         return std::make_pair(
-            [&model, &update_data, &scenario_sequence, &current_scenario_sequence_cache,
+            [&model, &update_data, scenario_sequence, &current_scenario_sequence_cache,
              do_update_cache_ = std::move(do_update_cache), &infos](Idx scenario_idx) {
                 Timer const t_update_model(infos[scenario_idx], 1200, "Update model");
                 current_scenario_sequence_cache =
                     model.get_sequence_idx_map(update_data, scenario_idx, do_update_cache_);
-                model.template update_component<cached_update_t>(update_data, scenario_idx, scenario_sequence);
+
+                model.template update_component<cached_update_t>(update_data, scenario_idx, scenario_sequence());
             },
-            [&model, &scenario_sequence, &current_scenario_sequence_cache, &infos](Idx scenario_idx) {
+            [&model, scenario_sequence, &current_scenario_sequence_cache, &infos](Idx scenario_idx) {
                 Timer const t_update_model(infos[scenario_idx], 1201, "Restore model");
-                model.restore_components(scenario_sequence);
+
+                model.restore_components(scenario_sequence());
                 std::ranges::for_each(current_scenario_sequence_cache,
                                       [](auto& comp_seq_idx) { comp_seq_idx.clear(); });
             });

power_grid_model_c/power_grid_model/include/power_grid_model/math_solver/observability.hpp

@@ -45,17 +45,17 @@ Idx count_bus_injection_sensors(const Idx n_bus, const MeasuredValues<sym>& meas
 
 template <symmetry_tag sym>
 std::tuple<Idx, Idx> count_voltage_sensors(const Idx n_bus, const MeasuredValues<sym>& measured_values) {
-    Idx n_voltage_magnitude{};
-    Idx n_voltage_phasor{};
+    Idx n_voltage_sensor{};
+    Idx n_voltage_phasor_sensor{};
     for (Idx bus = 0; bus != n_bus; ++bus) {
         if (measured_values.has_voltage(bus)) {
-            n_voltage_magnitude++;
+            n_voltage_sensor++;
             if (measured_values.has_angle_measurement(bus)) {
-                n_voltage_phasor++;
+                n_voltage_phasor_sensor++;
             }
         }
     }
-    return std::make_tuple(n_voltage_magnitude, n_voltage_phasor);
+    return std::make_tuple(n_voltage_sensor, n_voltage_phasor_sensor);
 }
 
 } // namespace detail
@@ -66,17 +66,19 @@ inline void necessary_observability_check(MeasuredValues<sym> const& measured_va
     Idx const n_bus{topo->n_bus()};
     std::vector<BranchIdx> const& branch_bus_idx{topo->branch_bus_idx};
 
-    auto const [n_voltage_magnitude, n_voltage_phasor] = detail::count_voltage_sensors(n_bus, measured_values);
-    if (n_voltage_magnitude + n_voltage_phasor < 1) {
+    auto const [n_voltage_sensor, n_voltage_phasor_sensor] = detail::count_voltage_sensors(n_bus, measured_values);
+    if (n_voltage_sensor < 1) {
         throw NotObservableError{};
     }
+
     Idx const n_injection_sensor = detail::count_bus_injection_sensors(n_bus, measured_values);
     Idx const n_branch_sensor = detail::count_branch_sensors(branch_bus_idx, n_bus, measured_values);
+    Idx const n_power_sensor = n_injection_sensor + n_branch_sensor;
 
-    if (n_voltage_phasor == 0 && n_branch_sensor + n_injection_sensor < n_bus - 1) {
+    if (n_voltage_phasor_sensor == 0 && n_power_sensor < n_bus - 1) {
         throw NotObservableError{};
     }
-    if (n_voltage_phasor > 0 && n_branch_sensor + n_injection_sensor + n_voltage_phasor < n_bus) {
+    if (n_voltage_phasor_sensor > 0 && n_power_sensor + n_voltage_phasor_sensor < n_bus) {
         throw NotObservableError{};
     }
 }

src/power_grid_model/__init__.py

@@ -5,7 +5,12 @@
 """Power Grid Model"""
 
 from power_grid_model._core.dataset_definitions import ComponentType, DatasetType
-from power_grid_model._core.power_grid_meta import initialize_array, power_grid_meta_data
+from power_grid_model._core.power_grid_meta import (
+    attribute_dtype,
+    attribute_empty_value,
+    initialize_array,
+    power_grid_meta_data,
+)
 from power_grid_model._core.power_grid_model import PowerGridModel
 from power_grid_model.enum import (
     Branch3Side,

src/power_grid_model/_core/error_handling.py

@@ -7,7 +7,6 @@
 """
 
 import re
-from typing import Optional
 
 import numpy as np
 
@@ -115,7 +114,7 @@ def _interpret_error(message: str, decode_error: bool = True) -> PowerGridError:
     return PowerGridError(message)
 
 
-def find_error(batch_size: int = 1, decode_error: bool = True) -> Optional[RuntimeError]:
+def find_error(batch_size: int = 1, decode_error: bool = True) -> RuntimeError | None:
     """
     Check if there is an error and return it
 
@@ -171,7 +170,7 @@ def assert_no_error(batch_size: int = 1, decode_error: bool = True):
 
 def handle_errors(
     continue_on_batch_error: bool, batch_size: int = 1, decode_error: bool = True
-) -> Optional[PowerGridBatchError]:
+) -> PowerGridBatchError | None:
     """
     Handle any errors in the way that is specified.
 
@@ -184,10 +183,10 @@ def handle_errors(
         error: Any errors previously encountered, unless it was a batch error and continue_on_batch_error was True.
 
     Returns:
-        Optional[PowerGridBatchError]: None if there were no errors, or the previously encountered
-                                       error if it was a batch error and continue_on_batch_error was True.
+        PowerGridBatchError | None: None if there were no errors, or the previously encountered
+                                    error if it was a batch error and continue_on_batch_error was True.
     """
-    error: Optional[RuntimeError] = find_error(batch_size=batch_size, decode_error=decode_error)
+    error: RuntimeError | None = find_error(batch_size=batch_size, decode_error=decode_error)
     if error is None:
         return None