Merge pull request #383 from stfc/aggregate_diff

Aggregate metadata diff sensor

Author: AlexCK-STFC

.github/workflows/unit_tests.yml

@@ -12,7 +12,7 @@ jobs:
     strategy:
       matrix:
         # 3.9+ is not currently supported due to https://github.com/freach/udatetime/issues/32
-        python-version: ["3.8", "3.9"]
+        python-version: ["3.8", "3.9", "3.10"]
     steps:
       - uses: actions/checkout@v5
       - name: Set up Python ${{ matrix.python-version }}

lib/apis/utils/__init__.py

@@ -0,0 +1,74 @@
+import deepdiff
+from deepdiff import extract
+
+
+def diff_to_tabulate_table(obj1, obj2, excluded_paths=None):
+    """
+    Formats the output from DeepDiff into a list of lists suitable for tabulate,
+    showing path, before, and after values.
+    """
+    diff_output = deepdiff.DeepDiff(
+        t1=obj1,
+        t2=obj2,
+        exclude_paths=excluded_paths,
+        threshold_to_diff_deeper=0,
+        ignore_order=True,
+    )
+
+    table_data = []
+
+    # Handle 'values_changed'
+    if "values_changed" in diff_output:
+        for path, change_info in diff_output["values_changed"].items():
+            table_data.append(
+                [
+                    path,
+                    change_info.get("old_value", "Not Present"),
+                    change_info.get("new_value", "Not Present"),
+                ]
+            )
+
+    # Handle 'dictionary_item_added'
+    if "dictionary_item_added" in diff_output:
+        for path in diff_output["dictionary_item_added"]:
+            table_data.append(
+                [
+                    path,
+                    "Not Present",  # No old value for added items
+                    extract(obj2, path),
+                ]
+            )
+
+    # Handle 'dictionary_item_removed'
+    if "dictionary_item_removed" in diff_output:
+        for path in diff_output["dictionary_item_removed"]:
+            table_data.append(
+                [
+                    path,
+                    extract(obj1, path),
+                    "Not Present",  # No new value for removed items
+                ]
+            )
+
+    # Handle 'iterable_item_added'
+    if "iterable_item_added" in diff_output:
+        for path, value in diff_output["iterable_item_added"].items():
+            table_data.append([path, "N/A", value])
+
+    # Handle 'iterable_item_removed'
+    if "iterable_item_removed" in diff_output:
+        for path, value in diff_output["iterable_item_removed"].items():
+            table_data.append([path, value, "N/A"])
+
+    # Handle 'type_changes'
+    if "type_changes" in diff_output:
+        for path, change_info in diff_output["type_changes"].items():
+            table_data.append(
+                [
+                    path,
+                    f"'{change_info.get('old_value')}' (Type: {change_info.get('old_type').__name__})",
+                    f"'{change_info.get('new_value')}' (Type: {change_info.get('new_type').__name__})",
+                ]
+            )
+
+    return table_data

lib/apis/utils/diff_table.py

@@ -0,0 +1,16 @@
+---
+name: "alert.aggregate.metadata.mismatch"
+pack: "stackstorm_openstack"
+description: "Sends Jira Ticket for Aggregate metadata mismatch between prod and dev clouds"
+enabled: true
+
+trigger:
+  type: "stackstorm_openstack.aggregate.metadata_mismatch"
+
+action:
+  ref: "jira.create_issue"
+  parameters:
+    summary: "[Stackstorm Alert]: {{trigger.aggregate_name}} Aggregate metadata mismatch between prod and dev clouds"
+    type: Problem
+    description: >
+      {{trigger.diff}}

rules/alert.aggregate.metadata.mismatch.yaml

@@ -0,0 +1,15 @@
+---
+class_name: HostAggregateSensor
+entry_point: src/host_aggregate_metadata_sensor.py
+description: Detect changes in host aggregates
+poll_interval: 604800 # 7 days
+enabled: false
+trigger_types:
+  - name: "aggregate.metadata_mismatch"
+    description: "Triggers for aggregate metadata mismatch"
+    payload_schema:
+      type: "object"
+      properties:
+        diff:
+          type: "string"
+          description: "diff between aggregate"

sensors/aggregate.aggregate_list.yaml

@@ -0,0 +1,124 @@
+import tabulate
+
+from apis.openstack_api.openstack_connection import OpenstackConnection
+from apis.utils.diff_table import diff_to_tabulate_table
+from st2reactor.sensor.base import PollingSensor
+
+
+class HostAggregateSensor(PollingSensor):
+    """
+    * self.sensor_service
+        - provides utilities like
+            get_logger() for writing to logs.
+            dispatch() for dispatching triggers into the system.
+    * self._config
+        - contains configuration that was specified as
+          config.yaml in the pack.
+    * self._poll_interval
+        - indicates the interval between two successive poll() calls.
+    """
+
+    def __init__(self, sensor_service, config=None, poll_interval=None):
+        super().__init__(
+            sensor_service=sensor_service, config=config, poll_interval=poll_interval
+        )
+        self._log = self._sensor_service.get_logger(__name__)
+        self.source_cloud = self.config["sensor_source_cloud"]
+        self.target_cloud = self.config["sensor_dest_cloud"]
+
+    def setup(self):
+        """
+        Stub
+        """
+
+    def poll(self):
+        """
+        Polls the dev cloud host aggregates and dispatches a payload containing
+        a list of aggregates.
+        """
+        with OpenstackConnection(self.source_cloud) as source_conn, OpenstackConnection(
+            self.target_cloud
+        ) as target_conn:
+
+            source_aggregates = {
+                agg.name: agg for agg in source_conn.compute.aggregates()
+            }
+            target_aggregates = {
+                agg.name: agg for agg in target_conn.compute.aggregates()
+            }
+
+            self._log.info(
+                "Compare source (%s) and target (%s) host aggregate metadata"
+            )
+            for aggregate_name, source_agg in source_aggregates.items():
+                target_agg = target_aggregates.get(aggregate_name)
+
+                if not target_agg:
+                    self._log.info(
+                        "aggregate %s doesn't exist in %s cloud",
+                        aggregate_name,
+                        self.target_cloud,
+                    )
+                    continue
+
+                diff = diff_to_tabulate_table(
+                    obj1=source_agg,
+                    obj2=target_agg,
+                    excluded_paths=[
+                        "root['hosts']",
+                        "root['created_at']",
+                        "root['updated_at']",
+                        "root['uuid']",
+                        "root['id']",
+                        "root['location']",
+                    ],
+                )
+
+                if diff:
+
+                    self._log.info(
+                        "aggregate metadata mismatch between source (%s) and target (%s): %s",
+                        self.source_cloud,
+                        self.target_cloud,
+                        aggregate_name,
+                    )
+
+                    headers = [
+                        "Path",
+                        self.source_cloud,
+                        self.target_cloud,
+                    ]
+
+                    payload = {
+                        "aggregate_name": source_agg.name,
+                        "diff": tabulate.tabulate(
+                            diff,
+                            headers=headers,
+                            tablefmt="jira",
+                        ),
+                    }
+
+                    self.sensor_service.dispatch(
+                        trigger="stackstorm_openstack.aggregate.metadata_mismatch",
+                        payload=payload,
+                    )
+
+    def cleanup(self):
+        """
+        Stub
+        """
+
+    def add_trigger(self, trigger):
+        """
+        Stub
+        """
+
+    def update_trigger(self, trigger):
+        """
+        Stub
+        """
+
+    def remove_trigger(self, trigger):
+        """
+        Stub
+        """