Add CF_TIP_BOOT_FAILED_COUNT metric for cuttlefish boot failures (#4713)

This commit introduces the `CF_TIP_BOOT_FAILED_COUNT` metric to track
failures when booting Cuttlefish with a tip-of-tree build. This metric
is intended to be used as the basis for an alerting mechanism. By
setting up an alerting policy on this metric, we can be notified when
the number of tip-of-tree boot failures exceeds a defined threshold.

Author: aditya-wazir

src/clusterfuzz/_internal/metrics/monitoring_metrics.py

@@ -23,6 +23,15 @@
         monitor.BooleanField('success'),
     ])
 
+CF_TIP_BOOT_FAILED_COUNT = monitor.CounterMetric(
+    'tip_boot_failure',
+    description=
+    'Count of failure in booting up cuttlefish with tip-of-the-tree build ',
+    field_spec=[
+        monitor.StringField('build_id'),
+        monitor.BooleanField('is_succeeded'),
+    ])
+
 JOB_BAD_BUILD_COUNT = monitor.CounterMetric(
     'task/fuzz/job/bad_build_count',
     description=("Count of fuzz task's bad build count "

src/clusterfuzz/_internal/platforms/android/flash.py

@@ -21,6 +21,7 @@
 from clusterfuzz._internal.base import persistent_cache
 from clusterfuzz._internal.datastore import locks
 from clusterfuzz._internal.metrics import logs
+from clusterfuzz._internal.metrics import monitoring_metrics
 from clusterfuzz._internal.system import archive
 from clusterfuzz._internal.system import environment
 from clusterfuzz._internal.system import shell
@@ -206,9 +207,17 @@ def flash_to_latest_build_if_needed():
     locks.release_lock(flash_lock_key_name, by_zone=True)
 
   if adb.get_device_state() != 'device':
+    monitoring_metrics.CF_TIP_BOOT_FAILED_COUNT.increment({
+        'build_id': build_info['bid'],
+        'is_succeeded': False
+    })
     logs.error('Unable to find device. Reimaging failed.')
     adb.bad_state_reached()
 
+  monitoring_metrics.CF_TIP_BOOT_FAILED_COUNT.increment({
+      'build_id': build_info['bid'],
+      'is_succeeded': True
+  })
   logs.info('Reimaging finished.')
 
   # Reset all of our persistent keys after wipe.

src/clusterfuzz/_internal/tests/core/metrics/monitor_test.py

@@ -15,11 +15,15 @@
 # pylint: disable=protected-access
 
 import os
+import queue
 import time
 import unittest
+from unittest.mock import patch
 
 from clusterfuzz._internal.metrics import monitor
 from clusterfuzz._internal.metrics import monitoring_metrics
+from clusterfuzz._internal.platforms.android import flash
+from clusterfuzz._internal.system import environment
 from clusterfuzz._internal.tests.test_libs import helpers
 
 
@@ -69,10 +73,82 @@ class MonitorTest(unittest.TestCase):
   def setUp(self):
     helpers.patch(self, [
         'clusterfuzz._internal.metrics.monitor.check_module_loaded',
+        'clusterfuzz._internal.base.persistent_cache.get_value',
+        'clusterfuzz._internal.base.persistent_cache.set_value',
+        'clusterfuzz._internal.base.persistent_cache.delete_value',
+        'clusterfuzz._internal.platforms.android.settings.is_google_device',
+        'clusterfuzz._internal.platforms.android.fetch_artifact.get_latest_artifact_info',
+        'clusterfuzz._internal.system.environment.is_android_cuttlefish',
+        'clusterfuzz._internal.platforms.android.flash.download_latest_build',
+        'clusterfuzz._internal.platforms.android.adb.connect_to_cuttlefish_device',
+        'clusterfuzz._internal.platforms.android.adb.recreate_cuttlefish_device',
+        'clusterfuzz._internal.platforms.android.adb.get_device_state',
+        'clusterfuzz._internal.platforms.android.adb.bad_state_reached',
     ])
     self.mock.check_module_loaded.return_value = True
+    self.mock.get_value.return_value = None
+    self.mock.is_google_device.return_value = True
+    self.mock.get_latest_artifact_info.return_value = {
+        'bid': 'test-bid',
+        'branch': 'test-branch',
+        'target': 'test-target'
+    }
+    self.mock.is_android_cuttlefish.return_value = True
+    environment.set_value('BUILD_BRANCH', 'test-branch')
+    environment.set_value('BUILD_TARGET', 'test-target')
     monitor.metrics_store().reset_for_testing()
 
+  def _setup_monitoring_daemon(self, mock_client):
+    """Setup and start monitoring daemon."""
+    monitor.credentials._use_anonymous_credentials = lambda: False
+    monitor._monitoring_v3_client = mock_client.return_value
+    monitor.FLUSH_INTERVAL_SECONDS = 10
+    monitor._monitoring_daemon = monitor._MonitoringDaemon(
+        monitor._flush_metrics, monitor.FLUSH_INTERVAL_SECONDS)
+    monitor.utils.get_application_id = lambda: 'google.com:clusterfuzz'
+    os.environ['BOT_NAME'] = 'bot-1'
+    monitor._initialize_monitored_resource()
+    monitor._monitored_resource.labels['zone'] = 'us-central1-b'
+    call_queue = queue.Queue()
+    mock_create_time_series = mock_client.return_value.create_time_series
+    mock_create_time_series.side_effect = (
+        lambda **kwargs: call_queue.put(kwargs))
+    monitor._monitoring_daemon.start()
+    return call_queue
+
+  def _assert_cuttlefish_boot_metric(self, time_series, is_succeeded):
+    """Asserts Cuttlefish boot failure metric presence and correctness in time series."""
+    for ts in time_series:
+      if ts.metric.type == "custom.googleapis.com/tip_boot_failure":
+        if is_succeeded is not None and ts.metric.labels['is_succeeded'] != str(
+            is_succeeded):
+          continue
+        self.assertEqual(ts.metric.labels['is_succeeded'], str(is_succeeded))
+        self.assertEqual(ts.metric.labels['build_id'], "test-bid")
+
+  @patch(
+      'clusterfuzz._internal.metrics.monitor.monitoring_v3.MetricServiceClient')
+  def test_cuttlefish_boot_success_metric(self, mock_client):
+    """Tests the metric emission for a successful Cuttlefish boot."""
+    call_queue = self._setup_monitoring_daemon(mock_client)
+    self.mock.get_device_state.return_value = 'device'
+    flash.flash_to_latest_build_if_needed()
+    args = call_queue.get(timeout=20)
+    time_series = args['time_series']
+    self._assert_cuttlefish_boot_metric(time_series, True)
+    monitor._monitoring_daemon.stop()
+
+  @patch(
+      'clusterfuzz._internal.metrics.monitor.monitoring_v3.MetricServiceClient')
+  def test_cuttlefish_boot_failure_metric(self, mock_client):
+    """Tests the metric emission for a failed Cuttlefish boot."""
+    call_queue = self._setup_monitoring_daemon(mock_client)
+    flash.flash_to_latest_build_if_needed()
+    args = call_queue.get(timeout=20)
+    time_series = args['time_series']
+    self._assert_cuttlefish_boot_metric(time_series, False)
+    monitor._monitoring_daemon.stop()
+
   def test_counter_metric_success(self):
     self.assertIsInstance(
         monitor.CounterMetric('t', 'desc', field_spec=None),