Downtimes: mark their histories as cancelled when removed from conf file

pkg/icingadb/sync.go

@@ -11,6 +11,7 @@ import (
 	"github.com/icinga/icinga-go-library/types"
 	"github.com/icinga/icingadb/pkg/common"
 	v1 "github.com/icinga/icingadb/pkg/icingadb/v1"
+	"github.com/icinga/icingadb/pkg/icingadb/v1/history"
 	"github.com/icinga/icingadb/pkg/icingaredis"
 	"github.com/icinga/icingadb/pkg/icingaredis/telemetry"
 	"github.com/pkg/errors"
@@ -162,9 +163,32 @@ func (s Sync) ApplyDelta(ctx context.Context, delta *Delta) error {
 	// Delete
 	if len(delta.Delete) > 0 {
 		s.logger.Infof("Deleting %d items of type %s", len(delta.Delete), strcase.Delimited(types.Name(delta.Subject.Entity()), ' '))
-		g.Go(func() error {
-			return s.db.Delete(ctx, delta.Subject.Entity(), delta.Delete.IDs(), database.OnSuccessIncrement[any](stat))
-		})
+
+		ids := delta.Delete.IDs()
+		if _, ok := delta.Subject.Entity().(*v1.Downtime); ok {
+			// Those downtimes are probably removed from the configuration files, i.e. Icinga 2 won't send
+			// the corresponding downtime end/removed events for them. So try to mark them as cancelled manually,
+			// so that they don't show up as if they were still active in the UI.
+			//
+			// The reason why we don't perform this in a separate goroutine like the other ones is that we don't
+			// want to delete the downtimes until we've successfully updated the downtime history records. So,
+			// if we fail to update the downtime history records, for whatever reason, forward the error to the
+			// config sync group and abort the config sync.
+			if err := history.SyncDowntimeHistoryEndEvent(ctx, s.db, ids); err != nil {
+				errCh := make(chan error, 1)
+				errCh <- err
+				close(errCh)
+				com.ErrgroupReceive(g, errCh)
+			}
+		}
+
+		// Start the deletion process only if we haven't aborted the config sync
+		// due to an error in the downtime history update above.
+		if ctx.Err() == nil {
+			g.Go(func() error {
+				return s.db.Delete(ctx, delta.Subject.Entity(), ids, database.OnSuccessIncrement[any](stat))
+			})
+		}
 	}
 
 	return g.Wait()

pkg/icingadb/v1/history/downtime.go

@@ -1,10 +1,13 @@
 package history
 
 import (
+	"context"
 	"database/sql/driver"
 	"github.com/icinga/icinga-go-library/database"
 	"github.com/icinga/icinga-go-library/types"
 	"github.com/icinga/icingadb/pkg/contracts"
+	"golang.org/x/sync/errgroup"
+	"time"
 )
 
 type DowntimeHistoryEntity struct {
@@ -147,6 +150,47 @@ func (et SlaDowntimeEndTime) Value() (driver.Value, error) {
 	}
 }
 
+// SyncDowntimeHistoryEndEvent updates the downtime history records with the given downtime IDs to mark them as cancelled.
+//
+// This function is used to mark downtimes as cancelled when the downtime configuration is removed from
+// the configuration files. In such cases, Icinga 2 won't send the corresponding downtime end/removed events,
+// so we need to mark the downtimes as cancelled manually.
+func SyncDowntimeHistoryEndEvent(ctx context.Context, db *database.DB, downtimeIds []any) error {
+	downtimes := make(chan database.Entity, len(downtimeIds))
+	g, ctx := errgroup.WithContext(ctx)
+	g.Go(func() error {
+		defer close(downtimes)
+
+		// Transform the downtime IDs into DowntimeHistory entities with the necessary fields
+		// and stream them into the downtimes channel.
+		for _, id := range downtimeIds {
+			// The downtimes channel is buffered, so this will never block, and we can just keep sending.
+			downtimes <- &DowntimeHistory{
+				DowntimeHistoryEntity: DowntimeHistoryEntity{DowntimeId: id.(types.Binary)},
+				DowntimeHistoryUpserter: DowntimeHistoryUpserter{
+					CancelledBy:      types.MakeString("Downtime Config Removed"),
+					HasBeenCancelled: types.Bool{Bool: true, Valid: true},
+					CancelTime:       types.UnixMilli(time.Now()),
+				},
+			}
+		}
+
+		return nil
+	})
+
+	g.Go(func() error {
+		// Each downtime removed in this manner should never have been cancelled before, so we
+		// don't need extra where clauses in the update statement other than the downtime ID.
+		stmt := `UPDATE downtime_history SET cancel_time = :cancel_time, has_been_cancelled = :has_been_cancelled, cancelled_by = :cancelled_by WHERE downtime_id = :downtime_id`
+
+		return db.NamedBulkExecTx(ctx, stmt, db.Options.MaxRowsPerTransaction, db.GetSemaphoreForTable("downtime_history"), downtimes)
+	})
+
+	// TODO: Fake the downtime end event in the regular history and SLA downtime history tables.
+
+	return g.Wait()
+}
+
 // Assert interface compliance.
 var (
 	_ database.Entity     = (*DowntimeHistoryEntity)(nil)