Add "Cancelled" Button for Paused Backfills

client/src/main/proto/app/cash/backfila/service.proto

@@ -102,5 +102,6 @@ message CheckBackfillStatusResponse {
     PAUSED = 1;
     RUNNING = 2;
     COMPLETE = 3;
+    CANCELLED = 4;
   }
 }

service/src/main/kotlin/app/cash/backfila/api/CheckBackfillStatusAction.kt

@@ -37,6 +37,7 @@ class CheckBackfillStatusAction @Inject constructor(
       BackfillState.PAUSED -> Status.PAUSED
       BackfillState.RUNNING -> Status.RUNNING
       BackfillState.COMPLETE -> Status.COMPLETE
+      BackfillState.CANCELLED -> Status.CANCELLED
     }
   }
 }

service/src/main/kotlin/app/cash/backfila/dashboard/CancelBackfillAction.kt

@@ -0,0 +1,62 @@
+package app.cash.backfila.dashboard
+
+import app.cash.backfila.service.listener.BackfillRunListener
+import app.cash.backfila.service.persistence.BackfilaDb
+import app.cash.backfila.service.persistence.BackfillState
+import app.cash.backfila.service.persistence.DbBackfillRun
+import app.cash.backfila.service.persistence.DbEventLog
+import javax.inject.Inject
+import misk.MiskCaller
+import misk.exceptions.BadRequestException
+import misk.hibernate.Id
+import misk.hibernate.Query
+import misk.hibernate.Transacter
+import misk.hibernate.load
+import misk.scope.ActionScoped
+import misk.security.authz.Authenticated
+import misk.web.PathParam
+import misk.web.Post
+import misk.web.RequestContentType
+import misk.web.ResponseContentType
+import misk.web.actions.WebAction
+import misk.web.mediatype.MediaTypes
+
+class CancelBackfillAction @Inject constructor(
+  @BackfilaDb private val transacter: Transacter,
+  private val queryFactory: Query.Factory,
+  private val backfillRunListeners: Set<BackfillRunListener>,
+  private val caller: @JvmSuppressWildcards ActionScoped<MiskCaller?>,
+) : WebAction {
+  @Post("/backfill/cancel/{id}")
+  @RequestContentType(MediaTypes.APPLICATION_JSON)
+  @ResponseContentType(MediaTypes.APPLICATION_JSON)
+  @Authenticated(capabilities = ["users"])
+  fun cancel(
+    @PathParam id: Long,
+  ) {
+    transacter.transaction { session ->
+      val backfillRun = session.load<DbBackfillRun>(Id(id))
+
+      if (backfillRun.state != BackfillState.PAUSED) {
+        throw BadRequestException("Cannot cancel a ${backfillRun.state.name.lowercase()} backfill")
+      }
+
+      // Update state in run_partitions table
+      backfillRun.setState(session, queryFactory, BackfillState.CANCELLED)
+
+      // Log the cancellation event
+      session.save(
+        DbEventLog(
+          backfillRun.id,
+          partition_id = null,
+          user = caller.get()?.user,
+          type = DbEventLog.Type.STATE_CHANGE,
+          message = "backfill cancelled",
+        ),
+      )
+    }
+
+    // Notify listeners
+    backfillRunListeners.forEach { it.runCancelled(Id(id), caller.get()?.principal ?: "") }
+  }
+}

service/src/main/kotlin/app/cash/backfila/service/listener/AuditClientListener.kt

@@ -71,6 +71,20 @@ internal class AuditClientListener @Inject constructor(
     )
   }
 
+  override fun runCancelled(id: Id<DbBackfillRun>, user: String) {
+    val (backfillName, serviceName, description) = transacter.transaction { session ->
+      val run = session.load<DbBackfillRun>(id)
+      AuditEventInputs(run.registered_backfill.name, serviceName(run), "Backfill cancelled by $user ${dryRunPrefix(run)}${nameAndId(run)}")
+    }
+    auditClient.logEvent(
+      target = backfillName,
+      description = description,
+      requestorLDAP = user,
+      applicationName = serviceName,
+      detailURL = idUrl(id),
+    )
+  }
+
   private fun serviceName(run: DbBackfillRun) = if (run.service.variant == "default") {
     run.service.registry_name
   } else {

service/src/main/kotlin/app/cash/backfila/service/listener/BackfillRunListener.kt

@@ -8,4 +8,5 @@ interface BackfillRunListener {
   fun runPaused(id: Id<DbBackfillRun>, user: String)
   fun runErrored(id: Id<DbBackfillRun>)
   fun runCompleted(id: Id<DbBackfillRun>)
+  fun runCancelled(id: Id<DbBackfillRun>, user: String)
 }

service/src/main/kotlin/app/cash/backfila/service/listener/SlackHelper.kt

@@ -52,6 +52,15 @@ class SlackHelper @Inject constructor(
     slackClient.postMessage("Backfila", ":backfila:", message, channel)
   }
 
+  override fun runCancelled(id: Id<DbBackfillRun>, user: String) {
+    val (message, channel) = transacter.transaction { session ->
+      val run = session.load(id)
+      val message = ":backfila_cancel:${dryRunEmoji(run)} ${nameAndId(run)} canceled by @$user"
+      message to run.service.slack_channel
+    }
+    slackClient.postMessage("Backfila", ":backfila:", message, channel)
+  }
+
   private fun nameAndId(run: DbBackfillRun) =
     "[${deployment.name}] ${run.service.registry_name} (${run.service.variant}) `${run.registered_backfill.name}` " +
       "(${idLink(run.id)})"

service/src/main/kotlin/app/cash/backfila/service/persistence/BackfillState.kt

@@ -4,4 +4,5 @@ enum class BackfillState {
   PAUSED,
   RUNNING,
   COMPLETE,
+  CANCELLED,
 }

service/src/main/kotlin/app/cash/backfila/service/persistence/DbBackfillRun.kt

@@ -135,6 +135,7 @@ class DbBackfillRun() : DbUnsharded<DbBackfillRun>, DbTimestampedEntity {
   fun setState(session: Session, queryFactory: Query.Factory, state: BackfillState) {
     // State can't be changed after being completed.
     checkState(this.state != BackfillState.COMPLETE)
+
     this.state = state
     // Set the state of all the partitions that are not complete
     val query = session.hibernateSession.createQuery(

service/src/main/kotlin/app/cash/backfila/service/runner/BackfillRunner.kt

@@ -171,7 +171,12 @@ class BackfillRunner private constructor(
 
       // Now that state is stored, check if we should exit.
       if (dbRunPartition.run_state != BackfillState.RUNNING) {
-        logger.info { "Backfill is no longer in RUNNING state, stopping runner ${logLabel()}" }
+        val stateChange = when (dbRunPartition.run_state) {
+          BackfillState.PAUSED -> "paused"
+          BackfillState.CANCELLED -> "cancelled"
+          else -> dbRunPartition.run_state.name.lowercase()
+        }
+        logger.info { "Backfill is no longer in RUNNING state (now $stateChange), stopping runner ${logLabel()}" }
         running = false
         return@transaction
       }

service/src/main/kotlin/app/cash/backfila/ui/actions/BackfillShowButtonHandlerAction.kt

@@ -1,5 +1,6 @@
 package app.cash.backfila.ui.actions
 
+import app.cash.backfila.dashboard.CancelBackfillAction
 import app.cash.backfila.dashboard.StartBackfillAction
 import app.cash.backfila.dashboard.StartBackfillRequest
 import app.cash.backfila.dashboard.StopBackfillAction
@@ -31,6 +32,7 @@ class BackfillShowButtonHandlerAction @Inject constructor(
   private val startBackfillAction: StartBackfillAction,
   private val stopBackfillAction: StopBackfillAction,
   private val updateBackfillAction: UpdateBackfillAction,
+  private val cancelBackfillAction: CancelBackfillAction,
 ) : WebAction {
   @Get(PATH)
   @ResponseContentType(MediaTypes.TEXT_HTML)
@@ -44,10 +46,16 @@ class BackfillShowButtonHandlerAction @Inject constructor(
       if (!field_id.isNullOrBlank()) {
         when (field_id) {
           "state" -> {
-            if (field_value == BackfillState.PAUSED.name) {
-              stopBackfillAction.stop(id.toLong(), StopBackfillRequest())
-            } else if (field_value == BackfillState.RUNNING.name) {
-              startBackfillAction.start(id.toLong(), StartBackfillRequest())
+            when (field_value) {
+              BackfillState.PAUSED.name -> {
+                stopBackfillAction.stop(id.toLong(), StopBackfillRequest())
+              }
+              BackfillState.RUNNING.name -> {
+                startBackfillAction.start(id.toLong(), StartBackfillRequest())
+              }
+              BackfillState.CANCELLED.name -> {
+                cancelBackfillAction.cancel(id.toLong())
+              }
             }
           }
 

service/src/main/kotlin/app/cash/backfila/ui/pages/BackfillShowAction.kt

@@ -259,6 +259,7 @@ class BackfillShowAction @Inject constructor(
     /* Value of the button click is provided through the button.href field. */
     val button: Link? = null,
     val updateFieldId: String? = null,
+    val cancelButton: Link? = null,
   )
 
   private fun getStateButton(state: BackfillState): Link? {
@@ -267,21 +268,33 @@ class BackfillShowAction @Inject constructor(
         label = START_STATE_BUTTON_LABEL,
         href = BackfillState.RUNNING.name,
       )
-
+      // COMPLETE and CANCELLED represent final states.
       BackfillState.COMPLETE -> null
+      BackfillState.CANCELLED -> null
       else -> Link(
         label = PAUSE_STATE_BUTTON_LABEL,
         href = BackfillState.PAUSED.name,
       )
     }
   }
 
+  private fun getCancelButton(state: BackfillState): Link? {
+    return when (state) {
+      BackfillState.PAUSED -> Link(
+        label = CANCEL_STATE_BUTTON_LABEL,
+        href = BackfillState.CANCELLED.name,
+      )
+      else -> null
+    }
+  }
+
   private fun GetBackfillStatusResponse.toConfigurationRows(id: Long) = listOf(
     DescriptionListRow(
       label = "State",
       description = state.name,
       button = getStateButton(state),
       updateFieldId = "state",
+      cancelButton = getCancelButton(state),
     ),
     DescriptionListRow(
       label = "Dry Run",
@@ -487,6 +500,36 @@ class BackfillShowAction @Inject constructor(
                   +button.label
                 }
               }
+
+              // Add cancel button if present
+              it.cancelButton?.let { cancelButton ->
+                span("ml-2") {
+                  form {
+                    action = BackfillShowButtonHandlerAction.path(id)
+
+                    it.updateFieldId?.let {
+                      input {
+                        type = InputType.hidden
+                        name = "field_id"
+                        value = it
+                      }
+
+                      input {
+                        type = InputType.hidden
+                        name = "field_value"
+                        value = cancelButton.href
+                      }
+                    }
+
+                    button(
+                      classes = "rounded-full bg-red-600 px-3 py-1.5 text-sm font-semibold text-white shadow-sm hover:bg-red-500 focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-red-600",
+                    ) {
+                      type = ButtonType.submit
+                      +cancelButton.label
+                    }
+                  }
+                }
+              }
             }
           }
         }
@@ -538,6 +581,7 @@ class BackfillShowAction @Inject constructor(
 
     const val START_STATE_BUTTON_LABEL = "Start"
     const val PAUSE_STATE_BUTTON_LABEL = "Pause"
+    const val CANCEL_STATE_BUTTON_LABEL = "Cancel"
     const val UPDATE_BUTTON_LABEL = "Update"
     const val VIEW_LOGS_BUTTON_LABEL = "View Logs"
   }

service/src/main/resources/migrations/v017__backfila.sql

@@ -0,0 +1,2 @@
+ALTER TABLE backfill_runs
+    MODIFY COLUMN `state` enum('PAUSED','RUNNING','COMPLETE','CANCELLED') NOT NULL;

service/src/main/resources/migrations/v018__backfila.sql

@@ -0,0 +1,2 @@
+ALTER TABLE run_partitions
+    MODIFY COLUMN `run_state` enum('PAUSED','RUNNING','COMPLETE','CANCELLED') NOT NULL;