Skip to content

Commit 0f66e33

Browse files
jeffhwang-sqjeffhwang-sq
committed
add-deprecation-reminder
1 parent 70a001c commit 0f66e33

File tree

9 files changed

+489
-0
lines changed

9 files changed

+489
-0
lines changed

‎service/src/main/kotlin/app/cash/backfila/service/BackfilaConfig.kt‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package app.cash.backfila.service
33
import misk.config.Config
44
import misk.jdbc.DataSourceClustersConfig
55
import misk.slack.SlackConfig
6+
import app.cash.backfila.service.deletion.DeleteByNotificationConfig
67

78
data class BackfilaConfig(
89
val backfill_runner_threads: Int?,
@@ -19,4 +20,6 @@ data class BackfilaConfig(
1920
val support_button_label: String? = null,
2021
/** Support banner shows up on all pages and can point to a Slack channel or other support method, if null banner not shown. */
2122
val support_button_url: String? = null,
23+
/** Configuration for delete-by notification system */
24+
val delete_by_notification: DeleteByNotificationConfig = DeleteByNotificationConfig()
2225
) : Config

‎service/src/main/kotlin/app/cash/backfila/service/BackfilaServiceModule.kt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ import app.cash.backfila.service.runner.BackfillRunnerLoggingSetupProvider
1515
import app.cash.backfila.service.runner.BackfillRunnerNoLoggingSetupProvider
1616
import app.cash.backfila.service.scheduler.ForBackfilaScheduler
1717
import app.cash.backfila.service.scheduler.RunnerSchedulerServiceModule
18+
import app.cash.backfila.service.deletion.DeleteByNotificationModule
1819
import com.google.common.util.concurrent.ListeningExecutorService
1920
import com.google.common.util.concurrent.MoreExecutors
2021
import com.google.common.util.concurrent.ThreadFactoryBuilder
@@ -53,6 +54,7 @@ class BackfilaServiceModule(
5354
install(ServiceWebActionsModule())
5455

5556
install(RunnerSchedulerServiceModule())
57+
install(DeleteByNotificationModule())
5658

5759
newMapBinder<String, BackfilaCallbackConnectorProvider>(ForConnectors::class)
5860
.addBinding(Connectors.HTTP)

‎service/src/main/kotlin/app/cash/backfila/service/deletion/DeleteByNotificationConfig.kt‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
package app.cash.backfila.service.deletion
2+
3+
import java.time.Duration
4+
import misk.config.Config
5+
6+
data class DeleteByNotificationConfig(
7+
// Default delete_by duration for backfills without an explicit date
8+
val defaultDeleteByDuration: Duration = Duration.ofDays(180), // 6 months
9+
10+
// Pre-deletion notification thresholds
11+
val promptAfterLastSuccessfulRun: Duration = Duration.ofDays(30),
12+
val promptAfterFailedOrNoRuns: Duration = Duration.ofDays(90),
13+
14+
// Pre-deletion notification stages
15+
val preDeleteStages: List<NotificationStage> = listOf(
16+
NotificationStage(Duration.ofDays(90), Duration.ofDays(30)), // 3 months out: Monthly
17+
NotificationStage(Duration.ofDays(60), Duration.ofDays(7)), // 2 months out: Weekly
18+
NotificationStage(Duration.ofDays(30), Duration.ofDays(1)) // 1 month out: Daily
19+
),
20+
21+
// Post-deletion notification configuration
22+
val postDeleteNotifications: PostDeleteNotifications = PostDeleteNotifications(
23+
initialDelay = Duration.ofDays(1), // First notification 1 day after delete_by
24+
followUpDelays = listOf( // Additional notifications after
25+
Duration.ofDays(7), // 1 week
26+
Duration.ofDays(30), // 1 month
27+
Duration.ofDays(90) // 3 months
28+
),
29+
maxAge = Duration.ofDays(180) // Stop notifications after 6 months
30+
)
31+
) : Config
32+
33+
data class NotificationStage(
34+
val threshold: Duration,
35+
val frequency: Duration
36+
)
37+
38+
data class PostDeleteNotifications(
39+
val initialDelay: Duration, // How long after delete_by to send first notification
40+
val followUpDelays: List<Duration>, // When to send follow-up notifications
41+
val maxAge: Duration // Stop notifications after this duration past delete_by
42+
)
43+
44+
enum class NotificationDecision(val emoji: String) {
45+
NONE(""),
46+
NOTIFY_INFO("ℹ️"),
47+
NOTIFY_WARNING("⚠️"),
48+
NOTIFY_URGENT("🚨"),
49+
NOTIFY_EXPIRED("⏰") // For post-deletion notifications
50+
}

‎service/src/main/kotlin/app/cash/backfila/service/deletion/DeleteByNotificationHelper.kt‎

Lines changed: 198 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,198 @@
1+
package app.cash.backfila.service.deletion
2+
3+
import app.cash.backfila.service.BackfilaConfig
4+
import app.cash.backfila.service.persistence.BackfilaDb
5+
import app.cash.backfila.service.persistence.DbRegisteredBackfill
6+
import app.cash.backfila.service.persistence.EventLogQuery
7+
import app.cash.backfila.service.persistence.RegisteredBackfillQuery
8+
import app.cash.backfila.service.listener.SlackHelper
9+
import app.cash.backfila.service.persistence.DbBackfillRun
10+
import app.cash.backfila.service.persistence.DbEventLog
11+
import misk.hibernate.Id
12+
import java.time.Clock
13+
import java.time.Duration
14+
import java.time.temporal.ChronoUnit
15+
import javax.inject.Inject
16+
import javax.inject.Singleton
17+
import misk.hibernate.Query
18+
import misk.hibernate.Transacter
19+
import misk.hibernate.newQuery
20+
import wisp.logging.getLogger
21+
22+
@Singleton
23+
class DeleteByNotificationHelper @Inject constructor(
24+
@BackfilaDb private val transacter: Transacter,
25+
private val queryFactory: Query.Factory,
26+
private val slackHelper: SlackHelper,
27+
private val backfilaConfig: BackfilaConfig,
28+
private val clock: Clock
29+
) {
30+
fun getBackfillsNeedingNotification(): List<DbRegisteredBackfill> {
31+
return transacter.transaction { session ->
32+
// Get all active backfills
33+
val backfills = queryFactory.newQuery<RegisteredBackfillQuery>()
34+
.active()
35+
.list(session)
36+
37+
// For backfills without a delete_by date, set it based on creation time
38+
backfills.forEach { backfill ->
39+
if (backfill.delete_by == null) {
40+
backfill.delete_by = backfill.created_at.plus(
41+
backfilaConfig.delete_by_notification.defaultDeleteByDuration
42+
)
43+
// Save the updated backfill
44+
session.save(backfill)
45+
}
46+
}
47+
backfills
48+
}
49+
}
50+
51+
fun evaluateBackfill(backfill: DbRegisteredBackfill): NotificationDecision {
52+
val deleteBy = backfill.delete_by
53+
?: return NotificationDecision.NONE
54+
55+
return transacter.transaction { session ->
56+
// Get relevant events
57+
val events = queryFactory.newQuery<EventLogQuery>()
58+
.backfillRunId(backfill.id as Id<DbBackfillRun>)
59+
.list(session)
60+
61+
val lastSuccessfulRun = events
62+
.filter { it.type == DbEventLog.Type.NOTIFICATION && it.message == "COMPLETED" }
63+
.maxByOrNull { it.created_at }
64+
65+
val lastNotification = events
66+
.filter { it.type == DbEventLog.Type.NOTIFICATION }
67+
.maxByOrNull { it.created_at }
68+
69+
// Calculate time until/since deletion
70+
val now = clock.instant()
71+
val timeUntilDeletion = Duration.between(now, deleteBy)
72+
73+
// If we're past the delete_by date
74+
if (timeUntilDeletion.isNegative) {
75+
val timeSinceDeletion = Duration.between(deleteBy, now)
76+
return@transaction evaluatePostDeleteNotification(timeSinceDeletion, lastNotification)
77+
}
78+
79+
// Find appropriate notification stage
80+
val stage = backfilaConfig.delete_by_notification.preDeleteStages.find {
81+
timeUntilDeletion <= it.threshold
82+
} ?: return@transaction NotificationDecision.NONE
83+
84+
// Check if we should notify based on the stage frequency
85+
val shouldNotify = lastNotification?.let {
86+
Duration.between(it.created_at, now) >= stage.frequency
87+
} ?: true
88+
89+
if (!shouldNotify) {
90+
return@transaction NotificationDecision.NONE
91+
}
92+
93+
// Determine notification urgency based on run status
94+
when {
95+
lastSuccessfulRun == null ||
96+
Duration.between(lastSuccessfulRun.created_at, now) > backfilaConfig.delete_by_notification.promptAfterFailedOrNoRuns ->
97+
NotificationDecision.NOTIFY_URGENT
98+
99+
Duration.between(lastSuccessfulRun.created_at, now) > backfilaConfig.delete_by_notification.promptAfterLastSuccessfulRun ->
100+
NotificationDecision.NOTIFY_WARNING
101+
102+
else -> NotificationDecision.NOTIFY_INFO
103+
}
104+
}
105+
}
106+
107+
private fun evaluatePostDeleteNotification(
108+
timeSinceDeletion: Duration,
109+
lastNotification: DbEventLog?
110+
): NotificationDecision {
111+
val config = backfilaConfig.delete_by_notification.postDeleteNotifications
112+
113+
// Stop notifications if we're past maxAge
114+
if (timeSinceDeletion > config.maxAge) {
115+
return NotificationDecision.NONE
116+
}
117+
118+
// If we haven't sent any notifications yet, check initial delay
119+
if (lastNotification == null) {
120+
return if (timeSinceDeletion >= config.initialDelay) {
121+
NotificationDecision.NOTIFY_EXPIRED
122+
} else {
123+
NotificationDecision.NONE
124+
}
125+
}
126+
127+
// Find the next notification delay that's applicable
128+
val nextDelay = config.followUpDelays.find { delay ->
129+
timeSinceDeletion <= delay &&
130+
Duration.between(lastNotification.created_at, clock.instant()) >= delay
131+
}
132+
133+
return if (nextDelay != null) NotificationDecision.NOTIFY_EXPIRED else NotificationDecision.NONE
134+
}
135+
136+
fun sendNotification(
137+
backfill: DbRegisteredBackfill,
138+
decision: NotificationDecision,
139+
channel: String
140+
) {
141+
val message = generateNotificationMessage(backfill, decision)
142+
143+
// Send to Slack
144+
slackHelper.sendDeletionNotification(message, channel)
145+
146+
// Record notification in event_logs
147+
transacter.transaction { session ->
148+
session.save(
149+
DbEventLog(
150+
backfill_run_id = backfill.id as Id<DbBackfillRun>,
151+
partition_id = null,
152+
type = DbEventLog.Type.NOTIFICATION,
153+
message = "Deletion notification sent to $channel",
154+
extra_data = message
155+
)
156+
)
157+
}
158+
}
159+
160+
fun determineNotificationChannel(backfill: DbRegisteredBackfill): String {
161+
// TODO: Make this configurable or determine from backfill metadata
162+
return "#backfila-notifications"
163+
}
164+
165+
private fun generateNotificationMessage(
166+
backfill: DbRegisteredBackfill,
167+
decision: NotificationDecision
168+
): String {
169+
val deleteBy = backfill.delete_by!!
170+
val daysUntilDeletion = ChronoUnit.DAYS.between(clock.instant(), deleteBy)
171+
172+
val lastInteraction = transacter.transaction { session ->
173+
queryFactory.newQuery<EventLogQuery>()
174+
.backfillRunId(backfill.id as Id<DbBackfillRun>)
175+
.list(session)
176+
.maxByOrNull { it.created_at }
177+
}
178+
179+
return """
180+
|${decision.emoji} *Backfill Deletion Notice*
181+
|Backfill `${backfill.name}` is scheduled for deletion on ${deleteBy}.
182+
|
183+
|• Days until deletion: $daysUntilDeletion
184+
|• Last activity: ${lastInteraction?.created_at ?: "Never"}
185+
|${if (lastInteraction?.user != null) "• Last touched by: ${lastInteraction.user}" else ""}
186+
|
187+
|To extend this backfill's lifetime, please:
188+
|1. Review if this backfill is still needed
189+
|2. Update the `@DeleteBy` annotation with a new date if needed
190+
|
191+
|_Note: Any activity on this backfill will reset the notification schedule._
192+
""".trimMargin()
193+
}
194+
195+
companion object {
196+
private val logger = getLogger<DeleteByNotificationHelper>()
197+
}
198+
}

‎service/src/main/kotlin/app/cash/backfila/service/deletion/DeleteByNotificationModule.kt‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
package app.cash.backfila.service.deletion
2+
3+
import com.google.inject.Provides
4+
import java.time.Clock
5+
import javax.inject.Singleton
6+
import misk.inject.KAbstractModule
7+
8+
class DeleteByNotificationModule : KAbstractModule() {
9+
override fun configure() {
10+
bind<DeleteByNotificationService>().asEagerSingleton()
11+
bind<DeleteByNotificationHelper>().asEagerSingleton()
12+
}
13+
14+
@Provides @Singleton
15+
fun provideClock(): Clock = Clock.systemUTC()
16+
}

‎service/src/main/kotlin/app/cash/backfila/service/deletion/DeleteByNotificationService.kt‎

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
package app.cash.backfila.service.deletion
2+
3+
import app.cash.backfila.service.persistence.DbRegisteredBackfill
4+
import com.google.common.util.concurrent.AbstractExecutionThreadService
5+
import java.time.Clock
6+
import java.time.DayOfWeek
7+
import java.time.ZoneId
8+
import java.util.Random
9+
import javax.inject.Inject
10+
import javax.inject.Singleton
11+
import wisp.logging.getLogger
12+
13+
/**
14+
* Service that periodically checks for backfills approaching their delete_by date
15+
* and sends notifications according to the configured schedule.
16+
*/
17+
@Singleton
18+
class DeleteByNotificationService @Inject constructor(
19+
private val notificationHelper: DeleteByNotificationHelper,
20+
private val clock: Clock
21+
) : AbstractExecutionThreadService() {
22+
@Volatile private var running = false
23+
private val random = Random()
24+
25+
override fun startUp() {
26+
running = true
27+
logger.info { "Starting DeleteByNotificationService" }
28+
}
29+
30+
override fun run() {
31+
while (running) {
32+
try {
33+
checkBackfills()
34+
} catch (e: Exception) {
35+
logger.error(e) { "Error checking backfills for deletion notifications" }
36+
}
37+
38+
// Sleep for an hour plus random jitter to avoid clustering
39+
Thread.sleep(HOUR_IN_MILLIS + random.nextInt(JITTER_RANGE_MILLIS))
40+
}
41+
}
42+
43+
override fun triggerShutdown() {
44+
running = false
45+
}
46+
47+
private fun checkBackfills() {
48+
notificationHelper.getBackfillsNeedingNotification().forEach { backfill ->
49+
val decision = notificationHelper.evaluateBackfill(backfill)
50+
51+
if (decision != NotificationDecision.NONE) {
52+
val channel = notificationHelper.determineNotificationChannel(backfill)
53+
54+
if (isBusinessHours(backfill)) {
55+
try {
56+
notificationHelper.sendNotification(
57+
backfill = backfill,
58+
decision = decision,
59+
channel = channel
60+
)
61+
logger.info { "Sent deletion notification for backfill: ${backfill.name}" }
62+
} catch (e: Exception) {
63+
logger.error(e) { "Failed to send notification for backfill: ${backfill.name}" }
64+
}
65+
} else {
66+
logger.info { "Skipping notification for ${backfill.name} outside business hours" }
67+
}
68+
}
69+
}
70+
}
71+
72+
private fun isBusinessHours(backfill: DbRegisteredBackfill): Boolean {
73+
val timeZone = ZoneId.of("America/Los_Angeles")
74+
75+
val localTime = clock.instant().atZone(timeZone)
76+
val hour = localTime.hour
77+
78+
return hour in 9..17 && // 9 AM to 5 PM
79+
localTime.dayOfWeek !in listOf(DayOfWeek.SATURDAY, DayOfWeek.SUNDAY)
80+
}
81+
82+
companion object {
83+
private val logger = getLogger<DeleteByNotificationService>()
84+
private const val HOUR_IN_MILLIS = 3_600_000L // 1 hour
85+
private const val JITTER_RANGE_MILLIS = 300_000 // 5 minutes
86+
}
87+
}

0 commit comments

Comments
 (0)