Refactor and clean up buildinfo, add heartbeat (#8892)

- BuildInfo field datastoreApiVersion was unused and out of date for a
long time, removed now
- Refactored BuildInfo out of the controller into separate service
- Add heartbeat event

------
- [x] Removed dev-only changes like prints and application.conf edits
- [x] Considered [common edge
cases](../blob/master/.github/common_edge_cases.md)
- [x] Needs datastore update after deployment

Author: fm3

app/WebknossosModule.scala

@@ -3,8 +3,8 @@ import com.scalableminds.webknossos.datastore.storage.DataVaultService
 import controllers.{Application, InitialDataService}
 import files.WkTempFileService
 import mail.MailchimpTicker
-import models.analytics.AnalyticsSessionService
-import models.annotation.{AnnotationMutexService, AnnotationStore, AnnotationDataSourceTemporaryStore}
+import models.analytics.{AnalyticsService, AnalyticsSessionService}
+import models.annotation.{AnnotationDataSourceTemporaryStore, AnnotationMutexService, AnnotationStore}
 import models.dataset.{DatasetService, ThumbnailCachingService}
 import models.job.{JobService, WorkerLivenessService}
 import models.organization.FreeCreditTransactionService
@@ -43,5 +43,6 @@ class WebknossosModule extends AbstractModule {
     bind(classOf[AnnotationDataSourceTemporaryStore]).asEagerSingleton()
     bind(classOf[CertificateValidationService]).asEagerSingleton()
     bind(classOf[FreeCreditTransactionService]).asEagerSingleton()
+    bind(classOf[AnalyticsService]).asEagerSingleton()
   }
 }

app/controllers/Application.scala

@@ -1,6 +1,5 @@
 package controllers
 
-import com.scalableminds.util.mvc.ApiVersioning
 import com.scalableminds.util.tools.{Fox, FoxImplicits}
 import com.typesafe.config.ConfigRenderOptions
 import mail.{DefaultMails, Send}
@@ -12,43 +11,29 @@ import play.api.mvc.{Action, AnyContent, Result}
 import play.silhouette.api.Silhouette
 import security.{CertificateValidationService, WkEnv}
 import utils.sql.{SimpleSQLDAO, SqlClient}
-import utils.{StoreModules, WkConf}
+import utils.{BuildInfoService, WkConf}
 
 import javax.inject.Inject
 import scala.concurrent.ExecutionContext
 
 class Application @Inject()(actorSystem: ActorSystem,
                             userService: UserService,
-                            releaseInformationDAO: ReleaseInformationDAO,
+                            buildInfoService: BuildInfoService,
                             organizationDAO: OrganizationDAO,
                             conf: WkConf,
                             defaultMails: DefaultMails,
-                            storeModules: StoreModules,
                             sil: Silhouette[WkEnv],
                             certificateValidationService: CertificateValidationService)(implicit ec: ExecutionContext)
-    extends Controller
-    with ApiVersioning {
+    extends Controller {
 
   private lazy val Mailer =
     actorSystem.actorSelection("/user/mailActor")
 
   // Note: This route is used by external applications, keep stable
   def buildInfo: Action[AnyContent] = sil.UserAwareAction.async {
     for {
-      schemaVersion <- releaseInformationDAO.getSchemaVersion.futureBox
-    } yield {
-      addRemoteOriginHeaders(
-        Ok(
-          Json.obj(
-            "webknossos" -> Json.toJson(
-              webknossos.BuildInfo.toMap.view.mapValues(_.toString).filterKeys(_ != "certificatePublicKey").toMap),
-            "schemaVersion" -> schemaVersion.toOption,
-            "httpApiVersioning" -> apiVersioningInfo,
-            "localDataStoreEnabled" -> storeModules.localDataStoreEnabled,
-            "localTracingStoreEnabled" -> storeModules.localTracingStoreEnabled
-          ))
-      )
-    }
+      buildInfoJson <- buildInfoService.buildInfoJson
+    } yield addRemoteOriginHeaders(Ok(buildInfoJson))
   }
 
   // This only changes on server restart, so we can cache the full result.

app/models/analytics/AnalyticsEvent.scala

@@ -230,3 +230,15 @@ case class FrontendAnalyticsEvent(user: User, eventType: String, eventProperties
   override def eventProperties(analyticsLookUpService: AnalyticsLookUpService): Fox[JsObject] =
     Fox.successful(eventProperties ++ Json.obj("is_frontend_event" -> true))
 }
+
+case class WebknossosHeartbeatAnalyticsEvent(user: User, buildInfoJson: JsObject)(implicit ec: ExecutionContext)
+    extends AnalyticsEvent {
+  def eventType: String = "webknossos_heartbeat"
+  def eventProperties(analyticsLookUpService: AnalyticsLookUpService): Fox[JsObject] =
+    Fox.successful(
+      Json.obj(
+        "build_info" -> buildInfoJson,
+      )
+    )
+
+}

app/models/analytics/AnalyticsService.scala

@@ -8,9 +8,12 @@ import com.scalableminds.webknossos.datastore.rpc.RPC
 import com.typesafe.scalalogging.LazyLogging
 import models.user.{MultiUserDAO, UserDAO}
 import com.scalableminds.util.tools.Box.tryo
+import com.scalableminds.webknossos.datastore.helpers.IntervalScheduler
+import org.apache.pekko.actor.ActorSystem
 import play.api.http.Status.UNAUTHORIZED
+import play.api.inject.ApplicationLifecycle
 import play.api.libs.json._
-import utils.WkConf
+import utils.{BuildInfoService, WkConf}
 
 import javax.inject.Inject
 import scala.concurrent.ExecutionContext
@@ -20,8 +23,13 @@ class AnalyticsService @Inject()(rpc: RPC,
                                  wkConf: WkConf,
                                  analyticsLookUpService: AnalyticsLookUpService,
                                  analyticsSessionService: AnalyticsSessionService,
-                                 analyticsDAO: AnalyticsDAO)(implicit ec: ExecutionContext)
+                                 analyticsDAO: AnalyticsDAO,
+                                 userDAO: UserDAO,
+                                 buildInfoService: BuildInfoService,
+                                 val actorSystem: ActorSystem,
+                                 val lifecycle: ApplicationLifecycle)(implicit val ec: ExecutionContext)
     extends LazyLogging
+    with IntervalScheduler
     with FoxImplicits {
 
   private lazy val conf = wkConf.BackendAnalytics
@@ -75,6 +83,17 @@ class AnalyticsService @Inject()(rpc: RPC,
     }
     Fox.successful(())
   }
+
+  override protected def tickerInterval: FiniteDuration = 24 hours
+
+  override protected def tick(): Fox[_] =
+    for {
+      oldestUser <- userDAO.findOldestActive
+      buildInfoJson <- Fox.fromFuture(buildInfoService.buildInfoJson)
+      event = WebknossosHeartbeatAnalyticsEvent(oldestUser, buildInfoJson)
+      _ = track(event)
+    } yield ()
+
 }
 
 class AnalyticsLookUpService @Inject()(userDAO: UserDAO, multiUserDAO: MultiUserDAO, wkConf: WkConf)

app/models/user/User.scala

@@ -168,6 +168,13 @@ class UserDAO @Inject()(sqlClient: SqlClient)(implicit ec: ExecutionContext)
   def findIdsByMultiUserId(multiUserId: ObjectId): Fox[Seq[ObjectId]] =
     run(q"SELECT _id FROM $existingCollectionName WHERE _multiUser = $multiUserId".as[ObjectId])
 
+  def findOldestActive: Fox[User] =
+    for {
+      r <- run(
+        q"SELECT $columns FROM $existingCollectionName WHERE NOT isDeactivated ORDER BY created LIMIT 1".as[UsersRow])
+      parsed <- parseFirst(r, "oldestActive")
+    } yield parsed
+
   def buildSelectionPredicates(isEditableOpt: Option[Boolean],
                                isTeamManagerOrAdminOpt: Option[Boolean],
                                isAdminOpt: Option[Boolean],

app/utils/BuildInfoService.scala

@@ -0,0 +1,25 @@
+package utils
+
+import com.scalableminds.util.mvc.ApiVersioning
+import controllers.ReleaseInformationDAO
+import play.api.libs.json.{JsObject, Json}
+
+import javax.inject.Inject
+import scala.concurrent.{ExecutionContext, Future}
+
+class BuildInfoService @Inject()(releaseInformationDAO: ReleaseInformationDAO, storeModules: StoreModules)
+    extends ApiVersioning {
+
+  def buildInfoJson(implicit ec: ExecutionContext): Future[JsObject] =
+    for {
+      schemaVersion <- releaseInformationDAO.getSchemaVersion.futureBox
+    } yield
+      Json.obj(
+        "webknossos" -> Json.toJson(
+          webknossos.BuildInfo.toMap.view.mapValues(_.toString).filterKeys(_ != "certificatePublicKey").toMap),
+        "schemaVersion" -> schemaVersion.toOption,
+        "httpApiVersioning" -> apiVersioningInfo,
+        "localDataStoreEnabled" -> storeModules.localDataStoreEnabled,
+        "localTracingStoreEnabled" -> storeModules.localTracingStoreEnabled
+      )
+}

project/BuildInfoSettings.scala

@@ -1,4 +1,5 @@
 import sbt.Keys.{name, sbtVersion, scalaVersion}
+import sbtbuildinfo.BuildInfoOption
 import sbtbuildinfo.BuildInfoPlugin.autoImport.*
 
 import scala.language.postfixOps
@@ -34,7 +35,6 @@ object BuildInfoSettings {
       "ciBuild" -> ciBuild,
       "ciTag" -> ciTag,
       "version" -> webknossosVersion,
-      "datastoreApiVersion" -> "2.0",
       "certificatePublicKey" -> certificatePublicKey
     ),
     buildInfoPackage := "webknossos",
@@ -51,7 +51,6 @@ object BuildInfoSettings {
       "ciBuild" -> ciBuild,
       "ciTag" -> ciTag,
       "version" -> webknossosVersion,
-      "datastoreApiVersion" -> "2.0"
     ),
     buildInfoPackage := "webknossosDatastore",
     buildInfoOptions := Seq(BuildInfoOption.ToJson)