storage quota size check fix: (#2966)

- add get_active_crawls_size() from db to check size of current crawls,
instead of relying on removed related object
- fixes regression caused by #2945
- bump version to 1.19.7

Author: ikreymer

backend/btrixcloud/crawls.py

@@ -354,6 +354,20 @@ async def get_active_crawls(self, oid: UUID, limit: int) -> list[str]:
         res_list = await res.to_list()
         return [res["_id"] for res in res_list]
 
+    async def get_active_crawls_size(self, oid: UUID) -> int:
+        """get size of all active (running, waiting, paused) crawls"""
+        cursor = self.crawls.aggregate(
+            [
+                {"$match": {"state": {"$in": RUNNING_AND_WAITING_STATES}, "oid": oid}},
+                {"$group": {"_id": None, "totalSum": {"$sum": "$stats.size"}}},
+            ]
+        )
+        results = await cursor.to_list(length=1)
+        if not results:
+            return 0
+
+        return results[0].get("totalSum") or 0
+
     async def delete_crawls(
         self,
         org: Organization,

backend/btrixcloud/operator/crawls.py

@@ -51,7 +51,6 @@
     POD,
     CMAP,
     PVC,
-    CJS,
 )
 
 
@@ -1031,7 +1030,7 @@ async def sync_crawl_state(
 
             # update stats and get status
             return await self.update_crawl_state(
-                redis, crawl, status, pods, pod_done_count, data
+                redis, crawl, status, pods, pod_done_count
             )
 
         # pylint: disable=broad-except
@@ -1397,7 +1396,7 @@ async def add_file_to_crawl(self, cc_data, crawl: CrawlSpec, redis):
         return True
 
     async def is_crawl_stopping(
-        self, crawl: CrawlSpec, status: CrawlStatus, data: MCSyncData
+        self, crawl: CrawlSpec, status: CrawlStatus
     ) -> Optional[StopReason]:
         """check if crawl is stopping and set reason"""
         # if user requested stop, then enter stopping phase
@@ -1428,19 +1427,11 @@ async def is_crawl_stopping(
             return "stopped_org_readonly"
 
         if org.quotas.storageQuota:
-            running_crawls_total_size = status.size
-            for crawl_job in data.related[CJS].values():
-                # if the job id matches current crawl job, then skip
-                # this job to avoid double-counting
-                # using the more up-to-date 'status.size' for this job
-                if crawl_job.get("spec", {}).get("id") == crawl.id:
-                    continue
-
-                crawl_status = crawl_job.get("status", {})
-                if crawl_status:
-                    running_crawls_total_size += crawl_status.get("size", 0)
+            active_crawls_total_size = await self.crawl_ops.get_active_crawls_size(
+                crawl.oid
+            )
 
-            if self.org_ops.storage_quota_reached(org, running_crawls_total_size):
+            if self.org_ops.storage_quota_reached(org, active_crawls_total_size):
                 return "stopped_storage_quota_reached"
 
         # gracefully stop crawl is execution time quota is reached
@@ -1479,7 +1470,6 @@ async def update_crawl_state(
         status: CrawlStatus,
         pods: dict[str, dict],
         pod_done_count: int,
-        data: MCSyncData,
     ) -> CrawlStatus:
         """update crawl state and check if crawl is now done"""
         results = await redis.hgetall(f"{crawl.id}:status")
@@ -1536,7 +1526,7 @@ async def update_crawl_state(
             await redis.delete(f"{crawl.id}:paused")
 
         if not status.stopping:
-            status.stopReason = await self.is_crawl_stopping(crawl, status, data)
+            status.stopReason = await self.is_crawl_stopping(crawl, status)
             status.stopping = status.stopReason is not None
 
             # mark crawl as stopping

backend/btrixcloud/version.py

@@ -1,3 +1,3 @@
 """current version"""
 
-__version__ = "1.19.6"
+__version__ = "1.19.7"

chart/Chart.yaml

@@ -5,7 +5,7 @@ type: application
 icon: https://webrecorder.net/assets/icon.png
 
 # Browsertrix and Chart Version
-version: v1.19.6
+version: v1.19.7
 
 dependencies:
   - name: btrix-admin-logging

chart/values.yaml

@@ -106,7 +106,7 @@ replica_deletion_delay_days: 0
 
 # API Image
 # =========================================
-backend_image: "docker.io/webrecorder/browsertrix-backend:1.19.6"
+backend_image: "docker.io/webrecorder/browsertrix-backend:1.19.7"
 backend_pull_policy: "IfNotPresent"
 
 backend_password_secret: "PASSWORD!"
@@ -171,7 +171,7 @@ cleanup_files_after_minutes: 1440
 
 # Emails Image
 # =========================================
-emails_image: "docker.io/webrecorder/browsertrix-emails:1.19.6"
+emails_image: "docker.io/webrecorder/browsertrix-emails:1.19.7"
 emails_pull_policy: "IfNotPresent"
 
 emails_cpu: "10m"
@@ -185,7 +185,7 @@ local_emails_port: 30872
 
 # Nginx Image
 # =========================================
-frontend_image: "docker.io/webrecorder/browsertrix-frontend:1.19.6"
+frontend_image: "docker.io/webrecorder/browsertrix-frontend:1.19.7"
 frontend_pull_policy: "IfNotPresent"
 
 frontend_cpu: "10m"

frontend/package.json

@@ -1,6 +1,6 @@
 {
   "name": "browsertrix-frontend",
-  "version": "1.19.6",
+  "version": "1.19.7",
   "main": "index.ts",
   "license": "AGPL-3.0-or-later",
   "dependencies": {

version.txt

@@ -1 +1 @@
-1.19.6
+1.19.7