compute top page origins for each collection:

- compute as part of existing collection stats compute
- store top 10 results in collection
- display in collection About sidebar

Author: ikreymer

backend/btrixcloud/colls.py

@@ -705,6 +705,8 @@ async def update_collection_counts_and_tags(self, collection_id: UUID):
 
         unique_page_count = await self.page_ops.get_unique_page_count(crawl_ids)
 
+        top_page_origins = await self.page_ops.get_top_page_origins(crawl_ids)
+
         await self.collections.find_one_and_update(
             {"_id": collection_id},
             {
@@ -715,6 +717,7 @@ async def update_collection_counts_and_tags(self, collection_id: UUID):
                     "totalSize": total_size,
                     "tags": sorted_tags,
                     "preloadResources": preload_resources,
+                    "topPageOrigins": top_page_origins,
                 }
             },
         )

backend/btrixcloud/models.py

@@ -1415,6 +1415,14 @@ class PreloadResource(BaseModel):
     crawlId: str
 
 
+# ============================================================================
+class OriginCount(BaseModel):
+    """Origin Count"""
+
+    origin: str
+    count: int
+
+
 # ============================================================================
 class Collection(BaseMongoModel):
     """Org collection structure"""
@@ -1513,6 +1521,8 @@ class CollOut(BaseMongoModel):
     pagesQueryUrl: str = ""
     downloadUrl: Optional[str] = None
 
+    topPageOrigins: List[OriginCount] = []
+
 
 # ============================================================================
 class PublicCollOut(BaseMongoModel):

backend/btrixcloud/pages.py

@@ -923,6 +923,30 @@ async def get_unique_page_count(self, crawl_ids: List[str]) -> int:
         res = await cursor.to_list(1)
         return res[0].get("urls") if res else 0
 
+    async def get_top_page_origins(
+        self, crawl_ids: List[str]
+    ) -> List[dict[str, str | int]]:
+        """Get count of top page origins across all archived items"""
+        cursor = self.pages.aggregate(
+            [
+                {"$match": {"crawl_id": {"$in": crawl_ids}}},
+                {
+                    "$addFields": {
+                        "origin": {
+                            "$regexFind": {
+                                "input": "$url",
+                                "regex": "^https?://([^/])+",
+                            }
+                        }
+                    }
+                },
+                {"$group": {"_id": "$origin.match", "count": {"$count": {}}}},
+                {"$sort": {"count": -1}},
+            ]
+        )
+        res = await cursor.to_list(10)
+        return [{"origin": x.get("_id"), "count": x.get("count")} for x in res]
+
     async def set_archived_item_page_counts(self, crawl_id: str):
         """Store archived item page and unique page counts in crawl document"""
         page_count = await self.pages.count_documents({"crawl_id": crawl_id})

frontend/src/layouts/collections/metadataColumn.ts

@@ -56,6 +56,20 @@ export function metadataColumn(collection?: Collection | PublicCollection) {
         label: metadata.totalSize,
         render: (col) => `${localize.bytes(col.totalSize)}`,
       })}
+      ${metadataItem({
+        label: metadata.topPageDomains,
+        render: (col) =>
+          html` <table>
+            ${col.topPageOrigins.map(
+              (x) => html`
+                <tr>
+                  <td>${x.origin}</td>
+                  <td class="pl-4">${x.count}</td>
+                </tr>
+              `,
+            )}
+          </table>`,
+      })}
     </btrix-desc-list>
   `;
 }

frontend/src/strings/collections/metadata.ts

@@ -5,4 +5,5 @@ export const metadata = {
   uniquePageCount: msg("Unique Pages in Collection"),
   pageCount: msg("Total Pages Crawled"),
   totalSize: msg("Collection Size"),
+  topPageDomains: msg("Top Page Domains"),
 };

frontend/src/types/collection.ts

@@ -41,6 +41,12 @@ export const publicCollectionSchema = z.object({
   crawlCount: z.number(),
   uniquePageCount: z.number(),
   pageCount: z.number(),
+  topPageOrigins: z.array(
+    z.object({
+      origin: z.string(),
+      count: z.number(),
+    }),
+  ),
   totalSize: z.number(),
   allowPublicDownload: z.boolean(),
   homeUrl: z.string().url().nullable(),