Include tag counts in tag filter & tag input autocomplete (#2711)

Author: emma-sg

backend/btrixcloud/crawlconfigs.py

@@ -25,6 +25,7 @@
     ConfigRevision,
     CrawlConfig,
     CrawlConfigOut,
+    CrawlConfigTags,
     CrawlOut,
     UpdateCrawlConfig,
     Organization,
@@ -976,8 +977,20 @@ async def remove_collection_from_all_configs(
 
     async def get_crawl_config_tags(self, org):
         """get distinct tags from all crawl configs for this org"""
-        tags = await self.crawl_configs.distinct("tags", {"oid": org.id})
-        return list(tags)
+        return await self.crawl_configs.distinct("tags", {"oid": org.id})
+
+    async def get_crawl_config_tag_counts(self, org):
+        """get distinct tags from all crawl configs for this org"""
+        tags = await self.crawl_configs.aggregate(
+            [
+                {"$match": {"oid": org.id}},
+                {"$unwind": "$tags"},
+                {"$group": {"_id": "$tags", "count": {"$sum": 1}}},
+                {"$project": {"tag": "$_id", "count": "$count", "_id": 0}},
+                {"$sort": {"count": -1, "tag": 1}},
+            ]
+        ).to_list()
+        return tags
 
     async def get_crawl_config_search_values(self, org):
         """List unique names, first seeds, and descriptions from all workflows in org"""
@@ -1399,10 +1412,17 @@ async def get_crawl_configs(
         )
         return paginated_format(crawl_configs, total, page, pageSize)
 
-    @router.get("/tags", response_model=List[str])
+    @router.get("/tags", response_model=List[str], deprecated=True)
     async def get_crawl_config_tags(org: Organization = Depends(org_viewer_dep)):
+        """
+        Deprecated - prefer /api/orgs/{oid}/crawlconfigs/tagCounts instead.
+        """
         return await ops.get_crawl_config_tags(org)
 
+    @router.get("/tagCounts", response_model=CrawlConfigTags)
+    async def get_crawl_config_tag_counts(org: Organization = Depends(org_viewer_dep)):
+        return {"tags": await ops.get_crawl_config_tag_counts(org)}
+
     @router.get("/search-values", response_model=CrawlConfigSearchValues)
     async def get_crawl_config_search_values(
         org: Organization = Depends(org_viewer_dep),

backend/btrixcloud/models.py

@@ -577,11 +577,19 @@ class CrawlConfigAddedResponse(BaseModel):
     execMinutesQuotaReached: bool
 
 
+# ============================================================================
+class CrawlConfigTagCount(BaseModel):
+    """Response model for crawlconfig tag count"""
+
+    tag: str
+    count: int
+
+
 # ============================================================================
 class CrawlConfigTags(BaseModel):
     """Response model for crawlconfig tags"""
 
-    tags: List[str]
+    tags: List[CrawlConfigTagCount]
 
 
 # ============================================================================

backend/test/test_crawl_config_tags.py

@@ -50,6 +50,22 @@ def test_get_config_by_tag_1(admin_auth_headers, default_org_id):
     assert sorted(data) == ["tag-1", "tag-2", "wr-test-1", "wr-test-2"]
 
 
+def test_get_config_by_tag_counts_1(admin_auth_headers, default_org_id):
+    r = requests.get(
+        f"{API_PREFIX}/orgs/{default_org_id}/crawlconfigs/tagCounts",
+        headers=admin_auth_headers,
+    )
+    data = r.json()
+    assert data == {
+        "tags": [
+            {"tag": "wr-test-2", "count": 2},
+            {"tag": "tag-1", "count": 1},
+            {"tag": "tag-2", "count": 1},
+            {"tag": "wr-test-1", "count": 1},
+        ]
+    }
+
+
 def test_create_new_config_2(admin_auth_headers, default_org_id):
     r = requests.post(
         f"{API_PREFIX}/orgs/{default_org_id}/crawlconfigs/",
@@ -84,6 +100,24 @@ def test_get_config_by_tag_2(admin_auth_headers, default_org_id):
     ]
 
 
+def test_get_config_by_tag_counts_2(admin_auth_headers, default_org_id):
+    r = requests.get(
+        f"{API_PREFIX}/orgs/{default_org_id}/crawlconfigs/tagCounts",
+        headers=admin_auth_headers,
+    )
+    data = r.json()
+    assert data == {
+        "tags": [
+            {"tag": "wr-test-2", "count": 2},
+            {"tag": "tag-0", "count": 1},
+            {"tag": "tag-1", "count": 1},
+            {"tag": "tag-2", "count": 1},
+            {"tag": "tag-3", "count": 1},
+            {"tag": "wr-test-1", "count": 1},
+        ]
+    }
+
+
 def test_get_config_2(admin_auth_headers, default_org_id):
     r = requests.get(
         f"{API_PREFIX}/orgs/{default_org_id}/crawlconfigs/{new_cid_2}",

frontend/src/components/ui/badge.ts

@@ -11,7 +11,7 @@ export type BadgeVariant =
   | "danger"
   | "neutral"
   | "primary"
-  | "blue"
+  | "cyan"
   | "high-contrast";
 
 /**
@@ -27,6 +27,12 @@ export class Badge extends TailwindElement {
   @property({ type: String })
   variant: BadgeVariant = "neutral";
 
+  @property({ type: Boolean })
+  outline = false;
+
+  @property({ type: Boolean })
+  pill = false;
+
   @property({ type: String, reflect: true })
   role: string | null = "status";
 
@@ -40,16 +46,32 @@ export class Badge extends TailwindElement {
     return html`
       <span
         class=${clsx(
-          tw`h-4.5 inline-flex items-center justify-center rounded-sm px-2 align-[1px] text-xs`,
-          {
-            success: tw`bg-success-500 text-neutral-0`,
-            warning: tw`bg-warning-600 text-neutral-0`,
-            danger: tw`bg-danger-500 text-neutral-0`,
-            neutral: tw`bg-neutral-100 text-neutral-600`,
-            "high-contrast": tw`bg-neutral-600 text-neutral-0`,
-            primary: tw`bg-primary text-neutral-0`,
-            blue: tw`bg-cyan-50 text-cyan-600`,
-          }[this.variant],
+          tw`inline-flex h-[1.125rem] items-center justify-center align-[1px] text-xs`,
+          this.outline
+            ? [
+                tw`ring-1`,
+                {
+                  success: tw`bg-success-500 text-success-500 ring-success-500`,
+                  warning: tw`bg-warning-600 text-warning-600 ring-warning-600`,
+                  danger: tw`bg-danger-500 text-danger-500 ring-danger-500`,
+                  neutral: tw`g-neutral-100 text-neutral-600 ring-neutral-600`,
+                  "high-contrast": tw`bg-neutral-600 text-neutral-0 ring-neutral-0`,
+                  primary: tw`bg-white text-primary ring-primary`,
+                  cyan: tw`bg-cyan-50 text-cyan-600 ring-cyan-600`,
+                  blue: tw`bg-blue-50 text-blue-600 ring-blue-600`,
+                }[this.variant],
+              ]
+            : {
+                success: tw`bg-success-500 text-neutral-0`,
+                warning: tw`bg-warning-600 text-neutral-0`,
+                danger: tw`bg-danger-500 text-neutral-0`,
+                neutral: tw`bg-neutral-100 text-neutral-600`,
+                "high-contrast": tw`bg-neutral-600 text-neutral-0`,
+                primary: tw`bg-primary text-neutral-0`,
+                cyan: tw`bg-cyan-50 text-cyan-600`,
+                blue: tw`bg-blue-50 text-blue-600`,
+              }[this.variant],
+          this.pill ? tw`min-w-[1.125rem] rounded-full px-1` : tw`rounded px-2`,
         )}
         part="base"
       >

frontend/src/components/ui/tag-input.ts

@@ -17,6 +17,7 @@ import { customElement, property, query, state } from "lit/decorators.js";
 import debounce from "lodash/fp/debounce";
 
 import type { UnderlyingFunction } from "@/types/utils";
+import { type WorkflowTag } from "@/types/workflow";
 import { dropdown } from "@/utils/css";
 
 export type Tags = string[];
@@ -80,7 +81,7 @@ export class TagInput extends LitElement {
       }
 
       sl-popup::part(popup) {
-        z-index: 3;
+        z-index: 5;
       }
 
       .shake {
@@ -116,7 +117,7 @@ export class TagInput extends LitElement {
   initialTags?: Tags;
 
   @property({ type: Array })
-  tagOptions: Tags = [];
+  tagOptions: WorkflowTag[] = [];
 
   @property({ type: Boolean })
   disabled = false;
@@ -224,6 +225,7 @@ export class TagInput extends LitElement {
               @paste=${this.onPaste}
               ?required=${this.required && !this.tags.length}
               placeholder=${placeholder}
+              autocomplete="off"
               role="combobox"
               aria-controls="dropdown"
               aria-expanded="${this.dropdownIsOpen === true}"
@@ -258,10 +260,14 @@ export class TagInput extends LitElement {
               >
                 ${this.tagOptions
                   .slice(0, 3)
+                  .filter(({ tag }) => !this.tags.includes(tag))
                   .map(
-                    (tag) => html`
+                    ({ tag, count }) => html`
                       <sl-menu-item role="option" value=${tag}
-                        >${tag}</sl-menu-item
+                        >${tag}
+                        <btrix-badge pill variant="cyan" slot="suffix"
+                          >${count}</btrix-badge
+                        ></sl-menu-item
                       >
                     `,
                   )}

frontend/src/features/archived-items/file-uploader.ts

@@ -17,6 +17,7 @@ import type {
   TagsChangeEvent,
 } from "@/components/ui/tag-input";
 import { type CollectionsChangeEvent } from "@/features/collections/collections-add";
+import { type WorkflowTag, type WorkflowTags } from "@/types/workflow";
 import { APIError } from "@/utils/api";
 import { maxLengthValidator } from "@/utils/form";
 
@@ -70,7 +71,7 @@ export class FileUploader extends BtrixElement {
   private collectionIds: string[] = [];
 
   @state()
-  private tagOptions: Tags = [];
+  private tagOptions: WorkflowTag[] = [];
 
   @state()
   private tagsToSave: Tags = [];
@@ -85,7 +86,8 @@ export class FileUploader extends BtrixElement {
   private readonly form!: Promise<HTMLFormElement>;
 
   // For fuzzy search:
-  private readonly fuse = new Fuse([], {
+  private readonly fuse = new Fuse<WorkflowTag>([], {
+    keys: ["tag"],
     shouldSort: false,
     threshold: 0.2, // stricter; default is 0.6
   });
@@ -361,8 +363,8 @@ export class FileUploader extends BtrixElement {
 
   private async fetchTags() {
     try {
-      const tags = await this.api.fetch<never>(
-        `/orgs/${this.orgId}/crawlconfigs/tags`,
+      const { tags } = await this.api.fetch<WorkflowTags>(
+        `/orgs/${this.orgId}/crawlconfigs/tagCounts`,
       );
 
       // Update search/filter collection

frontend/src/features/archived-items/item-metadata-editor.ts

@@ -10,6 +10,7 @@ import type {
 } from "@/components/ui/tag-input";
 import { type CollectionsChangeEvent } from "@/features/collections/collections-add";
 import type { ArchivedItem } from "@/types/crawler";
+import { type WorkflowTag, type WorkflowTags } from "@/types/workflow";
 import { maxLengthValidator } from "@/utils/form";
 import LiteElement, { html } from "@/utils/LiteElement";
 
@@ -46,7 +47,7 @@ export class CrawlMetadataEditor extends LiteElement {
   private includeName = false;
 
   @state()
-  private tagOptions: Tags = [];
+  private tagOptions: WorkflowTag[] = [];
 
   @state()
   private tagsToSave: Tags = [];
@@ -55,7 +56,8 @@ export class CrawlMetadataEditor extends LiteElement {
   private collectionsToSave: string[] = [];
 
   // For fuzzy search:
-  private readonly fuse = new Fuse<string>([], {
+  private readonly fuse = new Fuse<WorkflowTag>([], {
+    keys: ["tag"],
     shouldSort: false,
     threshold: 0.2, // stricter; default is 0.6
   });
@@ -164,8 +166,8 @@ export class CrawlMetadataEditor extends LiteElement {
   private async fetchTags() {
     if (!this.crawl) return;
     try {
-      const tags = await this.apiFetch<string[]>(
-        `/orgs/${this.crawl.oid}/crawlconfigs/tags`,
+      const { tags } = await this.apiFetch<WorkflowTags>(
+        `/orgs/${this.crawl.oid}/crawlconfigs/tagCounts`,
       );
 
       // Update search/filter collection

frontend/src/features/crawl-workflows/workflow-editor.ts

@@ -88,7 +88,11 @@ import {
   type WorkflowParams,
 } from "@/types/crawler";
 import type { UnderlyingFunction } from "@/types/utils";
-import { NewWorkflowOnlyScopeType } from "@/types/workflow";
+import {
+  NewWorkflowOnlyScopeType,
+  type WorkflowTag,
+  type WorkflowTags,
+} from "@/types/workflow";
 import { track } from "@/utils/analytics";
 import { isApiError, isApiErrorDetail } from "@/utils/api";
 import { DEPTH_SUPPORTED_SCOPES, isPageScopeType } from "@/utils/crawler";
@@ -258,7 +262,7 @@ export class WorkflowEditor extends BtrixElement {
   private showCrawlerChannels = false;
 
   @state()
-  private tagOptions: string[] = [];
+  private tagOptions: WorkflowTag[] = [];
 
   @state()
   private isSubmitting = false;
@@ -293,7 +297,8 @@ export class WorkflowEditor extends BtrixElement {
   });
 
   // For fuzzy search:
-  private readonly fuse = new Fuse<string>([], {
+  private readonly fuse = new Fuse<WorkflowTag>([], {
+    keys: ["tag"],
     shouldSort: false,
     threshold: 0.2, // stricter; default is 0.6
   });
@@ -2532,8 +2537,8 @@ https://archiveweb.page/images/${"logo.svg"}`}
   private async fetchTags() {
     this.tagOptions = [];
     try {
-      const tags = await this.api.fetch<string[]>(
-        `/orgs/${this.orgId}/crawlconfigs/tags`,
+      const { tags } = await this.api.fetch<WorkflowTags>(
+        `/orgs/${this.orgId}/crawlconfigs/tagCounts`,
       );
 
       // Update search/filter collection