feat: Env var for community-supported connectors

.vscode/env_template.txt

@@ -65,3 +65,6 @@ S3_ENDPOINT_URL=http://localhost:9004
 S3_FILE_STORE_BUCKET_NAME=onyx-file-store-bucket
 S3_AWS_ACCESS_KEY_ID=minioadmin
 S3_AWS_SECRET_ACCESS_KEY=minioadmin
+
+# Show extra/uncommon connectors
+SHOW_EXTRA_CONNECTORS=False

backend/onyx/configs/app_configs.py

@@ -38,6 +38,9 @@
 # Controls whether users can use User Knowledge (personal documents) in assistants
 DISABLE_USER_KNOWLEDGE = os.environ.get("DISABLE_USER_KNOWLEDGE", "").lower() == "true"
 
+# If set to true, will show extra/uncommon connectors in the "Other" category
+SHOW_EXTRA_CONNECTORS = os.environ.get("SHOW_EXTRA_CONNECTORS", "").lower() == "true"
+
 # Controls whether to allow admin query history reports with:
 # 1. associated user emails
 # 2. anonymized user emails

backend/onyx/configs/constants.py

@@ -216,9 +216,6 @@ class BlobType(str, Enum):
     GOOGLE_CLOUD_STORAGE = "google_cloud_storage"
     OCI_STORAGE = "oci_storage"
 
-    # Special case, for internet search
-    NOT_APPLICABLE = "not_applicable"
-
 
 class DocumentIndexType(str, Enum):
     COMBINED = "combined"  # Vespa

backend/onyx/server/settings/models.py

@@ -62,6 +62,9 @@ class Settings(BaseModel):
     # User Knowledge settings
     user_knowledge_enabled: bool | None = True
 
+    # Connector settings
+    show_extra_connectors: bool | None = True
+
 
 class UserSettings(Settings):
     notifications: list[Notification]

backend/onyx/server/settings/store.py

@@ -1,5 +1,6 @@
 from onyx.configs.app_configs import DISABLE_USER_KNOWLEDGE
 from onyx.configs.app_configs import ONYX_QUERY_HISTORY_TYPE
+from onyx.configs.app_configs import SHOW_EXTRA_CONNECTORS
 from onyx.configs.constants import KV_SETTINGS_KEY
 from onyx.configs.constants import OnyxRedisLocks
 from onyx.key_value_store.factory import get_kv_store
@@ -53,6 +54,9 @@ def load_settings() -> Settings:
     if DISABLE_USER_KNOWLEDGE:
         settings.user_knowledge_enabled = False
 
+    # Override show extra connectors setting based on environment variable
+    settings.show_extra_connectors = SHOW_EXTRA_CONNECTORS
+
     return settings
 
 

deployment/docker_compose/docker-compose.dev.yml

@@ -233,6 +233,7 @@ services:
       - GITHUB_CONNECTOR_BASE_URL=${GITHUB_CONNECTOR_BASE_URL:-}
       - MAX_DOCUMENT_CHARS=${MAX_DOCUMENT_CHARS:-}
       - MAX_FILE_SIZE_BYTES=${MAX_FILE_SIZE_BYTES:-}
+      - SHOW_EXTRA_CONNECTORS=${SHOW_EXTRA_CONNECTORS:-}
       # Egnyte OAuth Configs
       - EGNYTE_CLIENT_ID=${EGNYTE_CLIENT_ID:-}
       - EGNYTE_CLIENT_SECRET=${EGNYTE_CLIENT_SECRET:-}

deployment/docker_compose/docker-compose.gpu-dev.yml

@@ -145,6 +145,7 @@ services:
       - GENERATIVE_MODEL_ACCESS_CHECK_FREQ=${GENERATIVE_MODEL_ACCESS_CHECK_FREQ:-}
       - DISABLE_LITELLM_STREAMING=${DISABLE_LITELLM_STREAMING:-}
       - LITELLM_EXTRA_HEADERS=${LITELLM_EXTRA_HEADERS:-}
+      - EXA_API_KEY=${EXA_API_KEY:-}
       # Query Options
       - DOC_TIME_DECAY=${DOC_TIME_DECAY:-} # Recency Bias for search results, decay at 1 / (1 + DOC_TIME_DECAY * x years)
       - HYBRID_ALPHA=${HYBRID_ALPHA:-} # Hybrid Search Alpha (0 for entirely keyword, 1 for entirely vector)
@@ -191,6 +192,7 @@ services:
       - GONG_CONNECTOR_START_TIME=${GONG_CONNECTOR_START_TIME:-}
       - NOTION_CONNECTOR_ENABLE_RECURSIVE_PAGE_LOOKUP=${NOTION_CONNECTOR_ENABLE_RECURSIVE_PAGE_LOOKUP:-}
       - GITHUB_CONNECTOR_BASE_URL=${GITHUB_CONNECTOR_BASE_URL:-}
+      - SHOW_EXTRA_CONNECTORS=${SHOW_EXTRA_CONNECTORS:-}
       # Onyx SlackBot Configs
       - DANSWER_BOT_DISABLE_DOCS_ONLY_ANSWER=${DANSWER_BOT_DISABLE_DOCS_ONLY_ANSWER:-}
       - DANSWER_BOT_FEEDBACK_VISIBILITY=${DANSWER_BOT_FEEDBACK_VISIBILITY:-}

deployment/docker_compose/docker-compose.multitenant-dev.yml

@@ -214,6 +214,7 @@ services:
       - GITHUB_CONNECTOR_BASE_URL=${GITHUB_CONNECTOR_BASE_URL:-}
       - MAX_DOCUMENT_CHARS=${MAX_DOCUMENT_CHARS:-}
       - MAX_FILE_SIZE_BYTES=${MAX_FILE_SIZE_BYTES:-}
+      - SHOW_EXTRA_CONNECTORS=${SHOW_EXTRA_CONNECTORS:-}
       # Egnyte OAuth Configs
       - EGNYTE_CLIENT_ID=${EGNYTE_CLIENT_ID:-}
       - EGNYTE_CLIENT_SECRET=${EGNYTE_CLIENT_SECRET:-}

deployment/docker_compose/env.prod.template

@@ -8,7 +8,7 @@ WEB_DOMAIN=http://localhost:3000
 
 
 # NOTE: Generative AI configurations are done via the UI now
-
+EXA_API_KEY=
 
 # The following are for configuring User Authentication, supported flows are:
 # disabled
@@ -65,3 +65,7 @@ DB_READONLY_PASSWORD=password
 # If setting the vespa language is required, set this ('en', 'de', etc.).
 # See: https://docs.vespa.ai/en/linguistics.html 
 #VESPA_LANGUAGE_OVERRIDE=
+
+# Uncommon connectors supported by the community
+# See https://docs.onyx.app for list of these connectors
+SHOW_EXTRA_CONNECTORS=False

web/src/app/admin/add-connector/page.tsx

@@ -7,7 +7,14 @@ import { listSourceMetadata } from "@/lib/sources";
 import Title from "@/components/ui/title";
 import { Button } from "@/components/ui/button";
 import Link from "next/link";
-import { useCallback, useEffect, useMemo, useRef, useState } from "react";
+import {
+  useCallback,
+  useContext,
+  useEffect,
+  useMemo,
+  useRef,
+  useState,
+} from "react";
 import {
   Tooltip,
   TooltipContent,
@@ -24,6 +31,7 @@ import useSWR from "swr";
 import { errorHandlingFetcher } from "@/lib/fetcher";
 import { buildSimilarCredentialInfoURL } from "@/app/admin/connector/[ccPairId]/lib";
 import { Credential } from "@/lib/connectors/credentials";
+import { SettingsContext } from "@/components/settings/SettingsProvider";
 
 function SourceTile({
   sourceMetadata,
@@ -148,6 +156,7 @@ export default function Page() {
   const sources = useMemo(() => listSourceMetadata(), []);
   const [searchTerm, setSearchTerm] = useState("");
   const { data: federatedConnectors } = useFederatedConnectors();
+  const settings = useContext(SettingsContext);
 
   // Fetch Slack credentials to determine navigation behavior
   const { data: slackCredentials } = useSWR<Credential<any>[]>(
@@ -177,7 +186,7 @@ export default function Page() {
 
   const categorizedSources = useMemo(() => {
     const filtered = filterSources(sources);
-    return Object.values(SourceCategory).reduce(
+    const categories = Object.values(SourceCategory).reduce(
       (acc, category) => {
         acc[category] = sources.filter(
           (source) =>
@@ -189,7 +198,25 @@ export default function Page() {
       },
       {} as Record<SourceCategory, SourceMetadata[]>
     );
-  }, [sources, filterSources, searchTerm]);
+
+    // Filter out the "Other" category if show_extra_connectors is false
+    if (settings?.settings?.show_extra_connectors === false) {
+      const filteredCategories = Object.entries(categories).filter(
+        ([category]) => category !== SourceCategory.Other
+      );
+      return Object.fromEntries(filteredCategories) as Record<
+        SourceCategory,
+        SourceMetadata[]
+      >;
+    }
+
+    return categories;
+  }, [
+    sources,
+    filterSources,
+    searchTerm,
+    settings?.settings?.show_extra_connectors,
+  ]);
 
   const handleKeyPress = (e: React.KeyboardEvent<HTMLInputElement>) => {
     if (e.key === "Enter") {
@@ -251,7 +278,6 @@ export default function Page() {
             <div className="flex mt-8">
               <Title>{category}</Title>
             </div>
-            <p>{getCategoryDescription(category as SourceCategory)}</p>
             <div className="flex flex-wrap gap-4 p-4">
               {sources.map((source, sourceInd) => (
                 <SourceTile
@@ -270,26 +296,3 @@ export default function Page() {
     </div>
   );
 }
-
-function getCategoryDescription(category: SourceCategory): string {
-  switch (category) {
-    case SourceCategory.Messaging:
-      return "Integrate with messaging and communication platforms.";
-    case SourceCategory.ProjectManagement:
-      return "Link to project management and task tracking tools.";
-    case SourceCategory.CustomerSupport:
-      return "Connect to customer support and helpdesk systems.";
-    case SourceCategory.CustomerRelationshipManagement:
-      return "Integrate with customer relationship management platforms.";
-    case SourceCategory.CodeRepository:
-      return "Integrate with code repositories and version control systems.";
-    case SourceCategory.Storage:
-      return "Connect to cloud storage and file hosting services.";
-    case SourceCategory.Wiki:
-      return "Link to wiki and knowledge base platforms.";
-    case SourceCategory.Other:
-      return "Connect to other miscellaneous knowledge sources.";
-    default:
-      return "Connect to various knowledge sources.";
-  }
-}

web/src/app/admin/settings/interfaces.ts

@@ -30,6 +30,9 @@ export interface Settings {
 
   // User Knowledge settings
   user_knowledge_enabled?: boolean;
+
+  // Connector settings
+  show_extra_connectors?: boolean;
 }
 
 export enum NotificationType {

web/src/lib/search/interfaces.ts

@@ -162,10 +162,10 @@ export interface SearchResponse {
 }
 
 export enum SourceCategory {
-  Storage = "Storage",
-  Wiki = "Wiki",
+  Storage = "Web Crawl & File Storage",
+  Wiki = "Knowledge Base & Wiki",
   CustomerSupport = "Customer Support",
-  CustomerRelationshipManagement = "Customer Relationship Management",
+  SalesAndMarketing = "Sales & Marketing",
   Messaging = "Messaging",
   ProjectManagement = "Project Management",
   CodeRepository = "Code Repository",

web/src/lib/sources.ts

@@ -84,7 +84,7 @@ export const SOURCE_METADATA_MAP: SourceMap = {
   web: {
     icon: GlobeIcon2,
     displayName: "Web",
-    category: SourceCategory.Other,
+    category: SourceCategory.Storage,
     docs: "https://docs.onyx.app/connectors/web",
   },
   file: {
@@ -154,7 +154,7 @@ export const SOURCE_METADATA_MAP: SourceMap = {
   gong: {
     icon: GongIcon,
     displayName: "Gong",
-    category: SourceCategory.Other,
+    category: SourceCategory.SalesAndMarketing,
     docs: "https://docs.onyx.app/connectors/gong",
   },
   linear: {
@@ -190,7 +190,7 @@ export const SOURCE_METADATA_MAP: SourceMap = {
   hubspot: {
     icon: HubSpotIcon,
     displayName: "HubSpot",
-    category: SourceCategory.CustomerRelationshipManagement,
+    category: SourceCategory.SalesAndMarketing,
     docs: "https://docs.onyx.app/connectors/hubspot",
   },
   document360: {
@@ -214,7 +214,7 @@ export const SOURCE_METADATA_MAP: SourceMap = {
   loopio: {
     icon: LoopioIcon,
     displayName: "Loopio",
-    category: SourceCategory.Other,
+    category: SourceCategory.SalesAndMarketing,
   },
   dropbox: {
     icon: DropboxIcon,
@@ -225,7 +225,7 @@ export const SOURCE_METADATA_MAP: SourceMap = {
   salesforce: {
     icon: SalesforceIcon,
     displayName: "Salesforce",
-    category: SourceCategory.CustomerRelationshipManagement,
+    category: SourceCategory.SalesAndMarketing,
     docs: "https://docs.onyx.app/connectors/salesforce",
   },
   sharepoint: {
@@ -319,7 +319,7 @@ export const SOURCE_METADATA_MAP: SourceMap = {
   fireflies: {
     icon: FirefliesIcon,
     displayName: "Fireflies",
-    category: SourceCategory.Other,
+    category: SourceCategory.SalesAndMarketing,
     docs: "https://docs.onyx.app/connectors/fireflies",
   },
   egnyte: {
@@ -331,7 +331,7 @@ export const SOURCE_METADATA_MAP: SourceMap = {
   airtable: {
     icon: AirtableIcon,
     displayName: "Airtable",
-    category: SourceCategory.Other,
+    category: SourceCategory.ProjectManagement,
     docs: "https://docs.onyx.app/connectors/airtable",
   },
   gitbook: {
@@ -351,8 +351,7 @@ export const SOURCE_METADATA_MAP: SourceMap = {
     displayName: "Email",
     category: SourceCategory.Messaging,
   },
-  // currently used for the Internet Search tool docs, which is why
-  // a globe is used
+  // Placeholder used as non-null default
   not_applicable: {
     icon: GlobeIcon,
     displayName: "Not Applicable",