Merge api and telegram

backend/Dockerfile

@@ -97,6 +97,7 @@ COPY supervisord.conf /etc/supervisor/conf.d/supervisord.conf
 
 # Set up application files
 COPY ./onyx /app/onyx
+COPY ./telegram /app/telegram
 COPY ./shared_configs /app/shared_configs
 COPY ./alembic /app/alembic
 COPY ./alembic_tenants /app/alembic_tenants

backend/alembic/versions/5eb9b14f4656_add_telegram_tabls.py

@@ -0,0 +1,46 @@
+"""add telegram tabls
+
+Revision ID: 5eb9b14f4656
+Revises: 81f65b7f25ba
+Create Date: 2024-10-28 13:14:13.967004
+
+"""
+import fastapi_users_db_sqlalchemy
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = '5eb9b14f4656'
+down_revision = '81f65b7f25ba'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('telegram_user_settings',
+    sa.Column('user_id', sa.BigInteger(), nullable=False),
+    sa.Column('model', sa.JSON(), nullable=True),
+    sa.Column('persona_id', sa.Integer(), nullable=True),
+    sa.Column('prompt_id', sa.Integer(), nullable=True),
+    sa.PrimaryKeyConstraint('user_id')
+    )
+    op.create_table('telegram_user_api_key',
+    sa.Column('api_key', sa.String(), nullable=False),
+    sa.Column('user_id', fastapi_users_db_sqlalchemy.generics.GUID(), nullable=False),
+    sa.Column('telegram_user_id', sa.BigInteger(), nullable=True),
+    sa.ForeignKeyConstraint(['user_id'], ['user.id'], ),
+    sa.PrimaryKeyConstraint('api_key'),
+    sa.UniqueConstraint('api_key')
+    )
+    op.create_foreign_key(None, 'api_key', 'user', ['user_id'], ['id'])
+    op.create_foreign_key(None, 'api_key', 'user', ['owner_id'], ['id'])
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table('telegram_user_api_key')
+    op.drop_table('telegram_user_settings')
+    # ### end Alembic commands ###

backend/alembic/versions/81f65b7f25ba_add_knowledge_map.py

@@ -0,0 +1,47 @@
+"""add knowledge map
+
+Revision ID: 81f65b7f25ba
+Revises: e58d392624d2
+Create Date: 2024-10-07 12:52:27.434928
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = '81f65b7f25ba'
+down_revision = 'e58d392624d2'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('knowledge_map',
+    sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
+    sa.Column('name', sa.String(), nullable=False),
+    sa.Column('document_set_id', sa.Integer(), nullable=False),
+    sa.Column('flowise_pipeline_id', sa.String(), nullable=False),
+    sa.Column('description', sa.String(), nullable=False),
+    sa.ForeignKeyConstraint(['document_set_id'], ['document_set.id'], ondelete='CASCADE'),
+    sa.PrimaryKeyConstraint('id')
+    )
+    op.create_table('knowledge_map_answer',
+    sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
+    sa.Column('document_id', sa.String(), nullable=False),
+    sa.Column('knowledge_map_id', sa.Integer(), nullable=False),
+    sa.Column('topic', sa.String(), nullable=False),
+    sa.Column('answer', sa.String(), nullable=False),
+    sa.ForeignKeyConstraint(['document_id'], ['document.id'], ondelete='CASCADE'),
+    sa.ForeignKeyConstraint(['knowledge_map_id'], ['knowledge_map.id'], ondelete='CASCADE'),
+    sa.PrimaryKeyConstraint('id')
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table('knowledge_map_answer')
+    op.drop_table('knowledge_map')
+    # ### end Alembic commands ###

backend/alembic/versions/e58d392624d2_add_persona_pipeline_id_and_use_default.py

@@ -0,0 +1,32 @@
+"""Add persona pipeline_id and use_default
+
+Revision ID: e58d392624d2
+Revises: cf90764725d8
+Create Date: 2025-04-04 10:56:39.769294
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "e58d392624d2"
+down_revision = "cf90764725d8"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column(
+        "persona", sa.Column("pipeline_id", sa.String(), nullable=True)
+    )
+    op.add_column("persona", sa.Column('template_file', sa.LargeBinary(), nullable=True))
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column("persona", "pipeline_id")
+    op.drop_column("persona", 'template_file')
+    # ### end Alembic commands ###

backend/ee/onyx/server/manage/standard_answer.py

@@ -16,7 +16,7 @@
 from ee.onyx.server.manage.models import StandardAnswerCategory
 from ee.onyx.server.manage.models import StandardAnswerCategoryCreationRequest
 from ee.onyx.server.manage.models import StandardAnswerCreationRequest
-from onyx.auth.users import current_admin_user
+from onyx.auth.users import current_admin_user, current_curator_or_admin_user
 from onyx.db.engine import get_session
 from onyx.db.models import User
 
@@ -27,7 +27,7 @@
 def create_standard_answer(
     standard_answer_creation_request: StandardAnswerCreationRequest,
     db_session: Session = Depends(get_session),
-    _: User | None = Depends(current_admin_user),
+    _: User | None = Depends(current_curator_or_admin_user),
 ) -> StandardAnswer:
     standard_answer_model = insert_standard_answer(
         keyword=standard_answer_creation_request.keyword,
@@ -43,7 +43,7 @@ def create_standard_answer(
 @router.get("/admin/standard-answer")
 def list_standard_answers(
     db_session: Session = Depends(get_session),
-    _: User | None = Depends(current_admin_user),
+    _: User | None = Depends(current_curator_or_admin_user),
 ) -> list[StandardAnswer]:
     standard_answer_models = fetch_standard_answers(db_session=db_session)
     return [
@@ -57,7 +57,7 @@ def patch_standard_answer(
     standard_answer_id: int,
     standard_answer_creation_request: StandardAnswerCreationRequest,
     db_session: Session = Depends(get_session),
-    _: User | None = Depends(current_admin_user),
+    _: User | None = Depends(current_curator_or_admin_user),
 ) -> StandardAnswer:
     existing_standard_answer = fetch_standard_answer(
         standard_answer_id=standard_answer_id,
@@ -83,7 +83,7 @@ def patch_standard_answer(
 def delete_standard_answer(
     standard_answer_id: int,
     db_session: Session = Depends(get_session),
-    _: User | None = Depends(current_admin_user),
+    _: User | None = Depends(current_curator_or_admin_user),
 ) -> None:
     return remove_standard_answer(
         standard_answer_id=standard_answer_id,
@@ -95,7 +95,7 @@ def delete_standard_answer(
 def create_standard_answer_category(
     standard_answer_category_creation_request: StandardAnswerCategoryCreationRequest,
     db_session: Session = Depends(get_session),
-    _: User | None = Depends(current_admin_user),
+    _: User | None = Depends(current_curator_or_admin_user),
 ) -> StandardAnswerCategory:
     standard_answer_category_model = insert_standard_answer_category(
         category_name=standard_answer_category_creation_request.name,
@@ -107,7 +107,7 @@ def create_standard_answer_category(
 @router.get("/admin/standard-answer/category")
 def list_standard_answer_categories(
     db_session: Session = Depends(get_session),
-    _: User | None = Depends(current_admin_user),
+    _: User | None = Depends(current_curator_or_admin_user),
 ) -> list[StandardAnswerCategory]:
     standard_answer_category_models = fetch_standard_answer_categories(
         db_session=db_session
@@ -123,7 +123,7 @@ def patch_standard_answer_category(
     standard_answer_category_id: int,
     standard_answer_category_creation_request: StandardAnswerCategoryCreationRequest,
     db_session: Session = Depends(get_session),
-    _: User | None = Depends(current_admin_user),
+    _: User | None = Depends(current_curator_or_admin_user),
 ) -> StandardAnswerCategory:
     existing_standard_answer_category = fetch_standard_answer_category(
         standard_answer_category_id=standard_answer_category_id,

backend/ee/onyx/server/query_and_chat/chat_backend.py

@@ -14,7 +14,7 @@
     BasicCreateChatMessageWithHistoryRequest,
 )
 from ee.onyx.server.query_and_chat.models import ChatBasicResponse
-from onyx.auth.users import current_user
+from onyx.auth.users import current_user, api_key_dep
 from onyx.chat.chat_utils import combine_message_thread
 from onyx.chat.chat_utils import create_chat_chain
 from onyx.chat.models import AgentAnswerPiece
@@ -229,7 +229,7 @@ def remove_answer_citations(answer: str) -> str:
 @router.post("/send-message-simple-api")
 def handle_simplified_chat_message(
     chat_message_req: BasicCreateChatMessageRequest,
-    user: User | None = Depends(current_user),
+    user: User | None = Depends(api_key_dep),
     db_session: Session = Depends(get_session),
 ) -> ChatBasicResponse:
     """This is a Non-Streaming version that only gives back a minimal set of information"""
@@ -290,7 +290,7 @@ def handle_simplified_chat_message(
 @router.post("/send-message-simple-with-history")
 def handle_send_message_simple_with_history(
     req: BasicCreateChatMessageWithHistoryRequest,
-    user: User | None = Depends(current_user),
+    user: User | None = Depends(api_key_dep),
     db_session: Session = Depends(get_session),
 ) -> ChatBasicResponse:
     """This is a Non-Streaming version that only gives back a minimal set of information.

backend/onyx/auth/users.py

@@ -55,6 +55,7 @@
 from httpx_oauth.oauth2 import OAuth2Token
 from pydantic import BaseModel
 from sqlalchemy.ext.asyncio import AsyncSession
+from sqlalchemy.orm import Session
 
 from ee.onyx.configs.app_configs import ANONYMOUS_USER_COOKIE_NAME
 from onyx.auth.api_key import get_hashed_api_key_from_request
@@ -91,12 +92,13 @@
 from onyx.db.auth import get_user_count
 from onyx.db.auth import get_user_db
 from onyx.db.auth import SQLAlchemyUserAdminDB
-from onyx.db.engine import get_async_session
+from onyx.db.engine import get_async_session, get_session
 from onyx.db.engine import get_async_session_with_tenant
 from onyx.db.engine import get_session_with_tenant
 from onyx.db.models import AccessToken
 from onyx.db.models import OAuthAccount
 from onyx.db.models import User
+from onyx.db.telegram import get_user_by_telegram_api_key
 from onyx.db.users import get_user_by_email
 from onyx.redis.redis_pool import get_async_redis_connection
 from onyx.redis.redis_pool import get_redis_client
@@ -1381,3 +1383,23 @@ async def api_key_dep(
         raise HTTPException(status_code=401, detail="Invalid API key")
 
     return user
+
+
+async def telegram_api_key_dep(token: str,
+    request: Request, db_session: Session = Depends(get_session),
+) -> User | None:
+    if AUTH_TYPE == AuthType.DISABLED:
+        return None
+
+    user: User | None = None
+
+    if not token:
+        raise HTTPException(status_code=401, detail="Missing API key")
+
+    if token:
+        user = get_user_by_telegram_api_key(token, db_session)
+
+    if user is None:
+        raise HTTPException(status_code=401, detail="Invalid API key")
+
+    return user

backend/onyx/chat/models.py

@@ -226,6 +226,14 @@ class CustomToolResponse(BaseModel):
     response: ToolResultType
     tool_name: str
 
+class LangflowToolResponse(BaseModel):
+    response: ToolResultType
+    tool_name: str
+
+class ResumeToolResponse(BaseModel):
+    response: ToolResultType
+    tool_name: str
+
 
 class ToolConfig(BaseModel):
     id: int
@@ -265,6 +273,7 @@ class PersonaOverrideConfig(BaseModel):
     | CitationInfo
     | FileChatDisplay
     | CustomToolResponse
+    | LangflowToolResponse
     | StreamingError
     | StreamStopInfo
 )