From 94397e175bf72a8db9b32d0f96240c5a3a811d5d Mon Sep 17 00:00:00 2001 From: anmarhindi Date: Tue, 14 Jan 2025 14:18:39 +0100 Subject: [PATCH 01/76] Initial playground From 949c6e9e192881c42e71ab423c35d5f6ce046eff Mon Sep 17 00:00:00 2001 From: LennartSchmidtKern Date: Tue, 14 Jan 2025 14:49:33 +0100 Subject: [PATCH 02/76] Add playground route and search question model --- app.py | 6 ++++- fast_api/models.py | 5 ++++ fast_api/routes/playground.py | 49 +++++++++++++++++++++++++++++++++++ route_prefix.py | 1 + 4 files changed, 60 insertions(+), 1 deletion(-) create mode 100644 fast_api/routes/playground.py diff --git a/app.py b/app.py index f152011f..80969c89 100644 --- a/app.py +++ b/app.py @@ -30,6 +30,7 @@ from fast_api.routes.weak_supervision import router as weak_supervision_router from fast_api.routes.labeling_tasks import router as labeling_tasks_router from fast_api.routes.task_execution import router as task_execution_router +from fast_api.routes.playground import router as playground_router from middleware.database_session import handle_db_session from middleware.starlette_tmp_middleware import DatabaseSessionHandler from starlette.applications import Starlette @@ -55,6 +56,7 @@ PREFIX_WEAK_SUPERVISION, PREFIX_LABELING_TASKS, PREFIX_TASK_EXECUTION, + PREFIX_PLAYGROUND, ) from util import security, clean_up from middleware import log_storage @@ -106,7 +108,9 @@ fastapi_app.include_router( labeling_tasks_router, prefix=PREFIX_LABELING_TASKS, tags=["labeling-tasks"] ) - +fastapi_app.include_router( + playground_router, prefix=PREFIX_PLAYGROUND, tags=["playground"] +) fastapi_app_internal = FastAPI() fastapi_app_internal.include_router( task_execution_router, prefix=PREFIX_TASK_EXECUTION, tags=["task-execution"] diff --git a/fast_api/models.py b/fast_api/models.py index ac790911..d7ea5d34 100644 --- a/fast_api/models.py +++ b/fast_api/models.py @@ -444,3 +444,8 @@ class UpdateCustomerButton(BaseModel): class MissingUsersBody(BaseModel): user_ids: List[StrictStr] + + +class SearchQuestionBody(BaseModel): + question: StrictStr + embeddingId: StrictStr diff --git a/fast_api/routes/playground.py b/fast_api/routes/playground.py new file mode 100644 index 00000000..cbad6022 --- /dev/null +++ b/fast_api/routes/playground.py @@ -0,0 +1,49 @@ +from fastapi import APIRouter, Depends, Request, Body +from controller.auth import manager as auth_manager +from fast_api.routes.client_response import pack_json_result +from fast_api.models import SearchQuestionBody +from typing import List, Any, Optional, Tuple +import os +import requests + +NEURAL_SEARCH = os.getenv("NEURAL_SEARCH") +EMBEDDING_SERVICE = os.getenv("EMBEDDING_SERVICE") + +router = APIRouter() + + +@router.post( + "/{project_id}/question-playground" +) # dependencies=[Depends(auth_manager.check_project_access_dep)] +def get_search_results( + request: Request, + project_id: str, + search_question: SearchQuestionBody = Body(...), +): + embedding_id = search_question.embeddingId + question = search_question.question + + has_error, result, logs = get_tensors_for_texts( + project_id, embedding_id, [question] + ) + return pack_json_result({"has_error": has_error, "result": result, "logs": logs}) + + +def get_tensors_for_texts( + refinery_project_id: str, embedding_id: str, texts: List[str] +) -> Tuple[bool, Optional[List[Any]]]: + + url = f"{EMBEDDING_SERVICE}/calc-tensor-by-pkl/{refinery_project_id}/{embedding_id}" + + response = requests.post( + url, + headers={ + "accept": "application/json", + "content-type": "application/json", + }, + json={"texts": texts}, + ) + if response.ok: + return response.json()["tensor"] + else: + return None diff --git a/route_prefix.py b/route_prefix.py index b519c320..5c511fbe 100644 --- a/route_prefix.py +++ b/route_prefix.py @@ -16,3 +16,4 @@ PREFIX_WEAK_SUPERVISION = PREFIX + "/weak-supervision" PREFIX_LABELING_TASKS = PREFIX + "/labeling-tasks" PREFIX_TASK_EXECUTION = PREFIX + "/task-execution" +PREFIX_PLAYGROUND = PREFIX + "/playground" From e050d0ac22d5c0e0cb92292bf5e88c6790cb401f Mon Sep 17 00:00:00 2001 From: anmarhindi Date: Tue, 14 Jan 2025 17:19:00 +0100 Subject: [PATCH 03/76] initial eval set endpoint --- fast_api/routes/playground.py | 11 +++++++++++ submodules/model | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/fast_api/routes/playground.py b/fast_api/routes/playground.py index cbad6022..2336cb81 100644 --- a/fast_api/routes/playground.py +++ b/fast_api/routes/playground.py @@ -47,3 +47,14 @@ def get_tensors_for_texts( return response.json()["tensor"] else: return None + + +@router.post( + "/{project_id}/store-eval-set" +) # dependencies=[Depends(auth_manager.check_project_access_dep)] +def store_eval_set( + request: Request, + project_id: str, + search_question: SearchQuestionBody = Body(...), +): + pass diff --git a/submodules/model b/submodules/model index cf337c2d..23012d4d 160000 --- a/submodules/model +++ b/submodules/model @@ -1 +1 @@ -Subproject commit cf337c2d46a302f33ed35419c1d95a60871bbe61 +Subproject commit 23012d4dd5c32d2a2d845670a8abb7078f8c75d7 From b705cf5d6943729351c0730b61683b07bde2052e Mon Sep 17 00:00:00 2001 From: anmarhindi Date: Tue, 14 Jan 2025 17:20:40 +0100 Subject: [PATCH 04/76] Upgrade alembic version for eval set --- .../6ef1e5c90164_adds_evaluation_set.py | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 alembic/versions/6ef1e5c90164_adds_evaluation_set.py diff --git a/alembic/versions/6ef1e5c90164_adds_evaluation_set.py b/alembic/versions/6ef1e5c90164_adds_evaluation_set.py new file mode 100644 index 00000000..04cd3f7d --- /dev/null +++ b/alembic/versions/6ef1e5c90164_adds_evaluation_set.py @@ -0,0 +1,42 @@ +"""adds_evaluation_set + +Revision ID: 6ef1e5c90164 +Revises: eb5ecbee5090 +Create Date: 2025-01-14 16:19:32.940641 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = '6ef1e5c90164' +down_revision = 'eb5ecbee5090' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('evaluation_set', + sa.Column('id', postgresql.UUID(as_uuid=True), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('created_by', postgresql.UUID(as_uuid=True), nullable=True), + sa.Column('project_id', postgresql.UUID(as_uuid=True), nullable=True), + sa.Column('name', sa.String(), nullable=True), + sa.Column('data', sa.JSON(), nullable=True), + sa.ForeignKeyConstraint(['created_by'], ['user.id'], ondelete='SET NULL'), + sa.ForeignKeyConstraint(['project_id'], ['project.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_evaluation_set_created_by'), 'evaluation_set', ['created_by'], unique=False) + op.create_index(op.f('ix_evaluation_set_project_id'), 'evaluation_set', ['project_id'], unique=False) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index(op.f('ix_evaluation_set_project_id'), table_name='evaluation_set') + op.drop_index(op.f('ix_evaluation_set_created_by'), table_name='evaluation_set') + op.drop_table('evaluation_set') + # ### end Alembic commands ### From 3a3efc4a7827589425255ba39d1026d01a99f81c Mon Sep 17 00:00:00 2001 From: LennartSchmidtKern Date: Tue, 14 Jan 2025 17:35:01 +0100 Subject: [PATCH 05/76] similar records --- fast_api/routes/playground.py | 51 +++++++++++++++++++++++++++++++---- 1 file changed, 46 insertions(+), 5 deletions(-) diff --git a/fast_api/routes/playground.py b/fast_api/routes/playground.py index cbad6022..aa49ff45 100644 --- a/fast_api/routes/playground.py +++ b/fast_api/routes/playground.py @@ -1,8 +1,8 @@ from fastapi import APIRouter, Depends, Request, Body from controller.auth import manager as auth_manager -from fast_api.routes.client_response import pack_json_result +from fast_api.routes.client_response import pack_json_result, GENERIC_FAILURE_RESPONSE from fast_api.models import SearchQuestionBody -from typing import List, Any, Optional, Tuple +from typing import List, Any, Optional, Tuple, Dict import os import requests @@ -10,6 +10,9 @@ EMBEDDING_SERVICE = os.getenv("EMBEDDING_SERVICE") router = APIRouter() +LIMIT = 10 +FILTER = None +THRESHOLD = None @router.post( @@ -23,10 +26,17 @@ def get_search_results( embedding_id = search_question.embeddingId question = search_question.question - has_error, result, logs = get_tensors_for_texts( - project_id, embedding_id, [question] + question_tensor = get_tensors_for_texts(project_id, embedding_id, [question]) + + if question_tensor is None: + return GENERIC_FAILURE_RESPONSE + elif len(question_tensor) == 0: + return pack_json_result([]) + search_results = get_most_similar_records( + project_id, embedding_id, question_tensor[0], LIMIT ) - return pack_json_result({"has_error": has_error, "result": result, "logs": logs}) + + return pack_json_result(search_results) def get_tensors_for_texts( @@ -47,3 +57,34 @@ def get_tensors_for_texts( return response.json()["tensor"] else: return None + + +def get_most_similar_records( + project_id: str, + embedding_id: str, + embedding_tensor: List[float], + limit: int, + similarity_filter_option: Optional[List[Dict[str, Any]]] = None, + threshold: Optional[float] = None, +): + url = f"{NEURAL_SEARCH}/most_similar_by_embedding?include_scores=true" + + response = requests.post( + url, + headers={ + "accept": "application/json", + "content-type": "application/json", + }, + json={ + "project_id": project_id, + "embedding_id": embedding_id, + "embedding_tensor": embedding_tensor, + "limit": limit, + "att_filter": similarity_filter_option, + "threshold": threshold, + }, + ) + if response.ok: + return response.json() + else: + return None From a5d69019d64aaf1aab99c1f6ccd163830524c2e7 Mon Sep 17 00:00:00 2001 From: LennartSchmidtKern Date: Thu, 16 Jan 2025 10:56:11 +0100 Subject: [PATCH 06/76] restructure --- controller/playground/__init__.py | 0 controller/playground/manager.py | 89 +++++++++++++++++++++++++++++++ fast_api/models.py | 5 ++ fast_api/routes/playground.py | 85 +++++++---------------------- 4 files changed, 113 insertions(+), 66 deletions(-) create mode 100644 controller/playground/__init__.py create mode 100644 controller/playground/manager.py diff --git a/controller/playground/__init__.py b/controller/playground/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/controller/playground/manager.py b/controller/playground/manager.py new file mode 100644 index 00000000..ab7961eb --- /dev/null +++ b/controller/playground/manager.py @@ -0,0 +1,89 @@ +from fast_api.routes.client_response import pack_json_result, GENERIC_FAILURE_RESPONSE +from typing import List, Any, Optional, Tuple, Dict +import os +import requests + +NEURAL_SEARCH = os.getenv("NEURAL_SEARCH") +EMBEDDING_SERVICE = os.getenv("EMBEDDING_SERVICE") + +LIMIT = 10 +FILTER = None +THRESHOLD = None + + +def get_search_result_for_text(project_id: str, embedding_id: str, question: str): + question_tensor = __get_tensors_for_texts(project_id, embedding_id, [question]) + + if question_tensor is None: + return GENERIC_FAILURE_RESPONSE + elif len(question_tensor) == 0: + return pack_json_result([]) + + search_results = __get_most_similar_records( + project_id, embedding_id, question_tensor[0], LIMIT + ) + return search_results + +def get_search_result_for_eval_set(project_id: str, embedding_id: str, eval_set_id: str): + question_tensor = __get_tensors_for_texts(project_id, embedding_id, [question]) + + if question_tensor is None: + return GENERIC_FAILURE_RESPONSE + elif len(question_tensor) == 0: + return pack_json_result([]) + + search_results = __get_most_similar_records( + project_id, embedding_id, question_tensor[0], LIMIT + ) + return search_results + + +def __get_tensors_for_texts( + refinery_project_id: str, embedding_id: str, texts: List[str] +) -> Tuple[bool, Optional[List[Any]]]: + + url = f"{EMBEDDING_SERVICE}/calc-tensor-by-pkl/{refinery_project_id}/{embedding_id}" + + response = requests.post( + url, + headers={ + "accept": "application/json", + "content-type": "application/json", + }, + json={"texts": texts}, + ) + if response.ok: + return response.json()["tensor"] + else: + return None + + +def __get_most_similar_records( + project_id: str, + embedding_id: str, + embedding_tensor: List[float], + limit: int, + similarity_filter_option: Optional[List[Dict[str, Any]]] = None, + threshold: Optional[float] = None, +): + url = f"{NEURAL_SEARCH}/most_similar_by_embedding?include_scores=true" + + response = requests.post( + url, + headers={ + "accept": "application/json", + "content-type": "application/json", + }, + json={ + "project_id": project_id, + "embedding_id": embedding_id, + "embedding_tensor": embedding_tensor, + "limit": limit, + "att_filter": similarity_filter_option, + "threshold": threshold, + }, + ) + if response.ok: + return response.json() + else: + return None diff --git a/fast_api/models.py b/fast_api/models.py index d7ea5d34..aa6f1963 100644 --- a/fast_api/models.py +++ b/fast_api/models.py @@ -449,3 +449,8 @@ class MissingUsersBody(BaseModel): class SearchQuestionBody(BaseModel): question: StrictStr embeddingId: StrictStr + + +class SearchEvalSetBody(BaseModel): + evalSetId: StrictStr + embeddingId: StrictStr diff --git a/fast_api/routes/playground.py b/fast_api/routes/playground.py index c5d324d7..5fab61bc 100644 --- a/fast_api/routes/playground.py +++ b/fast_api/routes/playground.py @@ -1,24 +1,16 @@ from fastapi import APIRouter, Depends, Request, Body from controller.auth import manager as auth_manager -from fast_api.routes.client_response import pack_json_result, GENERIC_FAILURE_RESPONSE -from fast_api.models import SearchQuestionBody -from typing import List, Any, Optional, Tuple, Dict -import os -import requests - -NEURAL_SEARCH = os.getenv("NEURAL_SEARCH") -EMBEDDING_SERVICE = os.getenv("EMBEDDING_SERVICE") +from fast_api.routes.client_response import pack_json_result +from fast_api.models import SearchQuestionBody, SearchEvalSetBody +from controller.playground import manager as playground_manager router = APIRouter() -LIMIT = 10 -FILTER = None -THRESHOLD = None @router.post( - "/{project_id}/question-playground" + "/{project_id}/search" ) # dependencies=[Depends(auth_manager.check_project_access_dep)] -def get_search_results( +def get_search_results_question( request: Request, project_id: str, search_question: SearchQuestionBody = Body(...), @@ -26,68 +18,29 @@ def get_search_results( embedding_id = search_question.embeddingId question = search_question.question - question_tensor = get_tensors_for_texts(project_id, embedding_id, [question]) - - if question_tensor is None: - return GENERIC_FAILURE_RESPONSE - elif len(question_tensor) == 0: - return pack_json_result([]) - search_results = get_most_similar_records( - project_id, embedding_id, question_tensor[0], LIMIT + search_results = playground_manager.get_search_result_for_text( + project_id, embedding_id, question ) return pack_json_result(search_results) -def get_tensors_for_texts( - refinery_project_id: str, embedding_id: str, texts: List[str] -) -> Tuple[bool, Optional[List[Any]]]: - - url = f"{EMBEDDING_SERVICE}/calc-tensor-by-pkl/{refinery_project_id}/{embedding_id}" - - response = requests.post( - url, - headers={ - "accept": "application/json", - "content-type": "application/json", - }, - json={"texts": texts}, - ) - if response.ok: - return response.json()["tensor"] - else: - return None - - -def get_most_similar_records( +@router.post( + "/{project_id}/eval-set-search" +) # dependencies=[Depends(auth_manager.check_project_access_dep)] +def get_search_results_eval_set( + request: Request, project_id: str, - embedding_id: str, - embedding_tensor: List[float], - limit: int, - similarity_filter_option: Optional[List[Dict[str, Any]]] = None, - threshold: Optional[float] = None, + search_eval_set: SearchEvalSetBody = Body(...), ): - url = f"{NEURAL_SEARCH}/most_similar_by_embedding?include_scores=true" + embedding_id = search_eval_set.embeddingId + evaluation_set_id = search_eval_set.evalSetId - response = requests.post( - url, - headers={ - "accept": "application/json", - "content-type": "application/json", - }, - json={ - "project_id": project_id, - "embedding_id": embedding_id, - "embedding_tensor": embedding_tensor, - "limit": limit, - "att_filter": similarity_filter_option, - "threshold": threshold, - }, + search_results = playground_manager.get_search_result_for_evalset( + project_id, embedding_id, question ) - if response.ok: - return response.json() - else: - return None + + return pack_json_result(search_results) @router.post( From ce1c8644cc21b81ba3393d1147d4e5fd8b4c7bbd Mon Sep 17 00:00:00 2001 From: anmarhindi Date: Thu, 16 Jan 2025 12:04:05 +0100 Subject: [PATCH 07/76] Remove alembic upgrade due to db arch changes --- .../6ef1e5c90164_adds_evaluation_set.py | 42 ------------------- 1 file changed, 42 deletions(-) delete mode 100644 alembic/versions/6ef1e5c90164_adds_evaluation_set.py diff --git a/alembic/versions/6ef1e5c90164_adds_evaluation_set.py b/alembic/versions/6ef1e5c90164_adds_evaluation_set.py deleted file mode 100644 index 04cd3f7d..00000000 --- a/alembic/versions/6ef1e5c90164_adds_evaluation_set.py +++ /dev/null @@ -1,42 +0,0 @@ -"""adds_evaluation_set - -Revision ID: 6ef1e5c90164 -Revises: eb5ecbee5090 -Create Date: 2025-01-14 16:19:32.940641 - -""" -from alembic import op -import sqlalchemy as sa -from sqlalchemy.dialects import postgresql - -# revision identifiers, used by Alembic. -revision = '6ef1e5c90164' -down_revision = 'eb5ecbee5090' -branch_labels = None -depends_on = None - - -def upgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.create_table('evaluation_set', - sa.Column('id', postgresql.UUID(as_uuid=True), nullable=False), - sa.Column('created_at', sa.DateTime(), nullable=True), - sa.Column('created_by', postgresql.UUID(as_uuid=True), nullable=True), - sa.Column('project_id', postgresql.UUID(as_uuid=True), nullable=True), - sa.Column('name', sa.String(), nullable=True), - sa.Column('data', sa.JSON(), nullable=True), - sa.ForeignKeyConstraint(['created_by'], ['user.id'], ondelete='SET NULL'), - sa.ForeignKeyConstraint(['project_id'], ['project.id'], ondelete='CASCADE'), - sa.PrimaryKeyConstraint('id') - ) - op.create_index(op.f('ix_evaluation_set_created_by'), 'evaluation_set', ['created_by'], unique=False) - op.create_index(op.f('ix_evaluation_set_project_id'), 'evaluation_set', ['project_id'], unique=False) - # ### end Alembic commands ### - - -def downgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.drop_index(op.f('ix_evaluation_set_project_id'), table_name='evaluation_set') - op.drop_index(op.f('ix_evaluation_set_created_by'), table_name='evaluation_set') - op.drop_table('evaluation_set') - # ### end Alembic commands ### From df1c4d4988b231041c7e4d67a6e4a587a7f156b2 Mon Sep 17 00:00:00 2001 From: LennartSchmidtKern Date: Thu, 16 Jan 2025 14:18:46 +0100 Subject: [PATCH 08/76] dummy endpoints --- controller/playground/manager.py | 42 ++++++++++++++---- fast_api/models.py | 11 +++-- fast_api/routes/playground.py | 74 +++++++++++++++++++++++++------- 3 files changed, 100 insertions(+), 27 deletions(-) diff --git a/controller/playground/manager.py b/controller/playground/manager.py index ab7961eb..d79e1338 100644 --- a/controller/playground/manager.py +++ b/controller/playground/manager.py @@ -14,17 +14,17 @@ def get_search_result_for_text(project_id: str, embedding_id: str, question: str): question_tensor = __get_tensors_for_texts(project_id, embedding_id, [question]) - if question_tensor is None: - return GENERIC_FAILURE_RESPONSE - elif len(question_tensor) == 0: - return pack_json_result([]) + if question_tensor and question_tensor[0]: + return __get_most_similar_records( + project_id, embedding_id, question_tensor[0], LIMIT + ) + else: + return [] - search_results = __get_most_similar_records( - project_id, embedding_id, question_tensor[0], LIMIT - ) - return search_results -def get_search_result_for_eval_set(project_id: str, embedding_id: str, eval_set_id: str): +def get_search_result_for_eval_set( + project_id: str, embedding_id: str, eval_set_id: str +): question_tensor = __get_tensors_for_texts(project_id, embedding_id, [question]) if question_tensor is None: @@ -38,6 +38,30 @@ def get_search_result_for_eval_set(project_id: str, embedding_id: str, eval_set_ return search_results +def create_evaluation_set(project_id: str, question: str, record_ids: List[str]): + pass + + +def get_evaluation_set_by_id(project_id: str, set_id: str): + pass + + +def get_evaluation_sets(project_id: str): + pass + + +def create_evaluation_group(project_id: str, name: str, evaluation_set_ids: List[str]): + pass + + +def get_evaluation_group_by_id(project_id: str, group_id: str): + pass + + +def get_evaluation_groups(project_id: str): + pass + + def __get_tensors_for_texts( refinery_project_id: str, embedding_id: str, texts: List[str] ) -> Tuple[bool, Optional[List[Any]]]: diff --git a/fast_api/models.py b/fast_api/models.py index aa6f1963..39c9ad82 100644 --- a/fast_api/models.py +++ b/fast_api/models.py @@ -451,6 +451,11 @@ class SearchQuestionBody(BaseModel): embeddingId: StrictStr -class SearchEvalSetBody(BaseModel): - evalSetId: StrictStr - embeddingId: StrictStr +class EvaluationSetCreationBody(BaseModel): + question: StrictStr + recordIds: List[StrictStr] + + +class EvaluationGroupCreationBody(BaseModel): + matchingSetIds: List[StrictStr] + name: StrictStr diff --git a/fast_api/routes/playground.py b/fast_api/routes/playground.py index 5fab61bc..9fa58c69 100644 --- a/fast_api/routes/playground.py +++ b/fast_api/routes/playground.py @@ -1,7 +1,12 @@ from fastapi import APIRouter, Depends, Request, Body from controller.auth import manager as auth_manager -from fast_api.routes.client_response import pack_json_result -from fast_api.models import SearchQuestionBody, SearchEvalSetBody +from fast_api.routes.client_response import pack_json_result, get_silent_success +from fast_api.models import ( + SearchQuestionBody, + MatchingSetCreationBody, + EvaluationSetCreationBody, + EvaluationGroupCreationBody, +) from controller.playground import manager as playground_manager router = APIRouter() @@ -26,29 +31,68 @@ def get_search_results_question( @router.post( - "/{project_id}/eval-set-search" + "/{project_id}/evaluation-sets" ) # dependencies=[Depends(auth_manager.check_project_access_dep)] -def get_search_results_eval_set( +def create_evaluation_set( request: Request, project_id: str, - search_eval_set: SearchEvalSetBody = Body(...), + evaluation_set: EvaluationSetCreationBody = Body(...), ): - embedding_id = search_eval_set.embeddingId - evaluation_set_id = search_eval_set.evalSetId - - search_results = playground_manager.get_search_result_for_evalset( - project_id, embedding_id, question + playground_manager.create_evaluation_set( + project_id, evaluation_set.question, evaluation_set.recordIds ) + return get_silent_success() - return pack_json_result(search_results) + +@router.get( + "/{project_id}/evaluation-sets" +) # dependencies=[Depends(auth_manager.check_project_access_dep)] +def get_evaluation_sets( + request: Request, + project_id: str, + search_question: SearchQuestionBody = Body(...), +): + matching_sets = playground_manager.get_evaluation_sets(project_id) + return pack_json_result(matching_sets) + + +@router.get("/{project_id}/evaluation-sets/{set_id}") +def get_single_evaluation_set( + request: Request, + project_id: str, + set_id: str, +): + matching_set = playground_manager.get_evaluation_set_by_id(project_id, set_id) + return pack_json_result(matching_set) @router.post( - "/{project_id}/store-eval-set" + "/{project_id}/evaluation-groups" ) # dependencies=[Depends(auth_manager.check_project_access_dep)] -def store_eval_set( +def create_evaluation_group( request: Request, project_id: str, - search_question: SearchQuestionBody = Body(...), + evaluation_set: EvaluationGroupCreationBody = Body(...), +): + playground_manager.create_evaluation_set( + project_id, evaluation_set.name, evaluation_set.matchingSetIds + ) + return get_silent_success() + + +@router.get( + "/{project_id}/evaluation-groups" +) # dependencies=[Depends(auth_manager.check_project_access_dep)] +def get_evaluation_groups(request: Request, project_id: str): + evaluation_groups = playground_manager.get_evaluation_groups(project_id) + return pack_json_result(evaluation_groups) + + +@router.get("/{project_id}/evaluation--groups/{group_id}") +def get_single_evaluation_group( + request: Request, + project_id: str, + set_id: str, ): - pass + evaluation_set = playground_manager.get_evaluation_group_by_id(project_id, set_id) + return pack_json_result(evaluation_set) From 613c0c8bf46aea70d8e685fa834b5e45379fc238 Mon Sep 17 00:00:00 2001 From: anmarhindi Date: Thu, 16 Jan 2025 14:27:36 +0100 Subject: [PATCH 09/76] adds_refinery_playground_evaluation --- ...6d2_adds_refinery_playground_evaluation.py | 83 +++++++++++++++++++ submodules/model | 2 +- 2 files changed, 84 insertions(+), 1 deletion(-) create mode 100644 alembic/versions/ac97442726d2_adds_refinery_playground_evaluation.py diff --git a/alembic/versions/ac97442726d2_adds_refinery_playground_evaluation.py b/alembic/versions/ac97442726d2_adds_refinery_playground_evaluation.py new file mode 100644 index 00000000..fcad86f3 --- /dev/null +++ b/alembic/versions/ac97442726d2_adds_refinery_playground_evaluation.py @@ -0,0 +1,83 @@ +"""adds_refinery_playground_evaluation + +Revision ID: ac97442726d2 +Revises: eb5ecbee5090 +Create Date: 2025-01-16 13:26:43.059523 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = 'ac97442726d2' +down_revision = 'eb5ecbee5090' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('evaluation_group', + sa.Column('id', postgresql.UUID(as_uuid=True), nullable=False), + sa.Column('name', sa.String(), nullable=True), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('created_by', postgresql.UUID(as_uuid=True), nullable=True), + sa.Column('project_id', postgresql.UUID(as_uuid=True), nullable=True), + sa.Column('evaluation_set_ids', sa.JSON(), nullable=True), + sa.ForeignKeyConstraint(['created_by'], ['user.id'], ondelete='SET NULL'), + sa.ForeignKeyConstraint(['project_id'], ['project.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_evaluation_group_created_by'), 'evaluation_group', ['created_by'], unique=False) + op.create_index(op.f('ix_evaluation_group_project_id'), 'evaluation_group', ['project_id'], unique=False) + op.create_table('evaluation_set', + sa.Column('id', postgresql.UUID(as_uuid=True), nullable=False), + sa.Column('question', sa.String(), nullable=True), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('created_by', postgresql.UUID(as_uuid=True), nullable=True), + sa.Column('project_id', postgresql.UUID(as_uuid=True), nullable=True), + sa.Column('record_ids', sa.JSON(), nullable=True), + sa.ForeignKeyConstraint(['created_by'], ['user.id'], ondelete='SET NULL'), + sa.ForeignKeyConstraint(['project_id'], ['project.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_evaluation_set_created_by'), 'evaluation_set', ['created_by'], unique=False) + op.create_index(op.f('ix_evaluation_set_project_id'), 'evaluation_set', ['project_id'], unique=False) + op.create_table('evaluation_run', + sa.Column('id', postgresql.UUID(as_uuid=True), nullable=False), + sa.Column('evaluation_group_id', postgresql.UUID(as_uuid=True), nullable=True), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('created_by', postgresql.UUID(as_uuid=True), nullable=True), + sa.Column('project_id', postgresql.UUID(as_uuid=True), nullable=True), + sa.Column('embedding_id', postgresql.UUID(as_uuid=True), nullable=True), + sa.Column('state', sa.String(), nullable=True), + sa.Column('results', sa.JSON(), nullable=True), + sa.Column('meta_info', sa.JSON(), nullable=True), + sa.ForeignKeyConstraint(['created_by'], ['user.id'], ondelete='SET NULL'), + sa.ForeignKeyConstraint(['embedding_id'], ['embedding.id'], ondelete='SET NULL'), + sa.ForeignKeyConstraint(['evaluation_group_id'], ['evaluation_group.id'], ondelete='SET NULL'), + sa.ForeignKeyConstraint(['project_id'], ['project.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_evaluation_run_created_by'), 'evaluation_run', ['created_by'], unique=False) + op.create_index(op.f('ix_evaluation_run_embedding_id'), 'evaluation_run', ['embedding_id'], unique=False) + op.create_index(op.f('ix_evaluation_run_evaluation_group_id'), 'evaluation_run', ['evaluation_group_id'], unique=False) + op.create_index(op.f('ix_evaluation_run_project_id'), 'evaluation_run', ['project_id'], unique=False) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index(op.f('ix_evaluation_run_project_id'), table_name='evaluation_run') + op.drop_index(op.f('ix_evaluation_run_evaluation_group_id'), table_name='evaluation_run') + op.drop_index(op.f('ix_evaluation_run_embedding_id'), table_name='evaluation_run') + op.drop_index(op.f('ix_evaluation_run_created_by'), table_name='evaluation_run') + op.drop_table('evaluation_run') + op.drop_index(op.f('ix_evaluation_set_project_id'), table_name='evaluation_set') + op.drop_index(op.f('ix_evaluation_set_created_by'), table_name='evaluation_set') + op.drop_table('evaluation_set') + op.drop_index(op.f('ix_evaluation_group_project_id'), table_name='evaluation_group') + op.drop_index(op.f('ix_evaluation_group_created_by'), table_name='evaluation_group') + op.drop_table('evaluation_group') + # ### end Alembic commands ### diff --git a/submodules/model b/submodules/model index 23012d4d..3587c7d3 160000 --- a/submodules/model +++ b/submodules/model @@ -1 +1 @@ -Subproject commit 23012d4dd5c32d2a2d845670a8abb7078f8c75d7 +Subproject commit 3587c7d33a1023ca500812c62f088bb734c98893 From dfaf2206b33905a519da32fa455fd2ca5805c691 Mon Sep 17 00:00:00 2001 From: anmarhindi Date: Thu, 16 Jan 2025 14:29:18 +0100 Subject: [PATCH 10/76] Remove old matching set --- fast_api/routes/playground.py | 1 - 1 file changed, 1 deletion(-) diff --git a/fast_api/routes/playground.py b/fast_api/routes/playground.py index 9fa58c69..4e731a32 100644 --- a/fast_api/routes/playground.py +++ b/fast_api/routes/playground.py @@ -3,7 +3,6 @@ from fast_api.routes.client_response import pack_json_result, get_silent_success from fast_api.models import ( SearchQuestionBody, - MatchingSetCreationBody, EvaluationSetCreationBody, EvaluationGroupCreationBody, ) From 8024f48a8be97dba0de8d045c3bb45fa559279ae Mon Sep 17 00:00:00 2001 From: LennartSchmidtKern Date: Thu, 16 Jan 2025 16:00:00 +0100 Subject: [PATCH 11/76] correct names --- fast_api/routes/playground.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/fast_api/routes/playground.py b/fast_api/routes/playground.py index 9fa58c69..2638a03a 100644 --- a/fast_api/routes/playground.py +++ b/fast_api/routes/playground.py @@ -3,7 +3,6 @@ from fast_api.routes.client_response import pack_json_result, get_silent_success from fast_api.models import ( SearchQuestionBody, - MatchingSetCreationBody, EvaluationSetCreationBody, EvaluationGroupCreationBody, ) @@ -50,7 +49,6 @@ def create_evaluation_set( def get_evaluation_sets( request: Request, project_id: str, - search_question: SearchQuestionBody = Body(...), ): matching_sets = playground_manager.get_evaluation_sets(project_id) return pack_json_result(matching_sets) @@ -72,10 +70,10 @@ def get_single_evaluation_set( def create_evaluation_group( request: Request, project_id: str, - evaluation_set: EvaluationGroupCreationBody = Body(...), + evaluation_group: EvaluationGroupCreationBody = Body(...), ): playground_manager.create_evaluation_set( - project_id, evaluation_set.name, evaluation_set.matchingSetIds + project_id, evaluation_group.name, evaluation_group.matchingSetIds ) return get_silent_success() @@ -88,11 +86,13 @@ def get_evaluation_groups(request: Request, project_id: str): return pack_json_result(evaluation_groups) -@router.get("/{project_id}/evaluation--groups/{group_id}") +@router.get("/{project_id}/evaluation-groups/{group_id}") def get_single_evaluation_group( request: Request, project_id: str, - set_id: str, + group_id: str, ): - evaluation_set = playground_manager.get_evaluation_group_by_id(project_id, set_id) - return pack_json_result(evaluation_set) + evaluation_group = playground_manager.get_evaluation_group_by_id( + project_id, group_id + ) + return pack_json_result(evaluation_group) From d1e6550ec9f02ec922a8da75a8cb981714854a87 Mon Sep 17 00:00:00 2001 From: LennartSchmidtKern Date: Thu, 16 Jan 2025 16:45:27 +0100 Subject: [PATCH 12/76] evaluation runs --- controller/playground/manager.py | 24 ++++++++---------------- fast_api/models.py | 5 +++++ fast_api/routes/playground.py | 21 +++++++++++++++++++++ 3 files changed, 34 insertions(+), 16 deletions(-) diff --git a/controller/playground/manager.py b/controller/playground/manager.py index d79e1338..b892b97a 100644 --- a/controller/playground/manager.py +++ b/controller/playground/manager.py @@ -22,22 +22,6 @@ def get_search_result_for_text(project_id: str, embedding_id: str, question: str return [] -def get_search_result_for_eval_set( - project_id: str, embedding_id: str, eval_set_id: str -): - question_tensor = __get_tensors_for_texts(project_id, embedding_id, [question]) - - if question_tensor is None: - return GENERIC_FAILURE_RESPONSE - elif len(question_tensor) == 0: - return pack_json_result([]) - - search_results = __get_most_similar_records( - project_id, embedding_id, question_tensor[0], LIMIT - ) - return search_results - - def create_evaluation_set(project_id: str, question: str, record_ids: List[str]): pass @@ -62,6 +46,14 @@ def get_evaluation_groups(project_id: str): pass +def get_evaluation_runs(project_id: str): + pass + + +def init_evaluation_run(project_id: str, evaluation_group_id: str): + pass + + def __get_tensors_for_texts( refinery_project_id: str, embedding_id: str, texts: List[str] ) -> Tuple[bool, Optional[List[Any]]]: diff --git a/fast_api/models.py b/fast_api/models.py index 39c9ad82..7039720b 100644 --- a/fast_api/models.py +++ b/fast_api/models.py @@ -459,3 +459,8 @@ class EvaluationSetCreationBody(BaseModel): class EvaluationGroupCreationBody(BaseModel): matchingSetIds: List[StrictStr] name: StrictStr + + +class EvaluationRunCreationBody(BaseModel): + embeddingId: StrictStr + evaluationGroupId: StrictStr diff --git a/fast_api/routes/playground.py b/fast_api/routes/playground.py index 2638a03a..ddb40c3e 100644 --- a/fast_api/routes/playground.py +++ b/fast_api/routes/playground.py @@ -5,6 +5,7 @@ SearchQuestionBody, EvaluationSetCreationBody, EvaluationGroupCreationBody, + EvaluationRunCreationBody, ) from controller.playground import manager as playground_manager @@ -96,3 +97,23 @@ def get_single_evaluation_group( project_id, group_id ) return pack_json_result(evaluation_group) + + +@router.get( + "/{project_id}/evaluation-runs" +) # dependencies=[Depends(auth_manager.check_project_access_dep)] +def get_evaluation_runs(request: Request, project_id: str): + evaluation_runs = playground_manager.get_evaluation_runs(project_id) + return pack_json_result(evaluation_runs) + + +@router.post( + "/{project_id}/evaluation-runs" +) # dependencies=[Depends(auth_manager.check_project_access_dep)] +def create_evaluation_run( + request: Request, + project_id: str, + evaluation_run: EvaluationRunCreationBody = Body(...), +): + playground_manager.init_evaluation_run(project_id, evaluation_run.evaluationGroupId) + return get_silent_success() From 65bbe0f0faf3db7a78127f864fd2186af4918a7c Mon Sep 17 00:00:00 2001 From: anmarhindi Date: Thu, 16 Jan 2025 16:59:34 +0100 Subject: [PATCH 13/76] sub ref --- submodules/model | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/submodules/model b/submodules/model index 3587c7d3..74e3e05a 160000 --- a/submodules/model +++ b/submodules/model @@ -1 +1 @@ -Subproject commit 3587c7d33a1023ca500812c62f088bb734c98893 +Subproject commit 74e3e05a71e465d44f07603e0ad86066d0366be9 From fd643f6bd1bbeb43861260badbcc069e0608c2d6 Mon Sep 17 00:00:00 2001 From: anmarhindi Date: Thu, 16 Jan 2025 17:20:24 +0100 Subject: [PATCH 14/76] Add getters to playground manager --- controller/playground/manager.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/controller/playground/manager.py b/controller/playground/manager.py index b892b97a..5eca7ad1 100644 --- a/controller/playground/manager.py +++ b/controller/playground/manager.py @@ -1,4 +1,9 @@ from fast_api.routes.client_response import pack_json_result, GENERIC_FAILURE_RESPONSE +from submodules.model.business_objects import ( + evaluation_set, + evaluation_group, + evaluation_run, +) from typing import List, Any, Optional, Tuple, Dict import os import requests @@ -27,11 +32,11 @@ def create_evaluation_set(project_id: str, question: str, record_ids: List[str]) def get_evaluation_set_by_id(project_id: str, set_id: str): - pass + return evaluation_set.get(project_id, set_id) def get_evaluation_sets(project_id: str): - pass + return evaluation_set.get_all(project_id) def create_evaluation_group(project_id: str, name: str, evaluation_set_ids: List[str]): @@ -39,15 +44,15 @@ def create_evaluation_group(project_id: str, name: str, evaluation_set_ids: List def get_evaluation_group_by_id(project_id: str, group_id: str): - pass + return evaluation_group.get(project_id, group_id) def get_evaluation_groups(project_id: str): - pass + return evaluation_group.get_all(project_id) def get_evaluation_runs(project_id: str): - pass + return evaluation_run.get_all(project_id) def init_evaluation_run(project_id: str, evaluation_group_id: str): From 4b0e2541eb5d697fd990a4001621b73b7100eba0 Mon Sep 17 00:00:00 2001 From: LennartSchmidtKern Date: Fri, 17 Jan 2025 10:31:14 +0100 Subject: [PATCH 15/76] renaming. init run --- controller/playground/manager.py | 43 ++++++++++++++++++++++---------- submodules/model | 2 +- 2 files changed, 31 insertions(+), 14 deletions(-) diff --git a/controller/playground/manager.py b/controller/playground/manager.py index 5eca7ad1..b1a2de1f 100644 --- a/controller/playground/manager.py +++ b/controller/playground/manager.py @@ -1,12 +1,11 @@ -from fast_api.routes.client_response import pack_json_result, GENERIC_FAILURE_RESPONSE -from submodules.model.business_objects import ( - evaluation_set, - evaluation_group, - evaluation_run, -) from typing import List, Any, Optional, Tuple, Dict import os import requests +from submodules.model.business_objects import ( + evaluation_group as evaluation_group_db_bo, + evaluation_set as evaluation_set_db_bo, + evaluation_run as evaluation_run_db_bo, +) NEURAL_SEARCH = os.getenv("NEURAL_SEARCH") EMBEDDING_SERVICE = os.getenv("EMBEDDING_SERVICE") @@ -32,11 +31,11 @@ def create_evaluation_set(project_id: str, question: str, record_ids: List[str]) def get_evaluation_set_by_id(project_id: str, set_id: str): - return evaluation_set.get(project_id, set_id) + return evaluation_set_db_bo.get(project_id, set_id) def get_evaluation_sets(project_id: str): - return evaluation_set.get_all(project_id) + return evaluation_set_db_bo.get_all(project_id) def create_evaluation_group(project_id: str, name: str, evaluation_set_ids: List[str]): @@ -44,19 +43,37 @@ def create_evaluation_group(project_id: str, name: str, evaluation_set_ids: List def get_evaluation_group_by_id(project_id: str, group_id: str): - return evaluation_group.get(project_id, group_id) + return evaluation_group_db_bo.get(project_id, group_id) def get_evaluation_groups(project_id: str): - return evaluation_group.get_all(project_id) + return evaluation_group_db_bo.get_all(project_id) def get_evaluation_runs(project_id: str): - return evaluation_run.get_all(project_id) + return evaluation_run_db_bo.get_all(project_id) -def init_evaluation_run(project_id: str, evaluation_group_id: str): - pass +def init_evaluation_run(project_id: str, embedding_id: str, evaluation_group_id: str): + + evaluation_sets = evaluation_set_db_bo.get_by_evaluation_group_id( + project_id, evaluation_group_id + ) + questions_tensors = __get_tensors_for_texts( + project_id, + embedding_id, + [evaluation_set.question for evaluation_set in evaluation_sets], + ) + search_results = __get_most_similar_records( + project_id, embedding_id, questions_tensors, LIMIT + ) + + evaluation_results = {} + for evaluation_set, search_result in zip(evaluation_sets, search_results): + evaluation_results[evaluation_set.id] = search_result + ## ... add more + + return evaluation_results def __get_tensors_for_texts( diff --git a/submodules/model b/submodules/model index 74e3e05a..18b7d0a8 160000 --- a/submodules/model +++ b/submodules/model @@ -1 +1 @@ -Subproject commit 74e3e05a71e465d44f07603e0ad86066d0366be9 +Subproject commit 18b7d0a82ee658e8508aef99ac1b8246a623b093 From 1f0efe2a39f5db3a65596b14ebb5beb18b1b9c10 Mon Sep 17 00:00:00 2001 From: LennartSchmidtKern Date: Fri, 17 Jan 2025 10:47:12 +0100 Subject: [PATCH 16/76] improve results --- controller/playground/manager.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/controller/playground/manager.py b/controller/playground/manager.py index b1a2de1f..cdab72d1 100644 --- a/controller/playground/manager.py +++ b/controller/playground/manager.py @@ -70,8 +70,23 @@ def init_evaluation_run(project_id: str, embedding_id: str, evaluation_group_id: evaluation_results = {} for evaluation_set, search_result in zip(evaluation_sets, search_results): - evaluation_results[evaluation_set.id] = search_result - ## ... add more + + expected_record_ids = evaluation_set.record_ids + received_record_ids = [record["id"] for record in search_result] + + evaluation_by_record_id = {} + for record_id in expected_record_ids: + if record_id in received_record_ids: + evaluation_by_record_id[record_id] = {"evaluation_state": "true_p"} + else: + evaluation_by_record_id[record_id] = {"evaluation_state": "false_p"} + for record_id in received_record_ids: + if record_id not in expected_record_ids: + evaluation_by_record_id[record_id] = {"evaluation_state": "false_n"} + evaluation_results[evaluation_set.id] = { + "search_results": search_result, + "evaluation_by_record_id": evaluation_by_record_id, + } return evaluation_results From 35924b108a8fd4e1e0b45f1502fb9c41d9229425 Mon Sep 17 00:00:00 2001 From: LennartSchmidtKern Date: Fri, 17 Jan 2025 10:51:55 +0100 Subject: [PATCH 17/76] creation --- controller/playground/manager.py | 14 ++++++++++---- fast_api/models.py | 2 +- fast_api/routes/playground.py | 12 +++++++++--- 3 files changed, 20 insertions(+), 8 deletions(-) diff --git a/controller/playground/manager.py b/controller/playground/manager.py index cdab72d1..b7c96cdc 100644 --- a/controller/playground/manager.py +++ b/controller/playground/manager.py @@ -26,8 +26,10 @@ def get_search_result_for_text(project_id: str, embedding_id: str, question: str return [] -def create_evaluation_set(project_id: str, question: str, record_ids: List[str]): - pass +def create_evaluation_set( + project_id: str, question: str, record_ids: List[str], created_by: str +): + evaluation_set_db_bo.create(project_id, question, created_by, record_ids, True) def get_evaluation_set_by_id(project_id: str, set_id: str): @@ -38,8 +40,12 @@ def get_evaluation_sets(project_id: str): return evaluation_set_db_bo.get_all(project_id) -def create_evaluation_group(project_id: str, name: str, evaluation_set_ids: List[str]): - pass +def create_evaluation_group( + project_id: str, name: str, evaluation_set_ids: List[str], created_by: str +): + evaluation_group_db_bo.create( + project_id, name, created_by, evaluation_set_ids, True + ) def get_evaluation_group_by_id(project_id: str, group_id: str): diff --git a/fast_api/models.py b/fast_api/models.py index 7039720b..362dd826 100644 --- a/fast_api/models.py +++ b/fast_api/models.py @@ -457,7 +457,7 @@ class EvaluationSetCreationBody(BaseModel): class EvaluationGroupCreationBody(BaseModel): - matchingSetIds: List[StrictStr] + evaluationSetIds: List[StrictStr] name: StrictStr diff --git a/fast_api/routes/playground.py b/fast_api/routes/playground.py index ddb40c3e..55703721 100644 --- a/fast_api/routes/playground.py +++ b/fast_api/routes/playground.py @@ -39,7 +39,10 @@ def create_evaluation_set( evaluation_set: EvaluationSetCreationBody = Body(...), ): playground_manager.create_evaluation_set( - project_id, evaluation_set.question, evaluation_set.recordIds + project_id, + evaluation_set.question, + evaluation_set.recordIds, + request.state.user_id, ) return get_silent_success() @@ -73,8 +76,11 @@ def create_evaluation_group( project_id: str, evaluation_group: EvaluationGroupCreationBody = Body(...), ): - playground_manager.create_evaluation_set( - project_id, evaluation_group.name, evaluation_group.matchingSetIds + playground_manager.create_evaluation_group( + project_id, + evaluation_group.name, + evaluation_group.evaluationSetIds, + request.state.user_id, ) return get_silent_success() From cd047378770e6e88400b4fe1f37d23ae93edd483 Mon Sep 17 00:00:00 2001 From: LennartSchmidtKern Date: Fri, 17 Jan 2025 11:04:00 +0100 Subject: [PATCH 18/76] init run --- controller/playground/manager.py | 82 ++++++++++++++++++++------------ fast_api/routes/playground.py | 9 +++- 2 files changed, 59 insertions(+), 32 deletions(-) diff --git a/controller/playground/manager.py b/controller/playground/manager.py index b7c96cdc..27425490 100644 --- a/controller/playground/manager.py +++ b/controller/playground/manager.py @@ -15,6 +15,13 @@ THRESHOLD = None +class EVALUATION_RUN_STATE: + INITIATED = "INITIATED" + RUNNING = "RUNNING" + SUCCESS = "SUCCESS" + FAILED = "FAILED" + + def get_search_result_for_text(project_id: str, embedding_id: str, question: str): question_tensor = __get_tensors_for_texts(project_id, embedding_id, [question]) @@ -60,41 +67,56 @@ def get_evaluation_runs(project_id: str): return evaluation_run_db_bo.get_all(project_id) -def init_evaluation_run(project_id: str, embedding_id: str, evaluation_group_id: str): +def init_evaluation_run( + project_id: str, embedding_id: str, evaluation_group_id: str, created_by: str +): - evaluation_sets = evaluation_set_db_bo.get_by_evaluation_group_id( - project_id, evaluation_group_id - ) - questions_tensors = __get_tensors_for_texts( + evaluation_run = evaluation_run_db_bo.create( project_id, + evaluation_group_id, + created_by, embedding_id, - [evaluation_set.question for evaluation_set in evaluation_sets], - ) - search_results = __get_most_similar_records( - project_id, embedding_id, questions_tensors, LIMIT + EVALUATION_RUN_STATE.INITIATED, ) - + state = EVALUATION_RUN_STATE.RUNNING evaluation_results = {} - for evaluation_set, search_result in zip(evaluation_sets, search_results): - - expected_record_ids = evaluation_set.record_ids - received_record_ids = [record["id"] for record in search_result] - - evaluation_by_record_id = {} - for record_id in expected_record_ids: - if record_id in received_record_ids: - evaluation_by_record_id[record_id] = {"evaluation_state": "true_p"} - else: - evaluation_by_record_id[record_id] = {"evaluation_state": "false_p"} - for record_id in received_record_ids: - if record_id not in expected_record_ids: - evaluation_by_record_id[record_id] = {"evaluation_state": "false_n"} - evaluation_results[evaluation_set.id] = { - "search_results": search_result, - "evaluation_by_record_id": evaluation_by_record_id, - } - - return evaluation_results + try: + evaluation_sets = evaluation_set_db_bo.get_by_evaluation_group_id( + project_id, evaluation_group_id + ) + questions_tensors = __get_tensors_for_texts( + project_id, + [evaluation_set.question for evaluation_set in evaluation_sets], + ) + search_results = __get_most_similar_records( + project_id, embedding_id, questions_tensors, LIMIT + ) + + for evaluation_set, search_result in zip(evaluation_sets, search_results): + + expected_record_ids = evaluation_set.record_ids + received_record_ids = [record["id"] for record in search_result] + + evaluation_by_record_id = {} + for record_id in expected_record_ids: + if record_id in received_record_ids: + evaluation_by_record_id[record_id] = {"evaluation_state": "true_p"} + else: + evaluation_by_record_id[record_id] = {"evaluation_state": "false_p"} + for record_id in received_record_ids: + if record_id not in expected_record_ids: + evaluation_by_record_id[record_id] = {"evaluation_state": "false_n"} + evaluation_results[evaluation_set.id] = { + "search_results": search_result, + "evaluation_by_record_id": evaluation_by_record_id, + } + state = EVALUATION_RUN_STATE.SUCCESS + except Exception: + state = EVALUATION_RUN_STATE.FAILED + evaluation_run_db_bo.update( + evaluation_run.id, state, evaluation_results, None, True + ) + return evaluation_run def __get_tensors_for_texts( diff --git a/fast_api/routes/playground.py b/fast_api/routes/playground.py index 55703721..cc4c66b5 100644 --- a/fast_api/routes/playground.py +++ b/fast_api/routes/playground.py @@ -121,5 +121,10 @@ def create_evaluation_run( project_id: str, evaluation_run: EvaluationRunCreationBody = Body(...), ): - playground_manager.init_evaluation_run(project_id, evaluation_run.evaluationGroupId) - return get_silent_success() + evaluation_run = playground_manager.init_evaluation_run( + project_id, + evaluation_run.embeddingId, + evaluation_run.evaluationGroupId, + request.state.user_id, + ) + return evaluation_run From f178efa80865f8baafb0216e1186f5ac3339a12c Mon Sep 17 00:00:00 2001 From: LennartSchmidtKern Date: Fri, 17 Jan 2025 12:33:49 +0100 Subject: [PATCH 19/76] simple contains search --- controller/playground/manager.py | 49 ++++++++++++++++++++++++++++++++ fast_api/models.py | 6 ++++ fast_api/routes/playground.py | 19 +++++++++++++ 3 files changed, 74 insertions(+) diff --git a/controller/playground/manager.py b/controller/playground/manager.py index 27425490..ee52dde3 100644 --- a/controller/playground/manager.py +++ b/controller/playground/manager.py @@ -5,7 +5,10 @@ evaluation_group as evaluation_group_db_bo, evaluation_set as evaluation_set_db_bo, evaluation_run as evaluation_run_db_bo, + attribute as attribute_db_bo, ) +from service.search.search import resolve_extended_search +from submodules.model.util import sql_alchemy_to_dict NEURAL_SEARCH = os.getenv("NEURAL_SEARCH") EMBEDDING_SERVICE = os.getenv("EMBEDDING_SERVICE") @@ -168,3 +171,49 @@ def __get_most_similar_records( return response.json() else: return None + + +def get_records_by_content( + project_id: str, user_id: str, content: str, limit: int, offset: int +): + + filter_data = __build_contains_filter(project_id, content) + record_list = resolve_extended_search( + project_id, user_id, filter_data, limit, offset + ).record_list + record_list = sql_alchemy_to_dict(record_list, column_blacklist=["rla_data"]) + + return record_list + + +def __build_contains_filter(project_id: str, content: str): + attributes = attribute_db_bo.get_all(project_id) + + filter_each_attribute = [ + { + "RELATION": "OR", + "NEGATION": False, + "TARGET_TABLE": "RECORD", + "TARGET_COLUMN": "DATA", + "OPERATOR": "CONTAINS", + "VALUES": [f"{attribute.name}", f"{content}"], + } + for attribute in attributes + ] + + if len(filter_each_attribute) > 0: + filter_each_attribute[0]["RELATION"] = "NONE" + + final_filter = [ + { + "RELATION": "NONE", + "NEGATION": False, + "TARGET_TABLE": "RECORD", + "TARGET_COLUMN": "CATEGORY", + "OPERATOR": "EQUAL", + "VALUES": ["SCALE"], + }, + {"RELATION": "AND", "NEGATION": False, "FILTER": filter_each_attribute}, + ] + + return final_filter diff --git a/fast_api/models.py b/fast_api/models.py index 362dd826..3fec1256 100644 --- a/fast_api/models.py +++ b/fast_api/models.py @@ -464,3 +464,9 @@ class EvaluationGroupCreationBody(BaseModel): class EvaluationRunCreationBody(BaseModel): embeddingId: StrictStr evaluationGroupId: StrictStr + + +class RecordSearchContains(BaseModel): + query: StrictStr + offset: StrictInt + limit: StrictInt diff --git a/fast_api/routes/playground.py b/fast_api/routes/playground.py index cc4c66b5..bca89465 100644 --- a/fast_api/routes/playground.py +++ b/fast_api/routes/playground.py @@ -6,6 +6,7 @@ EvaluationSetCreationBody, EvaluationGroupCreationBody, EvaluationRunCreationBody, + RecordSearchContains, ) from controller.playground import manager as playground_manager @@ -128,3 +129,21 @@ def create_evaluation_run( request.state.user_id, ) return evaluation_run + + +@router.post( + "/{project_id}/record-search-contains" +) # dependencies=[Depends(auth_manager.check_project_access_dep)] +def get_record_by_content( + request: Request, + project_id: str, + record_search: RecordSearchContains = Body(...), +): + query = record_search.query + limit = record_search.limit + offset = record_search.offset + user = "52a09a36-5e3a-446a-a9b7-0104edecf62d" # request.state.user + records = playground_manager.get_records_by_content( + project_id, user, query, limit, offset + ) + return pack_json_result(records) From 74a0fc86442fa012ff66488d021081c8d35bb396 Mon Sep 17 00:00:00 2001 From: LennartSchmidtKern Date: Fri, 17 Jan 2025 13:22:34 +0100 Subject: [PATCH 20/76] user fix --- fast_api/routes/playground.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/fast_api/routes/playground.py b/fast_api/routes/playground.py index bca89465..5d85b366 100644 --- a/fast_api/routes/playground.py +++ b/fast_api/routes/playground.py @@ -9,6 +9,7 @@ RecordSearchContains, ) from controller.playground import manager as playground_manager +from controller.auth import manager as auth_manager router = APIRouter() @@ -39,11 +40,12 @@ def create_evaluation_set( project_id: str, evaluation_set: EvaluationSetCreationBody = Body(...), ): + user_id = auth_manager.get_user_id_by_info(request.state.info) playground_manager.create_evaluation_set( project_id, evaluation_set.question, evaluation_set.recordIds, - request.state.user_id, + user_id, ) return get_silent_success() @@ -77,11 +79,12 @@ def create_evaluation_group( project_id: str, evaluation_group: EvaluationGroupCreationBody = Body(...), ): + user_id = auth_manager.get_user_id_by_info(request.state.info) playground_manager.create_evaluation_group( project_id, evaluation_group.name, evaluation_group.evaluationSetIds, - request.state.user_id, + user_id, ) return get_silent_success() @@ -122,11 +125,12 @@ def create_evaluation_run( project_id: str, evaluation_run: EvaluationRunCreationBody = Body(...), ): + user_id = auth_manager.get_user_id_by_info(request.state.info) evaluation_run = playground_manager.init_evaluation_run( project_id, evaluation_run.embeddingId, evaluation_run.evaluationGroupId, - request.state.user_id, + user_id, ) return evaluation_run From fb9b97e633de922e8a68e671f7e9a0df067aee00 Mon Sep 17 00:00:00 2001 From: LennartSchmidtKern Date: Fri, 17 Jan 2025 14:23:44 +0100 Subject: [PATCH 21/76] fix search, expand --- controller/playground/manager.py | 17 +++++++++++++++-- fast_api/routes/playground.py | 2 +- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/controller/playground/manager.py b/controller/playground/manager.py index ee52dde3..847d9bcc 100644 --- a/controller/playground/manager.py +++ b/controller/playground/manager.py @@ -6,6 +6,7 @@ evaluation_set as evaluation_set_db_bo, evaluation_run as evaluation_run_db_bo, attribute as attribute_db_bo, + record as record_db_bo, ) from service.search.search import resolve_extended_search from submodules.model.util import sql_alchemy_to_dict @@ -27,11 +28,23 @@ class EVALUATION_RUN_STATE: def get_search_result_for_text(project_id: str, embedding_id: str, question: str): question_tensor = __get_tensors_for_texts(project_id, embedding_id, [question]) - if question_tensor and question_tensor[0]: - return __get_most_similar_records( + records = __get_most_similar_records( project_id, embedding_id, question_tensor[0], LIMIT ) + # make more efficient, own function + unfolded_records = [] + for record in records: + record_obj = record_db_bo.get(project_id, record["id"]) + record_dict = sql_alchemy_to_dict(record_obj) + unfolded_records.append( + { + "data": record_dict["data"], + "id": record["id"], + "score": record["score"], + } + ) + return unfolded_records else: return [] diff --git a/fast_api/routes/playground.py b/fast_api/routes/playground.py index 5d85b366..5417d3f1 100644 --- a/fast_api/routes/playground.py +++ b/fast_api/routes/playground.py @@ -29,7 +29,7 @@ def get_search_results_question( project_id, embedding_id, question ) - return pack_json_result(search_results) + return pack_json_result(search_results, wrap_for_frontend=False) @router.post( From bca7ddbe5a3e29f2fc468cdd5ba802db93297e02 Mon Sep 17 00:00:00 2001 From: LennartSchmidtKern Date: Mon, 20 Jan 2025 13:52:34 +0100 Subject: [PATCH 22/76] fix formato, add limit --- fast_api/models.py | 1 + fast_api/routes/playground.py | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/fast_api/models.py b/fast_api/models.py index 3fec1256..4fa62a01 100644 --- a/fast_api/models.py +++ b/fast_api/models.py @@ -449,6 +449,7 @@ class MissingUsersBody(BaseModel): class SearchQuestionBody(BaseModel): question: StrictStr embeddingId: StrictStr + limit: Optional[StrictInt] = None class EvaluationSetCreationBody(BaseModel): diff --git a/fast_api/routes/playground.py b/fast_api/routes/playground.py index 5417d3f1..ec32058f 100644 --- a/fast_api/routes/playground.py +++ b/fast_api/routes/playground.py @@ -24,9 +24,9 @@ def get_search_results_question( ): embedding_id = search_question.embeddingId question = search_question.question - + limit = search_question.limit search_results = playground_manager.get_search_result_for_text( - project_id, embedding_id, question + project_id, embedding_id, question, limit ) return pack_json_result(search_results, wrap_for_frontend=False) @@ -150,4 +150,4 @@ def get_record_by_content( records = playground_manager.get_records_by_content( project_id, user, query, limit, offset ) - return pack_json_result(records) + return pack_json_result(records, wrap_for_frontend=False) From ff3a840bff964b0c4f8faa506fc864009d210a03 Mon Sep 17 00:00:00 2001 From: LennartSchmidtKern Date: Mon, 20 Jan 2025 13:52:42 +0100 Subject: [PATCH 23/76] limit --- controller/playground/manager.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/controller/playground/manager.py b/controller/playground/manager.py index 847d9bcc..44a5a30b 100644 --- a/controller/playground/manager.py +++ b/controller/playground/manager.py @@ -9,12 +9,11 @@ record as record_db_bo, ) from service.search.search import resolve_extended_search -from submodules.model.util import sql_alchemy_to_dict +from submodules.model.util import sql_alchemy_to_dict, to_frontend_obj_raw NEURAL_SEARCH = os.getenv("NEURAL_SEARCH") EMBEDDING_SERVICE = os.getenv("EMBEDDING_SERVICE") -LIMIT = 10 FILTER = None THRESHOLD = None @@ -26,11 +25,13 @@ class EVALUATION_RUN_STATE: FAILED = "FAILED" -def get_search_result_for_text(project_id: str, embedding_id: str, question: str): +def get_search_result_for_text( + project_id: str, embedding_id: str, question: str, limit: int +): question_tensor = __get_tensors_for_texts(project_id, embedding_id, [question]) if question_tensor and question_tensor[0]: records = __get_most_similar_records( - project_id, embedding_id, question_tensor[0], LIMIT + project_id, embedding_id, question_tensor[0], limit ) # make more efficient, own function unfolded_records = [] @@ -194,8 +195,9 @@ def get_records_by_content( record_list = resolve_extended_search( project_id, user_id, filter_data, limit, offset ).record_list - record_list = sql_alchemy_to_dict(record_list, column_blacklist=["rla_data"]) - + record_list = to_frontend_obj_raw( + sql_alchemy_to_dict(record_list, column_blacklist=["rla_data"]) + ) return record_list From f8b16e73bacdae31f762caf0338dfa945c505e7e Mon Sep 17 00:00:00 2001 From: anmarhindi Date: Mon, 20 Jan 2025 16:35:54 +0100 Subject: [PATCH 24/76] Remove unused auth manager --- fast_api/routes/playground.py | 1 - 1 file changed, 1 deletion(-) diff --git a/fast_api/routes/playground.py b/fast_api/routes/playground.py index ec32058f..5f5e0157 100644 --- a/fast_api/routes/playground.py +++ b/fast_api/routes/playground.py @@ -9,7 +9,6 @@ RecordSearchContains, ) from controller.playground import manager as playground_manager -from controller.auth import manager as auth_manager router = APIRouter() From b419d8d48ddfe3b69eb4d2beb7d1e46e898ec49c Mon Sep 17 00:00:00 2001 From: anmarhindi Date: Mon, 20 Jan 2025 17:36:13 +0100 Subject: [PATCH 25/76] Add records batch endpoint --- fast_api/models.py | 4 ++++ fast_api/routes/playground.py | 4 +++- fast_api/routes/project_setting.py | 28 ++++++++++++++++++++++++++++ 3 files changed, 35 insertions(+), 1 deletion(-) diff --git a/fast_api/models.py b/fast_api/models.py index 4fa62a01..6bd73944 100644 --- a/fast_api/models.py +++ b/fast_api/models.py @@ -471,3 +471,7 @@ class RecordSearchContains(BaseModel): query: StrictStr offset: StrictInt limit: StrictInt + + +class RecordsBatchBody(BaseModel): + record_ids: List[StrictStr] diff --git a/fast_api/routes/playground.py b/fast_api/routes/playground.py index 5f5e0157..67d4ea09 100644 --- a/fast_api/routes/playground.py +++ b/fast_api/routes/playground.py @@ -60,7 +60,9 @@ def get_evaluation_sets( return pack_json_result(matching_sets) -@router.get("/{project_id}/evaluation-sets/{set_id}") +@router.get( + "/{project_id}/evaluation-sets/{set_id}" +) # dependencies=[Depends(auth_manager.check_project_access_dep)] def get_single_evaluation_set( request: Request, project_id: str, diff --git a/fast_api/routes/project_setting.py b/fast_api/routes/project_setting.py index 1d7b5144..b5b7a6b5 100644 --- a/fast_api/routes/project_setting.py +++ b/fast_api/routes/project_setting.py @@ -3,6 +3,7 @@ CreateNewAttributeBody, PrepareProjectExportBody, PrepareRecordExportBody, + RecordsBatchBody, UpdateAttributeBody, ) from fastapi import APIRouter, Body, Depends, Request @@ -148,6 +149,33 @@ def get_record_by_record_id( return pack_json_result(data) +@router.post( + "/{project_id}/records-batch", + dependencies=[Depends(auth_manager.check_project_access_dep)], +) +def get_records_batch( + project_id: str, + recordsBatchBody: RecordsBatchBody = Body(...), +): + results = [] + for record_id in recordsBatchBody.record_ids: + if record_id is None or record_id == "null": + continue + + record = record_manager.get_record(project_id, record_id) + + results.append( + { + "id": str(record.id), + "data": json.dumps(record.data), + "projectId": str(record.project_id), + "category": record.category, + } + ) + + return pack_json_result(results) + + @router.get( "/{project_id}/project-size", dependencies=[Depends(auth_manager.check_project_access_dep)], From 96dd695ce00ebd5ddcc6b3a9285bc3a2fff8bb06 Mon Sep 17 00:00:00 2001 From: LennartSchmidtKern Date: Mon, 20 Jan 2025 19:33:51 +0100 Subject: [PATCH 26/76] fix evaluation run --- controller/playground/manager.py | 17 ++++++++++------- fast_api/routes/playground.py | 9 ++++++--- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/controller/playground/manager.py b/controller/playground/manager.py index 44a5a30b..543ecf72 100644 --- a/controller/playground/manager.py +++ b/controller/playground/manager.py @@ -103,16 +103,18 @@ def init_evaluation_run( ) questions_tensors = __get_tensors_for_texts( project_id, + embedding_id, [evaluation_set.question for evaluation_set in evaluation_sets], ) - search_results = __get_most_similar_records( - project_id, embedding_id, questions_tensors, LIMIT - ) + search_results = [ + __get_most_similar_records(project_id, embedding_id, questions_tensor, 10) + for questions_tensor in questions_tensors + ] for evaluation_set, search_result in zip(evaluation_sets, search_results): - expected_record_ids = evaluation_set.record_ids - received_record_ids = [record["id"] for record in search_result] + expected_record_ids = [str(rid) for rid in evaluation_set.record_ids] + received_record_ids = [str(record["id"]) for record in search_result] evaluation_by_record_id = {} for record_id in expected_record_ids: @@ -123,7 +125,7 @@ def init_evaluation_run( for record_id in received_record_ids: if record_id not in expected_record_ids: evaluation_by_record_id[record_id] = {"evaluation_state": "false_n"} - evaluation_results[evaluation_set.id] = { + evaluation_results[str(evaluation_set.id)] = { "search_results": search_result, "evaluation_by_record_id": evaluation_by_record_id, } @@ -131,7 +133,7 @@ def init_evaluation_run( except Exception: state = EVALUATION_RUN_STATE.FAILED evaluation_run_db_bo.update( - evaluation_run.id, state, evaluation_results, None, True + project_id, evaluation_run.id, state, evaluation_results, None, True ) return evaluation_run @@ -181,6 +183,7 @@ def __get_most_similar_records( "threshold": threshold, }, ) + print("response.ok", response.ok, flush=True) if response.ok: return response.json() else: diff --git a/fast_api/routes/playground.py b/fast_api/routes/playground.py index 67d4ea09..8d1c36d9 100644 --- a/fast_api/routes/playground.py +++ b/fast_api/routes/playground.py @@ -1,6 +1,9 @@ from fastapi import APIRouter, Depends, Request, Body from controller.auth import manager as auth_manager -from fast_api.routes.client_response import pack_json_result, get_silent_success +from fast_api.routes.client_response import ( + pack_json_result, + get_silent_success, +) from fast_api.models import ( SearchQuestionBody, EvaluationSetCreationBody, @@ -127,13 +130,13 @@ def create_evaluation_run( evaluation_run: EvaluationRunCreationBody = Body(...), ): user_id = auth_manager.get_user_id_by_info(request.state.info) - evaluation_run = playground_manager.init_evaluation_run( + playground_manager.init_evaluation_run( project_id, evaluation_run.embeddingId, evaluation_run.evaluationGroupId, user_id, ) - return evaluation_run + return get_silent_success() @router.post( From 2205612cfac6aef86d10b2b8d86e66ca4f939a1f Mon Sep 17 00:00:00 2001 From: anmarhindi Date: Tue, 21 Jan 2025 11:14:21 +0100 Subject: [PATCH 27/76] Add Evaluation set by group endpoint --- controller/playground/manager.py | 6 ++++++ fast_api/routes/playground.py | 16 ++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/controller/playground/manager.py b/controller/playground/manager.py index 543ecf72..f971f983 100644 --- a/controller/playground/manager.py +++ b/controller/playground/manager.py @@ -64,6 +64,12 @@ def get_evaluation_sets(project_id: str): return evaluation_set_db_bo.get_all(project_id) +def get_evaluation_sets_by_group_id(project_id: str, evaluation_group_id: str): + return evaluation_set_db_bo.get_by_evaluation_group_id( + project_id, evaluation_group_id + ) + + def create_evaluation_group( project_id: str, name: str, evaluation_set_ids: List[str], created_by: str ): diff --git a/fast_api/routes/playground.py b/fast_api/routes/playground.py index 8d1c36d9..568b52fd 100644 --- a/fast_api/routes/playground.py +++ b/fast_api/routes/playground.py @@ -75,6 +75,22 @@ def get_single_evaluation_set( return pack_json_result(matching_set) +@router.get( + "/{project_id}/evaluation-sets-by-group/{evaluation_group_id}", + dependencies=[Depends(auth_manager.check_project_access_dep)], +) +def get_evaluation_sets_batch( + request: Request, + project_id: str, + evaluation_group_id: str, +): + evaluation_sets = playground_manager.get_evaluation_sets_by_group_id( + project_id, evaluation_group_id + ) + + return pack_json_result(evaluation_sets) + + @router.post( "/{project_id}/evaluation-groups" ) # dependencies=[Depends(auth_manager.check_project_access_dep)] From f4823dac1ee353d9ead3edd88cc0de6155ff6e64 Mon Sep 17 00:00:00 2001 From: anmarhindi Date: Tue, 21 Jan 2025 13:45:03 +0100 Subject: [PATCH 28/76] Add auth check for project access endpoints --- fast_api/routes/playground.py | 45 +++++++++++++++++++++-------------- 1 file changed, 27 insertions(+), 18 deletions(-) diff --git a/fast_api/routes/playground.py b/fast_api/routes/playground.py index 568b52fd..4bfce2b4 100644 --- a/fast_api/routes/playground.py +++ b/fast_api/routes/playground.py @@ -17,8 +17,9 @@ @router.post( - "/{project_id}/search" -) # dependencies=[Depends(auth_manager.check_project_access_dep)] + "/{project_id}/search", + dependencies=[Depends(auth_manager.check_project_access_dep)], +) def get_search_results_question( request: Request, project_id: str, @@ -35,8 +36,9 @@ def get_search_results_question( @router.post( - "/{project_id}/evaluation-sets" -) # dependencies=[Depends(auth_manager.check_project_access_dep)] + "/{project_id}/evaluation-sets", + dependencies=[Depends(auth_manager.check_project_access_dep)], +) def create_evaluation_set( request: Request, project_id: str, @@ -53,8 +55,9 @@ def create_evaluation_set( @router.get( - "/{project_id}/evaluation-sets" -) # dependencies=[Depends(auth_manager.check_project_access_dep)] + "/{project_id}/evaluation-sets", + dependencies=[Depends(auth_manager.check_project_access_dep)], +) def get_evaluation_sets( request: Request, project_id: str, @@ -64,8 +67,9 @@ def get_evaluation_sets( @router.get( - "/{project_id}/evaluation-sets/{set_id}" -) # dependencies=[Depends(auth_manager.check_project_access_dep)] + "/{project_id}/evaluation-sets/{set_id}", + dependencies=[Depends(auth_manager.check_project_access_dep)], +) def get_single_evaluation_set( request: Request, project_id: str, @@ -92,8 +96,9 @@ def get_evaluation_sets_batch( @router.post( - "/{project_id}/evaluation-groups" -) # dependencies=[Depends(auth_manager.check_project_access_dep)] + "/{project_id}/evaluation-groups", + dependencies=[Depends(auth_manager.check_project_access_dep)], +) def create_evaluation_group( request: Request, project_id: str, @@ -110,8 +115,9 @@ def create_evaluation_group( @router.get( - "/{project_id}/evaluation-groups" -) # dependencies=[Depends(auth_manager.check_project_access_dep)] + "/{project_id}/evaluation-groups", + dependencies=[Depends(auth_manager.check_project_access_dep)], +) def get_evaluation_groups(request: Request, project_id: str): evaluation_groups = playground_manager.get_evaluation_groups(project_id) return pack_json_result(evaluation_groups) @@ -130,16 +136,18 @@ def get_single_evaluation_group( @router.get( - "/{project_id}/evaluation-runs" -) # dependencies=[Depends(auth_manager.check_project_access_dep)] + "/{project_id}/evaluation-runs", + dependencies=[Depends(auth_manager.check_project_access_dep)], +) def get_evaluation_runs(request: Request, project_id: str): evaluation_runs = playground_manager.get_evaluation_runs(project_id) return pack_json_result(evaluation_runs) @router.post( - "/{project_id}/evaluation-runs" -) # dependencies=[Depends(auth_manager.check_project_access_dep)] + "/{project_id}/evaluation-runs", + dependencies=[Depends(auth_manager.check_project_access_dep)], +) def create_evaluation_run( request: Request, project_id: str, @@ -156,8 +164,9 @@ def create_evaluation_run( @router.post( - "/{project_id}/record-search-contains" -) # dependencies=[Depends(auth_manager.check_project_access_dep)] + "/{project_id}/record-search-contains", + dependencies=[Depends(auth_manager.check_project_access_dep)], +) def get_record_by_content( request: Request, project_id: str, From 8ccc7daa18fdafe3362ec78e203143a34e7333db Mon Sep 17 00:00:00 2001 From: LennartSchmidtKern Date: Tue, 21 Jan 2025 15:00:21 +0100 Subject: [PATCH 29/76] delete endpoints --- controller/playground/manager.py | 8 ++++++++ fast_api/models.py | 8 ++++++++ fast_api/routes/playground.py | 30 ++++++++++++++++++++++++++++++ submodules/model | 2 +- 4 files changed, 47 insertions(+), 1 deletion(-) diff --git a/controller/playground/manager.py b/controller/playground/manager.py index 543ecf72..15fc04de 100644 --- a/controller/playground/manager.py +++ b/controller/playground/manager.py @@ -56,6 +56,10 @@ def create_evaluation_set( evaluation_set_db_bo.create(project_id, question, created_by, record_ids, True) +def delete_evaluation_sets(project_id: str, set_ids: str): + evaluation_set_db_bo.delete_all(project_id, set_ids, True) + + def get_evaluation_set_by_id(project_id: str, set_id: str): return evaluation_set_db_bo.get(project_id, set_id) @@ -72,6 +76,10 @@ def create_evaluation_group( ) +def delete_evaluation_groups(project_id: str, group_ids: str): + evaluation_group_db_bo.delete_all(project_id, group_ids, True) + + def get_evaluation_group_by_id(project_id: str, group_id: str): return evaluation_group_db_bo.get(project_id, group_id) diff --git a/fast_api/models.py b/fast_api/models.py index 6bd73944..958efbfd 100644 --- a/fast_api/models.py +++ b/fast_api/models.py @@ -457,11 +457,19 @@ class EvaluationSetCreationBody(BaseModel): recordIds: List[StrictStr] +class EvaluationSetDeletionBody(BaseModel): + evaluationSetIds: List[StrictStr] + + class EvaluationGroupCreationBody(BaseModel): evaluationSetIds: List[StrictStr] name: StrictStr +class EvaluationGroupDeletionBody(BaseModel): + evaluationGroupIds: List[StrictStr] + + class EvaluationRunCreationBody(BaseModel): embeddingId: StrictStr evaluationGroupId: StrictStr diff --git a/fast_api/routes/playground.py b/fast_api/routes/playground.py index 8d1c36d9..94390477 100644 --- a/fast_api/routes/playground.py +++ b/fast_api/routes/playground.py @@ -10,6 +10,8 @@ EvaluationGroupCreationBody, EvaluationRunCreationBody, RecordSearchContains, + EvaluationSetDeletionBody, + EvaluationGroupDeletionBody, ) from controller.playground import manager as playground_manager @@ -52,6 +54,20 @@ def create_evaluation_set( return get_silent_success() +@router.delete( + "/{project_id}/evaluation-sets" +) # dependencies=[Depends(auth_manager.check_project_access_dep)] +def delete_evaluation_set( + request: Request, + project_id: str, + evaluation_set: EvaluationSetDeletionBody = Body(...), +): + playground_manager.delete_evaluation_sets( + project_id, evaluation_set.evaluationSetIds + ) + return get_silent_success() + + @router.get( "/{project_id}/evaluation-sets" ) # dependencies=[Depends(auth_manager.check_project_access_dep)] @@ -93,6 +109,20 @@ def create_evaluation_group( return get_silent_success() +@router.delete( + "/{project_id}/evaluation-groups" +) # dependencies=[Depends(auth_manager.check_project_access_dep)] +def delete_evaluation_group( + request: Request, + project_id: str, + evaluation_group: EvaluationGroupDeletionBody = Body(...), +): + playground_manager.delete_evaluation_groups( + project_id, evaluation_group.evaluationGroupIds + ) + return get_silent_success() + + @router.get( "/{project_id}/evaluation-groups" ) # dependencies=[Depends(auth_manager.check_project_access_dep)] diff --git a/submodules/model b/submodules/model index 18b7d0a8..f52dc80c 160000 --- a/submodules/model +++ b/submodules/model @@ -1 +1 @@ -Subproject commit 18b7d0a82ee658e8508aef99ac1b8246a623b093 +Subproject commit f52dc80c38bb4df25620768a240c987b48c35bca From 5cda5930e6ba7f8105befb4731d1825427624594 Mon Sep 17 00:00:00 2001 From: Lina Date: Tue, 21 Jan 2025 15:29:10 +0100 Subject: [PATCH 30/76] Get evaluation run by id --- controller/playground/manager.py | 4 ++++ fast_api/routes/playground.py | 10 ++++++++++ 2 files changed, 14 insertions(+) diff --git a/controller/playground/manager.py b/controller/playground/manager.py index f971f983..6a020832 100644 --- a/controller/playground/manager.py +++ b/controller/playground/manager.py @@ -90,6 +90,10 @@ def get_evaluation_runs(project_id: str): return evaluation_run_db_bo.get_all(project_id) +def get_evaluation_run_by_id(project_id: str, run_id: str): + return evaluation_run_db_bo.get(project_id, run_id) + + def init_evaluation_run( project_id: str, embedding_id: str, evaluation_group_id: str, created_by: str ): diff --git a/fast_api/routes/playground.py b/fast_api/routes/playground.py index 4bfce2b4..3b4ddc64 100644 --- a/fast_api/routes/playground.py +++ b/fast_api/routes/playground.py @@ -144,6 +144,16 @@ def get_evaluation_runs(request: Request, project_id: str): return pack_json_result(evaluation_runs) +@router.get("/{project_id}/evaluation-runs/{run_id}") +def get_single_evaluation_run( + request: Request, + project_id: str, + run_id: str, +): + evaluation_run = playground_manager.get_evaluation_run_by_id(project_id, run_id) + return pack_json_result(evaluation_run) + + @router.post( "/{project_id}/evaluation-runs", dependencies=[Depends(auth_manager.check_project_access_dep)], From c31ec8fa29b6d15c947c26239a77dbe3c209a844 Mon Sep 17 00:00:00 2001 From: LennartSchmidtKern Date: Tue, 21 Jan 2025 16:13:36 +0100 Subject: [PATCH 31/76] change result data --- controller/playground/manager.py | 35 ++++++++++++++++++++++++-------- 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/controller/playground/manager.py b/controller/playground/manager.py index bb3db891..2c8c3722 100644 --- a/controller/playground/manager.py +++ b/controller/playground/manager.py @@ -110,7 +110,7 @@ def init_evaluation_run( EVALUATION_RUN_STATE.INITIATED, ) state = EVALUATION_RUN_STATE.RUNNING - evaluation_results = {} + evaluation_results = [] try: evaluation_sets = evaluation_set_db_bo.get_by_evaluation_group_id( project_id, evaluation_group_id @@ -130,19 +130,38 @@ def init_evaluation_run( expected_record_ids = [str(rid) for rid in evaluation_set.record_ids] received_record_ids = [str(record["id"]) for record in search_result] - evaluation_by_record_id = {} + true_positives = [] + false_positives = [] + false_negatives = [] for record_id in expected_record_ids: if record_id in received_record_ids: - evaluation_by_record_id[record_id] = {"evaluation_state": "true_p"} + true_positives.append(record_id) else: - evaluation_by_record_id[record_id] = {"evaluation_state": "false_p"} + false_positives.append(record_id) + for record_id in received_record_ids: if record_id not in expected_record_ids: - evaluation_by_record_id[record_id] = {"evaluation_state": "false_n"} - evaluation_results[str(evaluation_set.id)] = { - "search_results": search_result, - "evaluation_by_record_id": evaluation_by_record_id, + false_negatives.append(record_id) + + result = { + "evaluation_set_id": str(evaluation_set.id), + "true_positives": to_frontend_obj_raw( + sql_alchemy_to_dict( + record_db_bo.get_by_record_ids(project_id, true_positives) + ) + ), + "false_positives": to_frontend_obj_raw( + sql_alchemy_to_dict( + record_db_bo.get_by_record_ids(project_id, false_positives) + ) + ), + "false_negatives": to_frontend_obj_raw( + sql_alchemy_to_dict( + record_db_bo.get_by_record_ids(project_id, false_negatives) + ) + ), } + evaluation_results.append(result) state = EVALUATION_RUN_STATE.SUCCESS except Exception: state = EVALUATION_RUN_STATE.FAILED From b1bcb8f8cede119d736c43393e643897f6007f42 Mon Sep 17 00:00:00 2001 From: LennartSchmidtKern Date: Tue, 21 Jan 2025 17:14:40 +0100 Subject: [PATCH 32/76] snake case results --- controller/playground/manager.py | 9 ++++++++- fast_api/routes/playground.py | 2 +- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/controller/playground/manager.py b/controller/playground/manager.py index b2f42080..844df205 100644 --- a/controller/playground/manager.py +++ b/controller/playground/manager.py @@ -95,7 +95,14 @@ def get_evaluation_groups(project_id: str): def get_evaluation_runs(project_id: str): - return evaluation_run_db_bo.get_all(project_id) + evaluation_runs_objects = evaluation_run_db_bo.get_all(project_id) + evaluation_runs = [] + for evaluation_run in evaluation_runs_objects: + results = to_frontend_obj_raw(evaluation_run.results) + evaluation_runs.append( + {**sql_alchemy_to_dict(evaluation_run, True), "results": results} + ) + return evaluation_runs def get_evaluation_run_by_id(project_id: str, run_id: str): diff --git a/fast_api/routes/playground.py b/fast_api/routes/playground.py index e5c1ef06..9fe9b28c 100644 --- a/fast_api/routes/playground.py +++ b/fast_api/routes/playground.py @@ -171,7 +171,7 @@ def get_single_evaluation_group( ) def get_evaluation_runs(request: Request, project_id: str): evaluation_runs = playground_manager.get_evaluation_runs(project_id) - return pack_json_result(evaluation_runs) + return pack_json_result(evaluation_runs, wrap_for_frontend=False) @router.get("/{project_id}/evaluation-runs/{run_id}") From af6c4d3df8be0c4298cc1d0c91c61b6ac30a0e08 Mon Sep 17 00:00:00 2001 From: LennartSchmidtKern Date: Thu, 23 Jan 2025 15:13:34 +0100 Subject: [PATCH 33/76] fix naming --- controller/playground/manager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/controller/playground/manager.py b/controller/playground/manager.py index 844df205..cf6835ce 100644 --- a/controller/playground/manager.py +++ b/controller/playground/manager.py @@ -148,11 +148,11 @@ def init_evaluation_run( if record_id in received_record_ids: true_positives.append(record_id) else: - false_positives.append(record_id) + false_negatives.append(record_id) for record_id in received_record_ids: if record_id not in expected_record_ids: - false_negatives.append(record_id) + false_positives.append(record_id) result = { "evaluation_set_id": str(evaluation_set.id), From ffb1661455cca3b80e927ee7cba3c716a3be369d Mon Sep 17 00:00:00 2001 From: LennartSchmidtKern Date: Thu, 23 Jan 2025 17:21:39 +0100 Subject: [PATCH 34/76] filter --- controller/playground/manager.py | 12 +++++++++--- fast_api/models.py | 1 + fast_api/routes/playground.py | 3 ++- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/controller/playground/manager.py b/controller/playground/manager.py index cf6835ce..9abb7b56 100644 --- a/controller/playground/manager.py +++ b/controller/playground/manager.py @@ -26,12 +26,17 @@ class EVALUATION_RUN_STATE: def get_search_result_for_text( - project_id: str, embedding_id: str, question: str, limit: int + project_id: str, + embedding_id: str, + question: str, + limit: int, + filter=None, + threshold=None, ): question_tensor = __get_tensors_for_texts(project_id, embedding_id, [question]) if question_tensor and question_tensor[0]: records = __get_most_similar_records( - project_id, embedding_id, question_tensor[0], limit + project_id, embedding_id, question_tensor[0], limit, filter, threshold ) # make more efficient, own function unfolded_records = [] @@ -106,7 +111,8 @@ def get_evaluation_runs(project_id: str): def get_evaluation_run_by_id(project_id: str, run_id: str): - return evaluation_run_db_bo.get(project_id, run_id) + evaluation_run = evaluation_run_db_bo.get(project_id, run_id) + evaluation_run_object = sql_alchemy_to_dict(evaluation_run, False) def init_evaluation_run( diff --git a/fast_api/models.py b/fast_api/models.py index 958efbfd..92041e73 100644 --- a/fast_api/models.py +++ b/fast_api/models.py @@ -450,6 +450,7 @@ class SearchQuestionBody(BaseModel): question: StrictStr embeddingId: StrictStr limit: Optional[StrictInt] = None + filter: Optional[List[Dict]] = None class EvaluationSetCreationBody(BaseModel): diff --git a/fast_api/routes/playground.py b/fast_api/routes/playground.py index 9fe9b28c..b28fc2cf 100644 --- a/fast_api/routes/playground.py +++ b/fast_api/routes/playground.py @@ -30,8 +30,9 @@ def get_search_results_question( embedding_id = search_question.embeddingId question = search_question.question limit = search_question.limit + filter = search_question.filter search_results = playground_manager.get_search_result_for_text( - project_id, embedding_id, question, limit + project_id, embedding_id, question, limit, filter ) return pack_json_result(search_results, wrap_for_frontend=False) From 5a163f6e5f0e2a2cfeeb86ebc4e86b99af6354c0 Mon Sep 17 00:00:00 2001 From: Lina Date: Fri, 24 Jan 2025 09:39:35 +0100 Subject: [PATCH 35/76] Fixed evaluation run by id request --- controller/playground/manager.py | 1 + 1 file changed, 1 insertion(+) diff --git a/controller/playground/manager.py b/controller/playground/manager.py index 9abb7b56..ce185222 100644 --- a/controller/playground/manager.py +++ b/controller/playground/manager.py @@ -113,6 +113,7 @@ def get_evaluation_runs(project_id: str): def get_evaluation_run_by_id(project_id: str, run_id: str): evaluation_run = evaluation_run_db_bo.get(project_id, run_id) evaluation_run_object = sql_alchemy_to_dict(evaluation_run, False) + return evaluation_run_object def init_evaluation_run( From 7a81b47aef1de4af41b1713f21761ad76d8d5d8f Mon Sep 17 00:00:00 2001 From: Lina Date: Fri, 24 Jan 2025 11:57:48 +0100 Subject: [PATCH 36/76] Delete evaluation runs --- controller/playground/manager.py | 4 ++++ fast_api/models.py | 4 ++++ fast_api/routes/playground.py | 16 ++++++++++++++++ submodules/model | 2 +- 4 files changed, 25 insertions(+), 1 deletion(-) diff --git a/controller/playground/manager.py b/controller/playground/manager.py index ce185222..e84efedb 100644 --- a/controller/playground/manager.py +++ b/controller/playground/manager.py @@ -99,6 +99,10 @@ def get_evaluation_groups(project_id: str): return evaluation_group_db_bo.get_all(project_id) +def delete_evaluation_runs(project_id: str, run_ids: str): + return evaluation_run_db_bo.delete_all(project_id, run_ids, True) + + def get_evaluation_runs(project_id: str): evaluation_runs_objects = evaluation_run_db_bo.get_all(project_id) evaluation_runs = [] diff --git a/fast_api/models.py b/fast_api/models.py index 92041e73..a0e1e9ec 100644 --- a/fast_api/models.py +++ b/fast_api/models.py @@ -484,3 +484,7 @@ class RecordSearchContains(BaseModel): class RecordsBatchBody(BaseModel): record_ids: List[StrictStr] + + +class EvaluationRunDeletionBody(BaseModel): + evaluationRunIds: List[StrictStr] diff --git a/fast_api/routes/playground.py b/fast_api/routes/playground.py index b28fc2cf..148ad957 100644 --- a/fast_api/routes/playground.py +++ b/fast_api/routes/playground.py @@ -5,6 +5,7 @@ get_silent_success, ) from fast_api.models import ( + EvaluationRunDeletionBody, SearchQuestionBody, EvaluationSetCreationBody, EvaluationGroupCreationBody, @@ -221,3 +222,18 @@ def get_record_by_content( project_id, user, query, limit, offset ) return pack_json_result(records, wrap_for_frontend=False) + + +@router.delete( + "/{project_id}/evaluation-runs", + dependencies=[Depends(auth_manager.check_project_access_dep)], +) +def delete_evaluation_run( + request: Request, + project_id: str, + evaluation_set: EvaluationRunDeletionBody = Body(...), +): + playground_manager.delete_evaluation_runs( + project_id, evaluation_set.evaluationRunIds + ) + return get_silent_success() diff --git a/submodules/model b/submodules/model index f52dc80c..ccf85a6f 160000 --- a/submodules/model +++ b/submodules/model @@ -1 +1 @@ -Subproject commit f52dc80c38bb4df25620768a240c987b48c35bca +Subproject commit ccf85a6fdebcaf2cac232bfcf553a834e9acb2ba From da5440c413986a02a0f6522e509aa60c549e0ccf Mon Sep 17 00:00:00 2001 From: LennartSchmidtKern Date: Mon, 27 Jan 2025 16:50:38 +0100 Subject: [PATCH 37/76] fix recrods --- controller/playground/manager.py | 6 +++++- fast_api/routes/playground.py | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/controller/playground/manager.py b/controller/playground/manager.py index e84efedb..3ae10261 100644 --- a/controller/playground/manager.py +++ b/controller/playground/manager.py @@ -116,7 +116,11 @@ def get_evaluation_runs(project_id: str): def get_evaluation_run_by_id(project_id: str, run_id: str): evaluation_run = evaluation_run_db_bo.get(project_id, run_id) - evaluation_run_object = sql_alchemy_to_dict(evaluation_run, False) + results = to_frontend_obj_raw(evaluation_run.results) + evaluation_run_object = { + **sql_alchemy_to_dict(evaluation_run, True), + "results": results, + } return evaluation_run_object diff --git a/fast_api/routes/playground.py b/fast_api/routes/playground.py index 148ad957..680619e7 100644 --- a/fast_api/routes/playground.py +++ b/fast_api/routes/playground.py @@ -183,7 +183,7 @@ def get_single_evaluation_run( run_id: str, ): evaluation_run = playground_manager.get_evaluation_run_by_id(project_id, run_id) - return pack_json_result(evaluation_run) + return pack_json_result(evaluation_run, wrap_for_frontend=False) @router.post( From 40452578065075b19e87cd122fa597e341c70dfe Mon Sep 17 00:00:00 2001 From: LennartSchmidtKern Date: Tue, 28 Jan 2025 10:50:22 +0100 Subject: [PATCH 38/76] concurrency and limits --- controller/playground/manager.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/controller/playground/manager.py b/controller/playground/manager.py index 3ae10261..f0ef39d6 100644 --- a/controller/playground/manager.py +++ b/controller/playground/manager.py @@ -10,6 +10,7 @@ ) from service.search.search import resolve_extended_search from submodules.model.util import sql_alchemy_to_dict, to_frontend_obj_raw +from concurrent.futures import ThreadPoolExecutor NEURAL_SEARCH = os.getenv("NEURAL_SEARCH") EMBEDDING_SERVICE = os.getenv("EMBEDDING_SERVICE") @@ -146,10 +147,20 @@ def init_evaluation_run( embedding_id, [evaluation_set.question for evaluation_set in evaluation_sets], ) - search_results = [ - __get_most_similar_records(project_id, embedding_id, questions_tensor, 10) - for questions_tensor in questions_tensors - ] + + with ThreadPoolExecutor(max_workers=min(10, len(evaluation_sets))) as executor: + futures = [ + executor.submit( + __get_most_similar_records, + project_id, + embedding_id, + questions_tensors[index], + len(evaluation_set.record_ids), + ) + for index, evaluation_set in enumerate(evaluation_sets) + ] + + search_results = [future.result() for future in futures] for evaluation_set, search_result in zip(evaluation_sets, search_results): From 3e2333bf6033fcbdd6a60299a4938eb5cc6d3644 Mon Sep 17 00:00:00 2001 From: LennartSchmidtKern Date: Tue, 28 Jan 2025 15:07:14 +0100 Subject: [PATCH 39/76] threshold --- controller/playground/manager.py | 1 - fast_api/models.py | 1 + fast_api/routes/playground.py | 3 ++- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/controller/playground/manager.py b/controller/playground/manager.py index f0ef39d6..c6bc58d1 100644 --- a/controller/playground/manager.py +++ b/controller/playground/manager.py @@ -253,7 +253,6 @@ def __get_most_similar_records( "threshold": threshold, }, ) - print("response.ok", response.ok, flush=True) if response.ok: return response.json() else: diff --git a/fast_api/models.py b/fast_api/models.py index a0e1e9ec..9eaeb3a8 100644 --- a/fast_api/models.py +++ b/fast_api/models.py @@ -451,6 +451,7 @@ class SearchQuestionBody(BaseModel): embeddingId: StrictStr limit: Optional[StrictInt] = None filter: Optional[List[Dict]] = None + threshold: Optional[StrictFloat] = None class EvaluationSetCreationBody(BaseModel): diff --git a/fast_api/routes/playground.py b/fast_api/routes/playground.py index 680619e7..21c613b7 100644 --- a/fast_api/routes/playground.py +++ b/fast_api/routes/playground.py @@ -32,8 +32,9 @@ def get_search_results_question( question = search_question.question limit = search_question.limit filter = search_question.filter + threshold = search_question.threshold search_results = playground_manager.get_search_result_for_text( - project_id, embedding_id, question, limit, filter + project_id, embedding_id, question, limit, filter, threshold ) return pack_json_result(search_results, wrap_for_frontend=False) From 44f2a8874f9b52ac16feed29c949528ad369737f Mon Sep 17 00:00:00 2001 From: LennartSchmidtKern Date: Tue, 28 Jan 2025 15:23:16 +0100 Subject: [PATCH 40/76] threshold run --- controller/playground/manager.py | 13 +++++++++---- fast_api/models.py | 1 + fast_api/routes/playground.py | 1 + 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/controller/playground/manager.py b/controller/playground/manager.py index c6bc58d1..43d4e5af 100644 --- a/controller/playground/manager.py +++ b/controller/playground/manager.py @@ -15,8 +15,7 @@ NEURAL_SEARCH = os.getenv("NEURAL_SEARCH") EMBEDDING_SERVICE = os.getenv("EMBEDDING_SERVICE") -FILTER = None -THRESHOLD = None +EVALUATION_RUN_LIMIT_DEFAULT = 100 class EVALUATION_RUN_STATE: @@ -126,7 +125,11 @@ def get_evaluation_run_by_id(project_id: str, run_id: str): def init_evaluation_run( - project_id: str, embedding_id: str, evaluation_group_id: str, created_by: str + project_id: str, + embedding_id: str, + evaluation_group_id: str, + created_by: str, + threshold: float, ): evaluation_run = evaluation_run_db_bo.create( @@ -155,7 +158,9 @@ def init_evaluation_run( project_id, embedding_id, questions_tensors[index], - len(evaluation_set.record_ids), + EVALUATION_RUN_LIMIT_DEFAULT, + None, + threshold, ) for index, evaluation_set in enumerate(evaluation_sets) ] diff --git a/fast_api/models.py b/fast_api/models.py index 9eaeb3a8..7bf695bf 100644 --- a/fast_api/models.py +++ b/fast_api/models.py @@ -475,6 +475,7 @@ class EvaluationGroupDeletionBody(BaseModel): class EvaluationRunCreationBody(BaseModel): embeddingId: StrictStr evaluationGroupId: StrictStr + threshold: StrictFloat class RecordSearchContains(BaseModel): diff --git a/fast_api/routes/playground.py b/fast_api/routes/playground.py index 21c613b7..6a82432d 100644 --- a/fast_api/routes/playground.py +++ b/fast_api/routes/playground.py @@ -202,6 +202,7 @@ def create_evaluation_run( evaluation_run.embeddingId, evaluation_run.evaluationGroupId, user_id, + evaluation_run.threshold, ) return get_silent_success() From 8b90ceb26e5379692cb6c639a93b129c400f0389 Mon Sep 17 00:00:00 2001 From: LennartSchmidtKern Date: Tue, 28 Jan 2025 15:36:48 +0100 Subject: [PATCH 41/76] user id fix --- fast_api/routes/playground.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fast_api/routes/playground.py b/fast_api/routes/playground.py index 6a82432d..4bc787d5 100644 --- a/fast_api/routes/playground.py +++ b/fast_api/routes/playground.py @@ -219,9 +219,9 @@ def get_record_by_content( query = record_search.query limit = record_search.limit offset = record_search.offset - user = "52a09a36-5e3a-446a-a9b7-0104edecf62d" # request.state.user + user_id = auth_manager.get_user_id_by_info(request.state.info) records = playground_manager.get_records_by_content( - project_id, user, query, limit, offset + project_id, user_id, query, limit, offset ) return pack_json_result(records, wrap_for_frontend=False) From dc9d467af587f1a7a233bc6b2a6bebba78929b8d Mon Sep 17 00:00:00 2001 From: LennartSchmidtKern Date: Thu, 30 Jan 2025 15:21:09 +0100 Subject: [PATCH 42/76] reformulation prompt --- controller/playground/manager.py | 7 ++++++ controller/playground/reformulation.py | 33 ++++++++++++++++++++++++++ fast_api/models.py | 4 ++++ fast_api/routes/playground.py | 16 +++++++++++++ 4 files changed, 60 insertions(+) create mode 100644 controller/playground/reformulation.py diff --git a/controller/playground/manager.py b/controller/playground/manager.py index 43d4e5af..d641fe0d 100644 --- a/controller/playground/manager.py +++ b/controller/playground/manager.py @@ -11,6 +11,7 @@ from service.search.search import resolve_extended_search from submodules.model.util import sql_alchemy_to_dict, to_frontend_obj_raw from concurrent.futures import ThreadPoolExecutor +from .reformulation import REFORMULATION_PROMPT NEURAL_SEARCH = os.getenv("NEURAL_SEARCH") EMBEDDING_SERVICE = os.getenv("EMBEDDING_SERVICE") @@ -309,3 +310,9 @@ def __build_contains_filter(project_id: str, content: str): ] return final_filter + + +def get_question_reformulation(question: str): + + reformulation_dict = {"reformulation": "Test 123"} + return reformulation_dict diff --git a/controller/playground/reformulation.py b/controller/playground/reformulation.py new file mode 100644 index 00000000..5f52a362 --- /dev/null +++ b/controller/playground/reformulation.py @@ -0,0 +1,33 @@ +REFORMULATION_PROMPT = """Generate a refined and optimized reformulation of a given question, ensuring it aligns better with the user's search intent and maximizes result relevance for RAG. The reformulated question should maintain the original meaning while improving clarity, specificity, and contextual precision. + +# Important Guidelines + +- **Enhanced Clarity**: Improve sentence structure and wording for better readability. +- **Intent Alignment**: Ensure the reformulated question accurately reflects the user's underlying intent. +- **Keyword Optimization**: Retain and enhance relevant keywords for improved search efficiency. +- **Context Preservation**: Maintain all necessary contextual details while removing ambiguity. +- **Concise & Precise**: Avoid unnecessary words while keeping the question well-structured. +- **Neutral Tone**: Ensure the tone remains neutral and professional. +- **Variant Suggestions**: If applicable, suggest alternative phrasings for different perspectives. +- **Avoid Unnecessary Expansions**: Do not introduce irrelevant details that alter the original intent. +- **Question Type Awareness**: Recognize if the question seeks factual, comparative, or exploratory information and optimize accordingly. +- **No Change in Core Meaning**: Ensure the fundamental intent and topic of the question remain intact. + +# Steps + +1. Analyze the original question to identify key intent and meaning. +2. Reformulate it with improved clarity, precision, and structure. +3. Optimize for search relevance while preserving the original inquiry’s core message. +4. Provide alternative phrasings if they offer significant improvement. +5. Ensure the final output is concise, neutral, and intent-aligned. + +# Output Format +Only the pure valid json with key "reformulation" and value as the improved version of the original question. +```json +{ + "reformulation": "", +} + +# Notes + +A poorly reformulated question may lead to irrelevant or misleading results. It is essential to preserve intent while enhancing clarity and relevance.""" diff --git a/fast_api/models.py b/fast_api/models.py index 7bf695bf..0a16d7c0 100644 --- a/fast_api/models.py +++ b/fast_api/models.py @@ -490,3 +490,7 @@ class RecordsBatchBody(BaseModel): class EvaluationRunDeletionBody(BaseModel): evaluationRunIds: List[StrictStr] + + +class SearchQuestionReformulationBody(BaseModel): + question: StrictStr diff --git a/fast_api/routes/playground.py b/fast_api/routes/playground.py index 4bc787d5..80f63d6b 100644 --- a/fast_api/routes/playground.py +++ b/fast_api/routes/playground.py @@ -7,6 +7,7 @@ from fast_api.models import ( EvaluationRunDeletionBody, SearchQuestionBody, + SearchQuestionReformulationBody, EvaluationSetCreationBody, EvaluationGroupCreationBody, EvaluationRunCreationBody, @@ -239,3 +240,18 @@ def delete_evaluation_run( project_id, evaluation_set.evaluationRunIds ) return get_silent_success() + + +@router.post( + "/{project_id}/reformulation", + dependencies=[Depends(auth_manager.check_project_access_dep)], +) +def get_question_reformulation( + request: Request, + project_id: str, + question_reformulation: SearchQuestionReformulationBody = Body(...), +): + reformulation = playground_manager.get_question_reformulation( + question_reformulation.question + ) + return pack_json_result(reformulation) From 4bedf4d56f19e8d2ecf887ca734132769ccc9066 Mon Sep 17 00:00:00 2001 From: LennartSchmidtKern Date: Fri, 31 Jan 2025 13:13:31 +0100 Subject: [PATCH 43/76] requirements --- controller/playground/manager.py | 6 ++-- controller/playground/reformulation.py | 47 ++++++++++++++++++++++++++ requirements.txt | 28 +++++++++++++-- requirements/requirements.in | 4 ++- 4 files changed, 79 insertions(+), 6 deletions(-) diff --git a/controller/playground/manager.py b/controller/playground/manager.py index d641fe0d..7817f6e3 100644 --- a/controller/playground/manager.py +++ b/controller/playground/manager.py @@ -11,7 +11,7 @@ from service.search.search import resolve_extended_search from submodules.model.util import sql_alchemy_to_dict, to_frontend_obj_raw from concurrent.futures import ThreadPoolExecutor -from .reformulation import REFORMULATION_PROMPT +from .reformulation import REFORMULATION_PROMPT, reformulate_question NEURAL_SEARCH = os.getenv("NEURAL_SEARCH") EMBEDDING_SERVICE = os.getenv("EMBEDDING_SERVICE") @@ -313,6 +313,6 @@ def __build_contains_filter(project_id: str, content: str): def get_question_reformulation(question: str): - - reformulation_dict = {"reformulation": "Test 123"} + reformulation = reformulate_question(question) + reformulation_dict = {"reformulation": reformulation} return reformulation_dict diff --git a/controller/playground/reformulation.py b/controller/playground/reformulation.py index 5f52a362..a3d2699b 100644 --- a/controller/playground/reformulation.py +++ b/controller/playground/reformulation.py @@ -1,3 +1,7 @@ +from openai import OpenAI +from openai.types.chat import ChatCompletion, ChatCompletionChunk +from typing import Union + REFORMULATION_PROMPT = """Generate a refined and optimized reformulation of a given question, ensuring it aligns better with the user's search intent and maximizes result relevance for RAG. The reformulated question should maintain the original meaning while improving clarity, specificity, and contextual precision. # Important Guidelines @@ -31,3 +35,46 @@ # Notes A poorly reformulated question may lead to irrelevant or misleading results. It is essential to preserve intent while enhancing clarity and relevance.""" +API_KEY = "" + + +def reformulate_question(question: str) -> str: + openai_client = OpenAI(api_key=API_KEY) + messages = [ + { + "role": "system", + "content": REFORMULATION_PROMPT, + }, + { + "role": "user", + "content": f"Question: {question}", + }, + ] + completion = openai_client.chat.completions.create( + model="gpt-4o-mini", messages=messages, stream=False + ) + return __get_openai_value_from(completion) + + +def __get_openai_value_from( + open_ai_obj: Union[ChatCompletion, ChatCompletionChunk], raise_me: bool = True +) -> str: + if not open_ai_obj: + return "" + if isinstance(open_ai_obj, ChatCompletion) or isinstance( + open_ai_obj, ChatCompletionChunk + ): + if hasattr(open_ai_obj, "choices") and len(open_ai_obj.choices) > 0: + t = open_ai_obj.choices[0] + if isinstance(open_ai_obj, ChatCompletion): + if hasattr(t, "message") and hasattr(t.message, "content"): + return t.message.content or "" + elif isinstance(open_ai_obj, ChatCompletionChunk): + if hasattr(t, "delta") and hasattr(t.delta, "content"): + return t.delta.content or "" + else: + raise ValueError("Unknown open_ai_obj:" + type(open_ai_obj)) + ## if we reach this point, we couldn't access the value + if raise_me: + raise ValueError("Couldn't access value from", open_ai_obj) + return "" diff --git a/requirements.txt b/requirements.txt index df8f6eb6..06d107b9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,6 +13,8 @@ annotated-types==0.7.0 anyio==4.6.2.post1 # via # -r requirements/common-requirements.txt + # httpx + # openai # starlette blis==0.7.11 # via thinc @@ -31,6 +33,8 @@ catalogue==2.0.10 certifi==2024.8.30 # via # -r requirements/common-requirements.txt + # httpcore + # httpx # minio # requests charset-normalizer==3.4.0 @@ -53,6 +57,8 @@ cymem==2.0.8 # preshed # spacy # thinc +distro==1.9.0 + # via openai docker==5.0.0 # via -r requirements/requirements.in et-xmlfile==1.1.0 @@ -63,14 +69,24 @@ exceptiongroup==1.2.2 # anyio fastapi==0.115.2 # via -r requirements/common-requirements.txt +greenlet==3.1.1 + # via sqlalchemy h11==0.14.0 # via # -r requirements/common-requirements.txt + # httpcore # uvicorn +httpcore==1.0.7 + # via httpx +httpx==0.27.2 + # via + # -r requirements/requirements.in + # openai idna==3.10 # via # -r requirements/common-requirements.txt # anyio + # httpx # requests jinja2==3.1.4 # via spacy @@ -92,7 +108,7 @@ markupsafe==2.1.5 # jinja2 # mako minio==7.1.12 - # via -r requirements/common-requirements.txt + # via -r DRF murmurhash==1.0.10 # via # preshed @@ -105,6 +121,8 @@ numpy==1.23.4 # pandas # spacy # thinc +openai==1.31.0 + # via -r requirements/requirements.in openpyxl==3.0.10 # via -r requirements/requirements.in packaging==24.0 @@ -128,6 +146,7 @@ pydantic==2.7.4 # -r requirements/requirements.in # confection # fastapi + # openai # spacy # thinc # weasel @@ -170,6 +189,8 @@ sniffio==1.3.1 # via # -r requirements/common-requirements.txt # anyio + # httpx + # openai spacy[ja]==3.7.5 # via -r requirements/requirements.in spacy-legacy==3.0.12 @@ -199,7 +220,9 @@ sudachipy==0.6.8 thinc==8.2.5 # via spacy tqdm==4.66.4 - # via spacy + # via + # openai + # spacy typer==0.4.2 # via # spacy @@ -210,6 +233,7 @@ typing-extensions==4.12.2 # anyio # cloudpathlib # fastapi + # openai # pydantic # pydantic-core # starlette diff --git a/requirements/requirements.in b/requirements/requirements.in index efc31e82..e98c34e4 100644 --- a/requirements/requirements.in +++ b/requirements/requirements.in @@ -9,4 +9,6 @@ pyjwt==2.4.0 spacy[ja]==3.7.5 pyminizip==0.2.6 rncryptor==3.3.0 -pydantic==2.7.4 \ No newline at end of file +pydantic==2.7.4 +httpx==0.27.2 +openai==1.31.0 \ No newline at end of file From 5fb8d6bb40118f9726c966bbca452d7a807fb2e5 Mon Sep 17 00:00:00 2001 From: LennartSchmidtKern Date: Fri, 31 Jan 2025 13:30:21 +0100 Subject: [PATCH 44/76] change output --- controller/playground/manager.py | 4 ++-- controller/playground/reformulation.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/controller/playground/manager.py b/controller/playground/manager.py index 7817f6e3..41d12489 100644 --- a/controller/playground/manager.py +++ b/controller/playground/manager.py @@ -12,6 +12,7 @@ from submodules.model.util import sql_alchemy_to_dict, to_frontend_obj_raw from concurrent.futures import ThreadPoolExecutor from .reformulation import REFORMULATION_PROMPT, reformulate_question +import json NEURAL_SEARCH = os.getenv("NEURAL_SEARCH") EMBEDDING_SERVICE = os.getenv("EMBEDDING_SERVICE") @@ -313,6 +314,5 @@ def __build_contains_filter(project_id: str, content: str): def get_question_reformulation(question: str): - reformulation = reformulate_question(question) - reformulation_dict = {"reformulation": reformulation} + reformulation_dict = json.loads(reformulate_question(question)) return reformulation_dict diff --git a/controller/playground/reformulation.py b/controller/playground/reformulation.py index a3d2699b..d2452d34 100644 --- a/controller/playground/reformulation.py +++ b/controller/playground/reformulation.py @@ -27,7 +27,7 @@ # Output Format Only the pure valid json with key "reformulation" and value as the improved version of the original question. -```json + { "reformulation": "", } From 372c2c4db473fd938921722d1c9633d2f84d957b Mon Sep 17 00:00:00 2001 From: LennartSchmidtKern Date: Fri, 31 Jan 2025 13:54:17 +0100 Subject: [PATCH 45/76] api key dyn --- controller/playground/manager.py | 4 ++-- controller/playground/reformulation.py | 5 ++--- fast_api/models.py | 1 + fast_api/routes/playground.py | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/controller/playground/manager.py b/controller/playground/manager.py index 41d12489..147cce8e 100644 --- a/controller/playground/manager.py +++ b/controller/playground/manager.py @@ -313,6 +313,6 @@ def __build_contains_filter(project_id: str, content: str): return final_filter -def get_question_reformulation(question: str): - reformulation_dict = json.loads(reformulate_question(question)) +def get_question_reformulation(question: str, api_key: str) -> Dict: + reformulation_dict = json.loads(reformulate_question(question, api_key)) return reformulation_dict diff --git a/controller/playground/reformulation.py b/controller/playground/reformulation.py index d2452d34..9072fef3 100644 --- a/controller/playground/reformulation.py +++ b/controller/playground/reformulation.py @@ -35,11 +35,10 @@ # Notes A poorly reformulated question may lead to irrelevant or misleading results. It is essential to preserve intent while enhancing clarity and relevance.""" -API_KEY = "" -def reformulate_question(question: str) -> str: - openai_client = OpenAI(api_key=API_KEY) +def reformulate_question(question: str, api_key: str) -> str: + openai_client = OpenAI(api_key=api_key) messages = [ { "role": "system", diff --git a/fast_api/models.py b/fast_api/models.py index 0a16d7c0..51fd812c 100644 --- a/fast_api/models.py +++ b/fast_api/models.py @@ -494,3 +494,4 @@ class EvaluationRunDeletionBody(BaseModel): class SearchQuestionReformulationBody(BaseModel): question: StrictStr + apiKey: StrictStr diff --git a/fast_api/routes/playground.py b/fast_api/routes/playground.py index 80f63d6b..a36ddb74 100644 --- a/fast_api/routes/playground.py +++ b/fast_api/routes/playground.py @@ -252,6 +252,6 @@ def get_question_reformulation( question_reformulation: SearchQuestionReformulationBody = Body(...), ): reformulation = playground_manager.get_question_reformulation( - question_reformulation.question + question_reformulation.question, question_reformulation.apiKey ) return pack_json_result(reformulation) From a1527a28d39da0e13d40410656b5086d8c4aa6ce Mon Sep 17 00:00:00 2001 From: anmarhindi Date: Fri, 31 Jan 2025 14:12:01 +0100 Subject: [PATCH 46/76] Simple error handling for reformulation --- controller/playground/manager.py | 7 +++-- controller/playground/reformulation.py | 36 +++++++++++++++----------- 2 files changed, 26 insertions(+), 17 deletions(-) diff --git a/controller/playground/manager.py b/controller/playground/manager.py index 147cce8e..9cae0ea9 100644 --- a/controller/playground/manager.py +++ b/controller/playground/manager.py @@ -11,7 +11,7 @@ from service.search.search import resolve_extended_search from submodules.model.util import sql_alchemy_to_dict, to_frontend_obj_raw from concurrent.futures import ThreadPoolExecutor -from .reformulation import REFORMULATION_PROMPT, reformulate_question +from .reformulation import reformulate_question import json NEURAL_SEARCH = os.getenv("NEURAL_SEARCH") @@ -314,5 +314,8 @@ def __build_contains_filter(project_id: str, content: str): def get_question_reformulation(question: str, api_key: str) -> Dict: - reformulation_dict = json.loads(reformulate_question(question, api_key)) + q_reformulated = reformulate_question(question, api_key) + if q_reformulated is None: + return None + reformulation_dict = json.loads() return reformulation_dict diff --git a/controller/playground/reformulation.py b/controller/playground/reformulation.py index 9072fef3..d3bb283a 100644 --- a/controller/playground/reformulation.py +++ b/controller/playground/reformulation.py @@ -1,3 +1,4 @@ +import traceback from openai import OpenAI from openai.types.chat import ChatCompletion, ChatCompletionChunk from typing import Union @@ -38,21 +39,26 @@ def reformulate_question(question: str, api_key: str) -> str: - openai_client = OpenAI(api_key=api_key) - messages = [ - { - "role": "system", - "content": REFORMULATION_PROMPT, - }, - { - "role": "user", - "content": f"Question: {question}", - }, - ] - completion = openai_client.chat.completions.create( - model="gpt-4o-mini", messages=messages, stream=False - ) - return __get_openai_value_from(completion) + try: + openai_client = OpenAI(api_key=api_key) + messages = [ + { + "role": "system", + "content": REFORMULATION_PROMPT, + }, + { + "role": "user", + "content": f"Question: {question}", + }, + ] + completion = openai_client.chat.completions.create( + model="gpt-4o-mini", messages=messages, stream=False + ) + return __get_openai_value_from(completion) + except Exception: + traceback.print_exc() + + return None def __get_openai_value_from( From 9c0444252f6936b54f3455dd69bd6364cef46c00 Mon Sep 17 00:00:00 2001 From: LennartSchmidtKern Date: Mon, 3 Feb 2025 12:05:09 +0100 Subject: [PATCH 47/76] group fetch --- controller/playground/manager.py | 39 ++++++++++++++++---------------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/controller/playground/manager.py b/controller/playground/manager.py index 147cce8e..f0fa13a4 100644 --- a/controller/playground/manager.py +++ b/controller/playground/manager.py @@ -11,7 +11,7 @@ from service.search.search import resolve_extended_search from submodules.model.util import sql_alchemy_to_dict, to_frontend_obj_raw from concurrent.futures import ThreadPoolExecutor -from .reformulation import REFORMULATION_PROMPT, reformulate_question +from .reformulation import reformulate_question import json NEURAL_SEARCH = os.getenv("NEURAL_SEARCH") @@ -36,26 +36,27 @@ def get_search_result_for_text( threshold=None, ): question_tensor = __get_tensors_for_texts(project_id, embedding_id, [question]) - if question_tensor and question_tensor[0]: - records = __get_most_similar_records( - project_id, embedding_id, question_tensor[0], limit, filter, threshold - ) - # make more efficient, own function - unfolded_records = [] - for record in records: - record_obj = record_db_bo.get(project_id, record["id"]) - record_dict = sql_alchemy_to_dict(record_obj) - unfolded_records.append( - { - "data": record_dict["data"], - "id": record["id"], - "score": record["score"], - } - ) - return unfolded_records - else: + if not question_tensor or not question_tensor[0]: return [] + records = __get_most_similar_records( + project_id, embedding_id, question_tensor[0], limit, filter, threshold + ) + + record_ids = [record["id"] for record in records] + record_obj_map = record_db_bo.get_full_record_data_for_id_group( + project_id, record_ids + ) + + return [ + { + "data": record_obj_map.get(record["id"]), + "id": record["id"], + "score": record["score"], + } + for record in records + ] + def create_evaluation_set( project_id: str, question: str, record_ids: List[str], created_by: str From ed9ab38101635bb3e3bb5519170719909251c3c3 Mon Sep 17 00:00:00 2001 From: LennartSchmidtKern Date: Mon, 3 Feb 2025 12:16:30 +0100 Subject: [PATCH 48/76] improve record fetching --- controller/playground/manager.py | 43 +++++++++++++------------------- 1 file changed, 18 insertions(+), 25 deletions(-) diff --git a/controller/playground/manager.py b/controller/playground/manager.py index 1a19a1db..366bc12c 100644 --- a/controller/playground/manager.py +++ b/controller/playground/manager.py @@ -134,15 +134,13 @@ def init_evaluation_run( created_by: str, threshold: float, ): - evaluation_run = evaluation_run_db_bo.create( project_id, evaluation_group_id, created_by, embedding_id, - EVALUATION_RUN_STATE.INITIATED, + EVALUATION_RUN_STATE.RUNNING, ) - state = EVALUATION_RUN_STATE.RUNNING evaluation_results = [] try: evaluation_sets = evaluation_set_db_bo.get_by_evaluation_group_id( @@ -170,40 +168,35 @@ def init_evaluation_run( search_results = [future.result() for future in futures] + all_record_ids = set() for evaluation_set, search_result in zip(evaluation_sets, search_results): + expected_record_ids = {str(rid) for rid in evaluation_set.record_ids} + received_record_ids = {str(record["id"]) for record in search_result} + all_record_ids.update(expected_record_ids | received_record_ids) - expected_record_ids = [str(rid) for rid in evaluation_set.record_ids] - received_record_ids = [str(record["id"]) for record in search_result] + all_records = record_db_bo.get_by_record_ids(project_id, list(all_record_ids)) + record_map = { + str(record.id): sql_alchemy_to_dict(record) for record in all_records + } - true_positives = [] - false_positives = [] - false_negatives = [] - for record_id in expected_record_ids: - if record_id in received_record_ids: - true_positives.append(record_id) - else: - false_negatives.append(record_id) + for evaluation_set, search_result in zip(evaluation_sets, search_results): + expected_record_ids = {str(rid) for rid in evaluation_set.record_ids} + received_record_ids = {str(record["id"]) for record in search_result} - for record_id in received_record_ids: - if record_id not in expected_record_ids: - false_positives.append(record_id) + true_positives = expected_record_ids & received_record_ids + false_negatives = expected_record_ids - received_record_ids + false_positives = received_record_ids - expected_record_ids result = { "evaluation_set_id": str(evaluation_set.id), "true_positives": to_frontend_obj_raw( - sql_alchemy_to_dict( - record_db_bo.get_by_record_ids(project_id, true_positives) - ) + [record_map[record_id] for record_id in true_positives] ), "false_positives": to_frontend_obj_raw( - sql_alchemy_to_dict( - record_db_bo.get_by_record_ids(project_id, false_positives) - ) + [record_map[record_id] for record_id in false_positives] ), "false_negatives": to_frontend_obj_raw( - sql_alchemy_to_dict( - record_db_bo.get_by_record_ids(project_id, false_negatives) - ) + [record_map[record_id] for record_id in false_negatives] ), } evaluation_results.append(result) From e1d97873daee33949e5475f0ab36b77de1a2dc1b Mon Sep 17 00:00:00 2001 From: LennartSchmidtKern Date: Mon, 3 Feb 2025 13:36:57 +0100 Subject: [PATCH 49/76] record batch --- controller/record/manager.py | 4 ++++ fast_api/routes/project_setting.py | 10 ++-------- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/controller/record/manager.py b/controller/record/manager.py index a02a7704..ad15a0e6 100644 --- a/controller/record/manager.py +++ b/controller/record/manager.py @@ -30,6 +30,10 @@ def get_record(project_id: str, record_id: str) -> Record: return record.get(project_id, record_id) +def get_record_by_ids(project_id: str, record_ids: List[str]) -> List[Record]: + return record.get_by_record_ids(project_id, record_ids) + + def get_records_by_similarity_search( project_id: str, user_id: str, diff --git a/fast_api/routes/project_setting.py b/fast_api/routes/project_setting.py index b5b7a6b5..82b87f93 100644 --- a/fast_api/routes/project_setting.py +++ b/fast_api/routes/project_setting.py @@ -158,18 +158,12 @@ def get_records_batch( recordsBatchBody: RecordsBatchBody = Body(...), ): results = [] - for record_id in recordsBatchBody.record_ids: - if record_id is None or record_id == "null": - continue - - record = record_manager.get_record(project_id, record_id) - + records = record_manager.get_record_by_ids(project_id, recordsBatchBody.record_ids) + for record in records: results.append( { "id": str(record.id), "data": json.dumps(record.data), - "projectId": str(record.project_id), - "category": record.category, } ) From 354f59daf8f0bc04e1da00885522e4b5808b0611 Mon Sep 17 00:00:00 2001 From: LennartSchmidtKern Date: Mon, 3 Feb 2025 13:40:22 +0100 Subject: [PATCH 50/76] simplifiy args --- fast_api/routes/playground.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/fast_api/routes/playground.py b/fast_api/routes/playground.py index a36ddb74..f3e1aa5e 100644 --- a/fast_api/routes/playground.py +++ b/fast_api/routes/playground.py @@ -29,13 +29,13 @@ def get_search_results_question( project_id: str, search_question: SearchQuestionBody = Body(...), ): - embedding_id = search_question.embeddingId - question = search_question.question - limit = search_question.limit - filter = search_question.filter - threshold = search_question.threshold search_results = playground_manager.get_search_result_for_text( - project_id, embedding_id, question, limit, filter, threshold + project_id, + search_question.embeddingId, + search_question.question, + search_question.limit, + search_question.filter, + search_question.threshold, ) return pack_json_result(search_results, wrap_for_frontend=False) @@ -217,12 +217,13 @@ def get_record_by_content( project_id: str, record_search: RecordSearchContains = Body(...), ): - query = record_search.query - limit = record_search.limit - offset = record_search.offset user_id = auth_manager.get_user_id_by_info(request.state.info) records = playground_manager.get_records_by_content( - project_id, user_id, query, limit, offset + project_id, + user_id, + record_search.query, + record_search.limit, + record_search.offset, ) return pack_json_result(records, wrap_for_frontend=False) From 0ff0f7104e2edc5f75ca9307c2546b954200893e Mon Sep 17 00:00:00 2001 From: LennartSchmidtKern Date: Mon, 3 Feb 2025 13:53:41 +0100 Subject: [PATCH 51/76] fix exception --- controller/playground/manager.py | 10 +++++++--- controller/playground/reformulation.py | 17 +++++------------ 2 files changed, 12 insertions(+), 15 deletions(-) diff --git a/controller/playground/manager.py b/controller/playground/manager.py index 366bc12c..d074aa37 100644 --- a/controller/playground/manager.py +++ b/controller/playground/manager.py @@ -307,9 +307,13 @@ def __build_contains_filter(project_id: str, content: str): return final_filter -def get_question_reformulation(question: str, api_key: str) -> Dict: +def get_question_reformulation(question: str, api_key: str) -> Optional[Dict]: q_reformulated = reformulate_question(question, api_key) if q_reformulated is None: return None - reformulation_dict = json.loads() - return reformulation_dict + + try: + reformulation_dict = json.loads(q_reformulated) + return reformulation_dict + except Exception: + return None diff --git a/controller/playground/reformulation.py b/controller/playground/reformulation.py index d3bb283a..0c124d90 100644 --- a/controller/playground/reformulation.py +++ b/controller/playground/reformulation.py @@ -1,7 +1,7 @@ import traceback from openai import OpenAI from openai.types.chat import ChatCompletion, ChatCompletionChunk -from typing import Union +from typing import Union, Optional REFORMULATION_PROMPT = """Generate a refined and optimized reformulation of a given question, ensuring it aligns better with the user's search intent and maximizes result relevance for RAG. The reformulated question should maintain the original meaning while improving clarity, specificity, and contextual precision. @@ -38,18 +38,12 @@ A poorly reformulated question may lead to irrelevant or misleading results. It is essential to preserve intent while enhancing clarity and relevance.""" -def reformulate_question(question: str, api_key: str) -> str: +def reformulate_question(question: str, api_key: str) -> Optional[str]: try: openai_client = OpenAI(api_key=api_key) messages = [ - { - "role": "system", - "content": REFORMULATION_PROMPT, - }, - { - "role": "user", - "content": f"Question: {question}", - }, + {"role": "system", "content": REFORMULATION_PROMPT}, + {"role": "user", "content": f"Question: {question}"}, ] completion = openai_client.chat.completions.create( model="gpt-4o-mini", messages=messages, stream=False @@ -57,8 +51,7 @@ def reformulate_question(question: str, api_key: str) -> str: return __get_openai_value_from(completion) except Exception: traceback.print_exc() - - return None + return None def __get_openai_value_from( From 6d5013a60dd9d055c2e44012547921470ca6c504 Mon Sep 17 00:00:00 2001 From: anmarhindi Date: Mon, 3 Feb 2025 14:10:16 +0100 Subject: [PATCH 52/76] Remove redundant check, delegate to json loads raise --- controller/playground/manager.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/controller/playground/manager.py b/controller/playground/manager.py index d074aa37..adb70706 100644 --- a/controller/playground/manager.py +++ b/controller/playground/manager.py @@ -309,9 +309,6 @@ def __build_contains_filter(project_id: str, content: str): def get_question_reformulation(question: str, api_key: str) -> Optional[Dict]: q_reformulated = reformulate_question(question, api_key) - if q_reformulated is None: - return None - try: reformulation_dict = json.loads(q_reformulated) return reformulation_dict From b0d60a2dcc4474b031cd975af732d117c8f61501 Mon Sep 17 00:00:00 2001 From: anmarhindi Date: Tue, 4 Feb 2025 10:49:20 +0100 Subject: [PATCH 53/76] Add playground question model --- .../0c8eb3ff1c71_adds_playground_question.py | 47 +++++++++++++++++++ submodules/model | 2 +- 2 files changed, 48 insertions(+), 1 deletion(-) create mode 100644 alembic/versions/0c8eb3ff1c71_adds_playground_question.py diff --git a/alembic/versions/0c8eb3ff1c71_adds_playground_question.py b/alembic/versions/0c8eb3ff1c71_adds_playground_question.py new file mode 100644 index 00000000..8deed107 --- /dev/null +++ b/alembic/versions/0c8eb3ff1c71_adds_playground_question.py @@ -0,0 +1,47 @@ +"""adds_playground_question + +Revision ID: 0c8eb3ff1c71 +Revises: ac97442726d2 +Create Date: 2025-02-04 09:48:41.971287 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = '0c8eb3ff1c71' +down_revision = 'ac97442726d2' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('playground_question', + sa.Column('id', postgresql.UUID(as_uuid=True), nullable=False), + sa.Column('question', sa.String(), nullable=True), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('created_by', postgresql.UUID(as_uuid=True), nullable=True), + sa.Column('project_id', postgresql.UUID(as_uuid=True), nullable=True), + sa.Column('embedding_id', postgresql.UUID(as_uuid=True), nullable=True), + sa.Column('record_ids', sa.JSON(), nullable=True), + sa.Column('meta_info', sa.JSON(), nullable=True), + sa.ForeignKeyConstraint(['created_by'], ['user.id'], ondelete='SET NULL'), + sa.ForeignKeyConstraint(['embedding_id'], ['embedding.id'], ondelete='SET NULL'), + sa.ForeignKeyConstraint(['project_id'], ['project.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_playground_question_created_by'), 'playground_question', ['created_by'], unique=False) + op.create_index(op.f('ix_playground_question_embedding_id'), 'playground_question', ['embedding_id'], unique=False) + op.create_index(op.f('ix_playground_question_project_id'), 'playground_question', ['project_id'], unique=False) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index(op.f('ix_playground_question_project_id'), table_name='playground_question') + op.drop_index(op.f('ix_playground_question_embedding_id'), table_name='playground_question') + op.drop_index(op.f('ix_playground_question_created_by'), table_name='playground_question') + op.drop_table('playground_question') + # ### end Alembic commands ### diff --git a/submodules/model b/submodules/model index ccf85a6f..1749dd66 160000 --- a/submodules/model +++ b/submodules/model @@ -1 +1 @@ -Subproject commit ccf85a6fdebcaf2cac232bfcf553a834e9acb2ba +Subproject commit 1749dd6668fd6cc24a1d18b979725d3e039a8359 From f01901a7d809c3845b6415ecb6e2e04f1a0912a6 Mon Sep 17 00:00:00 2001 From: anmarhindi Date: Tue, 4 Feb 2025 11:26:12 +0100 Subject: [PATCH 54/76] Change endpoint handling of saveQuestion --- controller/playground/manager.py | 13 +++++++++++++ fast_api/models.py | 1 + fast_api/routes/playground.py | 10 ++++++++++ 3 files changed, 24 insertions(+) diff --git a/controller/playground/manager.py b/controller/playground/manager.py index adb70706..d1ed759c 100644 --- a/controller/playground/manager.py +++ b/controller/playground/manager.py @@ -5,6 +5,7 @@ evaluation_group as evaluation_group_db_bo, evaluation_set as evaluation_set_db_bo, evaluation_run as evaluation_run_db_bo, + playground_question as playground_question_db_bo, attribute as attribute_db_bo, record as record_db_bo, ) @@ -314,3 +315,15 @@ def get_question_reformulation(question: str, api_key: str) -> Optional[Dict]: return reformulation_dict except Exception: return None + + +def create_playground_question( + project_id: str, + question: str, + record_ids: List[str], + created_by: str, + embedding_id: str, +): + playground_question_db_bo.create( + project_id, question, created_by, embedding_id, record_ids, with_commit=True + ) diff --git a/fast_api/models.py b/fast_api/models.py index 51fd812c..f261d6b5 100644 --- a/fast_api/models.py +++ b/fast_api/models.py @@ -452,6 +452,7 @@ class SearchQuestionBody(BaseModel): limit: Optional[StrictInt] = None filter: Optional[List[Dict]] = None threshold: Optional[StrictFloat] = None + saveQuestion: Optional[StrictBool] = None class EvaluationSetCreationBody(BaseModel): diff --git a/fast_api/routes/playground.py b/fast_api/routes/playground.py index f3e1aa5e..26208b64 100644 --- a/fast_api/routes/playground.py +++ b/fast_api/routes/playground.py @@ -38,6 +38,16 @@ def get_search_results_question( search_question.threshold, ) + if search_question.saveQuestion: + user_id = auth_manager.get_user_id_by_info(request.state.info) + playground_manager.create_playground_question( + project_id, + search_question.question, + search_results, + user_id, + search_question.embeddingId, + ) + return pack_json_result(search_results, wrap_for_frontend=False) From 6cc8651402c12a53909e245c47860ea5f0de0ea2 Mon Sep 17 00:00:00 2001 From: anmarhindi Date: Tue, 4 Feb 2025 11:34:24 +0100 Subject: [PATCH 55/76] Add initial get playground questions endpoint --- controller/playground/manager.py | 4 ++++ fast_api/routes/playground.py | 17 +++++++++++++++-- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/controller/playground/manager.py b/controller/playground/manager.py index d1ed759c..60db3577 100644 --- a/controller/playground/manager.py +++ b/controller/playground/manager.py @@ -327,3 +327,7 @@ def create_playground_question( playground_question_db_bo.create( project_id, question, created_by, embedding_id, record_ids, with_commit=True ) + + +def get_playground_questions(project_id: str): + return playground_question_db_bo.get_all(project_id) diff --git a/fast_api/routes/playground.py b/fast_api/routes/playground.py index 26208b64..7407bf4f 100644 --- a/fast_api/routes/playground.py +++ b/fast_api/routes/playground.py @@ -51,6 +51,18 @@ def get_search_results_question( return pack_json_result(search_results, wrap_for_frontend=False) +@router.get( + "/{project_id}/playground-questions", + dependencies=[Depends(auth_manager.check_project_access_dep)], +) +def get_playground_questions( + request: Request, + project_id: str, +): + questions = playground_manager.get_playground_questions(project_id) + return pack_json_result(questions) + + @router.post( "/{project_id}/evaluation-sets", dependencies=[Depends(auth_manager.check_project_access_dep)], @@ -71,8 +83,9 @@ def create_evaluation_set( @router.delete( - "/{project_id}/evaluation-sets" -) # dependencies=[Depends(auth_manager.check_project_access_dep)] + "/{project_id}/evaluation-sets", + dependencies=[Depends(auth_manager.check_project_access_dep)], +) def delete_evaluation_set( request: Request, project_id: str, From 51b11f0d76db44c44e7e43ae82d61d2cf9a8ed17 Mon Sep 17 00:00:00 2001 From: anmarhindi Date: Tue, 4 Feb 2025 13:37:42 +0100 Subject: [PATCH 56/76] sub ref --- submodules/model | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/submodules/model b/submodules/model index 1749dd66..2b1f7edb 160000 --- a/submodules/model +++ b/submodules/model @@ -1 +1 @@ -Subproject commit 1749dd6668fd6cc24a1d18b979725d3e039a8359 +Subproject commit 2b1f7edb1d0a615716c9b7dd946ebd5075be9b70 From ddbab82f1fb76b5c5d142dc4de0e15aa49e7b514 Mon Sep 17 00:00:00 2001 From: anmarhindi Date: Tue, 4 Feb 2025 13:40:28 +0100 Subject: [PATCH 57/76] sub ref --- submodules/model | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/submodules/model b/submodules/model index 2b1f7edb..d021d5a8 160000 --- a/submodules/model +++ b/submodules/model @@ -1 +1 @@ -Subproject commit 2b1f7edb1d0a615716c9b7dd946ebd5075be9b70 +Subproject commit d021d5a8116dd619018845aa9bd849c6ea4f3110 From 1204b2bdcdd6c0ffec89716e1fc94e077085fdc7 Mon Sep 17 00:00:00 2001 From: anmarhindi Date: Tue, 4 Feb 2025 15:55:50 +0100 Subject: [PATCH 58/76] Delegate save question to neural search --- controller/playground/manager.py | 18 +++++++++++++++++- fast_api/routes/playground.py | 13 +++---------- 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/controller/playground/manager.py b/controller/playground/manager.py index 60db3577..6f83485c 100644 --- a/controller/playground/manager.py +++ b/controller/playground/manager.py @@ -30,18 +30,28 @@ class EVALUATION_RUN_STATE: def get_search_result_for_text( project_id: str, + user_id: str, embedding_id: str, question: str, limit: int, filter=None, threshold=None, + save_question=None, ): question_tensor = __get_tensors_for_texts(project_id, embedding_id, [question]) if not question_tensor or not question_tensor[0]: return [] records = __get_most_similar_records( - project_id, embedding_id, question_tensor[0], limit, filter, threshold + project_id, + embedding_id, + question_tensor[0], + limit, + filter, + threshold, + save_question, + question, + user_id, ) record_ids = [record["id"] for record in records] @@ -237,6 +247,9 @@ def __get_most_similar_records( limit: int, similarity_filter_option: Optional[List[Dict[str, Any]]] = None, threshold: Optional[float] = None, + save_question: Optional[bool] = None, + question: Optional[str] = None, + user_id: Optional[str] = None, ): url = f"{NEURAL_SEARCH}/most_similar_by_embedding?include_scores=true" @@ -253,6 +266,9 @@ def __get_most_similar_records( "limit": limit, "att_filter": similarity_filter_option, "threshold": threshold, + "save_question": save_question, + "question": question, + "user_id": user_id, }, ) if response.ok: diff --git a/fast_api/routes/playground.py b/fast_api/routes/playground.py index 7407bf4f..21c3003f 100644 --- a/fast_api/routes/playground.py +++ b/fast_api/routes/playground.py @@ -29,25 +29,18 @@ def get_search_results_question( project_id: str, search_question: SearchQuestionBody = Body(...), ): + user_id = auth_manager.get_user_id_by_info(request.state.info) search_results = playground_manager.get_search_result_for_text( project_id, + str(user_id), search_question.embeddingId, search_question.question, search_question.limit, search_question.filter, search_question.threshold, + search_question.saveQuestion, ) - if search_question.saveQuestion: - user_id = auth_manager.get_user_id_by_info(request.state.info) - playground_manager.create_playground_question( - project_id, - search_question.question, - search_results, - user_id, - search_question.embeddingId, - ) - return pack_json_result(search_results, wrap_for_frontend=False) From 709728e4759b56ec8246d68b36055faa8275f1c3 Mon Sep 17 00:00:00 2001 From: anmarhindi Date: Tue, 4 Feb 2025 17:36:38 +0100 Subject: [PATCH 59/76] Update model and manager --- .../0c8eb3ff1c71_adds_playground_question.py | 43 +++++++++---------- controller/playground/manager.py | 20 --------- fast_api/models.py | 1 - fast_api/routes/playground.py | 3 -- submodules/model | 2 +- 5 files changed, 22 insertions(+), 47 deletions(-) diff --git a/alembic/versions/0c8eb3ff1c71_adds_playground_question.py b/alembic/versions/0c8eb3ff1c71_adds_playground_question.py index 8deed107..64eb7f3f 100644 --- a/alembic/versions/0c8eb3ff1c71_adds_playground_question.py +++ b/alembic/versions/0c8eb3ff1c71_adds_playground_question.py @@ -5,43 +5,42 @@ Create Date: 2025-02-04 09:48:41.971287 """ + from alembic import op import sqlalchemy as sa from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. -revision = '0c8eb3ff1c71' -down_revision = 'ac97442726d2' +revision = "0c8eb3ff1c71" +down_revision = "ac97442726d2" branch_labels = None depends_on = None def upgrade(): # ### commands auto generated by Alembic - please adjust! ### - op.create_table('playground_question', - sa.Column('id', postgresql.UUID(as_uuid=True), nullable=False), - sa.Column('question', sa.String(), nullable=True), - sa.Column('created_at', sa.DateTime(), nullable=True), - sa.Column('created_by', postgresql.UUID(as_uuid=True), nullable=True), - sa.Column('project_id', postgresql.UUID(as_uuid=True), nullable=True), - sa.Column('embedding_id', postgresql.UUID(as_uuid=True), nullable=True), - sa.Column('record_ids', sa.JSON(), nullable=True), - sa.Column('meta_info', sa.JSON(), nullable=True), - sa.ForeignKeyConstraint(['created_by'], ['user.id'], ondelete='SET NULL'), - sa.ForeignKeyConstraint(['embedding_id'], ['embedding.id'], ondelete='SET NULL'), - sa.ForeignKeyConstraint(['project_id'], ['project.id'], ondelete='CASCADE'), - sa.PrimaryKeyConstraint('id') + op.create_table( + "playground_question", + sa.Column("id", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column("question", sa.String(), nullable=True), + sa.Column("created_at", sa.DateTime(), nullable=True), + sa.Column("project_id", postgresql.UUID(as_uuid=True), nullable=True), + sa.ForeignKeyConstraint(["project_id"], ["project.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index( + op.f("ix_playground_question_project_id"), + "playground_question", + ["project_id"], + unique=False, ) - op.create_index(op.f('ix_playground_question_created_by'), 'playground_question', ['created_by'], unique=False) - op.create_index(op.f('ix_playground_question_embedding_id'), 'playground_question', ['embedding_id'], unique=False) - op.create_index(op.f('ix_playground_question_project_id'), 'playground_question', ['project_id'], unique=False) # ### end Alembic commands ### def downgrade(): # ### commands auto generated by Alembic - please adjust! ### - op.drop_index(op.f('ix_playground_question_project_id'), table_name='playground_question') - op.drop_index(op.f('ix_playground_question_embedding_id'), table_name='playground_question') - op.drop_index(op.f('ix_playground_question_created_by'), table_name='playground_question') - op.drop_table('playground_question') + op.drop_index( + op.f("ix_playground_question_project_id"), table_name="playground_question" + ) + op.drop_table("playground_question") # ### end Alembic commands ### diff --git a/controller/playground/manager.py b/controller/playground/manager.py index 6f83485c..c9db6dbb 100644 --- a/controller/playground/manager.py +++ b/controller/playground/manager.py @@ -30,13 +30,11 @@ class EVALUATION_RUN_STATE: def get_search_result_for_text( project_id: str, - user_id: str, embedding_id: str, question: str, limit: int, filter=None, threshold=None, - save_question=None, ): question_tensor = __get_tensors_for_texts(project_id, embedding_id, [question]) if not question_tensor or not question_tensor[0]: @@ -49,9 +47,7 @@ def get_search_result_for_text( limit, filter, threshold, - save_question, question, - user_id, ) record_ids = [record["id"] for record in records] @@ -247,9 +243,7 @@ def __get_most_similar_records( limit: int, similarity_filter_option: Optional[List[Dict[str, Any]]] = None, threshold: Optional[float] = None, - save_question: Optional[bool] = None, question: Optional[str] = None, - user_id: Optional[str] = None, ): url = f"{NEURAL_SEARCH}/most_similar_by_embedding?include_scores=true" @@ -266,9 +260,7 @@ def __get_most_similar_records( "limit": limit, "att_filter": similarity_filter_option, "threshold": threshold, - "save_question": save_question, "question": question, - "user_id": user_id, }, ) if response.ok: @@ -333,17 +325,5 @@ def get_question_reformulation(question: str, api_key: str) -> Optional[Dict]: return None -def create_playground_question( - project_id: str, - question: str, - record_ids: List[str], - created_by: str, - embedding_id: str, -): - playground_question_db_bo.create( - project_id, question, created_by, embedding_id, record_ids, with_commit=True - ) - - def get_playground_questions(project_id: str): return playground_question_db_bo.get_all(project_id) diff --git a/fast_api/models.py b/fast_api/models.py index f261d6b5..51fd812c 100644 --- a/fast_api/models.py +++ b/fast_api/models.py @@ -452,7 +452,6 @@ class SearchQuestionBody(BaseModel): limit: Optional[StrictInt] = None filter: Optional[List[Dict]] = None threshold: Optional[StrictFloat] = None - saveQuestion: Optional[StrictBool] = None class EvaluationSetCreationBody(BaseModel): diff --git a/fast_api/routes/playground.py b/fast_api/routes/playground.py index 21c3003f..76725957 100644 --- a/fast_api/routes/playground.py +++ b/fast_api/routes/playground.py @@ -29,16 +29,13 @@ def get_search_results_question( project_id: str, search_question: SearchQuestionBody = Body(...), ): - user_id = auth_manager.get_user_id_by_info(request.state.info) search_results = playground_manager.get_search_result_for_text( project_id, - str(user_id), search_question.embeddingId, search_question.question, search_question.limit, search_question.filter, search_question.threshold, - search_question.saveQuestion, ) return pack_json_result(search_results, wrap_for_frontend=False) diff --git a/submodules/model b/submodules/model index d021d5a8..59cbf483 160000 --- a/submodules/model +++ b/submodules/model @@ -1 +1 @@ -Subproject commit d021d5a8116dd619018845aa9bd849c6ea4f3110 +Subproject commit 59cbf483449f7f4e88b7aa6c5cd13cfb109ec56a From b9c4fd7af1a3a1df58a3b5248b2c9b1dde853f37 Mon Sep 17 00:00:00 2001 From: anmarhindi Date: Tue, 4 Feb 2025 17:37:18 +0100 Subject: [PATCH 60/76] sub ref --- submodules/model | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/submodules/model b/submodules/model index 59cbf483..4d1929b0 160000 --- a/submodules/model +++ b/submodules/model @@ -1 +1 @@ -Subproject commit 59cbf483449f7f4e88b7aa6c5cd13cfb109ec56a +Subproject commit 4d1929b023d5b301deb40588477a03dce0d89ec6 From 62ae6a07bde990ef1fcacc7113e2e7051b8b03bd Mon Sep 17 00:00:00 2001 From: anmarhindi Date: Tue, 4 Feb 2025 17:51:05 +0100 Subject: [PATCH 61/76] Enable project access check --- fast_api/routes/playground.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/fast_api/routes/playground.py b/fast_api/routes/playground.py index 76725957..9ecf65a6 100644 --- a/fast_api/routes/playground.py +++ b/fast_api/routes/playground.py @@ -148,8 +148,9 @@ def create_evaluation_group( @router.delete( - "/{project_id}/evaluation-groups" -) # dependencies=[Depends(auth_manager.check_project_access_dep)] + "/{project_id}/evaluation-groups", + dependencies=[Depends(auth_manager.check_project_access_dep)], +) def delete_evaluation_group( request: Request, project_id: str, From a0fcc4e1bef7da34603f6fefcfacd890c2ba059a Mon Sep 17 00:00:00 2001 From: anmarhindi Date: Tue, 4 Feb 2025 17:54:24 +0100 Subject: [PATCH 62/76] list comp --- fast_api/routes/project_setting.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/fast_api/routes/project_setting.py b/fast_api/routes/project_setting.py index 82b87f93..fd8774de 100644 --- a/fast_api/routes/project_setting.py +++ b/fast_api/routes/project_setting.py @@ -157,15 +157,14 @@ def get_records_batch( project_id: str, recordsBatchBody: RecordsBatchBody = Body(...), ): - results = [] records = record_manager.get_record_by_ids(project_id, recordsBatchBody.record_ids) - for record in records: - results.append( - { - "id": str(record.id), - "data": json.dumps(record.data), - } - ) + results = [ + { + "id": str(record.id), + "data": json.dumps(record.data), + } + for record in records + ] return pack_json_result(results) From b2df18bbd0bb388bcff8abd504ab709f3a0400f6 Mon Sep 17 00:00:00 2001 From: anmarhindi Date: Thu, 6 Feb 2025 09:51:10 +0100 Subject: [PATCH 63/76] Use adjusted run state from enum --- controller/playground/manager.py | 14 ++++---------- submodules/model | 2 +- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/controller/playground/manager.py b/controller/playground/manager.py index c9db6dbb..1f82f928 100644 --- a/controller/playground/manager.py +++ b/controller/playground/manager.py @@ -10,6 +10,7 @@ record as record_db_bo, ) from service.search.search import resolve_extended_search +from submodules.model.enums import EvaluationRunState from submodules.model.util import sql_alchemy_to_dict, to_frontend_obj_raw from concurrent.futures import ThreadPoolExecutor from .reformulation import reformulate_question @@ -21,13 +22,6 @@ EVALUATION_RUN_LIMIT_DEFAULT = 100 -class EVALUATION_RUN_STATE: - INITIATED = "INITIATED" - RUNNING = "RUNNING" - SUCCESS = "SUCCESS" - FAILED = "FAILED" - - def get_search_result_for_text( project_id: str, embedding_id: str, @@ -146,7 +140,7 @@ def init_evaluation_run( evaluation_group_id, created_by, embedding_id, - EVALUATION_RUN_STATE.RUNNING, + EvaluationRunState.RUNNING, ) evaluation_results = [] try: @@ -207,9 +201,9 @@ def init_evaluation_run( ), } evaluation_results.append(result) - state = EVALUATION_RUN_STATE.SUCCESS + state = EvaluationRunState.SUCCESS except Exception: - state = EVALUATION_RUN_STATE.FAILED + state = EvaluationRunState.FAILED evaluation_run_db_bo.update( project_id, evaluation_run.id, state, evaluation_results, None, True ) diff --git a/submodules/model b/submodules/model index 4d1929b0..c2acd126 160000 --- a/submodules/model +++ b/submodules/model @@ -1 +1 @@ -Subproject commit 4d1929b023d5b301deb40588477a03dce0d89ec6 +Subproject commit c2acd1268212385d243cc6d27e9df9fa4584e784 From 304da3e0c5f65058d9ce6cfc5d0f0bf9fad907f1 Mon Sep 17 00:00:00 2001 From: anmarhindi Date: Thu, 6 Feb 2025 10:15:10 +0100 Subject: [PATCH 64/76] Rename base uri --- controller/embedding/connector.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/controller/embedding/connector.py b/controller/embedding/connector.py index 96886321..1f5c5135 100644 --- a/controller/embedding/connector.py +++ b/controller/embedding/connector.py @@ -5,17 +5,17 @@ from util import service_requests import requests -BASE_URI = os.getenv("EMBEDDING_SERVICE") +EMBEDDING_BASE_URI = os.getenv("EMBEDDING_SERVICE") NEURAL_SEARCH_BASE_URI = os.getenv("NEURAL_SEARCH") def request_listing_recommended_encoders() -> Any: - url = f"{BASE_URI}/classification/recommend/TEXT" # TODO does here have to be a data type? + url = f"{EMBEDDING_BASE_URI}/classification/recommend/TEXT" # TODO does here have to be a data type? return service_requests.get_call_or_raise(url) def request_embedding(project_id: str, embedding_id: str) -> Any: - url = f"{BASE_URI}/embed" + url = f"{EMBEDDING_BASE_URI}/embed" data = { "project_id": str(project_id), "embedding_id": str(embedding_id), @@ -24,12 +24,12 @@ def request_embedding(project_id: str, embedding_id: str) -> Any: def request_deleting_embedding(project_id: str, embedding_id: str) -> Any: - url = f"{BASE_URI}/delete/{project_id}/{embedding_id}" + url = f"{EMBEDDING_BASE_URI}/delete/{project_id}/{embedding_id}" return service_requests.delete_call_or_raise(url) def request_tensor_upload(project_id: str, embedding_id: str) -> None: - url = f"{BASE_URI}/upload_tensor_data/{project_id}/{embedding_id}" + url = f"{EMBEDDING_BASE_URI}/upload_tensor_data/{project_id}/{embedding_id}" service_requests.post_call_or_raise(url, {}) @@ -39,7 +39,7 @@ def request_re_embed_records( # example changes structure: # {"":[{"record_id":"","attribute_name":"","sub_key":""}]} # note that sub_key is optional and only for embedding lists relevant - url = f"{BASE_URI}/re_embed_records/{project_id}" + url = f"{EMBEDDING_BASE_URI}/re_embed_records/{project_id}" service_requests.post_call_or_raise(url, {"changes": changes}) From 2ad52c6d6e7a13bacc0f7404694a646707f83992 Mon Sep 17 00:00:00 2001 From: LennartSchmidtKern Date: Thu, 6 Feb 2025 10:22:29 +0100 Subject: [PATCH 65/76] eval run function --- controller/playground/manager.py | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/controller/playground/manager.py b/controller/playground/manager.py index d074aa37..a01a97de 100644 --- a/controller/playground/manager.py +++ b/controller/playground/manager.py @@ -108,23 +108,12 @@ def delete_evaluation_runs(project_id: str, run_ids: str): def get_evaluation_runs(project_id: str): evaluation_runs_objects = evaluation_run_db_bo.get_all(project_id) - evaluation_runs = [] - for evaluation_run in evaluation_runs_objects: - results = to_frontend_obj_raw(evaluation_run.results) - evaluation_runs.append( - {**sql_alchemy_to_dict(evaluation_run, True), "results": results} - ) - return evaluation_runs + return [__pack_evaluation_run(run) for run in evaluation_runs_objects] def get_evaluation_run_by_id(project_id: str, run_id: str): - evaluation_run = evaluation_run_db_bo.get(project_id, run_id) - results = to_frontend_obj_raw(evaluation_run.results) - evaluation_run_object = { - **sql_alchemy_to_dict(evaluation_run, True), - "results": results, - } - return evaluation_run_object + evaluation_run_object = evaluation_run_db_bo.get(project_id, run_id) + return __pack_evaluation_run(evaluation_run_object) def init_evaluation_run( @@ -307,6 +296,13 @@ def __build_contains_filter(project_id: str, content: str): return final_filter +def __pack_evaluation_run(evaluation_run): + return { + **sql_alchemy_to_dict(evaluation_run, True), + "results": to_frontend_obj_raw(evaluation_run.results), + } + + def get_question_reformulation(question: str, api_key: str) -> Optional[Dict]: q_reformulated = reformulate_question(question, api_key) if q_reformulated is None: From 0848d56c069a87869366ed3b3d14348dfe430109 Mon Sep 17 00:00:00 2001 From: anmarhindi Date: Thu, 6 Feb 2025 10:36:25 +0100 Subject: [PATCH 66/76] Resolve tensor via connector --- controller/embedding/connector.py | 13 +++++++++++++ controller/playground/manager.py | 16 +++------------- 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/controller/embedding/connector.py b/controller/embedding/connector.py index 1f5c5135..78969ed3 100644 --- a/controller/embedding/connector.py +++ b/controller/embedding/connector.py @@ -96,3 +96,16 @@ def delete_embedding_from_neural_search(embedding_id: str) -> None: url = f"{NEURAL_SEARCH_BASE_URI}/delete_collection" params = {"embedding_id": embedding_id} requests.put(url, params=params) + + +def request_tensor_for_text( + refinery_project_id: str, embedding_id: str, texts: List[str] +) -> Any: + url = ( + f"{EMBEDDING_BASE_URI}/calc-tensor-by-pkl/{refinery_project_id}/{embedding_id}" + ) + + data = { + "texts": texts, + } + return service_requests.post_call_or_raise(url, data) diff --git a/controller/playground/manager.py b/controller/playground/manager.py index 1f82f928..774d38b3 100644 --- a/controller/playground/manager.py +++ b/controller/playground/manager.py @@ -1,6 +1,7 @@ from typing import List, Any, Optional, Tuple, Dict import os import requests +from controller.embedding.connector import request_tensor_for_text from submodules.model.business_objects import ( evaluation_group as evaluation_group_db_bo, evaluation_set as evaluation_set_db_bo, @@ -214,20 +215,9 @@ def __get_tensors_for_texts( refinery_project_id: str, embedding_id: str, texts: List[str] ) -> Tuple[bool, Optional[List[Any]]]: - url = f"{EMBEDDING_SERVICE}/calc-tensor-by-pkl/{refinery_project_id}/{embedding_id}" + obj = request_tensor_for_text(refinery_project_id, embedding_id, texts) - response = requests.post( - url, - headers={ - "accept": "application/json", - "content-type": "application/json", - }, - json={"texts": texts}, - ) - if response.ok: - return response.json()["tensor"] - else: - return None + return obj.get("tensor", None) def __get_most_similar_records( From 878595ddb8587f4e45388ce5649a95465927e651 Mon Sep 17 00:00:00 2001 From: anmarhindi Date: Thu, 6 Feb 2025 10:46:08 +0100 Subject: [PATCH 67/76] Get tensor for text via connector --- controller/embedding/connector.py | 25 +++++++++++++++++++-- controller/playground/manager.py | 36 +++++++++++-------------------- 2 files changed, 35 insertions(+), 26 deletions(-) diff --git a/controller/embedding/connector.py b/controller/embedding/connector.py index 78969ed3..453d237d 100644 --- a/controller/embedding/connector.py +++ b/controller/embedding/connector.py @@ -10,7 +10,7 @@ def request_listing_recommended_encoders() -> Any: - url = f"{EMBEDDING_BASE_URI}/classification/recommend/TEXT" # TODO does here have to be a data type? + url = f"{EMBEDDING_BASE_URI}/classification/recommend/TEXT" return service_requests.get_call_or_raise(url) @@ -104,8 +104,29 @@ def request_tensor_for_text( url = ( f"{EMBEDDING_BASE_URI}/calc-tensor-by-pkl/{refinery_project_id}/{embedding_id}" ) - data = { "texts": texts, } return service_requests.post_call_or_raise(url, data) + + +def request_most_similar_records( + project_id: str, + embedding_id: str, + embedding_tensor: List[float], + limit: int, + similarity_filter_option: Optional[List[Dict[str, Any]]] = None, + threshold: Optional[float] = None, + question: Optional[str] = None, +) -> Any: + url = f"{NEURAL_SEARCH_BASE_URI}/most_similar_by_embedding?include_scores=true" + data = { + "project_id": project_id, + "embedding_id": embedding_id, + "embedding_tensor": embedding_tensor, + "limit": limit, + "att_filter": similarity_filter_option, + "threshold": threshold, + "question": question, + } + return service_requests.post_call_or_raise(url, data) diff --git a/controller/playground/manager.py b/controller/playground/manager.py index 94c596dc..a14d121f 100644 --- a/controller/playground/manager.py +++ b/controller/playground/manager.py @@ -1,7 +1,10 @@ from typing import List, Any, Optional, Tuple, Dict import os import requests -from controller.embedding.connector import request_tensor_for_text +from controller.embedding.connector import ( + request_most_similar_records, + request_tensor_for_text, +) from submodules.model.business_objects import ( evaluation_group as evaluation_group_db_bo, evaluation_set as evaluation_set_db_bo, @@ -203,9 +206,7 @@ def init_evaluation_run( def __get_tensors_for_texts( refinery_project_id: str, embedding_id: str, texts: List[str] ) -> Tuple[bool, Optional[List[Any]]]: - obj = request_tensor_for_text(refinery_project_id, embedding_id, texts) - return obj.get("tensor", None) @@ -218,28 +219,15 @@ def __get_most_similar_records( threshold: Optional[float] = None, question: Optional[str] = None, ): - url = f"{NEURAL_SEARCH}/most_similar_by_embedding?include_scores=true" - - response = requests.post( - url, - headers={ - "accept": "application/json", - "content-type": "application/json", - }, - json={ - "project_id": project_id, - "embedding_id": embedding_id, - "embedding_tensor": embedding_tensor, - "limit": limit, - "att_filter": similarity_filter_option, - "threshold": threshold, - "question": question, - }, + return request_most_similar_records( + project_id, + embedding_id, + embedding_tensor, + limit, + similarity_filter_option, + threshold, + question, ) - if response.ok: - return response.json() - else: - return None def get_records_by_content( From 19a359bbd11c4e1a8e335ec4eaf2c8387d7378f2 Mon Sep 17 00:00:00 2001 From: anmarhindi Date: Thu, 6 Feb 2025 10:46:32 +0100 Subject: [PATCH 68/76] Remove unused import --- controller/playground/manager.py | 1 - 1 file changed, 1 deletion(-) diff --git a/controller/playground/manager.py b/controller/playground/manager.py index a14d121f..c140f36e 100644 --- a/controller/playground/manager.py +++ b/controller/playground/manager.py @@ -1,6 +1,5 @@ from typing import List, Any, Optional, Tuple, Dict import os -import requests from controller.embedding.connector import ( request_most_similar_records, request_tensor_for_text, From 7278172ebf66c7e21f4e5c205ab3f114b4bce8fd Mon Sep 17 00:00:00 2001 From: anmarhindi Date: Thu, 6 Feb 2025 11:19:48 +0100 Subject: [PATCH 69/76] Get enum value for state --- controller/playground/manager.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/controller/playground/manager.py b/controller/playground/manager.py index c140f36e..78028f82 100644 --- a/controller/playground/manager.py +++ b/controller/playground/manager.py @@ -132,7 +132,7 @@ def init_evaluation_run( evaluation_group_id, created_by, embedding_id, - EvaluationRunState.RUNNING, + EvaluationRunState.RUNNING.value, ) evaluation_results = [] try: @@ -193,9 +193,9 @@ def init_evaluation_run( ), } evaluation_results.append(result) - state = EvaluationRunState.SUCCESS + state = EvaluationRunState.SUCCESS.value except Exception: - state = EvaluationRunState.FAILED + state = EvaluationRunState.FAILED.value evaluation_run_db_bo.update( project_id, evaluation_run.id, state, evaluation_results, None, True ) From 1d8da3d448e5bc7f8aea492ad4b28f91f481dcd9 Mon Sep 17 00:00:00 2001 From: anmarhindi Date: Thu, 6 Feb 2025 11:21:35 +0100 Subject: [PATCH 70/76] sub ref --- submodules/model | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/submodules/model b/submodules/model index c2acd126..2ff8904b 160000 --- a/submodules/model +++ b/submodules/model @@ -1 +1 @@ -Subproject commit c2acd1268212385d243cc6d27e9df9fa4584e784 +Subproject commit 2ff8904b7ed095bed95c68032c399c4d70c1600f From c69865087d64bf5982f5f8b62fe549de94cc8e04 Mon Sep 17 00:00:00 2001 From: anmarhindi Date: Thu, 6 Feb 2025 11:24:39 +0100 Subject: [PATCH 71/76] sub ref --- submodules/model | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/submodules/model b/submodules/model index 2ff8904b..89bda2cd 160000 --- a/submodules/model +++ b/submodules/model @@ -1 +1 @@ -Subproject commit 2ff8904b7ed095bed95c68032c399c4d70c1600f +Subproject commit 89bda2cded0e4988497b678da32007e311cb6014 From 672eeb0197f067257a419b881ee9c442cb7c37dc Mon Sep 17 00:00:00 2001 From: anmarhindi Date: Thu, 6 Feb 2025 11:33:35 +0100 Subject: [PATCH 72/76] sub ref --- submodules/model | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/submodules/model b/submodules/model index 89bda2cd..86743c4b 160000 --- a/submodules/model +++ b/submodules/model @@ -1 +1 @@ -Subproject commit 89bda2cded0e4988497b678da32007e311cb6014 +Subproject commit 86743c4b8329477ada2eacbe6522c59bda5464da From 8bd9f9aaf113aee5c2944d39e4e3b184013c8df6 Mon Sep 17 00:00:00 2001 From: anmarhindi Date: Thu, 6 Feb 2025 11:46:04 +0100 Subject: [PATCH 73/76] sub ref --- submodules/model | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/submodules/model b/submodules/model index 86743c4b..57119687 160000 --- a/submodules/model +++ b/submodules/model @@ -1 +1 @@ -Subproject commit 86743c4b8329477ada2eacbe6522c59bda5464da +Subproject commit 57119687d11eaa459bc99cc5356c1054f570b0be From 79ad2df22a7560bdb94cce50c1ca850da119904f Mon Sep 17 00:00:00 2001 From: anmarhindi Date: Thu, 6 Feb 2025 14:16:29 +0100 Subject: [PATCH 74/76] Add traceback to exception handling --- controller/playground/manager.py | 2 ++ submodules/model | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/controller/playground/manager.py b/controller/playground/manager.py index 78028f82..5406e0d5 100644 --- a/controller/playground/manager.py +++ b/controller/playground/manager.py @@ -1,3 +1,4 @@ +import traceback from typing import List, Any, Optional, Tuple, Dict import os from controller.embedding.connector import ( @@ -195,6 +196,7 @@ def init_evaluation_run( evaluation_results.append(result) state = EvaluationRunState.SUCCESS.value except Exception: + traceback.print_exc() state = EvaluationRunState.FAILED.value evaluation_run_db_bo.update( project_id, evaluation_run.id, state, evaluation_results, None, True diff --git a/submodules/model b/submodules/model index 57119687..565c2295 160000 --- a/submodules/model +++ b/submodules/model @@ -1 +1 @@ -Subproject commit 57119687d11eaa459bc99cc5356c1054f570b0be +Subproject commit 565c2295d936f668fb29182ec057b191361ce931 From 2c4fc72867ada6449709e2c0ea1b7f1d08568ba9 Mon Sep 17 00:00:00 2001 From: Lina Date: Fri, 7 Feb 2025 11:19:26 +0100 Subject: [PATCH 75/76] Deleting questions and order desc --- controller/playground/manager.py | 4 ++++ fast_api/routes/playground.py | 13 +++++++++++++ submodules/model | 2 +- 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/controller/playground/manager.py b/controller/playground/manager.py index 5406e0d5..468ba327 100644 --- a/controller/playground/manager.py +++ b/controller/playground/manager.py @@ -296,3 +296,7 @@ def get_question_reformulation(question: str, api_key: str) -> Optional[Dict]: def get_playground_questions(project_id: str): return playground_question_db_bo.get_all(project_id) + + +def delete_playground_question(project_id: str, question_id: str): + playground_question_db_bo.delete(project_id, question_id) diff --git a/fast_api/routes/playground.py b/fast_api/routes/playground.py index 9ecf65a6..7bb16d5e 100644 --- a/fast_api/routes/playground.py +++ b/fast_api/routes/playground.py @@ -270,3 +270,16 @@ def get_question_reformulation( question_reformulation.question, question_reformulation.apiKey ) return pack_json_result(reformulation) + + +@router.delete( + "/{project_id}/playground-questions/{question_id}", + dependencies=[Depends(auth_manager.check_project_access_dep)], +) +def delete_playground_question( + request: Request, + project_id: str, + question_id: str, +): + playground_manager.delete_playground_question(project_id, question_id) + return get_silent_success() diff --git a/submodules/model b/submodules/model index 565c2295..0eeed19b 160000 --- a/submodules/model +++ b/submodules/model @@ -1 +1 @@ -Subproject commit 565c2295d936f668fb29182ec057b191361ce931 +Subproject commit 0eeed19bf1709c694c540fbf05fe3fa152cce3bd From 91ba23a50da03e597429efbf8237c5368f41ba8f Mon Sep 17 00:00:00 2001 From: anmarhindi Date: Thu, 13 Feb 2025 12:00:45 +0100 Subject: [PATCH 76/76] sub ref --- submodules/model | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/submodules/model b/submodules/model index 0eeed19b..8e59d8ef 160000 --- a/submodules/model +++ b/submodules/model @@ -1 +1 @@ -Subproject commit 0eeed19bf1709c694c540fbf05fe3fa152cce3bd +Subproject commit 8e59d8ef75b0ade84d1ef4d0b7e957eb324c2bc8