From 42cc96bed5435b9013ba6db81538d76ad772530b Mon Sep 17 00:00:00 2001 From: MustafaAkolawala Date: Sat, 7 Sep 2024 03:11:56 +0530 Subject: [PATCH 1/6] Add A flag to Toggle /voice-search on or off --- core_backend/app/config.py | 1 + core_backend/app/question_answer/routers.py | 230 +++++++++--------- .../docker-compose/template.core_backend.env | 5 + 3 files changed, 125 insertions(+), 111 deletions(-) diff --git a/core_backend/app/config.py b/core_backend/app/config.py index 6031f9b1a..9f21079da 100644 --- a/core_backend/app/config.py +++ b/core_backend/app/config.py @@ -83,6 +83,7 @@ BACKEND_ROOT_PATH = os.environ.get("BACKEND_ROOT_PATH", "") # Speech API +TOGGLE_SPEECH = os.environ.get("TOGGLE_SPEECH", None) CUSTOM_SPEECH_ENDPOINT = os.environ.get("CUSTOM_SPEECH_ENDPOINT", None) # Logging LANGFUSE = os.environ.get("LANGFUSE", "False") diff --git a/core_backend/app/question_answer/routers.py b/core_backend/app/question_answer/routers.py index 5ce853469..2a1209948 100644 --- a/core_backend/app/question_answer/routers.py +++ b/core_backend/app/question_answer/routers.py @@ -13,7 +13,12 @@ from sqlalchemy.ext.asyncio import AsyncSession from ..auth.dependencies import authenticate_key, rate_limiter -from ..config import CUSTOM_SPEECH_ENDPOINT, GCS_SPEECH_BUCKET, USE_CROSS_ENCODER +from ..config import ( + CUSTOM_SPEECH_ENDPOINT, + GCS_SPEECH_BUCKET, + TOGGLE_SPEECH, + USE_CROSS_ENCODER, +) from ..contents.models import ( get_similar_content_async, increment_query_count, @@ -157,134 +162,137 @@ async def search( ) -@router.post( - "/voice-search", - response_model=QueryAudioResponse, - responses={ - status.HTTP_400_BAD_REQUEST: { - "model": QueryResponseError, - "description": "Bad Request", - }, - status.HTTP_500_INTERNAL_SERVER_ERROR: { - "model": QueryResponseError, - "description": "Internal Server Error", +if TOGGLE_SPEECH is not None: + + @router.post( + "/voice-search", + response_model=QueryAudioResponse, + responses={ + status.HTTP_400_BAD_REQUEST: { + "model": QueryResponseError, + "description": "Bad Request", + }, + status.HTTP_500_INTERNAL_SERVER_ERROR: { + "model": QueryResponseError, + "description": "Internal Server Error", + }, }, - }, -) -async def voice_search( - file_url: str, - request: Request, - asession: AsyncSession = Depends(get_async_session), - user_db: UserDB = Depends(authenticate_key), -) -> QueryAudioResponse | JSONResponse: - """ - Endpoint to transcribe audio from a provided URL, - generate an LLM response, by default generate_tts is - set to true and return a public random URL of an audio - file containing the spoken version of the generated response. - """ - try: - file_stream, content_type, file_extension = await download_file_from_url( - file_url - ) + ) + async def voice_search( + file_url: str, + request: Request, + asession: AsyncSession = Depends(get_async_session), + user_db: UserDB = Depends(authenticate_key), + ) -> QueryAudioResponse | JSONResponse: + """ + Endpoint to transcribe audio from a provided URL, + generate an LLM response, by default generate_tts is + set to true and return a public random URL of an audio + file containing the spoken version of the generated response. + """ + try: + file_stream, content_type, file_extension = await download_file_from_url( + file_url + ) - unique_filename = generate_random_filename(file_extension) - destination_blob_name = f"stt-voice-notes/{unique_filename}" + unique_filename = generate_random_filename(file_extension) + destination_blob_name = f"stt-voice-notes/{unique_filename}" - await upload_file_to_gcs( - GCS_SPEECH_BUCKET, file_stream, destination_blob_name, content_type - ) + await upload_file_to_gcs( + GCS_SPEECH_BUCKET, file_stream, destination_blob_name, content_type + ) - file_path = f"temp/{unique_filename}" - with open(file_path, "wb") as f: + file_path = f"temp/{unique_filename}" + with open(file_path, "wb") as f: + file_stream.seek(0) + f.write(file_stream.read()) file_stream.seek(0) - f.write(file_stream.read()) - file_stream.seek(0) - - if CUSTOM_SPEECH_ENDPOINT is not None: - transcription = await post_to_speech(file_path, CUSTOM_SPEECH_ENDPOINT) - transcription_result = transcription["text"] - else: - transcription_result = await transcribe_audio(file_path) - - user_query = QueryBase( - generate_llm_response=True, - query_text=transcription_result, - query_metadata={}, - ) - ( - user_query_db, - user_query_refined_template, - response_template, - ) = await get_user_query_and_response( - user_id=user_db.user_id, - user_query=user_query, - asession=asession, - generate_tts=True, - ) + if CUSTOM_SPEECH_ENDPOINT is not None: + transcription = await post_to_speech(file_path, CUSTOM_SPEECH_ENDPOINT) + transcription_result = transcription["text"] + else: + transcription_result = await transcribe_audio(file_path) - response = await get_search_response( - query_refined=user_query_refined_template, - response=response_template, - user_id=user_db.user_id, - n_similar=int(N_TOP_CONTENT), - n_to_crossencoder=int(N_TOP_CONTENT_TO_CROSSENCODER), - asession=asession, - exclude_archived=True, - request=request, - ) + user_query = QueryBase( + generate_llm_response=True, + query_text=transcription_result, + query_metadata={}, + ) + + ( + user_query_db, + user_query_refined_template, + response_template, + ) = await get_user_query_and_response( + user_id=user_db.user_id, + user_query=user_query, + asession=asession, + generate_tts=True, + ) - if user_query.generate_llm_response: - response = await get_generation_response( + response = await get_search_response( query_refined=user_query_refined_template, - response=response, + response=response_template, + user_id=user_db.user_id, + n_similar=int(N_TOP_CONTENT), + n_to_crossencoder=int(N_TOP_CONTENT_TO_CROSSENCODER), + asession=asession, + exclude_archived=True, + request=request, ) - await save_query_response_to_db(user_query_db, response, asession) - await increment_query_count( - user_id=user_db.user_id, - contents=response.search_results, - asession=asession, - ) - await save_content_for_query_to_db( - user_id=user_db.user_id, - query_id=response.query_id, - session_id=user_query.session_id, - contents=response.search_results, - asession=asession, - ) + if user_query.generate_llm_response: + response = await get_generation_response( + query_refined=user_query_refined_template, + response=response, + ) + + await save_query_response_to_db(user_query_db, response, asession) + await increment_query_count( + user_id=user_db.user_id, + contents=response.search_results, + asession=asession, + ) + await save_content_for_query_to_db( + user_id=user_db.user_id, + query_id=response.query_id, + session_id=user_query.session_id, + contents=response.search_results, + asession=asession, + ) + + if os.path.exists(file_path): + os.remove(file_path) + file_stream.close() - if os.path.exists(file_path): - os.remove(file_path) - file_stream.close() + if type(response) is QueryAudioResponse: + return response - if type(response) is QueryAudioResponse: - return response + if type(response) is QueryResponseError: + return JSONResponse( + status_code=status.HTTP_400_BAD_REQUEST, + content=response.model_dump(), + ) - if type(response) is QueryResponseError: return JSONResponse( - status_code=status.HTTP_400_BAD_REQUEST, content=response.model_dump() + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + content={"error": "Internal server error"}, ) - return JSONResponse( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - content={"error": "Internal server error"}, - ) - - except ValueError as ve: - logger.error(f"ValueError: {str(ve)}") - return JSONResponse( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - content={"error": f"Value error: {str(ve)}"}, - ) + except ValueError as ve: + logger.error(f"ValueError: {str(ve)}") + return JSONResponse( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + content={"error": f"Value error: {str(ve)}"}, + ) - except Exception as e: - logger.error(f"Unexpected error: {str(e)}") - return JSONResponse( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - content={"error": "Internal server error"}, - ) + except Exception as e: + logger.error(f"Unexpected error: {str(e)}") + return JSONResponse( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + content={"error": "Internal server error"}, + ) async def download_file_from_url(file_url: str) -> tuple[BytesIO, str, str]: diff --git a/deployment/docker-compose/template.core_backend.env b/deployment/docker-compose/template.core_backend.env index 53d2b2266..59828f356 100644 --- a/deployment/docker-compose/template.core_backend.env +++ b/deployment/docker-compose/template.core_backend.env @@ -34,6 +34,11 @@ LITELLM_ENDPOINT="http://localhost:4000" #PGVECTOR_VECTOR_SIZE=1024 #### Speech APIs ############################################################### +# Set this variable to 'External' or 'Custom' accordingly to toggle the /voice-search endpoint +# By default it is not set so it defaults to None +# TOGGLE_SPEECH=External + +# if TOGGLE_SPEECH is set to 'Custom' then make sure to also set the Environment variables mentioned below # CUSTOM_SPEECH_ENDPOINT=http://speech_service:8001/transcribe #### Temporary folder for prometheus gunicorn multiprocess #################### From 63c52d794cf27a0a7ae90be2c0d45fac27685623 Mon Sep 17 00:00:00 2001 From: MustafaAkolawala Date: Tue, 10 Sep 2024 01:25:06 +0530 Subject: [PATCH 2/6] Fix Pytests --- Makefile | 1 + core_backend/Makefile | 2 +- core_backend/app/config.py | 2 +- core_backend/app/question_answer/routers.py | 4 ++-- core_backend/tests/api/test.env | 1 + deployment/docker-compose/template.core_backend.env | 6 +++--- 6 files changed, 9 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index b118ece4c..fa71aa15a 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,5 @@ #!make +SHELL := /bin/bash PROJECT_NAME = aaq CONDA_ACTIVATE=source $$(conda info --base)/etc/profile.d/conda.sh ; conda activate ; conda activate diff --git a/core_backend/Makefile b/core_backend/Makefile index 10153cb86..43d2b2acf 100644 --- a/core_backend/Makefile +++ b/core_backend/Makefile @@ -1,4 +1,5 @@ #!make +SHELL := /bin/bash .PHONY : tests @@ -49,4 +50,3 @@ teardown-redis-test: teardown-test-db: @docker stop testdb @docker rm testdb - diff --git a/core_backend/app/config.py b/core_backend/app/config.py index 9f21079da..703c182ed 100644 --- a/core_backend/app/config.py +++ b/core_backend/app/config.py @@ -83,7 +83,7 @@ BACKEND_ROOT_PATH = os.environ.get("BACKEND_ROOT_PATH", "") # Speech API -TOGGLE_SPEECH = os.environ.get("TOGGLE_SPEECH", None) +TOGGLE_VOICE = os.environ.get("TOGGLE_VOICE", None) CUSTOM_SPEECH_ENDPOINT = os.environ.get("CUSTOM_SPEECH_ENDPOINT", None) # Logging LANGFUSE = os.environ.get("LANGFUSE", "False") diff --git a/core_backend/app/question_answer/routers.py b/core_backend/app/question_answer/routers.py index 2a1209948..9ecdfaf53 100644 --- a/core_backend/app/question_answer/routers.py +++ b/core_backend/app/question_answer/routers.py @@ -16,7 +16,7 @@ from ..config import ( CUSTOM_SPEECH_ENDPOINT, GCS_SPEECH_BUCKET, - TOGGLE_SPEECH, + TOGGLE_VOICE, USE_CROSS_ENCODER, ) from ..contents.models import ( @@ -162,7 +162,7 @@ async def search( ) -if TOGGLE_SPEECH is not None: +if TOGGLE_VOICE is not None: @router.post( "/voice-search", diff --git a/core_backend/tests/api/test.env b/core_backend/tests/api/test.env index c359cbcbd..44a2d7a60 100644 --- a/core_backend/tests/api/test.env +++ b/core_backend/tests/api/test.env @@ -12,3 +12,4 @@ ALIGN_SCORE_API="http://localhost:5002/alignscore_base" # Speech Api endpoint # if u want to try the tests for the external TTS and STT apis then comment this out CUSTOM_SPEECH_ENDPOINT="http://localhost:8001/transcribe" +TOGGLE_VOICE=custom diff --git a/deployment/docker-compose/template.core_backend.env b/deployment/docker-compose/template.core_backend.env index 59828f356..f9f08bf10 100644 --- a/deployment/docker-compose/template.core_backend.env +++ b/deployment/docker-compose/template.core_backend.env @@ -34,11 +34,11 @@ LITELLM_ENDPOINT="http://localhost:4000" #PGVECTOR_VECTOR_SIZE=1024 #### Speech APIs ############################################################### -# Set this variable to 'External' or 'Custom' accordingly to toggle the /voice-search endpoint +# Set this variable to 'external' or 'custom' accordingly to toggle the /voice-search endpoint # By default it is not set so it defaults to None -# TOGGLE_SPEECH=External +# TOGGLE_VOICE=external -# if TOGGLE_SPEECH is set to 'Custom' then make sure to also set the Environment variables mentioned below +# if TOGGLE_VOICE is set to 'Custom' then make sure to also set the Environment variables mentioned below # CUSTOM_SPEECH_ENDPOINT=http://speech_service:8001/transcribe #### Temporary folder for prometheus gunicorn multiprocess #################### From 27570bca6b2b97e267de86d70e21189552b50b8e Mon Sep 17 00:00:00 2001 From: MustafaAkolawala Date: Tue, 10 Sep 2024 12:09:08 +0530 Subject: [PATCH 3/6] Fix issue in GHA workflow of backend unit-tests --- .github/workflows/tests.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index d98670ada..75caed620 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -12,6 +12,8 @@ env: POSTGRES_USER: postgres-test-user POSTGRES_DB: postgres-test-db REDIS_HOST: redis://redis:6379 + TOGGLE_VOICE: custom + jobs: container-job: runs-on: ubuntu-20.04 @@ -55,6 +57,7 @@ jobs: env: PROMETHEUS_MULTIPROC_DIR: /tmp REDIS_HOST: ${{ env.REDIS_HOST }} + TOGGLE_VOICE: ${{ env.TOGGLE_VOICE }} run: | cd core_backend export POSTGRES_HOST=postgres POSTGRES_USER=$POSTGRES_USER \ From 9740d8c4ff72a814b269def5e6f907f21d443aa2 Mon Sep 17 00:00:00 2001 From: MustafaAkolawala Date: Wed, 6 Nov 2024 22:17:06 +0530 Subject: [PATCH 4/6] change ENABLE_VOICE_SEARCH to bool --- core_backend/app/config.py | 3 ++- core_backend/app/question_answer/routers.py | 4 ++-- deployment/docker-compose/template.core_backend.env | 6 +++--- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/core_backend/app/config.py b/core_backend/app/config.py index 703c182ed..10923e31f 100644 --- a/core_backend/app/config.py +++ b/core_backend/app/config.py @@ -83,7 +83,8 @@ BACKEND_ROOT_PATH = os.environ.get("BACKEND_ROOT_PATH", "") # Speech API -TOGGLE_VOICE = os.environ.get("TOGGLE_VOICE", None) +ENABLE_VOICE_SEARCH = os.getenv("ENABLE_VOICE_SEARCH", "False").lower() == "true" + CUSTOM_SPEECH_ENDPOINT = os.environ.get("CUSTOM_SPEECH_ENDPOINT", None) # Logging LANGFUSE = os.environ.get("LANGFUSE", "False") diff --git a/core_backend/app/question_answer/routers.py b/core_backend/app/question_answer/routers.py index 9ecdfaf53..546f7872e 100644 --- a/core_backend/app/question_answer/routers.py +++ b/core_backend/app/question_answer/routers.py @@ -15,8 +15,8 @@ from ..auth.dependencies import authenticate_key, rate_limiter from ..config import ( CUSTOM_SPEECH_ENDPOINT, + ENABLE_VOICE_SEARCH, GCS_SPEECH_BUCKET, - TOGGLE_VOICE, USE_CROSS_ENCODER, ) from ..contents.models import ( @@ -162,7 +162,7 @@ async def search( ) -if TOGGLE_VOICE is not None: +if ENABLE_VOICE_SEARCH: @router.post( "/voice-search", diff --git a/deployment/docker-compose/template.core_backend.env b/deployment/docker-compose/template.core_backend.env index f9f08bf10..29916dbed 100644 --- a/deployment/docker-compose/template.core_backend.env +++ b/deployment/docker-compose/template.core_backend.env @@ -34,9 +34,9 @@ LITELLM_ENDPOINT="http://localhost:4000" #PGVECTOR_VECTOR_SIZE=1024 #### Speech APIs ############################################################### -# Set this variable to 'external' or 'custom' accordingly to toggle the /voice-search endpoint -# By default it is not set so it defaults to None -# TOGGLE_VOICE=external +# This variable controls whether the voice search endpoint is active (set to true) or inactive (set to false). Default is false. +# If enabled, we default to using external services unless `CUSTOM_SPEECH_ENDPOINT` is set, in which case the custom hosted APIs will be used. +# ENABLE_VOICE_SEARCH=True # if TOGGLE_VOICE is set to 'Custom' then make sure to also set the Environment variables mentioned below # CUSTOM_SPEECH_ENDPOINT=http://speech_service:8001/transcribe From 68f07b28348608b6038c276d4de7fd87ab9c110c Mon Sep 17 00:00:00 2001 From: MustafaAkolawala Date: Wed, 6 Nov 2024 23:33:47 +0530 Subject: [PATCH 5/6] fix the unit test workflow --- .github/workflows/tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 75caed620..a08f942e6 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -12,7 +12,7 @@ env: POSTGRES_USER: postgres-test-user POSTGRES_DB: postgres-test-db REDIS_HOST: redis://redis:6379 - TOGGLE_VOICE: custom + TOGGLE_VOICE: True jobs: container-job: From 8ac76e5205c22f4545d24f8baa1bd7814a10a12e Mon Sep 17 00:00:00 2001 From: MustafaAkolawala Date: Wed, 6 Nov 2024 23:45:17 +0530 Subject: [PATCH 6/6] fix variable name issue --- .github/workflows/tests.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index a08f942e6..99cc75194 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -12,7 +12,7 @@ env: POSTGRES_USER: postgres-test-user POSTGRES_DB: postgres-test-db REDIS_HOST: redis://redis:6379 - TOGGLE_VOICE: True + ENABLE_VOICE_SEARCH: True jobs: container-job: @@ -57,7 +57,7 @@ jobs: env: PROMETHEUS_MULTIPROC_DIR: /tmp REDIS_HOST: ${{ env.REDIS_HOST }} - TOGGLE_VOICE: ${{ env.TOGGLE_VOICE }} + ENABLE_VOICE_SEARCH: ${{ env.ENABLE_VOICE_SEARCH }} run: | cd core_backend export POSTGRES_HOST=postgres POSTGRES_USER=$POSTGRES_USER \