From 23012d4dd5c32d2a2d845670a8abb7078f8c75d7 Mon Sep 17 00:00:00 2001 From: anmarhindi Date: Tue, 14 Jan 2025 17:13:29 +0100 Subject: [PATCH 01/26] Add initial table --- enums.py | 1 + models.py | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/enums.py b/enums.py index 3e64a5e..74d5072 100644 --- a/enums.py +++ b/enums.py @@ -145,6 +145,7 @@ class Tablenames(Enum): PIPELINE_VERSION = ( "pipeline_version" # dump of previous versions to easily jump between ) + EVALUATION_SET = "evaluation_set" def snake_case_to_pascal_case(self): # the type name (written in PascalCase) of a table is needed to create backrefs diff --git a/models.py b/models.py index 33b4592..42ac106 100644 --- a/models.py +++ b/models.py @@ -1862,3 +1862,21 @@ class CustomerButton(Base): ForeignKey(f"{Tablenames.USER.value}.id", ondelete="SET NULL"), index=True, ) + + +class EvaluationSet(Base): + __tablename__ = Tablenames.EVALUATION_SET.value + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + created_at = Column(DateTime, default=sql.func.now()) + created_by = Column( + UUID(as_uuid=True), + ForeignKey(f"{Tablenames.USER.value}.id", ondelete="SET NULL"), + index=True, + ) + project_id = Column( + UUID(as_uuid=True), + ForeignKey(f"{Tablenames.PROJECT.value}.id", ondelete="CASCADE"), + index=True, + ) + name = Column(String) + data = Column(JSON) From 3587c7d33a1023ca500812c62f088bb734c98893 Mon Sep 17 00:00:00 2001 From: anmarhindi Date: Thu, 16 Jan 2025 14:11:56 +0100 Subject: [PATCH 02/26] Add evaluation models --- enums.py | 2 ++ models.py | 49 ++++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 50 insertions(+), 1 deletion(-) diff --git a/enums.py b/enums.py index 74d5072..2f2620c 100644 --- a/enums.py +++ b/enums.py @@ -146,6 +146,8 @@ class Tablenames(Enum): "pipeline_version" # dump of previous versions to easily jump between ) EVALUATION_SET = "evaluation_set" + EVALUATION_GROUP = "evaluation_group" + EVALUATION_RUN = "evaluation_run" def snake_case_to_pascal_case(self): # the type name (written in PascalCase) of a table is needed to create backrefs diff --git a/models.py b/models.py index 42ac106..898c43f 100644 --- a/models.py +++ b/models.py @@ -1867,6 +1867,7 @@ class CustomerButton(Base): class EvaluationSet(Base): __tablename__ = Tablenames.EVALUATION_SET.value id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + question = Column(String) created_at = Column(DateTime, default=sql.func.now()) created_by = Column( UUID(as_uuid=True), @@ -1878,5 +1879,51 @@ class EvaluationSet(Base): ForeignKey(f"{Tablenames.PROJECT.value}.id", ondelete="CASCADE"), index=True, ) + record_ids = Column(JSON) + + +class EvaluationGroup(Base): + __tablename__ = Tablenames.EVALUATION_GROUP.value + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) name = Column(String) - data = Column(JSON) + created_at = Column(DateTime, default=sql.func.now()) + created_by = Column( + UUID(as_uuid=True), + ForeignKey(f"{Tablenames.USER.value}.id", ondelete="SET NULL"), + index=True, + ) + project_id = Column( + UUID(as_uuid=True), + ForeignKey(f"{Tablenames.PROJECT.value}.id", ondelete="CASCADE"), + index=True, + ) + evaluation_set_ids = Column(JSON) + + +class EvaluationRun(Base): + __tablename__ = Tablenames.EVALUATION_RUN.value + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + evaluation_group_id = Column( + UUID(as_uuid=True), + ForeignKey(f"{Tablenames.EVALUATION_GROUP.value}.id", ondelete="SET NULL"), + index=True, + ) + created_at = Column(DateTime, default=sql.func.now()) + created_by = Column( + UUID(as_uuid=True), + ForeignKey(f"{Tablenames.USER.value}.id", ondelete="SET NULL"), + index=True, + ) + project_id = Column( + UUID(as_uuid=True), + ForeignKey(f"{Tablenames.PROJECT.value}.id", ondelete="CASCADE"), + index=True, + ) + embedding_id = Column( + UUID(as_uuid=True), + ForeignKey(f"{Tablenames.EMBEDDING.value}.id", ondelete="SET NULL"), + index=True, + ) + state = Column(String) + results = Column(JSON) + meta_info = Column(JSON) From 1654d716c6c7b0e3da9740d508dfc593d19a3ebf Mon Sep 17 00:00:00 2001 From: anmarhindi Date: Thu, 16 Jan 2025 16:16:32 +0100 Subject: [PATCH 03/26] add evaluation set accessors --- business_objects/evaluation_group.py | 0 business_objects/evaluation_set.py | 42 ++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+) create mode 100644 business_objects/evaluation_group.py create mode 100644 business_objects/evaluation_set.py diff --git a/business_objects/evaluation_group.py b/business_objects/evaluation_group.py new file mode 100644 index 0000000..e69de29 diff --git a/business_objects/evaluation_set.py b/business_objects/evaluation_set.py new file mode 100644 index 0000000..4b9f1e9 --- /dev/null +++ b/business_objects/evaluation_set.py @@ -0,0 +1,42 @@ +from typing import List + +from ..models import EvaluationSet +from ..session import session +from . import general + + +def get(project_id: str, evaluation_set_id: str) -> EvaluationSet: + query = session.query(EvaluationSet).filter( + EvaluationSet.project_id == project_id, + EvaluationSet.id == evaluation_set_id, + ) + return query.first() + + +def get_all(project_id: str) -> List[EvaluationSet]: + query = session.query(EvaluationSet).filter( + EvaluationSet.project_id == project_id, + ) + query = query.order_by(EvaluationSet.question) + return query.all() + + +def create( + project_id: str, + question: str, + created_by: str, + name: str, + record_ids: List[str], + with_commit: bool = False, +) -> EvaluationSet: + eval_set = EvaluationSet( + project_id=project_id, + question=question, + created_by=created_by, + name=name, + record_ids=record_ids, + ) + + general.add(eval_set, with_commit) + + return eval_set From 0a4cbd8f7981da1ff4bb818a617b95951eb1adce Mon Sep 17 00:00:00 2001 From: anmarhindi Date: Thu, 16 Jan 2025 16:21:35 +0100 Subject: [PATCH 04/26] add evaluation group accessors --- business_objects/evaluation_group.py | 40 ++++++++++++++++++++++++++++ business_objects/evaluation_set.py | 2 -- 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/business_objects/evaluation_group.py b/business_objects/evaluation_group.py index e69de29..0eb9380 100644 --- a/business_objects/evaluation_group.py +++ b/business_objects/evaluation_group.py @@ -0,0 +1,40 @@ +from typing import List + +from ..models import EvaluationGroup +from ..session import session +from . import general + + +def get(project_id: str, evaluation_group_id: str) -> EvaluationGroup: + query = session.query(EvaluationGroup).filter( + EvaluationGroup.project_id == project_id, + EvaluationGroup.id == evaluation_group_id, + ) + return query.first() + + +def get_all(project_id: str) -> List[EvaluationGroup]: + query = session.query(EvaluationGroup).filter( + EvaluationGroup.project_id == project_id, + ) + query = query.order_by(EvaluationGroup.name) + return query.all() + + +def create( + project_id: str, + name: str, + created_by: str, + evaluation_set_ids: List[str], + with_commit: bool = False, +) -> EvaluationGroup: + eval_group = EvaluationGroup( + project_id=project_id, + name=name, + created_by=created_by, + evaluation_set_ids=evaluation_set_ids, + ) + + general.add(eval_group, with_commit) + + return eval_group diff --git a/business_objects/evaluation_set.py b/business_objects/evaluation_set.py index 4b9f1e9..9f1633e 100644 --- a/business_objects/evaluation_set.py +++ b/business_objects/evaluation_set.py @@ -25,7 +25,6 @@ def create( project_id: str, question: str, created_by: str, - name: str, record_ids: List[str], with_commit: bool = False, ) -> EvaluationSet: @@ -33,7 +32,6 @@ def create( project_id=project_id, question=question, created_by=created_by, - name=name, record_ids=record_ids, ) From d7a140148b55d56f7722de51378835697b96b332 Mon Sep 17 00:00:00 2001 From: anmarhindi Date: Thu, 16 Jan 2025 16:51:54 +0100 Subject: [PATCH 05/26] add eval run bobj --- business_objects/evaluation_run.py | 72 ++++++++++++++++++++++++++++++ enums.py | 6 +++ 2 files changed, 78 insertions(+) create mode 100644 business_objects/evaluation_run.py diff --git a/business_objects/evaluation_run.py b/business_objects/evaluation_run.py new file mode 100644 index 0000000..1889bd4 --- /dev/null +++ b/business_objects/evaluation_run.py @@ -0,0 +1,72 @@ +from typing import List, Optional + +from submodules.model.enums import EvaluationRunState + +from ..models import EvaluationRun +from ..session import session +from . import general + + +def get(project_id: str, evaluation_run_id: str) -> EvaluationRun: + query = session.query(EvaluationRun).filter( + EvaluationRun.project_id == project_id, + EvaluationRun.id == evaluation_run_id, + ) + return query.first() + + +def get_all_by_embedding_id(project_id: str, embedding_id: str) -> EvaluationRun: + query = session.query(EvaluationRun).filter( + EvaluationRun.project_id == project_id, + EvaluationRun.embedding_id == embedding_id, + ) + query = query.order_by(EvaluationRun.created_at.asc()) + return query.all() + + +def get_all_by_evaluation_group_id( + project_id: str, evaluation_group_id: str +) -> EvaluationRun: + query = session.query(EvaluationRun).filter( + EvaluationRun.project_id == project_id, + EvaluationRun.evaluation_group_id == evaluation_group_id, + ) + query = query.order_by(EvaluationRun.created_at.asc()) + return query.all() + + +def get_all(project_id: str) -> List[EvaluationRun]: + query = session.query(EvaluationRun).filter( + EvaluationRun.project_id == project_id, + ) + query = query.order_by(EvaluationRun.created_at.asc()) + return query.all() + + +def create( + project_id: str, + evaluation_group_id: str, + created_by: str, + embedding_id: str, + state: EvaluationRunState, + results: Optional[str] = None, + meta_info: Optional[str] = None, + with_commit: bool = False, +) -> EvaluationRun: + eval_run = EvaluationRun( + evaluation_group_id=evaluation_group_id, + created_by=created_by, + project_id=project_id, + embedding_id=embedding_id, + state=state, + ) + + if results is not None: + eval_run.results = results + + if meta_info is not None: + eval_run.meta_info = meta_info + + general.add(eval_run, with_commit) + + return eval_run diff --git a/enums.py b/enums.py index 2f2620c..bd2a127 100644 --- a/enums.py +++ b/enums.py @@ -802,3 +802,9 @@ class ChangeAction(Enum): class PipelineVersionType(Enum): AUTO_SAVE = "AUTO_SAVE" # any save operation in relevant but only 10 per project NAMED_VERSION = "NAMED_VERSION" # any AUTO_SAVE that is considered worth keeping + + +class EvaluationRunState(Enum): + STARTED = "STARTED" + FINISHED = "FINISHED" + FAILED = "FAILED" From 74e3e05a71e465d44f07603e0ad86066d0366be9 Mon Sep 17 00:00:00 2001 From: anmarhindi Date: Thu, 16 Jan 2025 16:58:59 +0100 Subject: [PATCH 06/26] add update bobj --- business_objects/evaluation_run.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/business_objects/evaluation_run.py b/business_objects/evaluation_run.py index 1889bd4..e1c400e 100644 --- a/business_objects/evaluation_run.py +++ b/business_objects/evaluation_run.py @@ -70,3 +70,23 @@ def create( general.add(eval_run, with_commit) return eval_run + + +def update( + project_id: str, + evaluation_run_id: str, + state: EvaluationRunState, + results: Optional[str] = None, + meta_info: Optional[str] = None, + with_commit: bool = False, +) -> EvaluationRun: + eval_run: EvaluationRun = get(project_id, evaluation_run_id) + if state is not None: + eval_run.state = state + if results is not None: + eval_run.results = results + if meta_info is not None: + eval_run.meta_info = meta_info + + general.flush_or_commit(with_commit) + return eval_run From 18b7d0a82ee658e8508aef99ac1b8246a623b093 Mon Sep 17 00:00:00 2001 From: LennartSchmidtKern Date: Fri, 17 Jan 2025 10:31:09 +0100 Subject: [PATCH 07/26] by group id --- business_objects/evaluation_set.py | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/business_objects/evaluation_set.py b/business_objects/evaluation_set.py index 9f1633e..76e37dd 100644 --- a/business_objects/evaluation_set.py +++ b/business_objects/evaluation_set.py @@ -1,6 +1,6 @@ from typing import List -from ..models import EvaluationSet +from ..models import EvaluationSet, EvaluationGroup from ..session import session from . import general @@ -21,6 +21,26 @@ def get_all(project_id: str) -> List[EvaluationSet]: return query.all() +def get_by_evaluation_group_id( + project_id: str, evaluation_group_id: str +) -> List[EvaluationSet]: + + evaluation_group = ( + session.query(EvaluationGroup) + .filter( + EvaluationGroup.project_id == project_id, + EvaluationGroup.id == evaluation_group_id, + ) + .first() + ) + query = session.query(EvaluationSet).filter( + EvaluationSet.project_id == project_id, + EvaluationSet.id.in_(evaluation_group.evaluation_set_ids), + ) + query = query.order_by(EvaluationSet.question) + return query.all() + + def create( project_id: str, question: str, From f52dc80c38bb4df25620768a240c987b48c35bca Mon Sep 17 00:00:00 2001 From: LennartSchmidtKern Date: Tue, 21 Jan 2025 15:00:14 +0100 Subject: [PATCH 08/26] deletion --- business_objects/evaluation_group.py | 11 +++++++++++ business_objects/evaluation_set.py | 9 +++++++++ 2 files changed, 20 insertions(+) diff --git a/business_objects/evaluation_group.py b/business_objects/evaluation_group.py index 0eb9380..f4deb75 100644 --- a/business_objects/evaluation_group.py +++ b/business_objects/evaluation_group.py @@ -38,3 +38,14 @@ def create( general.add(eval_group, with_commit) return eval_group + + +def delete_all(project_id: str, group_ids: str, with_commit: bool = False): + query = session.query(EvaluationGroup).filter( + EvaluationGroup.project_id == project_id, + EvaluationGroup.id.in_(group_ids), + ) + query.delete(synchronize_session=False) + + if with_commit: + general.commit() diff --git a/business_objects/evaluation_set.py b/business_objects/evaluation_set.py index 76e37dd..c9f5fd1 100644 --- a/business_objects/evaluation_set.py +++ b/business_objects/evaluation_set.py @@ -58,3 +58,12 @@ def create( general.add(eval_set, with_commit) return eval_set + + +def delete_all(project_id: str, set_ids: List[str], with_commit: bool = False) -> None: + session.query(EvaluationSet).filter( + EvaluationSet.project_id == project_id, + EvaluationSet.id.in_(set_ids), + ).delete(synchronize_session=False) + if with_commit: + session.commit() From ccf85a6fdebcaf2cac232bfcf553a834e9acb2ba Mon Sep 17 00:00:00 2001 From: Lina Date: Fri, 24 Jan 2025 11:57:24 +0100 Subject: [PATCH 09/26] Delete evaluation runs --- business_objects/evaluation_run.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/business_objects/evaluation_run.py b/business_objects/evaluation_run.py index e1c400e..3968874 100644 --- a/business_objects/evaluation_run.py +++ b/business_objects/evaluation_run.py @@ -90,3 +90,14 @@ def update( general.flush_or_commit(with_commit) return eval_run + + +def delete_all(project_id: str, run_ids: str, with_commit: bool = False): + query = session.query(EvaluationRun).filter( + EvaluationRun.project_id == project_id, + EvaluationRun.id.in_(run_ids), + ) + query.delete(synchronize_session=False) + + if with_commit: + general.commit() From 1749dd6668fd6cc24a1d18b979725d3e039a8359 Mon Sep 17 00:00:00 2001 From: anmarhindi Date: Tue, 4 Feb 2025 10:47:41 +0100 Subject: [PATCH 10/26] Add playground question model --- business_objects/playground_question.py | 55 +++++++++++++++++++++++++ enums.py | 1 + models.py | 24 +++++++++++ 3 files changed, 80 insertions(+) create mode 100644 business_objects/playground_question.py diff --git a/business_objects/playground_question.py b/business_objects/playground_question.py new file mode 100644 index 0000000..49758f2 --- /dev/null +++ b/business_objects/playground_question.py @@ -0,0 +1,55 @@ +from typing import List, Optional + +from ..models import PlaygroundQuestion +from ..session import session +from . import general + + +def get(project_id: str, question_id: str) -> PlaygroundQuestion: + query = session.query(PlaygroundQuestion).filter( + PlaygroundQuestion.project_id == project_id, + PlaygroundQuestion.id == question_id, + ) + return query.first() + + +def get_all(project_id: str) -> List[PlaygroundQuestion]: + query = session.query(PlaygroundQuestion).filter( + PlaygroundQuestion.project_id == project_id, + ) + query = query.order_by(PlaygroundQuestion.created_at.asc()) + return query.all() + + +def create( + project_id: str, + question: str, + created_by: str, + embedding_id: str, + record_ids: List[str], + meta_info: Optional[str] = None, + with_commit: bool = False, +) -> PlaygroundQuestion: + q = PlaygroundQuestion( + project_id=project_id, + question=question, + created_by=created_by, + embedding_id=embedding_id, + record_ids=record_ids, + ) + + if meta_info is not None: + q.meta_info = meta_info + + general.add(q, with_commit) + + return q + + +def delete_all(project_id: str, set_ids: List[str], with_commit: bool = False) -> None: + session.query(PlaygroundQuestion).filter( + PlaygroundQuestion.project_id == project_id, + PlaygroundQuestion.id.in_(set_ids), + ).delete(synchronize_session=False) + if with_commit: + session.commit() diff --git a/enums.py b/enums.py index bd2a127..8c3e6f2 100644 --- a/enums.py +++ b/enums.py @@ -148,6 +148,7 @@ class Tablenames(Enum): EVALUATION_SET = "evaluation_set" EVALUATION_GROUP = "evaluation_group" EVALUATION_RUN = "evaluation_run" + PLAYGROUND_QUESTION = "playground_question" def snake_case_to_pascal_case(self): # the type name (written in PascalCase) of a table is needed to create backrefs diff --git a/models.py b/models.py index 898c43f..e4970c0 100644 --- a/models.py +++ b/models.py @@ -1927,3 +1927,27 @@ class EvaluationRun(Base): state = Column(String) results = Column(JSON) meta_info = Column(JSON) + + +class PlaygroundQuestion(Base): + __tablename__ = Tablenames.PLAYGROUND_QUESTION.value + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + question = Column(String) + created_at = Column(DateTime, default=sql.func.now()) + created_by = Column( + UUID(as_uuid=True), + ForeignKey(f"{Tablenames.USER.value}.id", ondelete="SET NULL"), + index=True, + ) + project_id = Column( + UUID(as_uuid=True), + ForeignKey(f"{Tablenames.PROJECT.value}.id", ondelete="CASCADE"), + index=True, + ) + embedding_id = Column( + UUID(as_uuid=True), + ForeignKey(f"{Tablenames.EMBEDDING.value}.id", ondelete="SET NULL"), + index=True, + ) + record_ids = Column(JSON) + meta_info = Column(JSON) From 2b1f7edb1d0a615716c9b7dd946ebd5075be9b70 Mon Sep 17 00:00:00 2001 From: anmarhindi Date: Tue, 4 Feb 2025 13:37:30 +0100 Subject: [PATCH 11/26] Add limit check for questions per project --- business_objects/playground_question.py | 33 +++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/business_objects/playground_question.py b/business_objects/playground_question.py index 49758f2..54e93b6 100644 --- a/business_objects/playground_question.py +++ b/business_objects/playground_question.py @@ -5,6 +5,9 @@ from . import general +MAX_SAVED_QUESTIONS_HISTORY_PER_PROJECT = 100 + + def get(project_id: str, question_id: str) -> PlaygroundQuestion: query = session.query(PlaygroundQuestion).filter( PlaygroundQuestion.project_id == project_id, @@ -30,6 +33,28 @@ def create( meta_info: Optional[str] = None, with_commit: bool = False, ) -> PlaygroundQuestion: + + current_count = ( + session.query(PlaygroundQuestion) + .filter( + PlaygroundQuestion.project_id == project_id, + ) + .count() + ) + + if current_count >= MAX_SAVED_QUESTIONS_HISTORY_PER_PROJECT: + oldest = ( + session.query(PlaygroundQuestion) + .filter( + PlaygroundQuestion.project_id == project_id, + ) + .order_by(PlaygroundQuestion.created_at.asc()) + .limit(current_count - MAX_SAVED_QUESTIONS_HISTORY_PER_PROJECT + 1) + .all() + ) + set_ids = [q.id for q in oldest] + delete_all(project_id, set_ids, False) + q = PlaygroundQuestion( project_id=project_id, question=question, @@ -53,3 +78,11 @@ def delete_all(project_id: str, set_ids: List[str], with_commit: bool = False) - ).delete(synchronize_session=False) if with_commit: session.commit() + + +def delete(project_id: str, id: str, with_commit: bool = True) -> None: + session.query(PlaygroundQuestion).filter( + PlaygroundQuestion.project_id == project_id, + PlaygroundQuestion.id == id, + ).delete() + general.flush_or_commit(with_commit) From d021d5a8116dd619018845aa9bd849c6ea4f3110 Mon Sep 17 00:00:00 2001 From: anmarhindi Date: Tue, 4 Feb 2025 13:40:15 +0100 Subject: [PATCH 12/26] Remove unused import --- business_objects/knowledge_base.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/business_objects/knowledge_base.py b/business_objects/knowledge_base.py index afb7efe..4e41db9 100644 --- a/business_objects/knowledge_base.py +++ b/business_objects/knowledge_base.py @@ -1,4 +1,4 @@ -from typing import List, List, Optional +from typing import List, Optional from ..models import KnowledgeBase from ..exceptions import EntityAlreadyExistsException, EntityNotFoundException @@ -14,7 +14,6 @@ def get(project_id: str, base_id: str) -> KnowledgeBase: ) - def get_all_by_project_id(project_id: str) -> List[KnowledgeBase]: return ( session.query(KnowledgeBase) From 59cbf483449f7f4e88b7aa6c5cd13cfb109ec56a Mon Sep 17 00:00:00 2001 From: anmarhindi Date: Tue, 4 Feb 2025 17:33:37 +0100 Subject: [PATCH 13/26] Remove unused fields --- business_objects/playground_question.py | 10 ---------- models.py | 12 ------------ 2 files changed, 22 deletions(-) diff --git a/business_objects/playground_question.py b/business_objects/playground_question.py index 54e93b6..94c597b 100644 --- a/business_objects/playground_question.py +++ b/business_objects/playground_question.py @@ -27,10 +27,6 @@ def get_all(project_id: str) -> List[PlaygroundQuestion]: def create( project_id: str, question: str, - created_by: str, - embedding_id: str, - record_ids: List[str], - meta_info: Optional[str] = None, with_commit: bool = False, ) -> PlaygroundQuestion: @@ -58,14 +54,8 @@ def create( q = PlaygroundQuestion( project_id=project_id, question=question, - created_by=created_by, - embedding_id=embedding_id, - record_ids=record_ids, ) - if meta_info is not None: - q.meta_info = meta_info - general.add(q, with_commit) return q diff --git a/models.py b/models.py index e4970c0..a160a08 100644 --- a/models.py +++ b/models.py @@ -1934,20 +1934,8 @@ class PlaygroundQuestion(Base): id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) question = Column(String) created_at = Column(DateTime, default=sql.func.now()) - created_by = Column( - UUID(as_uuid=True), - ForeignKey(f"{Tablenames.USER.value}.id", ondelete="SET NULL"), - index=True, - ) project_id = Column( UUID(as_uuid=True), ForeignKey(f"{Tablenames.PROJECT.value}.id", ondelete="CASCADE"), index=True, ) - embedding_id = Column( - UUID(as_uuid=True), - ForeignKey(f"{Tablenames.EMBEDDING.value}.id", ondelete="SET NULL"), - index=True, - ) - record_ids = Column(JSON) - meta_info = Column(JSON) From 4d1929b023d5b301deb40588477a03dce0d89ec6 Mon Sep 17 00:00:00 2001 From: anmarhindi Date: Tue, 4 Feb 2025 17:37:07 +0100 Subject: [PATCH 14/26] Remove unused import --- business_objects/playground_question.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/business_objects/playground_question.py b/business_objects/playground_question.py index 94c597b..55923e9 100644 --- a/business_objects/playground_question.py +++ b/business_objects/playground_question.py @@ -1,4 +1,4 @@ -from typing import List, Optional +from typing import List from ..models import PlaygroundQuestion from ..session import session From c2acd1268212385d243cc6d27e9df9fa4584e784 Mon Sep 17 00:00:00 2001 From: anmarhindi Date: Thu, 6 Feb 2025 09:50:42 +0100 Subject: [PATCH 15/26] Change model fields --- enums.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/enums.py b/enums.py index 8c3e6f2..df408e2 100644 --- a/enums.py +++ b/enums.py @@ -806,6 +806,7 @@ class PipelineVersionType(Enum): class EvaluationRunState(Enum): - STARTED = "STARTED" - FINISHED = "FINISHED" + INITIATED = "INITIATED" + RUNNING = "RUNNING" + SUCCESS = "SUCCESS" FAILED = "FAILED" From 2ff8904b7ed095bed95c68032c399c4d70c1600f Mon Sep 17 00:00:00 2001 From: anmarhindi Date: Thu, 6 Feb 2025 11:21:26 +0100 Subject: [PATCH 16/26] Remove ifs during eval run create --- business_objects/evaluation_run.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/business_objects/evaluation_run.py b/business_objects/evaluation_run.py index 3968874..223c458 100644 --- a/business_objects/evaluation_run.py +++ b/business_objects/evaluation_run.py @@ -59,14 +59,10 @@ def create( project_id=project_id, embedding_id=embedding_id, state=state, + results=results, + meta_info=meta_info, ) - if results is not None: - eval_run.results = results - - if meta_info is not None: - eval_run.meta_info = meta_info - general.add(eval_run, with_commit) return eval_run @@ -98,6 +94,5 @@ def delete_all(project_id: str, run_ids: str, with_commit: bool = False): EvaluationRun.id.in_(run_ids), ) query.delete(synchronize_session=False) - - if with_commit: - general.commit() + general.flush_or_commit(with_commit) + general.flush_or_commit(with_commit) From 89bda2cded0e4988497b678da32007e311cb6014 Mon Sep 17 00:00:00 2001 From: anmarhindi Date: Thu, 6 Feb 2025 11:24:25 +0100 Subject: [PATCH 17/26] Add optional state param --- business_objects/evaluation_run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/business_objects/evaluation_run.py b/business_objects/evaluation_run.py index 223c458..af86b5e 100644 --- a/business_objects/evaluation_run.py +++ b/business_objects/evaluation_run.py @@ -71,7 +71,7 @@ def create( def update( project_id: str, evaluation_run_id: str, - state: EvaluationRunState, + state: Optional[EvaluationRunState] = None, results: Optional[str] = None, meta_info: Optional[str] = None, with_commit: bool = False, From 85f761ba2b682d12036f86a4711520b8149f5833 Mon Sep 17 00:00:00 2001 From: anmarhindi Date: Thu, 6 Feb 2025 11:26:38 +0100 Subject: [PATCH 18/26] Add flush_or_commit --- business_objects/evaluation_group.py | 4 +--- business_objects/evaluation_run.py | 1 - business_objects/evaluation_set.py | 3 +-- business_objects/playground_question.py | 3 +-- 4 files changed, 3 insertions(+), 8 deletions(-) diff --git a/business_objects/evaluation_group.py b/business_objects/evaluation_group.py index f4deb75..b35c9f7 100644 --- a/business_objects/evaluation_group.py +++ b/business_objects/evaluation_group.py @@ -46,6 +46,4 @@ def delete_all(project_id: str, group_ids: str, with_commit: bool = False): EvaluationGroup.id.in_(group_ids), ) query.delete(synchronize_session=False) - - if with_commit: - general.commit() + general.flush_or_commit(with_commit) diff --git a/business_objects/evaluation_run.py b/business_objects/evaluation_run.py index af86b5e..ce81126 100644 --- a/business_objects/evaluation_run.py +++ b/business_objects/evaluation_run.py @@ -95,4 +95,3 @@ def delete_all(project_id: str, run_ids: str, with_commit: bool = False): ) query.delete(synchronize_session=False) general.flush_or_commit(with_commit) - general.flush_or_commit(with_commit) diff --git a/business_objects/evaluation_set.py b/business_objects/evaluation_set.py index c9f5fd1..4aaed77 100644 --- a/business_objects/evaluation_set.py +++ b/business_objects/evaluation_set.py @@ -65,5 +65,4 @@ def delete_all(project_id: str, set_ids: List[str], with_commit: bool = False) - EvaluationSet.project_id == project_id, EvaluationSet.id.in_(set_ids), ).delete(synchronize_session=False) - if with_commit: - session.commit() + general.flush_or_commit(with_commit) diff --git a/business_objects/playground_question.py b/business_objects/playground_question.py index 55923e9..bb23451 100644 --- a/business_objects/playground_question.py +++ b/business_objects/playground_question.py @@ -66,8 +66,7 @@ def delete_all(project_id: str, set_ids: List[str], with_commit: bool = False) - PlaygroundQuestion.project_id == project_id, PlaygroundQuestion.id.in_(set_ids), ).delete(synchronize_session=False) - if with_commit: - session.commit() + general.flush_or_commit(with_commit) def delete(project_id: str, id: str, with_commit: bool = True) -> None: From 77f856a9c85265f0fd3537e62391251779686fd3 Mon Sep 17 00:00:00 2001 From: anmarhindi Date: Thu, 6 Feb 2025 11:28:12 +0100 Subject: [PATCH 19/26] Rename to ids for readability --- business_objects/playground_question.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/business_objects/playground_question.py b/business_objects/playground_question.py index bb23451..740e43c 100644 --- a/business_objects/playground_question.py +++ b/business_objects/playground_question.py @@ -48,8 +48,8 @@ def create( .limit(current_count - MAX_SAVED_QUESTIONS_HISTORY_PER_PROJECT + 1) .all() ) - set_ids = [q.id for q in oldest] - delete_all(project_id, set_ids, False) + ids = [q.id for q in oldest] + delete_all(project_id, ids, False) q = PlaygroundQuestion( project_id=project_id, From 2f3646b0dc517abf6d8289cc04c2b3e5b7b2551f Mon Sep 17 00:00:00 2001 From: anmarhindi Date: Thu, 6 Feb 2025 11:28:47 +0100 Subject: [PATCH 20/26] Rename delete_all func param to ids for readability --- business_objects/playground_question.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/business_objects/playground_question.py b/business_objects/playground_question.py index 740e43c..b98b49c 100644 --- a/business_objects/playground_question.py +++ b/business_objects/playground_question.py @@ -61,10 +61,10 @@ def create( return q -def delete_all(project_id: str, set_ids: List[str], with_commit: bool = False) -> None: +def delete_all(project_id: str, ids: List[str], with_commit: bool = False) -> None: session.query(PlaygroundQuestion).filter( PlaygroundQuestion.project_id == project_id, - PlaygroundQuestion.id.in_(set_ids), + PlaygroundQuestion.id.in_(ids), ).delete(synchronize_session=False) general.flush_or_commit(with_commit) From 86743c4b8329477ada2eacbe6522c59bda5464da Mon Sep 17 00:00:00 2001 From: anmarhindi Date: Thu, 6 Feb 2025 11:33:13 +0100 Subject: [PATCH 21/26] Add comment in PlaygroundQuestion model for extension properties --- models.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/models.py b/models.py index a160a08..3dd742a 100644 --- a/models.py +++ b/models.py @@ -1939,3 +1939,15 @@ class PlaygroundQuestion(Base): ForeignKey(f"{Tablenames.PROJECT.value}.id", ondelete="CASCADE"), index=True, ) + """ + Playground question can be extended with the below properties to allow the following: + - User can see questions with specific results relating to the embedding used + - Can be used for comparison with new results using same question but different embedding + """ + # embedding_id = Column( + # UUID(as_uuid=True), + # ForeignKey(f"{Tablenames.EMBEDDING.value}.id", ondelete="SET NULL"), + # index=True, + # ) + # record_ids = Column(JSON) + # meta_info = Column(JSON) From 57119687d11eaa459bc99cc5356c1054f570b0be Mon Sep 17 00:00:00 2001 From: anmarhindi Date: Thu, 6 Feb 2025 11:45:33 +0100 Subject: [PATCH 22/26] join query for eval group --- business_objects/evaluation_set.py | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/business_objects/evaluation_set.py b/business_objects/evaluation_set.py index 4aaed77..d3e9d14 100644 --- a/business_objects/evaluation_set.py +++ b/business_objects/evaluation_set.py @@ -24,20 +24,15 @@ def get_all(project_id: str) -> List[EvaluationSet]: def get_by_evaluation_group_id( project_id: str, evaluation_group_id: str ) -> List[EvaluationSet]: - - evaluation_group = ( - session.query(EvaluationGroup) + query = ( + session.query(EvaluationSet) + .join(EvaluationGroup, EvaluationSet.id.in_(EvaluationGroup.evaluation_set_ids)) .filter( EvaluationGroup.project_id == project_id, EvaluationGroup.id == evaluation_group_id, ) - .first() - ) - query = session.query(EvaluationSet).filter( - EvaluationSet.project_id == project_id, - EvaluationSet.id.in_(evaluation_group.evaluation_set_ids), + .order_by(EvaluationSet.question) ) - query = query.order_by(EvaluationSet.question) return query.all() @@ -54,9 +49,7 @@ def create( created_by=created_by, record_ids=record_ids, ) - general.add(eval_set, with_commit) - return eval_set From 565c2295d936f668fb29182ec057b191361ce931 Mon Sep 17 00:00:00 2001 From: anmarhindi Date: Thu, 6 Feb 2025 14:15:58 +0100 Subject: [PATCH 23/26] Add join query for eval group id --- business_objects/evaluation_set.py | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/business_objects/evaluation_set.py b/business_objects/evaluation_set.py index d3e9d14..3dd7281 100644 --- a/business_objects/evaluation_set.py +++ b/business_objects/evaluation_set.py @@ -1,6 +1,8 @@ from typing import List -from ..models import EvaluationSet, EvaluationGroup +from submodules.model.util import prevent_sql_injection + +from ..models import EvaluationSet from ..session import session from . import general @@ -24,16 +26,23 @@ def get_all(project_id: str) -> List[EvaluationSet]: def get_by_evaluation_group_id( project_id: str, evaluation_group_id: str ) -> List[EvaluationSet]: - query = ( - session.query(EvaluationSet) - .join(EvaluationGroup, EvaluationSet.id.in_(EvaluationGroup.evaluation_set_ids)) - .filter( - EvaluationGroup.project_id == project_id, - EvaluationGroup.id == evaluation_group_id, - ) - .order_by(EvaluationSet.question) + project_id = prevent_sql_injection(project_id, isinstance(project_id, str)) + evaluation_group_id = prevent_sql_injection( + evaluation_group_id, isinstance(evaluation_group_id, str) ) - return query.all() + + query = f""" + SELECT es.* + FROM evaluation_group eg + JOIN LATERAL jsonb_array_elements_text(eg.evaluation_set_ids::jsonb) AS elem(evaluation_set_id) ON TRUE + JOIN evaluation_set es ON es.id = elem.evaluation_set_id::uuid + WHERE eg.project_id = '{project_id}' + AND eg.id = '{evaluation_group_id}' + AND es.project_id = '{project_id}' + ORDER BY es.question; + """ + + return general.execute_all(query) def create( From dc4f3d80f47bbf3e3d02254c984aeceada1ee48d Mon Sep 17 00:00:00 2001 From: anmarhindi Date: Thu, 6 Feb 2025 16:02:25 +0100 Subject: [PATCH 24/26] Add check for duplicate questions --- business_objects/playground_question.py | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/business_objects/playground_question.py b/business_objects/playground_question.py index b98b49c..d1f7226 100644 --- a/business_objects/playground_question.py +++ b/business_objects/playground_question.py @@ -1,4 +1,4 @@ -from typing import List +from typing import Any, List from ..models import PlaygroundQuestion from ..session import session @@ -28,16 +28,18 @@ def create( project_id: str, question: str, with_commit: bool = False, -) -> PlaygroundQuestion: +) -> Any: - current_count = ( + current_questions = ( session.query(PlaygroundQuestion) .filter( PlaygroundQuestion.project_id == project_id, ) - .count() + .all() ) + current_count = len(current_questions) + if current_count >= MAX_SAVED_QUESTIONS_HISTORY_PER_PROJECT: oldest = ( session.query(PlaygroundQuestion) @@ -51,14 +53,19 @@ def create( ids = [q.id for q in oldest] delete_all(project_id, ids, False) - q = PlaygroundQuestion( - project_id=project_id, - question=question, + current_count_question = sum( + 1 for q in current_questions if str(q.question).lower() == question.lower() ) - general.add(q, with_commit) + if current_count_question == 0: + q = PlaygroundQuestion( + project_id=project_id, + question=question, + ) + general.add(q, with_commit) + return q - return q + return None def delete_all(project_id: str, ids: List[str], with_commit: bool = False) -> None: From 0eeed19bf1709c694c540fbf05fe3fa152cce3bd Mon Sep 17 00:00:00 2001 From: Lina Date: Fri, 7 Feb 2025 11:19:04 +0100 Subject: [PATCH 25/26] Deleting questions and order desc --- business_objects/playground_question.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/business_objects/playground_question.py b/business_objects/playground_question.py index d1f7226..b913010 100644 --- a/business_objects/playground_question.py +++ b/business_objects/playground_question.py @@ -20,7 +20,7 @@ def get_all(project_id: str) -> List[PlaygroundQuestion]: query = session.query(PlaygroundQuestion).filter( PlaygroundQuestion.project_id == project_id, ) - query = query.order_by(PlaygroundQuestion.created_at.asc()) + query = query.order_by(PlaygroundQuestion.created_at.desc()) return query.all() From 80a3a370a40b1ceb30bedd2c2c91c48a1f5ddbbf Mon Sep 17 00:00:00 2001 From: anmarhindi Date: Thu, 13 Feb 2025 11:15:37 +0100 Subject: [PATCH 26/26] Add question check --- business_objects/playground_question.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/business_objects/playground_question.py b/business_objects/playground_question.py index b913010..195c174 100644 --- a/business_objects/playground_question.py +++ b/business_objects/playground_question.py @@ -53,17 +53,18 @@ def create( ids = [q.id for q in oldest] delete_all(project_id, ids, False) - current_count_question = sum( - 1 for q in current_questions if str(q.question).lower() == question.lower() - ) - - if current_count_question == 0: - q = PlaygroundQuestion( - project_id=project_id, - question=question, + if question: + current_count_question = sum( + 1 for q in current_questions if str(q.question).lower() == question.lower() ) - general.add(q, with_commit) - return q + + if current_count_question == 0: + q = PlaygroundQuestion( + project_id=project_id, + question=question, + ) + general.add(q, with_commit) + return q return None