From 1481b2f1ce0acad6e02969b5e06017accf1deba2 Mon Sep 17 00:00:00 2001 From: LennartSchmidtKern Date: Mon, 25 Nov 2024 14:11:36 +0100 Subject: [PATCH 1/6] new oclumns --- models.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/models.py b/models.py index 962b0a2..d374faf 100644 --- a/models.py +++ b/models.py @@ -156,6 +156,8 @@ class Organization(Base): # designed as opt out to ensure "forgotten" doesn't result in issues log_admin_requests = Column(String, default=AdminLogLevel.NO_GET.value) + conversation_lifespan_days = Column(Integer) + file_lifespan_days = Column(Integer) class User(Base): From f0a414d5ce5eb71c6071bb7e27d40a0bc2722d92 Mon Sep 17 00:00:00 2001 From: LennartSchmidtKern Date: Mon, 25 Nov 2024 18:17:12 +0100 Subject: [PATCH 2/6] all by org joined --- cognition_objects/conversation.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/cognition_objects/conversation.py b/cognition_objects/conversation.py index cc3d93d..1b408c7 100644 --- a/cognition_objects/conversation.py +++ b/cognition_objects/conversation.py @@ -25,6 +25,20 @@ def get(project_id: str, conversation_id: str) -> CognitionConversation: ) +def get_all_with_org() -> List[CognitionConversation]: + query = """ + SELECT + cc.*, + cp.organization_id + FROM + cognition.conversation cc + JOIN + cognition.project cp + ON + cc.project_id = cp.id;""" + return general.execute_all(query) + + def get_scoped(project_id: str, conversation_id: str, user_id) -> CognitionConversation: return ( session.query(CognitionConversation) From 1969c5ef1e9d33dc5ac18413f7fa706d3b3300b8 Mon Sep 17 00:00:00 2001 From: LennartSchmidtKern Date: Tue, 26 Nov 2024 11:29:20 +0100 Subject: [PATCH 3/6] remove gdpr compliant field --- models.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/models.py b/models.py index d374faf..c438445 100644 --- a/models.py +++ b/models.py @@ -140,8 +140,6 @@ class Organization(Base): # database entry is_paying = Column(Boolean, default=False) created_at = Column(DateTime, default=sql.func.now()) - gdpr_compliant = Column(Boolean, default=False) - projects = parent_to_child_relationship( Tablenames.ORGANIZATION, Tablenames.PROJECT, From f129a633fd520974b5dc0ec4ab8de01121b71985 Mon Sep 17 00:00:00 2001 From: LennartSchmidtKern Date: Tue, 26 Nov 2024 12:03:59 +0100 Subject: [PATCH 4/6] default file lifespan --- models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models.py b/models.py index c438445..bad8df7 100644 --- a/models.py +++ b/models.py @@ -155,7 +155,7 @@ class Organization(Base): # designed as opt out to ensure "forgotten" doesn't result in issues log_admin_requests = Column(String, default=AdminLogLevel.NO_GET.value) conversation_lifespan_days = Column(Integer) - file_lifespan_days = Column(Integer) + file_lifespan_days = Column(Integer, default=14) class User(Base): From 8e4459dd43a8731f71a7a82ba968c2eb0526cd30 Mon Sep 17 00:00:00 2001 From: LennartSchmidtKern Date: Wed, 27 Nov 2024 15:21:41 +0100 Subject: [PATCH 5/6] new joins --- cognition_objects/conversation.py | 35 ++++++++++++++++++++--------- cognition_objects/file_reference.py | 14 +++++++++++- 2 files changed, 38 insertions(+), 11 deletions(-) diff --git a/cognition_objects/conversation.py b/cognition_objects/conversation.py index 1b408c7..8545919 100644 --- a/cognition_objects/conversation.py +++ b/cognition_objects/conversation.py @@ -25,17 +25,32 @@ def get(project_id: str, conversation_id: str) -> CognitionConversation: ) -def get_all_with_org() -> List[CognitionConversation]: +def get_conversations_to_clean_up() -> List[Tuple[str, str, str]]: query = """ - SELECT - cc.*, - cp.organization_id - FROM - cognition.conversation cc - JOIN - cognition.project cp - ON - cc.project_id = cp.id;""" + SELECT cc.id, cc.project_id, o.id organization_id + FROM cognition.conversation cc + INNER JOIN cognition.project cp + ON cc.project_id = cp.id + INNER JOIN ( + SELECT o.*, NOW() - INTERVAL '1 DAY' * conversation_lifespan_days conversation_delete_by + FROM PUBLIC.organization o + ) o + ON cp.organization_id = o.id AND cc.created_at <= o.conversation_delete_by""" + return general.execute_all(query) + + +def get_conversation_files_to_clean_up() -> List[Tuple[str, str, str]]: + query = """ + SELECT cc.id, cc.project_id, o.id organization_id + FROM cognition.conversation cc + INNER JOIN cognition.project cp + ON cc.project_id = cp.id + INNER JOIN ( + SELECT o.*, NOW() - INTERVAL '1 DAY' * file_lifespan_days file_delete_by + FROM PUBLIC.organization o + ) o + ON cp.organization_id = o.id AND cc.created_at <= o.file_delete_by + WHERE NOT cc.archived AND cc.has_tmp_files """ return general.execute_all(query) diff --git a/cognition_objects/file_reference.py b/cognition_objects/file_reference.py index cec6da0..15b7a4e 100644 --- a/cognition_objects/file_reference.py +++ b/cognition_objects/file_reference.py @@ -1,7 +1,7 @@ from ..business_objects import general from ..session import session from ..models import FileReference -from typing import Dict, Any, List +from typing import Dict, Any, List, Tuple def get(org_id: str, hash: str, file_size_bytes) -> FileReference: @@ -54,6 +54,18 @@ def get_all() -> List[FileReference]: return session.query(FileReference).all() +def get_all_for_cleanup() -> List[Tuple[str, str]]: + query = """ + SELECT fr.id, fr.organization_id + FROM cognition.file_reference fr + INNER JOIN ( + SELECT o.*, NOW() - INTERVAL '1 DAY' * file_lifespan_days file_delete_by + FROM PUBLIC.organization o + ) o + ON fr.organization_id = o.id AND fr.last_used <= o.file_delete_by""" + return general.execute_all(query) + + def create( org_id: str, hash: str, From 99791571c22594bfc550ec4ad8c63fecbf769d3c Mon Sep 17 00:00:00 2001 From: LennartSchmidtKern Date: Wed, 27 Nov 2024 15:47:03 +0100 Subject: [PATCH 6/6] user count org --- business_objects/organization.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/business_objects/organization.py b/business_objects/organization.py index 2a0fcab..7e2e462 100644 --- a/business_objects/organization.py +++ b/business_objects/organization.py @@ -5,7 +5,7 @@ from ..session import session -from ..models import Organization, Project +from ..models import Organization, Project, User from ..business_objects import project, user, general from ..util import prevent_sql_injection @@ -62,6 +62,10 @@ def get_organization_overview_stats( return values[0] +def get_user_count(organization_id: str) -> int: + return session.query(User).filter(User.organization_id == organization_id).count() + + def __get_organization_overview_stats_query(organization_id: str): return f""" WITH labeled_records AS (