Removal of GraphQL Data Return Schema (#284)

* projectByProjectId

* allProjects

* labelDistribution, generalProjectStats

* projectTokenization, modelProviderInfo

* projectExportCredentials, uploadCredentialsAndId

* createSampleProject, updateProjectStatus, updateProjectTokenizer

* createProject, deleteProject

* uploadTaskById, updateProjectNameDescription

* attributes,embeddings, data slide

* lt query rework

* comment, data browser, heuristic

* labeling-tasks-by-project-id

* labeling, lookup

* fixes

* notifications, record, settings

* misc

* misc

* First part org queries

* More org endpoints

* More org changes

* allOrgs, user roles

* another endpoint

* perf: update organization endpoints to remove graphql syntax

* info source

* perf: update organization endpoints to remove graphql syntax

* silent success

* weak-supervision api change to silent success

* task_execution api change to silent success

* get silent success

* Change sync-records usage of responses

* embedding, fix camel case, heuristic

* labeling

* misc, look up

* rename get_custom_response function

* Update delete-project and update-project-name-description

* update-project-status & update-project-tokenizer silent response

* Fix: prepare-record-export

* Fix: calculate_user_attribute_all_records & update_attribute

* perf: eliminate get_embeddings_schema unused function
perf: update get_embeddings route for pgsql to handle data transformation

* fix: prepare_project_export

* proect settings

* fix: project-size, json cannot encode Decimal.

* remove duplicate import

* model

* gen fail

* remove unused import

* ref

* remove unused embedding

* perf: move column whitelist to submodules

* model

* silent success

* Handle exceptions with direct returns

* Remove type hint

* Remove print

* perf: update submodules

* Remove unused key

* Add list comprehension all-projects-mini

* PR comments

* PR embed

* model

---------

Co-authored-by: JWittmeyer <jens.wittmeyer@kern.ai>
Co-authored-by: andhreljaKern <andrea.hrelja@kern.ai>
Co-authored-by: anmarhindi <anmar.hindi@kern.ai>

Author: 4 people

controller/auth/kratos.py

@@ -152,7 +152,7 @@ def resolve_user_mail_by_id(user_id: str) -> str:
     return None
 
 
-def resolve_user_name_by_id(user_id: str) -> str:
+def resolve_user_name_by_id(user_id: str) -> Dict[str, str]:
     i = __get_identity(user_id, False)
     if i:
         i = i["identity"]

controller/embedding/manager.py

@@ -5,7 +5,6 @@
 from submodules.model import enums
 from submodules.model.models import Embedding
 from util import notification
-from . import util
 from . import connector
 from .terms import TERMS_INFO
 from controller.model_provider import manager as model_manager
@@ -14,11 +13,8 @@
     embedding,
     agreement,
     general,
-    project,
 )
 from submodules.model import daemon
-from submodules.model.util import sql_alchemy_to_dict
-from controller.embedding.connector import collection_on_qdrant
 
 
 def get_terms_info(
@@ -29,19 +25,6 @@ def get_terms_info(
     return list(TERMS_INFO.values())
 
 
-def get_current_terms_text(
-    platform: str,
-) -> Optional[str]:
-    terms = TERMS_INFO[platform]
-    term_text = terms.get("terms")
-    if not term_text:
-        return None
-    link = terms.get("link")
-    if link:
-        term_text = term_text.replace("@@PLACEHOLDER@@", link)
-    return term_text
-
-
 def get_recommended_encoders() -> List[Any]:
     # can run into circular import problems if directly resolved here by helper method
     recommendations = connector.request_listing_recommended_encoders()
@@ -71,17 +54,6 @@ def get_recommended_encoders() -> List[Any]:
     return recommendations
 
 
-def create_embedding(project_id: str, embedding_id: str) -> None:
-    daemon.run_without_db_token(connector.request_embedding, project_id, embedding_id)
-
-
-def create_embeddings_one_by_one(
-    project_id: str,
-    embeddings_ids: List[str],
-) -> None:
-    daemon.run_without_db_token(__embed_one_by_one_helper, project_id, embeddings_ids)
-
-
 def request_tensor_upload(project_id: str, embedding_id: str) -> Any:
     connector.request_tensor_upload(project_id, embedding_id)
 
@@ -92,21 +64,6 @@ def delete_embedding(project_id: str, embedding_id: str) -> None:
     connector.request_deleting_embedding(project_id, embedding_id)
 
 
-def __embed_one_by_one_helper(project_id: str, embeddings_ids: List[str]) -> None:
-    ctx_token = general.get_ctx_token()
-    for embedding_id in embeddings_ids:
-        connector.request_embedding(project_id, embedding_id)
-        time.sleep(5)
-        c = 1
-        while util.has_encoder_running(project_id):
-            c += 1
-            if c > 12:
-                ctx_token = general.remove_and_refresh_session(ctx_token, True)
-                c = 1
-            time.sleep(5)
-    general.remove_and_refresh_session(ctx_token, False)
-
-
 def get_embedding_name(
     project_id: str,
     attribute_id: str,
@@ -142,62 +99,6 @@ def get_embedding_name(
     return name
 
 
-EMBEDDING_SCHEMA_WHITELIST = [
-    "id",
-    "name",
-    "custom",
-    "type",
-    "state",
-    "progress",
-    "dimension",
-    "count",
-    "platform",
-    "model",
-    "filter_attributes",
-    "attribute_id",
-]
-
-
-def get_embedding_schema(project_id: str) -> List[Dict[str, Any]]:
-    embeddings = embedding.get_all_embeddings_by_project_id(project_id)
-    embedding_dict = sql_alchemy_to_dict(
-        embeddings, column_whitelist=EMBEDDING_SCHEMA_WHITELIST
-    )
-    number_records = len(project.get(project_id).records)
-    expanded_embeddings = []
-    for embed in embedding_dict:
-        count = embedding.get_tensor_count(embed["id"])
-        onQdrant = collection_on_qdrant(project_id, embed["id"])
-
-        embedding_item = embedding.get_tensor(embed["id"])
-        dimension = 0
-        if embedding_item is not None:
-            # distinguish between token and attribute embeddings
-            if type(embedding_item.data[0]) is list:
-                dimension = len(embedding_item.data[0])
-            else:
-                dimension = len(embedding_item.data)
-
-        if embed["state"] == "FINISHED":
-            progress = 1
-        elif embed["state"] == "INITIALIZING" or embed["state"] == "WAITING":
-            progress = 0.0
-        else:
-            progress = min(
-                0.1 + (count / number_records * 0.9),
-                0.99,
-            )
-        expanded_embed = {
-            **embed,
-            "progress": progress,
-            "count": count,
-            "dimension": dimension,
-            "onQdrant": onQdrant,
-        }
-        expanded_embeddings.append(expanded_embed)
-    return {"id": project_id, "embeddings": expanded_embeddings}
-
-
 def recreate_embeddings(
     project_id: str, embedding_ids: Optional[List[str]] = None, user_id: str = None
 ) -> None:

controller/embedding/util.py

@@ -1,8 +1,7 @@
 import os
 import logging
 import json
-import traceback
-from typing import Any, List, Optional, Dict
+from typing import Any, Optional, Dict
 
 from controller.transfer import export_parser
 from controller.transfer.knowledge_base_transfer_manager import (
@@ -23,7 +22,6 @@
     record_label_association,
     data_slice,
     knowledge_base,
-    upload_task,
 )
 from submodules.model.business_objects import general
 from controller.upload_task import manager as upload_task_manager
@@ -35,11 +33,6 @@
 from controller.labeling_task import manager as labeling_task_manager
 from controller.labeling_task_label import manager as labeling_task_label_manager
 from submodules.model.business_objects import record_label_association as rla
-from controller.task_master import manager as task_master_manager
-from submodules.model.enums import TaskType, RecordTokenizationScope
-
-
-from util.notification import create_notification
 
 logging.basicConfig(level=logging.DEBUG)
 logger = logging.getLogger(__name__)
@@ -65,7 +58,9 @@ def get_upload_credentials_and_id(
         key,
     )
     org_id = organization.get_id_by_project_id(project_id)
-    return s3.get_upload_credentials_and_id(org_id, project_id + "/" + str(task.id))
+    return s3.get_upload_credentials_and_id(
+        org_id, project_id + "/" + str(task.id), True, True
+    )
 
 
 def import_records_from_file(project_id: str, task: UploadTask) -> None:

controller/transfer/manager.py

@@ -2,7 +2,7 @@
 from controller.auth import manager as auth_manager
 from typing import List, Union
 from fast_api.models import DeleteUserAttributeBody
-from fast_api.routes.client_response import pack_json_result
+from fast_api.routes.client_response import pack_json_result, get_silent_success
 from fastapi import APIRouter, Body, Depends, Query, Request
 from submodules.model.enums import NotificationType
 from submodules.model.util import sql_alchemy_to_dict
@@ -34,7 +34,7 @@ def get_attributes(
 ):
     data = manager.get_all_attributes(project_id, state_filter)
     data_dict = sql_alchemy_to_dict(data, column_whitelist=ALL_ATTRIBUTES_WHITELIST)
-    return pack_json_result({"data": {"attributesByProjectId": data_dict}})
+    return pack_json_result(data_dict)
 
 
 @router.get(
@@ -54,7 +54,7 @@ def get_check_composite_key(
             project_id,
         )
 
-    return pack_json_result({"data": {"checkCompositeKey": is_valid}})
+    return pack_json_result(is_valid)
 
 
 @router.get(
@@ -71,12 +71,8 @@ def get_sample_records(
     )
     return pack_json_result(
         {
-            "data": {
-                "calculateUserAttributeSampleRecords": {
-                    "record_ids": record_ids,
-                    "calculated_attributes": calculated_attributes,
-                }
-            }
+            "record_ids": record_ids,
+            "calculated_attributes": calculated_attributes,
         }
     )
 
@@ -91,4 +87,4 @@ def delete_user_attribute(
     body: DeleteUserAttributeBody = Body(...),
 ):
     manager.delete_attribute(project_id, body.attribute_id)
-    return pack_json_result({"data": {"deleteUserAttribute": {"ok": True}}})
+    return get_silent_success()

fast_api/routes/attribute.py

@@ -60,5 +60,18 @@ def get_silent_success() -> JSONResponse:
     return SILENT_SUCCESS_RESPONSE
 
 
+def get_custom_response(status_code, content, type="text"):
+    if type == "json":
+        return JSONResponse(
+            status_code=status_code,
+            content=content,
+        )
+    else:
+        return PlainTextResponse(
+            status_code=status_code,
+            content=content,
+        )
+
+
 def to_json(obj: BaseModel):
     return json.loads(obj.json())

fast_api/routes/client_response.py

@@ -6,12 +6,11 @@
     DeleteCommentBody,
     UpdateCommentBody,
 )
-from fast_api.routes.client_response import pack_json_result
+from fast_api.routes.client_response import pack_json_result, get_silent_success
 from submodules.model.enums import CommentCategory
 from util import notification
 from middleware.log_storage import extend_state_get_like
 
-
 router = APIRouter()
 
 
@@ -56,9 +55,7 @@ def get_all_comments(request: Request):
             "add_info": add_info,
         }
 
-    return pack_json_result(
-        {"data": {"getAllComments": to_return}}, wrap_for_frontend=False
-    )
+    return pack_json_result(to_return, wrap_for_frontend=False)
 
 
 @router.post("/create-comment")
@@ -84,7 +81,7 @@ def create_comment(request: Request, body: CreateCommentBody = Body(...)):
             True,
         )
 
-    return pack_json_result({"data": {"createComment": {"ok": True}}})
+    return get_silent_success()
 
 
 @router.delete("/delete-comment")
@@ -109,7 +106,7 @@ def delete_comment(
             True,
         )
 
-    return pack_json_result({"data": {"deleteComment": {"ok": True}}})
+    return get_silent_success()
 
 
 @router.put("/update-comment")
@@ -134,7 +131,7 @@ def update_comment(
             True,
         )
 
-    return pack_json_result({"data": {"updateComment": {"ok": True}}})
+    return get_silent_success()
 
 
 @router.get("/get-unique-comments-keys-for")