Skip to content

Commit 7751b74

Browse files
LennartSchmidtKernJWittmeyerandhreljaKernanmarhindi
authored
Removal of GraphQL Data Return Schema (#284)
* projectByProjectId * allProjects * labelDistribution, generalProjectStats * projectTokenization, modelProviderInfo * projectExportCredentials, uploadCredentialsAndId * createSampleProject, updateProjectStatus, updateProjectTokenizer * createProject, deleteProject * uploadTaskById, updateProjectNameDescription * attributes,embeddings, data slide * lt query rework * comment, data browser, heuristic * labeling-tasks-by-project-id * labeling, lookup * fixes * notifications, record, settings * misc * misc * First part org queries * More org endpoints * More org changes * allOrgs, user roles * another endpoint * perf: update organization endpoints to remove graphql syntax * info source * perf: update organization endpoints to remove graphql syntax * silent success * weak-supervision api change to silent success * task_execution api change to silent success * get silent success * Change sync-records usage of responses * embedding, fix camel case, heuristic * labeling * misc, look up * rename get_custom_response function * Update delete-project and update-project-name-description * update-project-status & update-project-tokenizer silent response * Fix: prepare-record-export * Fix: calculate_user_attribute_all_records & update_attribute * perf: eliminate get_embeddings_schema unused function perf: update get_embeddings route for pgsql to handle data transformation * fix: prepare_project_export * proect settings * fix: project-size, json cannot encode Decimal. * remove duplicate import * model * gen fail * remove unused import * ref * remove unused embedding * perf: move column whitelist to submodules * model * silent success * Handle exceptions with direct returns * Remove type hint * Remove print * perf: update submodules * Remove unused key * Add list comprehension all-projects-mini * PR comments * PR embed * model --------- Co-authored-by: JWittmeyer <jens.wittmeyer@kern.ai> Co-authored-by: andhreljaKern <andrea.hrelja@kern.ai> Co-authored-by: anmarhindi <anmar.hindi@kern.ai>
1 parent 0b83b8e commit 7751b74

24 files changed

+350
-813
lines changed

controller/auth/kratos.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -152,7 +152,7 @@ def resolve_user_mail_by_id(user_id: str) -> str:
152152
return None
153153

154154

155-
def resolve_user_name_by_id(user_id: str) -> str:
155+
def resolve_user_name_by_id(user_id: str) -> Dict[str, str]:
156156
i = __get_identity(user_id, False)
157157
if i:
158158
i = i["identity"]

controller/embedding/manager.py

Lines changed: 0 additions & 99 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@
55
from submodules.model import enums
66
from submodules.model.models import Embedding
77
from util import notification
8-
from . import util
98
from . import connector
109
from .terms import TERMS_INFO
1110
from controller.model_provider import manager as model_manager
@@ -14,11 +13,8 @@
1413
embedding,
1514
agreement,
1615
general,
17-
project,
1816
)
1917
from submodules.model import daemon
20-
from submodules.model.util import sql_alchemy_to_dict
21-
from controller.embedding.connector import collection_on_qdrant
2218

2319

2420
def get_terms_info(
@@ -29,19 +25,6 @@ def get_terms_info(
2925
return list(TERMS_INFO.values())
3026

3127

32-
def get_current_terms_text(
33-
platform: str,
34-
) -> Optional[str]:
35-
terms = TERMS_INFO[platform]
36-
term_text = terms.get("terms")
37-
if not term_text:
38-
return None
39-
link = terms.get("link")
40-
if link:
41-
term_text = term_text.replace("@@PLACEHOLDER@@", link)
42-
return term_text
43-
44-
4528
def get_recommended_encoders() -> List[Any]:
4629
# can run into circular import problems if directly resolved here by helper method
4730
recommendations = connector.request_listing_recommended_encoders()
@@ -71,17 +54,6 @@ def get_recommended_encoders() -> List[Any]:
7154
return recommendations
7255

7356

74-
def create_embedding(project_id: str, embedding_id: str) -> None:
75-
daemon.run_without_db_token(connector.request_embedding, project_id, embedding_id)
76-
77-
78-
def create_embeddings_one_by_one(
79-
project_id: str,
80-
embeddings_ids: List[str],
81-
) -> None:
82-
daemon.run_without_db_token(__embed_one_by_one_helper, project_id, embeddings_ids)
83-
84-
8557
def request_tensor_upload(project_id: str, embedding_id: str) -> Any:
8658
connector.request_tensor_upload(project_id, embedding_id)
8759

@@ -92,21 +64,6 @@ def delete_embedding(project_id: str, embedding_id: str) -> None:
9264
connector.request_deleting_embedding(project_id, embedding_id)
9365

9466

95-
def __embed_one_by_one_helper(project_id: str, embeddings_ids: List[str]) -> None:
96-
ctx_token = general.get_ctx_token()
97-
for embedding_id in embeddings_ids:
98-
connector.request_embedding(project_id, embedding_id)
99-
time.sleep(5)
100-
c = 1
101-
while util.has_encoder_running(project_id):
102-
c += 1
103-
if c > 12:
104-
ctx_token = general.remove_and_refresh_session(ctx_token, True)
105-
c = 1
106-
time.sleep(5)
107-
general.remove_and_refresh_session(ctx_token, False)
108-
109-
11067
def get_embedding_name(
11168
project_id: str,
11269
attribute_id: str,
@@ -142,62 +99,6 @@ def get_embedding_name(
14299
return name
143100

144101

145-
EMBEDDING_SCHEMA_WHITELIST = [
146-
"id",
147-
"name",
148-
"custom",
149-
"type",
150-
"state",
151-
"progress",
152-
"dimension",
153-
"count",
154-
"platform",
155-
"model",
156-
"filter_attributes",
157-
"attribute_id",
158-
]
159-
160-
161-
def get_embedding_schema(project_id: str) -> List[Dict[str, Any]]:
162-
embeddings = embedding.get_all_embeddings_by_project_id(project_id)
163-
embedding_dict = sql_alchemy_to_dict(
164-
embeddings, column_whitelist=EMBEDDING_SCHEMA_WHITELIST
165-
)
166-
number_records = len(project.get(project_id).records)
167-
expanded_embeddings = []
168-
for embed in embedding_dict:
169-
count = embedding.get_tensor_count(embed["id"])
170-
onQdrant = collection_on_qdrant(project_id, embed["id"])
171-
172-
embedding_item = embedding.get_tensor(embed["id"])
173-
dimension = 0
174-
if embedding_item is not None:
175-
# distinguish between token and attribute embeddings
176-
if type(embedding_item.data[0]) is list:
177-
dimension = len(embedding_item.data[0])
178-
else:
179-
dimension = len(embedding_item.data)
180-
181-
if embed["state"] == "FINISHED":
182-
progress = 1
183-
elif embed["state"] == "INITIALIZING" or embed["state"] == "WAITING":
184-
progress = 0.0
185-
else:
186-
progress = min(
187-
0.1 + (count / number_records * 0.9),
188-
0.99,
189-
)
190-
expanded_embed = {
191-
**embed,
192-
"progress": progress,
193-
"count": count,
194-
"dimension": dimension,
195-
"onQdrant": onQdrant,
196-
}
197-
expanded_embeddings.append(expanded_embed)
198-
return {"id": project_id, "embeddings": expanded_embeddings}
199-
200-
201102
def recreate_embeddings(
202103
project_id: str, embedding_ids: Optional[List[str]] = None, user_id: str = None
203104
) -> None:

controller/embedding/util.py

Lines changed: 0 additions & 5 deletions
This file was deleted.

controller/transfer/manager.py

Lines changed: 4 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,7 @@
11
import os
22
import logging
33
import json
4-
import traceback
5-
from typing import Any, List, Optional, Dict
4+
from typing import Any, Optional, Dict
65

76
from controller.transfer import export_parser
87
from controller.transfer.knowledge_base_transfer_manager import (
@@ -23,7 +22,6 @@
2322
record_label_association,
2423
data_slice,
2524
knowledge_base,
26-
upload_task,
2725
)
2826
from submodules.model.business_objects import general
2927
from controller.upload_task import manager as upload_task_manager
@@ -35,11 +33,6 @@
3533
from controller.labeling_task import manager as labeling_task_manager
3634
from controller.labeling_task_label import manager as labeling_task_label_manager
3735
from submodules.model.business_objects import record_label_association as rla
38-
from controller.task_master import manager as task_master_manager
39-
from submodules.model.enums import TaskType, RecordTokenizationScope
40-
41-
42-
from util.notification import create_notification
4336

4437
logging.basicConfig(level=logging.DEBUG)
4538
logger = logging.getLogger(__name__)
@@ -65,7 +58,9 @@ def get_upload_credentials_and_id(
6558
key,
6659
)
6760
org_id = organization.get_id_by_project_id(project_id)
68-
return s3.get_upload_credentials_and_id(org_id, project_id + "/" + str(task.id))
61+
return s3.get_upload_credentials_and_id(
62+
org_id, project_id + "/" + str(task.id), True, True
63+
)
6964

7065

7166
def import_records_from_file(project_id: str, task: UploadTask) -> None:

fast_api/routes/attribute.py

Lines changed: 6 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
from controller.auth import manager as auth_manager
33
from typing import List, Union
44
from fast_api.models import DeleteUserAttributeBody
5-
from fast_api.routes.client_response import pack_json_result
5+
from fast_api.routes.client_response import pack_json_result, get_silent_success
66
from fastapi import APIRouter, Body, Depends, Query, Request
77
from submodules.model.enums import NotificationType
88
from submodules.model.util import sql_alchemy_to_dict
@@ -34,7 +34,7 @@ def get_attributes(
3434
):
3535
data = manager.get_all_attributes(project_id, state_filter)
3636
data_dict = sql_alchemy_to_dict(data, column_whitelist=ALL_ATTRIBUTES_WHITELIST)
37-
return pack_json_result({"data": {"attributesByProjectId": data_dict}})
37+
return pack_json_result(data_dict)
3838

3939

4040
@router.get(
@@ -54,7 +54,7 @@ def get_check_composite_key(
5454
project_id,
5555
)
5656

57-
return pack_json_result({"data": {"checkCompositeKey": is_valid}})
57+
return pack_json_result(is_valid)
5858

5959

6060
@router.get(
@@ -71,12 +71,8 @@ def get_sample_records(
7171
)
7272
return pack_json_result(
7373
{
74-
"data": {
75-
"calculateUserAttributeSampleRecords": {
76-
"record_ids": record_ids,
77-
"calculated_attributes": calculated_attributes,
78-
}
79-
}
74+
"record_ids": record_ids,
75+
"calculated_attributes": calculated_attributes,
8076
}
8177
)
8278

@@ -91,4 +87,4 @@ def delete_user_attribute(
9187
body: DeleteUserAttributeBody = Body(...),
9288
):
9389
manager.delete_attribute(project_id, body.attribute_id)
94-
return pack_json_result({"data": {"deleteUserAttribute": {"ok": True}}})
90+
return get_silent_success()

fast_api/routes/client_response.py

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,5 +60,18 @@ def get_silent_success() -> JSONResponse:
6060
return SILENT_SUCCESS_RESPONSE
6161

6262

63+
def get_custom_response(status_code, content, type="text"):
64+
if type == "json":
65+
return JSONResponse(
66+
status_code=status_code,
67+
content=content,
68+
)
69+
else:
70+
return PlainTextResponse(
71+
status_code=status_code,
72+
content=content,
73+
)
74+
75+
6376
def to_json(obj: BaseModel):
6477
return json.loads(obj.json())

fast_api/routes/comment.py

Lines changed: 5 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -6,12 +6,11 @@
66
DeleteCommentBody,
77
UpdateCommentBody,
88
)
9-
from fast_api.routes.client_response import pack_json_result
9+
from fast_api.routes.client_response import pack_json_result, get_silent_success
1010
from submodules.model.enums import CommentCategory
1111
from util import notification
1212
from middleware.log_storage import extend_state_get_like
1313

14-
1514
router = APIRouter()
1615

1716

@@ -56,9 +55,7 @@ def get_all_comments(request: Request):
5655
"add_info": add_info,
5756
}
5857

59-
return pack_json_result(
60-
{"data": {"getAllComments": to_return}}, wrap_for_frontend=False
61-
)
58+
return pack_json_result(to_return, wrap_for_frontend=False)
6259

6360

6461
@router.post("/create-comment")
@@ -84,7 +81,7 @@ def create_comment(request: Request, body: CreateCommentBody = Body(...)):
8481
True,
8582
)
8683

87-
return pack_json_result({"data": {"createComment": {"ok": True}}})
84+
return get_silent_success()
8885

8986

9087
@router.delete("/delete-comment")
@@ -109,7 +106,7 @@ def delete_comment(
109106
True,
110107
)
111108

112-
return pack_json_result({"data": {"deleteComment": {"ok": True}}})
109+
return get_silent_success()
113110

114111

115112
@router.put("/update-comment")
@@ -134,7 +131,7 @@ def update_comment(
134131
True,
135132
)
136133

137-
return pack_json_result({"data": {"updateComment": {"ok": True}}})
134+
return get_silent_success()
138135

139136

140137
@router.get("/get-unique-comments-keys-for")

0 commit comments

Comments
 (0)