Remove refinery-config (#266)

* Initial config addition

* adjust services

* edits

* Adjust config route

* remove check_config_service

* move implementation

* Change order of config

* Access values directly

* adjust fastapi

* Add new routes for config

* edit

* edit

* add direct default route for base_config

* Fix: outdated lines

* Fix: unnecessary async

* Remove unused service

* Moves change config to misc routing

* remove org createino endpoint for os version

* Removes unused config parts

* remove unused notify services

* Removes auto notify on startup

---------

Co-authored-by: JWittmeyer <jens.wittmeyer@kern.ai>

Author: anmarhindi

.gitignore

@@ -143,4 +143,6 @@ project_export*
 tmp/*
 !tmp/.gitkeep
 
-logs/*
+logs/*
+
+current_config.json

api/misc.py

@@ -1,15 +1,12 @@
-from controller.misc import config_service
 from starlette.endpoints import HTTPEndpoint
 from starlette.responses import JSONResponse
-from starlette import status
+from fastapi import Request
 
+from config_handler import (
+    full_config_json,
+)
 
-class IsManagedRest(HTTPEndpoint):
-    def get(self, request) -> JSONResponse:
-        is_managed = config_service.get_config_value("is_managed")
-        return JSONResponse(is_managed, status_code=status.HTTP_200_OK)
 
-class IsDemoRest(HTTPEndpoint):
-    def get(self, request) -> JSONResponse:
-        is_managed = config_service.get_config_value("is_demo")
-        return JSONResponse(is_managed, status_code=status.HTTP_200_OK)
+class FullConfigRest(HTTPEndpoint):
+    def get(self, request: Request) -> JSONResponse:
+        return full_config_json()

api/transfer.py

@@ -28,7 +28,6 @@
 from controller.upload_task import manager as upload_task_manager
 from controller.auth import manager as auth_manager
 from controller.transfer import association_transfer_manager
-from controller.auth import manager as auth
 from controller.project import manager as project_manager
 from controller.attribute import manager as attribute_manager
 
@@ -135,7 +134,6 @@ def get(self, request) -> JSONResponse:
 
 class PrepareFileImport(HTTPEndpoint):
     async def post(self, request) -> JSONResponse:
-        auth.check_is_demo_without_info()
         project_id = request.path_params["project_id"]
         request_body = await request.json()
 
@@ -168,7 +166,6 @@ async def post(self, request) -> JSONResponse:
 
 class JSONImport(HTTPEndpoint):
     async def post(self, request) -> JSONResponse:
-        auth.check_is_demo_without_info()
         project_id = request.path_params["project_id"]
         request_body = await request.json()
         user_id = request_body["user_id"]
@@ -272,7 +269,6 @@ async def post(self, request) -> JSONResponse:
 
 class UploadTaskInfo(HTTPEndpoint):
     def get(self, request) -> JSONResponse:
-        auth.check_is_demo_without_info()
         project_id = request.path_params["project_id"]
         task_id = request.path_params["task_id"]
         user_id = request.query_params["user_id"]

app.py

@@ -1,9 +1,10 @@
 import logging
-
 from fastapi import FastAPI
 from api.healthcheck import Healthcheck
 from starlette.middleware import Middleware
-from api.misc import IsDemoRest, IsManagedRest
+from api.misc import (
+    FullConfigRest,
+)
 from api.project import ProjectDetails
 from api.transfer import (
     AssociationsImport,
@@ -16,6 +17,9 @@
     CognitionImport,
     CognitionPrepareProject,
 )
+from config_handler import (
+    init_config,
+)
 from fast_api.routes.organization import router as org_router
 from fast_api.routes.project import router as project_router
 from fast_api.routes.project_setting import router as project_setting_router
@@ -65,8 +69,10 @@
 logging.basicConfig(level=logging.DEBUG)
 logger = logging.getLogger(__name__)
 
+init_config()
 fastapi_app = FastAPI()
 
+
 fastapi_app.include_router(
     org_router, prefix=PREFIX_ORGANIZATION, tags=["organization"]
 )
@@ -110,7 +116,9 @@
 fastapi_app_internal.include_router(
     task_execution_router, prefix=PREFIX_TASK_EXECUTION, tags=["task-execution"]
 )
+
 routes = [
+    Route("/full_config", FullConfigRest),
     Route("/notify/{path:path}", Notify),
     Route("/healthcheck", Healthcheck),
     Route("/project/{project_id:str}", ProjectDetails),
@@ -130,8 +138,6 @@
         CognitionPrepareProject,
     ),
     Route("/project/{project_id:str}/import/task/{task_id:str}", UploadTaskInfo),
-    Route("/is_managed", IsManagedRest),
-    Route("/is_demo", IsDemoRest),
     Mount("/api", app=fastapi_app, name="REST API"),
     Mount(
         "/internal/api", app=fastapi_app_internal, name="INTERNAL REST API"

base_config.json

@@ -0,0 +1,8 @@
+{
+    "s3_region": null,
+    "KERN_S3_ENDPOINT": null,
+    "spacy_downloads": [
+        "en_core_web_sm",
+        "de_core_news_sm"
+    ]
+}

check_config_service

@@ -0,0 +1,123 @@
+from typing import Dict, Any, Optional, Union
+import os
+import json
+from notify_handler import notify_others_about_change_thread
+from fastapi import responses, status
+
+__config = None
+
+BASE_CONFIG_PATH = "base_config.json"
+CURRENT_CONFIG_PATH = "/config/current_config.json"
+
+SERVICES_TO_NOTIFY = {
+    "TOKENIZER": "http://refinery-tokenizer:80",
+}
+
+
+def get_config_value(
+    key: str, subkey: Optional[str] = None
+) -> Union[str, Dict[str, str]]:
+    if key not in __config:
+        raise ValueError(f"Key {key} coudn't be found in config")
+    value = __config[key]
+
+    if not subkey:
+        return value
+
+    if isinstance(value, dict) and subkey in value:
+        return value[subkey]
+    else:
+        raise ValueError(f"Subkey {subkey} coudn't be found in config[{key}]")
+
+
+def __read_and_change_base_config():
+    print("reading base config file", flush=True)
+    global __config
+    f = open(BASE_CONFIG_PATH)
+    __config = json.load(f)
+    __config["s3_region"] = os.getenv("S3_REGION", "eu-west-1")
+    __save_current_config()
+
+
+def change_config(changes: Dict[str, Any]) -> bool:
+    global __config
+    something_changed = False
+    for key in changes:
+        if key == "KERN_S3_ENDPOINT":
+            continue
+        if key in __config:
+            if isinstance(changes[key], dict):
+                for subkey in changes[key]:
+                    if subkey in __config[key]:
+                        __config[key][subkey] = changes[key][subkey]
+                        something_changed = True
+            else:
+                __config[key] = changes[key]
+                something_changed = True
+    if something_changed:
+        __save_current_config()
+    else:
+        print("nothing was changed with input", changes, flush=True)
+    return something_changed
+
+
+def __save_current_config() -> None:
+    print("saving config file", flush=True)
+    with open(CURRENT_CONFIG_PATH, "w") as f:
+        json.dump(__config, f, indent=4)
+
+
+def init_config() -> None:
+    if not os.path.exists(CURRENT_CONFIG_PATH):
+        __read_and_change_base_config()
+    else:
+        __load_and_remove_outdated_config_keys()
+    # this one is to be set on every start to ensure its up to date
+    print("setting s3 endpoint", flush=True)
+    __config["KERN_S3_ENDPOINT"] = os.getenv("KERN_S3_ENDPOINT")
+
+
+def __load_and_remove_outdated_config_keys():
+    if not os.path.exists(CURRENT_CONFIG_PATH):
+        return
+
+    global __config
+    with open(CURRENT_CONFIG_PATH) as f:
+        __config = json.load(f)
+
+    with open(BASE_CONFIG_PATH) as f:
+        base_config = json.load(f)
+
+    to_remove = [key for key in __config if key not in base_config]
+
+    if len(to_remove) > 0:
+        print("removing outdated config keys", to_remove, flush=True)
+        for key in to_remove:
+            del __config[key]
+        __save_current_config()
+
+
+def get_config() -> Dict[str, Any]:
+    global __config
+    return __config
+
+
+def change_json(config_data) -> responses.PlainTextResponse:
+    try:
+        has_changed = change_config(config_data)
+
+        if has_changed:
+            notify_others_about_change_thread(SERVICES_TO_NOTIFY)
+
+        return responses.PlainTextResponse(
+            f"Did update: {has_changed}", status_code=status.HTTP_200_OK
+        )
+
+    except Exception as e:
+        return responses.PlainTextResponse(
+            f"Error: {str(e)}", status_code=status.HTTP_500_INTERNAL_SERVER_ERROR
+        )
+
+
+def full_config_json() -> responses.JSONResponse:
+    return responses.JSONResponse(status_code=status.HTTP_200_OK, content=get_config())

config/.gitkeep

@@ -1,10 +1,8 @@
 from typing import Any, Dict
 
 from fastapi import Request
-from controller.misc import config_service
 from exceptions.exceptions import (
     AuthManagerError,
-    NotAllowedInDemoError,
     ProjectAccessError,
 )
 import jwt
@@ -14,7 +12,6 @@
 from submodules.model import enums, exceptions
 from submodules.model.business_objects import organization
 from submodules.model.models import Organization, Project, User
-from controller.misc import manager as misc_manager
 import sqlalchemy
 
 DEV_USER_ID = "741df1c2-a531-43b6-b259-df23bc78e9a2"
@@ -127,28 +124,6 @@ def check_is_admin(request: Any) -> bool:
     return False
 
 
-def check_demo_access(info: Any) -> None:
-    if not check_is_admin(info.context["request"]) and config_service.get_config_value(
-        "is_demo"
-    ):
-        check_black_white(info)
-
-
-def check_black_white(info: Any):
-    black_white = misc_manager.get_black_white_demo()
-    if str(info.parent_type) == "Mutation":
-        if info.field_name not in black_white["mutations"]:
-            raise NotAllowedInDemoError
-    elif str(info.parent_type) == "Query":
-        if info.field_name in black_white["queries"]:
-            raise NotAllowedInDemoError
-
-
-def check_is_demo_without_info() -> None:
-    if config_service.get_config_value("is_demo"):
-        raise NotAllowedInDemoError
-
-
 def check_is_single_organization() -> bool:
     return len(organization_manager.get_all_organizations()) == 1
 

config_handler.py

@@ -42,14 +42,11 @@ def get_current_terms_text(
     return term_text
 
 
-def get_recommended_encoders(is_managed: bool) -> List[Any]:
-    # only use is_managed if it is really managed
+def get_recommended_encoders() -> List[Any]:
     # can run into circular import problems if directly resolved here by helper method
     recommendations = connector.request_listing_recommended_encoders()
-    if is_managed:
-        existing_models = model_manager.get_model_provider_info()
-    else:
-        existing_models = []
+    existing_models = model_manager.get_model_provider_info()
+
     for model in existing_models:
         not_yet_known = (
             len(

controller/auth/manager.py

@@ -7,7 +7,6 @@
     information_source,
     payload,
 )
-from controller.misc import config_service
 from controller.labeling_access_link import manager as link_manager
 from submodules.model import daemon
 
@@ -86,7 +85,6 @@ def delete_information_source(project_id: str, source_id: str) -> None:
     if (
         information_source_item.type
         == enums.InformationSourceType.ACTIVE_LEARNING.value
-        and config_service.get_config_value("is_managed")
     ):
         daemon.run_without_db_token(
             __delete_active_learner_from_inference_dir, project_id, source_id