refactor!: Replace Pydantic model for LLM config with plain dict

This change replaces the Pydantic model used for the LLM configuration (`llm`) with a plain dictionary, improving flexibility by exposing all parameters directly to the deployer.

While it would be preferable to annotate `llm` as `langchain_openai.ChatOpenAI`, there is a limitation: Pydantic raises an error ("cannot pickle '_thread.RLock' object"), likely due to incompatibility within `langchain_openai.ChatOpenAI`.

Using a plain dictionary removes validation at startup, but the default empty dict is sufficient to create a `ChatOpenAI` instance, making this an acceptable tradeoff for now.

Author: edwardzjl

api/chatbot/config.py

@@ -1,16 +1,10 @@
 import re
+from typing import Any
 
-from pydantic import BaseModel, HttpUrl, PostgresDsn
+from pydantic import Field, PostgresDsn
 from pydantic_settings import BaseSettings, SettingsConfigDict
 
 
-class LLMServiceSettings(BaseModel):
-    url: HttpUrl = "http://localhost:8080"
-    """llm service url"""
-    model: str = "cognitivecomputations/dolphin-2.6-mistral-7b-dpo-laser"
-    creds: str = "EMPTY"
-
-
 def remove_postgresql_variants(dsn: str) -> str:
     """Remove the 'driver' part from a connection string, if one is present in the URI scheme.
 
@@ -34,7 +28,7 @@ def remove_postgresql_variants(dsn: str) -> str:
 class Settings(BaseSettings):
     model_config = SettingsConfigDict(env_nested_delimiter="__")
 
-    llm: LLMServiceSettings = LLMServiceSettings()
+    llm: dict[str, Any] = Field(default_factory=dict)
     db_url: PostgresDsn = "postgresql+psycopg://postgres:postgres@localhost:5432/"
     """Database url. Must be a valid postgresql connection string."""
 

api/chatbot/state.py

@@ -19,10 +19,4 @@
     autoflush=False,
     class_=AsyncSession,
 )
-chat_model = ChatOpenAI(
-    openai_api_base=str(settings.llm.url),
-    model=settings.llm.model,
-    openai_api_key=settings.llm.creds,
-    max_tokens=1024,
-    streaming=True,
-)
+chat_model = ChatOpenAI(**settings.llm)

api/tests/test_config.py

@@ -1,19 +1,57 @@
-import os
 import unittest
-from unittest.mock import patch
 
-from chatbot.config import Settings
+from pydantic import ValidationError
+
+from chatbot.config import Settings, remove_postgresql_variants
+
+
+class TestRemovePostgresqlVariants(unittest.TestCase):
+
+    def test_remove_psycopg(self):
+        dsn = "postgresql+psycopg://user:pass@localhost/dbname"
+        expected = "postgresql://user:pass@localhost/dbname"
+        self.assertEqual(remove_postgresql_variants(dsn), expected)
+
+    def test_remove_psycopg2(self):
+        dsn = "postgresql+psycopg2://user:pass@localhost/dbname"
+        expected = "postgresql://user:pass@localhost/dbname"
+        self.assertEqual(remove_postgresql_variants(dsn), expected)
+
+    def test_remove_psycopg2cffi(self):
+        dsn = "postgresql+psycopg2cffi://user:pass@localhost/dbname"
+        expected = "postgresql://user:pass@localhost/dbname"
+        self.assertEqual(remove_postgresql_variants(dsn), expected)
+
+    def test_no_change(self):
+        dsn = "postgresql://user:pass@localhost/dbname"
+        expected = "postgresql://user:pass@localhost/dbname"
+        self.assertEqual(remove_postgresql_variants(dsn), expected)
 
 
 class TestSettings(unittest.TestCase):
-    def test_default_inferece_url(self):
+    def test_llm_default(self):
         settings = Settings()
-        self.assertEqual(str(settings.llm.url), "http://localhost:8080")
+        self.assertEqual(settings.llm, {})
+
+    def test_llm_custom(self):
+        custom_llm = {"model": "gpt-3", "version": "davinci"}
+        settings = Settings(llm=custom_llm)
+        self.assertEqual(settings.llm, custom_llm)
 
-    @patch.dict(os.environ, {"LLM__URL": "http://foo.bar.com"}, clear=True)
-    def test_inference_server_url(self):
+    def test_psycopg_url_default(self):
         settings = Settings()
-        self.assertEqual(str(settings.llm.url), "http://foo.bar.com/")
+        expected = "postgresql://postgres:postgres@localhost:5432/"
+        self.assertEqual(settings.psycopg_url, expected)
+
+    def test_psycopg_url_custom(self):
+        custom_url = "postgresql+psycopg2://custom_user:custom_pass@localhost/custom_db"
+        settings = Settings(db_url=custom_url)
+        expected = "postgresql://custom_user:custom_pass@localhost/custom_db"
+        self.assertEqual(settings.psycopg_url, expected)
+
+    def test_invalid_db_url(self):
+        with self.assertRaises(ValidationError):
+            Settings(db_url="invalid_url")
 
 
 if __name__ == "__main__":

manifests/base/params.env

@@ -1,2 +1 @@
-LLM__URL=http://qwen2dot5-72b-instruct.skynet.svc.cluster.local/v1
-LLM__MODEL=qwen2.5-72b-instruct
+LLM={"base_url": "http://qwen2dot5-72b-instruct.skynet.svc.cluster.local/v1", "model_name": "qwen2.5-72b-instruct", "api_key": "NOTHING", "temperature": "0.7", "top_p": "0.8", "max_tokens": "1024", "streaming": "True", "stream_usage": "True", "extra_body": {"repetition_penalty": "1.05"}}