Merge pull request #48 from mobiusml/num_workers

Make the Number of Request Workers Configurable

Author: movchan74

README.md

@@ -202,4 +202,13 @@ ORM models referenced in the rest of the code should be imported from `aana.mode
 not from that model's file for reasons explained in `aana/models/db/__init__.py`. This also means that 
 if you add a new model class, it should be imported by `__init__.py` in addition to creating a migration.
 
-Higher level code for interacting with the ORM is available in `aana.repository.data`.
+Higher level code for interacting with the ORM is available in `aana.repository.data`.
+
+## Settings
+
+Here are the environment variables that can be used to configure the Aaana SDK:
+- TMP_DATA_DIR: The directory to store temporary data. Default: `/tmp/aana`.
+- NUM_WORKERS: The number of request workers. Default: `2`.
+- DB_CONFIG: The database configuration in the format `{"datastore_type": "sqlite", "datastore_config": {"path": "/path/to/sqlite.db"}}`. Currently only SQLite and PostgreSQL are supported. Default: `{"datastore_type": "sqlite", "datastore_config": {"path": "/var/lib/aana_data"}}`.
+- USE_DEPLOYMENT_CACHE (testing only): If set to `true`, the tests will use the deployment cache to avoid downloading the models and running the deployments. Default: `false`.
+- SAVE_DEPLOYMENT_CACHE (testing only): If set to `true`, the tests will save the deployment cache after running the deployments. Default: `false`.

aana/configs/settings.py

@@ -19,6 +19,7 @@ class Settings(BaseSettings):
     tmp_data_dir: Path = Path("/tmp/aana_data")  # noqa: S108
     image_dir = tmp_data_dir / "images"
     video_dir = tmp_data_dir / "videos"
+    num_workers: int = 2
 
     db_config: DBConfig = {
         "datastore_type": "sqlite",

aana/main.py

@@ -53,7 +53,9 @@ def run():
         }
     }
     try:
-        server = RequestHandler.bind(endpoints, pipeline_nodes, context)
+        server = RequestHandler.options(num_replicas=aana_settings.num_workers).bind(
+            endpoints, pipeline_nodes, context
+        )
         serve.run(server, port=args.port, host=args.host)
         # TODO: add logging
         print("Deployed Serve app successfully.")

aana/tests/conftest.py

@@ -90,7 +90,8 @@ def start_ray_serve(endpoints, nodes, context, runtime_env=None):
         if runtime_env is None:
             runtime_env = {}
         server = RequestHandler.options(
-            ray_actor_options={"runtime_env": runtime_env}
+            num_replicas=aana_settings.num_workers,
+            ray_actor_options={"runtime_env": runtime_env},
         ).bind(endpoints, nodes, context)
         return setup_deployment(server)