Use all cpus for workers to run evaluators in parallel for supporting many pii detection calls at the same time and remove unsupported en piis

Author: rogeriochaves

evaluators/presidio/langevals_presidio/pii_detection.py

@@ -39,7 +39,6 @@ class PresidioEntities(BaseModel):
     us_ssn: bool = False
     uk_nhs: bool = False
     sg_nric_fin: bool = False
-    sg_uen: bool = False
     au_abn: bool = False
     au_acn: bool = False
     au_tfn: bool = False
@@ -49,7 +48,6 @@ class PresidioEntities(BaseModel):
     in_vehicle_registration: bool = False
     in_voter: bool = False
     in_passport: bool = False
-    fi_personal_identity_code: bool = False
 
 
 class PresidioPIIDetectionSettings(EvaluatorSettings):

langevals/server.py

@@ -3,14 +3,14 @@
 import dotenv
 
 from langevals.utils import (
+    get_cpu_count,
     get_evaluator_classes,
     get_evaluator_definitions,
     load_evaluator_packages,
 )
 
 dotenv.load_dotenv()
 
-import asyncio
 from fastapi import FastAPI, HTTPException, Request
 from typing import List, Optional
 from langevals_core.base_evaluator import (
@@ -106,16 +106,39 @@ def main():
             f.write(json.dumps(app.openapi(), indent=2))
         print("openapi.json exported")
         return
-    from hypercorn.config import Config
-    from hypercorn.asyncio import serve
+    import gunicorn.app.base
+
+    class StandaloneApplication(gunicorn.app.base.BaseApplication):
+        def __init__(self, app, options=None):
+            self.options = options or {}
+            self.application = app
+            super().__init__()
+
+        def load_config(self):
+            config = {key: value for key, value in self.options.items()
+                    if key in self.cfg.settings and value is not None} # type: ignore
+            for key, value in config.items():
+                self.cfg.set(key.lower(), value) # type: ignore
+
+        def load(self):
+            return self.application
+
 
     host = "0.0.0.0"
     port = int(os.getenv("PORT", 8000))
+    workers = get_cpu_count()
+
+    print(f"Starting server with {workers} workers")
 
-    config = Config()
-    config.bind = [f"{host}:{port}"]
+    options = {
+        'bind': f'{host}:{port}',
+        'workers': workers,
+        'worker_class': 'uvicorn.workers.UvicornWorker',
+        'preload_app': True,
+        'forwarded_allow_ips': '*',
+    }
 
-    asyncio.run(serve(app, config))  # type: ignore
+    StandaloneApplication(app, options).run()
 
 
 if __name__ == "__main__":

langevals/utils.py

@@ -1,5 +1,7 @@
 import importlib
 import importlib.metadata
+import math
+import os
 import pkgutil
 import re
 import textwrap
@@ -99,3 +101,21 @@ def get_evaluator_definitions(evaluator_cls: BaseEvaluator):
         is_guardrail=is_guardrail,
         category=category,
     )
+
+
+def get_cpu_count():
+    cpu_count = os.getenv("CPU_COUNT", None)
+    if cpu_count is not None:
+        return int(cpu_count)
+    try:
+        # Kubernetes
+        with open("/sys/fs/cgroup/cpu/cpu.shares") as f:
+            cpu_shares = int(f.read().strip())
+        return max(1, math.ceil(cpu_shares / 1024))
+    except FileNotFoundError:
+        try:
+            # Local for UNIX
+            return len(os.sched_getaffinity(0))
+        except AttributeError:
+            # Local fallback
+            return os.cpu_count() or 4

poetry.lock

@@ -17,12 +17,13 @@ priority = "explicit"
 pytest = "^8.1.2"
 python = "^3.11"
 fastapi = "^0.110.0"
-hypercorn = "^0.16.0"
 pydantic = "^2.6.4"
 python-dotenv = "1.0.1"
 mangum = "^0.17.0"
 pandas = "^2.2.2"
 markdown = "^3.6"
+uvicorn = "^0.30.6"
+gunicorn = "^23.0.0"
 [[tool.poetry.dependencies.torch]]
 version = "^2.2.2+cpu"
 optional = true

pyproject.toml

@@ -286,7 +286,6 @@ export type Evaluators = {
         us_ssn: boolean;
         uk_nhs: boolean;
         sg_nric_fin: boolean;
-        sg_uen: boolean;
         au_abn: boolean;
         au_acn: boolean;
         au_tfn: boolean;
@@ -296,7 +295,6 @@ export type Evaluators = {
         in_vehicle_registration: boolean;
         in_voter: boolean;
         in_passport: boolean;
-        fi_personal_identity_code: boolean;
       };
       min_threshold: number;
     };
@@ -897,7 +895,6 @@ social security numbers. It allows customization of the detection threshold and
           us_ssn: false,
           uk_nhs: false,
           sg_nric_fin: false,
-          sg_uen: false,
           au_abn: false,
           au_acn: false,
           au_tfn: false,
@@ -907,7 +904,6 @@ social security numbers. It allows customization of the detection threshold and
           in_vehicle_registration: false,
           in_voter: false,
           in_passport: false,
-          fi_personal_identity_code: false,
         },
       },
       min_threshold: {