Merge branch 'dev' into etl-other-providers

Author: anmarhindi

.github/workflows/az_acr_push.yml

@@ -0,0 +1,16 @@
+name: 'ACR: Docker Push'
+
+on:
+  workflow_dispatch:
+  
+  push:
+    branches:
+      - "**"
+      - "!dev"
+    tags-ignore:
+      - "**"
+    
+jobs: 
+  call-az-acr-push:
+    uses: code-kern-ai/cicd-deployment-scripts/.github/workflows/az_acr_push.yml@dev
+    secrets: inherit

.github/workflows/k8s_deploy.yml

@@ -0,0 +1,22 @@
+name: 'K8: Prepare Deployment' # rename
+
+on:
+  # workflow_dispatch:
+  release:
+    types: [prereleased]
+  pull_request:
+    branches:
+      - dev
+    types: [closed]
+
+# Special permissions required for OIDC authentication
+permissions:
+  id-token: write
+  contents: read
+  actions: read
+
+jobs:
+  call-k8-release:
+    uses: code-kern-ai/cicd-deployment-scripts/.github/workflows/k8s_release.yml@dev
+    if: github.event_name == 'pull_request' && github.event.pull_request.merged || github.event_name == 'release'
+    secrets: inherit

.github/workflows/k8s_test.yml

@@ -0,0 +1,19 @@
+name: 'K8: Test'
+
+on:
+  pull_request:
+    types: [opened, synchronize, reopened]
+
+# Special permissions required for OIDC authentication
+permissions:
+  id-token: write
+  contents: read
+  actions: read
+
+
+jobs:
+  call-k8-test:
+    uses: code-kern-ai/cicd-deployment-scripts/.github/workflows/k8s_test.yml@dev
+    secrets: inherit
+    with:
+      test_cmd: 'pytest'

CODEOWNERS

@@ -0,0 +1 @@
+.github/ @code-kern-ai/devops-admin @code-kern-ai/dev-admin

controller/task_manager.py

@@ -87,7 +87,7 @@ def start_rats_task(
     only_uploaded_attributes: bool = False,
     attribute_id: Optional[str] = None,
 ) -> int:
-    if tokenization.is_doc_bin_creation_running(project_id):
+    if tokenization.is_doc_bin_creation_running_or_queued(project_id, only_running=True):
         # at the end of doc bin creation rats will be calculated
         return
 

requirements.txt

@@ -2,13 +2,15 @@
 # This file is autogenerated by pip-compile with Python 3.9
 # by the following command:
 #
-#    pip-compile --output-file=requirements.txt requirements/requirements.in
+#    pip-compile --output-file=requirements.txt --strip-extras requirements/requirements.in
 #
+annotated-types==0.7.0
+    # via pydantic
 anyio==4.4.0
     # via
     #   -r requirements/common-requirements.txt
     #   starlette
-blis==0.7.9
+blis==0.7.11
     # via thinc
 boto3==1.25.0
     # via -r requirements/common-requirements.txt
@@ -17,7 +19,7 @@ botocore==1.28.5
     #   -r requirements/common-requirements.txt
     #   boto3
     #   s3transfer
-catalogue==2.0.8
+catalogue==2.0.10
     # via
     #   spacy
     #   srsly
@@ -36,9 +38,13 @@ click==8.1.7
     #   -r requirements/common-requirements.txt
     #   typer
     #   uvicorn
-confection==0.0.3
-    # via thinc
-cymem==2.0.7
+cloudpathlib==0.18.1
+    # via weasel
+confection==0.1.5
+    # via
+    #   thinc
+    #   weasel
+cymem==2.0.8
     # via
     #   preshed
     #   spacy
@@ -49,6 +55,8 @@ exceptiongroup==1.2.1
     #   anyio
 fastapi==0.110.3
     # via -r requirements/common-requirements.txt
+greenlet==3.0.3
+    # via sqlalchemy
 h11==0.14.0
     # via
     #   -r requirements/common-requirements.txt
@@ -58,20 +66,28 @@ idna==3.7
     #   -r requirements/common-requirements.txt
     #   anyio
     #   requests
-jinja2==3.1.3
+jinja2==3.1.4
     # via spacy
 jmespath==1.0.1
     # via
     #   -r requirements/common-requirements.txt
     #   boto3
     #   botocore
-langcodes==3.3.0
+langcodes==3.4.0
     # via spacy
-markupsafe==2.1.1
+language-data==1.2.0
+    # via langcodes
+marisa-trie==1.2.0
+    # via language-data
+markdown-it-py==3.0.0
+    # via rich
+markupsafe==2.1.5
     # via jinja2
+mdurl==0.1.2
+    # via markdown-it-py
 minio==7.1.12
     # via -r requirements/common-requirements.txt
-murmurhash==1.0.9
+murmurhash==1.0.10
     # via
     #   preshed
     #   spacy
@@ -83,27 +99,31 @@ numpy==1.23.4
     #   pandas
     #   spacy
     #   thinc
-packaging==21.3
-    # via spacy
+packaging==24.1
+    # via
+    #   spacy
+    #   thinc
+    #   weasel
 pandas==1.5.1
     # via -r requirements/common-requirements.txt
-pathy==0.6.2
-    # via spacy
-preshed==3.0.8
+preshed==3.0.9
     # via
     #   spacy
     #   thinc
 psycopg2-binary==2.9.9
     # via -r requirements/common-requirements.txt
-pydantic==1.10.13
+pydantic==2.7.4
     # via
-    #   -r requirements/common-requirements.txt
+    #   -r requirements/requirements.in
     #   confection
     #   fastapi
     #   spacy
     #   thinc
-pyparsing==3.0.9
-    # via packaging
+    #   weasel
+pydantic-core==2.18.4
+    # via pydantic
+pygments==2.18.0
+    # via rich
 python-dateutil==2.9.0.post0
     # via
     #   -r requirements/common-requirements.txt
@@ -117,52 +137,69 @@ requests==2.31.0
     # via
     #   -r requirements/common-requirements.txt
     #   spacy
+    #   weasel
+rich==13.7.1
+    # via typer
 s3transfer==0.6.2
     # via
     #   -r requirements/common-requirements.txt
     #   boto3
+shellingham==1.5.4
+    # via typer
 six==1.16.0
     # via
     #   -r requirements/common-requirements.txt
     #   python-dateutil
-smart-open==5.2.1
-    # via pathy
+smart-open==7.0.4
+    # via weasel
 sniffio==1.3.1
     # via
     #   -r requirements/common-requirements.txt
     #   anyio
-spacy==3.4.2
-    # via -r requirements/requirements.in
-spacy-legacy==3.0.10
+spacy==3.7.5
+    # via
+    #   -r requirements/requirements.in
+    #   spacy
+spacy-legacy==3.0.12
     # via spacy
-spacy-loggers==1.0.3
+spacy-loggers==1.0.5
     # via spacy
 sqlalchemy==1.4.42
     # via -r requirements/common-requirements.txt
-srsly==2.4.5
+srsly==2.4.8
     # via
     #   confection
     #   spacy
     #   thinc
+    #   weasel
 starlette==0.37.2
     # via
     #   -r requirements/common-requirements.txt
     #   fastapi
-thinc==8.1.5
+sudachidict-core==20240409
+    # via spacy
+sudachipy==0.6.8
+    # via
+    #   spacy
+    #   sudachidict-core
+thinc==8.2.5
     # via spacy
-tqdm==4.64.1
+tqdm==4.66.4
     # via spacy
-typer==0.4.2
+typer==0.12.3
     # via
-    #   pathy
     #   spacy
+    #   weasel
 typing-extensions==4.12.1
     # via
     #   -r requirements/common-requirements.txt
     #   anyio
+    #   cloudpathlib
     #   fastapi
     #   pydantic
+    #   pydantic-core
     #   starlette
+    #   typer
 urllib3==1.26.18
     # via
     #   -r requirements/common-requirements.txt
@@ -171,11 +208,15 @@ urllib3==1.26.18
     #   requests
 uvicorn==0.22.0
     # via -r requirements/common-requirements.txt
-wasabi==0.10.1
+wasabi==1.1.3
     # via
     #   spacy
-    #   spacy-loggers
     #   thinc
+    #   weasel
+weasel==0.4.1
+    # via spacy
+wrapt==1.16.0
+    # via smart-open
 
 # The following packages are considered to be unsafe in a requirements file:
 # setuptools

requirements/common-requirements.txt

@@ -44,7 +44,7 @@ pandas==1.5.1
     # via -r common-requirements.in
 psycopg2-binary==2.9.9
     # via -r common-requirements.in
-pydantic==1.10.13
+# pydantic==1.10.13
     # via
     #   -r mini-requirements.in
     #   fastapi

requirements/requirements.in

@@ -1,2 +1,3 @@
 -r common-requirements.txt
-spacy==3.4.2
+spacy[ja]==3.7.5
+pydantic==2.7.4