apache pinot support

Author: zschumacher

.github/workflows/test-pinot.yml

@@ -0,0 +1,63 @@
+name: test-pinot
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+
+jobs:
+
+
+
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [ '3.9', '3.10', '3.11', '3.12', '3.13' ]
+
+
+    steps:
+      - name: checkout
+        uses: actions/checkout@v4
+
+      - name: set up python
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: install poetry
+        env:
+          POETRY_VERSION: "2.0.1"
+        run: |
+          curl -sSL https://install.python-poetry.org | python -
+          poetry config virtualenvs.create false
+
+      - name: cache poetry.lock dependencies
+        id: cache-poetry-deps
+        uses: actions/cache@v4
+        with:
+          path: ${{ env.pythonLocation }}
+          key: ${{ runner.os }}-${{ env.pythonLocation }}-${{ hashFiles('poetry.lock') }}-test-pinot
+
+      - name: install dependencies
+        if: steps.cache-poetry-deps.outputs.cache-hit != 'true'
+        run: poetry install -E pinotdb
+
+      # we only have to start the cluster - no tables needed in ci bc http table queries are recorded w/ vcr
+      - name: start pinot cluster
+        run: make startpinot
+
+      - name: test
+        env:
+          PYTHON_VERSION: ${{ matrix.python-version }}
+        run: |
+          poetry run pytest -m "pinot"  --cov=. --cov-branch -v --durations=25 --cov-report=xml
+
+      - uses: codecov/codecov-action@v5
+        with:
+          fail_ci_if_error: true
+          flags: ${{ matrix.python-version}}-mssql
+          token: ${{ secrets.CODECOV_TOKEN }}

Makefile

@@ -71,3 +71,45 @@ resetlock: ## reset the poetry lock file from main
 	rm poetry.lock
 	git checkout main poetry.lock
 	poetry lock --no-update
+
+waitforpinot:
+	@echo "Waiting for Pinot container to be healthy..."
+	@timeout=60; \
+	while [ $$timeout -gt 0 ]; do \
+	    status=$$(docker inspect -f '{{.State.Health.Status}}' pinot-quickstart); \
+	    if [ "$$status" = "healthy" ]; then \
+	        echo "Pinot container is healthy and ready!"; \
+	        exit 0; \
+	    elif [ "$$status" = "unhealthy" ]; then \
+	        echo "ERROR: Pinot container became unhealthy."; \
+	        exit 1; \
+	    fi; \
+	    echo "Waiting for container to be healthy... ($$timeout seconds left)"; \
+	    sleep 2; \
+	    timeout=$$((timeout - 2)); \
+	done; \
+	echo "ERROR: Pinot container did not become healthy within the timeout period."; \
+	exit 1
+
+pinottables:
+	@poetry run python scripts/seed_pinot.py
+	echo "Pinot successfully started and seeded!"
+
+startpinot:  # starts apache pinot batch processing quick start
+	@echo "Staring apache pinot"
+	@docker run -d --name pinot-quickstart -p 9000:9000 \
+      -p 8099:8000 \
+      --health-cmd="curl -f http://localhost:9000/health || exit 1" \
+	  --health-interval=10s \
+	  --health-timeout=5s \
+	  --health-retries=3 \
+	  --health-start-period=5s \
+	  apachepinot/pinot:latest QuickStart -type empty
+
+stoppinot:
+	@docker stop pinot-quickstart
+	@docker rm pinot-quickstart
+
+pinot: startpinot waitforpinot pinottables
+
+resetpinot: stoppinot pinot

poetry.lock

@@ -7,6 +7,7 @@
 from .main import using_async
 from .mssql import PymssqlCommands as _PymssqlCommands
 from .mysql import MySqlConnectorPythonCommands as _MySqlConnectorPythonCommands
+from .pinot import PinotDbCommands as _PinotDbCommands
 from .postgresql import AiopgCommands as _AioPgCommand
 from .postgresql import Psycopg2Commands as _Psycopg2Commands
 from .postgresql import Psycopg3Commands as _Psycopg3Commands

pydapper/__init__.py

@@ -10,6 +10,7 @@
     "mysql": "mysql",
     "oracle": "oracledb",
     "bigquery": "google",
+    "pinot": "pinotdb",
 }
 
 

pydapper/dsn_parser.py

@@ -0,0 +1 @@
+from .pinotdb import PinotDbCommands