feat: use ollama

Author: gary-van-woerkens

.github/workflows/create_instance.yaml

@@ -3,10 +3,10 @@ name: Create instance
 on:
   workflow_dispatch:
     inputs:
-      s3ModelPath:
-        description: 'The model to fetch from S3'
+      modelName:
+        description: 'The model to use'
         required: true
-        default: 'distilgpt2'
+        default: 'llama3.1:70b'
       instanceNameSuffix:
         description: 'Instance name suffix'
         required: false
@@ -70,9 +70,5 @@ jobs:
           OVH_SSH_KEY_ID: ${{ secrets.OVH_SSH_KEY_ID }}
           OVH_SERVICE_NAME: ${{ secrets.OVH_SERVICE_NAME }}
           AUTH_TOKEN: ${{ secrets.AUTH_TOKEN }}
-          HUGGING_FACE_HUB_TOKEN: ${{ secrets.HUGGING_FACE_HUB_TOKEN }}
-          S3_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY_ID}}
-          S3_SECRET_ACCESS_KEY: ${{ secrets.S3_SECRET_ACCESS_KEY}}
-          S3_ENDPOINT_URL: ${{ vars.S3_ENDPOINT_URL }}
-          S3_MODEL_PATH: ${{ inputs.s3ModelPath }}
           INSTANCE_NAME_SUFFIX: ${{ inputs.instanceNameSuffix }}
+          MODEL_NAME: ${{ inputs.modelName }}

.github/workflows/delete_instance.yaml

@@ -41,9 +41,4 @@ jobs:
           OVH_SSH_KEY_ID: ${{ secrets.OVH_SSH_KEY_ID }}
           OVH_SERVICE_NAME: ${{ secrets.OVH_SERVICE_NAME }}
           AUTH_TOKEN: ${{ secrets.AUTH_TOKEN }}
-          HUGGING_FACE_HUB_TOKEN: ${{ secrets.HUGGING_FACE_HUB_TOKEN }}
-          S3_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY_ID}}
-          S3_SECRET_ACCESS_KEY: ${{ secrets.S3_SECRET_ACCESS_KEY}}
-          S3_ENDPOINT_URL: ${{ vars.S3_ENDPOINT_URL }}
-          S3_MODEL_PATH: "distilgpt2"
           INSTANCE_NAME_SUFFIX: ${{ inputs.instanceNameSuffix }}

docker-compose.yaml

@@ -1,25 +1,24 @@
 services:
-  llm-service:
+  ollama-service:
     restart: always
-    image: ghcr.io/socialgouv/llm-inference-server:main
-    environment:
-      HUGGING_FACE_HUB_TOKEN: "${HUGGING_FACE_HUB_TOKEN}"
-      S3_ACCESS_KEY_ID: "${S3_ACCESS_KEY_ID}"
-      S3_SECRET_ACCESS_KEY: "${S3_SECRET_ACCESS_KEY}"
-      S3_ENDPOINT_URL: ${S3_ENDPOINT_URL}
-      S3_MODEL_PATH: ${S3_MODEL_PATH}
+    image: ollama/ollama
+    tty: true
     expose:
-      - "8000"
+      - "11434"
     ports:
-      - "127.0.0.1:8000:8000"
+      - "127.0.0.1:11434:11434"
     labels:
-      - "traefik.http.routers.vllm-service.rule=Host(`${HOST}.nip.io`)"
-      - "traefik.http.routers.vllm-service.entrypoints=websecure"
-      - "traefik.http.routers.vllm-service.tls.certresolver=myresolver"
+      - "traefik.http.routers.ollama-service.rule=Host(`${HOST}.nip.io`)"
+      - "traefik.http.routers.ollama-service.entrypoints=websecure"
+      - "traefik.http.routers.ollama-service.tls.certresolver=myresolver"
       - "traefik.http.middlewares.main-auth.basicauth.users=${CREDENTIALS}"
-      - "traefik.http.routers.vllm-service.middlewares=main-auth@docker"
-    runtime: nvidia  # Set the desired runtime for GPU support
-    ipc: host  # Set the IPC namespace to host
+      - "traefik.http.routers.ollama-service.middlewares=main-auth@docker"
+      - "traefik.http.services.ollama-service.loadbalancer.server.port=11434"
+    environment:
+      OLLAMA_KEEP_ALIVE: "-1"
+      VERBOSE: "${VERBOSE:-0}"
+    runtime: nvidia
+    ipc: host
     deploy:
       resources:
         reservations:
@@ -28,13 +27,7 @@ services:
               count: all
               capabilities: [gpu]
     volumes:
-      - /opt/vllm/models:/app/models
-    # healthcheck:
-    #   test: curl --fail http://localhost:8000/v1/models || exit 1
-    #   interval: 10s
-    #   timeout: 30s
-    #   retries: 60
-    #   start_period: 60s
+      - "./.ollama:/root/.ollama"
 
   reverse-proxy:
     image: traefik:v2.4
@@ -53,14 +46,3 @@ services:
     volumes:
       - "/var/run/docker.sock:/var/run/docker.sock"
       - "./letsencrypt:/letsencrypt"
-
-  # autoheal:
-  #   image: willfarrell/autoheal:latest
-  #   tty: true
-  #   restart: always
-  #   environment:
-  #     - AUTOHEAL_INTERVAL=60
-  #     - AUTOHEAL_START_PERIOD=300
-  #     - AUTOHEAL_DEFAULT_STOP_TIMEOUT=10
-  #   volumes:
-  #     - /var/run/docker.sock:/var/run/docker.sock

main.py

@@ -41,19 +41,15 @@ def indentString(input_string, indent_level=4):
 if (instanceNameSuffix):
     instanceNameSuffix = f"-{instanceNameSuffix}"
 
-instanceName = "vllm-managed-instance" + instanceNameSuffix
+instanceName = "ollama-managed-instance" + instanceNameSuffix
 serviceName = getRequiredEnv("OVH_SERVICE_NAME")
 sshKeyId = getRequiredEnv("OVH_SSH_KEY_ID")
 flavorId = getRequiredEnv("OVH_INSTANCE_FLAVOR_ID")
 imageId = getRequiredEnv("OVH_INSTANCE_IMAGE_ID")
 region = getRequiredEnv("OVH_REGION")
 authToken = getRequiredEnv("AUTH_TOKEN")
-huggingFaceHubToken = getRequiredEnv("HUGGING_FACE_HUB_TOKEN")
-s3AccessKeyId = getRequiredEnv("S3_ACCESS_KEY_ID")
-s3SecretAccessKey = getRequiredEnv("S3_SECRET_ACCESS_KEY")
-s3ModelPath = getRequiredEnv("S3_MODEL_PATH")
-s3EndpointUrl = getRequiredEnv("S3_ENDPOINT_URL")
 users = os.getenv("USERS", "")
+modelName = getRequiredEnv("MODEL_NAME")
 
 if users:
   users = f"""
@@ -97,33 +93,32 @@ def indentString(input_string, indent_level=4):
     permissions: "0600"
     owner: ubuntu:ubuntu
     content: |
-  - path: /opt/vllm/init.sh
+  - path: /opt/ollama/init.sh
     owner: ubuntu:ubuntu
     permissions: "0775"
     content: |
         #!/bin/bash
 
         # init config
-        sudo mkdir -p /opt/vllm
+        sudo mkdir -p /opt/ollama
         sudo chown -R ubuntu:ubuntu /opt
         sudo chmod -R 0775 /opt
 
-        cd /opt/vllm
+        cd /opt/ollama
         cat <<'EOF' > docker-compose.yaml
 {dockerCompose}
         EOF
         echo "TOKEN={authToken}" >> .env
         echo "HOST=$(curl -4 ifconfig.me)" >> .env
         echo "CREDENTIALS='$(htpasswd -nBb user {authToken})'" >> .env
-        echo "HUGGING_FACE_HUB_TOKEN='{huggingFaceHubToken}'" >> .env
-        echo "S3_MODEL_PATH='{s3ModelPath}'" >> .env
-        echo "S3_ACCESS_KEY_ID='{s3AccessKeyId}'" >> .env
-        echo "S3_SECRET_ACCESS_KEY='{s3SecretAccessKey}'" >> .env
-        echo "S3_ENDPOINT_URL='{s3EndpointUrl}'" >> .env
-        
+
+        # Configure Docker to use Nvidia driver
+        nvidia-ctk runtime configure --runtime=docker
+        systemctl restart docker
+
         # up docker compose services
         docker compose up -d --build
-
+        docker exec ollama-ollama-service-1 ollama run {modelName}
         touch /tmp/runcmd_finished
 
   - path: /etc/ssh/sshd_config.d/90-custom-settings.conf
@@ -135,7 +130,7 @@ def indentString(input_string, indent_level=4):
       AllowGroups ubuntu
 
 runcmd:
-  - su - ubuntu -c '/opt/vllm/init.sh > /opt/vllm/init.log 2>&1'
+  - su - ubuntu -c '/opt/ollama/init.sh > /opt/ollama/init.log 2>&1'
 
 {users}
 """
@@ -226,27 +221,6 @@ def findStatusInInstance(response):
         sys.exit(1)
     logger.info(f"Instance domain: {ip}.nip.io")
 
-    # url = f"https://{ip}.nip.io/v1/models"
-    # max_attempts = 120
-    # wait_time = 5
-    # attempt = 1
-    # while attempt <= max_attempts:
-    #     try:
-    #         response = requests.get(url)
-    #         if response.status_code == 401:
-    #             logger.info(f"URL {url} is ready.")
-    #             break
-    #     except requests.ConnectionError:
-    #         pass
-    #     logger.info(f"Attempt {attempt}/{max_attempts}: URL not ready (HTTP status code: {response.status_code if 'response' in locals() else 'Connection error'}). Waiting {wait_time} seconds...")
-    #     time.sleep(wait_time)
-    #     attempt += 1
-    # else:
-    #     logger.info("URL is not ready after maximum attempts.")
-    #     sys.exit(1)
-
-
-
 elif action == "delete":
     instanceId = findInstance()
     if not instanceId:

pyproject.toml

@@ -1,5 +1,5 @@
 [tool.poetry]
-name = "vllm-managed-instance"
+name = "ollama-managed-instance"
 version = "0.1.0"
 description = ""
 authors = ["Matéo Mévollon <m.mevollon@proton.me>"]