Feat: initial appcontainer setup (#38)

Author: thekaveman

.devcontainer/Dockerfile

@@ -1,42 +1,8 @@
-ARG PYTHON_VERSION=3.12
+FROM caltrans/pems:app
 
-FROM python:${PYTHON_VERSION}
-
-ENV PYTHONDONTWRITEBYTECODE=1 \
-    PYTHONUNBUFFERED=1 \
-    USER=caltrans
-
-# create non-root $USER and home directory
-RUN useradd --create-home --shell /bin/bash $USER && \
-    chown -R $USER /home/$USER
-
-# switch to $USER
-USER $USER
-
-# enter src directory
-WORKDIR /home/$USER/src
-
-# update PATH for local pip installs
-ENV PATH="$PATH:/home/$USER/.local/bin"
-
-# upgrade pip
-RUN python -m pip install --upgrade pip
-
-# copy assets
 COPY . .
-
 # install devcontainer requirements
 RUN pip install -e .[dev,test]
 
 # install docs requirements
 RUN pip install --no-cache-dir -r docs/requirements.txt
-
-# install pre-commit environments in throwaway Git repository
-# https://stackoverflow.com/a/68758943
-RUN git init . && \
-    pre-commit install-hooks && \
-    rm -rf .git
-
-CMD sleep infinity
-
-ENTRYPOINT []

.devcontainer/devcontainer.json

@@ -3,7 +3,7 @@
   "dockerComposeFile": ["../compose.yml"],
   "service": "dev",
   "forwardPorts": ["docs:8000", "kibana:5601"],
-  "workspaceFolder": "/home/caltrans/src",
+  "workspaceFolder": "/caltrans/app",
   "postStartCommand": ["/bin/bash", "bin/reset_db.sh"],
   "postAttachCommand": ["/bin/bash", ".devcontainer/postAttach.sh"],
   "customizations": {

.devcontainer/postAttach.sh

@@ -3,4 +3,4 @@ set -eux
 
 # initialize pre-commit
 git config --global --add safe.directory /home/$USER/src
-pre-commit install --overwrite
+pre-commit install --install-hooks --overwrite

.dockerignore

@@ -1,3 +1,2 @@
-.git/
 *.egg-info
 *.db

.env.sample

@@ -8,6 +8,9 @@ DJANGO_DB_RESET=true
 DJANGO_STORAGE_DIR=.
 DJANGO_DB_FILE=django.db
 
+# Other Django config
+DJANGO_DEBUG=true
+
 # uncomment to start the elasticstack services with compose
 # COMPOSE_PROFILES=elasticstack
 

.vscode/settings.json

@@ -20,6 +20,9 @@
   "[python]": {
     "editor.defaultFormatter": "ms-python.black-formatter"
   },
+  "[toml]": {
+    "editor.defaultFormatter": "tamasfe.even-better-toml"
+  },
   "python.languageServer": "Pylance",
   "python.testing.pytestArgs": ["tests/pytest"],
   "python.testing.pytestEnabled": true,

appcontainer/Dockerfile

@@ -0,0 +1,90 @@
+ARG PYTHON_VERSION=3.12
+
+# multi-stage build
+#
+# stage 1: builds the pems package from source
+#          using the git metadata for version info
+FROM python:${PYTHON_VERSION} AS build_wheel
+WORKDIR /build
+
+# upgrade pip
+RUN python -m pip install --upgrade pip && \
+    pip install build
+
+# copy source files
+COPY . .
+RUN git config --global --add safe.directory /build
+
+# build package
+RUN python -m build
+
+# multi-stage build
+#
+# stage 2: installs the pems package in a fresh base container
+#          using the pre-built package, and copying only needed source
+FROM python:${PYTHON_VERSION} AS app_container
+
+ENV PYTHONDONTWRITEBYTECODE=1 \
+    PYTHONUNBUFFERED=1 \
+    USER=caltrans
+
+EXPOSE 8000
+
+    # create non-root $USER and home directory
+RUN useradd --create-home --shell /bin/bash $USER && \
+    # setup $USER permissions for nginx
+    mkdir -p /var/cache/nginx && \
+    chown -R $USER:$USER /var/cache/nginx && \
+    mkdir -p /var/lib/nginx && \
+    chown -R $USER:$USER /var/lib/nginx && \
+    mkdir -p /var/log/nginx && \
+    chown -R $USER:$USER /var/log/nginx && \
+    touch /var/log/nginx/error.log && \
+    chown $USER:$USER /var/log/nginx/error.log && \
+    touch /var/run/nginx.pid && \
+    chown -R $USER:$USER /var/run/nginx.pid && \
+    # setup directories and permissions for gunicorn, (eventual) app
+    mkdir -p /$USER/app && \
+    mkdir -p /$USER/run && \
+    chown -R $USER:$USER /$USER && \
+    # install server components
+    apt-get update && \
+    apt-get install -qq --no-install-recommends build-essential nginx gettext && \
+    python -m pip install --upgrade pip
+
+# enter source directory
+WORKDIR /$USER
+
+COPY LICENSE app/LICENSE
+
+# switch to non-root $USER
+USER $USER
+
+# update env for local pip installs
+# see https://docs.python.org/3/using/cmdline.html#envvar-PYTHONUSERBASE
+# since all `pip install` commands are in the context of $USER
+# $PYTHONUSERBASE is the location used by default
+ENV PATH="$PATH:/$USER/.local/bin" \
+    PYTHONUSERBASE="/$USER/.local"
+
+# copy gunicorn config file
+COPY appcontainer/gunicorn.conf.py run/gunicorn.conf.py
+ENV GUNICORN_CONF "/$USER/run/gunicorn.conf.py"
+
+# overwrite default nginx.conf
+COPY appcontainer/nginx.conf /etc/nginx/nginx.conf
+
+WORKDIR /$USER/app
+
+# copy runtime files
+COPY --from=build_wheel /build/dist /build/dist
+COPY manage.py manage.py
+COPY bin bin
+COPY pems pems
+
+# install source as a package
+RUN pip install $(find /build/dist -name pems*.whl)
+
+# configure container executable
+ENTRYPOINT ["/bin/bash"]
+CMD ["bin/start.sh"]

appcontainer/gunicorn.conf.py

@@ -0,0 +1,20 @@
+"""
+The Gunicorn configuration file
+More information: https://docs.gunicorn.org/en/stable/settings.html
+"""
+
+import multiprocessing
+
+# the unix socket defined in nginx.conf
+bind = "unix:/caltrans/run/gunicorn.sock"
+
+# Recommend (2 x $num_cores) + 1 as the number of workers to start off with
+workers = multiprocessing.cpu_count() * 2 + 1
+
+# send logs to stdout and stderr
+accesslog = "-"
+errorlog = "-"
+
+# Preloading can save some RAM resources as well as speed up server boot times,
+# at the cost of not being able to reload app code by restarting workers
+preload_app = True

appcontainer/nginx.conf

@@ -0,0 +1,80 @@
+worker_processes auto;
+error_log stderr warn;
+pid /var/run/nginx.pid;
+
+events {
+  worker_connections 1024;
+  accept_mutex on;
+}
+
+http {
+  include mime.types;
+  default_type application/octet-stream;
+  sendfile on;
+  gzip on;
+  keepalive_timeout 5;
+
+  log_format main '$remote_addr - $remote_user [$time_local] '
+                  '"$request" $status $body_bytes_sent '
+                  '"$http_referer" "$http_user_agent" "$gzip_ratio"';
+
+  access_log /dev/stdout main;
+
+  upstream app_server {
+    # fail_timeout=0 means we always retry an upstream even if it failed
+    # to return a good HTTP response
+    server unix:/caltrans/run/gunicorn.sock fail_timeout=0;
+  }
+
+  server {
+    listen 8000;
+
+    keepalive_timeout 65;
+
+    # 404 known scraping path targets
+    # case-insensitive regex matches the given path fragment anywhere in the request path
+    location ~* /(\.?git|api|app|assets|ats|bootstrap|bower|cgi|content|cpanel|credentials|debug|docker|doc|env|example|jenkins|robots|swagger|web|yq) {
+        access_log off;
+        log_not_found off;
+        return 404;
+    }
+
+    # 404 known scraping file targets
+    # case-insensitive regex matches the given file extension anywhere in the request path
+    location ~* /.*\.(ash|asp|axd|cgi|com|db|env|json|php|ping|sqlite|xml|ya?ml) {
+        access_log off;
+        log_not_found off;
+        return 404;
+    }
+
+    location /favicon.ico {
+      access_log off;
+      log_not_found off;
+      expires 1y;
+      add_header Cache-Control public;
+    }
+
+    # path for static files
+    location /static/ {
+      alias /caltrans/app/static/;
+      expires 1y;
+      add_header Cache-Control public;
+    }
+
+    location / {
+      # checks for static file, if not found proxy to app
+      try_files $uri @proxy_to_app;
+    }
+
+    # app path
+    location @proxy_to_app {
+      proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+      proxy_set_header X-Forwarded-Proto $scheme;
+      proxy_set_header Host $http_host;
+      # we don't want nginx trying to do something clever with
+      # redirects, we set the Host: header above already.
+      proxy_redirect off;
+      proxy_pass http://app_server;
+    }
+  }
+}

bin/start.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+set -eu
+
+# initialize Django
+
+bin/init.sh
+
+# start the web server
+
+nginx
+
+# start the application server
+
+python -m gunicorn -c $GUNICORN_CONF pems.wsgi

compose.yml

@@ -1,23 +1,34 @@
 name: pems
 
 services:
+  app:
+    build:
+      context: .
+      dockerfile: appcontainer/Dockerfile
+    image: caltrans/pems:app
+    env_file: .env
+    ports:
+      - "8000"
+
   dev:
     build:
       context: .
       dockerfile: .devcontainer/Dockerfile
-    image: caltrans/pems:main
+    image: caltrans/pems:dev
     env_file: .env
+    # https://code.visualstudio.com/docs/remote/create-dev-container#_use-docker-compose
+    entrypoint: sleep infinity
     volumes:
-      - ./:/home/caltrans/src
+      - ./:/caltrans/app
 
   docs:
-    image: caltrans/pems:main
+    image: caltrans/pems:dev
     entrypoint: mkdocs
     command: serve --dev-addr "0.0.0.0:8000"
     ports:
       - "8000"
     volumes:
-      - ./:/home/caltrans/src
+      - ./:/caltrans/app
 
   esconfig:
     profiles: ["elasticstack"]

manage.py

@@ -3,10 +3,17 @@
 import os
 import sys
 
+from pems import __version__ as VERSION
+
 
 def main():
     """Run administrative tasks."""
     os.environ.setdefault("DJANGO_SETTINGS_MODULE", "pems.settings")
+
+    if len(sys.argv) == 2 and sys.argv[1] == "--version" or sys.argv[1] == "-v":
+        print(f"pems, {VERSION}")
+        return
+
     try:
         from django.core.management import execute_from_command_line
     except ImportError as exc: