From 08f2c39e0a145474e22bd5d76ab72c58ec7b9059 Mon Sep 17 00:00:00 2001 From: Kegan Maher Date: Wed, 9 Apr 2025 08:56:17 -0700 Subject: [PATCH 1/9] chore(compose): remove local elasticstack transitioning to streamlit for viz --- .devcontainer/devcontainer.json | 2 +- .env.sample | 27 ---- compose.yml | 188 ----------------------- elasticstack/esconfig/instances.yml | 13 -- elasticstack/esconfig/setup.sh | 51 ------ elasticstack/filebeat01/Dockerfile | 9 -- elasticstack/filebeat01/filebeat.yml | 24 --- elasticstack/kibana/kibana.yml | 2 - elasticstack/logstash01/Dockerfile | 9 -- elasticstack/logstash01/logstash.conf | 20 --- elasticstack/metricbeat01/Dockerfile | 9 -- elasticstack/metricbeat01/metricbeat.yml | 56 ------- elasticstack/reset.sh | 8 - elasticstack/start.sh | 4 - 14 files changed, 1 insertion(+), 421 deletions(-) delete mode 100644 elasticstack/esconfig/instances.yml delete mode 100755 elasticstack/esconfig/setup.sh delete mode 100644 elasticstack/filebeat01/Dockerfile delete mode 100644 elasticstack/filebeat01/filebeat.yml delete mode 100644 elasticstack/kibana/kibana.yml delete mode 100644 elasticstack/logstash01/Dockerfile delete mode 100644 elasticstack/logstash01/logstash.conf delete mode 100644 elasticstack/metricbeat01/Dockerfile delete mode 100644 elasticstack/metricbeat01/metricbeat.yml delete mode 100755 elasticstack/reset.sh delete mode 100755 elasticstack/start.sh diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index ac6c6c0..3de76c4 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -2,7 +2,7 @@ "name": "caltrans/pems", "dockerComposeFile": ["../compose.yml"], "service": "dev", - "forwardPorts": ["docs:8000", "kibana:5601"], + "forwardPorts": ["docs:8000"], "workspaceFolder": "/caltrans/app", "postStartCommand": ["/bin/bash", "bin/reset_db.sh"], "postAttachCommand": ["/bin/bash", ".devcontainer/postAttach.sh"], diff --git a/.env.sample b/.env.sample index 6bdc79f..58583fe 100644 --- a/.env.sample +++ b/.env.sample @@ -7,30 +7,3 @@ DJANGO_SUPERUSER_PASSWORD=superuser12345! DJANGO_DB_RESET=true DJANGO_STORAGE_DIR=. DJANGO_DB_FILE=django.db - -# uncomment to start the elasticstack services with compose -# COMPOSE_PROFILES=elasticstack - -# Version of Elastic products -ELASTIC_STACK_VERSION=8.16.1 - -# Set to 'basic' or 'trial' to automatically start the 30-day trial -ELASTIC_LICENSE=basic - -# SAMPLE Predefined Key only to be used in POC environments -ELASTIC_ENCRYPTION_KEY=c34d38b3a14956121ff2170e5030b471551370178f43e5626eec58b04a30fae2 - -# Set the cluster name -ELASTIC_CLUSTER=eslocal - -# Increase or decrease based on the available host memory (in bytes) -ELASTIC_MEM_LIMIT=2147483648 - -# Password for the 'elastic' user (at least 6 characters) -ELASTIC_PASSWORD=elastic - -# Password for the 'kibana_system' user (at least 6 characters) -KIBANA_PASSWORD=kibana - -# Increase or decrease based on the available host memory (in bytes) -KIBANA_MEM_LIMIT=1073741824 diff --git a/compose.yml b/compose.yml index a634e4b..670fab2 100644 --- a/compose.yml +++ b/compose.yml @@ -29,191 +29,3 @@ services: - "8000" volumes: - ./:/caltrans/app - - esconfig: - profiles: ["elasticstack"] - image: docker.elastic.co/elasticsearch/elasticsearch:${ELASTIC_STACK_VERSION} - user: "0" - command: ["bash", "/.local/config/setup.sh"] - env_file: - - .env - volumes: - - certs:/usr/share/elasticsearch/config/certs - - ./elasticstack/esconfig:/.local/config - healthcheck: - test: ["CMD-SHELL", "[ -f config/certs/es01/es01.crt ]"] - interval: 1s - timeout: 5s - retries: 120 - - es01: - profiles: ["elasticstack"] - depends_on: - esconfig: - condition: service_healthy - image: docker.elastic.co/elasticsearch/elasticsearch:${ELASTIC_STACK_VERSION} - labels: - co.elastic.logs/module: elasticsearch - volumes: - - certs:/usr/share/elasticsearch/config/certs - - esdata01:/usr/share/elasticsearch/data - ports: - - "9200" - environment: - - node.name=es01 - - cluster.name=${ELASTIC_CLUSTER} - - discovery.type=single-node - - ELASTIC_PASSWORD=${ELASTIC_PASSWORD} - - bootstrap.memory_lock=true - - xpack.security.enabled=true - - xpack.security.http.ssl.enabled=true - - xpack.security.http.ssl.key=certs/es01/es01.key - - xpack.security.http.ssl.certificate=certs/es01/es01.crt - - xpack.security.http.ssl.certificate_authorities=certs/ca/ca.crt - - xpack.security.transport.ssl.enabled=true - - xpack.security.transport.ssl.key=certs/es01/es01.key - - xpack.security.transport.ssl.certificate=certs/es01/es01.crt - - xpack.security.transport.ssl.certificate_authorities=certs/ca/ca.crt - - xpack.security.transport.ssl.verification_mode=certificate - - xpack.license.self_generated.type=${ELASTIC_LICENSE} - mem_limit: ${ELASTIC_MEM_LIMIT} - ulimits: - memlock: - soft: -1 - hard: -1 - healthcheck: - test: - [ - "CMD-SHELL", - "curl -s --cacert config/certs/ca/ca.crt https://localhost:9200 | grep -q 'missing authentication credentials'", - ] - interval: 10s - timeout: 10s - retries: 120 - - kibana: - profiles: ["elasticstack"] - depends_on: - es01: - condition: service_healthy - image: docker.elastic.co/kibana/kibana:${ELASTIC_STACK_VERSION} - labels: - co.elastic.logs/module: kibana - volumes: - - certs:/usr/share/kibana/config/certs - - kibanadata:/usr/share/kibana/data - - ./elasticstack/kibana:/usr/share/kibana/config - ports: - - "5601" - environment: - - SERVERNAME=kibana - - ELASTICSEARCH_HOSTS=https://es01:9200 - - ELASTICSEARCH_USERNAME=kibana_system - - ELASTICSEARCH_PASSWORD=${KIBANA_PASSWORD} - - ELASTICSEARCH_SSL_CERTIFICATEAUTHORITIES=config/certs/ca/ca.crt - - XPACK_SECURITY_ENCRYPTIONKEY=${ELASTIC_ENCRYPTION_KEY} - - XPACK_ENCRYPTEDSAVEDOBJECTS_ENCRYPTIONKEY=${ELASTIC_ENCRYPTION_KEY} - - XPACK_REPORTING_ENCRYPTIONKEY=${ELASTIC_ENCRYPTION_KEY} - mem_limit: ${KIBANA_MEM_LIMIT} - healthcheck: - test: - [ - "CMD-SHELL", - "curl -s -I http://localhost:5601 | grep -q 'HTTP/1.1 302 Found'", - ] - interval: 10s - timeout: 10s - retries: 120 - - metricbeat01: - profiles: ["elasticstack"] - depends_on: - es01: - condition: service_healthy - kibana: - condition: service_healthy - image: caltrans/pems:metricbeat01 - build: - context: . - dockerfile: ./elasticstack/metricbeat01/Dockerfile - args: - - ELASTIC_STACK_VERSION=${ELASTIC_STACK_VERSION} - volumes: - - certs:/usr/share/metricbeat/certs - - metricbeatdata01:/usr/share/metricbeat/data - - "/var/run/docker.sock:/var/run/docker.sock:ro" - - "/sys/fs/cgroup:/hostfs/sys/fs/cgroup:ro" - - "/proc:/hostfs/proc:ro" - - "/:/hostfs:ro" - environment: - - ELASTIC_USER=elastic - - ELASTIC_PASSWORD=${ELASTIC_PASSWORD} - - ELASTIC_HOSTS=https://es01:9200 - - KIBANA_HOSTS=http://kibana:5601 - - LOGSTASH_HOSTS=http://logstash01:9600 - command: - - --strict.perms=false - - filebeat01: - profiles: ["elasticstack"] - depends_on: - es01: - condition: service_healthy - image: caltrans/pems:filebeat01 - build: - context: . - dockerfile: ./elasticstack/filebeat01/Dockerfile - args: - - ELASTIC_STACK_VERSION=${ELASTIC_STACK_VERSION} - volumes: - - certs:/usr/share/filebeat/certs - - filebeatdata01:/usr/share/filebeat/data - - "/var/lib/docker/containers:/var/lib/docker/containers:ro" - - "/var/run/docker.sock:/var/run/docker.sock:ro" - environment: - - ELASTIC_USER=elastic - - ELASTIC_PASSWORD=${ELASTIC_PASSWORD} - - ELASTIC_HOSTS=https://es01:9200 - - KIBANA_HOSTS=http://kibana:5601 - - LOGSTASH_HOSTS=http://logstash01:9600 - command: - - --strict.perms=false - - logstash01: - profiles: ["elasticstack"] - depends_on: - es01: - condition: service_healthy - kibana: - condition: service_healthy - image: caltrans/pems:logstash01 - build: - context: . - dockerfile: ./elasticstack/logstash01/Dockerfile - args: - - ELASTIC_STACK_VERSION=${ELASTIC_STACK_VERSION} - labels: - co.elastic.logs/module: logstash - volumes: - - certs:/usr/share/logstash/certs - - logstashdata01:/usr/share/logstash/data - - ./elasticstack/logstash01:/tmp/logstash01 - environment: - - xpack.monitoring.enabled=false - - ELASTIC_USER=elastic - - ELASTIC_PASSWORD=${ELASTIC_PASSWORD} - - ELASTIC_HOSTS=https://es01:9200 - -volumes: - certs: - driver: local - esdata01: - driver: local - kibanadata: - driver: local - metricbeatdata01: - driver: local - filebeatdata01: - driver: local - logstashdata01: - driver: local diff --git a/elasticstack/esconfig/instances.yml b/elasticstack/esconfig/instances.yml deleted file mode 100644 index 36df494..0000000 --- a/elasticstack/esconfig/instances.yml +++ /dev/null @@ -1,13 +0,0 @@ -instances: - - name: es01 - dns: - - es01 - - localhost - ip: - - 127.0.0.1 - - name: kibana - dns: - - kibana - - localhost - ip: - - 127.0.0.1 diff --git a/elasticstack/esconfig/setup.sh b/elasticstack/esconfig/setup.sh deleted file mode 100755 index ac82b9d..0000000 --- a/elasticstack/esconfig/setup.sh +++ /dev/null @@ -1,51 +0,0 @@ -#!/usr/bin/env bash -set -eu - -if [ x${ELASTIC_PASSWORD} == x ]; then - echo "Set the ELASTIC_PASSWORD environment variable in the .env file"; - exit 1; -elif [ x${KIBANA_PASSWORD} == x ]; then - echo "Set the KIBANA_PASSWORD environment variable in the .env file"; - exit 1; -fi; - -if [ ! -f config/certs/ca.zip ]; then - echo "Creating CA"; - bin/elasticsearch-certutil ca --silent --pem -out config/certs/ca.zip; - unzip config/certs/ca.zip -d config/certs; -fi; - -if [ ! -f config/certs/certs.zip ]; then - echo "Creating certs"; - cp /.local/config/instances.yml config/certs/instances.yml; - bin/elasticsearch-certutil cert --silent --pem \ - -out config/certs/certs.zip \ - --in config/certs/instances.yml \ - --ca-cert config/certs/ca/ca.crt \ - --ca-key config/certs/ca/ca.key; - unzip config/certs/certs.zip -d config/certs; -fi; - -echo "Setting file permissions" -chown -R root:root config/certs; - -find . -type d -exec chmod 755 \{\} \;; -find . -type f -exec chmod 644 \{\} \;; - -TIMEOUT=10 - -until - echo "Waiting for Elasticsearch availability (sleeping for ${TIMEOUT}s)"; - curl -s --cacert config/certs/ca/ca.crt https://es01:9200 | grep -q "missing authentication credentials"; -do sleep $TIMEOUT; done; - -until - echo "Setting kibana_system password (sleeping for ${TIMEOUT}s)"; - curl -s -X POST \ - --cacert config/certs/ca/ca.crt \ - -u "elastic:${ELASTIC_PASSWORD}" \ - -H "Content-Type: application/json" https://es01:9200/_security/user/kibana_system/_password \ - -d "{\"password\":\"${KIBANA_PASSWORD}\"}" | grep -q "^{}"; -do sleep $TIMEOUT; done; - -echo "All done!"; diff --git a/elasticstack/filebeat01/Dockerfile b/elasticstack/filebeat01/Dockerfile deleted file mode 100644 index 0156eac..0000000 --- a/elasticstack/filebeat01/Dockerfile +++ /dev/null @@ -1,9 +0,0 @@ -ARG ELASTIC_STACK_VERSION=8.16.1 - -FROM docker.elastic.co/beats/filebeat:${ELASTIC_STACK_VERSION} - -USER root - -COPY ./elasticstack/filebeat01/filebeat.yml filebeat.yml - -RUN chmod go-w filebeat.yml diff --git a/elasticstack/filebeat01/filebeat.yml b/elasticstack/filebeat01/filebeat.yml deleted file mode 100644 index ad5e254..0000000 --- a/elasticstack/filebeat01/filebeat.yml +++ /dev/null @@ -1,24 +0,0 @@ -filebeat.autodiscover: - providers: - - type: docker - hints.enabled: true - hints.default_config: - type: container - paths: - - /var/lib/docker/containers/${data.container.id}/*.log - -processors: - - add_docker_metadata: ~ - -setup.kibana: - host: ${KIBANA_HOSTS} - username: ${ELASTIC_USER} - password: ${ELASTIC_PASSWORD} - -output.elasticsearch: - hosts: ${ELASTIC_HOSTS} - username: ${ELASTIC_USER} - password: ${ELASTIC_PASSWORD} - ssl: - enabled: true - certificate_authorities: certs/ca/ca.crt diff --git a/elasticstack/kibana/kibana.yml b/elasticstack/kibana/kibana.yml deleted file mode 100644 index f6773f9..0000000 --- a/elasticstack/kibana/kibana.yml +++ /dev/null @@ -1,2 +0,0 @@ -server.host: "0.0.0.0" -telemetry.optIn: "false" diff --git a/elasticstack/logstash01/Dockerfile b/elasticstack/logstash01/Dockerfile deleted file mode 100644 index a5f2bcc..0000000 --- a/elasticstack/logstash01/Dockerfile +++ /dev/null @@ -1,9 +0,0 @@ -ARG ELASTIC_STACK_VERSION=8.16.1 - -FROM docker.elastic.co/logstash/logstash:${ELASTIC_STACK_VERSION} - -USER root - -COPY ./elasticstack/logstash01/logstash.conf pipeline/logstash.conf - -RUN chmod go-w pipeline/logstash.conf diff --git a/elasticstack/logstash01/logstash.conf b/elasticstack/logstash01/logstash.conf deleted file mode 100644 index a43da8c..0000000 --- a/elasticstack/logstash01/logstash.conf +++ /dev/null @@ -1,20 +0,0 @@ -input { - file { - # https://www.elastic.co/guide/en/logstash/current/plugins-inputs-file.html - mode => "read" - path => "/tmp/logstash01/*.log" - } -} - -filter { -} - -output { - elasticsearch { - index => "logstash-%{+YYYY.MM.dd}" - hosts=> "${ELASTIC_HOSTS}" - user=> "${ELASTIC_USER}" - password=> "${ELASTIC_PASSWORD}" - ssl_certificate_authorities=> "certs/ca/ca.crt" - } -} diff --git a/elasticstack/metricbeat01/Dockerfile b/elasticstack/metricbeat01/Dockerfile deleted file mode 100644 index d656dee..0000000 --- a/elasticstack/metricbeat01/Dockerfile +++ /dev/null @@ -1,9 +0,0 @@ -ARG ELASTIC_STACK_VERSION=8.16.1 - -FROM docker.elastic.co/beats/metricbeat:${ELASTIC_STACK_VERSION} - -USER root - -COPY ./elasticstack/metricbeat01/metricbeat.yml metricbeat.yml - -RUN chmod go-w metricbeat.yml diff --git a/elasticstack/metricbeat01/metricbeat.yml b/elasticstack/metricbeat01/metricbeat.yml deleted file mode 100644 index 3f00f07..0000000 --- a/elasticstack/metricbeat01/metricbeat.yml +++ /dev/null @@ -1,56 +0,0 @@ -metricbeat.config.modules: - path: ${path.config}/modules.d/*.yml - reload.enabled: false - -metricbeat.modules: - - module: elasticsearch - xpack.enabled: true - period: 10s - hosts: ${ELASTIC_HOSTS} - ssl.certificate_authorities: "certs/ca/ca.crt" - ssl.certificate: "certs/es01/es01.crt" - ssl.key: "certs/es01/es01.key" - username: ${ELASTIC_USER} - password: ${ELASTIC_PASSWORD} - ssl.enabled: true - - - module: logstash - xpack.enabled: true - period: 10s - hosts: ${LOGSTASH_HOSTS} - - - module: kibana - metricsets: - - stats - period: 10s - hosts: ${KIBANA_HOSTS} - username: ${ELASTIC_USER} - password: ${ELASTIC_PASSWORD} - xpack.enabled: true - - - module: docker - metricsets: - - "container" - - "cpu" - - "diskio" - - "healthcheck" - - "info" - - "image" - - "memory" - - "network" - hosts: ["unix:///var/run/docker.sock"] - period: 10s - enabled: true - -processors: - - add_host_metadata: ~ - - add_docker_metadata: ~ - -output.elasticsearch: - hosts: ${ELASTIC_HOSTS} - username: ${ELASTIC_USER} - password: ${ELASTIC_PASSWORD} - ssl: - certificate: "certs/es01/es01.crt" - certificate_authorities: "certs/ca/ca.crt" - key: "certs/es01/es01.key" diff --git a/elasticstack/reset.sh b/elasticstack/reset.sh deleted file mode 100755 index 7eefce4..0000000 --- a/elasticstack/reset.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env bash -set -u - -docker compose down - -docker volume ls -q | grep pems_ | xargs docker volume rm - -docker compose build metricbeat01 filebeat01 logstash01 diff --git a/elasticstack/start.sh b/elasticstack/start.sh deleted file mode 100755 index 2bb7cf6..0000000 --- a/elasticstack/start.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/usr/bin/env bash -set -u - -docker compose up -d metricbeat01 filebeat01 logstash01 From bcc9a9238e41ed93cfb1dbed28a777820a57d88f Mon Sep 17 00:00:00 2001 From: Kegan Maher Date: Fri, 11 Apr 2025 01:14:15 +0000 Subject: [PATCH 2/9] feat(streamlit): compose service and basic app --- .env.sample | 3 +++ .streamlit/config.toml | 5 +++++ bin/build.sh | 8 ++++++++ compose.yml | 11 +++++++++++ streamlit_app/Dockerfile | 11 +++++++++++ streamlit_app/__init__.py | 0 streamlit_app/main.py | 15 +++++++++++++++ streamlit_app/requirements.txt | 1 + 8 files changed, 54 insertions(+) create mode 100644 .streamlit/config.toml create mode 100755 bin/build.sh create mode 100644 streamlit_app/Dockerfile create mode 100644 streamlit_app/__init__.py create mode 100644 streamlit_app/main.py create mode 100644 streamlit_app/requirements.txt diff --git a/.env.sample b/.env.sample index 58583fe..c6bf00b 100644 --- a/.env.sample +++ b/.env.sample @@ -7,3 +7,6 @@ DJANGO_SUPERUSER_PASSWORD=superuser12345! DJANGO_DB_RESET=true DJANGO_STORAGE_DIR=. DJANGO_DB_FILE=django.db + +# Streamlit +STREAMLIT_LOCAL_PORT=8501 diff --git a/.streamlit/config.toml b/.streamlit/config.toml new file mode 100644 index 0000000..a403bf7 --- /dev/null +++ b/.streamlit/config.toml @@ -0,0 +1,5 @@ +[browser] +gatherUsageStats = false + +[theme] +base = "light" diff --git a/bin/build.sh b/bin/build.sh new file mode 100755 index 0000000..572aef7 --- /dev/null +++ b/bin/build.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +set -eux + +docker compose build app + +docker compose build dev + +docker compose build streamlit diff --git a/compose.yml b/compose.yml index 670fab2..ea9309c 100644 --- a/compose.yml +++ b/compose.yml @@ -29,3 +29,14 @@ services: - "8000" volumes: - ./:/caltrans/app + + streamlit: + build: + context: . + dockerfile: streamlit_app/Dockerfile + image: caltrans/pems:streamlit + env_file: .env + ports: + - "${STREAMLIT_LOCAL_PORT:-8501}:8501" + volumes: + - ./:/caltrans/app diff --git a/streamlit_app/Dockerfile b/streamlit_app/Dockerfile new file mode 100644 index 0000000..8e00b8d --- /dev/null +++ b/streamlit_app/Dockerfile @@ -0,0 +1,11 @@ +FROM caltrans/pems:app + +ENV PYTHONPATH="$PYTHONPATH:/$USER/app" + +EXPOSE 8501 + +COPY streamlit_app streamlit_app + +RUN pip install -r streamlit_app/requirements.txt + +ENTRYPOINT ["streamlit", "run", "streamlit_app/main.py", "--server.port=8501", "--server.address=0.0.0.0"] diff --git a/streamlit_app/__init__.py b/streamlit_app/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/streamlit_app/main.py b/streamlit_app/main.py new file mode 100644 index 0000000..ad686ca --- /dev/null +++ b/streamlit_app/main.py @@ -0,0 +1,15 @@ +import logging + +import streamlit as st + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + + +def main(): + logger.info("Starting streamlit") + st.write("https://pems.dot.ca.gov") + + +if __name__ == "__main__": # pragma: no cover + raise SystemExit(main()) diff --git a/streamlit_app/requirements.txt b/streamlit_app/requirements.txt new file mode 100644 index 0000000..66fb0c2 --- /dev/null +++ b/streamlit_app/requirements.txt @@ -0,0 +1 @@ +streamlit==1.44.1 From 909319b0845862fd92f7e606c98d741cfda8b961 Mon Sep 17 00:00:00 2001 From: Kegan Maher Date: Fri, 11 Apr 2025 02:52:32 +0000 Subject: [PATCH 3/9] feat(devcontainer): install streamlit, launch config --- .devcontainer/Dockerfile | 4 ++++ .vscode/launch.json | 11 +++++++++++ 2 files changed, 15 insertions(+) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 518e334..67b85fe 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1,8 +1,12 @@ FROM caltrans/pems:app COPY . . + # install devcontainer requirements RUN pip install -e .[dev,test] # install docs requirements RUN pip install --no-cache-dir -r docs/requirements.txt + +# install streamlit requirements +RUN pip install --no-cache-dir -r streamlit_app/requirements.txt diff --git a/.vscode/launch.json b/.vscode/launch.json index fdd58d2..00d74ed 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -27,6 +27,17 @@ "DJANGO_DEBUG": "false", "DJANGO_STATICFILES_STORAGE": "django.contrib.staticfiles.storage.StaticFilesStorage" } + }, + { + "name": "Streamlit", + "type": "debugpy", + "request": "launch", + "module": "streamlit", + "args": ["run", "streamlit_app/main.py"], + "env": { + "PYTHONBUFFERED": "1", + "PYTHONWARNINGS": "default" + } } ] } From 93a5c03b995924eed12d9ce91ab8b3de934fad1f Mon Sep 17 00:00:00 2001 From: Kegan Maher Date: Fri, 11 Apr 2025 02:53:58 +0000 Subject: [PATCH 4/9] chore(tests): pems subdirectory, cover streamlit_app but ignore streamlit_app/apps directory, with user-submitted code --- pyproject.toml | 3 ++- tests/pytest/{core => pems}/__init__.py | 0 tests/pytest/pems/conftest.py | 0 tests/pytest/pems/core/__init__.py | 0 tests/pytest/{ => pems}/core/test_middleware_healthcheck.py | 0 5 files changed, 2 insertions(+), 1 deletion(-) rename tests/pytest/{core => pems}/__init__.py (100%) create mode 100644 tests/pytest/pems/conftest.py create mode 100644 tests/pytest/pems/core/__init__.py rename tests/pytest/{ => pems}/core/test_middleware_healthcheck.py (100%) diff --git a/pyproject.toml b/pyproject.toml index 35e47fc..ce18a5b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,7 +31,8 @@ include = '\.pyi?$' [tool.coverage.run] branch = true relative_files = true -source = ["pems"] +source = ["pems", "streamlit_app"] +omit = ["streamlit_app/apps/*"] [tool.pytest.ini_options] DJANGO_SETTINGS_MODULE = "pems.settings" diff --git a/tests/pytest/core/__init__.py b/tests/pytest/pems/__init__.py similarity index 100% rename from tests/pytest/core/__init__.py rename to tests/pytest/pems/__init__.py diff --git a/tests/pytest/pems/conftest.py b/tests/pytest/pems/conftest.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/pytest/pems/core/__init__.py b/tests/pytest/pems/core/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/pytest/core/test_middleware_healthcheck.py b/tests/pytest/pems/core/test_middleware_healthcheck.py similarity index 100% rename from tests/pytest/core/test_middleware_healthcheck.py rename to tests/pytest/pems/core/test_middleware_healthcheck.py From 433179fae1951a2ef34384e22fdabef78e576b99 Mon Sep 17 00:00:00 2001 From: Kegan Maher Date: Fri, 11 Apr 2025 02:56:55 +0000 Subject: [PATCH 5/9] feat(streamlit): helpers for app discovery and tests with a folder of test apps --- .github/workflows/tests-pytest.yml | 4 +- streamlit_app/utils.py | 58 +++++++++++++ tests/pytest/streamlit_app/__init__.py | 0 tests/pytest/streamlit_app/conftest.py | 0 .../streamlit_app/test_apps/__init__.py | 0 .../pytest/streamlit_app/test_apps/app_one.py | 3 + .../pytest/streamlit_app/test_apps/app_two.py | 3 + .../streamlit_app/test_apps/not_an_app.py | 1 + tests/pytest/streamlit_app/test_utils.py | 81 +++++++++++++++++++ 9 files changed, 149 insertions(+), 1 deletion(-) create mode 100644 streamlit_app/utils.py create mode 100644 tests/pytest/streamlit_app/__init__.py create mode 100644 tests/pytest/streamlit_app/conftest.py create mode 100644 tests/pytest/streamlit_app/test_apps/__init__.py create mode 100644 tests/pytest/streamlit_app/test_apps/app_one.py create mode 100644 tests/pytest/streamlit_app/test_apps/app_two.py create mode 100644 tests/pytest/streamlit_app/test_apps/not_an_app.py create mode 100644 tests/pytest/streamlit_app/test_utils.py diff --git a/.github/workflows/tests-pytest.yml b/.github/workflows/tests-pytest.yml index 5c4a066..1987b62 100644 --- a/.github/workflows/tests-pytest.yml +++ b/.github/workflows/tests-pytest.yml @@ -29,7 +29,9 @@ jobs: cache-dependency-path: "**/pyproject.toml" - name: Install Python dependencies - run: pip install -e .[test] + run: | + pip install -e .[test] + pip install -r streamlit_app/requirements.txt - name: Run setup run: ./bin/init.sh diff --git a/streamlit_app/utils.py b/streamlit_app/utils.py new file mode 100644 index 0000000..355b972 --- /dev/null +++ b/streamlit_app/utils.py @@ -0,0 +1,58 @@ +import logging +from pathlib import Path + +import streamlit as st +from streamlit.navigation.page import StreamlitPage + +logger = logging.getLogger(__name__) + + +APP_DIR = Path(__file__).parent / "apps/" + + +def _convert_to_pages(apps: list[Path]) -> list[StreamlitPage]: + return list(map(_make_app_page, apps)) + + +def _default_app_page() -> StreamlitPage: + return st.Page(APP_DIR / "__init__.py", default=True, title="PeMS Streamlit apps") + + +def _discover_apps() -> list[Path]: + logger.info("Beginning streamlit app discovery") + + apps = [] + + for dirpath, _, filenames in APP_DIR.walk(top_down=True): + for app_file in filter(_is_app_file, filenames): + logger.info(f"Discovered streamlit app: {dirpath}/{app_file}") + apps.append(dirpath / app_file) + + logger.info(f"Discovered {len(apps)} streamlit apps") + return apps + + +def _is_app_file(filename: str) -> bool: + return filename.startswith("app_") and filename.endswith(".py") + + +def _make_app_page(app_path: Path) -> StreamlitPage: + relative_dir = app_path.parent.relative_to(APP_DIR) + app_name = app_path.name.replace("app_", "").replace(".py", "") + + parts = (*relative_dir.parts, app_name) + url = "--".join(parts) + + title = url.replace("--", " | ").capitalize() + + logger.info(f"Registering streamlit app: {url}") + + return st.Page(app_path, url_path=url, title=title) + + +def discover_apps() -> list[StreamlitPage]: + default_app_page = _default_app_page() + apps = _discover_apps() + pages = _convert_to_pages(apps) + pages.insert(0, default_app_page) + return pages diff --git a/tests/pytest/streamlit_app/__init__.py b/tests/pytest/streamlit_app/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/pytest/streamlit_app/conftest.py b/tests/pytest/streamlit_app/conftest.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/pytest/streamlit_app/test_apps/__init__.py b/tests/pytest/streamlit_app/test_apps/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/pytest/streamlit_app/test_apps/app_one.py b/tests/pytest/streamlit_app/test_apps/app_one.py new file mode 100644 index 0000000..10f3c81 --- /dev/null +++ b/tests/pytest/streamlit_app/test_apps/app_one.py @@ -0,0 +1,3 @@ +import streamlit as st + +st.write("app one") diff --git a/tests/pytest/streamlit_app/test_apps/app_two.py b/tests/pytest/streamlit_app/test_apps/app_two.py new file mode 100644 index 0000000..875ad33 --- /dev/null +++ b/tests/pytest/streamlit_app/test_apps/app_two.py @@ -0,0 +1,3 @@ +import streamlit as st + +st.write("app two") diff --git a/tests/pytest/streamlit_app/test_apps/not_an_app.py b/tests/pytest/streamlit_app/test_apps/not_an_app.py new file mode 100644 index 0000000..8c1ab01 --- /dev/null +++ b/tests/pytest/streamlit_app/test_apps/not_an_app.py @@ -0,0 +1 @@ +var = "not an app since it doesn't start with app_" diff --git a/tests/pytest/streamlit_app/test_utils.py b/tests/pytest/streamlit_app/test_utils.py new file mode 100644 index 0000000..0630789 --- /dev/null +++ b/tests/pytest/streamlit_app/test_utils.py @@ -0,0 +1,81 @@ +from pathlib import Path + +import pytest +from streamlit.navigation.page import StreamlitPage + +from streamlit_app import utils + + +@pytest.fixture +def test_apps_path(): + return Path(__file__).parent / "test_apps" + + +@pytest.fixture(autouse=True) +def use_test_apps(mocker, test_apps_path): + mocker.patch.object(utils, "APP_DIR", test_apps_path) + + +@pytest.fixture +def app_paths(test_apps_path): + return [ + test_apps_path / "app_one.py", + test_apps_path / "app_two.py", + ] + + +def test_convert_to_pages(mocker, app_paths): + page_factory = mocker.patch("streamlit_app.utils._make_app_page", return_value="page") + + pages = utils._convert_to_pages(app_paths) + + assert len(pages) == len(app_paths) + for path in app_paths: + assert mocker.call(path) in page_factory.mock_calls + + +def test_default_app_page(): + page = utils._default_app_page() + + assert page._default + assert page.url_path == "" + + +def test__discover_apps(app_paths): + discovered = utils._discover_apps() + + assert discovered == app_paths + + +@pytest.mark.parametrize( + "filename,result", + [ + ("app_yes.py", True), + ("app_definitely-ok.py", True), + ("not_an_app.py", False), + ("app_nope.csv", False), + ], +) +def test_is_app_file(filename, result): + assert utils._is_app_file(filename) == result + + +def test_make_app_page(app_paths): + for path in app_paths: + page = utils._make_app_page(path) + assert not page._default + + +def test_discover_apps(mocker): + mock_discover = mocker.patch("streamlit_app.utils._discover_apps", return_value=[]) + mock_convert = mocker.patch("streamlit_app.utils._convert_to_pages", return_value=[]) + + result = utils.discover_apps() + + mock_discover.assert_called_once() + mock_convert.assert_called_once() + + assert len(result) == 1 + page = result[0] + assert isinstance(page, StreamlitPage) + assert page._default From 6396ee6575ddbcac4fbbc065843f9b406d48108d Mon Sep 17 00:00:00 2001 From: Kegan Maher Date: Fri, 11 Apr 2025 03:01:09 +0000 Subject: [PATCH 6/9] feat(streamlit): add the bike share sample --- streamlit_app/apps/__init__.py | 3 + streamlit_app/apps/sample/__init__.py | 0 streamlit_app/apps/sample/app_bikeshare.py | 28 + streamlit_app/apps/sample/pygwalker.json | 686 +++++++++++++++++++++ streamlit_app/requirements.txt | 2 + 5 files changed, 719 insertions(+) create mode 100644 streamlit_app/apps/__init__.py create mode 100644 streamlit_app/apps/sample/__init__.py create mode 100644 streamlit_app/apps/sample/app_bikeshare.py create mode 100644 streamlit_app/apps/sample/pygwalker.json diff --git a/streamlit_app/apps/__init__.py b/streamlit_app/apps/__init__.py new file mode 100644 index 0000000..932f695 --- /dev/null +++ b/streamlit_app/apps/__init__.py @@ -0,0 +1,3 @@ +import streamlit as st + +st.write("https://pems.dot.ca.gov") diff --git a/streamlit_app/apps/sample/__init__.py b/streamlit_app/apps/sample/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/streamlit_app/apps/sample/app_bikeshare.py b/streamlit_app/apps/sample/app_bikeshare.py new file mode 100644 index 0000000..d9df11d --- /dev/null +++ b/streamlit_app/apps/sample/app_bikeshare.py @@ -0,0 +1,28 @@ +from pathlib import Path + +import pandas as pd +import streamlit as st +from pygwalker.api.streamlit import StreamlitRenderer + +APP_DIR = Path(__file__).parent + +st.set_page_config(layout="wide") + + +@st.cache_data() +def fetch_data(): + data = pd.read_csv("https://kanaries-app.s3.ap-northeast-1.amazonaws.com/public-datasets/bike_sharing_dc.csv") + return data + + +@st.cache_data() +def fetch_config(): + data = Path(APP_DIR / "pygwalker.json").read_text() + return data + + +df = fetch_data() +vis_spec = fetch_config() + +pyg_app = StreamlitRenderer(df, spec=vis_spec) +pyg_app.explorer() diff --git a/streamlit_app/apps/sample/pygwalker.json b/streamlit_app/apps/sample/pygwalker.json new file mode 100644 index 0000000..3f922a9 --- /dev/null +++ b/streamlit_app/apps/sample/pygwalker.json @@ -0,0 +1,686 @@ +{ + "config": [ + { + "config": { + "defaultAggregated": true, + "geoms": ["bar"], + "coordSystem": "generic", + "limit": -1, + "timezoneDisplayOffset": 0 + }, + "encodings": { + "dimensions": [ + { + "dragId": "gw_d0SN", + "fid": "date", + "name": "date", + "basename": "date", + "semanticType": "temporal", + "analyticType": "dimension", + "offset": 0 + }, + { + "dragId": "gw_fCVU", + "fid": "month", + "name": "month", + "basename": "month", + "semanticType": "ordinal", + "analyticType": "dimension", + "offset": 0 + }, + { + "dragId": "gw_xAWV", + "fid": "season", + "name": "season", + "basename": "season", + "semanticType": "nominal", + "analyticType": "dimension", + "offset": 0 + }, + { + "dragId": "gw_ho7q", + "fid": "year", + "name": "year", + "basename": "year", + "semanticType": "ordinal", + "analyticType": "dimension", + "offset": 0 + }, + { + "dragId": "gw_1bIC", + "fid": "holiday", + "name": "holiday", + "basename": "holiday", + "semanticType": "nominal", + "analyticType": "dimension", + "offset": 0 + }, + { + "dragId": "gw_K8Ek", + "fid": "work yes or not", + "name": "work yes or not", + "basename": "work yes or not", + "semanticType": "ordinal", + "analyticType": "dimension", + "offset": 0 + }, + { + "dragId": "gw_tORa", + "fid": "am or pm", + "name": "am or pm", + "basename": "am or pm", + "semanticType": "nominal", + "analyticType": "dimension", + "offset": 0 + }, + { + "dragId": "gw_RMSm", + "fid": "Day of the week", + "name": "Day of the week", + "basename": "Day of the week", + "semanticType": "quantitative", + "analyticType": "dimension", + "offset": 0 + }, + { + "dragId": "gw_mea_key_fid", + "fid": "gw_mea_key_fid", + "name": "Measure names", + "analyticType": "dimension", + "semanticType": "nominal", + "offset": 0 + }, + { + "fid": "gw_hYau", + "dragId": "gw_hYau", + "name": "Weekday [date]", + "semanticType": "ordinal", + "analyticType": "dimension", + "aggName": "sum", + "computed": true, + "expression": { + "op": "dateTimeFeature", + "as": "gw_hYau", + "params": [ + { "type": "field", "value": "date" }, + { "type": "value", "value": "weekday" }, + { "type": "format", "value": "%Y-%m-%d" } + ] + }, + "offset": 0 + }, + { + "fid": "gw_lSdd", + "dragId": "gw_lSdd", + "name": "Quarter [date]", + "semanticType": "ordinal", + "analyticType": "dimension", + "aggName": "sum", + "computed": true, + "expression": { + "op": "dateTimeFeature", + "as": "gw_lSdd", + "params": [ + { "type": "field", "value": "date" }, + { "type": "value", "value": "quarter" }, + { "type": "format", "value": "%Y-%m-%d" } + ] + }, + "offset": 0 + } + ], + "measures": [ + { + "dragId": "gw_oE-g", + "fid": "hour", + "name": "hour", + "basename": "hour", + "analyticType": "measure", + "semanticType": "quantitative", + "aggName": "sum", + "offset": 0 + }, + { + "dragId": "gw_LZNz", + "fid": "temperature", + "name": "temperature", + "basename": "temperature", + "analyticType": "measure", + "semanticType": "quantitative", + "aggName": "sum", + "offset": 0 + }, + { + "dragId": "gw_JbdF", + "fid": "feeling_temp", + "name": "feeling_temp", + "basename": "feeling_temp", + "analyticType": "measure", + "semanticType": "quantitative", + "aggName": "sum", + "offset": 0 + }, + { + "dragId": "gw_7hAr", + "fid": "humidity", + "name": "humidity", + "basename": "humidity", + "analyticType": "measure", + "semanticType": "quantitative", + "aggName": "sum", + "offset": 0 + }, + { + "dragId": "gw_a8mK", + "fid": "winspeed", + "name": "winspeed", + "basename": "winspeed", + "analyticType": "measure", + "semanticType": "quantitative", + "aggName": "sum", + "offset": 0 + }, + { + "dragId": "gw_Yb-_", + "fid": "casual", + "name": "casual", + "basename": "casual", + "analyticType": "measure", + "semanticType": "quantitative", + "aggName": "sum", + "offset": 0 + }, + { + "dragId": "gw_fdQ9", + "fid": "registered", + "name": "registered", + "basename": "registered", + "analyticType": "measure", + "semanticType": "quantitative", + "aggName": "sum", + "offset": 0 + }, + { + "dragId": "gw_Bdj1", + "fid": "count", + "name": "count", + "basename": "count", + "analyticType": "measure", + "semanticType": "quantitative", + "aggName": "sum", + "offset": 0 + }, + { + "dragId": "gw_count_fid", + "fid": "gw_count_fid", + "name": "Row count", + "analyticType": "measure", + "semanticType": "quantitative", + "aggName": "sum", + "computed": true, + "expression": { "op": "one", "params": [], "as": "gw_count_fid" }, + "offset": 0 + }, + { + "dragId": "gw_mea_val_fid", + "fid": "gw_mea_val_fid", + "name": "Measure values", + "analyticType": "measure", + "semanticType": "quantitative", + "aggName": "sum", + "offset": 0 + } + ], + "rows": [ + { + "dragId": "gw_XI5j", + "fid": "registered", + "name": "registered", + "basename": "registered", + "analyticType": "measure", + "semanticType": "quantitative", + "aggName": "sum", + "offset": 0 + } + ], + "columns": [ + { + "dragId": "gw_fCVU", + "fid": "month", + "name": "month", + "basename": "month", + "semanticType": "ordinal", + "analyticType": "dimension", + "offset": 0 + } + ], + "color": [ + { + "fid": "gw_lSdd", + "dragId": "gw_BZBe", + "name": "Quarter [date]", + "semanticType": "ordinal", + "analyticType": "dimension", + "aggName": "sum", + "computed": true, + "expression": { + "op": "dateTimeFeature", + "as": "gw_lSdd", + "params": [ + { "type": "field", "value": "date" }, + { "type": "value", "value": "quarter" }, + { "type": "format", "value": "%Y-%m-%d" } + ] + }, + "offset": 0 + } + ], + "opacity": [], + "size": [], + "shape": [], + "radius": [], + "theta": [], + "longitude": [], + "latitude": [], + "geoId": [], + "details": [], + "filters": [ + { + "dragId": "gw_U38p", + "fid": "month", + "name": "month", + "basename": "month", + "semanticType": "ordinal", + "analyticType": "dimension", + "rule": { "type": "range", "value": [1, 12] }, + "offset": 0 + } + ], + "text": [] + }, + "layout": { + "showActions": false, + "showTableSummary": false, + "stack": "stack", + "interactiveScale": false, + "zeroScale": true, + "size": { "mode": "fixed", "width": 350, "height": 345 }, + "format": {}, + "geoKey": "name", + "resolve": { + "x": false, + "y": false, + "color": false, + "opacity": false, + "shape": false, + "size": false + }, + "colorPalette": "paired", + "scale": { "opacity": {}, "size": {} }, + "scaleIncludeUnmatchedChoropleth": false, + "useSvg": false + }, + "visId": "gw_YvK3", + "name": "Chart 1" + }, + { + "config": { + "defaultAggregated": true, + "geoms": ["auto"], + "coordSystem": "generic", + "limit": -1, + "folds": ["registered", "casual"], + "timezoneDisplayOffset": 0 + }, + "encodings": { + "dimensions": [ + { + "dragId": "gw_iwKS", + "fid": "date", + "name": "date", + "basename": "date", + "semanticType": "temporal", + "analyticType": "dimension", + "offset": 0 + }, + { + "dragId": "gw_qttg", + "fid": "month", + "name": "month", + "basename": "month", + "semanticType": "ordinal", + "analyticType": "dimension", + "offset": 0 + }, + { + "dragId": "gw_FJZI", + "fid": "season", + "name": "season", + "basename": "season", + "semanticType": "nominal", + "analyticType": "dimension", + "offset": 0 + }, + { + "dragId": "gw_noqw", + "fid": "year", + "name": "year", + "basename": "year", + "semanticType": "ordinal", + "analyticType": "dimension", + "offset": 0 + }, + { + "dragId": "gw_S1Op", + "fid": "holiday", + "name": "holiday", + "basename": "holiday", + "semanticType": "nominal", + "analyticType": "dimension", + "offset": 0 + }, + { + "dragId": "gw_FECQ", + "fid": "work yes or not", + "name": "work yes or not", + "basename": "work yes or not", + "semanticType": "ordinal", + "analyticType": "dimension", + "offset": 0 + }, + { + "dragId": "gw_F4AV", + "fid": "am or pm", + "name": "am or pm", + "basename": "am or pm", + "semanticType": "nominal", + "analyticType": "dimension", + "offset": 0 + }, + { + "dragId": "gw_Srun", + "fid": "Day of the week", + "name": "Day of the week", + "basename": "Day of the week", + "semanticType": "quantitative", + "analyticType": "dimension", + "offset": 0 + }, + { + "dragId": "gw_mea_key_fid", + "fid": "gw_mea_key_fid", + "name": "Measure names", + "analyticType": "dimension", + "semanticType": "nominal", + "offset": 0 + } + ], + "measures": [ + { + "dragId": "gw_KeT-", + "fid": "hour", + "name": "hour", + "basename": "hour", + "analyticType": "measure", + "semanticType": "quantitative", + "aggName": "sum", + "offset": 0 + }, + { + "dragId": "gw_rDyp", + "fid": "temperature", + "name": "temperature", + "basename": "temperature", + "analyticType": "measure", + "semanticType": "quantitative", + "aggName": "sum", + "offset": 0 + }, + { + "dragId": "gw_G71D", + "fid": "feeling_temp", + "name": "feeling_temp", + "basename": "feeling_temp", + "analyticType": "measure", + "semanticType": "quantitative", + "aggName": "sum", + "offset": 0 + }, + { + "dragId": "gw_Gjrm", + "fid": "humidity", + "name": "humidity", + "basename": "humidity", + "analyticType": "measure", + "semanticType": "quantitative", + "aggName": "sum", + "offset": 0 + }, + { + "dragId": "gw_2SZj", + "fid": "winspeed", + "name": "winspeed", + "basename": "winspeed", + "analyticType": "measure", + "semanticType": "quantitative", + "aggName": "sum", + "offset": 0 + }, + { + "dragId": "gw_Pjzq", + "fid": "casual", + "name": "casual", + "basename": "casual", + "analyticType": "measure", + "semanticType": "quantitative", + "aggName": "sum", + "offset": 0 + }, + { + "dragId": "gw_dBk7", + "fid": "registered", + "name": "registered", + "basename": "registered", + "analyticType": "measure", + "semanticType": "quantitative", + "aggName": "sum", + "offset": 0 + }, + { + "dragId": "gw_Bju8", + "fid": "count", + "name": "count", + "basename": "count", + "analyticType": "measure", + "semanticType": "quantitative", + "aggName": "sum", + "offset": 0 + }, + { + "dragId": "gw_count_fid", + "fid": "gw_count_fid", + "name": "Row count", + "analyticType": "measure", + "semanticType": "quantitative", + "aggName": "sum", + "computed": true, + "expression": { "op": "one", "params": [], "as": "gw_count_fid" }, + "offset": 0 + }, + { + "dragId": "gw_mea_val_fid", + "fid": "gw_mea_val_fid", + "name": "Measure values", + "analyticType": "measure", + "semanticType": "quantitative", + "aggName": "sum", + "offset": 0 + } + ], + "rows": [ + { + "dragId": "gw_PW8u", + "fid": "gw_mea_val_fid", + "name": "Measure values", + "analyticType": "measure", + "semanticType": "quantitative", + "aggName": "sum", + "offset": 0 + } + ], + "columns": [ + { + "dragId": "gw_8tGb", + "fid": "date", + "name": "date", + "basename": "date", + "semanticType": "temporal", + "analyticType": "dimension", + "offset": 0 + } + ], + "color": [ + { + "dragId": "gw_NK9S", + "fid": "gw_mea_key_fid", + "name": "Measure names", + "analyticType": "dimension", + "semanticType": "nominal", + "offset": 0 + } + ], + "opacity": [], + "size": [], + "shape": [], + "radius": [], + "theta": [], + "longitude": [], + "latitude": [], + "geoId": [], + "details": [], + "filters": [ + { + "dragId": "gw_RBu-", + "fid": "date", + "name": "date", + "basename": "date", + "semanticType": "temporal", + "analyticType": "dimension", + "rule": { + "type": "temporal range", + "value": [1293811200000, 1356883200000], + "format": "%Y-%m-%d", + "offset": -480 + }, + "offset": 0 + } + ], + "text": [] + }, + "layout": { + "showActions": false, + "showTableSummary": false, + "stack": "stack", + "interactiveScale": false, + "zeroScale": true, + "size": { "mode": "fixed", "width": 752, "height": 360 }, + "format": {}, + "geoKey": "name", + "resolve": { + "x": false, + "y": false, + "color": false, + "opacity": false, + "shape": false, + "size": false + }, + "scaleIncludeUnmatchedChoropleth": false + }, + "visId": "gw_QwuS", + "name": "Chart 2" + } + ], + "chart_map": {}, + "workflow_list": [ + { + "workflow": [ + { + "type": "filter", + "filters": [ + { "fid": "month", "rule": { "type": "range", "value": [1, 12] } } + ] + }, + { + "type": "transform", + "transform": [ + { + "key": "gw_lSdd", + "expression": { + "op": "dateTimeFeature", + "as": "gw_lSdd", + "params": [ + { "type": "field", "value": "date" }, + { "type": "value", "value": "quarter" }, + { "type": "format", "value": "%Y-%m-%d" }, + { "type": "displayOffset", "value": 0 } + ] + } + } + ] + }, + { + "type": "view", + "query": [ + { + "op": "aggregate", + "groupBy": ["month", "gw_lSdd"], + "measures": [ + { + "field": "registered", + "agg": "sum", + "asFieldKey": "registered_sum" + } + ] + } + ] + } + ] + }, + { + "workflow": [ + { + "type": "filter", + "filters": [ + { + "fid": "date", + "rule": { + "type": "temporal range", + "value": [1293811200000, 1356883200000], + "offset": -480, + "format": "%Y-%m-%d" + } + } + ] + }, + { + "type": "view", + "query": [ + { + "op": "aggregate", + "groupBy": ["date"], + "measures": [ + { + "field": "registered", + "agg": "sum", + "asFieldKey": "registered_sum" + }, + { "field": "casual", "agg": "sum", "asFieldKey": "casual_sum" } + ] + } + ] + } + ] + } + ], + "version": "0.4.9.4a0" +} diff --git a/streamlit_app/requirements.txt b/streamlit_app/requirements.txt index 66fb0c2..a39870a 100644 --- a/streamlit_app/requirements.txt +++ b/streamlit_app/requirements.txt @@ -1 +1,3 @@ +pandas==2.2.3 +pygwalker==0.4.9.15 streamlit==1.44.1 From 9e5cf8a72634718b706ede285a8bc3179103abd2 Mon Sep 17 00:00:00 2001 From: Kegan Maher Date: Fri, 11 Apr 2025 03:25:44 +0000 Subject: [PATCH 7/9] feat(streamlit): discover and register apps walk through the streamlit_ap/apps/ directory and any subdirectory(ies) looking for app_*.py files, which are assumed to contain a streamlit app each construct a multipage streamlit site from all the discovered apps - page titles like Directory | subdirectory | app_name - page URLs like http://localhost:8501/directory--subdirectory--name update the launch config for with and without the sidebar --- .env.sample | 2 ++ .vscode/launch.json | 17 +++++++++++++-- streamlit_app/main.py | 20 +++++++++++++++-- tests/pytest/streamlit_app/test_main.py | 29 +++++++++++++++++++++++++ 4 files changed, 64 insertions(+), 4 deletions(-) create mode 100644 tests/pytest/streamlit_app/test_main.py diff --git a/.env.sample b/.env.sample index c6bf00b..b6483d2 100644 --- a/.env.sample +++ b/.env.sample @@ -10,3 +10,5 @@ DJANGO_DB_FILE=django.db # Streamlit STREAMLIT_LOCAL_PORT=8501 +# options: hidden, sidebar +STREAMLIT_NAV=hidden diff --git a/.vscode/launch.json b/.vscode/launch.json index 00d74ed..c784b79 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -29,14 +29,27 @@ } }, { - "name": "Streamlit", + "name": "Streamlit, yes sidebar", "type": "debugpy", "request": "launch", "module": "streamlit", "args": ["run", "streamlit_app/main.py"], "env": { "PYTHONBUFFERED": "1", - "PYTHONWARNINGS": "default" + "PYTHONWARNINGS": "default", + "STREAMLIT_NAV": "sidebar" + } + }, + { + "name": "Streamlit, no sidebar", + "type": "debugpy", + "request": "launch", + "module": "streamlit", + "args": ["run", "streamlit_app/main.py"], + "env": { + "PYTHONBUFFERED": "1", + "PYTHONWARNINGS": "default", + "STREAMLIT_NAV": "hidden" } } ] diff --git a/streamlit_app/main.py b/streamlit_app/main.py index ad686ca..dbe403e 100644 --- a/streamlit_app/main.py +++ b/streamlit_app/main.py @@ -2,13 +2,29 @@ import streamlit as st +from streamlit_app.utils import discover_apps + logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) def main(): - logger.info("Starting streamlit") - st.write("https://pems.dot.ca.gov") + logger.info("Streamlit initializing") + + # find apps in the ./streamlit_app/apps directory and subdirectories + # apps are python modules with a name starting with "app_" + apps = discover_apps() + # position=hidden hides the sidebar navigator + # https://docs.streamlit.io/develop/api-reference/navigation/st.navigation#stnavigation + nav_position = os.environ.get("STREAMLIT_NAV", "hidden") + if nav_position not in ("hidden", "sidebar"): + nav_position = "hidden" + site = st.navigation(apps, position=nav_position) + + logger.info("Initialization complete") + logger.info("Starting multipage streamlit app") + + site.run() if __name__ == "__main__": # pragma: no cover diff --git a/tests/pytest/streamlit_app/test_main.py b/tests/pytest/streamlit_app/test_main.py new file mode 100644 index 0000000..3924081 --- /dev/null +++ b/tests/pytest/streamlit_app/test_main.py @@ -0,0 +1,29 @@ +import pytest +from streamlit_app import main + + +@pytest.mark.parametrize( + "nav_option,expected_nav", + [ + (None, "hidden"), + ("hidden", "hidden"), + ("sidebar", "sidebar"), + ("invalid", "hidden"), + ], +) +def test_main(mocker, monkeypatch, nav_option, expected_nav): + nav_key = "STREAMLIT_NAV" + apps = ["app", "app"] + if nav_option: + monkeypatch.setenv(nav_key, nav_option) + else: + monkeypatch.delenv(nav_key, False) + + mock_discover = mocker.patch("streamlit_app.main.discover_apps", return_value=apps) + mock_navigate = mocker.patch("streamlit_app.main.st.navigation") + + main.main() + + mock_discover.assert_called_once() + mock_navigate.assert_called_once_with(apps, position=expected_nav) + mock_navigate.return_value.run.assert_called_once() From d7976a21fff0bc39f8d4bd6eb7d2382dffa6343f Mon Sep 17 00:00:00 2001 From: Kegan Maher Date: Fri, 11 Apr 2025 04:32:28 +0000 Subject: [PATCH 8/9] feat(streamlit): setup django during init so e.g. Django models can be queried from Streamlit apps --- streamlit_app/main.py | 13 +++++++++++++ tests/pytest/streamlit_app/test_main.py | 2 ++ 2 files changed, 15 insertions(+) diff --git a/streamlit_app/main.py b/streamlit_app/main.py index dbe403e..b6e6af6 100644 --- a/streamlit_app/main.py +++ b/streamlit_app/main.py @@ -1,5 +1,7 @@ import logging +import os +import django import streamlit as st from streamlit_app.utils import discover_apps @@ -8,9 +10,20 @@ logger = logging.getLogger(__name__) +def django_setup(): + """ + Configures Django in case it is not running in the same environment. Should be called before using e.g. PeMS models. + """ + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "pems.settings") + + django.setup() + + def main(): logger.info("Streamlit initializing") + django_setup() + # find apps in the ./streamlit_app/apps directory and subdirectories # apps are python modules with a name starting with "app_" apps = discover_apps() diff --git a/tests/pytest/streamlit_app/test_main.py b/tests/pytest/streamlit_app/test_main.py index 3924081..87646f0 100644 --- a/tests/pytest/streamlit_app/test_main.py +++ b/tests/pytest/streamlit_app/test_main.py @@ -19,11 +19,13 @@ def test_main(mocker, monkeypatch, nav_option, expected_nav): else: monkeypatch.delenv(nav_key, False) + mock_django = mocker.patch("streamlit_app.main.django.setup") mock_discover = mocker.patch("streamlit_app.main.discover_apps", return_value=apps) mock_navigate = mocker.patch("streamlit_app.main.st.navigation") main.main() + mock_django.assert_called_once() mock_discover.assert_called_once() mock_navigate.assert_called_once_with(apps, position=expected_nav) mock_navigate.return_value.run.assert_called_once() From 4e4d60603b0db1602a664c4062cdcc82715e111d Mon Sep 17 00:00:00 2001 From: Kegan Maher Date: Fri, 11 Apr 2025 15:53:13 +0000 Subject: [PATCH 9/9] feat(pems): integrate sample Streamlit app show the bikeshare sample in an iframe in a Django template served by a Django route/view --- pems/settings.py | 1 + pems/streamlit_sample/__init__.py | 0 pems/streamlit_sample/apps.py | 7 +++++++ .../streamlit_sample/templates/streamlit_sample/index.html | 6 ++++++ pems/streamlit_sample/urls.py | 6 ++++++ pems/urls.py | 3 ++- 6 files changed, 22 insertions(+), 1 deletion(-) create mode 100644 pems/streamlit_sample/__init__.py create mode 100644 pems/streamlit_sample/apps.py create mode 100644 pems/streamlit_sample/templates/streamlit_sample/index.html create mode 100644 pems/streamlit_sample/urls.py diff --git a/pems/settings.py b/pems/settings.py index d834a1f..4c3b06c 100644 --- a/pems/settings.py +++ b/pems/settings.py @@ -31,6 +31,7 @@ def _filter_empty(ls): "django.contrib.sessions", "django.contrib.messages", "django.contrib.staticfiles", + "pems.streamlit_sample", ] MIDDLEWARE = [ diff --git a/pems/streamlit_sample/__init__.py b/pems/streamlit_sample/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/pems/streamlit_sample/apps.py b/pems/streamlit_sample/apps.py new file mode 100644 index 0000000..ea424c0 --- /dev/null +++ b/pems/streamlit_sample/apps.py @@ -0,0 +1,7 @@ +from django.apps import AppConfig + + +class StreamlitSampleAppConfig(AppConfig): + name = "pems.streamlit_sample" + label = "streamlit_sample" + verbose_name = "Streamlit sample" diff --git a/pems/streamlit_sample/templates/streamlit_sample/index.html b/pems/streamlit_sample/templates/streamlit_sample/index.html new file mode 100644 index 0000000..fe2028c --- /dev/null +++ b/pems/streamlit_sample/templates/streamlit_sample/index.html @@ -0,0 +1,6 @@ +

Streamlit inside Django

+ + diff --git a/pems/streamlit_sample/urls.py b/pems/streamlit_sample/urls.py new file mode 100644 index 0000000..18c238c --- /dev/null +++ b/pems/streamlit_sample/urls.py @@ -0,0 +1,6 @@ +from django.urls import path +from django.views.generic import TemplateView + +urlpatterns = [ + path("", TemplateView.as_view(template_name="streamlit_sample/index.html")), +] diff --git a/pems/urls.py b/pems/urls.py index 6d14891..cc0c2b1 100644 --- a/pems/urls.py +++ b/pems/urls.py @@ -16,8 +16,9 @@ """ from django.contrib import admin -from django.urls import path +from django.urls import include, path urlpatterns = [ path("admin/", admin.site.urls), + path("streamlit/", include("pems.streamlit_sample.urls")), ]