From d4fae69991f2b066f45ac9965f4af167212fbcc2 Mon Sep 17 00:00:00 2001 From: ctyano Date: Sat, 8 Mar 2025 17:00:44 +0900 Subject: [PATCH 1/6] Added external container images --- docs/DISTRIBUTIONS.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/DISTRIBUTIONS.md b/docs/DISTRIBUTIONS.md index 15e3e7b2..e688eabf 100644 --- a/docs/DISTRIBUTIONS.md +++ b/docs/DISTRIBUTIONS.md @@ -21,6 +21,12 @@ External Docker(OCI) image distributions: - [athenz-plugins](https://github.com/users/ctyano/packages/container/package/athenz-plugins) - [k8s-athenz-sia](https://github.com/users/ctyano/packages/container/package/k8s-athenz-sia) + - [open-policy-agent](https://hub.docker.com/r/openpolicyagent/opa) + - [kube-mgmt](https://hub.docker.com/r/openpolicyagent/kube-mgmt) + - [envoy](https://hub.docker.com/r/envoyproxy/envoy) + - [ghostunnel](https://hub.docker.com/r/ghostunnel/ghostunnel) + - [oauth2-proxy](https://quay.io/repository/oauth2-proxy/oauth2-proxy) + - [dex](https://github.com/dexidp/dex/pkgs/container/dex) Third party Docker(OCI) images: From aa8ac1460e538423164378ed5761cdf1fbcf579f Mon Sep 17 00:00:00 2001 From: ctyano Date: Mon, 17 Mar 2025 21:32:02 +0900 Subject: [PATCH 2/6] Updated markdown documentation --- docs/DISTRIBUTIONS.md | 6 ------ 1 file changed, 6 deletions(-) diff --git a/docs/DISTRIBUTIONS.md b/docs/DISTRIBUTIONS.md index e688eabf..15e3e7b2 100644 --- a/docs/DISTRIBUTIONS.md +++ b/docs/DISTRIBUTIONS.md @@ -21,12 +21,6 @@ External Docker(OCI) image distributions: - [athenz-plugins](https://github.com/users/ctyano/packages/container/package/athenz-plugins) - [k8s-athenz-sia](https://github.com/users/ctyano/packages/container/package/k8s-athenz-sia) - - [open-policy-agent](https://hub.docker.com/r/openpolicyagent/opa) - - [kube-mgmt](https://hub.docker.com/r/openpolicyagent/kube-mgmt) - - [envoy](https://hub.docker.com/r/envoyproxy/envoy) - - [ghostunnel](https://hub.docker.com/r/ghostunnel/ghostunnel) - - [oauth2-proxy](https://quay.io/repository/oauth2-proxy/oauth2-proxy) - - [dex](https://github.com/dexidp/dex/pkgs/container/dex) Third party Docker(OCI) images: From 91b1d6dda27a74eecee89de7522d47942f032353 Mon Sep 17 00:00:00 2001 From: ctyano Date: Wed, 19 Mar 2025 00:05:09 +0900 Subject: [PATCH 3/6] Added k8sclient --- kubernetes/athenz-k8sclient/.gitignore | 8 + kubernetes/athenz-k8sclient/Makefile | 176 +++++++ kubernetes/athenz-k8sclient/README.md | 53 ++ .../athenz-k8sclient/kustomize/.gitignore | 1 + .../kustomize/athenz-sia/athenz-sia.env | 175 +++++++ .../kustomize/deployment.yaml | 197 ++++++++ .../kustomize/envoy/ca_sds.yaml | 13 + .../kustomize/envoy/config.yaml | 473 ++++++++++++++++++ .../kustomize/envoy/identity_sds.yaml | 15 + .../kustomize/envoy/role_sds.yaml | 15 + .../kustomize/kustomization.yaml | 55 ++ .../athenz-k8sclient/kustomize/namespace.yaml | 5 + .../athenz-k8sclient/kustomize/service.yaml | 15 + .../kustomize/serviceaccount.yaml | 7 + 14 files changed, 1208 insertions(+) create mode 100644 kubernetes/athenz-k8sclient/.gitignore create mode 100644 kubernetes/athenz-k8sclient/Makefile create mode 100644 kubernetes/athenz-k8sclient/README.md create mode 100644 kubernetes/athenz-k8sclient/kustomize/.gitignore create mode 100644 kubernetes/athenz-k8sclient/kustomize/athenz-sia/athenz-sia.env create mode 100644 kubernetes/athenz-k8sclient/kustomize/deployment.yaml create mode 100644 kubernetes/athenz-k8sclient/kustomize/envoy/ca_sds.yaml create mode 100644 kubernetes/athenz-k8sclient/kustomize/envoy/config.yaml create mode 100644 kubernetes/athenz-k8sclient/kustomize/envoy/identity_sds.yaml create mode 100644 kubernetes/athenz-k8sclient/kustomize/envoy/role_sds.yaml create mode 100644 kubernetes/athenz-k8sclient/kustomize/kustomization.yaml create mode 100644 kubernetes/athenz-k8sclient/kustomize/namespace.yaml create mode 100644 kubernetes/athenz-k8sclient/kustomize/service.yaml create mode 100644 kubernetes/athenz-k8sclient/kustomize/serviceaccount.yaml diff --git a/kubernetes/athenz-k8sclient/.gitignore b/kubernetes/athenz-k8sclient/.gitignore new file mode 100644 index 00000000..1cf8cdb3 --- /dev/null +++ b/kubernetes/athenz-k8sclient/.gitignore @@ -0,0 +1,8 @@ +*.pem +*.jks +*.pkcs12 +*.srl +*.jar +.ntoken +athenz.conf +admin diff --git a/kubernetes/athenz-k8sclient/Makefile b/kubernetes/athenz-k8sclient/Makefile new file mode 100644 index 00000000..5ffde8c3 --- /dev/null +++ b/kubernetes/athenz-k8sclient/Makefile @@ -0,0 +1,176 @@ +ifeq ($(DOCKER_REGISTRY),) +DOCKER_REGISTRY=ghcr.io/ctyano/ +endif + +clean-athenz-k8sclient: + kubectl delete -k kustomize + +register-athenz-k8sclient: + kubectl -n athenz exec deployment/athenz-cli -it -- \ + curl \ + -sv \ + -d"{\"name\":\"$$(cat kustomize/athenz-sia/athenz-sia.env | grep -E ^PROVIDER_SERVICE | sed -e 's/PROVIDER_SERVICE=\(.*\)\.\(.*\)/\1/g')\",\"adminUsers\":[\"user.athenz_admin\"]}" \ + -H"Content-Type: application/json" \ + --key /var/run/athenz/athenz_admin.private.pem \ + --cert /var/run/athenz/athenz_admin.cert.pem \ + "https://athenz-zms-server.athenz:4443/zms/v1/domain" + kubectl -n athenz exec deployment/athenz-cli -it -- \ + zms-cli \ + -z https://athenz-zms-server.athenz:4443/zms/v1 \ + -key /var/run/athenz/athenz_admin.private.pem \ + -cert /var/run/athenz/athenz_admin.cert.pem \ + -d \ + sys.auth \ + set-domain-template \ + instance_provider \ + provider="$$(cat kustomize/athenz-sia/athenz-sia.env | grep -E ^PROVIDER_SERVICE | sed -e 's/PROVIDER_SERVICE=\(.*\)/\1/g')" \ + dnssuffix="$$(cat kustomize/athenz-sia/athenz-sia.env | grep -E ^DNS_SUFFIX | sed -e 's/DNS_SUFFIX=\(.*\)/\1/g')" + kubectl -n athenz exec deployment/athenz-cli -it -- \ + zms-cli \ + -z https://athenz-zms-server.athenz:4443/zms/v1 \ + -key /var/run/athenz/athenz_admin.private.pem \ + -cert /var/run/athenz/athenz_admin.cert.pem \ + -d \ + $$(cat kustomize/athenz-sia/athenz-sia.env | grep -E ^PROVIDER_SERVICE | sed -e 's/PROVIDER_SERVICE=\(.*\)\.\(.*\)/\1/g') \ + add-service \ + $$(cat kustomize/athenz-sia/athenz-sia.env | grep -E ^SERVICEACCOUNT | sed -e 's/SERVICEACCOUNT=\(.*\)/\1/g') \ + 0 \ + $$(cat kustomize/keys/k8sclient.public.pem | base64 | tr -d '\r\n' | tr '\+\=\/' '\.\-\_') ||: + kubectl -n athenz exec deployment/athenz-cli -it -- \ + zms-cli \ + -z https://athenz-zms-server.athenz:4443/zms/v1 \ + -key /var/run/athenz/athenz_admin.private.pem \ + -cert /var/run/athenz/athenz_admin.cert.pem \ + -d \ + $$(cat kustomize/athenz-sia/athenz-sia.env | grep -E ^PROVIDER_SERVICE | sed -e 's/PROVIDER_SERVICE=\(.*\)\.\(.*\)/\1/g') \ + set-domain-template \ + identity_provisioning \ + instanceprovider="sys.auth.zts" \ + service="$$(cat kustomize/athenz-sia/athenz-sia.env | grep -E ^SERVICEACCOUNT | sed -e 's/SERVICEACCOUNT=\(.*\)/\1/g')" + kubectl -n athenz exec deployment/athenz-cli -it -- \ + zms-cli \ + -z https://athenz-zms-server.athenz:4443/zms/v1 \ + -key /var/run/athenz/athenz_admin.private.pem \ + -cert /var/run/athenz/athenz_admin.cert.pem \ + -d \ + $$(cat kustomize/athenz-sia/athenz-sia.env | grep -E ^PROVIDER_SERVICE | sed -e 's/PROVIDER_SERVICE=\(.*\)\.\(.*\)/\1/g') \ + set-domain-template \ + identity_provisioning \ + instanceprovider="$$(cat kustomize/athenz-sia/athenz-sia.env | grep -E ^PROVIDER_SERVICE | sed -e 's/PROVIDER_SERVICE=\(.*\)/\1/g')" \ + service="$$(cat kustomize/athenz-sia/athenz-sia.env | grep -E ^SERVICEACCOUNT | sed -e 's/SERVICEACCOUNT=\(.*\)/\1/g')" + kubectl -n athenz exec deployment/athenz-cli -it -- \ + zms-cli \ + -z https://athenz-zms-server.athenz:4443/zms/v1 \ + -key /var/run/athenz/athenz_admin.private.pem \ + -cert /var/run/athenz/athenz_admin.cert.pem \ + -d $$(cat kustomize/athenz-sia/athenz-sia.env | grep -E ^PROVIDER_SERVICE | sed -e 's/PROVIDER_SERVICE=\(.*\)\.\(.*\)/\1/g') \ + add-member \ + envoyclients \ + $$(cat kustomize/athenz-sia/athenz-sia.env | grep -E ^PROVIDER_SERVICE | sed -e 's/PROVIDER_SERVICE=\(.*\)\.\(.*\)/\1/g').$$(cat kustomize/athenz-sia/athenz-sia.env | grep -E ^SERVICEACCOUNT | sed -e 's/SERVICEACCOUNT=\(.*\)/\1/g') ||: + kubectl -n athenz exec deployment/athenz-cli -it -- \ + zms-cli \ + -z https://athenz-zms-server.athenz:4443/zms/v1 \ + -key /var/run/athenz/athenz_admin.private.pem \ + -cert /var/run/athenz/athenz_admin.cert.pem \ + -d $$(cat kustomize/athenz-sia/athenz-sia.env | grep -E ^PROVIDER_SERVICE | sed -e 's/PROVIDER_SERVICE=\(.*\)\.\(.*\)/\1/g') \ + add-member \ + envoywebhookclients \ + $$(cat kustomize/athenz-sia/athenz-sia.env | grep -E ^PROVIDER_SERVICE | sed -e 's/PROVIDER_SERVICE=\(.*\)\.\(.*\)/\1/g').$$(cat kustomize/athenz-sia/athenz-sia.env | grep -E ^SERVICEACCOUNT | sed -e 's/SERVICEACCOUNT=\(.*\)/\1/g') ||: + kubectl -n athenz exec deployment/athenz-cli -it -- \ + zms-cli \ + -z https://athenz-zms-server.athenz:4443/zms/v1 \ + -key /var/run/athenz/athenz_admin.private.pem \ + -cert /var/run/athenz/athenz_admin.cert.pem \ + -d $$(cat kustomize/athenz-sia/athenz-sia.env | grep -E ^PROVIDER_SERVICE | sed -e 's/PROVIDER_SERVICE=\(.*\)\.\(.*\)/\1/g') \ + add-member \ + authorization-proxy-clients \ + $$(cat kustomize/athenz-sia/athenz-sia.env | grep -E ^PROVIDER_SERVICE | sed -e 's/PROVIDER_SERVICE=\(.*\)\.\(.*\)/\1/g').$$(cat kustomize/athenz-sia/athenz-sia.env | grep -E ^SERVICEACCOUNT | sed -e 's/SERVICEACCOUNT=\(.*\)/\1/g') ||: + kubectl -n athenz exec deployment/athenz-cli -it -- \ + zms-cli \ + -z https://athenz-zms-server.athenz:4443/zms/v1 \ + -key /var/run/athenz/athenz_admin.private.pem \ + -cert /var/run/athenz/athenz_admin.cert.pem \ + -d $$(cat kustomize/athenz-sia/athenz-sia.env | grep -E ^PROVIDER_SERVICE | sed -e 's/PROVIDER_SERVICE=\(.*\)\.\(.*\)/\1/g') \ + show-domain + +check-register-athenz-k8sclient: + SLEEP_SECONDS=5; \ +WAITING_THRESHOLD=60; \ +i=0; \ +while true; do \ + printf "\n***** Waiting for athenz($$(( $$i * $${SLEEP_SECONDS} ))s/$${WAITING_THRESHOLD}s) *****\n"; \ + kubectl -n athenz exec deployment/athenz-cli -it -- \ + curl \ + -sf \ + -H"Content-type: application/json" \ + -H"X-Auth-Request-Preferred-Username: user.athenz_admin" \ + "https://athenz-zts-server.athenz:4443/zts/v1/domain/$$(cat kustomize/athenz-sia/athenz-sia.env | grep -E ^PROVIDER_SERVICE | sed -e 's/PROVIDER_SERVICE=\(.*\)\.\(.*\)/\1/g')/service/$$(cat kustomize/athenz-sia/athenz-sia.env | grep -E ^SERVICEACCOUNT | sed -e 's/SERVICEACCOUNT=\(.*\)/\1/g')" \ + && break \ + || echo "Waiting for ZTS to sync with ZMS..."; \ + sleep $${SLEEP_SECONDS}; \ + i=$$(( i + 1 )); \ + if [ $$i -eq $$(( $${WAITING_THRESHOLD} / $${SLEEP_SECONDS} )) ]; then \ + printf "\n\n** Waiting ($$(( $$i * $${SLEEP_SECONDS} ))s) reached to threshold($${WAITING_THRESHOLD}s) **\n\n"; \ + kubectl -n athenz get all | grep -E "pod/.*0/1" | sed -e 's/^\(pod\/[^ ]*\) *0\/1.*/\1/g' | xargs -I%% kubectl -n athenz logs %% --all-containers=true ||:; \ + kubectl -n athenz get all | grep -E "pod/.*0/1" | sed -e 's/^\(pod\/[^ ]*\) *0\/1.*/\1/g' | xargs -I%% kubectl -n athenz describe %% ||:; \ + kubectl -n athenz get all; \ + exit 1; \ + fi; \ +done + kubectl -n athenz get all + @echo "" + @echo "**************************************" + @echo "**** Athenz Client is functioning ***" + @echo "**************************************" + @echo "" + +kustomize-edit-athenz-k8sclient-image: + if [ "$(DOCKER_REGISTRY)" != "ghcr.io/ctyano/" ]; then \ + ( \ + cd kustomize \ + && kustomize edit set image ghcr.io/ctyano/athenz-cli:latest=$(DOCKER_REGISTRY)athenz-cli:latest \ + && kustomize edit set image ghcr.io/ctyano/k8s-athenz-sia:latest=$(DOCKER_REGISTRY)k8s-athenz-sia:latest \ + ) \ + fi + +deploy-athenz-k8sclient: kustomize-edit-athenz-k8sclient-image + kubectl apply -k kustomize + +test-athenz-k8sclient: + SLEEP_SECONDS=5; \ +WAITING_THRESHOLD=60; \ +i=0; \ +while true; do \ + printf "\n***** Waiting for athenz($$(( $$i * $${SLEEP_SECONDS} ))s/$${WAITING_THRESHOLD}s) *****\n"; \ + ( \ + test $$(( $$(kubectl -n athenz get all | grep pod/k8sclient- | grep -E "0/1" | wc -l) )) -eq 0 \ + && \ + kubectl -n athenz exec deployment/k8sclient-deployment -it -c sia -- \ + ls \ + -alF \ + /var/run/athenz/tls.key \ + /var/run/athenz/tls.crt \ + /var/run/athenz/ca.crt \ + /var/run/athenz/athenz:role.envoyclients.cert.pem \ + /var/run/athenz/athenz:role.envoyclients.roletoken \ + /var/run/athenz/athenz:role.envoyclients.accesstoken \ + ) \ + && break \ + || echo "Waiting for Identity Provisioning..."; \ + sleep $${SLEEP_SECONDS}; \ + i=$$(( i + 1 )); \ + if [ $$i -eq $$(( $${WAITING_THRESHOLD} / $${SLEEP_SECONDS} )) ]; then \ + printf "\n\n** Waiting ($$(( $$i * $${SLEEP_SECONDS} ))s) reached to threshold($${WAITING_THRESHOLD}s) **\n\n"; \ + kubectl -n athenz get all | grep -E "pod/k8sclient-" | sed -e 's/^\(pod\/[^ ]*\) *[0-9]\/[0-9].*/\1/g' | xargs -I%% kubectl -n athenz logs %% --all-containers=true ||:; \ + kubectl -n athenz get all | grep -E "pod/k8sclient-" | sed -e 's/^\(pod\/[^ ]*\) *[0-9]\/[0-9].*/\1/g' | xargs -I%% kubectl -n athenz describe %% ||:; \ + kubectl -n athenz get all | grep -E "pod/identityprovider-deployment-" | sed -e 's/^\(pod\/[^ ]*\) *[0-9]\/[0-9].*/\1/g' | xargs -I%% kubectl -n athenz logs %% -c opa | grep "Identity Provider OPA Rego" ||:; \ + kubectl -n athenz get all; \ + exit 1; \ + fi; \ +done + kubectl -n athenz get all + @echo "" + @echo "**************************************" + @echo "*** Client provisioning successful **" + @echo "**************************************" + @echo "" diff --git a/kubernetes/athenz-k8sclient/README.md b/kubernetes/athenz-k8sclient/README.md new file mode 100644 index 00000000..d601d289 --- /dev/null +++ b/kubernetes/athenz-k8sclient/README.md @@ -0,0 +1,53 @@ +# athenz-k8sclient + +## Configuration + +Files below must be configured for each use cases accordingly + +1. [athenz-sia.env](kustomize/athenz-sia/athenz-sia.env) + +## Deployment + +``` +kubectl -n athenz apply -k kustomize +``` + +## Registering Client Service to Athenz + +``` +make register-athenz-k8sclient +``` + +confirm registration with: + +``` +kubectl -n athenz exec deployment/athenz-cli -it -- \ + zms-cli \ + -z https://athenz-zms-server.athenz:4443/zms/v1 \ + -key /var/run/athenz/athenz_admin.private.pem \ + -cert /var/run/athenz/athenz_admin.cert.pem \ + -d $(cat kustomize/athenz-sia/athenz-sia.env | grep -E ^PROVIDER_SERVICE | sed -e 's/PROVIDER_SERVICE=\(.*\)\.\(.*\)/\1/g') \ + show-domain +``` + +## Debugging + +``` +kubectl -n athenz exec -it deployment/k8sclient-deployment -c athenz-cli -- /bin/sh -c "curl -sv http://localhost:8080/echoserver | jq -r .request" +``` + +``` +kubectl -n athenz exec -it deployment/k8sclient-deployment -c athenz-cli -- /bin/sh -c "curl -sv http://localhost:8080/k8sclient2echoserver | jq -r .request" +``` + +``` +kubectl -n athenz exec -it deployment/k8sclient-deployment -c athenz-cli -- /bin/sh -c "curl -sv http://localhost:8080/k8sclient2server | jq -r .request" +``` + +``` +kubectl -n athenz exec -it deployment/k8sclient-deployment -c athenz-cli -- /bin/sh -c "curl -sv http://localhost:8080/k8sclient2echoservermtls | jq -r .request" +``` + +``` +kubectl -n athenz exec -it deployment/k8sclient-deployment -c athenz-cli -- /bin/sh -c "curl -sv --resolve k8sclient.athenz.svc.cluster.local:443:127.0.0.1 https://k8sclient.athenz.svc.cluster.local/echoserver | jq -r .request" +``` diff --git a/kubernetes/athenz-k8sclient/kustomize/.gitignore b/kubernetes/athenz-k8sclient/kustomize/.gitignore new file mode 100644 index 00000000..4a424df5 --- /dev/null +++ b/kubernetes/athenz-k8sclient/kustomize/.gitignore @@ -0,0 +1 @@ +secret.yaml diff --git a/kubernetes/athenz-k8sclient/kustomize/athenz-sia/athenz-sia.env b/kubernetes/athenz-k8sclient/kustomize/athenz-sia/athenz-sia.env new file mode 100644 index 00000000..88b623ea --- /dev/null +++ b/kubernetes/athenz-k8sclient/kustomize/athenz-sia/athenz-sia.env @@ -0,0 +1,175 @@ +# +# Booting mode of SIA +# must be one of "init" or "refresh" +# "init": SIA will exit after a single credential retrieval +# "refresh": SIA will run as a continuous process to retrieve credentials periodically and also runs as an HTTP server to provide credentials and metrics +# +MODE=init +# +# Athenz ZTS URL for SIA to request for X.509 certificate +# https://:/zts/v1 +# +ENDPOINT=https://athenz-zts-server.athenz:4443/zts/v1 +# +# Athenz Service name for the cloud provider +# . (e.g. "cloud-provider-top-level-domain.cluster-name.client-service") +# +PROVIDER_SERVICE=athenz.identityprovider +# +# Suffix restriction for SANs(Subject Alternative Names) DNS field in X.509 certificate +# Subject Alternative Names: +# "DNS: ." +# "DNS: .instanceid.athenz." +# Note: must meat the suffix registered in Athenz (e.g. ".athenz.cloud") +# +DNS_SUFFIX=svc.cluster.local +# +# Refresh interval for SIA to request to ZTS periodically in containers to get X.509 identity certificate +# Note: This only applies for refresh mode +# +REFRESH_INTERVAL=30s +# +# Delay interval for SIA to boot +# This may be useful when you want to prevent large number of SIA to make requests to ZTS concurrently in a short period +# +DELAY_JITTER_SECONDS=0 +# +# File paths to store Athenz X.509 certificate key file +# +KEY_FILE=/var/run/athenz/tls.key +# +# File paths to store Athenz X.509 certificate file +# +CERT_FILE=/var/run/athenz/tls.crt +# +# File paths to store Athenz X.509 ca certificate file that can verify CERT_FILE +# +CA_CERT_FILE=/var/run/athenz/ca.crt +# +# Directory to store the log files +# +LOG_DIR=/dev/null +# +# Log level to print logs +# Available values are: TRACE, DEBUG, INFO, WARNING, ERROR, FATAL or PANIC +# +LOG_LEVEL=DEBUG +# +# Set "read" or "read+write" for storing backup of X.509 certificate to Kubernetes Secret +# Set "write" or "read+write" for reading backup of X.509 certificate from Kubernetes Secret +# Set "" for disabling Kubernetes Secret backup +# "write" or "read+write" must be run uniquely for each secret to prevent conflict +# +BACKUP="" +# +# Kubernetes TLS Secret to backup and load X.509 certificate files +# +CERT_SECRET=k8sclient-tls +# +# Cloud users Athenz Domain to map to SIA +# +# if is not "", Athenz Domain would be: +# "" +# must end with "." +# must start with "." +# Example: +# ATHENZ_PREFIX="some-tld.foo." +# ATHENZ_DOMAIN="cluster-bar" +# ATHENZ_SUFFIX=".baz" +# +# if is "", Athenz Domain would be: +# "" +# is Kubernetes Namespace set by default +# must end with "." +# must start with "." +# Example: +# ATHENZ_PREFIX="some-tld.foo." +# NAMESPACE="namespace-bar" +# ATHENZ_SUFFIX=".baz" +# +# NAMESPACE may be extracted from metadata.namespace in Kubernetes manifests +# +NAMESPACE= +ATHENZ_DOMAIN= +ATHENZ_PREFIX= +ATHENZ_SUFFIX= +# +# Kubernetes Service Account as same name as Athenz Service +# +# SERVICEACCOUNT may be extracted from spec.serviceAccountName in Kubernetes Pod manifests +# +SERVICEACCOUNT=k8sclient +# +# File path for Kubernetes Service Account Token +# +SA_TOKEN_FILE=/var/run/secrets/kubernetes.io/bound-serviceaccount/token +# +# Kubernetes Pod IP +# +# POD_IP may be extracted from status.podIP in Kubernetes manifests +# +POD_IP= +# +# Kubernetes Pod UID +# +# POD_UID may be extracted from metadata.uid in Kubernetes manifests +# +POD_UID= +# +# CA certificate to verify ZTS server certificate +# +SERVER_CA_CERT=/etc/ssl/certs/ca-certificates.crt +# +# Comma separated Athenz Roles to retrieve role certificates (if empty, role certificate retrieval will be skipped) +# :role.,:role.... +# +TARGET_DOMAIN_ROLES=athenz:role.envoyclients,athenz:role.authorization-proxy-clients +# +# Directory path to store Athenz X.509 role certificate files +# +ROLECERT_DIR=/var/run/athenz +# +# Directory path to store Athenz role token files +# +TOKEN_DIR=/var/run/athenz +# +# Token type for Athenz role tokens +# +TOKEN_TYPE=roletoken+accesstoken +# +# Refresh interval for SIA to request to ZTS periodically in containers to get role tokens +# +TOKEN_REFRESH_INTERVAL=30s +# +# Server address to listen as token provider sidecar (e.g. :8180) +# +TOKEN_SERVER_ADDR=:8180 +# +# Server address to listen as metrics exporter sidecar (e.g. :9999) +# +METRICS_SERVER_ADDR=:9999 +# +# Delete Instance ID at container shutdown +# +DELETE_INSTANCE_ID=true +# +# Comma separated Athenz Domains to retrieve policies (if empty, Athenz Policy retrieval will be skipped) +# ,... +# +AUTHORIZATION_POLICY_DOMAINS= +# +# Server address to listen as authorization sidecar (e.g. :8280) +# +AUTHORIZATION_SERVER_ADDR= +# +# Refresh interval for SIA to request to ZTS periodically in containers to get policies +# +POLICY_REFRESH_INTERVAL= +# +# Refresh interval for SIA to request to ZTS periodically in containers to get public keys +# +PUBLICKEY_REFRESH_INTERVAL= +# +# Cache interval for SIA to authorize requests without comparing with roles and policies +# +AUTHORIZATION_CACHE_INTERVAL= diff --git a/kubernetes/athenz-k8sclient/kustomize/deployment.yaml b/kubernetes/athenz-k8sclient/kustomize/deployment.yaml new file mode 100644 index 00000000..ad3eeceb --- /dev/null +++ b/kubernetes/athenz-k8sclient/kustomize/deployment.yaml @@ -0,0 +1,197 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: k8sclient-deployment + labels: + app: k8sclient-deployment +spec: + replicas: 1 + selector: + matchLabels: + app: k8sclient-deployment + template: + metadata: + labels: + app: k8sclient-deployment + spec: + serviceAccountName: athenz-sia + securityContext: + fsGroup: 1000 + volumes: + - name: attestation-data + projected: + sources: + - serviceAccountToken: + audience: https://kubernetes.default.svc + expirationSeconds: 3600 + path: token + - name: athenz-tls-certs + emptyDir: {} + - name: envoy + configMap: + name: k8sclient-envoy + - name: athenz-cacert + secret: + secretName: athenz-cacert + defaultMode: 0555 + items: + - key: ca.cert.pem + path: ca-certificates.crt + - name: athenz-admin-keys + secret: + secretName: athenz-admin-keys + defaultMode: 0555 + items: + - key: athenz_admin.cert.pem + path: athenz_admin.cert.pem + - key: athenz_admin.private.pem + path: athenz_admin.private.pem + initContainers: + # TODO: sia-wait should be implemented in sia-init in future + - name: sia-wait + image: &sia-image ghcr.io/ctyano/k8s-athenz-sia:latest + imagePullPolicy: &sia-imagePullPolicy IfNotPresent + resources: &sia-resources + limits: + cpu: 50m + memory: 64Mi + command: + - sh + - -c + args: + - | + while true; do nc -vzw3 athenz-zts-server.athenz.svc.cluster.local 4443 && nc -vzw3 identityprovider.athenz.svc.cluster.local 443 && break; done + - name: sia-init + image: *sia-image + imagePullPolicy: *sia-imagePullPolicy + resources: *sia-resources + args: &sia-args + - --mode=init + - --delay-jitter-seconds=5 + envFrom: &sia-envFrom + - configMapRef: + name: k8sclient-sia + env: &sia-env + - name: NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: SERVICEACCOUNT + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.serviceAccountName + - name: POD_UID + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.uid + - name: POD_IP + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP + volumeMounts: &sia-volumeMounts + - name: attestation-data + mountPath: /var/run/secrets/kubernetes.io/bound-serviceaccount + readOnly: true + - name: athenz-tls-certs + mountPath: /var/run/athenz + - name: athenz-cacert + mountPath: /etc/ssl/certs/ca-certificates.crt + subPath: ca-certificates.crt + readOnly: true + containers: + - name: sia + image: *sia-image + imagePullPolicy: *sia-imagePullPolicy + args: + - --mode=refresh + - --token-server-addr=:8180 + - --authorization-server-addr=:8280 + - --metrics-server-addr=:9999 + envFrom: *sia-envFrom + env: *sia-env + ports: + - name: token-sidecar + containerPort: 8180 + protocol: TCP + - name: authorization + containerPort: 8280 + protocol: TCP + - name: exporter + containerPort: 9999 + protocol: TCP + resources: *sia-resources + volumeMounts: *sia-volumeMounts + - name: athenz-cli + image: ghcr.io/ctyano/athenz-cli:latest + imagePullPolicy: IfNotPresent + env: + - name: TZ + value: Asia/Tokyo + - name: ZMS + value: https://athenz-zms-server.athenz:4443/zms/v1 + - name: ZTS + value: https://athenz-zts-server.athenz:4443/zts/v1 + resources: + limits: + memory: 1Gi + cpu: 1 + requests: + memory: 256Mi + cpu: 50m + volumeMounts: + - name: athenz-admin-keys + mountPath: /var/run/athenz + readOnly: true + - name: athenz-cacert + mountPath: /etc/ssl/certs/ca-certificates.crt + subPath: ca-certificates.crt + readOnly: true + - name: envoy + # Envoy crashes in Raspberry Pi 4 + image: docker.io/envoyproxy/envoy:v1.29-latest + #image: docker.io/thegrandpkizzle/envoy:1.26.1 + imagePullPolicy: IfNotPresent + ports: + - name: https + containerPort: 443 + protocol: TCP + - name: http + containerPort: 8080 + protocol: TCP + command: + - /usr/local/bin/envoy + args: + - -c + - /etc/envoy/config.yaml + - -l + - debug + resources: &envoy-resources + limits: + cpu: 500m + memory: 512Mi + requests: + cpu: 50m + memory: 64Mi + volumeMounts: + - name: envoy + mountPath: /etc/envoy + readOnly: true + - name: athenz-tls-certs + mountPath: /var/run/athenz + readOnly: true + - name: echoserver + image: docker.io/ealen/echo-server:latest + imagePullPolicy: IfNotPresent + ports: + - name: echoserver + containerPort: 3000 + protocol: TCP + env: + - name: PORT + value: "3000" + resources: *envoy-resources diff --git a/kubernetes/athenz-k8sclient/kustomize/envoy/ca_sds.yaml b/kubernetes/athenz-k8sclient/kustomize/envoy/ca_sds.yaml new file mode 100644 index 00000000..2af98e25 --- /dev/null +++ b/kubernetes/athenz-k8sclient/kustomize/envoy/ca_sds.yaml @@ -0,0 +1,13 @@ +--- +# SDS Configuration to watch X.509 Certificate changes +# This example shows how to set up xDS connection by sourcing SDS configuration from the filesystem. +# The certificate and key files are watched with inotify and reloaded automatically without restart. +# See: https://www.envoyproxy.io/docs/envoy/latest/configuration/security/secret.html?highlight=inotify#example-three-certificate-rotation-for-xds-grpc-connection +resources: + - "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.Secret" + name: ca_sds + validation_context: + trusted_ca: + filename: /var/run/athenz/ca.crt + watched_directory: + path: /var/run/athenz diff --git a/kubernetes/athenz-k8sclient/kustomize/envoy/config.yaml b/kubernetes/athenz-k8sclient/kustomize/envoy/config.yaml new file mode 100644 index 00000000..582dc40f --- /dev/null +++ b/kubernetes/athenz-k8sclient/kustomize/envoy/config.yaml @@ -0,0 +1,473 @@ +# athenz-k8sclient +--- +admin: + access_log: + - name: access_log_file + typed_config: + "@type": type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog + path: "/dev/stdout" + address: + socket_address: + address: 127.0.0.1 + port_value: 9901 +node: + id: envoy-0 + cluster: envoy-cluster +static_resources: + listeners: + - name: localhost_listener + address: + socket_address: + protocol: TCP + address: ::FFFF:127.0.0.1 + ipv4_compat: true + port_value: 8080 + filter_chains: &localhost_filter_chains + - filters: + - name: envoy.filters.network.http_connection_manager + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + codec_type: auto + stat_prefix: localhost_proxy_prefix + access_log: &access_log + - name: envoy.access_loggers.stdout + typed_config: + "@type": type.googleapis.com/envoy.extensions.access_loggers.stream.v3.StdoutAccessLog + route_config: &localhost_route_config + name: localhost_proxy_route + virtual_hosts: + - name: localhost_proxy_hosts + domains: + # Only a single wildcard domain is permitted in route https_proxy_route + - "*" + typed_per_filter_config: + envoy.filters.http.ext_authz: + "@type": type.googleapis.com/envoy.extensions.filters.http.ext_authz.v3.ExtAuthzPerRoute + check_settings: + context_extensions: + virtual_host: localhost_proxy_hosts + routes: &localhost_routes + - match: + path_separated_prefix: "/status" + direct_response: + status: 200 + body: + inline_string: "This is k8sclient egress proxy." + response_headers_to_add: + - header: + key: x-athenz-method + value: "%REQ(:METHOD)%" + - header: + key: x-athenz-path + value: "%REQ(:PATH)%" + typed_per_filter_config: &ext_authz_disabled + envoy.filters.http.ext_authz.envoyclients.token: &ext_authz_envoyclients_tokensidecar_disabled + "@type": type.googleapis.com/envoy.extensions.filters.http.ext_authz.v3.ExtAuthzPerRoute + disabled: true + envoy.filters.http.ext_authz.authorization-proxy-clients.token: &ext_authz_authorization-proxy-clients_tokensidecar_disabled + "@type": type.googleapis.com/envoy.extensions.filters.http.ext_authz.v3.ExtAuthzPerRoute + disabled: true + - match: + path_separated_prefix: "/zms/v1" + route: + cluster: zms + auto_host_rewrite: true + typed_per_filter_config: *ext_authz_disabled + - match: + path_separated_prefix: "/zts/v1" + route: + cluster: zts + auto_host_rewrite: true + typed_per_filter_config: *ext_authz_disabled + - match: + path_separated_prefix: "/tokensidecar" + route: + cluster: token-sidecar + auto_host_rewrite: true + typed_per_filter_config: *ext_authz_disabled + - match: + path_separated_prefix: "/k8sclient2echoserver" + route: + cluster: echoserver + auto_host_rewrite: true + typed_per_filter_config: + envoy.filters.http.ext_authz.authorization-proxy-clients.token: *ext_authz_authorization-proxy-clients_tokensidecar_disabled + - match: + path_separated_prefix: "/k8sclient2extauthz" + route: + cluster: envoy_extauthz_tls + prefix_rewrite: "/extauthz" + auto_host_rewrite: true + typed_per_filter_config: + envoy.filters.http.ext_authz.authorization-proxy-clients.token: *ext_authz_authorization-proxy-clients_tokensidecar_disabled + - match: + path_separated_prefix: "/k8sclient2extauthzmtls" + route: + cluster: envoy_extauthz_mtls + prefix_rewrite: "/extauthz" + typed_per_filter_config: + envoy.filters.http.ext_authz.authorization-proxy-clients.token: *ext_authz_authorization-proxy-clients_tokensidecar_disabled + - match: + path_separated_prefix: "/k8sclient2filterauthzmtls" + route: + cluster: envoy_filter_mtls + prefix_rewrite: "/mtlsauthn" + typed_per_filter_config: + envoy.filters.http.ext_authz.envoyclients.token: *ext_authz_envoyclients_tokensidecar_disabled + envoy.filters.http.ext_authz.authorization-proxy-clients.token: *ext_authz_authorization-proxy-clients_tokensidecar_disabled + - match: + path_separated_prefix: "/k8sclient2filterauthzjwt" + route: + cluster: envoy_filter_tls + prefix_rewrite: "/jwtauthn" + typed_per_filter_config: + envoy.filters.http.ext_authz.authorization-proxy-clients.token: *ext_authz_authorization-proxy-clients_tokensidecar_disabled + - match: + path_separated_prefix: "/k8sclient2filterauthzmtlsjwt" + route: + cluster: envoy_filter_mtls + prefix_rewrite: "/jwtauthn" + typed_per_filter_config: + envoy.filters.http.ext_authz.authorization-proxy-clients.token: *ext_authz_authorization-proxy-clients_tokensidecar_disabled + - match: + path_separated_prefix: "/k8sclient2webhookauthzmtls" + route: + cluster: envoy_webhook_mtls + prefix_rewrite: "/mtlsauthn" + typed_per_filter_config: + envoy.filters.http.ext_authz.envoyclients.token: *ext_authz_envoyclients_tokensidecar_disabled + envoy.filters.http.ext_authz.authorization-proxy-clients.token: *ext_authz_authorization-proxy-clients_tokensidecar_disabled + - match: + path_separated_prefix: "/k8sclient2webhookauthzjwt" + route: + cluster: envoy_webhook_tls + prefix_rewrite: "/jwtauthn" + typed_per_filter_config: + envoy.filters.http.ext_authz.authorization-proxy-clients.token: *ext_authz_authorization-proxy-clients_tokensidecar_disabled + - match: + path_separated_prefix: "/k8sclient2webhookauthzmtlsjwt" + route: + cluster: envoy_webhook_mtls + prefix_rewrite: "/jwtauthn" + typed_per_filter_config: + envoy.filters.http.ext_authz.authorization-proxy-clients.token: *ext_authz_authorization-proxy-clients_tokensidecar_disabled + - match: + path_separated_prefix: "/k8sclient2authzproxy" + route: + cluster: authzproxy + prefix_rewrite: "/echoserver" + auto_host_rewrite: true + typed_per_filter_config: + envoy.filters.http.ext_authz.envoyclients.token: *ext_authz_envoyclients_tokensidecar_disabled + - match: + prefix: "/echoserver" + route: + cluster: echoserver + typed_per_filter_config: *ext_authz_disabled + - match: + prefix: "/" + route: + cluster: envoy_admin + typed_per_filter_config: *ext_authz_disabled + http_filters: &localhost_http_filters + # Authorization could also be done with type.googleapis.com/envoy.extensions.filters.http.jwt_authn.v3.JwtAuthentication + # https://www.envoyproxy.io/docs/envoy/latest/api-v3/extensions/filters/http/jwt_authn/v3/config.proto.html + # https://developer.mamezou-tech.com/blogs/2022/02/20/envoy-authz/ + - name: envoy.filters.http.ext_authz.envoyclients.token + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.ext_authz.v3.ExtAuthz + # https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/core/v3/grpc_service.proto#envoy-v3-api-msg-config-core-v3-grpcservice + #grpc_service: + # google_grpc: # or envoy_grpc + # target_uri: "127.0.0.1:9191" + # https://www.envoyproxy.io/docs/envoy/latest/api-v3/extensions/filters/http/ext_authz/v3/ext_authz.proto.html#extensions-filters-http-ext-authz-v3-httpservice + http_service: + server_uri: &server_uri + uri: token-sidecar + cluster: token-sidecar + timeout: 5s + authorization_request: + headers_to_add: + - key: x-athenz-proxy + value: "egress" + - key: x-athenz-domain + value: "athenz" + - key: x-athenz-role + value: "envoyclients" + authorization_response: &authorization_response + allowed_upstream_headers: + patterns: + - exact: athenz-role-auth + - exact: authorization + transport_api_version: V3 + - name: envoy.filters.http.ext_authz.authorization-proxy-clients.token + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.ext_authz.v3.ExtAuthz + http_service: + server_uri: *server_uri + authorization_request: + headers_to_add: + - key: x-athenz-proxy + value: "egress" + - key: x-athenz-domain + value: "athenz" + - key: x-athenz-role + value: "authorization-proxy-clients" + authorization_response: *authorization_response + transport_api_version: V3 + - name: envoy.filters.http.router + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + - name: https_listener + address: + socket_address: + protocol: TCP + address: ::FFFF:0.0.0.0 + ipv4_compat: true + port_value: 443 + filter_chains: + - filters: + - name: envoy.filters.network.http_connection_manager + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + codec_type: auto + stat_prefix: https_proxy_prefix + access_log: *access_log + route_config: *localhost_route_config + http_filters: *localhost_http_filters + transport_socket: + name: envoy.transport_sockets.tls + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.DownstreamTlsContext + common_tls_context: &common_tls_context + tls_certificate_sds_secret_configs: &tls_certificate_sds_secret_configs_identity + - name: identity_sds + sds_config: + path_config_source: + path: /etc/envoy/identity_sds.yaml + resource_api_version: V3 + validation_context_sds_secret_config: &validation_context_sds_secret_config_ca + name: ca_sds + sds_config: + path_config_source: + path: /etc/envoy/ca_sds.yaml + resource_api_version: V3 + clusters: + - name: envoy_admin + connect_timeout: 0.25s + type: STATIC # LOGICAL_DNS or STATIC + lb_policy: ROUND_ROBIN + load_assignment: + cluster_name: envoy_admin + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: 127.0.0.1 + port_value: 9901 + - name: envoy_localhost + connect_timeout: 0.25s + type: STATIC # LOGICAL_DNS or STATIC + lb_policy: ROUND_ROBIN + load_assignment: + cluster_name: envoy_localhost + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: 127.0.0.1 + port_value: 8080 + - name: envoy_extauthz_tls + connect_timeout: 0.25s + type: LOGICAL_DNS + dns_lookup_family: V4_ONLY + lb_policy: ROUND_ROBIN + load_assignment: + cluster_name: envoy_extauthz_tls + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: authorizer.athenz.svc.cluster.local + port_value: 443 + transport_socket: + name: envoy.transport_sockets.tls + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext + common_tls_context: + validation_context_sds_secret_config: *validation_context_sds_secret_config_ca + - name: envoy_filter_tls + connect_timeout: 0.25s + type: LOGICAL_DNS + dns_lookup_family: V4_ONLY + lb_policy: ROUND_ROBIN + load_assignment: + cluster_name: envoy_filter_tls + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: authzenvoy.athenz.svc.cluster.local + port_value: 443 + transport_socket: + name: envoy.transport_sockets.tls + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext + common_tls_context: + validation_context_sds_secret_config: *validation_context_sds_secret_config_ca + - name: envoy_webhook_tls + connect_timeout: 0.25s + type: LOGICAL_DNS + dns_lookup_family: V4_ONLY + lb_policy: ROUND_ROBIN + load_assignment: + cluster_name: envoy_webhook_tls + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: authzwebhook.athenz.svc.cluster.local + port_value: 443 + transport_socket: + name: envoy.transport_sockets.tls + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext + common_tls_context: + validation_context_sds_secret_config: *validation_context_sds_secret_config_ca + - name: envoy_extauthz_mtls + connect_timeout: 0.25s + type: LOGICAL_DNS + dns_lookup_family: V4_ONLY + lb_policy: ROUND_ROBIN + load_assignment: + cluster_name: envoy_extauthz_mtls + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: authorizer.athenz.svc.cluster.local + port_value: 4443 + transport_socket: &transport_socket + name: envoy.transport_sockets.tls + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext + common_tls_context: + tls_certificate_sds_secret_configs: &tls_certificate_sds_secret_configs_role + - name: role_sds + sds_config: + path_config_source: + path: /etc/envoy/role_sds.yaml + resource_api_version: V3 + validation_context_sds_secret_config: *validation_context_sds_secret_config_ca + - name: envoy_filter_mtls + connect_timeout: 0.25s + type: LOGICAL_DNS + dns_lookup_family: V4_ONLY + lb_policy: ROUND_ROBIN + load_assignment: + cluster_name: envoy_filter_mtls + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: authzenvoy.athenz.svc.cluster.local + port_value: 4443 + transport_socket: *transport_socket + - name: envoy_webhook_mtls + connect_timeout: 0.25s + type: LOGICAL_DNS + dns_lookup_family: V4_ONLY + lb_policy: ROUND_ROBIN + load_assignment: + cluster_name: envoy_webhook_mtls + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: authzwebhook.athenz.svc.cluster.local + port_value: 4443 + transport_socket: *transport_socket + - name: authzproxy + connect_timeout: 0.25s + type: LOGICAL_DNS + dns_lookup_family: V4_ONLY + lb_policy: ROUND_ROBIN + load_assignment: + cluster_name: authzproxy + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: authzproxy.athenz.svc.cluster.local + port_value: 443 + transport_socket: + name: envoy.transport_sockets.tls + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext + common_tls_context: + validation_context_sds_secret_config: *validation_context_sds_secret_config_ca + - name: zms + connect_timeout: 0.25s + type: LOGICAL_DNS + dns_lookup_family: V4_ONLY + lb_policy: ROUND_ROBIN + load_assignment: + cluster_name: zms + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: athenz-zms-server.athenz + port_value: 4443 + transport_socket: *transport_socket + - name: zts + connect_timeout: 0.25s + type: LOGICAL_DNS + dns_lookup_family: V4_ONLY + lb_policy: ROUND_ROBIN + load_assignment: + cluster_name: zts + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: athenz-zts-server.athenz + port_value: 4443 + transport_socket: *transport_socket + - name: token-sidecar + connect_timeout: 5s + type: STATIC + lb_policy: ROUND_ROBIN + load_assignment: + cluster_name: token-sidecar + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: 127.0.0.1 + port_value: 8180 + - name: echoserver + connect_timeout: 0.25s + type: STATIC # LOGICAL_DNS or STATIC + lb_policy: ROUND_ROBIN + load_assignment: + cluster_name: echoserver + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: 127.0.0.1 + port_value: 3000 diff --git a/kubernetes/athenz-k8sclient/kustomize/envoy/identity_sds.yaml b/kubernetes/athenz-k8sclient/kustomize/envoy/identity_sds.yaml new file mode 100644 index 00000000..a9efc5ee --- /dev/null +++ b/kubernetes/athenz-k8sclient/kustomize/envoy/identity_sds.yaml @@ -0,0 +1,15 @@ +--- +# SDS Configuration to watch X.509 Certificate changes +# This example shows how to set up xDS connection by sourcing SDS configuration from the filesystem. +# The certificate and key files are watched with inotify and reloaded automatically without restart. +# See: https://www.envoyproxy.io/docs/envoy/latest/configuration/security/secret.html?highlight=inotify#example-three-certificate-rotation-for-xds-grpc-connection +resources: + - "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.Secret" + name: identity_sds + tls_certificate: + certificate_chain: + filename: /var/run/athenz/tls.crt + private_key: + filename: /var/run/athenz/tls.key + watched_directory: + path: /var/run/athenz diff --git a/kubernetes/athenz-k8sclient/kustomize/envoy/role_sds.yaml b/kubernetes/athenz-k8sclient/kustomize/envoy/role_sds.yaml new file mode 100644 index 00000000..010042ae --- /dev/null +++ b/kubernetes/athenz-k8sclient/kustomize/envoy/role_sds.yaml @@ -0,0 +1,15 @@ +--- +# SDS Configuration to watch X.509 Certificate changes +# This example shows how to set up xDS connection by sourcing SDS configuration from the filesystem. +# The certificate and key files are watched with inotify and reloaded automatically without restart. +# See: https://www.envoyproxy.io/docs/envoy/latest/configuration/security/secret.html?highlight=inotify#example-three-certificate-rotation-for-xds-grpc-connection +resources: + - "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.Secret" + name: role_sds + tls_certificate: + certificate_chain: + filename: "/var/run/athenz/athenz:role.envoyclients.cert.pem" + private_key: + filename: /var/run/athenz/tls.key + watched_directory: + path: /var/run/athenz diff --git a/kubernetes/athenz-k8sclient/kustomize/kustomization.yaml b/kubernetes/athenz-k8sclient/kustomize/kustomization.yaml new file mode 100644 index 00000000..14e090ba --- /dev/null +++ b/kubernetes/athenz-k8sclient/kustomize/kustomization.yaml @@ -0,0 +1,55 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: athenz + +resources: + - deployment.yaml + - namespace.yaml + - service.yaml + - serviceaccount.yaml + +generatorOptions: + disableNameSuffixHash: true + +#secretGenerator: +#- files: +# - ca.cert.pem=certs/ca.cert.pem +# name: athenz-cacert +# type: Opaque + +configMapGenerator: + - name: k8sclient-sia + envs: + - athenz-sia/athenz-sia.env + - name: k8sclient-envoy + files: + - config.yaml=envoy/config.yaml + - identity_sds.yaml=envoy/identity_sds.yaml + - role_sds.yaml=envoy/role_sds.yaml + - ca_sds.yaml=envoy/ca_sds.yaml + +replacements: + - source: + version: v1 + kind: ConfigMap + name: k8sclient-sia + fieldpath: data.SERVICEACCOUNT + targets: + - select: + version: v1 + kind: ServiceAccount + name: k8sclient-serviceaccount + fieldPaths: + - metadata.name + options: + create: true + - select: + version: v1 + kind: Deployment + name: k8sclient-deployment + fieldPaths: + - spec.template.spec.serviceAccountName + options: + create: true diff --git a/kubernetes/athenz-k8sclient/kustomize/namespace.yaml b/kubernetes/athenz-k8sclient/kustomize/namespace.yaml new file mode 100644 index 00000000..8df80985 --- /dev/null +++ b/kubernetes/athenz-k8sclient/kustomize/namespace.yaml @@ -0,0 +1,5 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: athenz diff --git a/kubernetes/athenz-k8sclient/kustomize/service.yaml b/kubernetes/athenz-k8sclient/kustomize/service.yaml new file mode 100644 index 00000000..50fcf304 --- /dev/null +++ b/kubernetes/athenz-k8sclient/kustomize/service.yaml @@ -0,0 +1,15 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: k8sclient + labels: + app: k8sclient-service +spec: + type: ClusterIP + selector: + app: k8sclient-deployment + ports: + - name: https + port: 443 + targetPort: https diff --git a/kubernetes/athenz-k8sclient/kustomize/serviceaccount.yaml b/kubernetes/athenz-k8sclient/kustomize/serviceaccount.yaml new file mode 100644 index 00000000..e8ddb021 --- /dev/null +++ b/kubernetes/athenz-k8sclient/kustomize/serviceaccount.yaml @@ -0,0 +1,7 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: k8sclient-serviceaccount + labels: + app: k8sclient-serviceaccount From ae22bad74a87522bb7ab9e83aca975aceea8ee6a Mon Sep 17 00:00:00 2001 From: ctyano Date: Wed, 19 Mar 2025 23:55:16 +0900 Subject: [PATCH 4/6] Fixed k8sclient --- kubernetes/Makefile | 25 +++++++++++++++++++++++++ kubernetes/athenz-client/Makefile | 2 +- 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/kubernetes/Makefile b/kubernetes/Makefile index 5f02ec0c..e29c2010 100644 --- a/kubernetes/Makefile +++ b/kubernetes/Makefile @@ -19,6 +19,7 @@ clean-certificates: rm -rf athenz-authzwebhook/kustomize/{keys,certs} rm -rf athenz-authzproxy/kustomize/{keys,certs} rm -rf athenz-client/kustomize/{keys,certs} + rm -rf athenz-k8sclient/kustomize/{keys,certs} clean-namespace: kubectl delete namespace athenz ||: @@ -36,6 +37,7 @@ copy-to-kustomization: cp -r ../keys ../certs athenz-authzwebhook/kustomize/ cp -r ../keys ../certs athenz-authzproxy/kustomize/ cp -r ../keys ../certs athenz-client/kustomize/ + cp -r ../keys ../certs athenz-k8sclient/kustomize/ kind-setup: kind create cluster @@ -316,17 +318,28 @@ deploy-athenz-client: test-athenz-identityprovider test-athenz-client: @$(MAKE) -C athenz-client test-athenz-client +setup-athenz-k8sclient: test-athenz-servers test-athenz-identityprovider + @$(MAKE) -C athenz-k8sclient register-athenz-k8sclient check-register-athenz-k8sclient + +deploy-athenz-k8sclient: test-athenz-identityprovider + @$(MAKE) -C athenz-k8sclient deploy-athenz-k8sclient + +test-athenz-k8sclient: + @$(MAKE) -C athenz-k8sclient test-athenz-k8sclient + setup-athenz-workloads: @$(MAKE) -C athenz-authorizer register-athenz-authorizer @$(MAKE) -C athenz-authzenvoy register-athenz-authzenvoy @$(MAKE) -C athenz-authzwebhook register-athenz-authzwebhook @$(MAKE) -C athenz-authzproxy register-athenz-authzproxy @$(MAKE) -C athenz-client register-athenz-client + @$(MAKE) -C athenz-k8sclient register-athenz-k8sclient @$(MAKE) -C athenz-authorizer check-register-athenz-authorizer @$(MAKE) -C athenz-authzenvoy check-register-athenz-authzenvoy @$(MAKE) -C athenz-authzwebhook check-register-athenz-authzwebhook @$(MAKE) -C athenz-authzproxy check-register-athenz-authzproxy @$(MAKE) -C athenz-client check-register-athenz-client + @$(MAKE) -C athenz-k8sclient check-register-athenz-k8sclient deploy-athenz-workloads: test-athenz-identityprovider @$(MAKE) -C athenz-authorizer deploy-athenz-authorizer @@ -334,6 +347,7 @@ deploy-athenz-workloads: test-athenz-identityprovider @$(MAKE) -C athenz-authzwebhook deploy-athenz-authzwebhook @$(MAKE) -C athenz-authzproxy deploy-athenz-authzproxy @$(MAKE) -C athenz-client deploy-athenz-client + @$(MAKE) -C athenz-k8sclient deploy-athenz-k8sclient test-athenz-workloads: @$(MAKE) -C athenz-authorizer test-athenz-authorizer @@ -341,6 +355,7 @@ test-athenz-workloads: @$(MAKE) -C athenz-authzwebhook test-athenz-authzwebhook @$(MAKE) -C athenz-authzproxy test-athenz-authzproxy @$(MAKE) -C athenz-client test-athenz-client + @$(MAKE) -C athenz-k8sclient test-athenz-k8sclient test-athenz-servers: test-athenz-zms-server test-athenz-zts-server @echo "" @@ -351,6 +366,7 @@ test-athenz-servers: test-athenz-zms-server test-athenz-zts-server test-athenz-envoy2echoserver: kubectl -n athenz exec -it deployment/athenz-cli -c athenz-cli -- /bin/sh -c "curl -s https://client.athenz.svc.cluster.local/client2echoserver" | jq -r .request || (kubectl -n athenz logs deployment/client-deployment --all-containers=true && false) + kubectl -n athenz exec -it deployment/athenz-cli -c athenz-cli -- /bin/sh -c "curl -s https://k8sclient.athenz.svc.cluster.local/k8sclient2echoserver" | jq -r .request || (kubectl -n athenz logs deployment/k8sclient-deployment --all-containers=true && false) @echo "" @echo "**************************************" @echo "**** Envoy Showcase is functioning ***" @@ -360,6 +376,8 @@ test-athenz-envoy2echoserver: test-athenz-envoy2envoyextauthz: kubectl -n athenz exec -it deployment/athenz-cli -c athenz-cli -- /bin/sh -c "curl -s https://client.athenz.svc.cluster.local/client2extauthz" | jq -r .request || (kubectl -n athenz logs deployment/authorizer-deployment --all-containers=true && false) kubectl -n athenz exec -it deployment/athenz-cli -c athenz-cli -- /bin/sh -c "curl -s https://client.athenz.svc.cluster.local/client2extauthzmtls" | jq -r .request || (kubectl -n athenz logs deployment/authorizer-deployment --all-containers=true && false) + kubectl -n athenz exec -it deployment/athenz-cli -c athenz-cli -- /bin/sh -c "curl -s https://k8sclient.athenz.svc.cluster.local/k8sclient2extauthz" | jq -r .request || (kubectl -n athenz logs deployment/authorizer-deployment --all-containers=true && false) + kubectl -n athenz exec -it deployment/athenz-cli -c athenz-cli -- /bin/sh -c "curl -s https://k8sclient.athenz.svc.cluster.local/k8sclient2extauthzmtls" | jq -r .request || (kubectl -n athenz logs deployment/authorizer-deployment --all-containers=true && false) @echo "" @echo "**************************************" @echo "**** Envoy Showcase is functioning ***" @@ -370,11 +388,17 @@ test-athenz-envoy2envoyfilter: kubectl -n athenz exec -it deployment/athenz-cli -c athenz-cli -- /bin/sh -c "curl -s https://client.athenz.svc.cluster.local/client2filterauthzmtls" | jq -r .request || (kubectl -n athenz logs deployment/authzenvoy-deployment --all-containers=true && false) kubectl -n athenz exec -it deployment/athenz-cli -c athenz-cli -- /bin/sh -c "curl -s https://client.athenz.svc.cluster.local/client2filterauthzjwt" | jq -r .request || (kubectl -n athenz logs deployment/authzenvoy-deployment --all-containers=true && false) kubectl -n athenz exec -it deployment/athenz-cli -c athenz-cli -- /bin/sh -c "curl -s https://client.athenz.svc.cluster.local/client2filterauthzmtlsjwt" | jq -r .request || (kubectl -n athenz logs deployment/authzenvoy-deployment --all-containers=true && false) + kubectl -n athenz exec -it deployment/athenz-cli -c athenz-cli -- /bin/sh -c "curl -s https://k8sclient.athenz.svc.cluster.local/k8sclient2filterauthzmtls" | jq -r .request || (kubectl -n athenz logs deployment/authzenvoy-deployment --all-containers=true && false) + kubectl -n athenz exec -it deployment/athenz-cli -c athenz-cli -- /bin/sh -c "curl -s https://k8sclient.athenz.svc.cluster.local/k8sclient2filterauthzjwt" | jq -r .request || (kubectl -n athenz logs deployment/authzenvoy-deployment --all-containers=true && false) + kubectl -n athenz exec -it deployment/athenz-cli -c athenz-cli -- /bin/sh -c "curl -s https://k8sclient.athenz.svc.cluster.local/k8sclient2filterauthzmtlsjwt" | jq -r .request || (kubectl -n athenz logs deployment/authzenvoy-deployment --all-containers=true && false) test-athenz-envoy2envoywebhook: kubectl -n athenz exec -it deployment/athenz-cli -c athenz-cli -- /bin/sh -c "curl -s https://client.athenz.svc.cluster.local/client2webhookauthzmtls" | jq -r .request || (kubectl -n athenz logs deployment/authzwebhook-deployment --all-containers=true && false) kubectl -n athenz exec -it deployment/athenz-cli -c athenz-cli -- /bin/sh -c "curl -s https://client.athenz.svc.cluster.local/client2webhookauthzjwt" | jq -r .request || (kubectl -n athenz logs deployment/authzwebhook-deployment --all-containers=true && false) kubectl -n athenz exec -it deployment/athenz-cli -c athenz-cli -- /bin/sh -c "curl -s https://client.athenz.svc.cluster.local/client2webhookauthzmtlsjwt" | jq -r .request || (kubectl -n athenz logs deployment/authzwebhook-deployment --all-containers=true && false) + kubectl -n athenz exec -it deployment/athenz-cli -c athenz-cli -- /bin/sh -c "curl -s https://k8sclient.athenz.svc.cluster.local/k8sclient2webhookauthzmtls" | jq -r .request || (kubectl -n athenz logs deployment/authzwebhook-deployment --all-containers=true && false) + kubectl -n athenz exec -it deployment/athenz-cli -c athenz-cli -- /bin/sh -c "curl -s https://k8sclient.athenz.svc.cluster.local/k8sclient2webhookauthzjwt" | jq -r .request || (kubectl -n athenz logs deployment/authzwebhook-deployment --all-containers=true && false) + kubectl -n athenz exec -it deployment/athenz-cli -c athenz-cli -- /bin/sh -c "curl -s https://k8sclient.athenz.svc.cluster.local/k8sclient2webhookauthzmtlsjwt" | jq -r .request || (kubectl -n athenz logs deployment/authzwebhook-deployment --all-containers=true && false) @echo "" @echo "**************************************" @echo "**** Envoy Showcase is functioning ***" @@ -383,6 +407,7 @@ test-athenz-envoy2envoywebhook: test-athenz-envoy2authzproxy: kubectl -n athenz exec -it deployment/athenz-cli -c athenz-cli -- /bin/sh -c "curl -s https://client.athenz.svc.cluster.local/client2authzproxy" | jq -r .request || (kubectl -n athenz logs deployment/authzproxy-deployment --all-containers=true && false) + kubectl -n athenz exec -it deployment/athenz-cli -c athenz-cli -- /bin/sh -c "curl -s https://k8sclient.athenz.svc.cluster.local/k8sclient2authzproxy" | jq -r .request || (kubectl -n athenz logs deployment/authzproxy-deployment --all-containers=true && false) @echo "" @echo "**************************************" @echo "**** Envoy Showcase is functioning ***" diff --git a/kubernetes/athenz-client/Makefile b/kubernetes/athenz-client/Makefile index 16c5ee3e..1f498cd2 100644 --- a/kubernetes/athenz-client/Makefile +++ b/kubernetes/athenz-client/Makefile @@ -143,7 +143,7 @@ i=0; \ while true; do \ printf "\n***** Waiting for athenz($$(( $$i * $${SLEEP_SECONDS} ))s/$${WAITING_THRESHOLD}s) *****\n"; \ ( \ - test $$(( $$(kubectl -n athenz get all | grep client | grep -E "0/1" | wc -l) )) -eq 0 \ + test $$(( $$(kubectl -n athenz get all | grep pod/client- | grep -E "0/1" | wc -l) )) -eq 0 \ && \ kubectl -n athenz exec deployment/client-deployment -it -c sia -- \ ls \ From aa8a4449835fdd028d73b368e376c8531799346e Mon Sep 17 00:00:00 2001 From: ctyano Date: Thu, 20 Mar 2025 23:14:51 +0900 Subject: [PATCH 5/6] Switched to k8sidentityprovider --- kubernetes/athenz-k8sclient/Makefile | 43 +++++++++++++++++++ .../kustomize/athenz-sia/athenz-sia.env | 2 +- 2 files changed, 44 insertions(+), 1 deletion(-) diff --git a/kubernetes/athenz-k8sclient/Makefile b/kubernetes/athenz-k8sclient/Makefile index 5ffde8c3..389b84b4 100644 --- a/kubernetes/athenz-k8sclient/Makefile +++ b/kubernetes/athenz-k8sclient/Makefile @@ -5,6 +5,49 @@ endif clean-athenz-k8sclient: kubectl delete -k kustomize +register-athenz-k8sprovider: + kubectl -n athenz exec deployment/athenz-cli -it -- \ + zms-cli \ + -z https://athenz-zms-server.athenz:4443/zms/v1 \ + -key /var/run/athenz/athenz_admin.private.pem \ + -cert /var/run/athenz/athenz_admin.cert.pem \ + -d \ + $$(cat ../athenz-identityprovider/kustomize/kustomization.yaml | yq .namespace) \ + add-service \ + k8s$$(cat ../athenz-identityprovider/kustomize/athenz-sia/athenz-sia.env | grep -E ^SERVICEACCOUNT | sed -e 's/SERVICEACCOUNT=\(.*\)/\1/g') + kubectl -n athenz exec deployment/athenz-cli -it -- \ + zms-cli \ + -z https://athenz-zms-server.athenz:4443/zms/v1 \ + -key /var/run/athenz/athenz_admin.private.pem \ + -cert /var/run/athenz/athenz_admin.cert.pem \ + -d \ + $$(cat ../athenz-identityprovider/kustomize/kustomization.yaml | yq .namespace) \ + set-service-endpoint \ + $$(cat kustomize/athenz-sia/athenz-sia.env | grep -E ^PROVIDER_SERVICE | sed -e 's/PROVIDER_SERVICE=\(.*\)/\1/g') \ + class://com.yahoo.athenz.instance.provider.impl.InstanceK8SProvider + kubectl -n athenz exec deployment/athenz-cli -it -- \ + zms-cli \ + -z https://athenz-zms-server.athenz:4443/zms/v1 \ + -key /var/run/athenz/athenz_admin.private.pem \ + -cert /var/run/athenz/athenz_admin.cert.pem \ + -d \ + sys.auth \ + set-domain-template \ + instance_provider \ + provider="$$(cat kustomize/athenz-sia/athenz-sia.env | grep -E ^PROVIDER_SERVICE | sed -e 's/PROVIDER_SERVICE=\(.*\)/\1/g')" \ + dnssuffix="$$(cat ../athenz-identityprovider/kustomize/athenz-sia/athenz-sia.env | grep -E ^DNS_SUFFIX | sed -e 's/DNS_SUFFIX=\(.*\)/\1/g')" + kubectl -n athenz exec deployment/athenz-cli -it -- \ + zms-cli \ + -z https://athenz-zms-server.athenz:4443/zms/v1 \ + -key /var/run/athenz/athenz_admin.private.pem \ + -cert /var/run/athenz/athenz_admin.cert.pem \ + -d \ + sys.auth \ + set-domain-template \ + instance_provider \ + provider="$$(cat kustomize/athenz-sia/athenz-sia.env | grep -E ^PROVIDER_SERVICE | sed -e 's/PROVIDER_SERVICE=\(.*\)/\1/g')" \ + dnssuffix="$$(cat kustomize/athenz-sia/athenz-sia.env | grep -E ^DNS_SUFFIX | sed -e 's/DNS_SUFFIX=\(.*\)/\1/g')" + register-athenz-k8sclient: kubectl -n athenz exec deployment/athenz-cli -it -- \ curl \ diff --git a/kubernetes/athenz-k8sclient/kustomize/athenz-sia/athenz-sia.env b/kubernetes/athenz-k8sclient/kustomize/athenz-sia/athenz-sia.env index 88b623ea..c720a856 100644 --- a/kubernetes/athenz-k8sclient/kustomize/athenz-sia/athenz-sia.env +++ b/kubernetes/athenz-k8sclient/kustomize/athenz-sia/athenz-sia.env @@ -14,7 +14,7 @@ ENDPOINT=https://athenz-zts-server.athenz:4443/zts/v1 # Athenz Service name for the cloud provider # . (e.g. "cloud-provider-top-level-domain.cluster-name.client-service") # -PROVIDER_SERVICE=athenz.identityprovider +PROVIDER_SERVICE=athenz.k8sidentityprovider # # Suffix restriction for SANs(Subject Alternative Names) DNS field in X.509 certificate # Subject Alternative Names: From 4566c1d167c6bf3c2b5d264cf4eac0d0773e844b Mon Sep 17 00:00:00 2001 From: ctyano Date: Fri, 16 May 2025 22:40:43 +0900 Subject: [PATCH 6/6] Fixed check-documentation.yaml --- .github/workflows/check-documentation.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/check-documentation.yaml b/.github/workflows/check-documentation.yaml index 10d77959..b88dbded 100644 --- a/.github/workflows/check-documentation.yaml +++ b/.github/workflows/check-documentation.yaml @@ -18,6 +18,7 @@ on: paths: - '*.md' - 'docs/**' + - '**/*.md' env: # Use docker.io for Docker Hub if empty @@ -110,7 +111,6 @@ jobs: uses: actions/ai-inference@main with: prompt-file: '../prompt' - max-tokens: 1000 # response max-tokens # A GitHub Action to comment on PR - name: Comment on PR