Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions .github/scripts/end2end/configure-e2e-ctst.sh
Original file line number Diff line number Diff line change
Expand Up @@ -106,3 +106,12 @@ kubectl run kafka-topics \
kafka-topics.sh --create --topic $AZURE_ARCHIVE_STATUS_TOPIC_2_NV --partitions 10 --bootstrap-server $KAFKA_HOST_PORT --if-not-exists ; \
kafka-topics.sh --create --topic $AZURE_ARCHIVE_STATUS_TOPIC_2_V --partitions 10 --bootstrap-server $KAFKA_HOST_PORT --if-not-exists ; \
kafka-topics.sh --create --topic $AZURE_ARCHIVE_STATUS_TOPIC_2_S --partitions 10 --bootstrap-server $KAFKA_HOST_PORT --if-not-exists"

# KMIP mock setup
# Deploy PyKMIP server (infra only, does NOT patch the CR).
# The CR is patched later, after file-backend SSE tests have run.
if ! kubectl get deployment pykmip &>/dev/null; then
bash "$(dirname "$0")/../mocks/setup-kmip.sh"
else
echo "PyKMIP deployment already exists, skipping setup-kmip.sh"
fi
120 changes: 120 additions & 0 deletions .github/scripts/mocks/setup-kmip.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,120 @@
#!/bin/bash
# setup-kmip.sh — Deploy PyKMIP mock server for KMIP SSE testing.
# Idempotent
#
# Deploys PyKMIP infra (certs, pod, service). The Zenko CR is patched
# by the CTST Before hook when @ServerSideEncryptionKmip tests start.

set -euo pipefail

ZENKO_NAME="${ZENKO_NAME:-end2end}"
NAMESPACE="${NAMESPACE:-default}"

# 1. Certs + secrets

if kubectl get secret "${ZENKO_NAME}-kmip-certs" -n "${NAMESPACE}" &>/dev/null; then
echo "KMIP secrets already exist, skipping cert generation"
else
echo "Generating KMIP TLS certificates..."
D=$(mktemp -d)
trap 'rm -rf "$D"' EXIT

openssl genrsa -out "$D/ca.key" 4096 2>/dev/null
openssl req -new -x509 -key "$D/ca.key" -out "$D/ca.pem" \
-days 3650 -subj "/CN=KMIP-CA" 2>/dev/null

openssl genrsa -out "$D/server.key" 4096 2>/dev/null
openssl req -new -key "$D/server.key" -out "$D/server.csr" \
-subj "/CN=pykmip" 2>/dev/null
openssl x509 -req -in "$D/server.csr" -CA "$D/ca.pem" -CAkey "$D/ca.key" \
-CAcreateserial -out "$D/server.crt" -days 3650 \
-extfile <(printf "subjectAltName=DNS:pykmip,DNS:pykmip.%s.svc.cluster.local" "$NAMESPACE") \
2>/dev/null

openssl genrsa -out "$D/client.key" 4096 2>/dev/null
openssl req -new -key "$D/client.key" -out "$D/client.csr" \
-subj "/CN=cloudserver-client" 2>/dev/null
openssl x509 -req -in "$D/client.csr" -CA "$D/ca.pem" -CAkey "$D/ca.key" \
-CAcreateserial -out "$D/client.crt" -days 3650 \
-extfile <(printf "extendedKeyUsage=clientAuth") 2>/dev/null

kubectl create secret generic "${ZENKO_NAME}-kmip-certs" \
--from-file=ca.pem="$D/ca.pem" --from-file=cert.pem="$D/client.crt" \
--from-file=key.pem="$D/client.key" \
--dry-run=client -o yaml | kubectl apply -f -

kubectl create secret generic pykmip-server-certs \
--from-file=ca.crt="$D/ca.pem" --from-file=server.crt="$D/server.crt" \
--from-file=server.key="$D/server.key" \
--dry-run=client -o yaml | kubectl apply -f -
fi

# 2. PyKMIP startup script

kubectl create configmap pykmip-server-script --dry-run=client -o yaml \
--from-literal=run_pykmip.py='
import logging; from kmip.services.server import KmipServer
logging.basicConfig(level=logging.INFO)
server = KmipServer(hostname="0.0.0.0", port=5696,
certificate_path="/certs/server.crt", key_path="/certs/server.key",
ca_path="/certs/ca.crt", auth_suite="TLS1.2", config_path=None,
enable_tls_client_auth=True, database_path="/tmp/pykmip.db")
with server: server.serve()
' | kubectl apply -f -

# 3. Deploy PyKMIP pod + service (inline YAML)

if ! kubectl get deployment pykmip -n "${NAMESPACE}" &>/dev/null; then
kubectl apply -n "${NAMESPACE}" -f - <<'YAML'
apiVersion: v1
kind: Service
metadata:
name: pykmip
spec:
selector: { name: pykmip }
ports: [{ name: kmip, port: 5696, targetPort: 5696 }]
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: pykmip
labels: { name: pykmip }
spec:
replicas: 1
selector:
matchLabels: { name: pykmip }
template:
metadata:
labels: { name: pykmip }
spec:
initContainers:
- name: install
image: docker.io/library/python:3.10-slim
command: [pip, install, --target=/pykmip-libs, pykmip==0.10.0, -q]
volumeMounts: [{ name: pykmip-libs, mountPath: /pykmip-libs }]
containers:
- name: pykmip
image: docker.io/library/python:3.10-slim
command: [python3, /scripts/run_pykmip.py]
env: [{ name: PYTHONPATH, value: /pykmip-libs }]
ports: [{ containerPort: 5696 }]
readinessProbe:
tcpSocket: { port: 5696 }
initialDelaySeconds: 5
periodSeconds: 3
volumeMounts:
- { name: certs, mountPath: /certs, readOnly: true }
- { name: scripts, mountPath: /scripts, readOnly: true }
- { name: pykmip-libs, mountPath: /pykmip-libs }
volumes:
- { name: certs, secret: { secretName: pykmip-server-certs } }
- { name: scripts, configMap: { name: pykmip-server-script } }
- { name: pykmip-libs, emptyDir: {} }
YAML
echo "Waiting for PyKMIP..."
kubectl wait --for=condition=Available deployment/pykmip -n "${NAMESPACE}" --timeout=5m
else
echo "PyKMIP already deployed"
fi

echo "PyKMIP infra ready"
2 changes: 1 addition & 1 deletion solution/deps.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -130,7 +130,7 @@ vault:
zenko-operator:
sourceRegistry: ghcr.io/scality
image: zenko-operator
tag: v1.8.9
tag: 15b35ecc60bc8b83e855e0cb17091b10e592aa07
Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

TO BE DONE
!

envsubst: ZENKO_OPERATOR_TAG
zookeeper:
sourceRegistry: ghcr.io/adobe/zookeeper-operator
Expand Down
63 changes: 62 additions & 1 deletion tests/ctst/common/hooks.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ import {
ITestCaseHookParameter,
} from '@cucumber/cucumber';
import Zenko from '../world/Zenko';
import { CacheHelper, Identity } from 'cli-testing';
import { CacheHelper, Identity, WorkCoordination } from 'cli-testing';
import { prepareQuotaScenarios, teardownQuotaScenarios } from 'steps/quotas/quotas';
import { prepareUtilizationScenarios } from 'steps/utilization/utilizationAPI';
import { prepareMetricsScenarios } from './utils';
Expand All @@ -16,6 +16,7 @@ import { displayDebuggingInformation, preparePRA } from 'steps/pra';
import {
cleanupAccount,
} from './utils';
import { createKubeCustomObjectClient, waitForZenkoToStabilize } from 'steps/utils/kubernetes';

import 'cli-testing/hooks/KeycloakSetup';
import 'cli-testing/hooks/Logger';
Expand All @@ -33,6 +34,7 @@ const noParallelRun = atMostOnePicklePerTag([
'@AfterAll',
'@PRA',
'@ColdStorage',
'@ServerSideEncryption',
...replicationLockTags
]);

Expand Down Expand Up @@ -75,6 +77,65 @@ Before({ tags: '@PrepareStorageUsageReportingScenarios', timeout: 1200000 }, asy
});
});

Before({ tags: '@ServerSideEncryptionKmip', timeout: 15 * 60 * 1000 },
Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

best i can come up with in CUCUMBER TESTS to have certains tests ran after others

// Patch the Zenko CR with KMIP configuration before running any KMIP-related tests
async function (this: Zenko) {
const lockName = `kmip-cr-patch-${process.ppid}`;
await WorkCoordination.runOnceAcrossWorkers(
{ lockName, logger: this.logger },
async () => {
const namespace = 'default';
const zenkoName = 'end2end';
const client = createKubeCustomObjectClient(this);
const cr = await client.getNamespacedCustomObject({
group: 'zenko.io',
version: 'v1alpha2',
namespace,
plural: 'zenkos',
name: zenkoName,
}) as {
spec?: {
kms?: {
kmip?: {
providerName?: string;
tlsSecretName?: string;
endpoints?: { host?: string; port?: number }[];
};
};
};
};
const alreadyConfigured =
cr?.spec?.kms?.kmip?.providerName === 'pykmip'
&& cr?.spec?.kms?.kmip?.endpoints?.some(
ep => ep.host === `pykmip.${namespace}.svc.cluster.local` && ep.port === 5696,
);
if (alreadyConfigured) {
return;
}

const kmipValue = {
providerName: 'pykmip',
tlsSecretName: `${zenkoName}-kmip-certs`,
endpoints: [{
host: `pykmip.${namespace}.svc.cluster.local`,
port: 5696,
}],
};

await client.patchNamespacedCustomObject({
group: 'zenko.io',
version: 'v1alpha2',
namespace,
plural: 'zenkos',
name: zenkoName,
body: [{ op: 'add', path: '/spec/kms', value: { kmip: kmipValue } }],
});
await waitForZenkoToStabilize(this, true);
},
);
},
);

After(async function (this: Zenko, results) {
// Reset any configuration set on the endpoint (ssl, port)
CacheHelper.parameters.ssl = this.parameters.ssl;
Expand Down
77 changes: 77 additions & 0 deletions tests/ctst/features/serverSideEncryption.feature
Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

again, might have added too many tests that aren't super useful for FUNCTIONAL testing 🤔

Original file line number Diff line number Diff line change
Expand Up @@ -85,3 +85,80 @@ Feature: Server Side Encryption
Given a "Non versioned" bucket
When the user gets bucket encryption
Then it should fail with error "ServerSideEncryptionConfigurationNotFoundError"

# KMIP backend tests
# These scenarios require a PyKMIP server to be deployed and Zenko to be
# reconfigured with spec.kms.kmip before running. The previous @ServerSideEncryptionFileBackend
# tests will not work once KMIP is configured on the ZENKO custom resource.

@2.14.0
@PreMerge
@ServerSideEncryption
@ServerSideEncryptionKmip
Scenario Outline: KMIP: should encrypt object when bucket encryption is <bucketAlgo> and object encryption is <objectAlgo>
Given a "Non versioned" bucket
And bucket encryption is set to "<bucketAlgo>" with key "<bucketKeyId>"
Then the bucket encryption is verified for algorithm "<bucketAlgo>" and key "<bucketKeyId>"
When an object "<objectName>" is uploaded with SSE algorithm "<objectAlgo>" and key "<objectKeyId>"
Then the PutObject response should have SSE algorithm "<expectedAlgo>" and KMS key "<expectedKeyId>"
Then the GetObject should return the uploaded body with SSE algorithm "<expectedAlgo>" and KMS key "<expectedKeyId>"

Examples: No bucket encryption
| objectName | bucketAlgo | bucketKeyId | objectAlgo | objectKeyId | expectedAlgo | expectedKeyId |
| kmip-none-none | | | | | | absent |
| kmip-none-aes | | | AES256 | | AES256 | absent |

Examples: No bucket encryption, aws:kms
| objectName | bucketAlgo | bucketKeyId | objectAlgo | objectKeyId | expectedAlgo | expectedKeyId |
| kmip-none-kms | | | aws:kms | | aws:kms | generated |

Examples: Bucket AES256
| objectName | bucketAlgo | bucketKeyId | objectAlgo | objectKeyId | expectedAlgo | expectedKeyId |
| kmip-aes-none | AES256 | | | | AES256 | absent |
| kmip-aes-aes | AES256 | | AES256 | | AES256 | absent |
| kmip-aes-kms | AES256 | | aws:kms | | aws:kms | generated |

Examples: Bucket aws:kms (default key)
| objectName | bucketAlgo | bucketKeyId | objectAlgo | objectKeyId | expectedAlgo | expectedKeyId |
| kmip-kms-none | aws:kms | | | | aws:kms | generated |
| kmip-kms-aes | aws:kms | | AES256 | | AES256 | absent |
| kmip-kms-kms | aws:kms | | aws:kms | | aws:kms | generated |

@2.14.0
@PreMerge
@ServerSideEncryption
@ServerSideEncryptionKmip
Scenario: KMIP: DeleteBucketEncryption removes default encryption
Given a "Non versioned" bucket
And bucket encryption is set to "AES256" with key ""
When an object "kmip-enc-obj" is uploaded with SSE algorithm "" and key ""
Then the GetObject should return the uploaded body with SSE algorithm "AES256" and KMS key "absent"
When the user deletes bucket encryption
Then the GetObject should return the uploaded body with SSE algorithm "AES256" and KMS key "absent"
When an object "kmip-plain-obj" is uploaded with SSE algorithm "" and key ""
Then the GetObject should return the uploaded body with SSE algorithm "" and KMS key "absent"

@2.14.0
@PreMerge
@ServerSideEncryption
@ServerSideEncryptionKmip
Scenario Outline: KMIP: PutObject with invalid SSE parameters returns an error: <objectName>
Given a "Non versioned" bucket
When an object "<objectName>" is uploaded with SSE algorithm "<algo>" and key "<keyId>"
Then it should fail with error "InvalidArgument"

Examples:
| objectName | algo | keyId |
| kmip-invalid-algo | INVALID_ALGO | |
| kmip-aes-kms-err | AES256 | some-key |

@2.14.0
@PreMerge
@ServerSideEncryption
@ServerSideEncryptionKmip
Scenario: KMIP: objects in same bucket share the same KMIP master key
Given a "Non versioned" bucket
And bucket encryption is set to "aws:kms" with key ""
When an object "kmip-shared-key-obj-a" is uploaded with SSE algorithm "" and key ""
And an object "kmip-shared-key-obj-b" is uploaded with SSE algorithm "" and key ""
Then objects "kmip-shared-key-obj-a" and "kmip-shared-key-obj-b" share the same KMS key
39 changes: 37 additions & 2 deletions tests/ctst/steps/serverSideEncryption.ts
Original file line number Diff line number Diff line change
Expand Up @@ -158,6 +158,15 @@ Then('the PutObject response should have SSE algorithm {string} and KMS key {str
if (expectedKey === 'absent') {
assert.strictEqual(result.sseKmsKeyId, undefined,
`PutObject: SSEKMSKeyId should be absent, got "${result.sseKmsKeyId}"`);
} else if (expectedKey === 'generated') {
assert.ok(result.sseKmsKeyId, 'PutObject: SSEKMSKeyId should be present');
// Accept either:
// - 64-char hex (file/internal KMS backend)
// - KMIP numeric ID or ARN: arn:scality:kms:external:kmip:<provider>:key/<n>
const isFileBackendKey = /^[a-f0-9]{64}$/.test(result.sseKmsKeyId);
const isKmipKey = /^(\d+|arn:scality:kms:external:kmip:[a-z0-9]+:key\/\d+)$/.test(result.sseKmsKeyId);
assert.ok(isFileBackendKey || isKmipKey,
`PutObject: expected a generated key (hex or KMIP), got "${result.sseKmsKeyId}"`);
} else {
assert.ok(result.sseKmsKeyId, 'PutObject: SSEKMSKeyId should be present');
}
Expand Down Expand Up @@ -189,8 +198,13 @@ Then('the GetObject should return the uploaded body with SSE algorithm {string}
`GetObject: SSEKMSKeyId should be absent, got "${resp.SSEKMSKeyId}"`);
} else if (expectedKey === 'generated') {
assert.ok(resp.SSEKMSKeyId, 'GetObject: SSEKMSKeyId should be present');
assert.match(resp.SSEKMSKeyId, /^[a-f0-9]{64}$/,
`GetObject: expected a generated hex key, got "${resp.SSEKMSKeyId}"`);
// Accept either:
// - 64-char hex (file/internal KMS backend)
// - KMIP numeric ID or ARN: arn:scality:kms:external:kmip:<provider>:key/<n>
const isFileBackendKey = /^[a-f0-9]{64}$/.test(resp.SSEKMSKeyId);
const isKmipKey = /^(\d+|arn:scality:kms:external:kmip:[a-z0-9]+:key\/\d+)$/.test(resp.SSEKMSKeyId);
assert.ok(isFileBackendKey || isKmipKey,
`GetObject: expected a generated key (hex or KMIP), got "${resp.SSEKMSKeyId}"`);
} else {
assert.strictEqual(resp.SSEKMSKeyId, expectedKey,
`GetObject: expected key "${expectedKey}", got "${resp.SSEKMSKeyId}"`);
Expand All @@ -208,3 +222,24 @@ Then('it should fail with error {string}',
`Expected error "${expectedError}" but got: ${result.err}`);
},
);

Then('objects {string} and {string} share the same KMS key',
async function (this: Zenko, objA: string, objB: string) {
const bucket = this.getSaved<string>('bucketName');
const client = buildS3Client();
try {
const [respA, respB] = await Promise.all([
client.send(new GetObjectCommand({ Bucket: bucket, Key: objA })),
client.send(new GetObjectCommand({ Bucket: bucket, Key: objB })),
]);
const keyA = respA.SSEKMSKeyId;
const keyB = respB.SSEKMSKeyId;
assert.ok(keyA, `Object "${objA}" has no SSEKMSKeyId`);
assert.ok(keyB, `Object "${objB}" has no SSEKMSKeyId`);
assert.strictEqual(keyA, keyB,
`Objects in same bucket should share the same KMIP key; got "${keyA}" vs "${keyB}"`);
} finally {
client.destroy();
}
},
);
Loading