From a3e8577c1bf67cb4ccf9d1bb5ad3841e12dcf310 Mon Sep 17 00:00:00 2001 From: upodroid Date: Fri, 17 Oct 2025 01:05:11 +0300 Subject: [PATCH] create new 5k dra job --- .../DRA/sig-scalability-periodic-dra.yaml | 217 ++++++++++++++++++ .../sig-scalability-periodic-jobs.yaml | 1 + ...sig-scalability-release-blocking-jobs.yaml | 2 + config/prow/config.yaml | 2 + 4 files changed, 222 insertions(+) diff --git a/config/jobs/kubernetes/sig-scalability/DRA/sig-scalability-periodic-dra.yaml b/config/jobs/kubernetes/sig-scalability/DRA/sig-scalability-periodic-dra.yaml index 7991ab014d81..bca56943188f 100644 --- a/config/jobs/kubernetes/sig-scalability/DRA/sig-scalability-periodic-dra.yaml +++ b/config/jobs/kubernetes/sig-scalability/DRA/sig-scalability-periodic-dra.yaml @@ -407,3 +407,220 @@ periodics: value: "gcr.io/k8s-staging-perf-tests/sleep:v0.0.3" - name: CL2_ENABLE_EXTENDED_RESOURCES value: "true" + + - name: ci-kubernetes-e2e-kops-gce-100-node-dra-with-workload-ipalias-using-cl2 + tags: + - "perfDashPrefix: gce-dra-100Nodes-with-workload-dev" + - "perfDashBuildsCount: 270" + - "perfDashJobType: performance" + cluster: k8s-infra-prow-build + interval: 4h + labels: + preset-k8s-ssh: "true" + preset-dind-enabled: "true" + decorate: true + decoration_config: + timeout: 480m + extra_refs: + - org: kubernetes + repo: kubernetes + base_ref: master + path_alias: k8s.io/kubernetes + - org: kubernetes + repo: perf-tests + base_ref: master + path_alias: k8s.io/perf-tests + - org: kubernetes + repo: kops + base_ref: master + path_alias: k8s.io/kops + workdir: true + annotations: + test.kops.k8s.io/cloud: gce + test.kops.k8s.io/distro: u2404 + test.kops.k8s.io/k8s_version: stable + test.kops.k8s.io/kops_channel: alpha + test.kops.k8s.io/networking: ipalias + testgrid-dashboards: kops-misc, sig-cluster-lifecycle-kops, sig-scalability-gce, sig-scalability-dra + testgrid-tab-name: gce-dra-with-workload-master-scalability-100-canary + description: "Uses kops to run k8s.io/perf-tests/run-e2e.sh against a 100-node cluster with DRA enabled" + spec: + serviceAccountName: prow-build + containers: + - image: us-central1-docker.pkg.dev/k8s-staging-test-infra/images/kubekins-e2e:v20250925-95b5a2c7a5-master + imagePullPolicy: Always + command: + - runner.sh + args: + - ./tests/e2e/scenarios/scalability/run-test.sh + securityContext: + privileged: true + env: + - name: KUBE_SSH_KEY_PATH + value: /etc/ssh-key-secret/ssh-private + - name: KUBE_SSH_USER + value: ubuntu + - name: GOPATH + value: /home/prow/go + - name: ARTIFACTS + value: $(ARTIFACTS) + - name: CNI_PLUGIN + value: gce + - name: KUBE_NODE_COUNT + value: "100" + - name: CL2_MODE + value: "Indexed" + - name: CL2_NODES_PER_NAMESPACE + value: "10" + - name: CL2_JOB_RUNNING_TIME + value: "3s" + - name: CL2_LONG_JOB_RUNNING_TIME + value: "45m" + - name: CL2_EXTENDED_RESOURCE_NAME + value: "example.com/gpu" + # - name: CL2_LOAD_TEST_THROUGHPUT + # value: "50" + # - name: CL2_DELETE_TEST_THROUGHPUT + # value: "50" + - name: CL2_RATE_LIMIT_POD_CREATION + value: "false" + - name: NODE_MODE + value: "master" + - name: CONTROL_PLANE_COUNT + value: "1" + - name: CONTROL_PLANE_SIZE + value: "c4-standard-96" + - name: KUBE_PROXY_MODE + value: "nftables" + - name: ENABLE_PROMETHEUS_SERVER + value: "true" + - name: PROMETHEUS_SCRAPE_KUBELETS + value: "true" + - name: PROMETHEUS_PVC_STORAGE_CLASS + value: "ssd-csi" + - name: CLOUD_PROVIDER + value: "gce" + - name: BOSKOS_RESOURCE_TYPE + value: "scalability-project" + - name: NODE_PRELOAD_IMAGES + value: "gcr.io/k8s-staging-perf-tests/sleep:v0.0.3" + - name: KOPS_CL2_TEST_CONFIG + value: testing/dra/config.yaml + - name: CL2_ENABLE_EXTENDED_RESOURCES + value: "true" + resources: + requests: + cpu: "7" + memory: "28Gi" + limits: + cpu: "7" + memory: "28Gi" + + - name: ci-kubernetes-e2e-kops-gce-5000-node-dra-with-workload-ipalias-using-cl2 + tags: + - "perfDashPrefix: gce-dra-5000Nodes-with-workload" + - "perfDashBuildsCount: 270" + - "perfDashJobType: performance" + cluster: k8s-infra-prow-build + cron: '0 3 * * *' # Run once a day at 03:00 UTC + labels: + preset-k8s-ssh: "true" + preset-dind-enabled: "true" + job_queue_name: "5k-gce-scale-test" # DON'T REMOVE THIS + decorate: true + decoration_config: + timeout: 480m + extra_refs: + - org: kubernetes + repo: kubernetes + base_ref: master + path_alias: k8s.io/kubernetes + - org: kubernetes + repo: perf-tests + base_ref: master + path_alias: k8s.io/perf-tests + - org: kubernetes + repo: kops + base_ref: master + path_alias: k8s.io/kops + workdir: true + annotations: + test.kops.k8s.io/cloud: gce + test.kops.k8s.io/distro: u2404 + test.kops.k8s.io/k8s_version: stable + test.kops.k8s.io/kops_channel: alpha + test.kops.k8s.io/networking: ipalias + testgrid-dashboards: kops-misc, sig-cluster-lifecycle-kops, sig-scalability-gce, sig-scalability-dra + testgrid-tab-name: gce-dra-with-workload-master-scalability-5000 + testgrid-alert-email: kubernetes-sig-scale@googlegroups.com, kubernetes-scalability-tickets@google.com + testgrid-num-failures-to-alert: '2' + description: "Uses kops to run k8s.io/perf-tests/run-e2e.sh against a 5000-node cluster with DRA enabled" + spec: + serviceAccountName: prow-build + containers: + - image: us-central1-docker.pkg.dev/k8s-staging-test-infra/images/kubekins-e2e:v20250925-95b5a2c7a5-master + imagePullPolicy: Always + command: + - runner.sh + args: + - ./tests/e2e/scenarios/scalability/run-test.sh + securityContext: + privileged: true + env: + - name: KUBE_SSH_KEY_PATH + value: /etc/ssh-key-secret/ssh-private + - name: KUBE_SSH_USER + value: ubuntu + - name: GOPATH + value: /home/prow/go + - name: ARTIFACTS + value: $(ARTIFACTS) + - name: CNI_PLUGIN + value: gce + - name: KUBE_NODE_COUNT + value: "5000" + - name: CL2_MODE + value: "Indexed" + - name: CL2_NODES_PER_NAMESPACE + value: "2500" + - name: CL2_JOB_RUNNING_TIME + value: "3s" + - name: CL2_LONG_JOB_RUNNING_TIME + value: "240m" + # - name: CL2_LOAD_TEST_THROUGHPUT + # value: "50" + # - name: CL2_DELETE_TEST_THROUGHPUT + # value: "50" + - name: CL2_RATE_LIMIT_POD_CREATION + value: "false" + - name: NODE_MODE + value: "master" + - name: CONTROL_PLANE_COUNT + value: "1" + - name: CONTROL_PLANE_SIZE + value: "c4-standard-96" + - name: KUBE_PROXY_MODE + value: "nftables" + - name: ENABLE_PROMETHEUS_SERVER + value: "true" + - name: PROMETHEUS_SCRAPE_KUBELETS + value: "true" + - name: PROMETHEUS_PVC_STORAGE_CLASS + value: "ssd-csi" + - name: CLOUD_PROVIDER + value: "gce" + - name: BOSKOS_RESOURCE_TYPE + value: "scalability-scale-project" + - name: NODE_PRELOAD_IMAGES + value: "gcr.io/k8s-staging-perf-tests/sleep:v0.0.3" + - name: CL2_ENABLE_EXTENDED_RESOURCES + value: "true" + - name: KOPS_CL2_TEST_CONFIG + value: testing/dra/config.yaml + resources: + requests: + cpu: "7" + memory: "28Gi" + limits: + cpu: "7" + memory: "28Gi" diff --git a/config/jobs/kubernetes/sig-scalability/sig-scalability-periodic-jobs.yaml b/config/jobs/kubernetes/sig-scalability/sig-scalability-periodic-jobs.yaml index 0eb09d12b88a..c0079c865399 100644 --- a/config/jobs/kubernetes/sig-scalability/sig-scalability-periodic-jobs.yaml +++ b/config/jobs/kubernetes/sig-scalability/sig-scalability-periodic-jobs.yaml @@ -1181,6 +1181,7 @@ periodics: preset-e2e-scalability-common: "true" preset-e2e-scalability-periodics: "true" preset-e2e-scalability-periodics-master: "true" + job_queue_name: "5k-gce-scale-test" # DON'T REMOVE THIS decorate: true decoration_config: timeout: 450m diff --git a/config/jobs/kubernetes/sig-scalability/sig-scalability-release-blocking-jobs.yaml b/config/jobs/kubernetes/sig-scalability/sig-scalability-release-blocking-jobs.yaml index 4cf27f52345a..689f2b23631f 100644 --- a/config/jobs/kubernetes/sig-scalability/sig-scalability-release-blocking-jobs.yaml +++ b/config/jobs/kubernetes/sig-scalability/sig-scalability-release-blocking-jobs.yaml @@ -11,6 +11,7 @@ periodics: preset-e2e-scalability-periodics: "true" preset-e2e-scalability-periodics-master: "true" decorate: true + job_queue_name: "5k-gce-scale-test" # DON'T REMOVE THIS decoration_config: timeout: 270m annotations: @@ -85,6 +86,7 @@ periodics: preset-e2e-scalability-periodics: "true" preset-e2e-scalability-periodics-master: "true" decorate: true + job_queue_name: "5k-gce-scale-test" # DON'T REMOVE THIS decoration_config: timeout: 450m extra_refs: diff --git a/config/prow/config.yaml b/config/prow/config.yaml index 8e0d80063946..d55024f441a6 100644 --- a/config/prow/config.yaml +++ b/config/prow/config.yaml @@ -11,6 +11,8 @@ plank: 'k8sio-image-promo': 1 # limits concurrency for k8s-test-infra-staging project 'test-infra-staging-image-push': 1 + # limits concurrency for 5k GCE jobs + '5k-gce-scale-test': 1 default_decoration_config_entries: - config: timeout: 2h