From f83926487835004917f02dbeabfd237e63bb378c Mon Sep 17 00:00:00 2001 From: patuwwy Date: Wed, 14 Dec 2022 16:09:24 +0000 Subject: [PATCH] Fix and enrich pods limit error message --- .../adapters/src/kubernetes-client-adapter.ts | 12 ++- .../src/kubernetes-instance-adapter.ts | 10 +++ packages/host/src/lib/csi-controller.ts | 77 ++++++++++++------- 3 files changed, 67 insertions(+), 32 deletions(-) diff --git a/packages/adapters/src/kubernetes-client-adapter.ts b/packages/adapters/src/kubernetes-client-adapter.ts index 8b0682add..907c50338 100644 --- a/packages/adapters/src/kubernetes-client-adapter.ts +++ b/packages/adapters/src/kubernetes-client-adapter.ts @@ -137,7 +137,7 @@ class KubernetesClientAdapter { return response.body.status?.containerStatuses?.[0].state?.terminated?.reason; } - async isPodsLimitReached(quotaName: string) { + async getPodsLimit(quotaName: string) { const kubeApi = this.config.makeApiClient(k8s.CoreV1Api); try { @@ -152,13 +152,19 @@ class KubernetesClientAdapter { this.logger.info("Pods limit quota", used, hard); - return used >= hard; + return { used, hard }; } } catch (e) { this.logger.warn("Can't get quota object. "); } - return false; + return { used: 0, hard: 0 }; // quota failed - no pods! + } + + async isPodsLimitReached(quotaName: string) { + const { used, hard } = await this.getPodsLimit(quotaName); + + return used >= hard; } private async runWithRetries(retries: number, name: string, callback: any) { diff --git a/packages/adapters/src/kubernetes-instance-adapter.ts b/packages/adapters/src/kubernetes-instance-adapter.ts index c64b61be1..32d8e9c34 100644 --- a/packages/adapters/src/kubernetes-instance-adapter.ts +++ b/packages/adapters/src/kubernetes-instance-adapter.ts @@ -88,6 +88,16 @@ IComponent { }; } + async getPodsLimit() { + if (this.adapterConfig.quotaName) { + return this.kubeClient.getPodsLimit(this.adapterConfig.quotaName); + } + + this.logger.warn("Quota name not provided"); + + return undefined; + } + async run(config: InstanceConfig, instancesServerPort: number, instanceId: string): Promise { if (config.type !== "kubernetes") { throw new Error(`Invalid config type for kubernetes adapter: ${config.type}`); diff --git a/packages/host/src/lib/csi-controller.ts b/packages/host/src/lib/csi-controller.ts index 0aca944f7..d61c11984 100644 --- a/packages/host/src/lib/csi-controller.ts +++ b/packages/host/src/lib/csi-controller.ts @@ -40,7 +40,7 @@ import { EventEmitter, once } from "events"; import { ServerResponse } from "http"; import { DuplexStream, getRouter } from "@scramjet/api-server"; -import { getInstanceAdapter } from "@scramjet/adapters"; +import { getInstanceAdapter, KubernetesInstanceAdapter } from "@scramjet/adapters"; import { cancellableDefer, CancellablePromise, defer, promiseTimeout, TypedEmitter } from "@scramjet/utility"; import { ObjLogger } from "@scramjet/obj-logger"; import { ReasonPhrases } from "http-status-codes"; @@ -330,71 +330,90 @@ export class CSIController extends TypedEmitter { }); } - private mapRunnerExitCode(exitcode: number): Promise< + async handlePodsLimitError(): Promise { + let msg = "Instance limit reached"; + + if (this.instanceAdapter instanceof KubernetesInstanceAdapter) { + const limit = await this.instanceAdapter.getPodsLimit(); + + msg += limit ? ` (${limit.used}/${limit.hard})` : ""; + + return Promise.reject({ + message: msg, + exitcode: RunnerExitCode.PODS_LIMIT_REACHED, + status: InstanceStatus.ERRORED + }); + } + + this.logger.warn("Incorrect Adapter exitcode"); + + return Promise.reject({ + message: msg, + exitcode: RunnerExitCode.PODS_LIMIT_REACHED, + status: InstanceStatus.ERRORED + }); + } + + // eslint-disable-next-line complexity + private async mapRunnerExitCode(exitcode: number): Promise< { message: string, exitcode: number, reason: TerminateReason} > { - // eslint-disable-next-line default-case switch (exitcode) { - case RunnerExitCode.INVALID_ENV_VARS: { + case RunnerExitCode.SUCCESS: + return Promise.resolve({ message: "Instance completed", exitcode, reason: TerminateReason.COMPLETED, status: InstanceStatus.COMPLETED }); + + case RunnerExitCode.INVALID_ENV_VARS: return Promise.reject({ message: "Runner was started with invalid configuration. This is probably a bug in STH.", exitcode: RunnerExitCode.INVALID_ENV_VARS, status: InstanceStatus.ERRORED }); - } - case RunnerExitCode.PODS_LIMIT_REACHED: { - return Promise.reject({ - message: "Instance limit reached", - exitcode: RunnerExitCode.PODS_LIMIT_REACHED, - status: InstanceStatus.ERRORED - }); - } - case RunnerExitCode.INVALID_SEQUENCE_PATH: { + + case RunnerExitCode.PODS_LIMIT_REACHED: + return this.handlePodsLimitError(); + + case RunnerExitCode.INVALID_SEQUENCE_PATH: return Promise.reject({ message: `Sequence entrypoint path ${this.sequence.config.entrypointPath} is invalid. ` + "Check `main` field in Sequence package.json", exitcode: RunnerExitCode.INVALID_SEQUENCE_PATH, status: InstanceStatus.ERRORED }); - } - case RunnerExitCode.SEQUENCE_FAILED_ON_START: { + + case RunnerExitCode.SEQUENCE_FAILED_ON_START: return Promise.reject({ message: "Sequence failed on start", exitcode: RunnerExitCode.SEQUENCE_FAILED_ON_START, status: InstanceStatus.ERRORED }); - } - case RunnerExitCode.SEQUENCE_FAILED_DURING_EXECUTION: { + + case RunnerExitCode.SEQUENCE_FAILED_DURING_EXECUTION: return Promise.reject({ message: "Sequence failed during execution", exitcode: RunnerExitCode.SEQUENCE_FAILED_DURING_EXECUTION, status: InstanceStatus.ERRORED }); - } - case RunnerExitCode.SEQUENCE_UNPACK_FAILED: { + + case RunnerExitCode.SEQUENCE_UNPACK_FAILED: return Promise.reject({ message: "Sequence unpack failed", exitcode: RunnerExitCode.SEQUENCE_UNPACK_FAILED, status: InstanceStatus.ERRORED }); - } - case RunnerExitCode.KILLED: { + + case RunnerExitCode.KILLED: return Promise.resolve({ message: "Instance killed", exitcode: RunnerExitCode.KILLED, reason: TerminateReason.KILLED, status: InstanceStatus.COMPLETED }); - } - case RunnerExitCode.STOPPED: { + + case RunnerExitCode.STOPPED: return Promise.resolve({ message: "Instance stopped", exitcode: RunnerExitCode.STOPPED, reason: TerminateReason.STOPPED, status: InstanceStatus.COMPLETED }); - } - } - if (exitcode > 0) { - return Promise.reject({ message: "Runner failed", exitcode, reason: TerminateReason.ERRORED, status: InstanceStatus.ERRORED }); + default: + return Promise.reject({ message: "Runner failed", exitcode, reason: TerminateReason.ERRORED, status: InstanceStatus.ERRORED }); } - - return Promise.resolve({ message: "Instance completed", exitcode, reason: TerminateReason.COMPLETED, status: InstanceStatus.COMPLETED }); } async cleanup() {