Merge pull request #5376 from bcgov/feat/5235

refactor(5235): encapsulate logic for cluster config generation

Author: junminahn

app/app/api/private-cloud/products/[licencePlate]/namespace-subnet/route.ts

@@ -4,7 +4,7 @@ import { GlobalRole } from '@/constants';
 import createApiHandler from '@/core/api-handler';
 import { OkResponse, UnauthorizedResponse } from '@/core/responses';
 import { models } from '@/services/db';
-import { getSubnet } from '@/services/k8s';
+import { getSubnet } from '@/services/k8s/metrics';
 import { getPathParamSchema } from '../schema';
 
 const queryParamSchema = z.object({

app/app/api/private-cloud/products/[licencePlate]/usage-metrics/route.ts

@@ -5,7 +5,7 @@ import { environmentLongNames, GlobalRole } from '@/constants';
 import createApiHandler from '@/core/api-handler';
 import { OkResponse, UnauthorizedResponse } from '@/core/responses';
 import { models } from '@/services/db';
-import { getPodMetrics } from '@/services/k8s';
+import { getPodMetrics } from '@/services/k8s/metrics';
 import { getPathParamSchema } from '../schema';
 
 const queryParamSchema = z.object({

app/helpers/auto-approval-check.ts

@@ -6,7 +6,7 @@ import {
   ResourceRequests,
 } from '@prisma/client';
 import _each from 'lodash-es/each';
-import { getResourceDetails } from '@/services/k8s';
+import { getResourceDetails } from '@/services/k8s/metrics';
 import { iterateObject } from '@/utils/js';
 
 const allowedAutoApprovalPercentage = 50;

app/jest.setup.ts

@@ -59,7 +59,7 @@ jest.mock('@/helpers/pdfs/emou/index', () => ({
   generateEmouPdf: jest.fn(async () => Buffer.alloc(0)),
 }));
 
-jest.mock('@/services/k8s', () => ({
+jest.mock('@/services/k8s/metrics', () => ({
   getResourceDetails: jest.fn(async () => ({
     env: 'dev',
     allocation: {

app/services/k8s/core.ts

@@ -0,0 +1,71 @@
+import { KubeConfig, CoreV1Api, Metrics } from '@kubernetes/client-node';
+import { Cluster } from '@prisma/client';
+
+function configureKubeConfig(cluster: string, token: string) {
+  const kc = new KubeConfig();
+  kc.loadFromOptions({
+    clusters: [
+      {
+        name: cluster,
+        server: `https://api.${cluster}.devops.gov.bc.ca:6443`,
+        skipTLSVerify: false,
+      },
+    ],
+    users: [
+      {
+        name: 'my-user',
+        token: token,
+      },
+    ],
+    contexts: [
+      {
+        name: `${cluster}-context`,
+        user: 'my-user',
+        cluster: cluster,
+      },
+    ],
+    currentContext: `${cluster}-context`,
+  });
+
+  return kc;
+}
+
+export function createK8sClusterConfigs(tokens: {
+  [Cluster.KLAB]: string;
+  [Cluster.CLAB]: string;
+  [Cluster.KLAB2]: string;
+  [Cluster.GOLDDR]: string;
+  [Cluster.GOLD]: string;
+  [Cluster.SILVER]: string;
+  [Cluster.EMERALD]: string;
+}) {
+  const k8sConfigs = {
+    [Cluster.KLAB]: configureKubeConfig(Cluster.KLAB, tokens[Cluster.KLAB]),
+    [Cluster.CLAB]: configureKubeConfig(Cluster.CLAB, tokens[Cluster.CLAB]),
+    [Cluster.KLAB2]: configureKubeConfig(Cluster.KLAB2, tokens[Cluster.KLAB2]),
+    [Cluster.GOLDDR]: configureKubeConfig(Cluster.GOLDDR, tokens[Cluster.GOLDDR]),
+    [Cluster.GOLD]: configureKubeConfig(Cluster.GOLD, tokens[Cluster.GOLD]),
+    [Cluster.SILVER]: configureKubeConfig(Cluster.SILVER, tokens[Cluster.SILVER]),
+    [Cluster.EMERALD]: configureKubeConfig(Cluster.EMERALD, tokens[Cluster.EMERALD]),
+  };
+
+  function getK8sClusterToken(cluster: Cluster) {
+    return k8sConfigs[cluster];
+  }
+
+  function getK8sClusterClients(cluster: Cluster) {
+    const kc = k8sConfigs[cluster];
+    const apiClient = kc.makeApiClient(CoreV1Api);
+    const metricsClient = new Metrics(kc);
+
+    return {
+      apiClient,
+      metricsClient,
+    };
+  }
+
+  return {
+    getK8sClusterToken,
+    getK8sClusterClients,
+  };
+}

app/services/k8s/helpers.ts

@@ -0,0 +1,50 @@
+import { Cluster } from '@prisma/client';
+import axios from 'axios';
+import {
+  CLAB_METRICS_READER_TOKEN,
+  KLAB_METRICS_READER_TOKEN,
+  KLAB2_METRICS_READER_TOKEN,
+  GOLDDR_METRICS_READER_TOKEN,
+  GOLD_METRICS_READER_TOKEN,
+  SILVER_METRICS_READER_TOKEN,
+  EMERALD_METRICS_READER_TOKEN,
+} from '@/config';
+import { createK8sClusterConfigs } from '../helpers';
+
+type PrometheusMetricResult = {
+  metric: Record<string, string>;
+  value: [number, string];
+};
+
+type PrometheusQueryData = {
+  resultType: string;
+  result: PrometheusMetricResult[];
+};
+
+type PrometheusQueryResponse = {
+  status: string;
+  data: PrometheusQueryData;
+};
+
+const { getK8sClusterToken, getK8sClusterClients } = createK8sClusterConfigs({
+  [Cluster.KLAB]: KLAB_METRICS_READER_TOKEN,
+  [Cluster.CLAB]: CLAB_METRICS_READER_TOKEN,
+  [Cluster.KLAB2]: KLAB2_METRICS_READER_TOKEN,
+  [Cluster.GOLDDR]: GOLDDR_METRICS_READER_TOKEN,
+  [Cluster.GOLD]: GOLD_METRICS_READER_TOKEN,
+  [Cluster.SILVER]: SILVER_METRICS_READER_TOKEN,
+  [Cluster.EMERALD]: EMERALD_METRICS_READER_TOKEN,
+});
+
+export const getK8sClients = getK8sClusterClients;
+
+export async function queryPrometheus(query: string, cluster: Cluster) {
+  const METRICS_URL = `https://prometheus-k8s-openshift-monitoring.apps.${cluster}.devops.gov.bc.ca`;
+  const METRICS_TOKEN = getK8sClusterToken(cluster);
+  const response = await axios.get<PrometheusQueryResponse>(`${METRICS_URL}/api/v1/query`, {
+    headers: { Authorization: `Bearer ${METRICS_TOKEN}` },
+    params: { query },
+  });
+
+  return response.data.data.result;
+}

app/services/k8s/metrics/core.ts

@@ -7,7 +7,7 @@ import {
 } from '@prisma/client';
 import _each from 'lodash-es/each';
 import { getTotalMetrics, memoryUnitMultipliers, cpuCoreToMillicoreMultiplier } from '@/helpers/resource-metrics';
-import { getPodMetrics } from '@/services/k8s';
+import { getPodMetrics } from '@/services/k8s/metrics';
 
 export async function getResourceDetails({
   licencePlate,