From 3fa09d24ddef8ab03feb3f2386b17a165eafc75d Mon Sep 17 00:00:00 2001
From: Stuart Pearson <1926002+stuartp44@users.noreply.github.com>
Date: Wed, 21 May 2025 18:39:55 +0200
Subject: [PATCH 01/23] feat(runners): add runnerId to RunnerList and implement
 untag functionality for incorrectly tagged runners

---
 .../control-plane/src/aws/runners.d.ts        |  8 +++
 .../control-plane/src/aws/runners.test.ts     | 67 +++++++++++++++++--
 .../control-plane/src/aws/runners.ts          |  8 +++
 .../src/scale-runners/scale-down.test.ts      | 54 ++++++++++++++-
 .../src/scale-runners/scale-down.ts           | 52 ++++++++++----
 .../src/scale-runners/scale-up.test.ts        |  5 +-
 .../src/scale-runners/scale-up.ts             | 18 ++++-
 7 files changed, 192 insertions(+), 20 deletions(-)

diff --git a/lambdas/functions/control-plane/src/aws/runners.d.ts b/lambdas/functions/control-plane/src/aws/runners.d.ts
index 3a1b31b1cf..752ff07edd 100644
--- a/lambdas/functions/control-plane/src/aws/runners.d.ts
+++ b/lambdas/functions/control-plane/src/aws/runners.d.ts
@@ -10,6 +10,14 @@ export interface RunnerList {
   repo?: string;
   org?: string;
   orphan?: boolean;
+  runnerId?: string;
+}
+
+export interface RunnerState {
+  id: number;
+  name: string;
+  busy: boolean;
+  status: string;
 }
 
 export interface RunnerInfo {
diff --git a/lambdas/functions/control-plane/src/aws/runners.test.ts b/lambdas/functions/control-plane/src/aws/runners.test.ts
index b927f98696..008f29433f 100644
--- a/lambdas/functions/control-plane/src/aws/runners.test.ts
+++ b/lambdas/functions/control-plane/src/aws/runners.test.ts
@@ -4,6 +4,7 @@ import {
   CreateFleetInstance,
   CreateFleetResult,
   CreateTagsCommand,
+  DeleteTagsCommand,
   DefaultTargetCapacityType,
   DescribeInstancesCommand,
   DescribeInstancesResult,
@@ -17,7 +18,7 @@ import { mockClient } from 'aws-sdk-client-mock';
 import 'aws-sdk-client-mock-jest/vitest';
 
 import ScaleError from './../scale-runners/ScaleError';
-import { createRunner, listEC2Runners, tag, terminateRunner } from './runners';
+import { createRunner, listEC2Runners, tag, untag, terminateRunner } from './runners';
 import { RunnerInfo, RunnerInputParameters, RunnerType } from './runners.d';
 import { describe, it, expect, beforeEach, vi } from 'vitest';
 
@@ -53,6 +54,26 @@ const mockRunningInstances: DescribeInstancesResult = {
     },
   ],
 };
+const mockRunningInstancesJit: DescribeInstancesResult = {
+  Reservations: [
+    {
+      Instances: [
+        {
+          LaunchTime: new Date('2020-10-10T14:48:00.000+09:00'),
+          InstanceId: 'i-1234',
+          Tags: [
+            { Key: 'ghr:Application', Value: 'github-action-runner' },
+            { Key: 'ghr:runner_name_prefix', Value: RUNNER_NAME_PREFIX },
+            { Key: 'ghr:created_by', Value: 'scale-up-lambda' },
+            { Key: 'ghr:Type', Value: 'Org' },
+            { Key: 'ghr:Owner', Value: 'CoderToCat' },
+            { Key: 'ghr:githubrunnerid', Value: '9876543210' },
+          ],
+        },
+      ],
+    },
+  ],
+};
 
 describe('list instances', () => {
   beforeEach(() => {
@@ -60,7 +81,7 @@ describe('list instances', () => {
     vi.clearAllMocks();
   });
 
-  it('returns a list of instances', async () => {
+  it('returns a list of instances (Non JIT)', async () => {
     mockEC2Client.on(DescribeInstancesCommand).resolves(mockRunningInstances);
     const resp = await listEC2Runners();
     expect(resp.length).toBe(1);
@@ -73,6 +94,20 @@ describe('list instances', () => {
     });
   });
 
+  it('returns a list of instances (JIT)', async () => {
+    mockEC2Client.on(DescribeInstancesCommand).resolves(mockRunningInstancesJit);
+    const resp = await listEC2Runners();
+    expect(resp.length).toBe(1);
+    expect(resp).toContainEqual({
+      instanceId: 'i-1234',
+      launchTime: new Date('2020-10-10T14:48:00.000+09:00'),
+      type: 'Org',
+      owner: 'CoderToCat',
+      orphan: false,
+      runnerId: '9876543210',
+    });
+  });
+
   it('check orphan tag.', async () => {
     const instances: DescribeInstancesResult = mockRunningInstances;
     instances.Reservations![0].Instances![0].Tags!.push({ Key: 'ghr:orphan', Value: 'true' });
@@ -229,11 +264,35 @@ describe('tag runner', () => {
       owner: 'owner-2',
       type: 'Repo',
     };
-    await tag(runner.instanceId, [{ Key: 'ghr:orphan', Value: 'truer' }]);
+    await tag(runner.instanceId, [{ Key: 'ghr:orphan', Value: 'true' }]);
+
+    expect(mockEC2Client).toHaveReceivedCommandWith(CreateTagsCommand, {
+      Resources: [runner.instanceId],
+      Tags: [{ Key: 'ghr:orphan', Value: 'true' }],
+    });
+  });
+});
 
+describe('untag runner', () => {
+  beforeEach(() => {
+    vi.clearAllMocks();
+  });
+  it('removing extra tag', async () => {
+    mockEC2Client.on(DeleteTagsCommand).resolves({});
+    const runner: RunnerInfo = {
+      instanceId: 'instance-2',
+      owner: 'owner-2',
+      type: 'Repo',
+    };
+    //await tag(runner.instanceId, [{ Key: 'ghr:orphan', Value: '' }]);
     expect(mockEC2Client).toHaveReceivedCommandWith(CreateTagsCommand, {
       Resources: [runner.instanceId],
-      Tags: [{ Key: 'ghr:orphan', Value: 'truer' }],
+      Tags: [{ Key: 'ghr:orphan', Value: 'true' }],
+    });
+    await untag(runner.instanceId, [{ Key: 'ghr:orphan', Value: 'true' }]);
+    expect(mockEC2Client).toHaveReceivedCommandWith(DeleteTagsCommand, {
+      Resources: [runner.instanceId],
+      Tags: [{ Key: 'ghr:orphan', Value: 'true' }],
     });
   });
 });
diff --git a/lambdas/functions/control-plane/src/aws/runners.ts b/lambdas/functions/control-plane/src/aws/runners.ts
index dfe4d99fcf..c8fb1465f7 100644
--- a/lambdas/functions/control-plane/src/aws/runners.ts
+++ b/lambdas/functions/control-plane/src/aws/runners.ts
@@ -2,6 +2,7 @@ import {
   CreateFleetCommand,
   CreateFleetResult,
   CreateTagsCommand,
+  DeleteTagsCommand,
   DescribeInstancesCommand,
   DescribeInstancesResult,
   EC2Client,
@@ -91,6 +92,7 @@ function getRunnerInfo(runningInstances: DescribeInstancesResult) {
             repo: i.Tags?.find((e) => e.Key === 'ghr:Repo')?.Value as string,
             org: i.Tags?.find((e) => e.Key === 'ghr:Org')?.Value as string,
             orphan: i.Tags?.find((e) => e.Key === 'ghr:orphan')?.Value === 'true',
+            runnerId: i.Tags?.find((e) => e.Key === 'ghr:githubrunnerid')?.Value as string,
           });
         }
       }
@@ -112,6 +114,12 @@ export async function tag(instanceId: string, tags: Tag[]): Promise<void> {
   await ec2.send(new CreateTagsCommand({ Resources: [instanceId], Tags: tags }));
 }
 
+export async function untag(instanceId: string, tags: Tag[]): Promise<void> {
+  logger.debug(`Untagging '${instanceId}'`, { tags });
+  const ec2 = getTracedAWSV3Client(new EC2Client({ region: process.env.AWS_REGION }));
+  await ec2.send(new DeleteTagsCommand({ Resources: [instanceId], Tags: tags }));
+}
+
 function generateFleetOverrides(
   subnetIds: string[],
   instancesTypes: string[],
diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-down.test.ts b/lambdas/functions/control-plane/src/scale-runners/scale-down.test.ts
index 87bab093cb..7823d0ec76 100644
--- a/lambdas/functions/control-plane/src/scale-runners/scale-down.test.ts
+++ b/lambdas/functions/control-plane/src/scale-runners/scale-down.test.ts
@@ -4,7 +4,7 @@ import nock from 'nock';
 
 import { RunnerInfo, RunnerList } from '../aws/runners.d';
 import * as ghAuth from '../github/auth';
-import { listEC2Runners, terminateRunner, tag } from './../aws/runners';
+import { listEC2Runners, terminateRunner, tag, untag } from './../aws/runners';
 import { githubCache } from './cache';
 import { newestFirstStrategy, oldestFirstStrategy, scaleDown } from './scale-down';
 import { describe, it, expect, beforeEach, vi } from 'vitest';
@@ -33,6 +33,7 @@ vi.mock('./../aws/runners', async (importOriginal) => {
   return {
     ...actual,
     tag: vi.fn(),
+    untag: vi.fn(),
     terminateRunner: vi.fn(),
     listEC2Runners: vi.fn(),
   };
@@ -62,6 +63,7 @@ const mockedInstallationAuth = vi.mocked(ghAuth.createGithubInstallationAuth);
 const mockCreateClient = vi.mocked(ghAuth.createOctokitClient);
 const mockListRunners = vi.mocked(listEC2Runners);
 const mockTagRunners = vi.mocked(tag);
+const mockUntagRunners = vi.mocked(untag);
 const mockTerminateRunners = vi.mocked(terminateRunner);
 
 export interface TestData {
@@ -312,7 +314,7 @@ describe('Scale down runners', () => {
         checkNonTerminated(runners);
       });
 
-      it(`Should terminate orphan.`, async () => {
+      it(`Should terminate orphan (Non JIT)`, async () => {
         // setup
         const orphanRunner = createRunnerTestData('orphan-1', type, MINIMUM_BOOT_TIME + 1, false, false, false);
         const idleRunner = createRunnerTestData('idle-1', type, MINIMUM_BOOT_TIME + 1, true, false, false);
@@ -334,6 +336,7 @@ describe('Scale down runners', () => {
             Value: 'true',
           },
         ]);
+
         expect(mockTagRunners).not.toHaveBeenCalledWith(idleRunner.instanceId, expect.anything());
 
         // next cycle, update test data set orphan to true and terminate should be true
@@ -348,6 +351,51 @@ describe('Scale down runners', () => {
         checkNonTerminated(runners);
       });
 
+    it('Should test if orphaned runner untag if online and busy, else terminate (JIT)', async () => {
+        // arrange
+        const orphanRunner = createRunnerTestData('orphan-jit', type, MINIMUM_BOOT_TIME + 1, false, true, false, undefined, 1234567890);
+        const runners = [orphanRunner];
+
+        mockGitHubRunners([]);
+        mockAwsRunners(runners);
+
+        if (type === 'Repo') {
+          mockOctokit.actions.getSelfHostedRunnerForRepo.mockResolvedValueOnce({
+            data: { runnerId: 1234567890, name: orphanRunner.instanceId, busy: true, status: 'online' },
+          });
+        } else {
+          mockOctokit.actions.getSelfHostedRunnerForOrg.mockResolvedValueOnce({
+            data: { runnerId: 1234567890, name: orphanRunner.instanceId, busy: true, status: 'online' },
+          });
+        }
+
+        // act
+        await scaleDown();
+
+        // assert
+        expect(mockUntagRunners).toHaveBeenCalledWith(orphanRunner.instanceId, [
+          { Key: 'ghr:orphan', Value: 'true' },
+        ]);
+        expect(mockTerminateRunners).not.toHaveBeenCalledWith(orphanRunner.instanceId);
+
+        // arrange
+        if (type === 'Repo') {
+          mockOctokit.actions.getSelfHostedRunnerForRepo.mockResolvedValueOnce({
+            data: { runnerId: 1234567890, name: orphanRunner.instanceId, busy: true, status: 'offline' },
+          });
+        } else {
+          mockOctokit.actions.getSelfHostedRunnerForOrg.mockResolvedValueOnce({
+            data: { runnerId: 1234567890, name: orphanRunner.instanceId, busy: true, status: 'offline' },
+          });
+        }
+
+        // act
+        await scaleDown();
+
+        // assert
+        expect(mockTerminateRunners).toHaveBeenCalledWith(orphanRunner.instanceId);
+      });
+
       it(`Should ignore errors when termination orphan fails.`, async () => {
         // setup
         const orphanRunner = createRunnerTestData('orphan-1', type, MINIMUM_BOOT_TIME + 1, false, true, true);
@@ -625,6 +673,7 @@ function createRunnerTestData(
   orphan: boolean,
   shouldBeTerminated: boolean,
   owner?: string,
+  runnerId?: number,
 ): RunnerTestItem {
   return {
     instanceId: `i-${name}-${type.toLowerCase()}`,
@@ -638,5 +687,6 @@ function createRunnerTestData(
     registered,
     orphan,
     shouldBeTerminated,
+    runnerId: runnerId !== undefined ? String(runnerId) : undefined,
   };
 }
diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-down.ts b/lambdas/functions/control-plane/src/scale-runners/scale-down.ts
index 10d5615b43..71820d9d5e 100644
--- a/lambdas/functions/control-plane/src/scale-runners/scale-down.ts
+++ b/lambdas/functions/control-plane/src/scale-runners/scale-down.ts
@@ -3,8 +3,8 @@ import { createChildLogger } from '@aws-github-runner/aws-powertools-util';
 import moment from 'moment';
 
 import { createGithubAppAuth, createGithubInstallationAuth, createOctokitClient } from '../github/auth';
-import { bootTimeExceeded, listEC2Runners, tag, terminateRunner } from './../aws/runners';
-import { RunnerInfo, RunnerList } from './../aws/runners.d';
+import { bootTimeExceeded, listEC2Runners, tag, untag, terminateRunner } from './../aws/runners';
+import { RunnerInfo, RunnerList, RunnerState } from './../aws/runners.d';
 import { GhRunners, githubCache } from './cache';
 import { ScalingDownConfig, getEvictionStrategy, getIdleRunnerCount } from './scale-down-config';
 import { metricGitHubAppRateLimit } from '../github/rate-limit';
@@ -46,7 +46,11 @@ async function getOrCreateOctokit(runner: RunnerInfo): Promise<Octokit> {
   return octokit;
 }
 
-async function getGitHubRunnerBusyState(client: Octokit, ec2runner: RunnerInfo, runnerId: number): Promise<boolean> {
+async function getGitHubSelfHostedRunnerState(
+  client: Octokit,
+  ec2runner: RunnerInfo,
+  runnerId: number,
+): Promise<RunnerState> {
   const state =
     ec2runner.type === 'Org'
       ? await client.actions.getSelfHostedRunnerForOrg({
@@ -59,11 +63,12 @@ async function getGitHubRunnerBusyState(client: Octokit, ec2runner: RunnerInfo,
           repo: ec2runner.owner.split('/')[1],
         });
 
-  logger.info(`Runner '${ec2runner.instanceId}' - GitHub Runner ID '${runnerId}' - Busy: ${state.data.busy}`);
-
-  metricGitHubAppRateLimit(state.headers);
+  return { id: state.data.id, name: state.data.name, busy: state.data.busy, status: state.data.status };
+}
 
-  return state.data.busy;
+async function getGitHubRunnerBusyState(client: Octokit, ec2runner: RunnerInfo, runnerId: number): Promise<boolean> {
+  const state = await getGitHubSelfHostedRunnerState(client, ec2runner, runnerId);
+  return state.busy;
 }
 
 async function listGitHubRunners(runner: RunnerInfo): Promise<GhRunners> {
@@ -205,13 +210,36 @@ async function terminateOrphan(environment: string): Promise<void> {
     const orphanRunners = await listEC2Runners({ environment, orphan: true });
 
     for (const runner of orphanRunners) {
-      logger.info(`Terminating orphan runner '${runner.instanceId}'`);
-      await terminateRunner(runner.instanceId).catch((e) => {
-        logger.error(`Failed to terminate orphan runner '${runner.instanceId}'`, { error: e });
-      });
+      // do we have a valid runnerId? then we are in a Jit Runner scenario else, use old method
+      logger.info(`Runner var us '${JSON.stringify(runner)}' `);
+      if (runner.runnerId) {
+        logger.info(`Runner '${runner.instanceId}' is orphan, but has a runnerId.`);
+        // build a runner instance
+        const client = await getOrCreateOctokit(runner as RunnerInfo);
+        const runnerId = parseInt(runner.runnerId);
+        const ec2Instance = runner as RunnerInfo;
+        const state = await getGitHubSelfHostedRunnerState(client, ec2Instance, runnerId);
+        logger.info(`Runner is currently '${runner.instanceId}' state: ${JSON.stringify(state)}`);
+        if (state.status === 'online' && state.busy) {
+          logger.info(`Runner '${runner.instanceId}' is orphan, but is online and busy.`);
+          await untag(runner.instanceId, [{ Key: 'ghr:orphan', Value: 'true' }]);
+        } else if (state.status === 'offline') {
+          logger.warn(`Runner '${runner.instanceId}' is orphan.`);
+          logger.info(`Terminating orphan runner '${runner.instanceId}'`);
+          await terminateRunner(runner.instanceId).catch((e) => {
+            logger.error(`Failed to terminate orphan runner '${runner.instanceId}'`, { error: e });
+          });
+        }
+      } else {
+        logger.warn(`Runner '${runner.instanceId}' is orphan, but no runnerId found.`);
+        logger.info(`Terminating orphan runner '${runner.instanceId}'`);
+        await terminateRunner(runner.instanceId).catch((e) => {
+          logger.error(`Failed to terminate orphan runner '${runner.instanceId}'`, { error: e });
+        });
+      }
     }
   } catch (e) {
-    logger.warn(`Failure during orphan runner termination.`, { error: e });
+    logger.warn(`Failure during orphan termination processing.`, { error: e });
   }
 }
 
diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-up.test.ts b/lambdas/functions/control-plane/src/scale-runners/scale-up.test.ts
index 0611a6e697..14c0a0422e 100644
--- a/lambdas/functions/control-plane/src/scale-runners/scale-up.test.ts
+++ b/lambdas/functions/control-plane/src/scale-runners/scale-up.test.ts
@@ -41,6 +41,7 @@ vi.mock('@octokit/rest', () => ({
 vi.mock('./../aws/runners', async () => ({
   createRunner: vi.fn(),
   listEC2Runners: vi.fn(),
+  tag: vi.fn(),
 }));
 
 vi.mock('./../github/auth', async () => ({
@@ -645,7 +646,7 @@ describe('scaleUp with public GH', () => {
       });
     });
 
-    it('JIT config is ingored for non-ephemeral runners.', async () => {
+    it('JIT config is ignored for non-ephemeral runners.', async () => {
       process.env.ENABLE_EPHEMERAL_RUNNERS = 'false';
       process.env.ENABLE_JIT_CONFIG = 'true';
       process.env.ENABLE_JOB_QUEUED_CHECK = 'false';
@@ -1008,11 +1009,13 @@ function defaultOctokitMockImpl() {
   ]);
   mockOctokit.actions.generateRunnerJitconfigForOrg.mockImplementation(() => ({
     data: {
+      runner: { id: 9876543210 },
       encoded_jit_config: 'TEST_JIT_CONFIG_ORG',
     },
   }));
   mockOctokit.actions.generateRunnerJitconfigForRepo.mockImplementation(() => ({
     data: {
+      runner: { id: 9876543210 },
       encoded_jit_config: 'TEST_JIT_CONFIG_REPO',
     },
   }));
diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-up.ts b/lambdas/functions/control-plane/src/scale-runners/scale-up.ts
index 08d16d682a..faf9e4aece 100644
--- a/lambdas/functions/control-plane/src/scale-runners/scale-up.ts
+++ b/lambdas/functions/control-plane/src/scale-runners/scale-up.ts
@@ -4,11 +4,12 @@ import { getParameter, putParameter } from '@aws-github-runner/aws-ssm-util';
 import yn from 'yn';
 
 import { createGithubAppAuth, createGithubInstallationAuth, createOctokitClient } from '../github/auth';
-import { createRunner, listEC2Runners } from './../aws/runners';
+import { createRunner, listEC2Runners, tag } from './../aws/runners';
 import { RunnerInputParameters } from './../aws/runners.d';
 import ScaleError from './ScaleError';
 import { publishRetryMessage } from './job-retry';
 import { metricGitHubAppRateLimit } from '../github/rate-limit';
+import { run } from '../local-down';
 
 const logger = createChildLogger('scale-up');
 
@@ -416,6 +417,15 @@ async function createRegistrationTokenConfig(
   }
 }
 
+async function addGhRunnerIdToEC2InstanceTag(instanceId: string, runnerId: string): Promise<void> {
+  try {
+    await tag(instanceId, [{ Key: 'ghr:githubrunnerid', Value: runnerId }]);
+    logger.info(`Runner '${instanceId}' marked with ${runnerId}.`);
+  } catch (e) {
+    logger.error(`Failed to mark runner '${instanceId}' with ${runnerId}.`, { error: e });
+  }
+}
+
 async function createJitConfig(githubRunnerConfig: CreateGitHubRunnerConfig, instances: string[], ghClient: Octokit) {
   const runnerGroupId = await getRunnerGroupId(githubRunnerConfig, ghClient);
   const { isDelay, delay } = addDelay(instances);
@@ -449,6 +459,12 @@ async function createJitConfig(githubRunnerConfig: CreateGitHubRunnerConfig, ins
 
     metricGitHubAppRateLimit(runnerConfig.headers);
 
+    // tag the EC2 instance with the Github runner id
+    logger.debug('Tagging instance with GitHub runner ID', {
+      runnerId: runnerConfig.data.runner.id,
+    });
+    await addGhRunnerIdToEC2InstanceTag(instance, runnerConfig.data.runner.id.toString());
+
     // store jit config in ssm parameter store
     logger.debug('Runner JIT config for ephemeral runner generated.', {
       instance: instance,

From 14df9fe6c22371a321f5888551d44e04922888ad Mon Sep 17 00:00:00 2001
From: Stuart Pearson <1926002+stuartp44@users.noreply.github.com>
Date: Thu, 22 May 2025 09:22:04 +0200
Subject: [PATCH 02/23] fix(tests): improve clarity of orphaned runner
 untagging test description

---
 .../control-plane/src/scale-runners/scale-down.test.ts          | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-down.test.ts b/lambdas/functions/control-plane/src/scale-runners/scale-down.test.ts
index 7823d0ec76..212a4ebe0d 100644
--- a/lambdas/functions/control-plane/src/scale-runners/scale-down.test.ts
+++ b/lambdas/functions/control-plane/src/scale-runners/scale-down.test.ts
@@ -351,7 +351,7 @@ describe('Scale down runners', () => {
         checkNonTerminated(runners);
       });
 
-    it('Should test if orphaned runner untag if online and busy, else terminate (JIT)', async () => {
+    it('Should test if orphaned runner, untag if online and busy, else terminate (JIT)', async () => {
         // arrange
         const orphanRunner = createRunnerTestData('orphan-jit', type, MINIMUM_BOOT_TIME + 1, false, true, false, undefined, 1234567890);
         const runners = [orphanRunner];

From 9d7c89a339793f0ee2353e2033a5d03a6181d04a Mon Sep 17 00:00:00 2001
From: Stuart Pearson <1926002+stuartp44@users.noreply.github.com>
Date: Thu, 22 May 2025 10:40:07 +0200
Subject: [PATCH 03/23] fmt

---
 .../src/scale-runners/scale-down.test.ts        | 17 ++++++++++++-----
 1 file changed, 12 insertions(+), 5 deletions(-)

diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-down.test.ts b/lambdas/functions/control-plane/src/scale-runners/scale-down.test.ts
index 212a4ebe0d..318808b95c 100644
--- a/lambdas/functions/control-plane/src/scale-runners/scale-down.test.ts
+++ b/lambdas/functions/control-plane/src/scale-runners/scale-down.test.ts
@@ -351,9 +351,18 @@ describe('Scale down runners', () => {
         checkNonTerminated(runners);
       });
 
-    it('Should test if orphaned runner, untag if online and busy, else terminate (JIT)', async () => {
+      it('Should test if orphaned runner, untag if online and busy, else terminate (JIT)', async () => {
         // arrange
-        const orphanRunner = createRunnerTestData('orphan-jit', type, MINIMUM_BOOT_TIME + 1, false, true, false, undefined, 1234567890);
+        const orphanRunner = createRunnerTestData(
+          'orphan-jit',
+          type,
+          MINIMUM_BOOT_TIME + 1,
+          false,
+          true,
+          false,
+          undefined,
+          1234567890,
+        );
         const runners = [orphanRunner];
 
         mockGitHubRunners([]);
@@ -373,9 +382,7 @@ describe('Scale down runners', () => {
         await scaleDown();
 
         // assert
-        expect(mockUntagRunners).toHaveBeenCalledWith(orphanRunner.instanceId, [
-          { Key: 'ghr:orphan', Value: 'true' },
-        ]);
+        expect(mockUntagRunners).toHaveBeenCalledWith(orphanRunner.instanceId, [{ Key: 'ghr:orphan', Value: 'true' }]);
         expect(mockTerminateRunners).not.toHaveBeenCalledWith(orphanRunner.instanceId);
 
         // arrange

From e09337f34888361e7720c0465c176ede63ad3c78 Mon Sep 17 00:00:00 2001
From: Stuart Pearson <1926002+stuartp44@users.noreply.github.com>
Date: Thu, 22 May 2025 10:42:55 +0200
Subject: [PATCH 04/23] fix(scale-down): remove unnecessary logging of runner
 variable in terminateOrphan function

---
 lambdas/functions/control-plane/src/scale-runners/scale-down.ts | 1 -
 1 file changed, 1 deletion(-)

diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-down.ts b/lambdas/functions/control-plane/src/scale-runners/scale-down.ts
index 71820d9d5e..d384744a12 100644
--- a/lambdas/functions/control-plane/src/scale-runners/scale-down.ts
+++ b/lambdas/functions/control-plane/src/scale-runners/scale-down.ts
@@ -211,7 +211,6 @@ async function terminateOrphan(environment: string): Promise<void> {
 
     for (const runner of orphanRunners) {
       // do we have a valid runnerId? then we are in a Jit Runner scenario else, use old method
-      logger.info(`Runner var us '${JSON.stringify(runner)}' `);
       if (runner.runnerId) {
         logger.info(`Runner '${runner.instanceId}' is orphan, but has a runnerId.`);
         // build a runner instance

From 9064512e58620cb0a06c3b5eec2df8fedf818142 Mon Sep 17 00:00:00 2001
From: Stuart Pearson <1926002+stuartp44@users.noreply.github.com>
Date: Thu, 22 May 2025 10:46:15 +0200
Subject: [PATCH 05/23] fix(scale-up): remove unused import of run function

---
 lambdas/functions/control-plane/src/scale-runners/scale-up.ts | 1 -
 1 file changed, 1 deletion(-)

diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-up.ts b/lambdas/functions/control-plane/src/scale-runners/scale-up.ts
index faf9e4aece..747417cb8f 100644
--- a/lambdas/functions/control-plane/src/scale-runners/scale-up.ts
+++ b/lambdas/functions/control-plane/src/scale-runners/scale-up.ts
@@ -9,7 +9,6 @@ import { RunnerInputParameters } from './../aws/runners.d';
 import ScaleError from './ScaleError';
 import { publishRetryMessage } from './job-retry';
 import { metricGitHubAppRateLimit } from '../github/rate-limit';
-import { run } from '../local-down';
 
 const logger = createChildLogger('scale-up');
 

From 716e0790fb557b2d3202ab0b51cdaf410788da84 Mon Sep 17 00:00:00 2001
From: Stuart Pearson <1926002+stuartp44@users.noreply.github.com>
Date: Fri, 23 May 2025 08:50:26 +0200
Subject: [PATCH 06/23] fix(scale-down): remove unused import of
 metricGitHubAppRateLimit

---
 lambdas/functions/control-plane/src/scale-runners/scale-down.ts | 1 -
 1 file changed, 1 deletion(-)

diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-down.ts b/lambdas/functions/control-plane/src/scale-runners/scale-down.ts
index d384744a12..0ae8a87ee2 100644
--- a/lambdas/functions/control-plane/src/scale-runners/scale-down.ts
+++ b/lambdas/functions/control-plane/src/scale-runners/scale-down.ts
@@ -7,7 +7,6 @@ import { bootTimeExceeded, listEC2Runners, tag, untag, terminateRunner } from '.
 import { RunnerInfo, RunnerList, RunnerState } from './../aws/runners.d';
 import { GhRunners, githubCache } from './cache';
 import { ScalingDownConfig, getEvictionStrategy, getIdleRunnerCount } from './scale-down-config';
-import { metricGitHubAppRateLimit } from '../github/rate-limit';
 import { getGitHubEnterpriseApiUrl } from './scale-up';
 
 const logger = createChildLogger('scale-down');

From a5fcc88e8d95e147cd8fb5c9824501087f76cdeb Mon Sep 17 00:00:00 2001
From: Stuart Pearson <1926002+stuartp44@users.noreply.github.com>
Date: Mon, 26 May 2025 11:34:36 +0200
Subject: [PATCH 07/23] Update
 lambdas/functions/control-plane/src/scale-runners/scale-down.ts

Co-authored-by: Niek Palm <npalm@users.noreply.github.com>
---
 lambdas/functions/control-plane/src/scale-runners/scale-down.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-down.ts b/lambdas/functions/control-plane/src/scale-runners/scale-down.ts
index 0ae8a87ee2..f48333e43b 100644
--- a/lambdas/functions/control-plane/src/scale-runners/scale-down.ts
+++ b/lambdas/functions/control-plane/src/scale-runners/scale-down.ts
@@ -211,7 +211,7 @@ async function terminateOrphan(environment: string): Promise<void> {
     for (const runner of orphanRunners) {
       // do we have a valid runnerId? then we are in a Jit Runner scenario else, use old method
       if (runner.runnerId) {
-        logger.info(`Runner '${runner.instanceId}' is orphan, but has a runnerId.`);
+        logger.debug(`Runner '${runner.instanceId}' is orphan, but has a runnerId.`);
         // build a runner instance
         const client = await getOrCreateOctokit(runner as RunnerInfo);
         const runnerId = parseInt(runner.runnerId);

From bb8ba2bf56af9d461eaec6befa5b68f38a7bcd69 Mon Sep 17 00:00:00 2001
From: Stuart Pearson <1926002+stuartp44@users.noreply.github.com>
Date: Mon, 26 May 2025 11:34:49 +0200
Subject: [PATCH 08/23] Update
 lambdas/functions/control-plane/src/scale-runners/scale-down.ts

Co-authored-by: Niek Palm <npalm@users.noreply.github.com>
---
 lambdas/functions/control-plane/src/scale-runners/scale-down.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-down.ts b/lambdas/functions/control-plane/src/scale-runners/scale-down.ts
index f48333e43b..9a02e79ec4 100644
--- a/lambdas/functions/control-plane/src/scale-runners/scale-down.ts
+++ b/lambdas/functions/control-plane/src/scale-runners/scale-down.ts
@@ -217,7 +217,7 @@ async function terminateOrphan(environment: string): Promise<void> {
         const runnerId = parseInt(runner.runnerId);
         const ec2Instance = runner as RunnerInfo;
         const state = await getGitHubSelfHostedRunnerState(client, ec2Instance, runnerId);
-        logger.info(`Runner is currently '${runner.instanceId}' state: ${JSON.stringify(state)}`);
+        logger.debug(`Runner is currently '${runner.instanceId}' state: ${JSON.stringify(state)}`);
         if (state.status === 'online' && state.busy) {
           logger.info(`Runner '${runner.instanceId}' is orphan, but is online and busy.`);
           await untag(runner.instanceId, [{ Key: 'ghr:orphan', Value: 'true' }]);

From 97de234cdaf3151e6f8a957d202a03cbd92b9f10 Mon Sep 17 00:00:00 2001
From: Stuart Pearson <1926002+stuartp44@users.noreply.github.com>
Date: Mon, 26 May 2025 11:35:34 +0200
Subject: [PATCH 09/23] Update
 lambdas/functions/control-plane/src/scale-runners/scale-up.ts

Co-authored-by: Niek Palm <npalm@users.noreply.github.com>
---
 lambdas/functions/control-plane/src/scale-runners/scale-up.ts | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-up.ts b/lambdas/functions/control-plane/src/scale-runners/scale-up.ts
index 747417cb8f..cc1aefc8c7 100644
--- a/lambdas/functions/control-plane/src/scale-runners/scale-up.ts
+++ b/lambdas/functions/control-plane/src/scale-runners/scale-up.ts
@@ -459,9 +459,6 @@ async function createJitConfig(githubRunnerConfig: CreateGitHubRunnerConfig, ins
     metricGitHubAppRateLimit(runnerConfig.headers);
 
     // tag the EC2 instance with the Github runner id
-    logger.debug('Tagging instance with GitHub runner ID', {
-      runnerId: runnerConfig.data.runner.id,
-    });
     await addGhRunnerIdToEC2InstanceTag(instance, runnerConfig.data.runner.id.toString());
 
     // store jit config in ssm parameter store

From 8b61d39203e6f4b92dcddc6109dc5c872ac5d603 Mon Sep 17 00:00:00 2001
From: Stuart Pearson <1926002+stuartp44@users.noreply.github.com>
Date: Mon, 26 May 2025 11:35:59 +0200
Subject: [PATCH 10/23] Update
 lambdas/functions/control-plane/src/scale-runners/scale-up.ts

Co-authored-by: Niek Palm <npalm@users.noreply.github.com>
---
 lambdas/functions/control-plane/src/scale-runners/scale-up.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-up.ts b/lambdas/functions/control-plane/src/scale-runners/scale-up.ts
index cc1aefc8c7..38485d48f5 100644
--- a/lambdas/functions/control-plane/src/scale-runners/scale-up.ts
+++ b/lambdas/functions/control-plane/src/scale-runners/scale-up.ts
@@ -418,7 +418,7 @@ async function createRegistrationTokenConfig(
 
 async function addGhRunnerIdToEC2InstanceTag(instanceId: string, runnerId: string): Promise<void> {
   try {
-    await tag(instanceId, [{ Key: 'ghr:githubrunnerid', Value: runnerId }]);
+    await tag(instanceId, [{ Key: 'ghr:github_runner_id', Value: runnerId }]);
     logger.info(`Runner '${instanceId}' marked with ${runnerId}.`);
   } catch (e) {
     logger.error(`Failed to mark runner '${instanceId}' with ${runnerId}.`, { error: e });

From 9883b0b8490cde1dcfdb4ba5a623c2fe693aa450 Mon Sep 17 00:00:00 2001
From: Stuart Pearson <1926002+stuartp44@users.noreply.github.com>
Date: Thu, 29 May 2025 09:03:34 +0100
Subject: [PATCH 11/23] Remove warning log for orphan runners without runnerId
 in scale-down function

---
 lambdas/functions/control-plane/src/scale-runners/scale-down.ts | 1 -
 1 file changed, 1 deletion(-)

diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-down.ts b/lambdas/functions/control-plane/src/scale-runners/scale-down.ts
index 9a02e79ec4..f797735d53 100644
--- a/lambdas/functions/control-plane/src/scale-runners/scale-down.ts
+++ b/lambdas/functions/control-plane/src/scale-runners/scale-down.ts
@@ -229,7 +229,6 @@ async function terminateOrphan(environment: string): Promise<void> {
           });
         }
       } else {
-        logger.warn(`Runner '${runner.instanceId}' is orphan, but no runnerId found.`);
         logger.info(`Terminating orphan runner '${runner.instanceId}'`);
         await terminateRunner(runner.instanceId).catch((e) => {
           logger.error(`Failed to terminate orphan runner '${runner.instanceId}'`, { error: e });

From 43468a738990da4e4efc936d0beed252e1ff8ad2 Mon Sep 17 00:00:00 2001
From: Stuart Pearson <1926002+stuartp44@users.noreply.github.com>
Date: Thu, 29 May 2025 09:04:58 +0100
Subject: [PATCH 12/23] Remove logging of runner ID marking in
 addGhRunnerIdToEC2InstanceTag function

---
 lambdas/functions/control-plane/src/scale-runners/scale-up.ts | 1 -
 1 file changed, 1 deletion(-)

diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-up.ts b/lambdas/functions/control-plane/src/scale-runners/scale-up.ts
index 38485d48f5..62da8b79b8 100644
--- a/lambdas/functions/control-plane/src/scale-runners/scale-up.ts
+++ b/lambdas/functions/control-plane/src/scale-runners/scale-up.ts
@@ -419,7 +419,6 @@ async function createRegistrationTokenConfig(
 async function addGhRunnerIdToEC2InstanceTag(instanceId: string, runnerId: string): Promise<void> {
   try {
     await tag(instanceId, [{ Key: 'ghr:github_runner_id', Value: runnerId }]);
-    logger.info(`Runner '${instanceId}' marked with ${runnerId}.`);
   } catch (e) {
     logger.error(`Failed to mark runner '${instanceId}' with ${runnerId}.`, { error: e });
   }

From bc995ef3e0fb301a8b357c517f7d6e01680c08b9 Mon Sep 17 00:00:00 2001
From: Stuart Pearson <1926002+stuartp44@users.noreply.github.com>
Date: Tue, 3 Jun 2025 15:05:02 +0100
Subject: [PATCH 13/23] readded metricGitHubAppRateLimit

---
 .../functions/control-plane/src/scale-runners/scale-down.ts   | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-down.ts b/lambdas/functions/control-plane/src/scale-runners/scale-down.ts
index f797735d53..54aa24a4d8 100644
--- a/lambdas/functions/control-plane/src/scale-runners/scale-down.ts
+++ b/lambdas/functions/control-plane/src/scale-runners/scale-down.ts
@@ -7,10 +7,12 @@ import { bootTimeExceeded, listEC2Runners, tag, untag, terminateRunner } from '.
 import { RunnerInfo, RunnerList, RunnerState } from './../aws/runners.d';
 import { GhRunners, githubCache } from './cache';
 import { ScalingDownConfig, getEvictionStrategy, getIdleRunnerCount } from './scale-down-config';
+import { metricGitHubAppRateLimit } from '../github/rate-limit';
 import { getGitHubEnterpriseApiUrl } from './scale-up';
 
 const logger = createChildLogger('scale-down');
 
+
 async function getOrCreateOctokit(runner: RunnerInfo): Promise<Octokit> {
   const key = runner.owner;
   const cachedOctokit = githubCache.clients.get(key);
@@ -67,6 +69,8 @@ async function getGitHubSelfHostedRunnerState(
 
 async function getGitHubRunnerBusyState(client: Octokit, ec2runner: RunnerInfo, runnerId: number): Promise<boolean> {
   const state = await getGitHubSelfHostedRunnerState(client, ec2runner, runnerId);
+  logger.info(`Runner '${ec2runner.instanceId}' - GitHub Runner ID '${runnerId}' - Busy: ${state.data.busy}`);
+  metricGitHubAppRateLimit(state.headers);
   return state.busy;
 }
 

From 6e1c72ca3d0826535bba182dd7a088ed25866d66 Mon Sep 17 00:00:00 2001
From: Stuart Pearson <1926002+stuartp44@users.noreply.github.com>
Date: Tue, 3 Jun 2025 16:47:52 +0100
Subject: [PATCH 14/23] Refactor runner interfaces: remove RunnerState
 interface and update imports in scale-down.ts

---
 .../functions/control-plane/src/aws/runners.d.ts    |  7 -------
 .../control-plane/src/scale-runners/scale-down.ts   | 13 ++++++++++++-
 2 files changed, 12 insertions(+), 8 deletions(-)

diff --git a/lambdas/functions/control-plane/src/aws/runners.d.ts b/lambdas/functions/control-plane/src/aws/runners.d.ts
index 752ff07edd..72ff9e3e1a 100644
--- a/lambdas/functions/control-plane/src/aws/runners.d.ts
+++ b/lambdas/functions/control-plane/src/aws/runners.d.ts
@@ -13,13 +13,6 @@ export interface RunnerList {
   runnerId?: string;
 }
 
-export interface RunnerState {
-  id: number;
-  name: string;
-  busy: boolean;
-  status: string;
-}
-
 export interface RunnerInfo {
   instanceId: string;
   launchTime?: Date;
diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-down.ts b/lambdas/functions/control-plane/src/scale-runners/scale-down.ts
index 54aa24a4d8..df1527e6af 100644
--- a/lambdas/functions/control-plane/src/scale-runners/scale-down.ts
+++ b/lambdas/functions/control-plane/src/scale-runners/scale-down.ts
@@ -4,14 +4,25 @@ import moment from 'moment';
 
 import { createGithubAppAuth, createGithubInstallationAuth, createOctokitClient } from '../github/auth';
 import { bootTimeExceeded, listEC2Runners, tag, untag, terminateRunner } from './../aws/runners';
-import { RunnerInfo, RunnerList, RunnerState } from './../aws/runners.d';
+import { RunnerInfo, RunnerList } from './../aws/runners.d';
 import { GhRunners, githubCache } from './cache';
 import { ScalingDownConfig, getEvictionStrategy, getIdleRunnerCount } from './scale-down-config';
 import { metricGitHubAppRateLimit } from '../github/rate-limit';
 import { getGitHubEnterpriseApiUrl } from './scale-up';
+import { GetResponseDataTypeFromEndpointMethod } from '@octokit/types/dist-types/GetResponseTypeFromEndpointMethod';
 
 const logger = createChildLogger('scale-down');
 
+type OrgRunnerList = GetResponseDataTypeFromEndpointMethod<
+  typeof Octokit.prototype.actions.listSelfHostedRunnersForOrg
+>;
+
+type RepoRunnerList = GetResponseDataTypeFromEndpointMethod<
+  typeof Octokit.prototype.actions.listSelfHostedRunnersForRepo
+>;
+
+// Derive the shape of an individual runner
+type RunnerState = (OrgRunnerList | RepoRunnerList)['runners'][number];
 
 async function getOrCreateOctokit(runner: RunnerInfo): Promise<Octokit> {
   const key = runner.owner;

From 097c14d8c5b095777e24dd44942ea73942a9076a Mon Sep 17 00:00:00 2001
From: Stuart Pearson <1926002+stuartp44@users.noreply.github.com>
Date: Wed, 4 Jun 2025 10:13:47 +0100
Subject: [PATCH 15/23] Add headers to runner state return and update logging
 for busy state

---
 .../functions/control-plane/src/scale-runners/scale-down.ts   | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-down.ts b/lambdas/functions/control-plane/src/scale-runners/scale-down.ts
index df1527e6af..814c6928c0 100644
--- a/lambdas/functions/control-plane/src/scale-runners/scale-down.ts
+++ b/lambdas/functions/control-plane/src/scale-runners/scale-down.ts
@@ -75,12 +75,12 @@ async function getGitHubSelfHostedRunnerState(
           repo: ec2runner.owner.split('/')[1],
         });
 
-  return { id: state.data.id, name: state.data.name, busy: state.data.busy, status: state.data.status };
+  return { id: state.data.id, name: state.data.name, busy: state.data.busy, status: state.data.status, headers: state.headers };
 }
 
 async function getGitHubRunnerBusyState(client: Octokit, ec2runner: RunnerInfo, runnerId: number): Promise<boolean> {
   const state = await getGitHubSelfHostedRunnerState(client, ec2runner, runnerId);
-  logger.info(`Runner '${ec2runner.instanceId}' - GitHub Runner ID '${runnerId}' - Busy: ${state.data.busy}`);
+  logger.info(`Runner '${ec2runner.instanceId}' - GitHub Runner ID '${runnerId}' - Busy: ${state.busy}`);
   metricGitHubAppRateLimit(state.headers);
   return state.busy;
 }

From 971ec2d8b1deffbb2d01146aa5e4d7ec16c67877 Mon Sep 17 00:00:00 2001
From: Stuart Pearson <1926002+stuartp44@users.noreply.github.com>
Date: Wed, 4 Jun 2025 10:14:25 +0100
Subject: [PATCH 16/23] Remove redundant comment describing RunnerState type

---
 lambdas/functions/control-plane/src/scale-runners/scale-down.ts | 1 -
 1 file changed, 1 deletion(-)

diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-down.ts b/lambdas/functions/control-plane/src/scale-runners/scale-down.ts
index 814c6928c0..631074f8a5 100644
--- a/lambdas/functions/control-plane/src/scale-runners/scale-down.ts
+++ b/lambdas/functions/control-plane/src/scale-runners/scale-down.ts
@@ -21,7 +21,6 @@ type RepoRunnerList = GetResponseDataTypeFromEndpointMethod<
   typeof Octokit.prototype.actions.listSelfHostedRunnersForRepo
 >;
 
-// Derive the shape of an individual runner
 type RunnerState = (OrgRunnerList | RepoRunnerList)['runners'][number];
 
 async function getOrCreateOctokit(runner: RunnerInfo): Promise<Octokit> {

From 5a275e5bf706b519dc84fb07819257a642702cdf Mon Sep 17 00:00:00 2001
From: Stuart Pearson <1926002+stuartp44@users.noreply.github.com>
Date: Wed, 4 Jun 2025 10:25:04 +0100
Subject: [PATCH 17/23] Implement last chance check for orphan runners and
 refactor termination logic

---
 .../src/scale-runners/scale-down.ts           | 35 ++++++++++---------
 1 file changed, 19 insertions(+), 16 deletions(-)

diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-down.ts b/lambdas/functions/control-plane/src/scale-runners/scale-down.ts
index 631074f8a5..4e5cd38b17 100644
--- a/lambdas/functions/control-plane/src/scale-runners/scale-down.ts
+++ b/lambdas/functions/control-plane/src/scale-runners/scale-down.ts
@@ -218,6 +218,24 @@ async function markOrphan(instanceId: string): Promise<void> {
   }
 }
 
+async function lastChanceCheckOrphanRunner(runner: RunnerList): Promise<void> {
+  const client = await getOrCreateOctokit(runner as RunnerInfo);
+  const runnerId = parseInt(runner.runnerId);
+  const ec2Instance = runner as RunnerInfo;
+  const state = await getGitHubSelfHostedRunnerState(client, ec2Instance, runnerId);
+  logger.debug(`Runner is currently '${runner.instanceId}' state: ${JSON.stringify(state)}`);
+  if (state.status === 'online' && state.busy) {
+    logger.info(`Runner '${runner.instanceId}' is orphan, but is online and busy.`);
+    await untag(runner.instanceId, [{ Key: 'ghr:orphan', Value: 'true' }]);
+  } else if (state.status === 'offline') {
+    logger.warn(`Runner '${runner.instanceId}' is orphan.`);
+    logger.info(`Terminating orphan runner '${runner.instanceId}'`);
+    await terminateRunner(runner.instanceId).catch((e) => {
+      logger.error(`Failed to terminate orphan runner '${runner.instanceId}'`, { error: e });
+    });
+  }
+}
+
 async function terminateOrphan(environment: string): Promise<void> {
   try {
     const orphanRunners = await listEC2Runners({ environment, orphan: true });
@@ -226,22 +244,7 @@ async function terminateOrphan(environment: string): Promise<void> {
       // do we have a valid runnerId? then we are in a Jit Runner scenario else, use old method
       if (runner.runnerId) {
         logger.debug(`Runner '${runner.instanceId}' is orphan, but has a runnerId.`);
-        // build a runner instance
-        const client = await getOrCreateOctokit(runner as RunnerInfo);
-        const runnerId = parseInt(runner.runnerId);
-        const ec2Instance = runner as RunnerInfo;
-        const state = await getGitHubSelfHostedRunnerState(client, ec2Instance, runnerId);
-        logger.debug(`Runner is currently '${runner.instanceId}' state: ${JSON.stringify(state)}`);
-        if (state.status === 'online' && state.busy) {
-          logger.info(`Runner '${runner.instanceId}' is orphan, but is online and busy.`);
-          await untag(runner.instanceId, [{ Key: 'ghr:orphan', Value: 'true' }]);
-        } else if (state.status === 'offline') {
-          logger.warn(`Runner '${runner.instanceId}' is orphan.`);
-          logger.info(`Terminating orphan runner '${runner.instanceId}'`);
-          await terminateRunner(runner.instanceId).catch((e) => {
-            logger.error(`Failed to terminate orphan runner '${runner.instanceId}'`, { error: e });
-          });
-        }
+        await lastChanceCheckOrphanRunner(runner);
       } else {
         logger.info(`Terminating orphan runner '${runner.instanceId}'`);
         await terminateRunner(runner.instanceId).catch((e) => {

From 9f59abe1668ef10ffcab27b301468d5dfa879656 Mon Sep 17 00:00:00 2001
From: Stuart Pearson <1926002+stuartp44@users.noreply.github.com>
Date: Wed, 4 Jun 2025 10:32:49 +0100
Subject: [PATCH 18/23] Format return object in getGitHubSelfHostedRunnerState
 for improved readability

---
 .../control-plane/src/scale-runners/scale-down.ts         | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-down.ts b/lambdas/functions/control-plane/src/scale-runners/scale-down.ts
index 4e5cd38b17..b257dd14f5 100644
--- a/lambdas/functions/control-plane/src/scale-runners/scale-down.ts
+++ b/lambdas/functions/control-plane/src/scale-runners/scale-down.ts
@@ -74,7 +74,13 @@ async function getGitHubSelfHostedRunnerState(
           repo: ec2runner.owner.split('/')[1],
         });
 
-  return { id: state.data.id, name: state.data.name, busy: state.data.busy, status: state.data.status, headers: state.headers };
+  return {
+    id: state.data.id,
+    name: state.data.name,
+    busy: state.data.busy,
+    status: state.data.status,
+    headers: state.headers,
+  };
 }
 
 async function getGitHubRunnerBusyState(client: Octokit, ec2runner: RunnerInfo, runnerId: number): Promise<boolean> {

From 65c0b0ee86cd46bb0ac9a92b8ddfefa5cd9e9cb8 Mon Sep 17 00:00:00 2001
From: Stuart Pearson <1926002+stuartp44@users.noreply.github.com>
Date: Wed, 4 Jun 2025 10:57:24 +0100
Subject: [PATCH 19/23] Refactor runner state types to use Endpoints for
 improved type safety and clarity

---
 .../src/scale-runners/scale-down.ts           | 23 ++++++++-----------
 1 file changed, 10 insertions(+), 13 deletions(-)

diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-down.ts b/lambdas/functions/control-plane/src/scale-runners/scale-down.ts
index b257dd14f5..41d0e51495 100644
--- a/lambdas/functions/control-plane/src/scale-runners/scale-down.ts
+++ b/lambdas/functions/control-plane/src/scale-runners/scale-down.ts
@@ -1,4 +1,5 @@
 import { Octokit } from '@octokit/rest';
+import { Endpoints } from '@octokit/types';
 import { createChildLogger } from '@aws-github-runner/aws-powertools-util';
 import moment from 'moment';
 
@@ -9,19 +10,12 @@ import { GhRunners, githubCache } from './cache';
 import { ScalingDownConfig, getEvictionStrategy, getIdleRunnerCount } from './scale-down-config';
 import { metricGitHubAppRateLimit } from '../github/rate-limit';
 import { getGitHubEnterpriseApiUrl } from './scale-up';
-import { GetResponseDataTypeFromEndpointMethod } from '@octokit/types/dist-types/GetResponseTypeFromEndpointMethod';
 
 const logger = createChildLogger('scale-down');
 
-type OrgRunnerList = GetResponseDataTypeFromEndpointMethod<
-  typeof Octokit.prototype.actions.listSelfHostedRunnersForOrg
->;
-
-type RepoRunnerList = GetResponseDataTypeFromEndpointMethod<
-  typeof Octokit.prototype.actions.listSelfHostedRunnersForRepo
->;
-
-type RunnerState = (OrgRunnerList | RepoRunnerList)['runners'][number];
+type OrgRunnerList = Endpoints["GET /orgs/{org}/actions/runners"]["response"]["data"]["runners"];
+type RepoRunnerList = Endpoints["GET /repos/{owner}/{repo}/actions/runners"]["response"]["data"]["runners"];
+type RunnerState = OrgRunnerList[number] | RepoRunnerList[number];
 
 async function getOrCreateOctokit(runner: RunnerInfo): Promise<Octokit> {
   const key = runner.owner;
@@ -73,20 +67,23 @@ async function getGitHubSelfHostedRunnerState(
           owner: ec2runner.owner.split('/')[0],
           repo: ec2runner.owner.split('/')[1],
         });
+        metricGitHubAppRateLimit(state.headers);
 
   return {
     id: state.data.id,
     name: state.data.name,
     busy: state.data.busy,
     status: state.data.status,
-    headers: state.headers,
+    os: state.data.os,
+    labels: state.data.labels,
+    runner_group_id: state.data.runner_group_id,
+    ephemeral: state.data.ephemeral,
   };
 }
 
 async function getGitHubRunnerBusyState(client: Octokit, ec2runner: RunnerInfo, runnerId: number): Promise<boolean> {
   const state = await getGitHubSelfHostedRunnerState(client, ec2runner, runnerId);
   logger.info(`Runner '${ec2runner.instanceId}' - GitHub Runner ID '${runnerId}' - Busy: ${state.busy}`);
-  metricGitHubAppRateLimit(state.headers);
   return state.busy;
 }
 
@@ -226,7 +223,7 @@ async function markOrphan(instanceId: string): Promise<void> {
 
 async function lastChanceCheckOrphanRunner(runner: RunnerList): Promise<void> {
   const client = await getOrCreateOctokit(runner as RunnerInfo);
-  const runnerId = parseInt(runner.runnerId);
+  const runnerId = parseInt(runner.runnerId || '0');
   const ec2Instance = runner as RunnerInfo;
   const state = await getGitHubSelfHostedRunnerState(client, ec2Instance, runnerId);
   logger.debug(`Runner is currently '${runner.instanceId}' state: ${JSON.stringify(state)}`);

From 8ead598a3fd69152ada71863e2f4df60c2546191 Mon Sep 17 00:00:00 2001
From: Stuart Pearson <1926002+stuartp44@users.noreply.github.com>
Date: Wed, 4 Jun 2025 10:58:51 +0100
Subject: [PATCH 20/23] Fix formatting of type definitions and adjust
 indentation in getGitHubSelfHostedRunnerState for consistency

---
 .../functions/control-plane/src/scale-runners/scale-down.ts | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-down.ts b/lambdas/functions/control-plane/src/scale-runners/scale-down.ts
index 41d0e51495..1c8955d268 100644
--- a/lambdas/functions/control-plane/src/scale-runners/scale-down.ts
+++ b/lambdas/functions/control-plane/src/scale-runners/scale-down.ts
@@ -13,8 +13,8 @@ import { getGitHubEnterpriseApiUrl } from './scale-up';
 
 const logger = createChildLogger('scale-down');
 
-type OrgRunnerList = Endpoints["GET /orgs/{org}/actions/runners"]["response"]["data"]["runners"];
-type RepoRunnerList = Endpoints["GET /repos/{owner}/{repo}/actions/runners"]["response"]["data"]["runners"];
+type OrgRunnerList = Endpoints['GET /orgs/{org}/actions/runners']['response']['data']['runners'];
+type RepoRunnerList = Endpoints['GET /repos/{owner}/{repo}/actions/runners']['response']['data']['runners'];
 type RunnerState = OrgRunnerList[number] | RepoRunnerList[number];
 
 async function getOrCreateOctokit(runner: RunnerInfo): Promise<Octokit> {
@@ -67,7 +67,7 @@ async function getGitHubSelfHostedRunnerState(
           owner: ec2runner.owner.split('/')[0],
           repo: ec2runner.owner.split('/')[1],
         });
-        metricGitHubAppRateLimit(state.headers);
+  metricGitHubAppRateLimit(state.headers);
 
   return {
     id: state.data.id,

From ab5b6b0b8b708eb8289f77396a55d6de6c4ce2c6 Mon Sep 17 00:00:00 2001
From: Stuart Pearson <1926002+stuartp44@users.noreply.github.com>
Date: Wed, 4 Jun 2025 11:49:48 +0100
Subject: [PATCH 21/23] Update
 lambdas/functions/control-plane/src/aws/runners.ts

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
---
 lambdas/functions/control-plane/src/aws/runners.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lambdas/functions/control-plane/src/aws/runners.ts b/lambdas/functions/control-plane/src/aws/runners.ts
index c8fb1465f7..6779dd39d2 100644
--- a/lambdas/functions/control-plane/src/aws/runners.ts
+++ b/lambdas/functions/control-plane/src/aws/runners.ts
@@ -92,7 +92,7 @@ function getRunnerInfo(runningInstances: DescribeInstancesResult) {
             repo: i.Tags?.find((e) => e.Key === 'ghr:Repo')?.Value as string,
             org: i.Tags?.find((e) => e.Key === 'ghr:Org')?.Value as string,
             orphan: i.Tags?.find((e) => e.Key === 'ghr:orphan')?.Value === 'true',
-            runnerId: i.Tags?.find((e) => e.Key === 'ghr:githubrunnerid')?.Value as string,
+            runnerId: i.Tags?.find((e) => e.Key === 'ghr:github_runner_id')?.Value as string,
           });
         }
       }

From 0b462bbab9ad48084fa7d9696c1c96ccd6d54c2d Mon Sep 17 00:00:00 2001
From: Stuart Pearson <1926002+stuartp44@users.noreply.github.com>
Date: Wed, 4 Jun 2025 11:50:10 +0100
Subject: [PATCH 22/23] Update
 lambdas/functions/control-plane/src/scale-runners/scale-down.test.ts

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
---
 .../control-plane/src/scale-runners/scale-down.test.ts        | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-down.test.ts b/lambdas/functions/control-plane/src/scale-runners/scale-down.test.ts
index 318808b95c..8dd25323a6 100644
--- a/lambdas/functions/control-plane/src/scale-runners/scale-down.test.ts
+++ b/lambdas/functions/control-plane/src/scale-runners/scale-down.test.ts
@@ -370,11 +370,11 @@ describe('Scale down runners', () => {
 
         if (type === 'Repo') {
           mockOctokit.actions.getSelfHostedRunnerForRepo.mockResolvedValueOnce({
-            data: { runnerId: 1234567890, name: orphanRunner.instanceId, busy: true, status: 'online' },
+            data: { id: 1234567890, name: orphanRunner.instanceId, busy: true, status: 'online' },
           });
         } else {
           mockOctokit.actions.getSelfHostedRunnerForOrg.mockResolvedValueOnce({
-            data: { runnerId: 1234567890, name: orphanRunner.instanceId, busy: true, status: 'online' },
+            data: { id: 1234567890, name: orphanRunner.instanceId, busy: true, status: 'online' },
           });
         }
 

From 102edf0afcba291f2e6ec288c3a432cb1628a915 Mon Sep 17 00:00:00 2001
From: Stuart Pearson <1926002+stuartp44@users.noreply.github.com>
Date: Wed, 4 Jun 2025 11:52:43 +0100
Subject: [PATCH 23/23] Fix typo in key for GitHub runner ID in mock running
 instances

---
 lambdas/functions/control-plane/src/aws/runners.test.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lambdas/functions/control-plane/src/aws/runners.test.ts b/lambdas/functions/control-plane/src/aws/runners.test.ts
index 008f29433f..9d218605bb 100644
--- a/lambdas/functions/control-plane/src/aws/runners.test.ts
+++ b/lambdas/functions/control-plane/src/aws/runners.test.ts
@@ -67,7 +67,7 @@ const mockRunningInstancesJit: DescribeInstancesResult = {
             { Key: 'ghr:created_by', Value: 'scale-up-lambda' },
             { Key: 'ghr:Type', Value: 'Org' },
             { Key: 'ghr:Owner', Value: 'CoderToCat' },
-            { Key: 'ghr:githubrunnerid', Value: '9876543210' },
+            { Key: 'ghr:github_runner_id', Value: '9876543210' },
           ],
         },
       ],