Use script pod template custom resource to create pods and containers

Author: APErebus

source/Octopus.Tentacle/Kubernetes/KubernetesCustomResourceService.cs

@@ -0,0 +1,58 @@
+using System.Collections.Generic;
+using System.Linq;
+using System.Net;
+using System.Threading;
+using System.Threading.Tasks;
+using k8s;
+using k8s.Autorest;
+using k8s.Models;
+using Octopus.Tentacle.Core.Diagnostics;
+
+namespace Octopus.Tentacle.Kubernetes
+{
+    public interface IKubernetesCustomResourceService
+    {
+        Task<ScriptPodTemplateCustomResource?> GetOldestScriptPodTemplateCustomResource(CancellationToken cancellationToken);
+    }
+
+    public class KubernetesCustomResourceService : KubernetesService, IKubernetesCustomResourceService
+    {
+        public KubernetesCustomResourceService(IKubernetesClientConfigProvider configProvider, ISystemLog log)
+            : base(configProvider, log)
+        {
+        }
+
+        public async Task<ScriptPodTemplateCustomResource?> GetOldestScriptPodTemplateCustomResource(CancellationToken cancellationToken)
+        {
+            return await RetryPolicy.ExecuteAsync(async () =>
+            {
+                ScriptPodTemplateCustomResourceList resourceList;
+                try
+                {
+                    resourceList = await Client.CustomObjects.ListNamespacedCustomObjectAsync<ScriptPodTemplateCustomResourceList>(
+                        "agent.octopus.com",
+                        "v1beta1",
+                        KubernetesConfig.Namespace,
+                        "scriptpodtemplates",
+                        cancellationToken: cancellationToken);
+                }
+                catch (HttpOperationException opException)
+                    when (opException.Response.StatusCode == HttpStatusCode.NotFound)
+                {
+                    return null;
+                }
+
+                return resourceList.Items.OrderBy(r => r.Metadata.CreationTimestamp).FirstOrDefault();
+            });
+        }
+        
+        //These are only ever deserialized from JSON
+#pragma warning disable CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider adding the 'required' modifier or declaring as nullable.
+        class ScriptPodTemplateCustomResourceList : KubernetesObject
+        {
+            public V1ListMeta Metadata { get; set; }
+            public List<ScriptPodTemplateCustomResource> Items { get; set; }
+        }
+#pragma warning restore CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider adding the 'required' modifier or declaring as nullable.
+    }
+}

source/Octopus.Tentacle/Kubernetes/KubernetesModule.cs

@@ -34,6 +34,7 @@ protected override void Load(ContainerBuilder builder)
             builder.RegisterType<KubernetesPodContainerResolver>().As<IKubernetesPodContainerResolver>().SingleInstance();
             builder.RegisterType<KubernetesConfigMapService>().As<IKubernetesConfigMapService>().SingleInstance();
             builder.RegisterType<KubernetesSecretService>().As<IKubernetesSecretService>().SingleInstance();
+            builder.RegisterType<KubernetesCustomResourceService>().As<IKubernetesCustomResourceService>().SingleInstance();
 
             builder.RegisterType<KubernetesScriptPodCreator>().As<IKubernetesScriptPodCreator>().SingleInstance();
             builder.RegisterType<KubernetesRawScriptPodCreator>().As<IKubernetesRawScriptPodCreator>().SingleInstance();

source/Octopus.Tentacle/Kubernetes/KubernetesRawScriptPodCreator.cs

@@ -25,6 +25,7 @@ public KubernetesRawScriptPodCreator(
             IKubernetesPodService podService,
             IKubernetesPodMonitor podMonitor,
             IKubernetesSecretService secretService,
+            IKubernetesCustomResourceService customResourceService,
             IKubernetesPodContainerResolver containerResolver,
             IApplicationInstanceSelector appInstanceSelector,
             ISystemLog log,
@@ -33,31 +34,40 @@ public KubernetesRawScriptPodCreator(
             KubernetesPhysicalFileSystem kubernetesPhysicalFileSystem,
             IScriptPodLogEncryptionKeyProvider scriptPodLogEncryptionKeyProvider,
             ScriptIsolationMutex scriptIsolationMutex)
-            : base(podService, podMonitor, secretService, containerResolver, appInstanceSelector, log, scriptLogProvider, homeConfiguration, kubernetesPhysicalFileSystem, scriptPodLogEncryptionKeyProvider, scriptIsolationMutex)
+            : base(podService, podMonitor, secretService, customResourceService, containerResolver, appInstanceSelector, log, scriptLogProvider, homeConfiguration, kubernetesPhysicalFileSystem, scriptPodLogEncryptionKeyProvider, scriptIsolationMutex)
         {
             this.containerResolver = containerResolver;
         }
 
-        protected override async Task<IList<V1Container>> CreateInitContainers(StartKubernetesScriptCommandV1 command, string podName, string homeDir, string workspacePath, InMemoryTentacleScriptLog tentacleScriptLog)
+        protected override async Task<IList<V1Container>> CreateInitContainers(StartKubernetesScriptCommandV1 command, string podName, string homeDir, string workspacePath, InMemoryTentacleScriptLog tentacleScriptLog, V1Container? containerSpec)
         {
-            var container = new V1Container
+            V1Container container;
+            if (containerSpec is not null)
             {
-                Name = $"{podName}-init",
-                Image = command.PodImageConfiguration?.Image ?? await containerResolver.GetContainerImageForCluster(),
-                ImagePullPolicy = KubernetesConfig.ScriptPodPullPolicy,
-                Command = new List<string> { "sh", "-c", GetInitExecutionScript("/nfs-mount", homeDir, workspacePath) },
-                VolumeMounts = new List<V1VolumeMount> { new("/nfs-mount", "init-nfs-volume"), new(homeDir, "tentacle-home") },
-                Resources = GetScriptPodResourceRequirements(tentacleScriptLog)
-            };
+                container = containerSpec;
+            }
+            else
+            {
+                container = new V1Container
+                {
+                    Resources = GetScriptPodResourceRequirements(tentacleScriptLog)
+                };
+            }
+
+            container.Name = $"{podName}-init";
+            container.Image = command.PodImageConfiguration?.Image ?? await containerResolver.GetContainerImageForCluster();
+            container.ImagePullPolicy = KubernetesConfig.ScriptPodPullPolicy;
+            container.Command = new List<string> { "sh", "-c", GetInitExecutionScript("/nfs-mount", homeDir, workspacePath) };
+            container.VolumeMounts = new List<V1VolumeMount> { new("/nfs-mount", "init-nfs-volume"), new(homeDir, "tentacle-home") };
 
             return new List<V1Container> { container };
         }
-        
-        protected override async Task<IList<V1Container>> CreateScriptContainers(StartKubernetesScriptCommandV1 command, string podName, string scriptName, string homeDir, string workspacePath, string[]? scriptArguments, InMemoryTentacleScriptLog tentacleScriptLog)
+
+        protected override async Task<IList<V1Container>> CreateScriptContainers(StartKubernetesScriptCommandV1 command, string podName, string scriptName, string homeDir, string workspacePath, string[]? scriptArguments, InMemoryTentacleScriptLog tentacleScriptLog, ScriptPodTemplateSpec? spec)
         {
             return new List<V1Container>
             {
-                await CreateScriptContainer(command, podName, scriptName, homeDir, workspacePath, scriptArguments, tentacleScriptLog)
+                await CreateScriptContainer(command, podName, scriptName, homeDir, workspacePath, scriptArguments, tentacleScriptLog, spec?.ScriptContainerSpec)
             };
         }
 
@@ -82,7 +92,7 @@ protected override IList<V1Volume> CreateVolumes(StartKubernetesScriptCommandV1
             };
         }
 
-        string GetInitExecutionScript(string nfsVolumeDirectory, string homeDir, string workspacePath)
+        static string GetInitExecutionScript(string nfsVolumeDirectory, string homeDir, string workspacePath)
         {
             var nfsWorkspacePath = Path.Combine(nfsVolumeDirectory, workspacePath);
             var homeWorkspacePath = Path.Combine(homeDir, workspacePath);