Use script pod template custom resource to create pods and containers

source/Octopus.Tentacle/Kubernetes/KubernetesContainerExtensionMethods.cs

@@ -0,0 +1,21 @@
+using System.Diagnostics.CodeAnalysis;
+using k8s;
+using k8s.Models;
+
+namespace Octopus.Tentacle.Kubernetes
+{
+    public static class KubernetesContainerExtensionMethods
+    {
+        [return: NotNullIfNotNull(nameof(source))]
+        public static V1Container? Clone(this V1Container? source)
+        {
+            if (source is null)
+            {
+                return null;
+            }
+            // Use JSON serialization for deep cloning
+            var json = KubernetesJson.Serialize(source);
+            return KubernetesJson.Deserialize<V1Container>(json);
+        }
+    }
+}

source/Octopus.Tentacle/Kubernetes/KubernetesCustomResourceService.cs

@@ -0,0 +1,59 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Net;
+using System.Threading;
+using System.Threading.Tasks;
+using k8s;
+using k8s.Models;
+using Octopus.Tentacle.Core.Diagnostics;
+
+namespace Octopus.Tentacle.Kubernetes
+{
+    public interface IKubernetesCustomResourceService
+    {
+        Task<ScriptPodTemplateCustomResource?> GetOldestScriptPodTemplateCustomResource(CancellationToken cancellationToken);
+    }
+
+    public class KubernetesCustomResourceService : KubernetesService, IKubernetesCustomResourceService
+    {
+        public KubernetesCustomResourceService(IKubernetesClientConfigProvider configProvider, ISystemLog log)
+            : base(configProvider, log)
+        {
+        }
+
+        public async Task<ScriptPodTemplateCustomResource?> GetOldestScriptPodTemplateCustomResource(CancellationToken cancellationToken)
+        {
+            return await RetryPolicy.ExecuteAsync(async () =>
+            {
+                ScriptPodTemplateCustomResourceList resourceList;
+                try
+                {
+                    resourceList = await Client.CustomObjects.ListNamespacedCustomObjectAsync<ScriptPodTemplateCustomResourceList>(
+                        "agent.octopus.com",
+                        "v1beta1",
+                        KubernetesConfig.Namespace,
+                        "scriptpodtemplates",
+                        cancellationToken: cancellationToken);
+                }
+                catch (Exception ex)
+                {
+                    // we are happy to handle all exceptions here and just fallback
+                    Log.WarnFormat(ex, "Failed to retrieve 'scriptpodtemplates' custom resource");
+                    return null;
+                }
+
+                return resourceList.Items.OrderBy(r => r.Metadata.CreationTimestamp).FirstOrDefault();
+            });
+        }
+
+        //These are only ever deserialized from JSON
+#pragma warning disable CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider adding the 'required' modifier or declaring as nullable.
+        class ScriptPodTemplateCustomResourceList : KubernetesObject
+        {
+            public V1ListMeta Metadata { get; set; }
+            public List<ScriptPodTemplateCustomResource> Items { get; set; }
+        }
+#pragma warning restore CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider adding the 'required' modifier or declaring as nullable.
+    }
+}

source/Octopus.Tentacle/Kubernetes/KubernetesModule.cs

@@ -34,6 +34,7 @@ protected override void Load(ContainerBuilder builder)
             builder.RegisterType<KubernetesPodContainerResolver>().As<IKubernetesPodContainerResolver>().SingleInstance();
             builder.RegisterType<KubernetesConfigMapService>().As<IKubernetesConfigMapService>().SingleInstance();
             builder.RegisterType<KubernetesSecretService>().As<IKubernetesSecretService>().SingleInstance();
+            builder.RegisterType<KubernetesCustomResourceService>().As<IKubernetesCustomResourceService>().SingleInstance();
 
             builder.RegisterType<KubernetesScriptPodCreator>().As<IKubernetesScriptPodCreator>().SingleInstance();
             builder.RegisterType<KubernetesRawScriptPodCreator>().As<IKubernetesRawScriptPodCreator>().SingleInstance();

source/Octopus.Tentacle/Kubernetes/KubernetesRawScriptPodCreator.cs

@@ -9,7 +9,6 @@
 using Octopus.Tentacle.Core.Diagnostics;
 using Octopus.Tentacle.Core.Services.Scripts.Locking;
 using Octopus.Tentacle.Kubernetes.Crypto;
-using Octopus.Tentacle.Scripts;
 
 namespace Octopus.Tentacle.Kubernetes
 {
@@ -25,6 +24,7 @@ public KubernetesRawScriptPodCreator(
             IKubernetesPodService podService,
             IKubernetesPodMonitor podMonitor,
             IKubernetesSecretService secretService,
+            IKubernetesCustomResourceService customResourceService,
             IKubernetesPodContainerResolver containerResolver,
             IApplicationInstanceSelector appInstanceSelector,
             ISystemLog log,
@@ -33,31 +33,34 @@ public KubernetesRawScriptPodCreator(
             KubernetesPhysicalFileSystem kubernetesPhysicalFileSystem,
             IScriptPodLogEncryptionKeyProvider scriptPodLogEncryptionKeyProvider,
             ScriptIsolationMutex scriptIsolationMutex)
-            : base(podService, podMonitor, secretService, containerResolver, appInstanceSelector, log, scriptLogProvider, homeConfiguration, kubernetesPhysicalFileSystem, scriptPodLogEncryptionKeyProvider, scriptIsolationMutex)
+            : base(podService, podMonitor, secretService, customResourceService, containerResolver, appInstanceSelector, log, scriptLogProvider, homeConfiguration, kubernetesPhysicalFileSystem, scriptPodLogEncryptionKeyProvider, scriptIsolationMutex)
         {
             this.containerResolver = containerResolver;
         }
 
-        protected override async Task<IList<V1Container>> CreateInitContainers(StartKubernetesScriptCommandV1 command, string podName, string homeDir, string workspacePath, InMemoryTentacleScriptLog tentacleScriptLog)
+        protected override async Task<IList<V1Container>> CreateInitContainers(StartKubernetesScriptCommandV1 command, string podName, string homeDir, string workspacePath, InMemoryTentacleScriptLog tentacleScriptLog, V1Container? containerSpec)
         {
-            var container = new V1Container
-            {
-                Name = $"{podName}-init",
-                Image = command.PodImageConfiguration?.Image ?? await containerResolver.GetContainerImageForCluster(),
-                ImagePullPolicy = KubernetesConfig.ScriptPodPullPolicy,
-                Command = new List<string> { "sh", "-c", GetInitExecutionScript("/nfs-mount", homeDir, workspacePath) },
-                VolumeMounts = new List<V1VolumeMount> { new("/nfs-mount", "init-nfs-volume"), new(homeDir, "tentacle-home") },
-                Resources = GetScriptPodResourceRequirements(tentacleScriptLog)
-            };
+            // Deep clone the container spec to avoid modifying the original
+            var container = containerSpec.Clone() ??
+                new V1Container
+                {
+                    Resources = GetScriptPodResourceRequirements(tentacleScriptLog)
+                };
+
+            container.Name = $"{podName}-init";
+            container.Image = command.PodImageConfiguration?.Image ?? await containerResolver.GetContainerImageForCluster();
+            container.ImagePullPolicy = KubernetesConfig.ScriptPodPullPolicy;
+            container.Command = new List<string> { "sh", "-c", GetInitExecutionScript("/nfs-mount", homeDir, workspacePath) };
+            container.VolumeMounts = Merge(container.VolumeMounts, new[] { new V1VolumeMount("/nfs-mount", "init-nfs-volume"), new V1VolumeMount(homeDir, "tentacle-home") });
 
             return new List<V1Container> { container };
         }
-        
-        protected override async Task<IList<V1Container>> CreateScriptContainers(StartKubernetesScriptCommandV1 command, string podName, string scriptName, string homeDir, string workspacePath, string[]? scriptArguments, InMemoryTentacleScriptLog tentacleScriptLog)
+
+        protected override async Task<IList<V1Container>> CreateScriptContainers(StartKubernetesScriptCommandV1 command, string podName, string scriptName, string homeDir, string workspacePath, string[]? scriptArguments, InMemoryTentacleScriptLog tentacleScriptLog, ScriptPodTemplateSpec? spec)
         {
             return new List<V1Container>
             {
-                await CreateScriptContainer(command, podName, scriptName, homeDir, workspacePath, scriptArguments, tentacleScriptLog)
+                await CreateScriptContainer(command, podName, scriptName, homeDir, workspacePath, scriptArguments, tentacleScriptLog, spec?.ScriptContainerSpec)
             };
         }
 
@@ -82,7 +85,7 @@ protected override IList<V1Volume> CreateVolumes(StartKubernetesScriptCommandV1
             };
         }
 
-        string GetInitExecutionScript(string nfsVolumeDirectory, string homeDir, string workspacePath)
+        static string GetInitExecutionScript(string nfsVolumeDirectory, string homeDir, string workspacePath)
         {
             var nfsWorkspacePath = Path.Combine(nfsVolumeDirectory, workspacePath);
             var homeWorkspacePath = Path.Combine(homeDir, workspacePath);