fix: skip pod creation when workload is being deleted (#213)

* feat: skip pod creation when workload is being deleted

* fix lint

Author: 0x5457

internal/controller/tensorfusionworkload_controller.go

@@ -123,6 +123,11 @@ func (r *TensorFusionWorkloadReconciler) Reconcile(ctx context.Context, req ctrl
 		return ctrl.Result{RequeueAfter: constants.PendingRequeueDuration}, nil
 	}
 
+	if !workload.DeletionTimestamp.IsZero() {
+		log.Info("Workload is being deleted, skipping pod creation", "name", workload.Name, "namespace", workload.Namespace)
+		return ctrl.Result{}, nil
+	}
+
 	// Fetch the GPUPool
 	pool := &tfv1.GPUPool{}
 	if err := r.Get(ctx, client.ObjectKey{Name: workload.Spec.PoolName}, pool); err != nil {
@@ -144,6 +149,27 @@ func (r *TensorFusionWorkloadReconciler) Reconcile(ctx context.Context, req ctrl
 		}
 	}
 
+	result, err := r.reconcileScaling(ctx, workload, podList, workerGenerator, podTemplateHash)
+	if err != nil || !result.IsZero() {
+		return result, err
+	}
+
+	if err := r.updateStatus(ctx, workload, podList.Items, workerGenerator); err != nil {
+		return ctrl.Result{}, err
+	}
+
+	return ctrl.Result{}, nil
+}
+
+// reconcileScaling handles scaling up and down of worker pods and updates replica status
+func (r *TensorFusionWorkloadReconciler) reconcileScaling(
+	ctx context.Context,
+	workload *tfv1.TensorFusionWorkload,
+	podList *corev1.PodList,
+	workerGenerator *worker.WorkerGenerator,
+	podTemplateHash string,
+) (ctrl.Result, error) {
+	log := log.FromContext(ctx)
 	// Check if there are any Pods using the old podTemplateHash and delete them if any
 	if len(podList.Items) > 0 {
 		var outdatedPods []corev1.Pod
@@ -210,10 +236,6 @@ func (r *TensorFusionWorkloadReconciler) Reconcile(ctx context.Context, req ctrl
 		}
 	}
 
-	if err := r.updateStatus(ctx, workload, podList.Items, workerGenerator); err != nil {
-		return ctrl.Result{}, err
-	}
-
 	return ctrl.Result{}, nil
 }