From 4d7dd424872288191b217426deacc918a8adaa3a Mon Sep 17 00:00:00 2001 From: hirenko-v <132065511+hirenko-v@users.noreply.github.com> Date: Wed, 3 Jul 2024 14:53:56 +0300 Subject: [PATCH 1/2] Add remediateLastFailure --- api/v1alpha2/terraform_types.go | 15 +++++++++++++++ charts/tofu-controller/crds/crds.yaml | 6 ++++++ controllers/tf_controller.go | 10 ++++++++++ 3 files changed, 31 insertions(+) diff --git a/api/v1alpha2/terraform_types.go b/api/v1alpha2/terraform_types.go index 9998bda2..599a235e 100644 --- a/api/v1alpha2/terraform_types.go +++ b/api/v1alpha2/terraform_types.go @@ -286,6 +286,12 @@ type Remediation struct { // retries. // +optional Retries int64 `json:"retries,omitempty"` + + // RemediateLastFailure tells the controller to remediate the last failure, when + // no retries remain. Defaults to 'false' unless 'Retries' is greater than 0. + // +optional + RemediateLastFailure *bool `json:"remediateLastFailure,omitempty"` + } type CloudSpec struct { @@ -954,6 +960,15 @@ func (in *Terraform) GetRetries() int64 { return in.Spec.Remediation.Retries } +// MustRemediateLastFailure returns whether to remediate the last failure when +// no retries remain. +func (in *Terraform) MustRemediateLastFailure() bool { + if in.Spec.Remediation.RemediateLastFailure == nil { + return false + } + return *in.Spec.Remediation.RemediateLastFailure +} + func (in *Terraform) GetReconciliationFailures() int64 { return in.Status.ReconciliationFailures } diff --git a/charts/tofu-controller/crds/crds.yaml b/charts/tofu-controller/crds/crds.yaml index 3ff5696f..baa35b27 100644 --- a/charts/tofu-controller/crds/crds.yaml +++ b/charts/tofu-controller/crds/crds.yaml @@ -5082,6 +5082,12 @@ spec: retries. format: int64 type: integer + remediateLastFailure: + default: false + description: |- + remediateLastFailure instructs the controller to remediate the last failure + when no retries remain. Defaults to false + type: boolean type: object retryInterval: description: |- diff --git a/controllers/tf_controller.go b/controllers/tf_controller.go index eab88c0c..c0fcfa31 100644 --- a/controllers/tf_controller.go +++ b/controllers/tf_controller.go @@ -449,6 +449,16 @@ func (r *TerraformReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( terraform.GetGeneration(), )) + if terraform.MustRemediateLastFailure() { + log.Info("RemediateLastFailure is true, reseting retries, requeue after interval", "interval", terraform.Spec.Interval.Duration.String()) + terraform = infrav1.TerraformResetRetry(terraform) + if err := r.patchStatus(ctx, req.NamespacedName, terraform.Status); err != nil { + log.Error(err, "unable to update status after maximum number of retries reached") + return ctrl.Result{Requeue: true}, err + } + return ctrl.Result{RequeueAfter: terraform.Spec.Interval.Duration}, nil + } + terraform = infrav1.TerraformReachedLimit(terraform) traceLog.Info("Patch the status of the Terraform resource") From 3fb273733cb7512aa9dfef461a0159074e1cdeac Mon Sep 17 00:00:00 2001 From: hirenko-v Date: Sun, 7 Jul 2024 22:26:22 +0300 Subject: [PATCH 2/2] Add doc --- docs/References/terraform.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/docs/References/terraform.md b/docs/References/terraform.md index b447fe30..560e0567 100644 --- a/docs/References/terraform.md +++ b/docs/References/terraform.md @@ -702,6 +702,19 @@ before bailing. Defaults to ‘0’, a negative integer denotes unlimite retries.

+ + +remediateLastFailure
+ +bool + + + +(Optional) +

RemediateLastFailure tells the controller to remediate the last failure, when +no retries remain. Defaults to ‘false’ unless ‘Retries’ is greater than 0.

+ +