diff --git a/client/argocdServer/ArgoClientWrapperService.go b/client/argocdServer/ArgoClientWrapperService.go index a6d4cf0c61..78274d321b 100644 --- a/client/argocdServer/ArgoClientWrapperService.go +++ b/client/argocdServer/ArgoClientWrapperService.go @@ -12,6 +12,7 @@ import ( "github.com/devtron-labs/devtron/client/argocdServer/application" "github.com/devtron-labs/devtron/client/argocdServer/bean" "github.com/devtron-labs/devtron/client/argocdServer/repository" + "github.com/devtron-labs/devtron/internal/util" "github.com/devtron-labs/devtron/pkg/deployment/gitOps/config" "github.com/devtron-labs/devtron/pkg/deployment/gitOps/git" "github.com/devtron-labs/devtron/util/retryFunc" @@ -25,6 +26,14 @@ type ACDConfig struct { ArgoCDAutoSyncEnabled bool `env:"ARGO_AUTO_SYNC_ENABLED" envDefault:"true"` // will gradually switch this flag to false in enterprise } +func (config *ACDConfig) IsManualSyncEnabled() bool { + return config.ArgoCDAutoSyncEnabled == false +} + +func (config *ACDConfig) IsAutoSyncEnabled() bool { + return config.ArgoCDAutoSyncEnabled == true +} + func GetACDDeploymentConfig() (*ACDConfig, error) { cfg := &ACDConfig{} err := env.Parse(cfg) @@ -34,6 +43,10 @@ func GetACDDeploymentConfig() (*ACDConfig, error) { return cfg, err } +const ( + ErrorOperationAlreadyInProgress = "another operation is already in progress" // this string is returned from argocd +) + type ArgoClientWrapperService interface { // GetArgoAppWithNormalRefresh - refresh app at argocd side @@ -96,17 +109,45 @@ func (impl *ArgoClientWrapperServiceImpl) GetArgoAppWithNormalRefresh(context co } func (impl *ArgoClientWrapperServiceImpl) SyncArgoCDApplicationIfNeededAndRefresh(context context.Context, argoAppName string) error { + impl.logger.Info("argocd manual sync for app started", "argoAppName", argoAppName) - if !impl.ACDConfig.ArgoCDAutoSyncEnabled { + if impl.ACDConfig.IsManualSyncEnabled() { + impl.logger.Debugw("syncing argocd app as manual sync is enabled", "argoAppName", argoAppName) revision := "master" pruneResources := true - _, syncErr := impl.acdClient.Sync(context, &application2.ApplicationSyncRequest{Name: &argoAppName, Revision: &revision, Prune: &pruneResources}) + _, syncErr := impl.acdClient.Sync(context, &application2.ApplicationSyncRequest{Name: &argoAppName, + Revision: &revision, + Prune: &pruneResources, + }) if syncErr != nil { - impl.logger.Errorw("cannot get application with refresh", "app", argoAppName) - return syncErr + impl.logger.Errorw("error in syncing argoCD app", "app", argoAppName, "err", syncErr) + statusCode, msg := util.GetClientDetailedError(syncErr) + if statusCode.IsFailedPreconditionCode() && msg == ErrorOperationAlreadyInProgress { + impl.logger.Info("terminating ongoing sync operation and retrying manual sync", "argoAppName", argoAppName) + _, terminationErr := impl.acdClient.TerminateOperation(context, &application2.OperationTerminateRequest{ + Name: &argoAppName, + }) + if terminationErr != nil { + impl.logger.Errorw("error in terminating sync operation") + return fmt.Errorf("error in terminating existing sync, err: %w", terminationErr) + } + _, syncErr = impl.acdClient.Sync(context, &application2.ApplicationSyncRequest{Name: &argoAppName, + Revision: &revision, + Prune: &pruneResources, + RetryStrategy: &v1alpha1.RetryStrategy{ + Limit: 1, + }, + }) + if syncErr != nil { + impl.logger.Errorw("error in syncing argoCD app", "app", argoAppName, "err", syncErr) + return syncErr + } + } else { + return syncErr + } } - impl.logger.Debugw("argocd sync completed", "argoAppName", argoAppName) + impl.logger.Infow("argocd sync completed", "argoAppName", argoAppName) } refreshErr := impl.GetArgoAppWithNormalRefresh(context, argoAppName) if refreshErr != nil { @@ -129,10 +170,9 @@ func (impl *ArgoClientWrapperServiceImpl) UpdateArgoCDSyncModeIfNeeded(ctx conte } func (impl *ArgoClientWrapperServiceImpl) isArgoAppSyncModeMigrationNeeded(argoApplication *v1alpha1.Application) bool { - if !impl.ACDConfig.ArgoCDAutoSyncEnabled && argoApplication.Spec.SyncPolicy.Automated != nil { + if impl.ACDConfig.IsManualSyncEnabled() && argoApplication.Spec.SyncPolicy.Automated != nil { return true - } - if impl.ACDConfig.ArgoCDAutoSyncEnabled && argoApplication.Spec.SyncPolicy.Automated == nil { + } else if impl.ACDConfig.IsAutoSyncEnabled() && argoApplication.Spec.SyncPolicy.Automated == nil { return true } return false @@ -141,7 +181,7 @@ func (impl *ArgoClientWrapperServiceImpl) isArgoAppSyncModeMigrationNeeded(argoA func (impl *ArgoClientWrapperServiceImpl) CreateRequestForArgoCDSyncModeUpdateRequest(argoApplication *v1alpha1.Application) *v1alpha1.Application { // set automated field in update request var automated *v1alpha1.SyncPolicyAutomated - if impl.ACDConfig.ArgoCDAutoSyncEnabled { + if impl.ACDConfig.IsAutoSyncEnabled() { automated = &v1alpha1.SyncPolicyAutomated{ Prune: true, } diff --git a/client/argocdServer/application/Application.go b/client/argocdServer/application/Application.go index ca7e3e9ef9..8026a40fde 100644 --- a/client/argocdServer/application/Application.go +++ b/client/argocdServer/application/Application.go @@ -51,6 +51,8 @@ type ServiceClient interface { // Delete deletes an application Delete(ctx context.Context, query *application.ApplicationDeleteRequest) (*application.ApplicationResponse, error) + + TerminateOperation(ctx context.Context, query *application.OperationTerminateRequest) (*application.OperationTerminateResponse, error) } type ServiceClientImpl struct { @@ -349,3 +351,17 @@ func (c ServiceClientImpl) buildPodMetadata(resp *v1alpha1.ApplicationTree, resp } return } + +func (c ServiceClientImpl) TerminateOperation(ctxt context.Context, query *application.OperationTerminateRequest) (*application.OperationTerminateResponse, error) { + ctx, cancel := context.WithTimeout(ctxt, argoApplication.TimeoutFast) + defer cancel() + token, ok := ctxt.Value("token").(string) + if !ok { + return nil, argoApplication.NewErrUnauthorized("Unauthorized") + } + conn := c.argoCDConnectionManager.GetConnection(token) + defer util.Close(conn, c.logger) + asc := application.NewApplicationServiceClient(conn) + resp, err := asc.TerminateOperation(ctx, query) + return resp, err +} diff --git a/pkg/app/AppService.go b/pkg/app/AppService.go index ae76e30596..8a109d4551 100644 --- a/pkg/app/AppService.go +++ b/pkg/app/AppService.go @@ -442,7 +442,7 @@ func (impl *AppServiceImpl) CheckIfPipelineUpdateEventIsValidForAppStore(gitOpsA // drop event return isValid, installedAppVersionHistory, appId, envId, nil } - if !impl.acdConfig.ArgoCDAutoSyncEnabled { + if impl.acdConfig.IsManualSyncEnabled() { isArgoAppSynced := impl.pipelineStatusTimelineService.GetArgoAppSyncStatusForAppStore(installedAppVersionHistory.Id) if !isArgoAppSynced { return isValid, installedAppVersionHistory, appId, envId, nil @@ -491,7 +491,7 @@ func (impl *AppServiceImpl) CheckIfPipelineUpdateEventIsValid(argoAppName, gitHa // drop event return isValid, pipeline, cdWfr, pipelineOverride, nil } - if !impl.acdConfig.ArgoCDAutoSyncEnabled { + if impl.acdConfig.IsManualSyncEnabled() { // if manual sync, proceed only if ARGOCD_SYNC_COMPLETED timeline is created isArgoAppSynced := impl.pipelineStatusTimelineService.GetArgoAppSyncStatus(cdWfr.Id) if !isArgoAppSynced { diff --git a/pkg/app/ManifestPushService.go b/pkg/app/ManifestPushService.go index 5ec6e1da37..af0838f4d7 100644 --- a/pkg/app/ManifestPushService.go +++ b/pkg/app/ManifestPushService.go @@ -148,7 +148,7 @@ func (impl *GitOpsManifestPushServiceImpl) PushChart(manifestPushTemplate *bean. gitCommitTimeline := impl.pipelineStatusTimelineService.GetTimelineDbObjectByTimelineStatusAndTimelineDescription(manifestPushTemplate.WorkflowRunnerId, 0, pipelineConfig.TIMELINE_STATUS_GIT_COMMIT, "Git commit done successfully.", manifestPushTemplate.UserId, time.Now()) timelines := []*pipelineConfig.PipelineStatusTimeline{gitCommitTimeline} - if !impl.acdConfig.ArgoCDAutoSyncEnabled { + if impl.acdConfig.IsManualSyncEnabled() { // if manual sync is enabled, add ARGOCD_SYNC_INITIATED_TIMELINE argoCDSyncInitiatedTimeline := impl.pipelineStatusTimelineService.GetTimelineDbObjectByTimelineStatusAndTimelineDescription(manifestPushTemplate.WorkflowRunnerId, 0, pipelineConfig.TIMELINE_STATUS_ARGOCD_SYNC_INITIATED, "argocd sync initiated.", manifestPushTemplate.UserId, time.Now()) timelines = append(timelines, argoCDSyncInitiatedTimeline) diff --git a/pkg/appStore/chartGroup/ChartGroupService.go b/pkg/appStore/chartGroup/ChartGroupService.go index ac1d59160e..0df6ccdbc4 100644 --- a/pkg/appStore/chartGroup/ChartGroupService.go +++ b/pkg/appStore/chartGroup/ChartGroupService.go @@ -993,7 +993,7 @@ func (impl *ChartGroupServiceImpl) performDeployStageOnAcd(installedAppVersion * GetTimelineDbObjectByTimelineStatusAndTimelineDescription(0, installedAppVersion.InstalledAppVersionHistoryId, pipelineConfig.TIMELINE_STATUS_GIT_COMMIT, "Git commit done successfully.", installedAppVersion.UserId, time.Now()) timelines := []*pipelineConfig.PipelineStatusTimeline{GitCommitSuccessTimeline} - if !impl.acdConfig.ArgoCDAutoSyncEnabled { + if impl.acdConfig.IsManualSyncEnabled() { ArgocdSyncInitiatedTimeline := impl.pipelineStatusTimelineService. GetTimelineDbObjectByTimelineStatusAndTimelineDescription(0, installedAppVersion.InstalledAppVersionHistoryId, pipelineConfig.TIMELINE_STATUS_ARGOCD_SYNC_INITIATED, "ArgoCD sync initiated.", installedAppVersion.UserId, time.Now()) diff --git a/pkg/appStore/installedApp/service/AppStoreDeploymentService.go b/pkg/appStore/installedApp/service/AppStoreDeploymentService.go index 109c35280a..9c04787dce 100644 --- a/pkg/appStore/installedApp/service/AppStoreDeploymentService.go +++ b/pkg/appStore/installedApp/service/AppStoreDeploymentService.go @@ -164,7 +164,7 @@ func (impl *AppStoreDeploymentServiceImpl) InstallApp(installAppVersionRequest * } if util.IsAcdApp(installAppVersionRequest.DeploymentAppType) { _ = impl.fullModeDeploymentService.SaveTimelineForHelmApps(installAppVersionRequest, pipelineConfig.TIMELINE_STATUS_GIT_COMMIT, "Git commit done successfully.", time.Now(), tx) - if !impl.aCDConfig.ArgoCDAutoSyncEnabled { + if impl.aCDConfig.IsManualSyncEnabled() { _ = impl.fullModeDeploymentService.SaveTimelineForHelmApps(installAppVersionRequest, pipelineConfig.TIMELINE_STATUS_ARGOCD_SYNC_INITIATED, "argocd sync initiated.", time.Now(), tx) } } @@ -695,7 +695,7 @@ func (impl *AppStoreDeploymentServiceImpl) UpdateInstalledApp(ctx context.Contex upgradeAppRequest.GitHash = gitOpsResponse.GitHash _ = impl.fullModeDeploymentService.SaveTimelineForHelmApps(upgradeAppRequest, pipelineConfig.TIMELINE_STATUS_GIT_COMMIT, "Git commit done successfully.", time.Now(), tx) - if !impl.aCDConfig.ArgoCDAutoSyncEnabled { + if impl.aCDConfig.IsManualSyncEnabled() { _ = impl.fullModeDeploymentService.SaveTimelineForHelmApps(upgradeAppRequest, pipelineConfig.TIMELINE_STATUS_ARGOCD_SYNC_INITIATED, "Argocd sync initiated", time.Now(), tx) } installedAppVersionHistory.GitHash = gitOpsResponse.GitHash diff --git a/pkg/appStore/installedApp/service/FullMode/deployment/FullModeDeploymentService.go b/pkg/appStore/installedApp/service/FullMode/deployment/FullModeDeploymentService.go index fdd6cecb43..59044d3949 100644 --- a/pkg/appStore/installedApp/service/FullMode/deployment/FullModeDeploymentService.go +++ b/pkg/appStore/installedApp/service/FullMode/deployment/FullModeDeploymentService.go @@ -156,7 +156,7 @@ func (impl *FullModeDeploymentServiceImpl) InstallApp(installAppVersionRequest * impl.Logger.Errorw("error in getting the argo application with normal refresh", "err", err) return nil, err } - if !impl.acdConfig.ArgoCDAutoSyncEnabled { + if impl.acdConfig.IsManualSyncEnabled() { timeline := &pipelineConfig.PipelineStatusTimeline{ InstalledAppVersionHistoryId: installAppVersionRequest.InstalledAppVersionHistoryId, Status: pipelineConfig.TIMELINE_STATUS_ARGOCD_SYNC_COMPLETED, @@ -304,7 +304,7 @@ func (impl *FullModeDeploymentServiceImpl) RollbackRelease(ctx context.Context, return installedApp, false, err } - isManualSync := !impl.acdConfig.ArgoCDAutoSyncEnabled + isManualSync := impl.acdConfig.IsManualSyncEnabled() GitCommitSuccessTimeline := impl.pipelineStatusTimelineService. GetTimelineDbObjectByTimelineStatusAndTimelineDescription(0, installedApp.InstalledAppVersionHistoryId, pipelineConfig.TIMELINE_STATUS_GIT_COMMIT, "Git commit done successfully.", installedApp.UserId, time.Now()) diff --git a/pkg/appStore/installedApp/service/FullMode/deployment/InstalledAppArgoCdService.go b/pkg/appStore/installedApp/service/FullMode/deployment/InstalledAppArgoCdService.go index ae21946d34..12b27f8358 100644 --- a/pkg/appStore/installedApp/service/FullMode/deployment/InstalledAppArgoCdService.go +++ b/pkg/appStore/installedApp/service/FullMode/deployment/InstalledAppArgoCdService.go @@ -127,7 +127,7 @@ func (impl *FullModeDeploymentServiceImpl) UpdateAndSyncACDApps(installAppVersio } return err } - if !impl.acdConfig.ArgoCDAutoSyncEnabled { + if impl.acdConfig.IsManualSyncEnabled() { err = impl.SaveTimelineForHelmApps(installAppVersionRequest, pipelineConfig.TIMELINE_STATUS_ARGOCD_SYNC_COMPLETED, "argocd sync completed", syncTime, tx) if err != nil { impl.Logger.Errorw("error in saving timeline for acd helm apps", "err", err) diff --git a/pkg/deployment/trigger/devtronApps/TriggerService.go b/pkg/deployment/trigger/devtronApps/TriggerService.go index f670eabe13..183892bdae 100644 --- a/pkg/deployment/trigger/devtronApps/TriggerService.go +++ b/pkg/deployment/trigger/devtronApps/TriggerService.go @@ -1041,7 +1041,7 @@ func (impl *TriggerServiceImpl) deployArgocdApp(overrideRequest *bean3.ValuesOve impl.logger.Errorw("error in getting argo application with normal refresh", "argoAppName", valuesOverrideResponse.Pipeline.DeploymentAppName) return fmt.Errorf("%s. err: %s", bean.ARGOCD_SYNC_ERROR, err.Error()) } - if !impl.ACDConfig.ArgoCDAutoSyncEnabled { + if impl.ACDConfig.IsManualSyncEnabled() { timeline := &pipelineConfig.PipelineStatusTimeline{ CdWorkflowRunnerId: overrideRequest.WfrId, StatusTime: syncTime, diff --git a/pkg/workflow/status/WorkflowStatusService.go b/pkg/workflow/status/WorkflowStatusService.go index 19a9d1bfcf..e69e677d5e 100644 --- a/pkg/workflow/status/WorkflowStatusService.go +++ b/pkg/workflow/status/WorkflowStatusService.go @@ -193,7 +193,7 @@ func (impl *WorkflowStatusServiceImpl) UpdatePipelineTimelineAndStatusByLiveAppl return nil, isTimelineUpdated } - if !impl.acdConfig.ArgoCDAutoSyncEnabled { + if impl.acdConfig.IsManualSyncEnabled() { // if manual sync check for application sync status isArgoAppSynced := impl.pipelineStatusTimelineService.GetArgoAppSyncStatus(cdWfr.Id) if !isArgoAppSynced { @@ -277,7 +277,7 @@ func (impl *WorkflowStatusServiceImpl) UpdatePipelineTimelineAndStatusByLiveAppl // drop event return nil, isTimelineUpdated } - if !impl.acdConfig.ArgoCDAutoSyncEnabled { + if impl.acdConfig.IsManualSyncEnabled() { isArgoAppSynced := impl.pipelineStatusTimelineService.GetArgoAppSyncStatusForAppStore(installedAppVersionHistory.Id) if !isArgoAppSynced { return nil, isTimelineUpdated @@ -469,7 +469,7 @@ func (impl *WorkflowStatusServiceImpl) CheckArgoPipelineTimelineStatusPeriodical } func (impl *WorkflowStatusServiceImpl) syncACDDevtronApps(deployedBeforeMinutes int, pipelineId int) error { - if impl.acdConfig.ArgoCDAutoSyncEnabled { + if impl.acdConfig.IsAutoSyncEnabled() { // don't check for apps if auto sync is enabled return nil } @@ -528,7 +528,7 @@ func (impl *WorkflowStatusServiceImpl) syncACDDevtronApps(deployedBeforeMinutes } func (impl *WorkflowStatusServiceImpl) syncACDHelmApps(deployedBeforeMinutes int, installedAppVersionId int) error { - if impl.acdConfig.ArgoCDAutoSyncEnabled { + if impl.acdConfig.IsAutoSyncEnabled() { // don't check for apps if auto sync is enabled return nil }