Skip to content

Commit 7609112

Browse files
iamayushmkomalreddy3
iamayushm
authored and
komalreddy3
committed
fix: terminate sync if in progress (#4946)
* terminate operation before sync * using method for checking manual sync * terminating only in case of error * handling termination error * updating sync function
1 parent 2c569bd commit 7609112

File tree

10 files changed

+79
-23
lines changed

10 files changed

+79
-23
lines changed

‎client/argocdServer/ArgoClientWrapperService.go

Lines changed: 49 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ import (
1212
"github.com/devtron-labs/devtron/client/argocdServer/application"
1313
"github.com/devtron-labs/devtron/client/argocdServer/bean"
1414
"github.com/devtron-labs/devtron/client/argocdServer/repository"
15+
"github.com/devtron-labs/devtron/internal/util"
1516
"github.com/devtron-labs/devtron/pkg/deployment/gitOps/config"
1617
"github.com/devtron-labs/devtron/pkg/deployment/gitOps/git"
1718
"github.com/devtron-labs/devtron/util/retryFunc"
@@ -25,6 +26,14 @@ type ACDConfig struct {
2526
ArgoCDAutoSyncEnabled bool `env:"ARGO_AUTO_SYNC_ENABLED" envDefault:"true"` // will gradually switch this flag to false in enterprise
2627
}
2728

29+
func (config *ACDConfig) IsManualSyncEnabled() bool {
30+
return config.ArgoCDAutoSyncEnabled == false
31+
}
32+
33+
func (config *ACDConfig) IsAutoSyncEnabled() bool {
34+
return config.ArgoCDAutoSyncEnabled == true
35+
}
36+
2837
func GetACDDeploymentConfig() (*ACDConfig, error) {
2938
cfg := &ACDConfig{}
3039
err := env.Parse(cfg)
@@ -34,6 +43,10 @@ func GetACDDeploymentConfig() (*ACDConfig, error) {
3443
return cfg, err
3544
}
3645

46+
const (
47+
ErrorOperationAlreadyInProgress = "another operation is already in progress" // this string is returned from argocd
48+
)
49+
3750
type ArgoClientWrapperService interface {
3851

3952
// GetArgoAppWithNormalRefresh - refresh app at argocd side
@@ -96,17 +109,45 @@ func (impl *ArgoClientWrapperServiceImpl) GetArgoAppWithNormalRefresh(context co
96109
}
97110

98111
func (impl *ArgoClientWrapperServiceImpl) SyncArgoCDApplicationIfNeededAndRefresh(context context.Context, argoAppName string) error {
112+
99113
impl.logger.Info("argocd manual sync for app started", "argoAppName", argoAppName)
100-
if !impl.ACDConfig.ArgoCDAutoSyncEnabled {
114+
if impl.ACDConfig.IsManualSyncEnabled() {
115+
101116
impl.logger.Debugw("syncing argocd app as manual sync is enabled", "argoAppName", argoAppName)
102117
revision := "master"
103118
pruneResources := true
104-
_, syncErr := impl.acdClient.Sync(context, &application2.ApplicationSyncRequest{Name: &argoAppName, Revision: &revision, Prune: &pruneResources})
119+
_, syncErr := impl.acdClient.Sync(context, &application2.ApplicationSyncRequest{Name: &argoAppName,
120+
Revision: &revision,
121+
Prune: &pruneResources,
122+
})
105123
if syncErr != nil {
106-
impl.logger.Errorw("cannot get application with refresh", "app", argoAppName)
107-
return syncErr
124+
impl.logger.Errorw("error in syncing argoCD app", "app", argoAppName, "err", syncErr)
125+
statusCode, msg := util.GetClientDetailedError(syncErr)
126+
if statusCode.IsFailedPreconditionCode() && msg == ErrorOperationAlreadyInProgress {
127+
impl.logger.Info("terminating ongoing sync operation and retrying manual sync", "argoAppName", argoAppName)
128+
_, terminationErr := impl.acdClient.TerminateOperation(context, &application2.OperationTerminateRequest{
129+
Name: &argoAppName,
130+
})
131+
if terminationErr != nil {
132+
impl.logger.Errorw("error in terminating sync operation")
133+
return fmt.Errorf("error in terminating existing sync, err: %w", terminationErr)
134+
}
135+
_, syncErr = impl.acdClient.Sync(context, &application2.ApplicationSyncRequest{Name: &argoAppName,
136+
Revision: &revision,
137+
Prune: &pruneResources,
138+
RetryStrategy: &v1alpha1.RetryStrategy{
139+
Limit: 1,
140+
},
141+
})
142+
if syncErr != nil {
143+
impl.logger.Errorw("error in syncing argoCD app", "app", argoAppName, "err", syncErr)
144+
return syncErr
145+
}
146+
} else {
147+
return syncErr
148+
}
108149
}
109-
impl.logger.Debugw("argocd sync completed", "argoAppName", argoAppName)
150+
impl.logger.Infow("argocd sync completed", "argoAppName", argoAppName)
110151
}
111152
refreshErr := impl.GetArgoAppWithNormalRefresh(context, argoAppName)
112153
if refreshErr != nil {
@@ -129,10 +170,9 @@ func (impl *ArgoClientWrapperServiceImpl) UpdateArgoCDSyncModeIfNeeded(ctx conte
129170
}
130171

131172
func (impl *ArgoClientWrapperServiceImpl) isArgoAppSyncModeMigrationNeeded(argoApplication *v1alpha1.Application) bool {
132-
if !impl.ACDConfig.ArgoCDAutoSyncEnabled && argoApplication.Spec.SyncPolicy.Automated != nil {
173+
if impl.ACDConfig.IsManualSyncEnabled() && argoApplication.Spec.SyncPolicy.Automated != nil {
133174
return true
134-
}
135-
if impl.ACDConfig.ArgoCDAutoSyncEnabled && argoApplication.Spec.SyncPolicy.Automated == nil {
175+
} else if impl.ACDConfig.IsAutoSyncEnabled() && argoApplication.Spec.SyncPolicy.Automated == nil {
136176
return true
137177
}
138178
return false
@@ -141,7 +181,7 @@ func (impl *ArgoClientWrapperServiceImpl) isArgoAppSyncModeMigrationNeeded(argoA
141181
func (impl *ArgoClientWrapperServiceImpl) CreateRequestForArgoCDSyncModeUpdateRequest(argoApplication *v1alpha1.Application) *v1alpha1.Application {
142182
// set automated field in update request
143183
var automated *v1alpha1.SyncPolicyAutomated
144-
if impl.ACDConfig.ArgoCDAutoSyncEnabled {
184+
if impl.ACDConfig.IsAutoSyncEnabled() {
145185
automated = &v1alpha1.SyncPolicyAutomated{
146186
Prune: true,
147187
}

‎client/argocdServer/application/Application.go

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,8 @@ type ServiceClient interface {
5151

5252
// Delete deletes an application
5353
Delete(ctx context.Context, query *application.ApplicationDeleteRequest) (*application.ApplicationResponse, error)
54+
55+
TerminateOperation(ctx context.Context, query *application.OperationTerminateRequest) (*application.OperationTerminateResponse, error)
5456
}
5557

5658
type ServiceClientImpl struct {
@@ -349,3 +351,17 @@ func (c ServiceClientImpl) buildPodMetadata(resp *v1alpha1.ApplicationTree, resp
349351
}
350352
return
351353
}
354+
355+
func (c ServiceClientImpl) TerminateOperation(ctxt context.Context, query *application.OperationTerminateRequest) (*application.OperationTerminateResponse, error) {
356+
ctx, cancel := context.WithTimeout(ctxt, argoApplication.TimeoutFast)
357+
defer cancel()
358+
token, ok := ctxt.Value("token").(string)
359+
if !ok {
360+
return nil, argoApplication.NewErrUnauthorized("Unauthorized")
361+
}
362+
conn := c.argoCDConnectionManager.GetConnection(token)
363+
defer util.Close(conn, c.logger)
364+
asc := application.NewApplicationServiceClient(conn)
365+
resp, err := asc.TerminateOperation(ctx, query)
366+
return resp, err
367+
}

‎pkg/app/AppService.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -442,7 +442,7 @@ func (impl *AppServiceImpl) CheckIfPipelineUpdateEventIsValidForAppStore(gitOpsA
442442
// drop event
443443
return isValid, installedAppVersionHistory, appId, envId, nil
444444
}
445-
if !impl.acdConfig.ArgoCDAutoSyncEnabled {
445+
if impl.acdConfig.IsManualSyncEnabled() {
446446
isArgoAppSynced := impl.pipelineStatusTimelineService.GetArgoAppSyncStatusForAppStore(installedAppVersionHistory.Id)
447447
if !isArgoAppSynced {
448448
return isValid, installedAppVersionHistory, appId, envId, nil
@@ -491,7 +491,7 @@ func (impl *AppServiceImpl) CheckIfPipelineUpdateEventIsValid(argoAppName, gitHa
491491
// drop event
492492
return isValid, pipeline, cdWfr, pipelineOverride, nil
493493
}
494-
if !impl.acdConfig.ArgoCDAutoSyncEnabled {
494+
if impl.acdConfig.IsManualSyncEnabled() {
495495
// if manual sync, proceed only if ARGOCD_SYNC_COMPLETED timeline is created
496496
isArgoAppSynced := impl.pipelineStatusTimelineService.GetArgoAppSyncStatus(cdWfr.Id)
497497
if !isArgoAppSynced {

‎pkg/app/ManifestPushService.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -148,7 +148,7 @@ func (impl *GitOpsManifestPushServiceImpl) PushChart(manifestPushTemplate *bean.
148148
gitCommitTimeline := impl.pipelineStatusTimelineService.GetTimelineDbObjectByTimelineStatusAndTimelineDescription(manifestPushTemplate.WorkflowRunnerId, 0, pipelineConfig.TIMELINE_STATUS_GIT_COMMIT, "Git commit done successfully.", manifestPushTemplate.UserId, time.Now())
149149

150150
timelines := []*pipelineConfig.PipelineStatusTimeline{gitCommitTimeline}
151-
if !impl.acdConfig.ArgoCDAutoSyncEnabled {
151+
if impl.acdConfig.IsManualSyncEnabled() {
152152
// if manual sync is enabled, add ARGOCD_SYNC_INITIATED_TIMELINE
153153
argoCDSyncInitiatedTimeline := impl.pipelineStatusTimelineService.GetTimelineDbObjectByTimelineStatusAndTimelineDescription(manifestPushTemplate.WorkflowRunnerId, 0, pipelineConfig.TIMELINE_STATUS_ARGOCD_SYNC_INITIATED, "argocd sync initiated.", manifestPushTemplate.UserId, time.Now())
154154
timelines = append(timelines, argoCDSyncInitiatedTimeline)

‎pkg/appStore/chartGroup/ChartGroupService.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -993,7 +993,7 @@ func (impl *ChartGroupServiceImpl) performDeployStageOnAcd(installedAppVersion *
993993
GetTimelineDbObjectByTimelineStatusAndTimelineDescription(0, installedAppVersion.InstalledAppVersionHistoryId, pipelineConfig.TIMELINE_STATUS_GIT_COMMIT, "Git commit done successfully.", installedAppVersion.UserId, time.Now())
994994

995995
timelines := []*pipelineConfig.PipelineStatusTimeline{GitCommitSuccessTimeline}
996-
if !impl.acdConfig.ArgoCDAutoSyncEnabled {
996+
if impl.acdConfig.IsManualSyncEnabled() {
997997
ArgocdSyncInitiatedTimeline := impl.pipelineStatusTimelineService.
998998
GetTimelineDbObjectByTimelineStatusAndTimelineDescription(0, installedAppVersion.InstalledAppVersionHistoryId, pipelineConfig.TIMELINE_STATUS_ARGOCD_SYNC_INITIATED, "ArgoCD sync initiated.", installedAppVersion.UserId, time.Now())
999999

‎pkg/appStore/installedApp/service/AppStoreDeploymentService.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -164,7 +164,7 @@ func (impl *AppStoreDeploymentServiceImpl) InstallApp(installAppVersionRequest *
164164
}
165165
if util.IsAcdApp(installAppVersionRequest.DeploymentAppType) {
166166
_ = impl.fullModeDeploymentService.SaveTimelineForHelmApps(installAppVersionRequest, pipelineConfig.TIMELINE_STATUS_GIT_COMMIT, "Git commit done successfully.", time.Now(), tx)
167-
if !impl.aCDConfig.ArgoCDAutoSyncEnabled {
167+
if impl.aCDConfig.IsManualSyncEnabled() {
168168
_ = impl.fullModeDeploymentService.SaveTimelineForHelmApps(installAppVersionRequest, pipelineConfig.TIMELINE_STATUS_ARGOCD_SYNC_INITIATED, "argocd sync initiated.", time.Now(), tx)
169169
}
170170
}
@@ -695,7 +695,7 @@ func (impl *AppStoreDeploymentServiceImpl) UpdateInstalledApp(ctx context.Contex
695695

696696
upgradeAppRequest.GitHash = gitOpsResponse.GitHash
697697
_ = impl.fullModeDeploymentService.SaveTimelineForHelmApps(upgradeAppRequest, pipelineConfig.TIMELINE_STATUS_GIT_COMMIT, "Git commit done successfully.", time.Now(), tx)
698-
if !impl.aCDConfig.ArgoCDAutoSyncEnabled {
698+
if impl.aCDConfig.IsManualSyncEnabled() {
699699
_ = impl.fullModeDeploymentService.SaveTimelineForHelmApps(upgradeAppRequest, pipelineConfig.TIMELINE_STATUS_ARGOCD_SYNC_INITIATED, "Argocd sync initiated", time.Now(), tx)
700700
}
701701
installedAppVersionHistory.GitHash = gitOpsResponse.GitHash

‎pkg/appStore/installedApp/service/FullMode/deployment/FullModeDeploymentService.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -156,7 +156,7 @@ func (impl *FullModeDeploymentServiceImpl) InstallApp(installAppVersionRequest *
156156
impl.Logger.Errorw("error in getting the argo application with normal refresh", "err", err)
157157
return nil, err
158158
}
159-
if !impl.acdConfig.ArgoCDAutoSyncEnabled {
159+
if impl.acdConfig.IsManualSyncEnabled() {
160160
timeline := &pipelineConfig.PipelineStatusTimeline{
161161
InstalledAppVersionHistoryId: installAppVersionRequest.InstalledAppVersionHistoryId,
162162
Status: pipelineConfig.TIMELINE_STATUS_ARGOCD_SYNC_COMPLETED,
@@ -304,7 +304,7 @@ func (impl *FullModeDeploymentServiceImpl) RollbackRelease(ctx context.Context,
304304
return installedApp, false, err
305305
}
306306

307-
isManualSync := !impl.acdConfig.ArgoCDAutoSyncEnabled
307+
isManualSync := impl.acdConfig.IsManualSyncEnabled()
308308

309309
GitCommitSuccessTimeline := impl.pipelineStatusTimelineService.
310310
GetTimelineDbObjectByTimelineStatusAndTimelineDescription(0, installedApp.InstalledAppVersionHistoryId, pipelineConfig.TIMELINE_STATUS_GIT_COMMIT, "Git commit done successfully.", installedApp.UserId, time.Now())

‎pkg/appStore/installedApp/service/FullMode/deployment/InstalledAppArgoCdService.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -127,7 +127,7 @@ func (impl *FullModeDeploymentServiceImpl) UpdateAndSyncACDApps(installAppVersio
127127
}
128128
return err
129129
}
130-
if !impl.acdConfig.ArgoCDAutoSyncEnabled {
130+
if impl.acdConfig.IsManualSyncEnabled() {
131131
err = impl.SaveTimelineForHelmApps(installAppVersionRequest, pipelineConfig.TIMELINE_STATUS_ARGOCD_SYNC_COMPLETED, "argocd sync completed", syncTime, tx)
132132
if err != nil {
133133
impl.Logger.Errorw("error in saving timeline for acd helm apps", "err", err)

‎pkg/deployment/trigger/devtronApps/TriggerService.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1041,7 +1041,7 @@ func (impl *TriggerServiceImpl) deployArgocdApp(overrideRequest *bean3.ValuesOve
10411041
impl.logger.Errorw("error in getting argo application with normal refresh", "argoAppName", valuesOverrideResponse.Pipeline.DeploymentAppName)
10421042
return fmt.Errorf("%s. err: %s", bean.ARGOCD_SYNC_ERROR, err.Error())
10431043
}
1044-
if !impl.ACDConfig.ArgoCDAutoSyncEnabled {
1044+
if impl.ACDConfig.IsManualSyncEnabled() {
10451045
timeline := &pipelineConfig.PipelineStatusTimeline{
10461046
CdWorkflowRunnerId: overrideRequest.WfrId,
10471047
StatusTime: syncTime,

‎pkg/workflow/status/WorkflowStatusService.go

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -193,7 +193,7 @@ func (impl *WorkflowStatusServiceImpl) UpdatePipelineTimelineAndStatusByLiveAppl
193193
return nil, isTimelineUpdated
194194
}
195195

196-
if !impl.acdConfig.ArgoCDAutoSyncEnabled {
196+
if impl.acdConfig.IsManualSyncEnabled() {
197197
// if manual sync check for application sync status
198198
isArgoAppSynced := impl.pipelineStatusTimelineService.GetArgoAppSyncStatus(cdWfr.Id)
199199
if !isArgoAppSynced {
@@ -277,7 +277,7 @@ func (impl *WorkflowStatusServiceImpl) UpdatePipelineTimelineAndStatusByLiveAppl
277277
// drop event
278278
return nil, isTimelineUpdated
279279
}
280-
if !impl.acdConfig.ArgoCDAutoSyncEnabled {
280+
if impl.acdConfig.IsManualSyncEnabled() {
281281
isArgoAppSynced := impl.pipelineStatusTimelineService.GetArgoAppSyncStatusForAppStore(installedAppVersionHistory.Id)
282282
if !isArgoAppSynced {
283283
return nil, isTimelineUpdated
@@ -469,7 +469,7 @@ func (impl *WorkflowStatusServiceImpl) CheckArgoPipelineTimelineStatusPeriodical
469469
}
470470

471471
func (impl *WorkflowStatusServiceImpl) syncACDDevtronApps(deployedBeforeMinutes int, pipelineId int) error {
472-
if impl.acdConfig.ArgoCDAutoSyncEnabled {
472+
if impl.acdConfig.IsAutoSyncEnabled() {
473473
// don't check for apps if auto sync is enabled
474474
return nil
475475
}
@@ -528,7 +528,7 @@ func (impl *WorkflowStatusServiceImpl) syncACDDevtronApps(deployedBeforeMinutes
528528
}
529529

530530
func (impl *WorkflowStatusServiceImpl) syncACDHelmApps(deployedBeforeMinutes int, installedAppVersionId int) error {
531-
if impl.acdConfig.ArgoCDAutoSyncEnabled {
531+
if impl.acdConfig.IsAutoSyncEnabled() {
532532
// don't check for apps if auto sync is enabled
533533
return nil
534534
}

0 commit comments

Comments
 (0)