From 25179619b35d43b2c9e866329835b7322cf74b16 Mon Sep 17 00:00:00 2001 From: prakhar katiyar Date: Wed, 2 Apr 2025 16:14:57 +0530 Subject: [PATCH 01/34] refactor --- .../sql/repository/AppListingRepository.go | 4 +-- pkg/app/AppListingService.go | 25 +++++++++++++------ 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/internal/sql/repository/AppListingRepository.go b/internal/sql/repository/AppListingRepository.go index 026aed924a..20a786708a 100644 --- a/internal/sql/repository/AppListingRepository.go +++ b/internal/sql/repository/AppListingRepository.go @@ -51,7 +51,7 @@ type AppListingRepository interface { DeploymentDetailByArtifactId(ciArtifactId int, envId int) (AppView.DeploymentDetailContainer, error) FindAppCount(isProd bool) (int, error) FetchAppsByEnvironmentV2(appListingFilter helper.AppListingFilter) ([]*AppView.AppEnvironmentContainer, int, error) - FetchOverviewAppsByEnvironment(envId, limit, offset int) ([]*AppView.AppEnvironmentContainer, error) + FetchAppsEnvContainers(envId, limit, offset int) ([]*AppView.AppEnvironmentContainer, error) FetchLastDeployedImage(appId, envId int) (*LastDeployed, error) } @@ -137,7 +137,7 @@ func (impl *AppListingRepositoryImpl) FetchOverviewCiPipelines(jobId int) ([]*Ap return jobContainers, nil } -func (impl *AppListingRepositoryImpl) FetchOverviewAppsByEnvironment(envId, limit, offset int) ([]*AppView.AppEnvironmentContainer, error) { +func (impl *AppListingRepositoryImpl) FetchAppsEnvContainers(envId, limit, offset int) ([]*AppView.AppEnvironmentContainer, error) { query := ` SELECT a.id as app_id,a.app_name,aps.status as app_status, ld.last_deployed_time, p.id as pipeline_id FROM app a INNER JOIN pipeline p ON p.app_id = a.id and p.deleted = false and p.environment_id = ? diff --git a/pkg/app/AppListingService.go b/pkg/app/AppListingService.go index ad2659551b..b48e99788f 100644 --- a/pkg/app/AppListingService.go +++ b/pkg/app/AppListingService.go @@ -233,15 +233,9 @@ func (impl AppListingServiceImpl) FetchOverviewAppsByEnvironment(envId, limit, o resp.CreatedBy = fmt.Sprintf("%s (inactive)", createdBy.EmailId) } } - envContainers, err := impl.appListingRepository.FetchOverviewAppsByEnvironment(envId, limit, offset) + envContainers, err := impl.FetchAppsEnvContainers(envId, limit, offset) if err != nil { - impl.Logger.Errorw("failed to fetch environment containers", "err", err, "envId", envId) - return resp, err - } - - err = impl.updateAppStatusForHelmTypePipelines(envContainers) - if err != nil { - impl.Logger.Errorw("err, updateAppStatusForHelmTypePipelines", "envId", envId, "err", err) + impl.Logger.Errorw("failed to fetch env containers", "err", err, "envId", envId) return resp, err } @@ -293,6 +287,21 @@ func getUniqueArtifacts(artifactIds []int) (uniqueArtifactIds []int) { return uniqueArtifactIds } +func (impl AppListingServiceImpl) FetchAppsEnvContainers(envId, limit, offset int) ([]*AppView.AppEnvironmentContainer, error) { + envContainers, err := impl.appListingRepository.FetchAppsEnvContainers(envId, limit, offset) + if err != nil { + impl.Logger.Errorw("failed to fetch environment containers", "err", err, "envId", envId) + return nil, err + } + + err = impl.updateAppStatusForHelmTypePipelines(envContainers) + if err != nil { + impl.Logger.Errorw("err, updateAppStatusForHelmTypePipelines", "envId", envId, "err", err) + return nil, err + } + return envContainers, nil +} + func (impl AppListingServiceImpl) FetchAllDevtronManagedApps() ([]AppNameTypeIdContainer, error) { impl.Logger.Debug("reached at FetchAllDevtronManagedApps:") apps := make([]AppNameTypeIdContainer, 0) From de43a47a24397bf4b6ba2d76ba327f66f4d8a5e0 Mon Sep 17 00:00:00 2001 From: prakhar katiyar Date: Wed, 2 Apr 2025 17:57:59 +0530 Subject: [PATCH 02/34] refactor --- .../sql/repository/AppListingRepository.go | 22 ++++++++++++++----- pkg/app/AppListingService.go | 7 +++--- wire_gen.go | 6 ++--- 3 files changed, 23 insertions(+), 12 deletions(-) diff --git a/internal/sql/repository/AppListingRepository.go b/internal/sql/repository/AppListingRepository.go index 20a786708a..e0a44a0b82 100644 --- a/internal/sql/repository/AppListingRepository.go +++ b/internal/sql/repository/AppListingRepository.go @@ -51,7 +51,7 @@ type AppListingRepository interface { DeploymentDetailByArtifactId(ciArtifactId int, envId int) (AppView.DeploymentDetailContainer, error) FindAppCount(isProd bool) (int, error) FetchAppsByEnvironmentV2(appListingFilter helper.AppListingFilter) ([]*AppView.AppEnvironmentContainer, int, error) - FetchAppsEnvContainers(envId, limit, offset int) ([]*AppView.AppEnvironmentContainer, error) + FetchAppsEnvContainers(envId, limit, offset int, appIds []int) ([]*AppView.AppEnvironmentContainer, error) FetchLastDeployedImage(appId, envId int) (*LastDeployed, error) } @@ -137,7 +137,7 @@ func (impl *AppListingRepositoryImpl) FetchOverviewCiPipelines(jobId int) ([]*Ap return jobContainers, nil } -func (impl *AppListingRepositoryImpl) FetchAppsEnvContainers(envId, limit, offset int) ([]*AppView.AppEnvironmentContainer, error) { +func (impl *AppListingRepositoryImpl) FetchAppsEnvContainers(envId int, limit, offset int, appIds []int) ([]*AppView.AppEnvironmentContainer, error) { query := ` SELECT a.id as app_id,a.app_name,aps.status as app_status, ld.last_deployed_time, p.id as pipeline_id FROM app a INNER JOIN pipeline p ON p.app_id = a.id and p.deleted = false and p.environment_id = ? @@ -145,17 +145,27 @@ func (impl *AppListingRepositoryImpl) FetchAppsEnvContainers(envId, limit, offse LEFT JOIN (SELECT pco.pipeline_id,MAX(pco.created_on) as last_deployed_time from pipeline_config_override pco GROUP BY pco.pipeline_id) ld ON ld.pipeline_id = p.id - WHERE a.active = true - ORDER BY a.app_name ` + WHERE a.active = true` + queryParams := []interface{}{envId, envId} + + // Add app IDs filter if provided + if len(appIds) > 0 { + query += " AND a.id IN (?)" + queryParams = append(queryParams, pg.In(appIds)) + } + + query += " ORDER BY a.app_name" + if limit > 0 { - query += fmt.Sprintf("LIMIT ? ") + query += " LIMIT ?" queryParams = append(queryParams, limit) } if offset > 0 { - query += fmt.Sprintf("OFFSET ? ") + query += " OFFSET ?" queryParams = append(queryParams, offset) } + var envContainers []*AppView.AppEnvironmentContainer _, err := impl.dbConnection.Query(&envContainers, query, queryParams...) return envContainers, err diff --git a/pkg/app/AppListingService.go b/pkg/app/AppListingService.go index b48e99788f..3a6cc17041 100644 --- a/pkg/app/AppListingService.go +++ b/pkg/app/AppListingService.go @@ -233,7 +233,8 @@ func (impl AppListingServiceImpl) FetchOverviewAppsByEnvironment(envId, limit, o resp.CreatedBy = fmt.Sprintf("%s (inactive)", createdBy.EmailId) } } - envContainers, err := impl.FetchAppsEnvContainers(envId, limit, offset) + var appIds []int + envContainers, err := impl.FetchAppsEnvContainers(envId, limit, offset, appIds) if err != nil { impl.Logger.Errorw("failed to fetch env containers", "err", err, "envId", envId) return resp, err @@ -287,8 +288,8 @@ func getUniqueArtifacts(artifactIds []int) (uniqueArtifactIds []int) { return uniqueArtifactIds } -func (impl AppListingServiceImpl) FetchAppsEnvContainers(envId, limit, offset int) ([]*AppView.AppEnvironmentContainer, error) { - envContainers, err := impl.appListingRepository.FetchAppsEnvContainers(envId, limit, offset) +func (impl AppListingServiceImpl) FetchAppsEnvContainers(envId, limit, offset int, appIds []int) ([]*AppView.AppEnvironmentContainer, error) { + envContainers, err := impl.appListingRepository.FetchAppsEnvContainers(envId, limit, offset, appIds) if err != nil { impl.Logger.Errorw("failed to fetch environment containers", "err", err, "envId", envId) return nil, err diff --git a/wire_gen.go b/wire_gen.go index b1ec4db5c5..679957d5ec 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -1,6 +1,6 @@ // Code generated by Wire. DO NOT EDIT. -//go:generate go run github.com/google/wire/cmd/wire +//go:generate go run -mod=mod github.com/google/wire/cmd/wire //go:build !wireinject // +build !wireinject @@ -701,7 +701,7 @@ func InitializeApp() (*App, error) { deploymentGroupRepositoryImpl := repository2.NewDeploymentGroupRepositoryImpl(sugaredLogger, db) pipelineStrategyHistoryRepositoryImpl := repository21.NewPipelineStrategyHistoryRepositoryImpl(sugaredLogger, db) pipelineStrategyHistoryServiceImpl := history.NewPipelineStrategyHistoryServiceImpl(sugaredLogger, pipelineStrategyHistoryRepositoryImpl, userServiceImpl) - propertiesConfigServiceImpl := pipeline.NewPropertiesConfigServiceImpl(sugaredLogger, envConfigOverrideRepositoryImpl, chartRepositoryImpl, environmentRepositoryImpl, deploymentTemplateHistoryServiceImpl, scopedVariableManagerImpl, deployedAppMetricsServiceImpl, envConfigOverrideReadServiceImpl, deploymentConfigServiceImpl) + propertiesConfigServiceImpl := pipeline.NewPropertiesConfigServiceImpl(sugaredLogger, envConfigOverrideRepositoryImpl, chartRepositoryImpl, environmentRepositoryImpl, deploymentTemplateHistoryServiceImpl, scopedVariableManagerImpl, deployedAppMetricsServiceImpl, envConfigOverrideReadServiceImpl, deploymentConfigServiceImpl, chartServiceImpl) installedAppDBExtendedServiceImpl := FullMode.NewInstalledAppDBExtendedServiceImpl(installedAppDBServiceImpl, appStatusServiceImpl, gitOpsConfigReadServiceImpl) gitOpsValidationServiceImpl := validation.NewGitOpsValidationServiceImpl(sugaredLogger, gitFactory, gitOperationServiceImpl, gitOpsConfigReadServiceImpl, chartTemplateServiceImpl, chartServiceImpl, installedAppDBExtendedServiceImpl) imageDigestPolicyServiceImpl := imageDigestPolicy.NewImageDigestPolicyServiceImpl(sugaredLogger, qualifierMappingServiceImpl, devtronResourceSearchableKeyServiceImpl) @@ -731,7 +731,7 @@ func InitializeApp() (*App, error) { deploymentTemplateValidationServiceImpl := deploymentTemplate.NewDeploymentTemplateValidationServiceImpl(sugaredLogger, chartRefServiceImpl, scopedVariableManagerImpl) devtronAppGitOpConfigServiceImpl := gitOpsConfig.NewDevtronAppGitOpConfigServiceImpl(sugaredLogger, chartRepositoryImpl, chartServiceImpl, gitOpsConfigReadServiceImpl, gitOpsValidationServiceImpl, argoClientWrapperServiceImpl, deploymentConfigServiceImpl, chartReadServiceImpl) cdHandlerImpl := pipeline.NewCdHandlerImpl(sugaredLogger, userServiceImpl, cdWorkflowRepositoryImpl, ciLogServiceImpl, ciArtifactRepositoryImpl, ciPipelineMaterialRepositoryImpl, pipelineRepositoryImpl, environmentRepositoryImpl, ciWorkflowRepositoryImpl, enforcerUtilImpl, resourceGroupServiceImpl, imageTaggingServiceImpl, k8sServiceImpl, workflowServiceImpl, clusterServiceImplExtended, blobStorageConfigServiceImpl, customTagServiceImpl, deploymentConfigServiceImpl, workFlowStageStatusServiceImpl, cdWorkflowRunnerServiceImpl) - appWorkflowServiceImpl := appWorkflow2.NewAppWorkflowServiceImpl(sugaredLogger, appWorkflowRepositoryImpl, ciCdPipelineOrchestratorImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, enforcerUtilImpl, resourceGroupServiceImpl, appRepositoryImpl, userAuthServiceImpl, chartServiceImpl, deploymentConfigServiceImpl) + appWorkflowServiceImpl := appWorkflow2.NewAppWorkflowServiceImpl(sugaredLogger, appWorkflowRepositoryImpl, ciCdPipelineOrchestratorImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, enforcerUtilImpl, resourceGroupServiceImpl, appRepositoryImpl, userAuthServiceImpl, chartServiceImpl, deploymentConfigServiceImpl, pipelineBuilderImpl) appCloneServiceImpl := appClone.NewAppCloneServiceImpl(sugaredLogger, pipelineBuilderImpl, attributesServiceImpl, chartServiceImpl, configMapServiceImpl, appWorkflowServiceImpl, appListingServiceImpl, propertiesConfigServiceImpl, pipelineStageServiceImpl, ciTemplateReadServiceImpl, appRepositoryImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, ciPipelineConfigServiceImpl, gitOpsConfigReadServiceImpl, chartReadServiceImpl) deploymentTemplateRepositoryImpl := repository2.NewDeploymentTemplateRepositoryImpl(db, sugaredLogger) deploymentTemplateHistoryReadServiceImpl := read7.NewDeploymentTemplateHistoryReadServiceImpl(sugaredLogger, deploymentTemplateHistoryRepositoryImpl, scopedVariableManagerImpl) From 31cdfec481b08b698e4b94705a8429dc59d058f4 Mon Sep 17 00:00:00 2001 From: prakhar katiyar Date: Thu, 3 Apr 2025 13:29:32 +0530 Subject: [PATCH 03/34] fix for Hibernating Filter --- pkg/app/AppListingService.go | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/pkg/app/AppListingService.go b/pkg/app/AppListingService.go index 3a6cc17041..917ca2db48 100644 --- a/pkg/app/AppListingService.go +++ b/pkg/app/AppListingService.go @@ -36,6 +36,7 @@ import ( "go.opentelemetry.io/otel" "golang.org/x/exp/slices" "net/http" + "sigs.k8s.io/kustomize/kyaml/sliceutil" "strconv" "strings" "time" @@ -378,6 +379,14 @@ func (impl AppListingServiceImpl) FetchAppsByEnvironmentV2(fetchAppListingReques if len(fetchAppListingRequest.Namespaces) != 0 && len(fetchAppListingRequest.Environments) == 0 { return []*AppView.AppEnvironmentContainer{}, 0, nil } + + // fun to check if "HIBERNATING" exists in fetchAppListingRequest.AppStatuses + isFilteredOnHibernatingStatus := impl.isFilteredOnHibernatingStatus(fetchAppListingRequest) + // remove ""HIBERNATING" from fetchAppListingRequest.AppStatuses + if isFilteredOnHibernatingStatus { + fetchAppListingRequest.AppStatuses = sliceutil.Remove(fetchAppListingRequest.AppStatuses, "HIBERNATING") + } + appListingFilter := helper.AppListingFilter{ Environments: fetchAppListingRequest.Environments, Statuses: fetchAppListingRequest.Statuses, @@ -431,9 +440,29 @@ func (impl AppListingServiceImpl) FetchAppsByEnvironmentV2(fetchAppListingReques if err != nil { impl.Logger.Errorw("error, UpdateAppStatusForHelmTypePipelines", "envIds", envIds, "err", err) } + + // apply filter for "HIBERNATING" status + if isFilteredOnHibernatingStatus { + filteredContainers := make([]*AppView.AppEnvironmentContainer, 0) + for _, container := range envContainers { + if container.AppStatus == "HIBERNATING" { + filteredContainers = append(filteredContainers, container) + } + } + envContainers = filteredContainers + } return envContainers, appSize, nil } +func (impl AppListingServiceImpl) isFilteredOnHibernatingStatus(fetchAppListingRequest FetchAppListingRequest) bool { + if fetchAppListingRequest.AppStatuses != nil && len(fetchAppListingRequest.AppStatuses) > 0 { + if slices.Contains(fetchAppListingRequest.AppStatuses, "HIBERNATING") { + return true + } + } + return false +} + func (impl AppListingServiceImpl) ISLastReleaseStopType(appId, envId int) (bool, error) { override, err := impl.pipelineOverrideRepository.GetLatestRelease(appId, envId) if err != nil && !util.IsErrNoRows(err) { From 63b37ed3e35990e72c7ffd01b59f94fc63a2a158 Mon Sep 17 00:00:00 2001 From: prakhar katiyar Date: Thu, 3 Apr 2025 13:47:19 +0530 Subject: [PATCH 04/34] fix for Hibernating Filter --- pkg/app/AppListingService.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/app/AppListingService.go b/pkg/app/AppListingService.go index 917ca2db48..23873e26ad 100644 --- a/pkg/app/AppListingService.go +++ b/pkg/app/AppListingService.go @@ -450,6 +450,7 @@ func (impl AppListingServiceImpl) FetchAppsByEnvironmentV2(fetchAppListingReques } } envContainers = filteredContainers + appSize = len(filteredContainers) } return envContainers, appSize, nil } From 02a3d61a6c0d9f06c514235f5989365e91fab777 Mon Sep 17 00:00:00 2001 From: prakhar katiyar Date: Thu, 3 Apr 2025 13:50:08 +0530 Subject: [PATCH 05/34] fix for Hibernating Filter --- pkg/app/AppListingService.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/app/AppListingService.go b/pkg/app/AppListingService.go index 23873e26ad..ab35ffda6d 100644 --- a/pkg/app/AppListingService.go +++ b/pkg/app/AppListingService.go @@ -380,6 +380,8 @@ func (impl AppListingServiceImpl) FetchAppsByEnvironmentV2(fetchAppListingReques return []*AppView.AppEnvironmentContainer{}, 0, nil } + // Currently AppStatus is available in Db for only ArgoApps + // We fetch AppStatus on the fly for Helm Apps from scoop, So AppStatus filter will be applied in last // fun to check if "HIBERNATING" exists in fetchAppListingRequest.AppStatuses isFilteredOnHibernatingStatus := impl.isFilteredOnHibernatingStatus(fetchAppListingRequest) // remove ""HIBERNATING" from fetchAppListingRequest.AppStatuses From 89109d3046cb0f2b9515a45ec6dddd9d297ad852 Mon Sep 17 00:00:00 2001 From: prakhar katiyar Date: Thu, 3 Apr 2025 15:53:30 +0530 Subject: [PATCH 06/34] fix for Hibernating Filter --- pkg/app/AppListingService.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pkg/app/AppListingService.go b/pkg/app/AppListingService.go index ab35ffda6d..1ec6a680e3 100644 --- a/pkg/app/AppListingService.go +++ b/pkg/app/AppListingService.go @@ -20,6 +20,7 @@ import ( "context" "fmt" "github.com/devtron-labs/devtron/api/bean/AppView" + bean2 "github.com/devtron-labs/devtron/client/argocdServer/bean" "github.com/devtron-labs/devtron/internal/middleware" "github.com/devtron-labs/devtron/internal/sql/repository/app" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow/cdWorkflow" @@ -386,7 +387,7 @@ func (impl AppListingServiceImpl) FetchAppsByEnvironmentV2(fetchAppListingReques isFilteredOnHibernatingStatus := impl.isFilteredOnHibernatingStatus(fetchAppListingRequest) // remove ""HIBERNATING" from fetchAppListingRequest.AppStatuses if isFilteredOnHibernatingStatus { - fetchAppListingRequest.AppStatuses = sliceutil.Remove(fetchAppListingRequest.AppStatuses, "HIBERNATING") + fetchAppListingRequest.AppStatuses = sliceutil.Remove(fetchAppListingRequest.AppStatuses, bean2.HIBERNATING) } appListingFilter := helper.AppListingFilter{ @@ -447,7 +448,7 @@ func (impl AppListingServiceImpl) FetchAppsByEnvironmentV2(fetchAppListingReques if isFilteredOnHibernatingStatus { filteredContainers := make([]*AppView.AppEnvironmentContainer, 0) for _, container := range envContainers { - if container.AppStatus == "HIBERNATING" { + if container.AppStatus == bean2.HIBERNATING { filteredContainers = append(filteredContainers, container) } } @@ -459,7 +460,7 @@ func (impl AppListingServiceImpl) FetchAppsByEnvironmentV2(fetchAppListingReques func (impl AppListingServiceImpl) isFilteredOnHibernatingStatus(fetchAppListingRequest FetchAppListingRequest) bool { if fetchAppListingRequest.AppStatuses != nil && len(fetchAppListingRequest.AppStatuses) > 0 { - if slices.Contains(fetchAppListingRequest.AppStatuses, "HIBERNATING") { + if slices.Contains(fetchAppListingRequest.AppStatuses, bean2.HIBERNATING) { return true } } From cf86d1982981ba3807a17c2ba0b18ed36711bea7 Mon Sep 17 00:00:00 2001 From: prakhar katiyar Date: Thu, 3 Apr 2025 17:00:16 +0530 Subject: [PATCH 07/34] fix for Hibernating Filter --- pkg/app/AppListingService.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pkg/app/AppListingService.go b/pkg/app/AppListingService.go index 1ec6a680e3..4cf62ee723 100644 --- a/pkg/app/AppListingService.go +++ b/pkg/app/AppListingService.go @@ -37,7 +37,6 @@ import ( "go.opentelemetry.io/otel" "golang.org/x/exp/slices" "net/http" - "sigs.k8s.io/kustomize/kyaml/sliceutil" "strconv" "strings" "time" @@ -386,8 +385,10 @@ func (impl AppListingServiceImpl) FetchAppsByEnvironmentV2(fetchAppListingReques // fun to check if "HIBERNATING" exists in fetchAppListingRequest.AppStatuses isFilteredOnHibernatingStatus := impl.isFilteredOnHibernatingStatus(fetchAppListingRequest) // remove ""HIBERNATING" from fetchAppListingRequest.AppStatuses + appStatusesFilter := make([]string, 0) if isFilteredOnHibernatingStatus { - fetchAppListingRequest.AppStatuses = sliceutil.Remove(fetchAppListingRequest.AppStatuses, bean2.HIBERNATING) + appStatusesFilter = fetchAppListingRequest.AppStatuses + fetchAppListingRequest.AppStatuses = []string{} } appListingFilter := helper.AppListingFilter{ @@ -448,7 +449,7 @@ func (impl AppListingServiceImpl) FetchAppsByEnvironmentV2(fetchAppListingReques if isFilteredOnHibernatingStatus { filteredContainers := make([]*AppView.AppEnvironmentContainer, 0) for _, container := range envContainers { - if container.AppStatus == bean2.HIBERNATING { + if slices.Contains(appStatusesFilter, container.AppStatus) { filteredContainers = append(filteredContainers, container) } } From 15270c204b28cf9017605c4bc6b85b52177c1a54 Mon Sep 17 00:00:00 2001 From: Neha Sharma Date: Sun, 6 Apr 2025 19:09:05 +0530 Subject: [PATCH 08/34] updtae the descriptions of env variables --- api/bean/AutocompleteConfig.go | 2 +- api/helm-app/gRPC/applicationClient.go | 2 +- api/helm-app/service/HelmAppService.go | 6 ++-- .../configure/PipelineConfigRestHandler.go | 2 +- .../argocdServer/ArgoClientWrapperService.go | 6 ++-- client/argocdServer/bean/connection.go | 4 +-- client/cron/CiStatusUpdateCron.go | 4 +-- client/cron/CiTriggerCron.go | 4 +-- client/dashboard/Config.go | 6 ++-- client/events/EventClient.go | 4 +-- client/gitSensor/GitSensorClient.go | 6 ++-- client/grafana/Config.go | 6 ++-- client/grafana/GrafanaClient.go | 8 ++--- client/lens/LensClient.go | 4 +-- client/proxy/ProxyRouter.go | 2 +- internal/sql/models/UserTerminalSession.go | 8 ++--- internal/util/BasicProviders.go | 2 +- otel/OtelTracingService.go | 2 +- pkg/app/AppCrudOperationService.go | 2 +- pkg/app/AppService.go | 18 +++++----- pkg/appStatus/AppStatusService_test.go | 14 ++++---- pkg/auth/authorisation/casbin/rbac.go | 8 ++--- pkg/auth/user/UserAuthService.go | 18 +++++----- pkg/auth/user/UserCommonService.go | 2 +- .../artifacts/imageTagging/read/Config.go | 2 +- .../git/commandManager/GitCommandManager.go | 4 +-- pkg/generateManifest/bean.go | 4 +-- pkg/infraConfig/bean/v1/infra_config.go | 2 +- pkg/k8s/K8sCommonService.go | 4 +-- pkg/k8s/application/k8sApplicationService.go | 2 +- pkg/module/bean/bean.go | 2 +- pkg/pipeline/CiService.go | 4 +-- pkg/server/config/ServerEnvConfig.go | 36 +++++++++---------- pkg/sql/connection.go | 6 ++-- pkg/terminal/terminalSesion.go | 4 +-- pkg/util/TokenCache.go | 14 ++++---- pkg/variables/ScopedVariableService.go | 8 ++--- .../parsers/VariableTemplateParser.go | 6 ++-- util/GlobalConfig.go | 23 ++++++------ .../authenticator/client/k8sClient.go | 4 +-- .../authenticator/client/oidcClient.go | 10 +++--- .../common-lib/pubsub-lib/JetStreamUtil.go | 4 +-- .../common-lib/pubsub-lib/NatsClient.go | 12 +++---- .../common-lib/utils/bean/bean.go | 6 ++-- .../common-lib/utils/grpc/GrpcConfig.go | 4 +-- .../devtron-labs/common-lib/utils/k8s/bean.go | 12 +++---- 46 files changed, 157 insertions(+), 156 deletions(-) diff --git a/api/bean/AutocompleteConfig.go b/api/bean/AutocompleteConfig.go index 3bb8b22a79..62d9191c0f 100644 --- a/api/bean/AutocompleteConfig.go +++ b/api/bean/AutocompleteConfig.go @@ -17,5 +17,5 @@ package bean type Config struct { - IgnoreAuthCheck bool `env:"IGNORE_AUTOCOMPLETE_AUTH_CHECK" envDefault:"false"` + IgnoreAuthCheck bool `env:"IGNORE_AUTOCOMPLETE_AUTH_CHECK" envDefault:"false" description: "flag for ignoring auth check in autocomplete apis."` } diff --git a/api/helm-app/gRPC/applicationClient.go b/api/helm-app/gRPC/applicationClient.go index 77df9019f2..cabe07bd6b 100644 --- a/api/helm-app/gRPC/applicationClient.go +++ b/api/helm-app/gRPC/applicationClient.go @@ -75,7 +75,7 @@ func NewHelmAppClientImpl(logger *zap.SugaredLogger, // CATEGORY=INFRA_SETUP type HelmClientConfig struct { - Url string `env:"HELM_CLIENT_URL" envDefault:"127.0.0.1:50051"` + Url string `env:"HELM_CLIENT_URL" envDefault:"127.0.0.1:50051" description: "Kubelink micro-service url "` } func GetConfig() (*HelmClientConfig, error) { diff --git a/api/helm-app/service/HelmAppService.go b/api/helm-app/service/HelmAppService.go index e31186f36f..72e13fbcef 100644 --- a/api/helm-app/service/HelmAppService.go +++ b/api/helm-app/service/HelmAppService.go @@ -159,9 +159,9 @@ func NewHelmAppServiceImpl(Logger *zap.SugaredLogger, clusterService cluster.Clu // CATEGORY=CD type HelmReleaseConfig struct { - RevisionHistoryLimitDevtronApp int `env:"REVISION_HISTORY_LIMIT_DEVTRON_APP" envDefault:"1"` - RevisionHistoryLimitHelmApp int `env:"REVISION_HISTORY_LIMIT_HELM_APP" envDefault:"1"` - RevisionHistoryLimitExternalHelmApp int `env:"REVISION_HISTORY_LIMIT_EXTERNAL_HELM_APP" envDefault:"0"` + RevisionHistoryLimitDevtronApp int `env:"REVISION_HISTORY_LIMIT_DEVTRON_APP" envDefault:"1" description: "Count for devtron application rivision history"` + RevisionHistoryLimitHelmApp int `env:"REVISION_HISTORY_LIMIT_HELM_APP" envDefault:"1" description: "To set the history limit for the helm app being deployed through devtron"` + RevisionHistoryLimitExternalHelmApp int `env:"REVISION_HISTORY_LIMIT_EXTERNAL_HELM_APP" envDefault:"0" description: "Count for external helm application rivision history"` RevisionHistoryLimitLinkedHelmApp int `env:"REVISION_HISTORY_LIMIT_LINKED_HELM_APP" envDefault:"15"` } diff --git a/api/restHandler/app/pipeline/configure/PipelineConfigRestHandler.go b/api/restHandler/app/pipeline/configure/PipelineConfigRestHandler.go index 5e3e81defd..5711a35b4c 100644 --- a/api/restHandler/app/pipeline/configure/PipelineConfigRestHandler.go +++ b/api/restHandler/app/pipeline/configure/PipelineConfigRestHandler.go @@ -68,7 +68,7 @@ import ( ) type PipelineRestHandlerEnvConfig struct { - UseArtifactListApiV2 bool `env:"USE_ARTIFACT_LISTING_API_V2" envDefault:"true"` //deprecated + UseArtifactListApiV2 bool `env:"USE_ARTIFACT_LISTING_API_V2" envDefault:"true" description: "To use the V2 API for listing artifacts in Listing the images in pipeline"` //deprecated } type DevtronAppRestHandler interface { diff --git a/client/argocdServer/ArgoClientWrapperService.go b/client/argocdServer/ArgoClientWrapperService.go index 4e3b634a13..4d19894aaf 100644 --- a/client/argocdServer/ArgoClientWrapperService.go +++ b/client/argocdServer/ArgoClientWrapperService.go @@ -51,9 +51,9 @@ import ( ) type ACDConfig struct { - ArgoCDAutoSyncEnabled bool `env:"ARGO_AUTO_SYNC_ENABLED" envDefault:"true"` // will gradually switch this flag to false in enterprise - RegisterRepoMaxRetryCount int `env:"ARGO_REPO_REGISTER_RETRY_COUNT" envDefault:"3"` - RegisterRepoMaxRetryDelay int `env:"ARGO_REPO_REGISTER_RETRY_DELAY" envDefault:"10"` + ArgoCDAutoSyncEnabled bool `env:"ARGO_AUTO_SYNC_ENABLED" envDefault:"true" description: "If enabled all argocd application will have auto sync enabled"` // will gradually switch this flag to false in enterprise + RegisterRepoMaxRetryCount int `env:"ARGO_REPO_REGISTER_RETRY_COUNT" envDefault:"3" description: "Argo app registration in argo retries on deployment"` + RegisterRepoMaxRetryDelay int `env:"ARGO_REPO_REGISTER_RETRY_DELAY" envDefault:"10" description: "Argo app registration in argo cd on deployment delay between retry"` } func (config *ACDConfig) IsManualSyncEnabled() bool { diff --git a/client/argocdServer/bean/connection.go b/client/argocdServer/bean/connection.go index 320c233a3f..7aaeb67f92 100644 --- a/client/argocdServer/bean/connection.go +++ b/client/argocdServer/bean/connection.go @@ -25,8 +25,8 @@ type AcdAuthConfig struct { } type Config struct { - Host string `env:"CD_HOST" envDefault:"localhost"` - Port string `env:"CD_PORT" envDefault:"8000"` + Host string `env:"CD_HOST" envDefault:"localhost" description: "Host for the devtron stack"` + Port string `env:"CD_PORT" envDefault:"8000" description: "Port for pre/post-cd" ` Namespace string `env:"CD_NAMESPACE" envDefault:"devtroncd"` } diff --git a/client/cron/CiStatusUpdateCron.go b/client/cron/CiStatusUpdateCron.go index 696feaed10..9ab20d1432 100644 --- a/client/cron/CiStatusUpdateCron.go +++ b/client/cron/CiStatusUpdateCron.go @@ -66,8 +66,8 @@ func NewCiStatusUpdateCronImpl(logger *zap.SugaredLogger, appService app.AppServ } type CiWorkflowStatusUpdateConfig struct { - CiWorkflowStatusUpdateCron string `env:"CI_WORKFLOW_STATUS_UPDATE_CRON" envDefault:"*/5 * * * *"` - TimeoutForFailedCiBuild string `env:"TIMEOUT_FOR_FAILED_CI_BUILD" envDefault:"15"` //in minutes + CiWorkflowStatusUpdateCron string `env:"CI_WORKFLOW_STATUS_UPDATE_CRON" envDefault:"*/5 * * * *" description: "Cron schedule for CI pipeline status"` + TimeoutForFailedCiBuild string `env:"TIMEOUT_FOR_FAILED_CI_BUILD" envDefault:"15" description: "Timeout for Failed CI build "` //in minutes } func GetCiWorkflowStatusUpdateConfig() (*CiWorkflowStatusUpdateConfig, error) { diff --git a/client/cron/CiTriggerCron.go b/client/cron/CiTriggerCron.go index c208ff8cc8..fd958236c3 100644 --- a/client/cron/CiTriggerCron.go +++ b/client/cron/CiTriggerCron.go @@ -68,8 +68,8 @@ func NewCiTriggerCronImpl(logger *zap.SugaredLogger, cfg *CiTriggerCronConfig, p } type CiTriggerCronConfig struct { - SourceControllerCronTime int `env:"CI_TRIGGER_CRON_TIME" envDefault:"2"` - PluginName string `env:"PLUGIN_NAME" envDefault:"Pull images from container repository"` + SourceControllerCronTime int `env:"CI_TRIGGER_CRON_TIME" envDefault:"2" description: "For image poll plugin"` + PluginName string `env:"PLUGIN_NAME" envDefault:"Pull images from container repository" description: "Handles image retrieval from a container repository and triggers subsequent CI processes upon detecting new images.Current default plugin name: Pull Images from Container Repository."` } func GetCiTriggerCronConfig() (*CiTriggerCronConfig, error) { diff --git a/client/dashboard/Config.go b/client/dashboard/Config.go index c961d56a26..2c71b39982 100644 --- a/client/dashboard/Config.go +++ b/client/dashboard/Config.go @@ -22,9 +22,9 @@ import ( // CATEGORY=INFRA_SETUP type Config struct { - Host string `env:"DASHBOARD_HOST" envDefault:"localhost"` - Port string `env:"DASHBOARD_PORT" envDefault:"3000"` - Namespace string `env:"DASHBOARD_NAMESPACE" envDefault:"devtroncd"` + Host string `env:"DASHBOARD_HOST" envDefault:"localhost" description: "Dashboard micro-service URL"` + Port string `env:"DASHBOARD_PORT" envDefault:"3000" description: "Port for dashboard micro-service"` + Namespace string `env:"DASHBOARD_NAMESPACE" envDefault:"devtroncd" description: "Dashboard micro-service namespace"` } func GetConfig() (*Config, error) { diff --git a/client/events/EventClient.go b/client/events/EventClient.go index a58cadea82..2b472c1d77 100644 --- a/client/events/EventClient.go +++ b/client/events/EventClient.go @@ -38,8 +38,8 @@ import ( ) type EventClientConfig struct { - DestinationURL string `env:"EVENT_URL" envDefault:"http://localhost:3000/notify"` - NotificationMedium NotificationMedium `env:"NOTIFICATION_MEDIUM" envDefault:"rest"` + DestinationURL string `env:"EVENT_URL" envDefault:"http://localhost:3000/notify" description: "Notifier service url"` + NotificationMedium NotificationMedium `env:"NOTIFICATION_MEDIUM" envDefault:"rest" description: "notification medium"` } type NotificationMedium string diff --git a/client/gitSensor/GitSensorClient.go b/client/gitSensor/GitSensorClient.go index 6539fe5ea2..f12b029f29 100644 --- a/client/gitSensor/GitSensorClient.go +++ b/client/gitSensor/GitSensorClient.go @@ -81,9 +81,9 @@ func NewGitSensorClient(logger *zap.SugaredLogger, config *ClientConfig) (*Clien // CATEGORY=INFRA_SETUP type ClientConfig struct { - Url string `env:"GIT_SENSOR_URL" envDefault:"127.0.0.1:7070"` - Protocol string `env:"GIT_SENSOR_PROTOCOL" envDefault:"REST"` - Timeout int `env:"GIT_SENSOR_TIMEOUT" envDefault:"0"` // in seconds + Url string `env:"GIT_SENSOR_URL" envDefault:"127.0.0.1:7070" description: "git-sensor micro-service url "` + Protocol string `env:"GIT_SENSOR_PROTOCOL" envDefault:"REST" description: "Protocol to connect with git-sensor micro-service"` + Timeout int `env:"GIT_SENSOR_TIMEOUT" envDefault:"0" description: "Timeout for getting response from the git-sensor"` // in seconds } func GetConfig() (*ClientConfig, error) { diff --git a/client/grafana/Config.go b/client/grafana/Config.go index 1e0cb359cc..37dee8ef5d 100644 --- a/client/grafana/Config.go +++ b/client/grafana/Config.go @@ -19,9 +19,9 @@ package grafana import "github.com/caarlos0/env" type Config struct { - Host string `env:"GRAFANA_HOST" envDefault:"localhost"` - Port string `env:"GRAFANA_PORT" envDefault:"8090"` - Namespace string `env:"GRAFANA_NAMESPACE" envDefault:"devtroncd"` + Host string `env:"GRAFANA_HOST" envDefault:"localhost" description: "Host URL for the grafana dashboard"` + Port string `env:"GRAFANA_PORT" envDefault:"8090" description: "Port for grafana micro-service"` + Namespace string `env:"GRAFANA_NAMESPACE" envDefault:"devtroncd" description: "Namespace for grafana"` } func GetConfig() (*Config, error) { diff --git a/client/grafana/GrafanaClient.go b/client/grafana/GrafanaClient.go index 6e903615af..bb2ff7e004 100644 --- a/client/grafana/GrafanaClient.go +++ b/client/grafana/GrafanaClient.go @@ -34,10 +34,10 @@ import ( ) type GrafanaClientConfig struct { - GrafanaUsername string `env:"GRAFANA_USERNAME" envDefault:"admin"` - GrafanaPassword string `env:"GRAFANA_PASSWORD" envDefault:"prom-operator"` - GrafanaOrgId int `env:"GRAFANA_ORG_ID" envDefault:"2"` - DestinationURL string `env:"GRAFANA_URL" envDefault:""` + GrafanaUsername string `env:"GRAFANA_USERNAME" envDefault:"admin" description: "Username for grafana "` + GrafanaPassword string `env:"GRAFANA_PASSWORD" envDefault:"prom-operator" description: "Password for grafana dashboard"` + GrafanaOrgId int `env:"GRAFANA_ORG_ID" envDefault:"2" description: "Org ID for grafana for application metrics"` + DestinationURL string `env:"GRAFANA_URL" envDefault:"" description: "Host URL for the grafana dashboard"` } const PromDatasource = "/api/datasources" diff --git a/client/lens/LensClient.go b/client/lens/LensClient.go index 707d41423c..c5c64ebd27 100644 --- a/client/lens/LensClient.go +++ b/client/lens/LensClient.go @@ -30,8 +30,8 @@ import ( ) type LensConfig struct { - Url string `env:"LENS_URL" envDefault:"http://lens-milandevtron-service:80"` - Timeout int `env:"LENS_TIMEOUT" envDefault:"0"` // in seconds + Url string `env:"LENS_URL" envDefault:"http://lens-milandevtron-service:80" description: "Lens micro-service URL"` + Timeout int `env:"LENS_TIMEOUT" envDefault:"0" description: "Lens microservice timeout."` // in seconds } type StatusCode int diff --git a/client/proxy/ProxyRouter.go b/client/proxy/ProxyRouter.go index 6f57457f87..63ce696788 100644 --- a/client/proxy/ProxyRouter.go +++ b/client/proxy/ProxyRouter.go @@ -38,7 +38,7 @@ type ProxyConnection struct { } type Config struct { - ProxyServiceConfig string `env:"PROXY_SERVICE_CONFIG" envDefault:"{}"` + ProxyServiceConfig string `env:"PROXY_SERVICE_CONFIG" envDefault:"{}" description: "Proxy configuration for micro-service to be accessible on orhcestrator ingress"` } func GetProxyConfig() (*Config, error) { diff --git a/internal/sql/models/UserTerminalSession.go b/internal/sql/models/UserTerminalSession.go index 4ece768395..4db1013ea7 100644 --- a/internal/sql/models/UserTerminalSession.go +++ b/internal/sql/models/UserTerminalSession.go @@ -55,10 +55,10 @@ type UserTerminalPodEvents struct { } type UserTerminalSessionConfig struct { - MaxSessionPerUser int `env:"MAX_SESSION_PER_USER" envDefault:"5"` - TerminalPodStatusSyncTimeInSecs int `env:"TERMINAL_POD_STATUS_SYNC_In_SECS" envDefault:"600"` - TerminalPodDefaultNamespace string `env:"TERMINAL_POD_DEFAULT_NAMESPACE" envDefault:"default"` - TerminalPodInActiveDurationInMins int `env:"TERMINAL_POD_INACTIVE_DURATION_IN_MINS" envDefault:"10"` + MaxSessionPerUser int `env:"MAX_SESSION_PER_USER" envDefault:"5" description: "max no of cluster terminal pods can be created by an user"` + TerminalPodStatusSyncTimeInSecs int `env:"TERMINAL_POD_STATUS_SYNC_In_SECS" envDefault:"600" description: "this is the time interval at which the status of the cluster terminal pod"` + TerminalPodDefaultNamespace string `env:"TERMINAL_POD_DEFAULT_NAMESPACE" envDefault:"default" description: "Cluster terminal default namespace"` + TerminalPodInActiveDurationInMins int `env:"TERMINAL_POD_INACTIVE_DURATION_IN_MINS" envDefault:"10" description: "Timeout for cluster terminal to be inactive"` } type UserTerminalSessionResponse struct { diff --git a/internal/util/BasicProviders.go b/internal/util/BasicProviders.go index 1d9c8f5521..65e1b19c80 100644 --- a/internal/util/BasicProviders.go +++ b/internal/util/BasicProviders.go @@ -39,7 +39,7 @@ func GetLogger() *zap.SugaredLogger { type LogConfig struct { Level int `env:"LOG_LEVEL" envDefault:"0"` // default info - DevMode bool `env:"LOGGER_DEV_MODE" envDefault:"false"` + DevMode bool `env:"LOGGER_DEV_MODE" envDefault:"false" description: "Enables a different logger theme."` } func InitLogger() (*zap.SugaredLogger, error) { diff --git a/otel/OtelTracingService.go b/otel/OtelTracingService.go index a6b5a2639f..9457258462 100644 --- a/otel/OtelTracingService.go +++ b/otel/OtelTracingService.go @@ -48,7 +48,7 @@ func NewOtelTracingServiceImpl(logger *zap.SugaredLogger) *OtelTracingServiceImp } type OtelConfig struct { - OtelCollectorUrl string `env:"OTEL_COLLECTOR_URL" envDefault:""` + OtelCollectorUrl string `env:"OTEL_COLLECTOR_URL" envDefault:"" description: "Opentelemetry URL "` } // Init configures an OpenTelemetry exporter and trace provider diff --git a/pkg/app/AppCrudOperationService.go b/pkg/app/AppCrudOperationService.go index ff4399513a..e03b123c00 100644 --- a/pkg/app/AppCrudOperationService.go +++ b/pkg/app/AppCrudOperationService.go @@ -53,7 +53,7 @@ const ( ) type CrudOperationServiceConfig struct { - PropagateExtraLabels bool `env:"PROPAGATE_EXTRA_LABELS" envDefault:"false"` + PropagateExtraLabels bool `env:"PROPAGATE_EXTRA_LABELS" envDefault:"false" description: "Add additional propagate labels like api.devtron.ai/appName, api.devtron.ai/envName, api.devtron.ai/project along with the user defined ones."` } type AppCrudOperationService interface { diff --git a/pkg/app/AppService.go b/pkg/app/AppService.go index 5bb33db0c8..0d29403b1d 100644 --- a/pkg/app/AppService.go +++ b/pkg/app/AppService.go @@ -72,16 +72,16 @@ import ( // CATEGORY=CD type AppServiceConfig struct { - CdPipelineStatusCronTime string `env:"CD_PIPELINE_STATUS_CRON_TIME" envDefault:"*/2 * * * *"` - CdHelmPipelineStatusCronTime string `env:"CD_HELM_PIPELINE_STATUS_CRON_TIME" envDefault:"*/2 * * * *"` - CdPipelineStatusTimeoutDuration string `env:"CD_PIPELINE_STATUS_TIMEOUT_DURATION" envDefault:"20"` // in minutes - PipelineDegradedTime string `env:"PIPELINE_DEGRADED_TIME" envDefault:"10"` // in minutes - GetPipelineDeployedWithinHours int `env:"DEPLOY_STATUS_CRON_GET_PIPELINE_DEPLOYED_WITHIN_HOURS" envDefault:"12"` // in hours - HelmPipelineStatusCheckEligibleTime string `env:"HELM_PIPELINE_STATUS_CHECK_ELIGIBLE_TIME" envDefault:"120"` // in seconds + CdPipelineStatusCronTime string `env:"CD_PIPELINE_STATUS_CRON_TIME" envDefault:"*/2 * * * *" description: "Cron time for CD pipeline status"` + CdHelmPipelineStatusCronTime string `env:"CD_HELM_PIPELINE_STATUS_CRON_TIME" envDefault:"*/2 * * * *" description: "Cron time to check the pipeline status "` + CdPipelineStatusTimeoutDuration string `env:"CD_PIPELINE_STATUS_TIMEOUT_DURATION" envDefault:"20" description: "Timeout for CD pipeline to get healthy" ` // in minutes + PipelineDegradedTime string `env:"PIPELINE_DEGRADED_TIME" envDefault:"10" description: "Time to mark a pipeline degraded if not healthy in defined time"` // in minutes + GetPipelineDeployedWithinHours int `env:"DEPLOY_STATUS_CRON_GET_PIPELINE_DEPLOYED_WITHIN_HOURS" envDefault:"12" description: "This flag is used to fetch the deployment status of the application. It retrieves the status of deployments that occurred between 12 hours and 10 minutes prior to the current time. It fetches non-terminal statuses."` // in hours + HelmPipelineStatusCheckEligibleTime string `env:"HELM_PIPELINE_STATUS_CHECK_ELIGIBLE_TIME" envDefault:"120" description: "eligible time for checking helm app status periodically and update in db, value is in seconds., default is 120, if wfr is updated within configured time i.e. HELM_PIPELINE_STATUS_CHECK_ELIGIBLE_TIME then do not include for this cron cycle."` // in seconds ExposeCDMetrics bool `env:"EXPOSE_CD_METRICS" envDefault:"false"` - DevtronChartHelmInstallRequestTimeout int `env:"DEVTRON_CHART_INSTALL_REQUEST_TIMEOUT" envDefault:"6"` // in minutes - DevtronChartArgoCdInstallRequestTimeout int `env:"DEVTRON_CHART_ARGO_CD_INSTALL_REQUEST_TIMEOUT" envDefault:"1"` // in minutes - ArgoCdManualSyncCronPipelineDeployedBefore int `env:"ARGO_APP_MANUAL_SYNC_TIME" envDefault:"3"` // in minutes + DevtronChartHelmInstallRequestTimeout int `env:"DEVTRON_CHART_INSTALL_REQUEST_TIMEOUT" envDefault:"6" description: "Context timeout for no gitops concurrent async deployments"` // in minutes + DevtronChartArgoCdInstallRequestTimeout int `env:"DEVTRON_CHART_ARGO_CD_INSTALL_REQUEST_TIMEOUT" envDefault:"1" description: "Context timeout for gitops concurrent async deployments"` // in minutes + ArgoCdManualSyncCronPipelineDeployedBefore int `env:"ARGO_APP_MANUAL_SYNC_TIME" envDefault:"3" description: "retry argocd app manual sync if the timeline is stuck in ARGOCD_SYNC_INITIATED state for more than this defined time (in mins)"` // in minutes } func GetAppServiceConfig() (*AppServiceConfig, error) { diff --git a/pkg/appStatus/AppStatusService_test.go b/pkg/appStatus/AppStatusService_test.go index 1dcd0b6a53..537a1bce54 100644 --- a/pkg/appStatus/AppStatusService_test.go +++ b/pkg/appStatus/AppStatusService_test.go @@ -189,13 +189,13 @@ func TestDeleteWithAppIdEnvId(t *testing.T) { } type Config struct { - Addr string `env:"TEST_PG_ADDR" envDefault:"127.0.0.1"` - Port string `env:"TEST_PG_PORT" envDefault:"55000"` - User string `env:"TEST_PG_USER" envDefault:"postgres"` - Password string `env:"TEST_PG_PASSWORD" envDefault:"postgrespw" secretData:"-"` - Database string `env:"TEST_PG_DATABASE" envDefault:"orchestrator"` - ApplicationName string `env:"TEST_APP" envDefault:"orchestrator"` - LogQuery bool `env:"TEST_PG_LOG_QUERY" envDefault:"true"` + Addr string `env:"TEST_PG_ADDR" envDefault:"127.0.0.1" description: "Postgres URL"` + Port string `env:"TEST_PG_PORT" envDefault:"55000" description: "Postgres port number"` + User string `env:"TEST_PG_USER" envDefault:"postgres"description: "Postgres user"` + Password string `env:"TEST_PG_PASSWORD" envDefault:"postgrespw" secretData:"-" description: "Postgres user password"` + Database string `env:"TEST_PG_DATABASE" envDefault:"orchestrator" description: "Postgres database name"` + ApplicationName string `env:"TEST_APP" envDefault:"orchestrator" description: "application name"` + LogQuery bool `env:"TEST_PG_LOG_QUERY" envDefault:"true" description: "Log PostgreSQL queries."` } func getDbConn() (*pg.DB, error) { diff --git a/pkg/auth/authorisation/casbin/rbac.go b/pkg/auth/authorisation/casbin/rbac.go index 7edf521df3..1921e2500b 100644 --- a/pkg/auth/authorisation/casbin/rbac.go +++ b/pkg/auth/authorisation/casbin/rbac.go @@ -74,10 +74,10 @@ type CacheData struct { // CATEGORY=RBAC type EnforcerConfig struct { - CacheEnabled bool `env:"ENFORCER_CACHE" envDefault:"false"` - CacheExpirationInSecs int `env:"ENFORCER_CACHE_EXPIRATION_IN_SEC" envDefault:"86400"` - EnforcerBatchSize int `env:"ENFORCER_MAX_BATCH_SIZE" envDefault:"1"` - UseCasbinV2 bool `env:"USE_CASBIN_V2" envDefault:"false"` + CacheEnabled bool `env:"ENFORCER_CACHE" envDefault:"false" description: "To Enable enforcer cache."` + CacheExpirationInSecs int `env:"ENFORCER_CACHE_EXPIRATION_IN_SEC" envDefault:"86400" description: "Expiration time (in seconds) for enforcer cache. "` + EnforcerBatchSize int `env:"ENFORCER_MAX_BATCH_SIZE" envDefault:"1" description: "Maximum batch size for the enforcer."` + UseCasbinV2 bool `env:"USE_CASBIN_V2" envDefault:"false" description: "To enable casbin V2 API"` } func getConfig() (*EnforcerConfig, error) { diff --git a/pkg/auth/user/UserAuthService.go b/pkg/auth/user/UserAuthService.go index 4e735dbd35..e1e88a092f 100644 --- a/pkg/auth/user/UserAuthService.go +++ b/pkg/auth/user/UserAuthService.go @@ -94,18 +94,18 @@ var Claims struct { } type DexConfig struct { - RedirectURL string `env:"DEX_RURL" envDefault:"http://127.0.0.1:8080/callback"` - ClientID string `env:"DEX_CID" envDefault:"example-app"` - ClientSecret string `env:"DEX_SECRET" ` - DexURL string `env:"DEX_URL" ` - DexJwtKey string `env:"DEX_JWTKEY" ` - CStoreKey string `env:"DEX_CSTOREKEY"` - CookieExpirationTime int `env:"CExpirationTime" envDefault:"600"` - JwtExpirationTime int `env:"JwtExpirationTime" envDefault:"120"` + RedirectURL string `env:"DEX_RURL" envDefault:"http://127.0.0.1:8080/callback" description: "Dex redirect URL(http://argocd-dex-server.devtroncd:8080/callback)"` + ClientID string `env:"DEX_CID" envDefault:"example-app" description: "dex client id "` + ClientSecret string `env:"DEX_SECRET" description: "Dex secret"` + DexURL string `env:"DEX_URL" description: "Dex service endpoint with dex path(http://argocd-dex-server.devtroncd:5556/dex)"` + DexJwtKey string `env:"DEX_JWTKEY" description: "DEX JWT key. "` + CStoreKey string `env:"DEX_CSTOREKEY" description: "DEX CSTOREKEY."` + CookieExpirationTime int `env:"CExpirationTime" envDefault:"600" description: "Caching expiration time."` + JwtExpirationTime int `env:"JwtExpirationTime" envDefault:"120" description: "JWT expiration time."` } type WebhookToken struct { - WebhookToken string `env:"WEBHOOK_TOKEN" envDefault:""` + WebhookToken string `env:"WEBHOOK_TOKEN" envDefault:"" description: "If you want to continue using jenkins for CI then please provide this for authentication of requests"` } func NewUserAuthServiceImpl(userAuthRepository repository.UserAuthRepository, sessionManager *middleware.SessionManager, diff --git a/pkg/auth/user/UserCommonService.go b/pkg/auth/user/UserCommonService.go index 7f5b0961e4..5756eb9e4b 100644 --- a/pkg/auth/user/UserCommonService.go +++ b/pkg/auth/user/UserCommonService.go @@ -105,7 +105,7 @@ func NewUserCommonServiceImpl(userAuthRepository repository.UserAuthRepository, } type UserRbacConfig struct { - UseRbacCreationV2 bool `env:"USE_RBAC_CREATION_V2" envDefault:"true"` + UseRbacCreationV2 bool `env:"USE_RBAC_CREATION_V2" envDefault:"true" description: "To use the V2 for RBAC creation"` } func (impl UserCommonServiceImpl) CreateDefaultPoliciesForAllTypes(roleFieldsDto *bean.RoleModelFieldsDto, userId int32) (bool, error, []bean3.Policy) { diff --git a/pkg/build/artifacts/imageTagging/read/Config.go b/pkg/build/artifacts/imageTagging/read/Config.go index 7f74dc3a2c..f29424e53e 100644 --- a/pkg/build/artifacts/imageTagging/read/Config.go +++ b/pkg/build/artifacts/imageTagging/read/Config.go @@ -19,7 +19,7 @@ package read // ImageTaggingServiceConfig holds the configuration for the image tagging service type ImageTaggingServiceConfig struct { // HideImageTaggingHardDelete is a flag to hide the hard delete option in the image tagging service - HideImageTaggingHardDelete bool `env:"HIDE_IMAGE_TAGGING_HARD_DELETE" envDefault:"false"` + HideImageTaggingHardDelete bool `env:"HIDE_IMAGE_TAGGING_HARD_DELETE" envDefault:"false" description: "Flag to hide the hard delete option in the image tagging service"` } func (c *ImageTaggingServiceConfig) IsHardDeleteHidden() bool { diff --git a/pkg/deployment/gitOps/git/commandManager/GitCommandManager.go b/pkg/deployment/gitOps/git/commandManager/GitCommandManager.go index d62b315e4b..21fed3d942 100644 --- a/pkg/deployment/gitOps/git/commandManager/GitCommandManager.go +++ b/pkg/deployment/gitOps/git/commandManager/GitCommandManager.go @@ -45,8 +45,8 @@ func NewGitCommandManager(logger *zap.SugaredLogger) GitCommandManager { } type configuration struct { - UseGitCli bool `env:"USE_GIT_CLI" envDefault:"false"` - CliCmdTimeoutGlobal int `env:"CLI_CMD_TIMEOUT_GLOBAL_SECONDS" envDefault:"0"` + UseGitCli bool `env:"USE_GIT_CLI" envDefault:"false" description: "To enable git cli"` + CliCmdTimeoutGlobal int `env:"CLI_CMD_TIMEOUT_GLOBAL_SECONDS" envDefault:"0" description: "Used in git cli opeartion timeout"` } func ParseConfiguration() (*configuration, error) { diff --git a/pkg/generateManifest/bean.go b/pkg/generateManifest/bean.go index bb9b0d397b..a87317a914 100644 --- a/pkg/generateManifest/bean.go +++ b/pkg/generateManifest/bean.go @@ -92,6 +92,6 @@ type ResourceMetadata struct { GroupVersionKind schema.GroupVersionKind `json:"groupVersionKind"` } type RestartWorkloadConfig struct { - WorkerPoolSize int `env:"FEATURE_RESTART_WORKLOAD_WORKER_POOL_SIZE" envDefault:"5"` - RequestBatchSize int `env:"FEATURE_RESTART_WORKLOAD_BATCH_SIZE" envDefault:"1"` + WorkerPoolSize int `env:"FEATURE_RESTART_WORKLOAD_WORKER_POOL_SIZE" envDefault:"5" description: "restart workload retrieval pool size"` + RequestBatchSize int `env:"FEATURE_RESTART_WORKLOAD_BATCH_SIZE" envDefault:"1" description: "restart workload retrieval batch size "` } diff --git a/pkg/infraConfig/bean/v1/infra_config.go b/pkg/infraConfig/bean/v1/infra_config.go index 3f57e57fae..9e5a75e3f6 100644 --- a/pkg/infraConfig/bean/v1/infra_config.go +++ b/pkg/infraConfig/bean/v1/infra_config.go @@ -31,7 +31,7 @@ type InfraConfig struct { CiReqMem string `env:"REQ_CI_MEM" envDefault:"3G"` // CiDefaultTimeout is the default timeout for CI jobs in seconds // Earlier it was in int64, but now it is in float64 - CiDefaultTimeout float64 `env:"DEFAULT_TIMEOUT" envDefault:"3600"` + CiDefaultTimeout float64 `env:"DEFAULT_TIMEOUT" envDefault:"3600" description: "Timeout for CI to be completed"` // cm and cs ConfigMaps []bean.ConfigSecretMap `env:"-"` diff --git a/pkg/k8s/K8sCommonService.go b/pkg/k8s/K8sCommonService.go index 0c3a2a9401..cdca257bc5 100644 --- a/pkg/k8s/K8sCommonService.go +++ b/pkg/k8s/K8sCommonService.go @@ -77,8 +77,8 @@ type K8sCommonServiceImpl struct { ClusterReadService read.ClusterReadService } type K8sApplicationServiceConfig struct { - BatchSize int `env:"BATCH_SIZE" envDefault:"5"` - TimeOutInSeconds int `env:"TIMEOUT_IN_SECONDS" envDefault:"5"` + BatchSize int `env:"BATCH_SIZE" envDefault:"5" description: "there is feature to get URL's of services/ingresses. so to extract those, we need to parse all the servcie and ingress objects of the application. this BATCH_SIZE flag controls the no of these objects get parsed in one go."` + TimeOutInSeconds int `env:"TIMEOUT_IN_SECONDS" envDefault:"5" description: "timeout to compute the urls from services and ingress objects of an application"` } func NewK8sCommonServiceImpl(Logger *zap.SugaredLogger, k8sUtils *k8s.K8sServiceImpl, diff --git a/pkg/k8s/application/k8sApplicationService.go b/pkg/k8s/application/k8sApplicationService.go index 39588ef15e..c2648fffcc 100644 --- a/pkg/k8s/application/k8sApplicationService.go +++ b/pkg/k8s/application/k8sApplicationService.go @@ -145,7 +145,7 @@ func NewK8sApplicationServiceImpl(Logger *zap.SugaredLogger, clusterService clus } type EphemeralContainerConfig struct { - EphemeralServerVersionRegex string `env:"EPHEMERAL_SERVER_VERSION_REGEX" envDefault:"v[1-9]\\.\\b(2[3-9]|[3-9][0-9])\\b.*"` + EphemeralServerVersionRegex string `env:"EPHEMERAL_SERVER_VERSION_REGEX" envDefault:"v[1-9]\\.\\b(2[3-9]|[3-9][0-9])\\b.*" description: "ephemeral containers support version regex that is compared with k8sServerVersion"` } func (impl *K8sApplicationServiceImpl) ValidatePodLogsRequestQuery(r *http.Request) (*bean4.ResourceRequestBean, error) { diff --git a/pkg/module/bean/bean.go b/pkg/module/bean/bean.go index 2cd5a95ff4..fd874781b3 100644 --- a/pkg/module/bean/bean.go +++ b/pkg/module/bean/bean.go @@ -52,7 +52,7 @@ type ActionResponse struct { } type ModuleEnvConfig struct { - ModuleStatusHandlingCronDurationInMin int `env:"MODULE_STATUS_HANDLING_CRON_DURATION_MIN" envDefault:"3"` // default 3 minutes + ModuleStatusHandlingCronDurationInMin int `env:"MODULE_STATUS_HANDLING_CRON_DURATION_MIN" envDefault:"3" description: "Module status update cron, used in stack manager, to update the module installation status."` // default 3 minutes } type ModuleResourceStatusDto struct { diff --git a/pkg/pipeline/CiService.go b/pkg/pipeline/CiService.go index d5282add0b..0aea4c435a 100644 --- a/pkg/pipeline/CiService.go +++ b/pkg/pipeline/CiService.go @@ -83,8 +83,8 @@ type CiService interface { UpdateCiWorkflowWithStage(wf *pipelineConfig.CiWorkflow) error } type BuildxCacheFlags struct { - BuildxCacheModeMin bool `env:"BUILDX_CACHE_MODE_MIN" envDefault:"false"` - AsyncBuildxCacheExport bool `env:"ASYNC_BUILDX_CACHE_EXPORT" envDefault:"false"` + BuildxCacheModeMin bool `env:"BUILDX_CACHE_MODE_MIN" envDefault:"false" description: "To set build cache mode to minimum in buildx" ` + AsyncBuildxCacheExport bool `env:"ASYNC_BUILDX_CACHE_EXPORT" envDefault:"false" description: "To enable async container image cache export"` } type CiServiceImpl struct { diff --git a/pkg/server/config/ServerEnvConfig.go b/pkg/server/config/ServerEnvConfig.go index 87de843bc0..faeb8a8241 100644 --- a/pkg/server/config/ServerEnvConfig.go +++ b/pkg/server/config/ServerEnvConfig.go @@ -22,24 +22,24 @@ import ( ) type ServerEnvConfig struct { - DevtronInstallationType string `env:"DEVTRON_INSTALLATION_TYPE"` - InstallerCrdObjectGroupName string `env:"INSTALLER_CRD_OBJECT_GROUP_NAME" envDefault:"installer.devtron.ai"` - InstallerCrdObjectVersion string `env:"INSTALLER_CRD_OBJECT_VERSION" envDefault:"v1alpha1"` - InstallerCrdObjectResource string `env:"INSTALLER_CRD_OBJECT_RESOURCE" envDefault:"installers"` - InstallerCrdNamespace string `env:"INSTALLER_CRD_NAMESPACE" envDefault:"devtroncd"` - DevtronHelmRepoName string `env:"DEVTRON_HELM_REPO_NAME" envDefault:"devtron"` - DevtronHelmRepoUrl string `env:"DEVTRON_HELM_REPO_URL" envDefault:"https://helm.devtron.ai"` - DevtronHelmReleaseName string `env:"DEVTRON_HELM_RELEASE_NAME" envDefault:"devtron"` - DevtronHelmReleaseNamespace string `env:"DEVTRON_HELM_RELEASE_NAMESPACE" envDefault:"devtroncd"` - DevtronHelmReleaseChartName string `env:"DEVTRON_HELM_RELEASE_CHART_NAME" envDefault:"devtron-operator"` - DevtronVersionIdentifierInHelmValues string `env:"DEVTRON_VERSION_IDENTIFIER_IN_HELM_VALUES" envDefault:"installer.release"` - DevtronModulesIdentifierInHelmValues string `env:"DEVTRON_MODULES_IDENTIFIER_IN_HELM_VALUES" envDefault:"installer.modules"` - DevtronBomUrl string `env:"DEVTRON_BOM_URL" envDefault:"https://raw.githubusercontent.com/devtron-labs/devtron/%s/charts/devtron/devtron-bom.yaml"` - AppSyncImage string `env:"APP_SYNC_IMAGE" envDefault:"quay.io/devtron/chart-sync:1227622d-132-3775"` - AppSyncServiceAccount string `env:"APP_SYNC_SERVICE_ACCOUNT" envDefault:"chart-sync"` - AppSyncJobResourcesObj string `env:"APP_SYNC_JOB_RESOURCES_OBJ"` - ModuleMetaDataApiUrl string `env:"MODULE_METADATA_API_URL" envDefault:"https://api.devtron.ai/module?name=%s"` - ParallelismLimitForTagProcessing int `env:"PARALLELISM_LIMIT_FOR_TAG_PROCESSING"` + DevtronInstallationType string `env:"DEVTRON_INSTALLATION_TYPE" description: "Devtron Installation type(EA/Full)"` + InstallerCrdObjectGroupName string `env:"INSTALLER_CRD_OBJECT_GROUP_NAME" envDefault:"installer.devtron.ai" description: "Devtron installer CRD group name, partially deprecated."` + InstallerCrdObjectVersion string `env:"INSTALLER_CRD_OBJECT_VERSION" envDefault:"v1alpha1" description: "version of the CRDs. default is v1alpha1"` + InstallerCrdObjectResource string `env:"INSTALLER_CRD_OBJECT_RESOURCE" envDefault:"installers" description: "Devtron installer CRD resource name, partially deprecated"` + InstallerCrdNamespace string `env:"INSTALLER_CRD_NAMESPACE" envDefault:"devtroncd" description: "namespace where Custom Resource Definitions get installed"` + DevtronHelmRepoName string `env:"DEVTRON_HELM_REPO_NAME" envDefault:"devtron" description: "Is used to install modules (stack manager)"` + DevtronHelmRepoUrl string `env:"DEVTRON_HELM_REPO_URL" envDefault:"https://helm.devtron.ai" description: "Is used to install modules (stack manager)"` + DevtronHelmReleaseName string `env:"DEVTRON_HELM_RELEASE_NAME" envDefault:"devtron" description: "Name of the Devtron Helm release. "` + DevtronHelmReleaseNamespace string `env:"DEVTRON_HELM_RELEASE_NAMESPACE" envDefault:"devtroncd" description: "Namespace of the Devtron Helm release"` + DevtronHelmReleaseChartName string `env:"DEVTRON_HELM_RELEASE_CHART_NAME" envDefault:"devtron-operator" description: ""` + DevtronVersionIdentifierInHelmValues string `env:"DEVTRON_VERSION_IDENTIFIER_IN_HELM_VALUES" envDefault:"installer.release" description: "devtron operator version identifier in helm values yaml"` + DevtronModulesIdentifierInHelmValues string `env:"DEVTRON_MODULES_IDENTIFIER_IN_HELM_VALUES" envDefault:"installer.modules" ` + DevtronBomUrl string `env:"DEVTRON_BOM_URL" envDefault:"https://raw.githubusercontent.com/devtron-labs/devtron/%s/charts/devtron/devtron-bom.yaml" description: "Path to devtron-bom.yaml of devtron charts, used for module installation and devtron upgrade"` + AppSyncImage string `env:"APP_SYNC_IMAGE" envDefault:"quay.io/devtron/chart-sync:1227622d-132-3775" description: "For the app sync image, this image will be used in app-manual sync job"` + AppSyncServiceAccount string `env:"APP_SYNC_SERVICE_ACCOUNT" envDefault:"chart-sync" description: "Service account to be used in app sync Job"` + AppSyncJobResourcesObj string `env:"APP_SYNC_JOB_RESOURCES_OBJ" description: "To pass the resource of app sync"` + ModuleMetaDataApiUrl string `env:"MODULE_METADATA_API_URL" envDefault:"https://api.devtron.ai/module?name=%s" description: "Modules list and meta info will be fetched from this server, that is central api server of devtron."` + ParallelismLimitForTagProcessing int `env:"PARALLELISM_LIMIT_FOR_TAG_PROCESSING" description: "App manual sync job parallel tag processing count."` AppSyncJobShutDownWaitDuration int `env:"APP_SYNC_SHUTDOWN_WAIT_DURATION" envDefault:"120"` ErrorEncounteredOnGettingDevtronHelmRelease error } diff --git a/pkg/sql/connection.go b/pkg/sql/connection.go index fa6858e5de..478d327d23 100644 --- a/pkg/sql/connection.go +++ b/pkg/sql/connection.go @@ -34,10 +34,10 @@ type Config struct { User string `env:"PG_USER" envDefault:"" description:"user for postgres" example:"postgres"` Password string `env:"PG_PASSWORD" envDefault:"" secretData:"-" description:"password for postgres, associated with PG_USER" example:"confidential ;)"` Database string `env:"PG_DATABASE" envDefault:"orchestrator" description:"postgres database to be made connection with" example:"orchestrator, casbin, git_sensor, lens"` - CasbinDatabase string `env:"CASBIN_DATABASE" envDefault:"casbin""` + CasbinDatabase string `env:"CASBIN_DATABASE" envDefault:"casbin" description: "Database for casbin"` ApplicationName string `env:"APP" envDefault:"orchestrator" description:"Application name"` - ReadTimeout int64 `env:"PG_READ_TIMEOUT" envDefault:"30"` - WriteTimeout int64 `env:"PG_WRITE_TIMEOUT" envDefault:"30"` + ReadTimeout int64 `env:"PG_READ_TIMEOUT" envDefault:"30" description: "Time out for read operation in postgres"` + WriteTimeout int64 `env:"PG_WRITE_TIMEOUT" envDefault:"30" description: "Time out for write operation in postgres"` bean.PgQueryMonitoringConfig } diff --git a/pkg/terminal/terminalSesion.go b/pkg/terminal/terminalSesion.go index b2b5f2841a..e6c87c5cd9 100644 --- a/pkg/terminal/terminalSesion.go +++ b/pkg/terminal/terminalSesion.go @@ -282,8 +282,8 @@ func handleTerminalSession(session sockjs.Session) { } type SocketConfig struct { - SocketHeartbeatSeconds int `env:"SOCKET_HEARTBEAT_SECONDS" envDefault:"25"` - SocketDisconnectDelay int `env:"SOCKET_DISCONNECT_DELAY_SECONDS" envDefault:"5"` + SocketHeartbeatSeconds int `env:"SOCKET_HEARTBEAT_SECONDS" envDefault:"25" description: "In order to keep proxies and load balancers from closing long running http requests we need to pretend that the connection is active and send a heartbeat packet once in a while. This setting controls how often this is done. By default a heartbeat packet is sent every 25 seconds."` + SocketDisconnectDelay int `env:"SOCKET_DISCONNECT_DELAY_SECONDS" envDefault:"5" description: "The server closes a session when a client receiving connection have not been seen for a while.This delay is configured by this setting. By default the session is closed when a receiving connection wasn't seen for 5 seconds."` } var cfg *SocketConfig diff --git a/pkg/util/TokenCache.go b/pkg/util/TokenCache.go index 4d0c52e34b..bcae68cf82 100644 --- a/pkg/util/TokenCache.go +++ b/pkg/util/TokenCache.go @@ -62,13 +62,13 @@ func (impl *TokenCache) BuildACDSynchContext() (acdContext context.Context, err // CATEGORY=GITOPS type ACDAuthConfig struct { - ACDUsername string `env:"ACD_USERNAME" envDefault:"admin"` - ACDPassword string `env:"ACD_PASSWORD" ` - ACDConfigMapName string `env:"ACD_CM" envDefault:"argocd-cm"` - ACDConfigMapNamespace string `env:"ACD_NAMESPACE" envDefault:"devtroncd"` - GitOpsSecretName string `env:"GITOPS_SECRET_NAME" envDefault:"devtron-gitops-secret"` - ResourceListForReplicas string `env:"RESOURCE_LIST_FOR_REPLICAS" envDefault:"Deployment,Rollout,StatefulSet,ReplicaSet"` - ResourceListForReplicasBatchSize int `env:"RESOURCE_LIST_FOR_REPLICAS_BATCH_SIZE" envDefault:"5"` + ACDUsername string `env:"ACD_USERNAME" envDefault:"admin" description: "User name for argocd"` + ACDPassword string `env:"ACD_PASSWORD" description: "Password for the Argocd (deprecated)"` + ACDConfigMapName string `env:"ACD_CM" envDefault:"argocd-cm" description: "Name of the argocd CM"` + ACDConfigMapNamespace string `env:"ACD_NAMESPACE" envDefault:"devtroncd" description: "To pass the argocd namespace"` + GitOpsSecretName string `env:"GITOPS_SECRET_NAME" envDefault:"devtron-gitops-secret" description: "devtron-gitops-secret"` + ResourceListForReplicas string `env:"RESOURCE_LIST_FOR_REPLICAS" envDefault:"Deployment,Rollout,StatefulSet,ReplicaSet" description: "this holds the list of k8s resource names which support replicas key. this list used in hibernate/un hibernate process"` + ResourceListForReplicasBatchSize int `env:"RESOURCE_LIST_FOR_REPLICAS_BATCH_SIZE" envDefault:"5" description: "this the batch size to control no of above resources can be parsed in one go to determine hibernate status"` } func GetACDAuthConfig() (*ACDAuthConfig, error) { diff --git a/pkg/variables/ScopedVariableService.go b/pkg/variables/ScopedVariableService.go index a9e851edcb..b5361e4992 100644 --- a/pkg/variables/ScopedVariableService.go +++ b/pkg/variables/ScopedVariableService.go @@ -75,10 +75,10 @@ func NewScopedVariableServiceImpl(logger *zap.SugaredLogger, scopedVariableRepos } type VariableConfig struct { - VariableNameRegex string `env:"SCOPED_VARIABLE_NAME_REGEX" envDefault:"^[a-zA-Z][a-zA-Z0-9_-]{0,62}[a-zA-Z0-9]$"` - VariableCacheEnabled bool `env:"VARIABLE_CACHE_ENABLED" envDefault:"true"` - SystemVariablePrefix string `env:"SYSTEM_VAR_PREFIX" envDefault:"DEVTRON_"` - ScopedVariableFormat string `env:"SCOPED_VARIABLE_FORMAT" envDefault:"@{{%s}}"` + VariableNameRegex string `env:"SCOPED_VARIABLE_NAME_REGEX" envDefault:"^[a-zA-Z][a-zA-Z0-9_-]{0,62}[a-zA-Z0-9]$" description: "Regex for scoped variable name that must passed this regex."` + VariableCacheEnabled bool `env:"VARIABLE_CACHE_ENABLED" envDefault:"true" description: "This is used to control caching of all the scope variables defined in the system."` + SystemVariablePrefix string `env:"SYSTEM_VAR_PREFIX" envDefault:"DEVTRON_" description: "Scoped variable prefix, variable name must have this prefix."` + ScopedVariableFormat string `env:"SCOPED_VARIABLE_FORMAT" envDefault:"@{{%s}}" description: "Its a scope format for varialbe name."` } func loadVariableCache(cfg *VariableConfig, service *ScopedVariableServiceImpl) { diff --git a/pkg/variables/parsers/VariableTemplateParser.go b/pkg/variables/parsers/VariableTemplateParser.go index 68447a0b6f..fcfc023705 100644 --- a/pkg/variables/parsers/VariableTemplateParser.go +++ b/pkg/variables/parsers/VariableTemplateParser.go @@ -57,9 +57,9 @@ func NewVariableTemplateParserImpl(logger *zap.SugaredLogger) (*VariableTemplate } type VariableTemplateParserConfig struct { - ScopedVariableEnabled bool `env:"SCOPED_VARIABLE_ENABLED" envDefault:"false"` - ScopedVariableHandlePrimitives bool `env:"SCOPED_VARIABLE_HANDLE_PRIMITIVES" envDefault:"false"` - VariableExpressionRegex string `env:"VARIABLE_EXPRESSION_REGEX" envDefault:"@{{([^}]+)}}"` + ScopedVariableEnabled bool `env:"SCOPED_VARIABLE_ENABLED" envDefault:"false" description: "To enable scoped variable option"` + ScopedVariableHandlePrimitives bool `env:"SCOPED_VARIABLE_HANDLE_PRIMITIVES" envDefault:"false" description: "This describe should we handle primitives or not in scoped variable template parsing."` + VariableExpressionRegex string `env:"VARIABLE_EXPRESSION_REGEX" envDefault:"@{{([^}]+)}}" description: "Scoped variable expression regex"` } func (cfg VariableTemplateParserConfig) isScopedVariablesDisabled() bool { diff --git a/util/GlobalConfig.go b/util/GlobalConfig.go index 4a58fae1e9..1e2d02b3c8 100644 --- a/util/GlobalConfig.go +++ b/util/GlobalConfig.go @@ -31,7 +31,7 @@ type EnvironmentVariables struct { // CATEGORY=CD type DeploymentServiceTypeConfig struct { - ExternallyManagedDeploymentType bool `env:"IS_INTERNAL_USE" envDefault:"false"` + ExternallyManagedDeploymentType bool `env:"IS_INTERNAL_USE" envDefault:"false" description: "If enabled then cd pipeline and helm apps will not need the deployment app type mandatorily. Couple this flag with HIDE_GITOPS_OR_HELM_OPTION (in Dashborad) and if gitops is configured and allowed for the env, pipeline/ helm app will gitops else no-gitops."` HelmInstallASyncMode bool `env:"RUN_HELM_INSTALL_IN_ASYNC_MODE_HELM_APPS" envDefault:"false"` UseDeploymentConfigData bool `env:"USE_DEPLOYMENT_CONFIG_DATA" envDefault:"false" description:"use deployment config data from deployment_config table" deprecated:"true"` MigrateDeploymentConfigData bool `env:"MIGRATE_DEPLOYMENT_CONFIG_DATA" envDefault:"false" description:"migrate deployment config data from charts table to deployment_config table" deprecated:"false"` @@ -47,26 +47,27 @@ func (d *DeploymentServiceTypeConfig) IsFeatureMigrateArgoCdApplicationEnable() } type GlobalEnvVariables struct { - GitOpsRepoPrefix string `env:"GITOPS_REPO_PREFIX" envDefault:""` - EnableAsyncHelmInstallDevtronChart bool `env:"ENABLE_ASYNC_INSTALL_DEVTRON_CHART" envDefault:"false"` - EnableAsyncArgoCdInstallDevtronChart bool `env:"ENABLE_ASYNC_ARGO_CD_INSTALL_DEVTRON_CHART" envDefault:"false"` - ArgoGitCommitRetryCountOnConflict int `env:"ARGO_GIT_COMMIT_RETRY_COUNT_ON_CONFLICT" envDefault:"3"` - ArgoGitCommitRetryDelayOnConflict int `env:"ARGO_GIT_COMMIT_RETRY_DELAY_ON_CONFLICT" envDefault:"1"` - ExposeCiMetrics bool `env:"EXPOSE_CI_METRICS" envDefault:"false"` - ExecuteWireNilChecker bool `env:"EXECUTE_WIRE_NIL_CHECKER" envDefault:"false"` + GitOpsRepoPrefix string `env:"GITOPS_REPO_PREFIX" envDefault:"" description: "Prefix for Gitops repo being creation for argocd application"` + EnableAsyncHelmInstallDevtronChart bool `env:"ENABLE_ASYNC_INSTALL_DEVTRON_CHART" envDefault:"false" description: "To enable async installation of no-gitops application"` + EnableAsyncArgoCdInstallDevtronChart bool `env:"ENABLE_ASYNC_ARGO_CD_INSTALL_DEVTRON_CHART" envDefault:"false" description: "To enable async installation of gitops application"` + ArgoGitCommitRetryCountOnConflict int `env:"ARGO_GIT_COMMIT_RETRY_COUNT_ON_CONFLICT" envDefault:"3" description: "retry argocd app manual sync if the timeline is stuck in ARGOCD_SYNC_INITIATED state for more than this defined time (in mins)" + ` + ArgoGitCommitRetryDelayOnConflict int `env:"ARGO_GIT_COMMIT_RETRY_DELAY_ON_CONFLICT" envDefault:"1" description: "Delay on retrying the maifest commit the on gitops"` + ExposeCiMetrics bool `env:"EXPOSE_CI_METRICS" envDefault:"false" description: "To expose CI metrics"` + ExecuteWireNilChecker bool `env:"EXECUTE_WIRE_NIL_CHECKER" envDefault:"false" description: "checks for any nil pointer in wire.go"` } type GlobalClusterConfig struct { - ClusterStatusCronTime int `env:"CLUSTER_STATUS_CRON_TIME" envDefault:"15"` + ClusterStatusCronTime int `env:"CLUSTER_STATUS_CRON_TIME" envDefault:"15" description: "Cron schedule for cluster status on resource browser"` } type DevtronSecretConfig struct { DevtronSecretName string `env:"DEVTRON_SECRET_NAME" envDefault:"devtron-secret"` - DevtronDexSecretNamespace string `env:"DEVTRON_DEX_SECRET_NAMESPACE" envDefault:"devtroncd"` + DevtronDexSecretNamespace string `env:"DEVTRON_DEX_SECRET_NAMESPACE" envDefault:"devtroncd" description: "Namespace of dex secret"` } type TerminalEnvVariables struct { - RestrictTerminalAccessForNonSuperUser bool `env:"RESTRICT_TERMINAL_ACCESS_FOR_NON_SUPER_USER" envDefault:"false"` + RestrictTerminalAccessForNonSuperUser bool `env:"RESTRICT_TERMINAL_ACCESS_FOR_NON_SUPER_USER" envDefault:"false" description: "To restrict the cluster terminal from user having non-super admin acceess"` } type InternalEnvVariables struct { diff --git a/vendor/github.com/devtron-labs/authenticator/client/k8sClient.go b/vendor/github.com/devtron-labs/authenticator/client/k8sClient.go index 09f4b8171d..abca34ff00 100644 --- a/vendor/github.com/devtron-labs/authenticator/client/k8sClient.go +++ b/vendor/github.com/devtron-labs/authenticator/client/k8sClient.go @@ -47,8 +47,8 @@ type K8sClient struct { } type RuntimeConfig struct { - LocalDevMode LocalDevMode `env:"RUNTIME_CONFIG_LOCAL_DEV" envDefault:"false"` - DevtronDefaultNamespaceName string `env:"DEVTRON_DEFAULT_NAMESPACE" envDefault:"devtroncd"` + LocalDevMode LocalDevMode `env:"RUNTIME_CONFIG_LOCAL_DEV" envDefault:"false" description: "Used in local development process. Allows to read from local kube config file path for default cluster operations."` + DevtronDefaultNamespaceName string `env:"DEVTRON_DEFAULT_NAMESPACE" envDefault:"devtroncd" description: "default namespace for devtron"` } func GetRuntimeConfig() (*RuntimeConfig, error) { diff --git a/vendor/github.com/devtron-labs/authenticator/client/oidcClient.go b/vendor/github.com/devtron-labs/authenticator/client/oidcClient.go index b7dcba2f99..25bd1ab23c 100644 --- a/vendor/github.com/devtron-labs/authenticator/client/oidcClient.go +++ b/vendor/github.com/devtron-labs/authenticator/client/oidcClient.go @@ -77,18 +77,18 @@ func getOidcClient(dexServerAddress string, settings *oidc.Settings, userVerifie const dexProxyUri = "api/dex" type DexConfig struct { - DexHost string `env:"DEX_HOST" envDefault:"http://localhost"` - DexPort string `env:"DEX_PORT" envDefault:"5556"` - DexClientID string `env:"DEX_CLIENT_ID" envDefault:"argo-cd"` + DexHost string `env:"DEX_HOST" envDefault:"http://localhost" description: "Dex service endpoint(http://argocd-dex-server.devtroncd)"` + DexPort string `env:"DEX_PORT" envDefault:"5556" description: "Dex service port"` + DexClientID string `env:"DEX_CLIENT_ID" envDefault:"argo-cd" description: "dex client id to be used when connecting to dex server"` DexServerAddress string Url string DexClientSecret string ServerSecret string // Specifies token expiration duration - UserSessionDurationSeconds int `env:"USER_SESSION_DURATION_SECONDS" envDefault:"86400"` + UserSessionDurationSeconds int `env:"USER_SESSION_DURATION_SECONDS" envDefault:"86400" description: "Session time after which user have to reLogin"` AdminPasswordMtime time.Time `json:"ADMIN_PASSWORD_MTIME"` DexConfigRaw string - DevtronSecretName string `env:"DEVTRON_SECRET_NAME" envDefault:"devtron-secret"` + DevtronSecretName string `env:"DEVTRON_SECRET_NAME" envDefault:"devtron-secret" description: "Secret name for orchestrator micro-services"` } func (c *DexConfig) GetDexProxyUrl() (string, error) { diff --git a/vendor/github.com/devtron-labs/common-lib/pubsub-lib/JetStreamUtil.go b/vendor/github.com/devtron-labs/common-lib/pubsub-lib/JetStreamUtil.go index a4fb42eefa..d0114e1519 100644 --- a/vendor/github.com/devtron-labs/common-lib/pubsub-lib/JetStreamUtil.go +++ b/vendor/github.com/devtron-labs/common-lib/pubsub-lib/JetStreamUtil.go @@ -125,10 +125,10 @@ type NatsTopic struct { } type ConfigJson struct { // StreamConfigJson is a json string of map[string]NatsStreamConfig - StreamConfigJson string `env:"STREAM_CONFIG_JSON"` + StreamConfigJson string `env:"STREAM_CONFIG_JSON" description: "this is json map of nats stream configurations per stream. eg: {"ORCHESTRATOR":{"max_age":3600,"replicas":1}}"` // ConsumerConfigJson is a json string of map[string]NatsConsumerConfig // eg: "{\"ARGO_PIPELINE_STATUS_UPDATE_DURABLE-1\" : \"{\"natsMsgProcessingBatchSize\" : 3, \"natsMsgBufferSize\" : 3, \"ackWaitInSecs\": 300}\"}" - ConsumerConfigJson string `env:"CONSUMER_CONFIG_JSON"` + ConsumerConfigJson string `env:"CONSUMER_CONFIG_JSON" description: "ConsumerConfigJson is a json string of map[string]NatsConsumerConfig eg: "{\"ARGO_PIPELINE_STATUS_UPDATE_DURABLE-1\" : \"{\"natsMsgProcessingBatchSize\" : 3, \"natsMsgBufferSize\" : 3, \"ackWaitInSecs\": 300}\"}""` } var natsTopicMapping = map[string]NatsTopic{ diff --git a/vendor/github.com/devtron-labs/common-lib/pubsub-lib/NatsClient.go b/vendor/github.com/devtron-labs/common-lib/pubsub-lib/NatsClient.go index 1da3710cd4..e82559c238 100644 --- a/vendor/github.com/devtron-labs/common-lib/pubsub-lib/NatsClient.go +++ b/vendor/github.com/devtron-labs/common-lib/pubsub-lib/NatsClient.go @@ -32,11 +32,11 @@ type NatsClient struct { } type NatsClientConfig struct { - NatsServerHost string `env:"NATS_SERVER_HOST" envDefault:"nats://devtron-nats.devtroncd:4222"` + NatsServerHost string `env:"NATS_SERVER_HOST" envDefault:"nats://devtron-nats.devtroncd:4222" description: "Nats micro-service URL "` // consumer wise // NatsMsgProcessingBatchSize is the number of messages that will be processed in one go - NatsMsgProcessingBatchSize int `env:"NATS_MSG_PROCESSING_BATCH_SIZE" envDefault:"1"` + NatsMsgProcessingBatchSize int `env:"NATS_MSG_PROCESSING_BATCH_SIZE" envDefault:"1" description: "NatsMsgProcessingBatchSize is the number of messages that will be processed in one go"` // NatsMsgBufferSize is the number of messages that will be buffered in memory (channel size) // it is recommended to set this value equal to NatsMsgProcessingBatchSize as we want to process maximum messages in the buffer in one go. @@ -45,10 +45,10 @@ type NatsClientConfig struct { // NatsMsgBufferSize can be configured independently of NatsMsgProcessingBatchSize if needed by setting its value to positive value in env. // if NatsMsgBufferSize set to a non-positive value then it will take the value of NatsMsgProcessingBatchSize. // Note: always get this value by calling GetNatsMsgBufferSize method - NatsMsgBufferSize int `env:"NATS_MSG_BUFFER_SIZE" envDefault:"-1"` - NatsMsgMaxAge int `env:"NATS_MSG_MAX_AGE" envDefault:"86400"` - NatsMsgAckWaitInSecs int `env:"NATS_MSG_ACK_WAIT_IN_SECS" envDefault:"120"` - NatsMsgReplicas int `env:"NATS_MSG_REPLICAS" envDefault:"0"` + NatsMsgBufferSize int `env:"NATS_MSG_BUFFER_SIZE" envDefault:"-1" description: "NatsMsgBufferSize is the number of messages that will be buffered in memory (channel size)"` + NatsMsgMaxAge int `env:"NATS_MSG_MAX_AGE" envDefault:"86400" description: "Age for the message to persist "` + NatsMsgAckWaitInSecs int `env:"NATS_MSG_ACK_WAIT_IN_SECS" envDefault:"120" description: "Time to wait for acknowledging the message"` + NatsMsgReplicas int `env:"NATS_MSG_REPLICAS" envDefault:"0" description: "Replica count for runnings nats instance"` } func (ncc NatsClientConfig) GetNatsMsgBufferSize() int { diff --git a/vendor/github.com/devtron-labs/common-lib/utils/bean/bean.go b/vendor/github.com/devtron-labs/common-lib/utils/bean/bean.go index 50b122e495..ac75e25e90 100644 --- a/vendor/github.com/devtron-labs/common-lib/utils/bean/bean.go +++ b/vendor/github.com/devtron-labs/common-lib/utils/bean/bean.go @@ -66,10 +66,10 @@ type DockerRegistryInfo struct { type PgQueryMonitoringConfig struct { LogSlowQuery bool `env:"PG_LOG_SLOW_QUERY" envDefault:"true"` - LogAllQuery bool `env:"PG_LOG_ALL_QUERY" envDefault:"false"` + LogAllQuery bool `env:"PG_LOG_ALL_QUERY" envDefault:"false" description: "Log all query ran by orchestrator in logs"` LogAllFailureQueries bool `env:"PG_LOG_ALL_FAILURE_QUERIES" envDefault:"true"` - ExportPromMetrics bool `env:"PG_EXPORT_PROM_METRICS" envDefault:"true"` - QueryDurationThreshold int64 `env:"PG_QUERY_DUR_THRESHOLD" envDefault:"5000"` + ExportPromMetrics bool `env:"PG_EXPORT_PROM_METRICS" envDefault:"true" description: "To expose postgres metrics"` + QueryDurationThreshold int64 `env:"PG_QUERY_DUR_THRESHOLD" envDefault:"5000" description: "if the particular querry execution should cross the threshold , then the querry should log."` ServiceName string } diff --git a/vendor/github.com/devtron-labs/common-lib/utils/grpc/GrpcConfig.go b/vendor/github.com/devtron-labs/common-lib/utils/grpc/GrpcConfig.go index 0ac82eedac..3e7a8df31d 100644 --- a/vendor/github.com/devtron-labs/common-lib/utils/grpc/GrpcConfig.go +++ b/vendor/github.com/devtron-labs/common-lib/utils/grpc/GrpcConfig.go @@ -3,8 +3,8 @@ package grpc import "github.com/caarlos0/env" type Configuration struct { - KubelinkMaxRecvMsgSize int `env:"KUBELINK_GRPC_MAX_RECEIVE_MSG_SIZE" envDefault:"20"` // In mb - KubelinkMaxSendMsgSize int `env:"KUBELINK_GRPC_MAX_SEND_MSG_SIZE" envDefault:"4"` // In mb + KubelinkMaxRecvMsgSize int `env:"KUBELINK_GRPC_MAX_RECEIVE_MSG_SIZE" envDefault:"20" description: "Message size to receive from kubelink"` // In mb + KubelinkMaxSendMsgSize int `env:"KUBELINK_GRPC_MAX_SEND_MSG_SIZE" envDefault:"4" description: "Message size to send to kubelink"` // In mb } func GetConfiguration() (*Configuration, error) { diff --git a/vendor/github.com/devtron-labs/common-lib/utils/k8s/bean.go b/vendor/github.com/devtron-labs/common-lib/utils/k8s/bean.go index 21f0a21be3..a69121b368 100644 --- a/vendor/github.com/devtron-labs/common-lib/utils/k8s/bean.go +++ b/vendor/github.com/devtron-labs/common-lib/utils/k8s/bean.go @@ -169,12 +169,12 @@ func GetResourceKey(obj *unstructured.Unstructured) ResourceKey { } type CustomK8sHttpTransportConfig struct { - UseCustomTransport bool `env:"USE_CUSTOM_HTTP_TRANSPORT" envDefault:"false"` - TimeOut int `env:"K8s_TCP_TIMEOUT" envDefault:"30"` - KeepAlive int `env:"K8s_TCP_KEEPALIVE" envDefault:"30"` - TLSHandshakeTimeout int `env:"K8s_TLS_HANDSHAKE_TIMEOUT" envDefault:"10"` - MaxIdleConnsPerHost int `env:"K8s_CLIENT_MAX_IDLE_CONNS_PER_HOST" envDefault:"25"` - IdleConnTimeout int `env:"K8s_TCP_IDLE_CONN_TIMEOUT" envDefault:"300"` + UseCustomTransport bool `env:"USE_CUSTOM_HTTP_TRANSPORT" envDefault:"false" description: "There is an issue in updating cluster bearer token with same cluster url due to transport layer token caching in k8s client lib. so we added a custom transport while calling k8s api server. this flag controls the usage of this transport.(custom or k8s)"` + TimeOut int `env:"K8s_TCP_TIMEOUT" envDefault:"30" description: "is the maximum amount of time a dial will wait for a connect to complete, required only if USE_CUSTOM_HTTP_TRANSPORT is true."` + KeepAlive int `env:"K8s_TCP_KEEPALIVE" envDefault:"30" description: "specifies the interval between keep-alive probes for an active, required only if USE_CUSTOM_HTTP_TRANSPORT is true."` + TLSHandshakeTimeout int `env:"K8s_TLS_HANDSHAKE_TIMEOUT" envDefault:"10" description: "specifies the maximum amount of time to wait for a TLS handshake. Zero means no timeout, required only if USE_CUSTOM_HTTP_TRANSPORT is true."` + MaxIdleConnsPerHost int `env:"K8s_CLIENT_MAX_IDLE_CONNS_PER_HOST" envDefault:"25" description: "k8s client max idle connections per host"` + IdleConnTimeout int `env:"K8s_TCP_IDLE_CONN_TIMEOUT" envDefault:"300" description: "maximum amount of time an idle (keep-alive) connection will remain idle before closing itself, required only if USE_CUSTOM_HTTP_TRANSPORT is true."` } type LocalDevMode bool From 315c6b50a6f9a915567193fde86cde1ff4ebcbc1 Mon Sep 17 00:00:00 2001 From: prakhar katiyar Date: Tue, 8 Apr 2025 19:04:50 +0530 Subject: [PATCH 09/34] GetAppMetadataListByEnvironment --- .../configure/BuildPipelineRestHandler.go | 71 +++++++++++++++++++ api/router/ResourceGroupingRouter.go | 1 + cmd/external-app/wire_gen.go | 2 +- pkg/app/AppListingService.go | 1 + pkg/pipeline/BuildPipelineConfigService.go | 28 +++++++- pkg/pipeline/bean/pipelineStage.go | 11 +++ wire_gen.go | 2 +- 7 files changed, 113 insertions(+), 3 deletions(-) diff --git a/api/restHandler/app/pipeline/configure/BuildPipelineRestHandler.go b/api/restHandler/app/pipeline/configure/BuildPipelineRestHandler.go index 32448e0abf..5203896a59 100644 --- a/api/restHandler/app/pipeline/configure/BuildPipelineRestHandler.go +++ b/api/restHandler/app/pipeline/configure/BuildPipelineRestHandler.go @@ -77,6 +77,7 @@ type DevtronAppBuildRestHandler interface { CancelWorkflow(w http.ResponseWriter, r *http.Request) UpdateBranchCiPipelinesWithRegex(w http.ResponseWriter, r *http.Request) + GetAppMetadataListByEnvironment(w http.ResponseWriter, r *http.Request) GetCiPipelineByEnvironment(w http.ResponseWriter, r *http.Request) GetCiPipelineByEnvironmentMin(w http.ResponseWriter, r *http.Request) GetExternalCiByEnvironment(w http.ResponseWriter, r *http.Request) @@ -2294,3 +2295,73 @@ func (handler *PipelineConfigRestHandlerImpl) GetSourceCiDownStreamInfo(w http.R } common.WriteJsonResp(w, err, linkedCIDetails, http.StatusOK) } + +func (handler *PipelineConfigRestHandlerImpl) GetAppMetadataListByEnvironment(w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) + token := r.Header.Get("token") + userId, err := handler.userAuthService.GetLoggedInUser(r) + if userId == 0 || err != nil { + common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) + return + } + envId, err := strconv.Atoi(vars["envId"]) + if err != nil { + handler.Logger.Errorw("request err, GetAppMetadataListByEnvironment", "err", err, "envId", envId) + common.WriteJsonResp(w, err, nil, http.StatusBadRequest) + return + } + v := r.URL.Query() + appIdsString := v.Get("appIds") + var appIds []int + if len(appIdsString) > 0 { + appIdsSlices := strings.Split(appIdsString, ",") + for _, appId := range appIdsSlices { + id, err := strconv.Atoi(appId) + if err != nil { + common.WriteJsonResp(w, err, "please provide valid appIds", http.StatusBadRequest) + return + } + appIds = append(appIds, id) + } + } + + resp, err := handler.pipelineBuilder.GetAppMetadataListByEnvironment(envId, appIds) + if err != nil { + handler.Logger.Errorw("service err, GetAppMetadataListByEnvironment", "err", err, "envId", envId) + common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) + return + } + // return all if user is super admin + if isActionUserSuperAdmin := handler.enforcer.Enforce(token, casbin.ResourceGlobal, casbin.ActionGet, "*"); isActionUserSuperAdmin { + common.WriteJsonResp(w, err, resp, http.StatusOK) + return + } + + // get all the appIds + appIds = make([]int, 0) + appContainers := resp.Apps + for _, appBean := range resp.Apps { + appIds = append(appIds, appBean.AppId) + } + + // get rbac objects for the appids + rbacObjectsWithAppId := handler.enforcerUtil.GetRbacObjectsByAppIds(appIds) + rbacObjects := make([]string, len(rbacObjectsWithAppId)) + itr := 0 + for _, object := range rbacObjectsWithAppId { + rbacObjects[itr] = object + itr++ + } + // enforce rbac in batch + rbacResult := handler.enforcer.EnforceInBatch(token, casbin.ResourceApplications, casbin.ActionGet, rbacObjects) + // filter out rbac passed apps + resp.Apps = make([]*bean1.AppMetaData, 0) + for _, appBean := range appContainers { + rbacObject := rbacObjectsWithAppId[appBean.AppId] + if rbacResult[rbacObject] { + resp.Apps = append(resp.Apps, appBean) + } + } + resp.AppCount = len(resp.Apps) + common.WriteJsonResp(w, err, resp, http.StatusOK) +} diff --git a/api/router/ResourceGroupingRouter.go b/api/router/ResourceGroupingRouter.go index 00c22eda15..ffc8418a69 100644 --- a/api/router/ResourceGroupingRouter.go +++ b/api/router/ResourceGroupingRouter.go @@ -45,6 +45,7 @@ func NewResourceGroupingRouterImpl(restHandler configure.PipelineConfigRestHandl func (router ResourceGroupingRouterImpl) InitResourceGroupingRouter(resourceGroupingRouter *mux.Router) { resourceGroupingRouter.Path("/{envId}/app-wf"). HandlerFunc(router.appWorkflowRestHandler.FindAppWorkflowByEnvironment).Methods("GET") + resourceGroupingRouter.Path("/{envId}/app-metadata").HandlerFunc(router.pipelineConfigRestHandler.GetAppMetadataListByEnvironment).Methods("GET") resourceGroupingRouter.Path("/{envId}/ci-pipeline").HandlerFunc(router.pipelineConfigRestHandler.GetCiPipelineByEnvironment).Methods("GET") resourceGroupingRouter.Path("/{envId}/cd-pipeline").HandlerFunc(router.pipelineConfigRestHandler.GetCdPipelinesByEnvironment).Methods("GET") resourceGroupingRouter.Path("/{envId}/external-ci").HandlerFunc(router.pipelineConfigRestHandler.GetExternalCiByEnvironment).Methods("GET") diff --git a/cmd/external-app/wire_gen.go b/cmd/external-app/wire_gen.go index b0a26de083..96354c4ad0 100644 --- a/cmd/external-app/wire_gen.go +++ b/cmd/external-app/wire_gen.go @@ -1,6 +1,6 @@ // Code generated by Wire. DO NOT EDIT. -//go:generate go run github.com/google/wire/cmd/wire +//go:generate go run -mod=mod github.com/google/wire/cmd/wire //go:build !wireinject // +build !wireinject diff --git a/pkg/app/AppListingService.go b/pkg/app/AppListingService.go index 4cf62ee723..284bb14e30 100644 --- a/pkg/app/AppListingService.go +++ b/pkg/app/AppListingService.go @@ -73,6 +73,7 @@ type AppListingService interface { FetchAppsByEnvironmentV2(fetchAppListingRequest FetchAppListingRequest, w http.ResponseWriter, r *http.Request, token string) ([]*AppView.AppEnvironmentContainer, int, error) FetchOverviewAppsByEnvironment(envId, limit, offset int) (*OverviewAppsByEnvironmentBean, error) + FetchAppsEnvContainers(envId, limit, offset int, appIds []int) ([]*AppView.AppEnvironmentContainer, error) } const ( diff --git a/pkg/pipeline/BuildPipelineConfigService.go b/pkg/pipeline/BuildPipelineConfigService.go index fca115326e..6ed411c631 100644 --- a/pkg/pipeline/BuildPipelineConfigService.go +++ b/pkg/pipeline/BuildPipelineConfigService.go @@ -28,6 +28,7 @@ import ( "github.com/devtron-labs/devtron/internal/sql/repository/helper" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" "github.com/devtron-labs/devtron/internal/util" + "github.com/devtron-labs/devtron/pkg/app" "github.com/devtron-labs/devtron/pkg/attributes" bean2 "github.com/devtron-labs/devtron/pkg/attributes/bean" "github.com/devtron-labs/devtron/pkg/bean" @@ -118,6 +119,7 @@ type CiPipelineConfigService interface { GetExternalCiByEnvironment(request resourceGroup2.ResourceGroupingRequest, token string) (ciConfig []*bean.ExternalCiConfig, err error) DeleteCiPipeline(request *bean.CiPatchRequest) (*bean.CiPipeline, error) CreateExternalCiAndAppWorkflowMapping(appId, appWorkflowId int, userId int32, tx *pg.Tx) (int, *appWorkflow.AppWorkflowMapping, error) + GetAppMetadataListByEnvironment(envId int, appIds []int) (appMetadataListBean pipelineConfigBean.AppMetadataListBean, err error) } type CiPipelineConfigServiceImpl struct { @@ -148,6 +150,7 @@ type CiPipelineConfigServiceImpl struct { buildPipelineSwitchService BuildPipelineSwitchService pipelineStageRepository repository.PipelineStageRepository globalPluginRepository repository2.GlobalPluginRepository + appListingService app.AppListingService } func NewCiPipelineConfigServiceImpl(logger *zap.SugaredLogger, @@ -175,7 +178,8 @@ func NewCiPipelineConfigServiceImpl(logger *zap.SugaredLogger, cdWorkflowRepository pipelineConfig.CdWorkflowRepository, buildPipelineSwitchService BuildPipelineSwitchService, pipelineStageRepository repository.PipelineStageRepository, - globalPluginRepository repository2.GlobalPluginRepository) *CiPipelineConfigServiceImpl { + globalPluginRepository repository2.GlobalPluginRepository, + appListingService app.AppListingService) *CiPipelineConfigServiceImpl { securityConfig := &SecurityConfig{} err := env.Parse(securityConfig) if err != nil { @@ -209,6 +213,7 @@ func NewCiPipelineConfigServiceImpl(logger *zap.SugaredLogger, buildPipelineSwitchService: buildPipelineSwitchService, pipelineStageRepository: pipelineStageRepository, globalPluginRepository: globalPluginRepository, + appListingService: appListingService, } } @@ -2157,3 +2162,24 @@ func (impl *CiPipelineConfigServiceImpl) CreateExternalCiAndAppWorkflowMapping(a } return externalCiPipeline.Id, appWorkflowMap, nil } + +func (impl *CiPipelineConfigServiceImpl) GetAppMetadataListByEnvironment(envId int, appIds []int) (appMetadataListBean pipelineConfigBean.AppMetadataListBean, err error) { + appMetadataListBean = pipelineConfigBean.AppMetadataListBean{} + envContainers, err := impl.appListingService.FetchAppsEnvContainers(envId, 0, 0, appIds) + if err != nil { + impl.logger.Errorw("failed to fetch env containers", "err", err, "envId", envId) + return appMetadataListBean, err + } + appMetadataList := make([]*pipelineConfigBean.AppMetaData, 0) + for _, envContainer := range envContainers { + appMetaData := &pipelineConfigBean.AppMetaData{ + AppId: envContainer.AppId, + AppName: envContainer.AppName, + AppStatus: envContainer.AppStatus, + } + appMetadataList = append(appMetadataList, appMetaData) + } + appMetadataListBean.Apps = appMetadataList + appMetadataListBean.AppCount = len(envContainers) + return appMetadataListBean, nil +} diff --git a/pkg/pipeline/bean/pipelineStage.go b/pkg/pipeline/bean/pipelineStage.go index d28c466d5c..f18e699284 100644 --- a/pkg/pipeline/bean/pipelineStage.go +++ b/pkg/pipeline/bean/pipelineStage.go @@ -222,3 +222,14 @@ func (v *VariableAndConditionDataForStep) GetTriggerSkipConditions() []*Conditio func (v *VariableAndConditionDataForStep) GetSuccessFailureConditions() []*ConditionObject { return v.successFailureConditions } + +type AppMetadataListBean struct { + AppCount int `json:"appCount"` + Apps []*AppMetaData `json:"apps"` +} + +type AppMetaData struct { + AppId int `json:"appId"` + AppName string `json:"appName"` + AppStatus string `json:"appStatus"` +} diff --git a/wire_gen.go b/wire_gen.go index ac459536ce..f83e41258c 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -696,7 +696,7 @@ func InitializeApp() (*App, error) { ciTemplateHistoryRepositoryImpl := repository21.NewCiTemplateHistoryRepositoryImpl(db, sugaredLogger) ciTemplateHistoryServiceImpl := history.NewCiTemplateHistoryServiceImpl(ciTemplateHistoryRepositoryImpl, sugaredLogger) buildPipelineSwitchServiceImpl := pipeline.NewBuildPipelineSwitchServiceImpl(sugaredLogger, ciPipelineConfigReadServiceImpl, ciPipelineRepositoryImpl, ciCdPipelineOrchestratorImpl, pipelineRepositoryImpl, ciWorkflowRepositoryImpl, appWorkflowRepositoryImpl, ciPipelineHistoryServiceImpl, ciTemplateOverrideRepositoryImpl, ciPipelineMaterialRepositoryImpl) - ciPipelineConfigServiceImpl := pipeline.NewCiPipelineConfigServiceImpl(sugaredLogger, ciCdPipelineOrchestratorImpl, dockerArtifactStoreRepositoryImpl, gitMaterialReadServiceImpl, appRepositoryImpl, pipelineRepositoryImpl, ciPipelineConfigReadServiceImpl, ciPipelineRepositoryImpl, ecrConfig, appWorkflowRepositoryImpl, ciCdConfig, attributesServiceImpl, pipelineStageServiceImpl, ciPipelineMaterialRepositoryImpl, ciTemplateServiceImpl, ciTemplateReadServiceImpl, ciTemplateOverrideRepositoryImpl, ciTemplateHistoryServiceImpl, enforcerUtilImpl, ciWorkflowRepositoryImpl, resourceGroupServiceImpl, customTagServiceImpl, cdWorkflowRepositoryImpl, buildPipelineSwitchServiceImpl, pipelineStageRepositoryImpl, globalPluginRepositoryImpl) + ciPipelineConfigServiceImpl := pipeline.NewCiPipelineConfigServiceImpl(sugaredLogger, ciCdPipelineOrchestratorImpl, dockerArtifactStoreRepositoryImpl, gitMaterialReadServiceImpl, appRepositoryImpl, pipelineRepositoryImpl, ciPipelineConfigReadServiceImpl, ciPipelineRepositoryImpl, ecrConfig, appWorkflowRepositoryImpl, ciCdConfig, attributesServiceImpl, pipelineStageServiceImpl, ciPipelineMaterialRepositoryImpl, ciTemplateServiceImpl, ciTemplateReadServiceImpl, ciTemplateOverrideRepositoryImpl, ciTemplateHistoryServiceImpl, enforcerUtilImpl, ciWorkflowRepositoryImpl, resourceGroupServiceImpl, customTagServiceImpl, cdWorkflowRepositoryImpl, buildPipelineSwitchServiceImpl, pipelineStageRepositoryImpl, globalPluginRepositoryImpl, appListingServiceImpl) ciMaterialConfigServiceImpl := pipeline.NewCiMaterialConfigServiceImpl(sugaredLogger, materialRepositoryImpl, ciTemplateReadServiceImpl, ciCdPipelineOrchestratorImpl, ciPipelineRepositoryImpl, gitMaterialHistoryServiceImpl, pipelineRepositoryImpl, ciPipelineMaterialRepositoryImpl, transactionUtilImpl, gitMaterialReadServiceImpl) deploymentGroupRepositoryImpl := repository2.NewDeploymentGroupRepositoryImpl(sugaredLogger, db) pipelineStrategyHistoryRepositoryImpl := repository21.NewPipelineStrategyHistoryRepositoryImpl(sugaredLogger, db) From 7b9cc286a90b32ec1b5694eeb6c57f7cadd99dd4 Mon Sep 17 00:00:00 2001 From: prakhar katiyar Date: Tue, 8 Apr 2025 23:57:23 +0530 Subject: [PATCH 10/34] GetAppMetadataListByEnvironment --- .../configure/BuildPipelineRestHandler.go | 465 +++++++----------- .../BuildPipelineRestHandlerHelper.go | 153 ++++++ 2 files changed, 328 insertions(+), 290 deletions(-) create mode 100644 api/restHandler/app/pipeline/configure/BuildPipelineRestHandlerHelper.go diff --git a/api/restHandler/app/pipeline/configure/BuildPipelineRestHandler.go b/api/restHandler/app/pipeline/configure/BuildPipelineRestHandler.go index 5203896a59..381a47308e 100644 --- a/api/restHandler/app/pipeline/configure/BuildPipelineRestHandler.go +++ b/api/restHandler/app/pipeline/configure/BuildPipelineRestHandler.go @@ -111,126 +111,91 @@ type ImageTaggingRestHandler interface { } func (handler *PipelineConfigRestHandlerImpl) CreateCiConfig(w http.ResponseWriter, r *http.Request) { - decoder := json.NewDecoder(r.Body) - userId, err := handler.userAuthService.GetLoggedInUser(r) - if userId == 0 || err != nil { - common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) + userId, ok := handler.getUserIdOrUnauthorized(w, r) + if !ok { return } + var createRequest bean.CiConfigRequest - err = decoder.Decode(&createRequest) - createRequest.UserId = userId - if err != nil { - handler.Logger.Errorw("request err, create ci config", "err", err, "create request", createRequest) - common.WriteJsonResp(w, err, nil, http.StatusBadRequest) + if !handler.decodeJsonBody(w, r, &createRequest, "create ci config") { return } + createRequest.UserId = userId + handler.Logger.Infow("request payload, create ci config", "create request", createRequest) - err = handler.validator.Struct(createRequest) - if err != nil { - handler.Logger.Errorw("validation err, create ci config", "err", err, "create request", createRequest) - common.WriteJsonResp(w, err, nil, http.StatusBadRequest) + + if !handler.validateRequestBody(w, createRequest, "create ci config") { return } + // validates if the dockerRegistry can store CONTAINER isValid := handler.dockerRegistryConfig.ValidateRegistryStorageType(createRequest.DockerRegistry, dockerRegistryRepository.OCI_REGISRTY_REPO_TYPE_CONTAINER, dockerRegistryRepository.STORAGE_ACTION_TYPE_PUSH, dockerRegistryRepository.STORAGE_ACTION_TYPE_PULL_AND_PUSH) if !isValid { - err = fmt.Errorf("invalid registry type") + err := fmt.Errorf("invalid registry type") handler.Logger.Errorw("validation err, create ci config", "err", err, "create request", createRequest) common.WriteJsonResp(w, err, nil, http.StatusBadRequest) return } + token := r.Header.Get("token") - app, err := handler.pipelineBuilder.GetApp(createRequest.AppId) - if err != nil { - common.WriteJsonResp(w, err, nil, http.StatusBadRequest) - return - } - resourceName := handler.enforcerUtil.GetAppRBACName(app.AppName) - if ok := handler.enforcer.Enforce(token, casbin.ResourceApplications, casbin.ActionCreate, resourceName); !ok { - common.WriteJsonResp(w, fmt.Errorf("unauthorized user"), "Unauthorized User", http.StatusForbidden) + _, authorized := handler.getAppAndCheckAuthForAction(w, createRequest.AppId, token, casbin.ActionCreate) + if !authorized { return } + createResp, err := handler.pipelineBuilder.CreateCiPipeline(&createRequest) - if err != nil { - handler.Logger.Errorw("service err, create", "err", err, "create request", createRequest) - common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) - return - } - common.WriteJsonResp(w, err, createResp, http.StatusOK) + handler.handleServiceError(w, err, createResp, "create ci config", createRequest) } func (handler *PipelineConfigRestHandlerImpl) UpdateCiTemplate(w http.ResponseWriter, r *http.Request) { - decoder := json.NewDecoder(r.Body) - userId, err := handler.userAuthService.GetLoggedInUser(r) - if userId == 0 || err != nil { - common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) + userId, ok := handler.getUserIdOrUnauthorized(w, r) + if !ok { return } + var configRequest bean.CiConfigRequest - err = decoder.Decode(&configRequest) - configRequest.UserId = userId - if err != nil { - handler.Logger.Errorw("request err, UpdateCiTemplate", "err", err, "UpdateCiTemplate", configRequest) - common.WriteJsonResp(w, err, nil, http.StatusBadRequest) + if !handler.decodeJsonBody(w, r, &configRequest, "UpdateCiTemplate") { return } + configRequest.UserId = userId + handler.Logger.Infow("request payload, update ci template", "UpdateCiTemplate", configRequest, "userId", userId) - err = handler.validator.Struct(configRequest) - if err != nil { - handler.Logger.Errorw("validation err, UpdateCiTemplate", "err", err, "UpdateCiTemplate", configRequest) - common.WriteJsonResp(w, err, nil, http.StatusBadRequest) + + if !handler.validateRequestBody(w, configRequest, "UpdateCiTemplate") { return } + token := r.Header.Get("token") - app, err := handler.pipelineBuilder.GetApp(configRequest.AppId) - if err != nil { - common.WriteJsonResp(w, err, nil, http.StatusBadRequest) - return - } - resourceName := handler.enforcerUtil.GetAppRBACName(app.AppName) - if ok := handler.enforcer.Enforce(token, casbin.ResourceApplications, casbin.ActionCreate, resourceName); !ok { - common.WriteJsonResp(w, fmt.Errorf("unauthorized user"), "Unauthorized User", http.StatusForbidden) + _, authorized := handler.getAppAndCheckAuthForAction(w, configRequest.AppId, token, casbin.ActionCreate) + if !authorized { return } + createResp, err := handler.pipelineBuilder.UpdateCiTemplate(&configRequest) - if err != nil { - handler.Logger.Errorw("service err, UpdateCiTemplate", "err", err, "UpdateCiTemplate", configRequest) - common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) - return - } - common.WriteJsonResp(w, err, createResp, http.StatusOK) + handler.handleServiceError(w, err, createResp, "UpdateCiTemplate", configRequest) } func (handler *PipelineConfigRestHandlerImpl) UpdateBranchCiPipelinesWithRegex(w http.ResponseWriter, r *http.Request) { - decoder := json.NewDecoder(r.Body) - userId, err := handler.userAuthService.GetLoggedInUser(r) - if userId == 0 || err != nil { - common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) + userId, ok := handler.getUserIdOrUnauthorized(w, r) + if !ok { return } + var patchRequest bean.CiRegexPatchRequest - err = decoder.Decode(&patchRequest) - patchRequest.UserId = userId - if err != nil { - handler.Logger.Errorw("request err, PatchCiPipelines", "err", err, "PatchCiPipelines", patchRequest) - common.WriteJsonResp(w, err, nil, http.StatusBadRequest) + if !handler.decodeJsonBody(w, r, &patchRequest, "PatchCiPipelines") { return } + patchRequest.UserId = userId handler.Logger.Debugw("update request ", "req", patchRequest) + token := r.Header.Get("token") - app, err := handler.pipelineBuilder.GetApp(patchRequest.AppId) - if err != nil { - common.WriteJsonResp(w, err, nil, http.StatusBadRequest) - return - } - resourceName := handler.enforcerUtil.GetAppRBACName(app.AppName) - if ok := handler.enforcer.Enforce(token, casbin.ResourceApplications, casbin.ActionTrigger, resourceName); !ok { - common.WriteJsonResp(w, fmt.Errorf("unauthorized user"), "Unauthorized User", http.StatusForbidden) + _, authorized := handler.getAppAndCheckAuthForAction(w, patchRequest.AppId, token, casbin.ActionTrigger) + if !authorized { return } + // Filter materials that have regex configured var materialList []*bean.CiPipelineMaterial for _, material := range patchRequest.CiPipelineMaterial { if handler.ciPipelineMaterialRepository.CheckRegexExistsForMaterial(material.Id) { @@ -243,60 +208,61 @@ func (handler *PipelineConfigRestHandlerImpl) UpdateBranchCiPipelinesWithRegex(w } patchRequest.CiPipelineMaterial = materialList - err = handler.pipelineBuilder.PatchRegexCiPipeline(&patchRequest) + // Update the pipeline + err := handler.pipelineBuilder.PatchRegexCiPipeline(&patchRequest) if err != nil { handler.Logger.Errorw("service err, PatchCiPipelines", "err", err, "PatchCiPipelines", patchRequest) common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) return } + //if include/exclude configured showAll will include excluded materials also in list, if not configured it will ignore this flag resp, err := handler.ciHandler.FetchMaterialsByPipelineId(patchRequest.Id, false) - if err != nil { - handler.Logger.Errorw("service err, FetchMaterials", "err", err, "pipelineId", patchRequest.Id) - common.WriteJsonResp(w, err, resp, http.StatusInternalServerError) - return - } - common.WriteJsonResp(w, err, resp, http.StatusOK) + handler.handleServiceError(w, err, resp, "FetchMaterials", map[string]interface{}{"pipelineId": patchRequest.Id}) } func (handler *PipelineConfigRestHandlerImpl) parseSourceChangeRequest(w http.ResponseWriter, r *http.Request) (*bean.CiMaterialPatchRequest, int32, error) { - decoder := json.NewDecoder(r.Body) userId, err := handler.userAuthService.GetLoggedInUser(r) if userId == 0 || err != nil { common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) return nil, 0, err } + var patchRequest bean.CiMaterialPatchRequest + decoder := json.NewDecoder(r.Body) err = decoder.Decode(&patchRequest) - if err != nil { handler.Logger.Errorw("request err, PatchCiPipeline", "err", err, "PatchCiPipeline", patchRequest) common.WriteJsonResp(w, err, nil, http.StatusBadRequest) return nil, 0, err } + return &patchRequest, userId, nil } func (handler *PipelineConfigRestHandlerImpl) parseBulkSourceChangeRequest(w http.ResponseWriter, r *http.Request) (*bean.CiMaterialBulkPatchRequest, int32, error) { - decoder := json.NewDecoder(r.Body) userId, err := handler.userAuthService.GetLoggedInUser(r) if userId == 0 || err != nil { common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) return nil, 0, err } + var patchRequest bean.CiMaterialBulkPatchRequest + decoder := json.NewDecoder(r.Body) err = decoder.Decode(&patchRequest) if err != nil { handler.Logger.Errorw("request err, BulkPatchCiPipeline", "err", err, "BulkPatchCiPipeline", patchRequest) common.WriteJsonResp(w, err, nil, http.StatusBadRequest) return nil, 0, err } + err = handler.validator.Struct(patchRequest) if err != nil { handler.Logger.Errorw("request err, BulkPatchCiPipeline", "err", err, "BulkPatchCiPipeline", patchRequest) common.WriteJsonResp(w, err, nil, http.StatusBadRequest) return nil, 0, err } + return &patchRequest, userId, nil } @@ -548,95 +514,57 @@ func (handler *PipelineConfigRestHandlerImpl) checkCiPatchAccess(token string, r func (handler *PipelineConfigRestHandlerImpl) GetCiPipeline(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) - appId, err := strconv.Atoi(vars["appId"]) - if err != nil { - common.WriteJsonResp(w, err, nil, http.StatusBadRequest) + appId, ok := handler.getIntPathParam(w, vars, "appId") + if !ok { return } + token := r.Header.Get("token") - app, err := handler.pipelineBuilder.GetApp(appId) - if err != nil { - handler.Logger.Errorw("service err, GetCiPipeline", "err", err, "appId", appId) - common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) - return - } - resourceName := handler.enforcerUtil.GetAppRBACName(app.AppName) - ok := handler.enforcerUtil.CheckAppRbacForAppOrJob(token, resourceName, casbin.ActionGet) - if !ok { - common.WriteJsonResp(w, fmt.Errorf("unauthorized user"), "Unauthorized User", http.StatusForbidden) + if !handler.checkAppRbacForAppOrJob(w, token, appId, casbin.ActionGet) { return } + ciConf, err := handler.pipelineBuilder.GetCiPipelineRespResolved(appId) - if err != nil { - handler.Logger.Errorw("service err, GetCiPipelineRespResolved", "appId", appId, "err", err) - common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) - return - } - common.WriteJsonResp(w, err, ciConf, http.StatusOK) + handler.handleServiceError(w, err, ciConf, "GetCiPipelineRespResolved", map[string]interface{}{"appId": appId}) } func (handler *PipelineConfigRestHandlerImpl) GetExternalCi(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) - appId, err := strconv.Atoi(vars["appId"]) - if err != nil { - common.WriteJsonResp(w, err, nil, http.StatusBadRequest) + appId, ok := handler.getIntPathParam(w, vars, "appId") + if !ok { return } + token := r.Header.Get("token") - app, err := handler.pipelineBuilder.GetApp(appId) - if err != nil { - handler.Logger.Errorw("service err, GetExternalCi", "err", err, "appId", appId) - common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) - return - } - resourceName := handler.enforcerUtil.GetAppRBACName(app.AppName) - if ok := handler.enforcer.Enforce(token, casbin.ResourceApplications, casbin.ActionGet, resourceName); !ok { - common.WriteJsonResp(w, fmt.Errorf("unauthorized user"), "Unauthorized User", http.StatusForbidden) - return - } - ciConf, err := handler.pipelineBuilder.GetExternalCi(appId) - if err != nil { - handler.Logger.Errorw("service err, GetExternalCi", "err", err, "appId", appId) - common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) + _, authorized := handler.getAppAndCheckAuthForAction(w, appId, token, casbin.ActionGet) + if !authorized { return } - common.WriteJsonResp(w, err, ciConf, http.StatusOK) + ciConf, err := handler.pipelineBuilder.GetExternalCi(appId) + handler.handleServiceError(w, err, ciConf, "GetExternalCi", map[string]interface{}{"appId": appId}) } func (handler *PipelineConfigRestHandlerImpl) GetExternalCiById(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) - appId, err := strconv.Atoi(vars["appId"]) - if err != nil { - common.WriteJsonResp(w, err, nil, http.StatusBadRequest) + appId, ok := handler.getIntPathParam(w, vars, "appId") + if !ok { return } - externalCiId, err := strconv.Atoi(vars["externalCiId"]) - if err != nil { - common.WriteJsonResp(w, err, nil, http.StatusBadRequest) + externalCiId, ok := handler.getIntPathParam(w, vars, "externalCiId") + if !ok { return } + token := r.Header.Get("token") - app, err := handler.pipelineBuilder.GetApp(appId) - if err != nil { - handler.Logger.Errorw("service err, GetExternalCiById", "err", err, "appId", appId) - common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) - return - } - resourceName := handler.enforcerUtil.GetAppRBACName(app.AppName) - if ok := handler.enforcer.Enforce(token, casbin.ResourceApplications, casbin.ActionGet, resourceName); !ok { - common.WriteJsonResp(w, fmt.Errorf("unauthorized user"), "Unauthorized User", http.StatusForbidden) - return - } - ciConf, err := handler.pipelineBuilder.GetExternalCiById(appId, externalCiId) - if err != nil { - handler.Logger.Errorw("service err, GetExternalCiById", "err", err, "appId", appId) - common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) + _, authorized := handler.getAppAndCheckAuthForAction(w, appId, token, casbin.ActionGet) + if !authorized { return } - common.WriteJsonResp(w, err, ciConf, http.StatusOK) + ciConf, err := handler.pipelineBuilder.GetExternalCiById(appId, externalCiId) + handler.handleServiceError(w, err, ciConf, "GetExternalCiById", map[string]interface{}{"appId": appId, "externalCiId": externalCiId}) } func (handler *PipelineConfigRestHandlerImpl) validateCiTriggerRBAC(token string, ciPipelineId, triggerEnvironmentId int) error { @@ -712,271 +640,230 @@ func (handler *PipelineConfigRestHandlerImpl) validateCiTriggerRBAC(token string } func (handler *PipelineConfigRestHandlerImpl) TriggerCiPipeline(w http.ResponseWriter, r *http.Request) { - userId, err := handler.userAuthService.GetLoggedInUser(r) - if userId == 0 || err != nil { - common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) + userId, ok := handler.getUserIdOrUnauthorized(w, r) + if !ok { return } - decoder := json.NewDecoder(r.Body) + var ciTriggerRequest bean.CiTriggerRequest - err = decoder.Decode(&ciTriggerRequest) - if err != nil { - handler.Logger.Errorw("request err, TriggerCiPipeline", "err", err, "payload", ciTriggerRequest) - common.WriteJsonResp(w, err, nil, http.StatusBadRequest) + if !handler.decodeJsonBody(w, r, &ciTriggerRequest, "TriggerCiPipeline") { return } + token := r.Header.Get("token") // RBAC block starts - err = handler.validateCiTriggerRBAC(token, ciTriggerRequest.PipelineId, ciTriggerRequest.EnvironmentId) + err := handler.validateCiTriggerRBAC(token, ciTriggerRequest.PipelineId, ciTriggerRequest.EnvironmentId) if err != nil { common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) return } // RBAC block ends + if !handler.validForMultiMaterial(ciTriggerRequest) { handler.Logger.Errorw("invalid req, commit hash not present for multi-git", "payload", ciTriggerRequest) common.WriteJsonResp(w, errors.New("invalid req, commit hash not present for multi-git"), nil, http.StatusBadRequest) + return } + ciTriggerRequest.TriggeredBy = userId handler.Logger.Infow("request payload, TriggerCiPipeline", "payload", ciTriggerRequest) + response := make(map[string]string) resp, err := handler.ciHandler.HandleCIManual(ciTriggerRequest) + if errors.Is(err, bean1.ErrImagePathInUse) { handler.Logger.Errorw("service err duplicate image tag, TriggerCiPipeline", "err", err, "payload", ciTriggerRequest) common.WriteJsonResp(w, err, err, http.StatusConflict) return } + if err != nil { handler.Logger.Errorw("service err, TriggerCiPipeline", "err", err, "payload", ciTriggerRequest) common.WriteJsonResp(w, err, response, http.StatusInternalServerError) return } + response["apiResponse"] = strconv.Itoa(resp) common.WriteJsonResp(w, err, response, http.StatusOK) } func (handler *PipelineConfigRestHandlerImpl) FetchMaterials(w http.ResponseWriter, r *http.Request) { - userId, err := handler.userAuthService.GetLoggedInUser(r) - if userId == 0 || err != nil { - common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) + _, ok := handler.getUserIdOrUnauthorized(w, r) + if !ok { return } + vars := mux.Vars(r) - pipelineId, err := strconv.Atoi(vars["pipelineId"]) - if err != nil { - common.WriteJsonResp(w, err, nil, http.StatusBadRequest) + pipelineId, ok := handler.getIntPathParam(w, vars, "pipelineId") + if !ok { return } - v := r.URL.Query() - showAll := false - show := v.Get("showAll") - if len(show) > 0 { - showAll, err = strconv.ParseBool(show) - if err != nil { - showAll = true - err = nil - //ignore error, apply rbac by default - } - } + + // Get showAll query parameter + showAll := handler.getQueryParamBool(r, "showAll", false) + handler.Logger.Infow("request payload, FetchMaterials", "pipelineId", pipelineId) + ciPipeline, err := handler.ciPipelineRepository.FindById(pipelineId) if err != nil { - handler.Logger.Errorw("service err, UpdateCiTemplate", "err", err, "pipelineId", pipelineId) + handler.Logger.Errorw("service err, FindById", "err", err, "pipelineId", pipelineId) common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) return } - //RBAC + + // RBAC check token := r.Header.Get("token") - object := handler.enforcerUtil.GetAppRBACNameByAppId(ciPipeline.AppId) - ok := handler.enforcerUtil.CheckAppRbacForAppOrJob(token, object, casbin.ActionGet) - if !ok { - common.WriteJsonResp(w, err, "Unauthorized User", http.StatusForbidden) + if !handler.checkAppRbacForAppOrJob(w, token, ciPipeline.AppId, casbin.ActionGet) { return } - //RBAC + resp, err := handler.ciHandler.FetchMaterialsByPipelineId(pipelineId, showAll) - if err != nil { - handler.Logger.Errorw("service err, FetchMaterials", "err", err, "pipelineId", pipelineId) - common.WriteJsonResp(w, err, resp, http.StatusInternalServerError) - return - } - common.WriteJsonResp(w, err, resp, http.StatusOK) + handler.handleServiceError(w, err, resp, "FetchMaterials", map[string]interface{}{"pipelineId": pipelineId}) } func (handler *PipelineConfigRestHandlerImpl) FetchMaterialsByMaterialId(w http.ResponseWriter, r *http.Request) { - userId, err := handler.userAuthService.GetLoggedInUser(r) - if userId == 0 || err != nil { - common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) + _, ok := handler.getUserIdOrUnauthorized(w, r) + if !ok { return } + vars := mux.Vars(r) - pipelineId, err := strconv.Atoi(vars["pipelineId"]) - if err != nil { - common.WriteJsonResp(w, err, nil, http.StatusBadRequest) + pipelineId, ok := handler.getIntPathParam(w, vars, "pipelineId") + if !ok { return } - gitMaterialId, err := strconv.Atoi(vars["gitMaterialId"]) - if err != nil { - common.WriteJsonResp(w, err, nil, http.StatusBadRequest) + + gitMaterialId, ok := handler.getIntPathParam(w, vars, "gitMaterialId") + if !ok { return } - v := r.URL.Query() - showAll := false - show := v.Get("showAll") - if len(show) > 0 { - showAll, err = strconv.ParseBool(show) - if err != nil { - showAll = true - err = nil - //ignore error, apply rbac by default - } - } - handler.Logger.Infow("request payload, FetchMaterials", "pipelineId", pipelineId) + + // Get showAll query parameter + showAll := handler.getQueryParamBool(r, "showAll", false) + + handler.Logger.Infow("request payload, FetchMaterialsByMaterialId", "pipelineId", pipelineId, "gitMaterialId", gitMaterialId) + ciPipeline, err := handler.ciPipelineRepository.FindById(pipelineId) if err != nil { - handler.Logger.Errorw("service err, UpdateCiTemplate", "err", err, "pipelineId", pipelineId) + handler.Logger.Errorw("service err, FindById", "err", err, "pipelineId", pipelineId) common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) return } - //RBAC + + // RBAC check token := r.Header.Get("token") - object := handler.enforcerUtil.GetAppRBACNameByAppId(ciPipeline.AppId) - ok := handler.enforcerUtil.CheckAppRbacForAppOrJob(token, object, casbin.ActionGet) - if !ok { - common.WriteJsonResp(w, err, "Unauthorized User", http.StatusForbidden) + if !handler.checkAppRbacForAppOrJob(w, token, ciPipeline.AppId, casbin.ActionGet) { return } - //RBAC + resp, err := handler.ciHandler.FetchMaterialsByPipelineIdAndGitMaterialId(pipelineId, gitMaterialId, showAll) - if err != nil { - handler.Logger.Errorw("service err, FetchMaterials", "err", err, "pipelineId", pipelineId) - common.WriteJsonResp(w, err, resp, http.StatusInternalServerError) - return - } - common.WriteJsonResp(w, err, resp, http.StatusOK) + handler.handleServiceError(w, err, resp, "FetchMaterialsByMaterialId", map[string]interface{}{"pipelineId": pipelineId, "gitMaterialId": gitMaterialId}) } func (handler *PipelineConfigRestHandlerImpl) RefreshMaterials(w http.ResponseWriter, r *http.Request) { - userId, err := handler.userAuthService.GetLoggedInUser(r) - if userId == 0 || err != nil { - common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) + _, ok := handler.getUserIdOrUnauthorized(w, r) + if !ok { return } + vars := mux.Vars(r) - gitMaterialId, err := strconv.Atoi(vars["gitMaterialId"]) - if err != nil { - handler.Logger.Error(err) - common.WriteJsonResp(w, err, nil, http.StatusBadRequest) + gitMaterialId, ok := handler.getIntPathParam(w, vars, "gitMaterialId") + if !ok { return } + handler.Logger.Infow("request payload, RefreshMaterials", "gitMaterialId", gitMaterialId) + material, err := handler.gitMaterialReadService.FindById(gitMaterialId) if err != nil { handler.Logger.Errorw("service err, RefreshMaterials", "err", err, "gitMaterialId", gitMaterialId) common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) return } - //RBAC + + // RBAC check token := r.Header.Get("token") - object := handler.enforcerUtil.GetAppRBACNameByAppId(material.AppId) - ok := handler.enforcerUtil.CheckAppRbacForAppOrJob(token, object, casbin.ActionGet) - if !ok { - common.WriteJsonResp(w, err, "Unauthorized User", http.StatusForbidden) + if !handler.checkAppRbacForAppOrJob(w, token, material.AppId, casbin.ActionGet) { return } - //RBAC resp, err := handler.ciHandler.RefreshMaterialByCiPipelineMaterialId(material.Id) - if err != nil { - handler.Logger.Errorw("service err, RefreshMaterials", "err", err, "gitMaterialId", gitMaterialId) - common.WriteJsonResp(w, err, resp, http.StatusInternalServerError) - return - } - common.WriteJsonResp(w, err, resp, http.StatusOK) + handler.handleServiceError(w, err, resp, "RefreshMaterials", map[string]interface{}{"gitMaterialId": gitMaterialId}) } func (handler *PipelineConfigRestHandlerImpl) GetCiPipelineMin(w http.ResponseWriter, r *http.Request) { - userId, err := handler.userAuthService.GetLoggedInUser(r) - if userId == 0 || err != nil { - common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) + _, ok := handler.getUserIdOrUnauthorized(w, r) + if !ok { return } + vars := mux.Vars(r) - appId, err := strconv.Atoi(vars["appId"]) - if err != nil { - common.WriteJsonResp(w, err, nil, http.StatusBadRequest) + appId, ok := handler.getIntPathParam(w, vars, "appId") + if !ok { return } + + // Parse environment IDs from query parameter v := r.URL.Query() envIdsString := v.Get("envIds") envIds := make([]int, 0) if len(envIdsString) > 0 { + var err error envIds, err = stringsUtil.SplitCommaSeparatedIntValues(envIdsString) if err != nil { common.WriteJsonResp(w, err, "please provide valid envIds", http.StatusBadRequest) return } } - //RBAC + handler.Logger.Infow("request payload, GetCiPipelineMin", "appId", appId) + + // RBAC check token := r.Header.Get("token") - object := handler.enforcerUtil.GetAppRBACNameByAppId(appId) - ok := handler.enforcerUtil.CheckAppRbacForAppOrJob(token, object, casbin.ActionGet) - if !ok { - common.WriteJsonResp(w, err, "Unauthorized User", http.StatusForbidden) + if !handler.checkAppRbacForAppOrJob(w, token, appId, casbin.ActionGet) { return } - //RBAC + ciPipelines, err := handler.pipelineBuilder.GetCiPipelineMin(appId, envIds) - if err != nil { - handler.Logger.Errorw("service err, GetCiPipelineMin", "err", err, "appId", appId) - if util.IsErrNoRows(err) { - err = &util.ApiError{Code: "404", HttpStatusCode: http.StatusNotFound, UserMessage: "no data found"} - common.WriteJsonResp(w, err, nil, http.StatusOK) - } else { - common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) - } - return - } - common.WriteJsonResp(w, err, ciPipelines, http.StatusOK) + handler.handleServiceError(w, err, ciPipelines, "GetCiPipelineMin", map[string]interface{}{"appId": appId}) } func (handler *PipelineConfigRestHandlerImpl) DownloadCiWorkflowArtifacts(w http.ResponseWriter, r *http.Request) { - userId, err := handler.userAuthService.GetLoggedInUser(r) - if userId == 0 || err != nil { - common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) + _, ok := handler.getUserIdOrUnauthorized(w, r) + if !ok { return } + vars := mux.Vars(r) - pipelineId, err := strconv.Atoi(vars["pipelineId"]) - if err != nil { - common.WriteJsonResp(w, err, nil, http.StatusBadRequest) + pipelineId, ok := handler.getIntPathParam(w, vars, "pipelineId") + if !ok { return } - buildId, err := strconv.Atoi(vars["workflowId"]) - if err != nil { - common.WriteJsonResp(w, err, nil, http.StatusBadRequest) + + buildId, ok := handler.getIntPathParam(w, vars, "workflowId") + if !ok { return } + handler.Logger.Infow("request payload, DownloadCiWorkflowArtifacts", "pipelineId", pipelineId, "buildId", buildId) + ciPipeline, err := handler.ciPipelineRepository.FindById(pipelineId) if err != nil { handler.Logger.Error(err) common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) return } - //RBAC + + // RBAC check token := r.Header.Get("token") object := handler.enforcerUtil.GetAppRBACNameByAppId(ciPipeline.AppId) if ok := handler.enforcer.Enforce(token, casbin.ResourceApplications, casbin.ActionGet, object); !ok { - common.WriteJsonResp(w, err, "Unauthorized User", http.StatusForbidden) + common.WriteJsonResp(w, nil, "Unauthorized User", http.StatusForbidden) return } - //RBAC file, err := handler.ciHandler.DownloadCiWorkflowArtifacts(pipelineId, buildId) - defer file.Close() if err != nil { handler.Logger.Errorw("service err, DownloadCiWorkflowArtifacts", "err", err, "pipelineId", pipelineId, "buildId", buildId) if util.IsErrNoRows(err) { @@ -987,9 +874,12 @@ func (handler *PipelineConfigRestHandlerImpl) DownloadCiWorkflowArtifacts(w http } return } + defer file.Close() + w.Header().Set("Content-Disposition", "attachment; filename="+strconv.Itoa(buildId)+".zip") w.Header().Set("Content-Type", "application/octet-stream") w.Header().Set("Content-Length", r.Header.Get("Content-Length")) + _, err = io.Copy(w, file) if err != nil { handler.Logger.Errorw("service err, DownloadCiWorkflowArtifacts", "err", err, "pipelineId", pipelineId, "buildId", buildId) @@ -997,46 +887,41 @@ func (handler *PipelineConfigRestHandlerImpl) DownloadCiWorkflowArtifacts(w http } func (handler *PipelineConfigRestHandlerImpl) GetHistoricBuildLogs(w http.ResponseWriter, r *http.Request) { - userId, err := handler.userAuthService.GetLoggedInUser(r) - if userId == 0 || err != nil { - common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) + _, ok := handler.getUserIdOrUnauthorized(w, r) + if !ok { return } + vars := mux.Vars(r) - pipelineId, err := strconv.Atoi(vars["pipelineId"]) - if err != nil { - handler.Logger.Error(err) - common.WriteJsonResp(w, err, nil, http.StatusBadRequest) + pipelineId, ok := handler.getIntPathParam(w, vars, "pipelineId") + if !ok { return } - workflowId, err := strconv.Atoi(vars["workflowId"]) - if err != nil { - handler.Logger.Error(err) - common.WriteJsonResp(w, err, nil, http.StatusBadRequest) + + workflowId, ok := handler.getIntPathParam(w, vars, "workflowId") + if !ok { return } + handler.Logger.Infow("request payload, GetHistoricBuildLogs", "pipelineId", pipelineId, "workflowId", workflowId) + ciPipeline, err := handler.ciPipelineRepository.FindById(pipelineId) if err != nil { handler.Logger.Errorw("service err, GetHistoricBuildLogs", "err", err, "pipelineId", pipelineId, "workflowId", workflowId) common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) return } - //RBAC + + // RBAC check token := r.Header.Get("token") object := handler.enforcerUtil.GetAppRBACNameByAppId(ciPipeline.AppId) if ok := handler.enforcer.Enforce(token, casbin.ResourceApplications, casbin.ActionGet, object); !ok { - common.WriteJsonResp(w, err, "Unauthorized User", http.StatusForbidden) + common.WriteJsonResp(w, nil, "Unauthorized User", http.StatusForbidden) return } - //RBAC + resp, err := handler.ciHandler.GetHistoricBuildLogs(workflowId, nil) - if err != nil { - handler.Logger.Errorw("service err, GetHistoricBuildLogs", "err", err, "pipelineId", pipelineId, "workflowId", workflowId) - common.WriteJsonResp(w, err, resp, http.StatusInternalServerError) - return - } - common.WriteJsonResp(w, err, resp, http.StatusOK) + handler.handleServiceError(w, err, resp, "GetHistoricBuildLogs", map[string]interface{}{"pipelineId": pipelineId, "workflowId": workflowId}) } func (handler *PipelineConfigRestHandlerImpl) GetBuildHistory(w http.ResponseWriter, r *http.Request) { diff --git a/api/restHandler/app/pipeline/configure/BuildPipelineRestHandlerHelper.go b/api/restHandler/app/pipeline/configure/BuildPipelineRestHandlerHelper.go new file mode 100644 index 0000000000..f59829f57e --- /dev/null +++ b/api/restHandler/app/pipeline/configure/BuildPipelineRestHandlerHelper.go @@ -0,0 +1,153 @@ +/* + * Copyright (c) 2024. Devtron Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package configure + +import ( + "encoding/json" + "fmt" + "github.com/devtron-labs/devtron/api/restHandler/common" + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" + "github.com/devtron-labs/devtron/internal/util" + "github.com/devtron-labs/devtron/pkg/auth/authorisation/casbin" + "github.com/devtron-labs/devtron/pkg/bean" + "net/http" + "strconv" +) + +// Helper functions for common operations in BuildPipelineRestHandler.go + +// getUserIdOrUnauthorized gets the logged-in user ID or returns an unauthorized response +func (handler *PipelineConfigRestHandlerImpl) getUserIdOrUnauthorized(w http.ResponseWriter, r *http.Request) (int32, bool) { + userId, err := handler.userAuthService.GetLoggedInUser(r) + if userId == 0 || err != nil { + common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized) + return 0, false + } + return userId, true +} + +// getIntPathParam gets an integer path parameter from the request +func (handler *PipelineConfigRestHandlerImpl) getIntPathParam(w http.ResponseWriter, vars map[string]string, paramName string) (int, bool) { + paramValue, err := strconv.Atoi(vars[paramName]) + if err != nil { + handler.Logger.Errorw("request err, invalid path param", "err", err, "paramName", paramName) + common.WriteJsonResp(w, err, nil, http.StatusBadRequest) + return 0, false + } + return paramValue, true +} + +// decodeJsonBody decodes the request body into the provided struct +func (handler *PipelineConfigRestHandlerImpl) decodeJsonBody(w http.ResponseWriter, r *http.Request, obj interface{}, logContext string) bool { + decoder := json.NewDecoder(r.Body) + err := decoder.Decode(obj) + if err != nil { + handler.Logger.Errorw("request err, decode json body", "err", err, "context", logContext, "payload", obj) + common.WriteJsonResp(w, err, nil, http.StatusBadRequest) + return false + } + return true +} + +// validateRequestBody validates the request body against struct validation tags +func (handler *PipelineConfigRestHandlerImpl) validateRequestBody(w http.ResponseWriter, obj interface{}, logContext string) bool { + err := handler.validator.Struct(obj) + if err != nil { + handler.Logger.Errorw("validation err", "err", err, "context", logContext, "payload", obj) + common.WriteJsonResp(w, err, nil, http.StatusBadRequest) + return false + } + return true +} + +// getAppAndCheckAuthForAction gets the app and checks if the user has the required permission +func (handler *PipelineConfigRestHandlerImpl) getAppAndCheckAuthForAction(w http.ResponseWriter, appId int, token string, action string) (app *bean.CreateAppDTO, authorized bool) { + app, err := handler.pipelineBuilder.GetApp(appId) + if err != nil { + handler.Logger.Errorw("service err, GetApp", "err", err, "appId", appId) + common.WriteJsonResp(w, err, nil, http.StatusBadRequest) + return nil, false + } + + resourceName := handler.enforcerUtil.GetAppRBACName(app.AppName) + if ok := handler.enforcer.Enforce(token, casbin.ResourceApplications, action, resourceName); !ok { + common.WriteJsonResp(w, fmt.Errorf("unauthorized user"), "Unauthorized User", http.StatusForbidden) + return nil, false + } + + return app, true +} + +// checkAppRbacForAppOrJob checks if the user has the required permission for app or job +func (handler *PipelineConfigRestHandlerImpl) checkAppRbacForAppOrJob(w http.ResponseWriter, token string, appId int, action string) bool { + object := handler.enforcerUtil.GetAppRBACNameByAppId(appId) + ok := handler.enforcerUtil.CheckAppRbacForAppOrJob(token, object, action) + if !ok { + common.WriteJsonResp(w, nil, "Unauthorized User", http.StatusForbidden) + return false + } + return true +} + +// getCiPipelineWithAuth gets the CI pipeline and checks if the user has the required permission +func (handler *PipelineConfigRestHandlerImpl) getCiPipelineWithAuth(w http.ResponseWriter, pipelineId int, token string, action string) (*pipelineConfig.CiPipeline, bool) { + ciPipeline, err := handler.ciPipelineRepository.FindById(pipelineId) + if err != nil { + handler.Logger.Errorw("service err, FindById", "err", err, "pipelineId", pipelineId) + common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) + return nil, false + } + + object := handler.enforcerUtil.GetAppRBACNameByAppId(ciPipeline.AppId) + ok := handler.enforcerUtil.CheckAppRbacForAppOrJob(token, object, action) + if !ok { + common.WriteJsonResp(w, nil, "Unauthorized User", http.StatusForbidden) + return nil, false + } + + return ciPipeline, true +} + +// handleServiceError handles service errors and writes appropriate response +func (handler *PipelineConfigRestHandlerImpl) handleServiceError(w http.ResponseWriter, err error, resp interface{}, logContext string, logData interface{}) { + if err != nil { + if util.IsErrNoRows(err) { + err = &util.ApiError{Code: "404", HttpStatusCode: http.StatusNotFound, UserMessage: "no data found"} + common.WriteJsonResp(w, err, nil, http.StatusOK) + } else { + handler.Logger.Errorw("service err", "err", err, "context", logContext, "data", logData) + common.WriteJsonResp(w, err, resp, http.StatusInternalServerError) + } + return + } + common.WriteJsonResp(w, nil, resp, http.StatusOK) +} + +// getQueryParamBool gets a boolean query parameter from the request +func (handler *PipelineConfigRestHandlerImpl) getQueryParamBool(r *http.Request, paramName string, defaultValue bool) bool { + v := r.URL.Query() + paramValue := defaultValue + paramStr := v.Get(paramName) + if len(paramStr) > 0 { + var err error + paramValue, err = strconv.ParseBool(paramStr) + if err != nil { + paramValue = defaultValue + } + } + return paramValue +} From 764a34d8d4980d7e58da893ea221a4c8d8339c8a Mon Sep 17 00:00:00 2001 From: prakhar katiyar Date: Wed, 9 Apr 2025 11:27:32 +0530 Subject: [PATCH 11/34] GetAppMetadataListByEnvironment --- internal/sql/repository/AppListingRepository.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/sql/repository/AppListingRepository.go b/internal/sql/repository/AppListingRepository.go index e0a44a0b82..06ba8a2d94 100644 --- a/internal/sql/repository/AppListingRepository.go +++ b/internal/sql/repository/AppListingRepository.go @@ -158,11 +158,11 @@ func (impl *AppListingRepositoryImpl) FetchAppsEnvContainers(envId int, limit, o query += " ORDER BY a.app_name" if limit > 0 { - query += " LIMIT ?" + query += fmt.Sprintf("LIMIT ? ") queryParams = append(queryParams, limit) } if offset > 0 { - query += " OFFSET ?" + query += fmt.Sprintf("OFFSET ? ") queryParams = append(queryParams, offset) } From 430a65e6e2c7982685d9acf44b3749235193b0b3 Mon Sep 17 00:00:00 2001 From: prakhar katiyar Date: Wed, 9 Apr 2025 11:33:34 +0530 Subject: [PATCH 12/34] GetAppMetadataListByEnvironment --- internal/sql/repository/AppListingRepository.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/internal/sql/repository/AppListingRepository.go b/internal/sql/repository/AppListingRepository.go index 06ba8a2d94..b0706d20cf 100644 --- a/internal/sql/repository/AppListingRepository.go +++ b/internal/sql/repository/AppListingRepository.go @@ -145,7 +145,8 @@ func (impl *AppListingRepositoryImpl) FetchAppsEnvContainers(envId int, limit, o LEFT JOIN (SELECT pco.pipeline_id,MAX(pco.created_on) as last_deployed_time from pipeline_config_override pco GROUP BY pco.pipeline_id) ld ON ld.pipeline_id = p.id - WHERE a.active = true` + WHERE a.active = true + ORDER BY a.app_name` queryParams := []interface{}{envId, envId} @@ -155,8 +156,6 @@ func (impl *AppListingRepositoryImpl) FetchAppsEnvContainers(envId int, limit, o queryParams = append(queryParams, pg.In(appIds)) } - query += " ORDER BY a.app_name" - if limit > 0 { query += fmt.Sprintf("LIMIT ? ") queryParams = append(queryParams, limit) From 3cd0abb1061a5350d9746f89c5096bff97677789 Mon Sep 17 00:00:00 2001 From: prakhar katiyar Date: Wed, 9 Apr 2025 11:34:38 +0530 Subject: [PATCH 13/34] GetAppMetadataListByEnvironment --- internal/sql/repository/AppListingRepository.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/sql/repository/AppListingRepository.go b/internal/sql/repository/AppListingRepository.go index b0706d20cf..b628363e4d 100644 --- a/internal/sql/repository/AppListingRepository.go +++ b/internal/sql/repository/AppListingRepository.go @@ -146,7 +146,7 @@ func (impl *AppListingRepositoryImpl) FetchAppsEnvContainers(envId int, limit, o (SELECT pco.pipeline_id,MAX(pco.created_on) as last_deployed_time from pipeline_config_override pco GROUP BY pco.pipeline_id) ld ON ld.pipeline_id = p.id WHERE a.active = true - ORDER BY a.app_name` + ORDER BY a.app_name ` queryParams := []interface{}{envId, envId} From 25981301bb54238706123376d939dcfc53951bd1 Mon Sep 17 00:00:00 2001 From: prakhar katiyar Date: Wed, 9 Apr 2025 11:35:30 +0530 Subject: [PATCH 14/34] GetAppMetadataListByEnvironment --- internal/sql/repository/AppListingRepository.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/sql/repository/AppListingRepository.go b/internal/sql/repository/AppListingRepository.go index b628363e4d..b8e0df09ab 100644 --- a/internal/sql/repository/AppListingRepository.go +++ b/internal/sql/repository/AppListingRepository.go @@ -145,7 +145,7 @@ func (impl *AppListingRepositoryImpl) FetchAppsEnvContainers(envId int, limit, o LEFT JOIN (SELECT pco.pipeline_id,MAX(pco.created_on) as last_deployed_time from pipeline_config_override pco GROUP BY pco.pipeline_id) ld ON ld.pipeline_id = p.id - WHERE a.active = true + WHERE a.active = true ORDER BY a.app_name ` queryParams := []interface{}{envId, envId} From b0c54b96b690887b3d852fd42d111a460c8ebb19 Mon Sep 17 00:00:00 2001 From: prakhar katiyar Date: Wed, 9 Apr 2025 13:15:18 +0530 Subject: [PATCH 15/34] GetAppMetadataListByEnvironment --- .../app/pipeline/configure/BuildPipelineRestHandler.go | 9 ++------- internal/sql/repository/AppListingRepository.go | 4 ++-- pkg/app/AppListingService.go | 8 ++++---- pkg/pipeline/BuildPipelineConfigService.go | 2 +- 4 files changed, 9 insertions(+), 14 deletions(-) diff --git a/api/restHandler/app/pipeline/configure/BuildPipelineRestHandler.go b/api/restHandler/app/pipeline/configure/BuildPipelineRestHandler.go index 5203896a59..b6febbb59d 100644 --- a/api/restHandler/app/pipeline/configure/BuildPipelineRestHandler.go +++ b/api/restHandler/app/pipeline/configure/BuildPipelineRestHandler.go @@ -2327,7 +2327,7 @@ func (handler *PipelineConfigRestHandlerImpl) GetAppMetadataListByEnvironment(w resp, err := handler.pipelineBuilder.GetAppMetadataListByEnvironment(envId, appIds) if err != nil { - handler.Logger.Errorw("service err, GetAppMetadataListByEnvironment", "err", err, "envId", envId) + handler.Logger.Errorw("service err, GetAppMetadataListByEnvironment", "envId", envId, "err", err) common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) return } @@ -2346,12 +2346,7 @@ func (handler *PipelineConfigRestHandlerImpl) GetAppMetadataListByEnvironment(w // get rbac objects for the appids rbacObjectsWithAppId := handler.enforcerUtil.GetRbacObjectsByAppIds(appIds) - rbacObjects := make([]string, len(rbacObjectsWithAppId)) - itr := 0 - for _, object := range rbacObjectsWithAppId { - rbacObjects[itr] = object - itr++ - } + rbacObjects := maps.Values(rbacObjectsWithAppId) // enforce rbac in batch rbacResult := handler.enforcer.EnforceInBatch(token, casbin.ResourceApplications, casbin.ActionGet, rbacObjects) // filter out rbac passed apps diff --git a/internal/sql/repository/AppListingRepository.go b/internal/sql/repository/AppListingRepository.go index b8e0df09ab..1866badee5 100644 --- a/internal/sql/repository/AppListingRepository.go +++ b/internal/sql/repository/AppListingRepository.go @@ -51,7 +51,7 @@ type AppListingRepository interface { DeploymentDetailByArtifactId(ciArtifactId int, envId int) (AppView.DeploymentDetailContainer, error) FindAppCount(isProd bool) (int, error) FetchAppsByEnvironmentV2(appListingFilter helper.AppListingFilter) ([]*AppView.AppEnvironmentContainer, int, error) - FetchAppsEnvContainers(envId, limit, offset int, appIds []int) ([]*AppView.AppEnvironmentContainer, error) + FetchAppsEnvContainers(envId int, appIds []int, limit, offset int) ([]*AppView.AppEnvironmentContainer, error) FetchLastDeployedImage(appId, envId int) (*LastDeployed, error) } @@ -137,7 +137,7 @@ func (impl *AppListingRepositoryImpl) FetchOverviewCiPipelines(jobId int) ([]*Ap return jobContainers, nil } -func (impl *AppListingRepositoryImpl) FetchAppsEnvContainers(envId int, limit, offset int, appIds []int) ([]*AppView.AppEnvironmentContainer, error) { +func (impl *AppListingRepositoryImpl) FetchAppsEnvContainers(envId int, appIds []int, limit, offset int) ([]*AppView.AppEnvironmentContainer, error) { query := ` SELECT a.id as app_id,a.app_name,aps.status as app_status, ld.last_deployed_time, p.id as pipeline_id FROM app a INNER JOIN pipeline p ON p.app_id = a.id and p.deleted = false and p.environment_id = ? diff --git a/pkg/app/AppListingService.go b/pkg/app/AppListingService.go index 284bb14e30..48c5728c91 100644 --- a/pkg/app/AppListingService.go +++ b/pkg/app/AppListingService.go @@ -73,7 +73,7 @@ type AppListingService interface { FetchAppsByEnvironmentV2(fetchAppListingRequest FetchAppListingRequest, w http.ResponseWriter, r *http.Request, token string) ([]*AppView.AppEnvironmentContainer, int, error) FetchOverviewAppsByEnvironment(envId, limit, offset int) (*OverviewAppsByEnvironmentBean, error) - FetchAppsEnvContainers(envId, limit, offset int, appIds []int) ([]*AppView.AppEnvironmentContainer, error) + FetchAppsEnvContainers(envId int, appIds []int, limit, offset int) ([]*AppView.AppEnvironmentContainer, error) } const ( @@ -236,7 +236,7 @@ func (impl AppListingServiceImpl) FetchOverviewAppsByEnvironment(envId, limit, o } } var appIds []int - envContainers, err := impl.FetchAppsEnvContainers(envId, limit, offset, appIds) + envContainers, err := impl.FetchAppsEnvContainers(envId, appIds, limit, offset) if err != nil { impl.Logger.Errorw("failed to fetch env containers", "err", err, "envId", envId) return resp, err @@ -290,8 +290,8 @@ func getUniqueArtifacts(artifactIds []int) (uniqueArtifactIds []int) { return uniqueArtifactIds } -func (impl AppListingServiceImpl) FetchAppsEnvContainers(envId, limit, offset int, appIds []int) ([]*AppView.AppEnvironmentContainer, error) { - envContainers, err := impl.appListingRepository.FetchAppsEnvContainers(envId, limit, offset, appIds) +func (impl AppListingServiceImpl) FetchAppsEnvContainers(envId int, appIds []int, limit, offset int) ([]*AppView.AppEnvironmentContainer, error) { + envContainers, err := impl.appListingRepository.FetchAppsEnvContainers(envId, appIds, limit, offset) if err != nil { impl.Logger.Errorw("failed to fetch environment containers", "err", err, "envId", envId) return nil, err diff --git a/pkg/pipeline/BuildPipelineConfigService.go b/pkg/pipeline/BuildPipelineConfigService.go index 6ed411c631..c320898bdc 100644 --- a/pkg/pipeline/BuildPipelineConfigService.go +++ b/pkg/pipeline/BuildPipelineConfigService.go @@ -2165,7 +2165,7 @@ func (impl *CiPipelineConfigServiceImpl) CreateExternalCiAndAppWorkflowMapping(a func (impl *CiPipelineConfigServiceImpl) GetAppMetadataListByEnvironment(envId int, appIds []int) (appMetadataListBean pipelineConfigBean.AppMetadataListBean, err error) { appMetadataListBean = pipelineConfigBean.AppMetadataListBean{} - envContainers, err := impl.appListingService.FetchAppsEnvContainers(envId, 0, 0, appIds) + envContainers, err := impl.appListingService.FetchAppsEnvContainers(envId, appIds, 0, 0) if err != nil { impl.logger.Errorw("failed to fetch env containers", "err", err, "envId", envId) return appMetadataListBean, err From ba512415a18263efcfd9925461f7aa5a4da3e57d Mon Sep 17 00:00:00 2001 From: prakhar katiyar Date: Wed, 9 Apr 2025 13:21:44 +0530 Subject: [PATCH 16/34] env_gen.md changes --- env_gen.json | 2 +- env_gen.md | 1 + wire_gen.go | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/env_gen.json b/env_gen.json index b4b889fc79..ad32c0da18 100644 --- a/env_gen.json +++ b/env_gen.json @@ -1 +1 @@ -[{"Category":"CD","Fields":[{"Env":"ARGO_APP_MANUAL_SYNC_TIME","EnvType":"int","EnvValue":"3","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_HELM_PIPELINE_STATUS_CRON_TIME","EnvType":"string","EnvValue":"*/2 * * * *","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_PIPELINE_STATUS_CRON_TIME","EnvType":"string","EnvValue":"*/2 * * * *","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_PIPELINE_STATUS_TIMEOUT_DURATION","EnvType":"string","EnvValue":"20","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEPLOY_STATUS_CRON_GET_PIPELINE_DEPLOYED_WITHIN_HOURS","EnvType":"int","EnvValue":"12","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEVTRON_CHART_ARGO_CD_INSTALL_REQUEST_TIMEOUT","EnvType":"int","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEVTRON_CHART_INSTALL_REQUEST_TIMEOUT","EnvType":"int","EnvValue":"6","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"EXPOSE_CD_METRICS","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"FEATURE_MIGRATE_ARGOCD_APPLICATION_ENABLE","EnvType":"bool","EnvValue":"false","EnvDescription":"enable migration of external argocd application to devtron pipeline","Example":"","Deprecated":"false"},{"Env":"HELM_PIPELINE_STATUS_CHECK_ELIGIBLE_TIME","EnvType":"string","EnvValue":"120","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"IS_INTERNAL_USE","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"MIGRATE_DEPLOYMENT_CONFIG_DATA","EnvType":"bool","EnvValue":"false","EnvDescription":"migrate deployment config data from charts table to deployment_config table","Example":"","Deprecated":"false"},{"Env":"PIPELINE_DEGRADED_TIME","EnvType":"string","EnvValue":"10","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"REVISION_HISTORY_LIMIT_DEVTRON_APP","EnvType":"int","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"REVISION_HISTORY_LIMIT_EXTERNAL_HELM_APP","EnvType":"int","EnvValue":"0","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"REVISION_HISTORY_LIMIT_HELM_APP","EnvType":"int","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"REVISION_HISTORY_LIMIT_LINKED_HELM_APP","EnvType":"int","EnvValue":"15","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"RUN_HELM_INSTALL_IN_ASYNC_MODE_HELM_APPS","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"SHOULD_CHECK_NAMESPACE_ON_CLONE","EnvType":"bool","EnvValue":"false","EnvDescription":"should we check if namespace exists or not while cloning app","Example":"","Deprecated":"false"},{"Env":"USE_DEPLOYMENT_CONFIG_DATA","EnvType":"bool","EnvValue":"false","EnvDescription":"use deployment config data from deployment_config table","Example":"","Deprecated":"true"}]},{"Category":"CI_RUNNER","Fields":[{"Env":"AZURE_ACCOUNT_KEY","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"AZURE_ACCOUNT_NAME","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"AZURE_BLOB_CONTAINER_CI_CACHE","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"AZURE_BLOB_CONTAINER_CI_LOG","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"AZURE_GATEWAY_CONNECTION_INSECURE","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"AZURE_GATEWAY_URL","EnvType":"string","EnvValue":"http://devtron-minio.devtroncd:9000","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BASE_LOG_LOCATION_PATH","EnvType":"string","EnvValue":"/home/devtron/","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_GCP_CREDENTIALS_JSON","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_PROVIDER","EnvType":"","EnvValue":"S3","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_S3_ACCESS_KEY","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_S3_BUCKET_VERSIONED","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_S3_ENDPOINT","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_S3_ENDPOINT_INSECURE","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_S3_SECRET_KEY","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BUILDX_CACHE_PATH","EnvType":"string","EnvValue":"/var/lib/devtron/buildx","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BUILDX_K8S_DRIVER_OPTIONS","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BUILDX_PROVENANCE_MODE","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BUILD_LOG_TTL_VALUE_IN_SECS","EnvType":"int","EnvValue":"3600","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CACHE_LIMIT","EnvType":"int64","EnvValue":"5000000000","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_DEFAULT_ADDRESS_POOL_BASE_CIDR","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_DEFAULT_ADDRESS_POOL_SIZE","EnvType":"int","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_LIMIT_CI_CPU","EnvType":"string","EnvValue":"0.5","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_LIMIT_CI_MEM","EnvType":"string","EnvValue":"3G","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_NODE_LABEL_SELECTOR","EnvType":"","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_NODE_TAINTS_KEY","EnvType":"string","EnvValue":"dedicated","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_NODE_TAINTS_VALUE","EnvType":"string","EnvValue":"ci","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_REQ_CI_CPU","EnvType":"string","EnvValue":"0.5","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_REQ_CI_MEM","EnvType":"string","EnvValue":"3G","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_WORKFLOW_EXECUTOR_TYPE","EnvType":"","EnvValue":"AWF","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_WORKFLOW_SERVICE_ACCOUNT","EnvType":"string","EnvValue":"cd-runner","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CI_DEFAULT_ADDRESS_POOL_BASE_CIDR","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CI_DEFAULT_ADDRESS_POOL_SIZE","EnvType":"int","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CI_IGNORE_DOCKER_CACHE","EnvType":"bool","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CI_LOGS_KEY_PREFIX","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CI_NODE_LABEL_SELECTOR","EnvType":"","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CI_NODE_TAINTS_KEY","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CI_NODE_TAINTS_VALUE","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CI_RUNNER_DOCKER_MTU_VALUE","EnvType":"int","EnvValue":"-1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CI_SUCCESS_AUTO_TRIGGER_BATCH_SIZE","EnvType":"int","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CI_VOLUME_MOUNTS_JSON","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CI_WORKFLOW_EXECUTOR_TYPE","EnvType":"","EnvValue":"AWF","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_ARTIFACT_KEY_LOCATION","EnvType":"string","EnvValue":"arsenal-v1/ci-artifacts","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_BUILD_LOGS_BUCKET","EnvType":"string","EnvValue":"devtron-pro-ci-logs","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_BUILD_LOGS_KEY_PREFIX","EnvType":"string","EnvValue":"arsenal-v1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_CACHE_BUCKET","EnvType":"string","EnvValue":"ci-caching","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_CACHE_BUCKET_REGION","EnvType":"string","EnvValue":"us-east-2","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_CD_ARTIFACT_KEY_LOCATION","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_CD_LOGS_BUCKET_REGION","EnvType":"string","EnvValue":"us-east-2","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_CD_NAMESPACE","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_CD_TIMEOUT","EnvType":"int64","EnvValue":"3600","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_CI_IMAGE","EnvType":"string","EnvValue":"686244538589.dkr.ecr.us-east-2.amazonaws.com/cirunner:47","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_NAMESPACE","EnvType":"string","EnvValue":"devtron-ci","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_TARGET_PLATFORM","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DOCKER_BUILD_CACHE_PATH","EnvType":"string","EnvValue":"/var/lib/docker","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ENABLE_BUILD_CONTEXT","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ENABLE_WORKFLOW_EXECUTION_STAGE","EnvType":"bool","EnvValue":"true","EnvDescription":"if enabled then we will display build stages separately for CI/Job/Pre-Post CD","Example":"true","Deprecated":"false"},{"Env":"EXTERNAL_BLOB_STORAGE_CM_NAME","EnvType":"string","EnvValue":"blob-storage-cm","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"EXTERNAL_BLOB_STORAGE_SECRET_NAME","EnvType":"string","EnvValue":"blob-storage-secret","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"EXTERNAL_CD_NODE_LABEL_SELECTOR","EnvType":"","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"EXTERNAL_CD_NODE_TAINTS_KEY","EnvType":"string","EnvValue":"dedicated","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"EXTERNAL_CD_NODE_TAINTS_VALUE","EnvType":"string","EnvValue":"ci","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"EXTERNAL_CI_API_SECRET","EnvType":"string","EnvValue":"devtroncd-secret","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"EXTERNAL_CI_PAYLOAD","EnvType":"string","EnvValue":"{\"ciProjectDetails\":[{\"gitRepository\":\"https://github.com/vikram1601/getting-started-nodejs.git\",\"checkoutPath\":\"./abc\",\"commitHash\":\"239077135f8cdeeccb7857e2851348f558cb53d3\",\"commitTime\":\"2022-10-30T20:00:00\",\"branch\":\"master\",\"message\":\"Update README.md\",\"author\":\"User Name \"}],\"dockerImage\":\"445808685819.dkr.ecr.us-east-2.amazonaws.com/orch:23907713-2\"}","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"EXTERNAL_CI_WEB_HOOK_URL","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"IGNORE_CM_CS_IN_CI_JOB","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"IMAGE_RETRY_COUNT","EnvType":"int","EnvValue":"0","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"IMAGE_RETRY_INTERVAL","EnvType":"int","EnvValue":"5","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"IMAGE_SCANNER_ENDPOINT","EnvType":"string","EnvValue":"http://image-scanner-new-demo-devtroncd-service.devtroncd:80","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"IMAGE_SCAN_MAX_RETRIES","EnvType":"int","EnvValue":"3","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"IMAGE_SCAN_RETRY_DELAY","EnvType":"int","EnvValue":"5","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"IN_APP_LOGGING_ENABLED","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"MAX_CD_WORKFLOW_RUNNER_RETRIES","EnvType":"int","EnvValue":"0","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"MAX_CI_WORKFLOW_RETRIES","EnvType":"int","EnvValue":"0","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"MODE","EnvType":"string","EnvValue":"DEV","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_SERVER_HOST","EnvType":"string","EnvValue":"localhost:4222","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ORCH_HOST","EnvType":"string","EnvValue":"http://devtroncd-orchestrator-service-prod.devtroncd/webhook/msg/nats","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ORCH_TOKEN","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PRE_CI_CACHE_PATH","EnvType":"string","EnvValue":"/devtroncd-cache","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"SHOW_DOCKER_BUILD_ARGS","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"SKIP_CI_JOB_BUILD_CACHE_PUSH_PULL","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"SKIP_CREATING_ECR_REPO","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TERMINATION_GRACE_PERIOD_SECS","EnvType":"int","EnvValue":"180","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"USE_ARTIFACT_LISTING_QUERY_V2","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"USE_BLOB_STORAGE_CONFIG_IN_CD_WORKFLOW","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"USE_BLOB_STORAGE_CONFIG_IN_CI_WORKFLOW","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"USE_BUILDX","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"USE_DOCKER_API_TO_GET_DIGEST","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"USE_EXTERNAL_NODE","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"USE_IMAGE_TAG_FROM_GIT_PROVIDER_FOR_TAG_BASED_BUILD","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"WF_CONTROLLER_INSTANCE_ID","EnvType":"string","EnvValue":"devtron-runner","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"WORKFLOW_CACHE_CONFIG","EnvType":"string","EnvValue":"{}","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"WORKFLOW_SERVICE_ACCOUNT","EnvType":"string","EnvValue":"ci-runner","EnvDescription":"","Example":"","Deprecated":"false"}]},{"Category":"DEVTRON","Fields":[{"Env":"-","EnvType":"","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ADDITIONAL_NODE_GROUP_LABELS","EnvType":"","EnvValue":"","EnvDescription":"Add comma separated list of additional node group labels to default labels","Example":"karpenter.sh/nodepool,cloud.google.com/gke-nodepool","Deprecated":"false"},{"Env":"APP_SYNC_IMAGE","EnvType":"string","EnvValue":"quay.io/devtron/chart-sync:1227622d-132-3775","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"APP_SYNC_JOB_RESOURCES_OBJ","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"APP_SYNC_SERVICE_ACCOUNT","EnvType":"string","EnvValue":"chart-sync","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"APP_SYNC_SHUTDOWN_WAIT_DURATION","EnvType":"int","EnvValue":"120","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ARGO_AUTO_SYNC_ENABLED","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ARGO_GIT_COMMIT_RETRY_COUNT_ON_CONFLICT","EnvType":"int","EnvValue":"3","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ARGO_GIT_COMMIT_RETRY_DELAY_ON_CONFLICT","EnvType":"int","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ARGO_REPO_REGISTER_RETRY_COUNT","EnvType":"int","EnvValue":"3","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ARGO_REPO_REGISTER_RETRY_DELAY","EnvType":"int","EnvValue":"10","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ASYNC_BUILDX_CACHE_EXPORT","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BATCH_SIZE","EnvType":"int","EnvValue":"5","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_ENABLED","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BUILDX_CACHE_MODE_MIN","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_HOST","EnvType":"string","EnvValue":"localhost","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_NAMESPACE","EnvType":"string","EnvValue":"devtroncd","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_PORT","EnvType":"string","EnvValue":"8000","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CExpirationTime","EnvType":"int","EnvValue":"600","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CI_TRIGGER_CRON_TIME","EnvType":"int","EnvValue":"2","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CI_WORKFLOW_STATUS_UPDATE_CRON","EnvType":"string","EnvValue":"*/5 * * * *","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CLI_CMD_TIMEOUT_GLOBAL_SECONDS","EnvType":"int","EnvValue":"0","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CLUSTER_STATUS_CRON_TIME","EnvType":"int","EnvValue":"15","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CONSUMER_CONFIG_JSON","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_LOG_TIME_LIMIT","EnvType":"int64","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_TIMEOUT","EnvType":"float64","EnvValue":"3600","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEVTRON_BOM_URL","EnvType":"string","EnvValue":"https://raw.githubusercontent.com/devtron-labs/devtron/%s/charts/devtron/devtron-bom.yaml","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEVTRON_DEFAULT_NAMESPACE","EnvType":"string","EnvValue":"devtroncd","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEVTRON_DEX_SECRET_NAMESPACE","EnvType":"string","EnvValue":"devtroncd","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEVTRON_HELM_RELEASE_CHART_NAME","EnvType":"string","EnvValue":"devtron-operator","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEVTRON_HELM_RELEASE_NAME","EnvType":"string","EnvValue":"devtron","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEVTRON_HELM_RELEASE_NAMESPACE","EnvType":"string","EnvValue":"devtroncd","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEVTRON_HELM_REPO_NAME","EnvType":"string","EnvValue":"devtron","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEVTRON_HELM_REPO_URL","EnvType":"string","EnvValue":"https://helm.devtron.ai","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEVTRON_INSTALLATION_TYPE","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEVTRON_MODULES_IDENTIFIER_IN_HELM_VALUES","EnvType":"string","EnvValue":"installer.modules","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEVTRON_SECRET_NAME","EnvType":"string","EnvValue":"devtron-secret","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEVTRON_VERSION_IDENTIFIER_IN_HELM_VALUES","EnvType":"string","EnvValue":"installer.release","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEX_CID","EnvType":"string","EnvValue":"example-app","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEX_CLIENT_ID","EnvType":"string","EnvValue":"argo-cd","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEX_CSTOREKEY","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEX_JWTKEY","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEX_RURL","EnvType":"string","EnvValue":"http://127.0.0.1:8080/callback","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEX_SECRET","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEX_URL","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ECR_REPO_NAME_PREFIX","EnvType":"string","EnvValue":"test/","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ENABLE_ASYNC_ARGO_CD_INSTALL_DEVTRON_CHART","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ENABLE_ASYNC_INSTALL_DEVTRON_CHART","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"EPHEMERAL_SERVER_VERSION_REGEX","EnvType":"string","EnvValue":"v[1-9]\\.\\b(2[3-9]\\|[3-9][0-9])\\b.*","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"EVENT_URL","EnvType":"string","EnvValue":"http://localhost:3000/notify","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"EXECUTE_WIRE_NIL_CHECKER","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"EXPOSE_CI_METRICS","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"FEATURE_RESTART_WORKLOAD_BATCH_SIZE","EnvType":"int","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"FEATURE_RESTART_WORKLOAD_WORKER_POOL_SIZE","EnvType":"int","EnvValue":"5","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"FORCE_SECURITY_SCANNING","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"GITOPS_REPO_PREFIX","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"GO_RUNTIME_ENV","EnvType":"string","EnvValue":"production","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"GRAFANA_HOST","EnvType":"string","EnvValue":"localhost","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"GRAFANA_NAMESPACE","EnvType":"string","EnvValue":"devtroncd","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"GRAFANA_ORG_ID","EnvType":"int","EnvValue":"2","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"GRAFANA_PASSWORD","EnvType":"string","EnvValue":"prom-operator","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"GRAFANA_PORT","EnvType":"string","EnvValue":"8090","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"GRAFANA_URL","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"GRAFANA_USERNAME","EnvType":"string","EnvValue":"admin","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"HIDE_IMAGE_TAGGING_HARD_DELETE","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"IGNORE_AUTOCOMPLETE_AUTH_CHECK","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"INSTALLED_MODULES","EnvType":"","EnvValue":"","EnvDescription":"List of installed modules given in helm values/yaml are written in cm and used by devtron to know which modules are given","Example":"security.trivy,security.clair","Deprecated":"false"},{"Env":"INSTALLER_CRD_NAMESPACE","EnvType":"string","EnvValue":"devtroncd","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"INSTALLER_CRD_OBJECT_GROUP_NAME","EnvType":"string","EnvValue":"installer.devtron.ai","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"INSTALLER_CRD_OBJECT_RESOURCE","EnvType":"string","EnvValue":"installers","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"INSTALLER_CRD_OBJECT_VERSION","EnvType":"string","EnvValue":"v1alpha1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"IS_AIR_GAP_ENVIRONMENT","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"JwtExpirationTime","EnvType":"int","EnvValue":"120","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"K8s_CLIENT_MAX_IDLE_CONNS_PER_HOST","EnvType":"int","EnvValue":"25","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"K8s_TCP_IDLE_CONN_TIMEOUT","EnvType":"int","EnvValue":"300","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"K8s_TCP_KEEPALIVE","EnvType":"int","EnvValue":"30","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"K8s_TCP_TIMEOUT","EnvType":"int","EnvValue":"30","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"K8s_TLS_HANDSHAKE_TIMEOUT","EnvType":"int","EnvValue":"10","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"KUBELINK_GRPC_MAX_RECEIVE_MSG_SIZE","EnvType":"int","EnvValue":"20","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"KUBELINK_GRPC_MAX_SEND_MSG_SIZE","EnvType":"int","EnvValue":"4","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"LENS_TIMEOUT","EnvType":"int","EnvValue":"0","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"LENS_URL","EnvType":"string","EnvValue":"http://lens-milandevtron-service:80","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"LIMIT_CI_CPU","EnvType":"string","EnvValue":"0.5","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"LIMIT_CI_MEM","EnvType":"string","EnvValue":"3G","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"LOGGER_DEV_MODE","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"LOG_LEVEL","EnvType":"int","EnvValue":"-1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"MAX_SESSION_PER_USER","EnvType":"int","EnvValue":"5","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"MODULE_METADATA_API_URL","EnvType":"string","EnvValue":"https://api.devtron.ai/module?name=%s","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"MODULE_STATUS_HANDLING_CRON_DURATION_MIN","EnvType":"int","EnvValue":"3","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_ACK_WAIT_IN_SECS","EnvType":"int","EnvValue":"120","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_BUFFER_SIZE","EnvType":"int","EnvValue":"-1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_MAX_AGE","EnvType":"int","EnvValue":"86400","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_PROCESSING_BATCH_SIZE","EnvType":"int","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_REPLICAS","EnvType":"int","EnvValue":"0","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NOTIFICATION_MEDIUM","EnvType":"NotificationMedium","EnvValue":"rest","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"OTEL_COLLECTOR_URL","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PARALLELISM_LIMIT_FOR_TAG_PROCESSING","EnvType":"int","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_EXPORT_PROM_METRICS","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_LOG_ALL_FAILURE_QUERIES","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_LOG_ALL_QUERY","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_LOG_SLOW_QUERY","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_QUERY_DUR_THRESHOLD","EnvType":"int64","EnvValue":"5000","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PLUGIN_NAME","EnvType":"string","EnvValue":"Pull images from container repository","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PROPAGATE_EXTRA_LABELS","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PROXY_SERVICE_CONFIG","EnvType":"string","EnvValue":"{}","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"REQ_CI_CPU","EnvType":"string","EnvValue":"0.5","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"REQ_CI_MEM","EnvType":"string","EnvValue":"3G","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"RESTRICT_TERMINAL_ACCESS_FOR_NON_SUPER_USER","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"RUNTIME_CONFIG_LOCAL_DEV","EnvType":"LocalDevMode","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"SCOPED_VARIABLE_ENABLED","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"SCOPED_VARIABLE_FORMAT","EnvType":"string","EnvValue":"@{{%s}}","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"SCOPED_VARIABLE_HANDLE_PRIMITIVES","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"SCOPED_VARIABLE_NAME_REGEX","EnvType":"string","EnvValue":"^[a-zA-Z][a-zA-Z0-9_-]{0,62}[a-zA-Z0-9]$","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"SOCKET_DISCONNECT_DELAY_SECONDS","EnvType":"int","EnvValue":"5","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"SOCKET_HEARTBEAT_SECONDS","EnvType":"int","EnvValue":"25","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"STREAM_CONFIG_JSON","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"SYSTEM_VAR_PREFIX","EnvType":"string","EnvValue":"DEVTRON_","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TERMINAL_POD_DEFAULT_NAMESPACE","EnvType":"string","EnvValue":"default","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TERMINAL_POD_INACTIVE_DURATION_IN_MINS","EnvType":"int","EnvValue":"10","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TERMINAL_POD_STATUS_SYNC_In_SECS","EnvType":"int","EnvValue":"600","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TEST_APP","EnvType":"string","EnvValue":"orchestrator","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TEST_PG_ADDR","EnvType":"string","EnvValue":"127.0.0.1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TEST_PG_DATABASE","EnvType":"string","EnvValue":"orchestrator","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TEST_PG_LOG_QUERY","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TEST_PG_PASSWORD","EnvType":"string","EnvValue":"postgrespw","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TEST_PG_PORT","EnvType":"string","EnvValue":"55000","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TEST_PG_USER","EnvType":"string","EnvValue":"postgres","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TIMEOUT_FOR_FAILED_CI_BUILD","EnvType":"string","EnvValue":"15","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TIMEOUT_IN_SECONDS","EnvType":"int","EnvValue":"5","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"USER_SESSION_DURATION_SECONDS","EnvType":"int","EnvValue":"86400","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"USE_ARTIFACT_LISTING_API_V2","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"USE_CUSTOM_HTTP_TRANSPORT","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"USE_GIT_CLI","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"USE_RBAC_CREATION_V2","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"VARIABLE_CACHE_ENABLED","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"VARIABLE_EXPRESSION_REGEX","EnvType":"string","EnvValue":"@{{([^}]+)}}","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"WEBHOOK_TOKEN","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"}]},{"Category":"GITOPS","Fields":[{"Env":"ACD_CM","EnvType":"string","EnvValue":"argocd-cm","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ACD_NAMESPACE","EnvType":"string","EnvValue":"devtroncd","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ACD_PASSWORD","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ACD_USERNAME","EnvType":"string","EnvValue":"admin","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"GITOPS_SECRET_NAME","EnvType":"string","EnvValue":"devtron-gitops-secret","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"RESOURCE_LIST_FOR_REPLICAS","EnvType":"string","EnvValue":"Deployment,Rollout,StatefulSet,ReplicaSet","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"RESOURCE_LIST_FOR_REPLICAS_BATCH_SIZE","EnvType":"int","EnvValue":"5","EnvDescription":"","Example":"","Deprecated":"false"}]},{"Category":"INFRA_SETUP","Fields":[{"Env":"DASHBOARD_HOST","EnvType":"string","EnvValue":"localhost","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DASHBOARD_NAMESPACE","EnvType":"string","EnvValue":"devtroncd","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DASHBOARD_PORT","EnvType":"string","EnvValue":"3000","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEX_HOST","EnvType":"string","EnvValue":"http://localhost","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEX_PORT","EnvType":"string","EnvValue":"5556","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"GIT_SENSOR_PROTOCOL","EnvType":"string","EnvValue":"REST","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"GIT_SENSOR_TIMEOUT","EnvType":"int","EnvValue":"0","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"GIT_SENSOR_URL","EnvType":"string","EnvValue":"127.0.0.1:7070","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"HELM_CLIENT_URL","EnvType":"string","EnvValue":"127.0.0.1:50051","EnvDescription":"","Example":"","Deprecated":"false"}]},{"Category":"POSTGRES","Fields":[{"Env":"APP","EnvType":"string","EnvValue":"orchestrator","EnvDescription":"Application name","Example":"","Deprecated":"false"},{"Env":"CASBIN_DATABASE","EnvType":"string","EnvValue":"casbin","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_ADDR","EnvType":"string","EnvValue":"127.0.0.1","EnvDescription":"address of postgres service","Example":"postgresql-postgresql.devtroncd","Deprecated":"false"},{"Env":"PG_DATABASE","EnvType":"string","EnvValue":"orchestrator","EnvDescription":"postgres database to be made connection with","Example":"orchestrator, casbin, git_sensor, lens","Deprecated":"false"},{"Env":"PG_PASSWORD","EnvType":"string","EnvValue":"{password}","EnvDescription":"password for postgres, associated with PG_USER","Example":"confidential ;)","Deprecated":"false"},{"Env":"PG_PORT","EnvType":"string","EnvValue":"5432","EnvDescription":"port of postgresql service","Example":"5432","Deprecated":"false"},{"Env":"PG_READ_TIMEOUT","EnvType":"int64","EnvValue":"30","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_USER","EnvType":"string","EnvValue":"postgres","EnvDescription":"user for postgres","Example":"postgres","Deprecated":"false"},{"Env":"PG_WRITE_TIMEOUT","EnvType":"int64","EnvValue":"30","EnvDescription":"","Example":"","Deprecated":"false"}]},{"Category":"RBAC","Fields":[{"Env":"ENFORCER_CACHE","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ENFORCER_CACHE_EXPIRATION_IN_SEC","EnvType":"int","EnvValue":"86400","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ENFORCER_MAX_BATCH_SIZE","EnvType":"int","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"USE_CASBIN_V2","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"}]}] \ No newline at end of file +[{"Category":"CD","Fields":[{"Env":"ARGO_APP_MANUAL_SYNC_TIME","EnvType":"int","EnvValue":"3","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_HELM_PIPELINE_STATUS_CRON_TIME","EnvType":"string","EnvValue":"*/2 * * * *","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_PIPELINE_STATUS_CRON_TIME","EnvType":"string","EnvValue":"*/2 * * * *","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_PIPELINE_STATUS_TIMEOUT_DURATION","EnvType":"string","EnvValue":"20","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEPLOY_STATUS_CRON_GET_PIPELINE_DEPLOYED_WITHIN_HOURS","EnvType":"int","EnvValue":"12","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEVTRON_CHART_ARGO_CD_INSTALL_REQUEST_TIMEOUT","EnvType":"int","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEVTRON_CHART_INSTALL_REQUEST_TIMEOUT","EnvType":"int","EnvValue":"6","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"EXPOSE_CD_METRICS","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"FEATURE_MIGRATE_ARGOCD_APPLICATION_ENABLE","EnvType":"bool","EnvValue":"false","EnvDescription":"enable migration of external argocd application to devtron pipeline","Example":"","Deprecated":"false"},{"Env":"HELM_PIPELINE_STATUS_CHECK_ELIGIBLE_TIME","EnvType":"string","EnvValue":"120","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"IS_INTERNAL_USE","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"MIGRATE_DEPLOYMENT_CONFIG_DATA","EnvType":"bool","EnvValue":"false","EnvDescription":"migrate deployment config data from charts table to deployment_config table","Example":"","Deprecated":"false"},{"Env":"PIPELINE_DEGRADED_TIME","EnvType":"string","EnvValue":"10","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"REVISION_HISTORY_LIMIT_DEVTRON_APP","EnvType":"int","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"REVISION_HISTORY_LIMIT_EXTERNAL_HELM_APP","EnvType":"int","EnvValue":"0","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"REVISION_HISTORY_LIMIT_HELM_APP","EnvType":"int","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"REVISION_HISTORY_LIMIT_LINKED_HELM_APP","EnvType":"int","EnvValue":"15","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"RUN_HELM_INSTALL_IN_ASYNC_MODE_HELM_APPS","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"SHOULD_CHECK_NAMESPACE_ON_CLONE","EnvType":"bool","EnvValue":"false","EnvDescription":"should we check if namespace exists or not while cloning app","Example":"","Deprecated":"false"},{"Env":"USE_DEPLOYMENT_CONFIG_DATA","EnvType":"bool","EnvValue":"false","EnvDescription":"use deployment config data from deployment_config table","Example":"","Deprecated":"true"}]},{"Category":"CI_RUNNER","Fields":[{"Env":"AZURE_ACCOUNT_KEY","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"AZURE_ACCOUNT_NAME","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"AZURE_BLOB_CONTAINER_CI_CACHE","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"AZURE_BLOB_CONTAINER_CI_LOG","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"AZURE_GATEWAY_CONNECTION_INSECURE","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"AZURE_GATEWAY_URL","EnvType":"string","EnvValue":"http://devtron-minio.devtroncd:9000","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BASE_LOG_LOCATION_PATH","EnvType":"string","EnvValue":"/home/devtron/","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_GCP_CREDENTIALS_JSON","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_PROVIDER","EnvType":"","EnvValue":"S3","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_S3_ACCESS_KEY","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_S3_BUCKET_VERSIONED","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_S3_ENDPOINT","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_S3_ENDPOINT_INSECURE","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_S3_SECRET_KEY","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BUILDX_CACHE_PATH","EnvType":"string","EnvValue":"/var/lib/devtron/buildx","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BUILDX_K8S_DRIVER_OPTIONS","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BUILDX_PROVENANCE_MODE","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BUILD_LOG_TTL_VALUE_IN_SECS","EnvType":"int","EnvValue":"3600","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CACHE_LIMIT","EnvType":"int64","EnvValue":"5000000000","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_DEFAULT_ADDRESS_POOL_BASE_CIDR","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_DEFAULT_ADDRESS_POOL_SIZE","EnvType":"int","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_LIMIT_CI_CPU","EnvType":"string","EnvValue":"0.5","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_LIMIT_CI_MEM","EnvType":"string","EnvValue":"3G","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_NODE_LABEL_SELECTOR","EnvType":"","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_NODE_TAINTS_KEY","EnvType":"string","EnvValue":"dedicated","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_NODE_TAINTS_VALUE","EnvType":"string","EnvValue":"ci","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_REQ_CI_CPU","EnvType":"string","EnvValue":"0.5","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_REQ_CI_MEM","EnvType":"string","EnvValue":"3G","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_WORKFLOW_EXECUTOR_TYPE","EnvType":"","EnvValue":"AWF","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_WORKFLOW_SERVICE_ACCOUNT","EnvType":"string","EnvValue":"cd-runner","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CI_DEFAULT_ADDRESS_POOL_BASE_CIDR","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CI_DEFAULT_ADDRESS_POOL_SIZE","EnvType":"int","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CI_IGNORE_DOCKER_CACHE","EnvType":"bool","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CI_LOGS_KEY_PREFIX","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CI_NODE_LABEL_SELECTOR","EnvType":"","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CI_NODE_TAINTS_KEY","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CI_NODE_TAINTS_VALUE","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CI_RUNNER_DOCKER_MTU_VALUE","EnvType":"int","EnvValue":"-1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CI_SUCCESS_AUTO_TRIGGER_BATCH_SIZE","EnvType":"int","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CI_VOLUME_MOUNTS_JSON","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CI_WORKFLOW_EXECUTOR_TYPE","EnvType":"","EnvValue":"AWF","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_ARTIFACT_KEY_LOCATION","EnvType":"string","EnvValue":"arsenal-v1/ci-artifacts","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_BUILD_LOGS_BUCKET","EnvType":"string","EnvValue":"devtron-pro-ci-logs","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_BUILD_LOGS_KEY_PREFIX","EnvType":"string","EnvValue":"arsenal-v1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_CACHE_BUCKET","EnvType":"string","EnvValue":"ci-caching","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_CACHE_BUCKET_REGION","EnvType":"string","EnvValue":"us-east-2","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_CD_ARTIFACT_KEY_LOCATION","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_CD_LOGS_BUCKET_REGION","EnvType":"string","EnvValue":"us-east-2","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_CD_NAMESPACE","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_CD_TIMEOUT","EnvType":"int64","EnvValue":"3600","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_CI_IMAGE","EnvType":"string","EnvValue":"686244538589.dkr.ecr.us-east-2.amazonaws.com/cirunner:47","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_NAMESPACE","EnvType":"string","EnvValue":"devtron-ci","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_TARGET_PLATFORM","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DOCKER_BUILD_CACHE_PATH","EnvType":"string","EnvValue":"/var/lib/docker","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ENABLE_BUILD_CONTEXT","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ENABLE_WORKFLOW_EXECUTION_STAGE","EnvType":"bool","EnvValue":"true","EnvDescription":"if enabled then we will display build stages separately for CI/Job/Pre-Post CD","Example":"true","Deprecated":"false"},{"Env":"EXTERNAL_BLOB_STORAGE_CM_NAME","EnvType":"string","EnvValue":"blob-storage-cm","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"EXTERNAL_BLOB_STORAGE_SECRET_NAME","EnvType":"string","EnvValue":"blob-storage-secret","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"EXTERNAL_CD_NODE_LABEL_SELECTOR","EnvType":"","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"EXTERNAL_CD_NODE_TAINTS_KEY","EnvType":"string","EnvValue":"dedicated","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"EXTERNAL_CD_NODE_TAINTS_VALUE","EnvType":"string","EnvValue":"ci","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"EXTERNAL_CI_API_SECRET","EnvType":"string","EnvValue":"devtroncd-secret","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"EXTERNAL_CI_PAYLOAD","EnvType":"string","EnvValue":"{\"ciProjectDetails\":[{\"gitRepository\":\"https://github.com/vikram1601/getting-started-nodejs.git\",\"checkoutPath\":\"./abc\",\"commitHash\":\"239077135f8cdeeccb7857e2851348f558cb53d3\",\"commitTime\":\"2022-10-30T20:00:00\",\"branch\":\"master\",\"message\":\"Update README.md\",\"author\":\"User Name \"}],\"dockerImage\":\"445808685819.dkr.ecr.us-east-2.amazonaws.com/orch:23907713-2\"}","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"EXTERNAL_CI_WEB_HOOK_URL","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"IGNORE_CM_CS_IN_CI_JOB","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"IMAGE_RETRY_COUNT","EnvType":"int","EnvValue":"0","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"IMAGE_RETRY_INTERVAL","EnvType":"int","EnvValue":"5","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"IMAGE_SCANNER_ENDPOINT","EnvType":"string","EnvValue":"http://image-scanner-new-demo-devtroncd-service.devtroncd:80","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"IMAGE_SCAN_MAX_RETRIES","EnvType":"int","EnvValue":"3","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"IMAGE_SCAN_RETRY_DELAY","EnvType":"int","EnvValue":"5","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"IN_APP_LOGGING_ENABLED","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"MAX_CD_WORKFLOW_RUNNER_RETRIES","EnvType":"int","EnvValue":"0","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"MAX_CI_WORKFLOW_RETRIES","EnvType":"int","EnvValue":"0","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"MODE","EnvType":"string","EnvValue":"DEV","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_SERVER_HOST","EnvType":"string","EnvValue":"localhost:4222","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ORCH_HOST","EnvType":"string","EnvValue":"http://devtroncd-orchestrator-service-prod.devtroncd/webhook/msg/nats","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ORCH_TOKEN","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PRE_CI_CACHE_PATH","EnvType":"string","EnvValue":"/devtroncd-cache","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"SHOW_DOCKER_BUILD_ARGS","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"SKIP_CI_JOB_BUILD_CACHE_PUSH_PULL","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"SKIP_CREATING_ECR_REPO","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TERMINATION_GRACE_PERIOD_SECS","EnvType":"int","EnvValue":"180","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"USE_ARTIFACT_LISTING_QUERY_V2","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"USE_BLOB_STORAGE_CONFIG_IN_CD_WORKFLOW","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"USE_BLOB_STORAGE_CONFIG_IN_CI_WORKFLOW","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"USE_BUILDX","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"USE_DOCKER_API_TO_GET_DIGEST","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"USE_EXTERNAL_NODE","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"USE_IMAGE_TAG_FROM_GIT_PROVIDER_FOR_TAG_BASED_BUILD","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"WF_CONTROLLER_INSTANCE_ID","EnvType":"string","EnvValue":"devtron-runner","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"WORKFLOW_CACHE_CONFIG","EnvType":"string","EnvValue":"{}","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"WORKFLOW_SERVICE_ACCOUNT","EnvType":"string","EnvValue":"ci-runner","EnvDescription":"","Example":"","Deprecated":"false"}]},{"Category":"DEVTRON","Fields":[{"Env":"-","EnvType":"","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ADDITIONAL_NODE_GROUP_LABELS","EnvType":"","EnvValue":"","EnvDescription":"Add comma separated list of additional node group labels to default labels","Example":"karpenter.sh/nodepool,cloud.google.com/gke-nodepool","Deprecated":"false"},{"Env":"APP_SYNC_IMAGE","EnvType":"string","EnvValue":"quay.io/devtron/chart-sync:1227622d-132-3775","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"APP_SYNC_JOB_RESOURCES_OBJ","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"APP_SYNC_SERVICE_ACCOUNT","EnvType":"string","EnvValue":"chart-sync","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"APP_SYNC_SHUTDOWN_WAIT_DURATION","EnvType":"int","EnvValue":"120","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ARGO_AUTO_SYNC_ENABLED","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ARGO_GIT_COMMIT_RETRY_COUNT_ON_CONFLICT","EnvType":"int","EnvValue":"3","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ARGO_GIT_COMMIT_RETRY_DELAY_ON_CONFLICT","EnvType":"int","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ARGO_REPO_REGISTER_RETRY_COUNT","EnvType":"int","EnvValue":"3","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ARGO_REPO_REGISTER_RETRY_DELAY","EnvType":"int","EnvValue":"10","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ASYNC_BUILDX_CACHE_EXPORT","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BATCH_SIZE","EnvType":"int","EnvValue":"5","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_ENABLED","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BUILDX_CACHE_MODE_MIN","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_HOST","EnvType":"string","EnvValue":"localhost","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_NAMESPACE","EnvType":"string","EnvValue":"devtroncd","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_PORT","EnvType":"string","EnvValue":"8000","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CExpirationTime","EnvType":"int","EnvValue":"600","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CI_TRIGGER_CRON_TIME","EnvType":"int","EnvValue":"2","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CI_WORKFLOW_STATUS_UPDATE_CRON","EnvType":"string","EnvValue":"*/5 * * * *","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CLI_CMD_TIMEOUT_GLOBAL_SECONDS","EnvType":"int","EnvValue":"0","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CLUSTER_STATUS_CRON_TIME","EnvType":"int","EnvValue":"15","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CONSUMER_CONFIG_JSON","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_LOG_TIME_LIMIT","EnvType":"int64","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_TIMEOUT","EnvType":"float64","EnvValue":"3600","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEVTRON_BOM_URL","EnvType":"string","EnvValue":"https://raw.githubusercontent.com/devtron-labs/devtron/%s/charts/devtron/devtron-bom.yaml","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEVTRON_DEFAULT_NAMESPACE","EnvType":"string","EnvValue":"devtroncd","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEVTRON_DEX_SECRET_NAMESPACE","EnvType":"string","EnvValue":"devtroncd","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEVTRON_HELM_RELEASE_CHART_NAME","EnvType":"string","EnvValue":"devtron-operator","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEVTRON_HELM_RELEASE_NAME","EnvType":"string","EnvValue":"devtron","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEVTRON_HELM_RELEASE_NAMESPACE","EnvType":"string","EnvValue":"devtroncd","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEVTRON_HELM_REPO_NAME","EnvType":"string","EnvValue":"devtron","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEVTRON_HELM_REPO_URL","EnvType":"string","EnvValue":"https://helm.devtron.ai","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEVTRON_INSTALLATION_TYPE","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEVTRON_MODULES_IDENTIFIER_IN_HELM_VALUES","EnvType":"string","EnvValue":"installer.modules","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEVTRON_SECRET_NAME","EnvType":"string","EnvValue":"devtron-secret","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEVTRON_VERSION_IDENTIFIER_IN_HELM_VALUES","EnvType":"string","EnvValue":"installer.release","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEX_CID","EnvType":"string","EnvValue":"example-app","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEX_CLIENT_ID","EnvType":"string","EnvValue":"argo-cd","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEX_CSTOREKEY","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEX_JWTKEY","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEX_RURL","EnvType":"string","EnvValue":"http://127.0.0.1:8080/callback","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEX_SCOPES","EnvType":"","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEX_SECRET","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEX_URL","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ECR_REPO_NAME_PREFIX","EnvType":"string","EnvValue":"test/","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ENABLE_ASYNC_ARGO_CD_INSTALL_DEVTRON_CHART","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ENABLE_ASYNC_INSTALL_DEVTRON_CHART","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"EPHEMERAL_SERVER_VERSION_REGEX","EnvType":"string","EnvValue":"v[1-9]\\.\\b(2[3-9]\\|[3-9][0-9])\\b.*","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"EVENT_URL","EnvType":"string","EnvValue":"http://localhost:3000/notify","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"EXECUTE_WIRE_NIL_CHECKER","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"EXPOSE_CI_METRICS","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"FEATURE_RESTART_WORKLOAD_BATCH_SIZE","EnvType":"int","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"FEATURE_RESTART_WORKLOAD_WORKER_POOL_SIZE","EnvType":"int","EnvValue":"5","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"FORCE_SECURITY_SCANNING","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"GITOPS_REPO_PREFIX","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"GO_RUNTIME_ENV","EnvType":"string","EnvValue":"production","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"GRAFANA_HOST","EnvType":"string","EnvValue":"localhost","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"GRAFANA_NAMESPACE","EnvType":"string","EnvValue":"devtroncd","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"GRAFANA_ORG_ID","EnvType":"int","EnvValue":"2","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"GRAFANA_PASSWORD","EnvType":"string","EnvValue":"prom-operator","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"GRAFANA_PORT","EnvType":"string","EnvValue":"8090","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"GRAFANA_URL","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"GRAFANA_USERNAME","EnvType":"string","EnvValue":"admin","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"HIDE_IMAGE_TAGGING_HARD_DELETE","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"IGNORE_AUTOCOMPLETE_AUTH_CHECK","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"INSTALLED_MODULES","EnvType":"","EnvValue":"","EnvDescription":"List of installed modules given in helm values/yaml are written in cm and used by devtron to know which modules are given","Example":"security.trivy,security.clair","Deprecated":"false"},{"Env":"INSTALLER_CRD_NAMESPACE","EnvType":"string","EnvValue":"devtroncd","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"INSTALLER_CRD_OBJECT_GROUP_NAME","EnvType":"string","EnvValue":"installer.devtron.ai","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"INSTALLER_CRD_OBJECT_RESOURCE","EnvType":"string","EnvValue":"installers","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"INSTALLER_CRD_OBJECT_VERSION","EnvType":"string","EnvValue":"v1alpha1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"IS_AIR_GAP_ENVIRONMENT","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"JwtExpirationTime","EnvType":"int","EnvValue":"120","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"K8s_CLIENT_MAX_IDLE_CONNS_PER_HOST","EnvType":"int","EnvValue":"25","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"K8s_TCP_IDLE_CONN_TIMEOUT","EnvType":"int","EnvValue":"300","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"K8s_TCP_KEEPALIVE","EnvType":"int","EnvValue":"30","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"K8s_TCP_TIMEOUT","EnvType":"int","EnvValue":"30","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"K8s_TLS_HANDSHAKE_TIMEOUT","EnvType":"int","EnvValue":"10","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"KUBELINK_GRPC_MAX_RECEIVE_MSG_SIZE","EnvType":"int","EnvValue":"20","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"KUBELINK_GRPC_MAX_SEND_MSG_SIZE","EnvType":"int","EnvValue":"4","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"LENS_TIMEOUT","EnvType":"int","EnvValue":"0","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"LENS_URL","EnvType":"string","EnvValue":"http://lens-milandevtron-service:80","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"LIMIT_CI_CPU","EnvType":"string","EnvValue":"0.5","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"LIMIT_CI_MEM","EnvType":"string","EnvValue":"3G","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"LOGGER_DEV_MODE","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"LOG_LEVEL","EnvType":"int","EnvValue":"-1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"MAX_SESSION_PER_USER","EnvType":"int","EnvValue":"5","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"MODULE_METADATA_API_URL","EnvType":"string","EnvValue":"https://api.devtron.ai/module?name=%s","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"MODULE_STATUS_HANDLING_CRON_DURATION_MIN","EnvType":"int","EnvValue":"3","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_ACK_WAIT_IN_SECS","EnvType":"int","EnvValue":"120","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_BUFFER_SIZE","EnvType":"int","EnvValue":"-1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_MAX_AGE","EnvType":"int","EnvValue":"86400","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_PROCESSING_BATCH_SIZE","EnvType":"int","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_REPLICAS","EnvType":"int","EnvValue":"0","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NOTIFICATION_MEDIUM","EnvType":"NotificationMedium","EnvValue":"rest","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"OTEL_COLLECTOR_URL","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PARALLELISM_LIMIT_FOR_TAG_PROCESSING","EnvType":"int","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_EXPORT_PROM_METRICS","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_LOG_ALL_FAILURE_QUERIES","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_LOG_ALL_QUERY","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_LOG_SLOW_QUERY","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_QUERY_DUR_THRESHOLD","EnvType":"int64","EnvValue":"5000","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PLUGIN_NAME","EnvType":"string","EnvValue":"Pull images from container repository","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PROPAGATE_EXTRA_LABELS","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PROXY_SERVICE_CONFIG","EnvType":"string","EnvValue":"{}","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"REQ_CI_CPU","EnvType":"string","EnvValue":"0.5","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"REQ_CI_MEM","EnvType":"string","EnvValue":"3G","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"RESTRICT_TERMINAL_ACCESS_FOR_NON_SUPER_USER","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"RUNTIME_CONFIG_LOCAL_DEV","EnvType":"LocalDevMode","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"SCOPED_VARIABLE_ENABLED","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"SCOPED_VARIABLE_FORMAT","EnvType":"string","EnvValue":"@{{%s}}","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"SCOPED_VARIABLE_HANDLE_PRIMITIVES","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"SCOPED_VARIABLE_NAME_REGEX","EnvType":"string","EnvValue":"^[a-zA-Z][a-zA-Z0-9_-]{0,62}[a-zA-Z0-9]$","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"SOCKET_DISCONNECT_DELAY_SECONDS","EnvType":"int","EnvValue":"5","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"SOCKET_HEARTBEAT_SECONDS","EnvType":"int","EnvValue":"25","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"STREAM_CONFIG_JSON","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"SYSTEM_VAR_PREFIX","EnvType":"string","EnvValue":"DEVTRON_","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TERMINAL_POD_DEFAULT_NAMESPACE","EnvType":"string","EnvValue":"default","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TERMINAL_POD_INACTIVE_DURATION_IN_MINS","EnvType":"int","EnvValue":"10","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TERMINAL_POD_STATUS_SYNC_In_SECS","EnvType":"int","EnvValue":"600","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TEST_APP","EnvType":"string","EnvValue":"orchestrator","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TEST_PG_ADDR","EnvType":"string","EnvValue":"127.0.0.1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TEST_PG_DATABASE","EnvType":"string","EnvValue":"orchestrator","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TEST_PG_LOG_QUERY","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TEST_PG_PASSWORD","EnvType":"string","EnvValue":"postgrespw","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TEST_PG_PORT","EnvType":"string","EnvValue":"55000","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TEST_PG_USER","EnvType":"string","EnvValue":"postgres","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TIMEOUT_FOR_FAILED_CI_BUILD","EnvType":"string","EnvValue":"15","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TIMEOUT_IN_SECONDS","EnvType":"int","EnvValue":"5","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"USER_SESSION_DURATION_SECONDS","EnvType":"int","EnvValue":"86400","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"USE_ARTIFACT_LISTING_API_V2","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"USE_CUSTOM_HTTP_TRANSPORT","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"USE_GIT_CLI","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"USE_RBAC_CREATION_V2","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"VARIABLE_CACHE_ENABLED","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"VARIABLE_EXPRESSION_REGEX","EnvType":"string","EnvValue":"@{{([^}]+)}}","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"WEBHOOK_TOKEN","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"}]},{"Category":"GITOPS","Fields":[{"Env":"ACD_CM","EnvType":"string","EnvValue":"argocd-cm","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ACD_NAMESPACE","EnvType":"string","EnvValue":"devtroncd","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ACD_PASSWORD","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ACD_USERNAME","EnvType":"string","EnvValue":"admin","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"GITOPS_SECRET_NAME","EnvType":"string","EnvValue":"devtron-gitops-secret","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"RESOURCE_LIST_FOR_REPLICAS","EnvType":"string","EnvValue":"Deployment,Rollout,StatefulSet,ReplicaSet","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"RESOURCE_LIST_FOR_REPLICAS_BATCH_SIZE","EnvType":"int","EnvValue":"5","EnvDescription":"","Example":"","Deprecated":"false"}]},{"Category":"INFRA_SETUP","Fields":[{"Env":"DASHBOARD_HOST","EnvType":"string","EnvValue":"localhost","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DASHBOARD_NAMESPACE","EnvType":"string","EnvValue":"devtroncd","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DASHBOARD_PORT","EnvType":"string","EnvValue":"3000","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEX_HOST","EnvType":"string","EnvValue":"http://localhost","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEX_PORT","EnvType":"string","EnvValue":"5556","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"GIT_SENSOR_PROTOCOL","EnvType":"string","EnvValue":"REST","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"GIT_SENSOR_TIMEOUT","EnvType":"int","EnvValue":"0","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"GIT_SENSOR_URL","EnvType":"string","EnvValue":"127.0.0.1:7070","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"HELM_CLIENT_URL","EnvType":"string","EnvValue":"127.0.0.1:50051","EnvDescription":"","Example":"","Deprecated":"false"}]},{"Category":"POSTGRES","Fields":[{"Env":"APP","EnvType":"string","EnvValue":"orchestrator","EnvDescription":"Application name","Example":"","Deprecated":"false"},{"Env":"CASBIN_DATABASE","EnvType":"string","EnvValue":"casbin","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_ADDR","EnvType":"string","EnvValue":"127.0.0.1","EnvDescription":"address of postgres service","Example":"postgresql-postgresql.devtroncd","Deprecated":"false"},{"Env":"PG_DATABASE","EnvType":"string","EnvValue":"orchestrator","EnvDescription":"postgres database to be made connection with","Example":"orchestrator, casbin, git_sensor, lens","Deprecated":"false"},{"Env":"PG_PASSWORD","EnvType":"string","EnvValue":"{password}","EnvDescription":"password for postgres, associated with PG_USER","Example":"confidential ;)","Deprecated":"false"},{"Env":"PG_PORT","EnvType":"string","EnvValue":"5432","EnvDescription":"port of postgresql service","Example":"5432","Deprecated":"false"},{"Env":"PG_READ_TIMEOUT","EnvType":"int64","EnvValue":"30","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_USER","EnvType":"string","EnvValue":"postgres","EnvDescription":"user for postgres","Example":"postgres","Deprecated":"false"},{"Env":"PG_WRITE_TIMEOUT","EnvType":"int64","EnvValue":"30","EnvDescription":"","Example":"","Deprecated":"false"}]},{"Category":"RBAC","Fields":[{"Env":"ENFORCER_CACHE","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ENFORCER_CACHE_EXPIRATION_IN_SEC","EnvType":"int","EnvValue":"86400","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ENFORCER_MAX_BATCH_SIZE","EnvType":"int","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"USE_CASBIN_V2","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"}]}] \ No newline at end of file diff --git a/env_gen.md b/env_gen.md index f247bf8a1f..63d998590a 100644 --- a/env_gen.md +++ b/env_gen.md @@ -168,6 +168,7 @@ | DEX_CSTOREKEY | string | | | | false | | DEX_JWTKEY | string | | | | false | | DEX_RURL | string |http://127.0.0.1:8080/callback | | | false | + | DEX_SCOPES | | | | | false | | DEX_SECRET | string | | | | false | | DEX_URL | string | | | | false | | ECR_REPO_NAME_PREFIX | string |test/ | | | false | diff --git a/wire_gen.go b/wire_gen.go index 7f38930f06..205046253d 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -1,6 +1,6 @@ // Code generated by Wire. DO NOT EDIT. -//go:generate go run github.com/google/wire/cmd/wire +//go:generate go run -mod=mod github.com/google/wire/cmd/wire //go:build !wireinject // +build !wireinject From 28ff909087c49e1edfe0f90a6f9b9a642de7c988 Mon Sep 17 00:00:00 2001 From: Ash-exp Date: Wed, 9 Apr 2025 23:32:35 +0530 Subject: [PATCH 17/34] fix: postgresql skip unique constraints error and handle network connection error --- go.mod | 4 +- go.sum | 8 +- .../common-lib/utils/CommonUtils.go | 56 ------- .../devtron-labs/common-lib/utils/SqlUtil.go | 139 ++++++++++++++++++ .../common-lib/utils/bean/bean.go | 1 + vendor/modules.txt | 8 +- 6 files changed, 150 insertions(+), 66 deletions(-) create mode 100644 vendor/github.com/devtron-labs/common-lib/utils/SqlUtil.go diff --git a/go.mod b/go.mod index 45efcdd373..392128e0b2 100644 --- a/go.mod +++ b/go.mod @@ -307,8 +307,8 @@ require ( replace ( github.com/argoproj/argo-workflows/v3 v3.5.13 => github.com/devtron-labs/argo-workflows/v3 v3.5.13 - github.com/devtron-labs/authenticator => github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250408072617-f637b61dcc6b - github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250408072617-f637b61dcc6b + github.com/devtron-labs/authenticator => github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250409180044-56ed13f8f5e5 + github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250409180044-56ed13f8f5e5 github.com/go-check/check => github.com/go-check/check v0.0.0-20180628173108-788fd7840127 github.com/googleapis/gnostic => github.com/googleapis/gnostic v0.5.5 k8s.io/api => k8s.io/api v0.29.7 diff --git a/go.sum b/go.sum index e5cd50a567..f0c1d38972 100644 --- a/go.sum +++ b/go.sum @@ -829,10 +829,10 @@ github.com/denisenkom/go-mssqldb v0.0.0-20200428022330-06a60b6afbbc h1:VRRKCwnzq github.com/denisenkom/go-mssqldb v0.0.0-20200428022330-06a60b6afbbc/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= github.com/devtron-labs/argo-workflows/v3 v3.5.13 h1:3pINq0gXOSeTw2z/vYe+j80lRpSN5Rp/8mfQORh8SmU= github.com/devtron-labs/argo-workflows/v3 v3.5.13/go.mod h1:/vqxcovDPT4zqr4DjR5v7CF8ggpY1l3TSa2CIG3jmjA= -github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250408072617-f637b61dcc6b h1:dgLyiFv/RJ5msxkXt8yP1Pmsq0ajvm2uVP0dErvAhLI= -github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250408072617-f637b61dcc6b/go.mod h1:5lv4Wfj5ERhhvDGXe2IeES6qxjvUVCcohaRwKnWBMNo= -github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250408072617-f637b61dcc6b h1:VvpEWGUZEmtqMYYrDgZwbxzaHzdVJXhBK4/NCm8bo/M= -github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250408072617-f637b61dcc6b/go.mod h1:ceFKgQ2qm40PR95g5Xp2EClq7nDBKFTcglJ0JdsgClA= +github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250409180044-56ed13f8f5e5 h1:J/PT2HoFj3LtMJzMoUNDFFCGbPcVEt6X2SpEDgAqmFQ= +github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250409180044-56ed13f8f5e5/go.mod h1:5lv4Wfj5ERhhvDGXe2IeES6qxjvUVCcohaRwKnWBMNo= +github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250409180044-56ed13f8f5e5 h1:olj+0hnWMAg4lqqGAtX7sAdvAs6RbzQEcjHmER5MZTI= +github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250409180044-56ed13f8f5e5/go.mod h1:ceFKgQ2qm40PR95g5Xp2EClq7nDBKFTcglJ0JdsgClA= github.com/devtron-labs/go-bitbucket v0.9.60-beta h1:VEx1jvDgdtDPS6A1uUFoaEi0l1/oLhbr+90xOwr6sDU= github.com/devtron-labs/go-bitbucket v0.9.60-beta/go.mod h1:GnuiCesvh8xyHeMCb+twm8lBR/kQzJYSKL28ZfObp1Y= github.com/devtron-labs/protos v0.0.3-0.20250323220609-ecf8a0f7305e h1:U6UdYbW8a7xn5IzFPd8cywjVVPfutGJCudjePAfL/Hs= diff --git a/vendor/github.com/devtron-labs/common-lib/utils/CommonUtils.go b/vendor/github.com/devtron-labs/common-lib/utils/CommonUtils.go index ad3cbbda0d..17ccda0615 100644 --- a/vendor/github.com/devtron-labs/common-lib/utils/CommonUtils.go +++ b/vendor/github.com/devtron-labs/common-lib/utils/CommonUtils.go @@ -17,13 +17,9 @@ package utils import ( - "errors" "fmt" "github.com/devtron-labs/common-lib/git-manager/util" "github.com/devtron-labs/common-lib/utils/bean" - "github.com/go-pg/pg" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promauto" "log" "math/rand" "os" @@ -96,53 +92,6 @@ func BuildDockerImagePath(dockerInfo bean.DockerRegistryInfo) (string, error) { return dest, nil } -func GetPGPostQueryProcessor(cfg bean.PgQueryMonitoringConfig) func(event *pg.QueryProcessedEvent) { - return func(event *pg.QueryProcessedEvent) { - query, err := event.FormattedQuery() - if err != nil { - log.Println("Error formatting query", "err", err) - return - } - ExecutePGQueryProcessor(cfg, bean.PgQueryEvent{ - StartTime: event.StartTime, - Error: event.Error, - Query: query, - }) - } -} - -func ExecutePGQueryProcessor(cfg bean.PgQueryMonitoringConfig, event bean.PgQueryEvent) { - queryDuration := time.Since(event.StartTime) - var queryError bool - pgError := event.Error - if pgError != nil && !errors.Is(pgError, pg.ErrNoRows) { - queryError = true - } - // Expose prom metrics - if cfg.ExportPromMetrics { - var status string - if queryError { - status = "FAIL" - } else { - status = "SUCCESS" - } - PgQueryDuration.WithLabelValues(status, cfg.ServiceName).Observe(queryDuration.Seconds()) - } - - // Log pg query if enabled - logThresholdQueries := cfg.LogSlowQuery && queryDuration.Milliseconds() > cfg.QueryDurationThreshold - logFailureQuery := queryError && cfg.LogAllFailureQueries - if logFailureQuery { - log.Println("PG_QUERY_FAIL - query time", "duration", queryDuration.Seconds(), "query", event.Query, "pgError", pgError) - } - if logThresholdQueries { - log.Println("PG_QUERY_SLOW - query time", "duration", queryDuration.Seconds(), "query", event.Query) - } - if cfg.LogAllQuery { - log.Println("query time", "duration", queryDuration.Seconds(), "query", event.Query) - } -} - func GetSelfK8sUID() string { return os.Getenv(DEVTRON_SELF_POD_UID) } @@ -151,11 +100,6 @@ func GetSelfK8sPodName() string { return os.Getenv(DEVTRON_SELF_POD_NAME) } -var PgQueryDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ - Name: "pg_query_duration_seconds", - Help: "Duration of PG queries", -}, []string{"status", "serviceName"}) - func ConvertTargetPlatformStringToObject(targetPlatformString string) []*bean.TargetPlatform { targetPlatforms := ConvertTargetPlatformStringToList(targetPlatformString) targetPlatformObject := []*bean.TargetPlatform{} diff --git a/vendor/github.com/devtron-labs/common-lib/utils/SqlUtil.go b/vendor/github.com/devtron-labs/common-lib/utils/SqlUtil.go new file mode 100644 index 0000000000..3bdec67be5 --- /dev/null +++ b/vendor/github.com/devtron-labs/common-lib/utils/SqlUtil.go @@ -0,0 +1,139 @@ +/* + * Copyright (c) 2024. Devtron Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package utils + +import ( + "errors" + "fmt" + "github.com/devtron-labs/common-lib/utils/bean" + "github.com/go-pg/pg" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + "io" + "log" + "net" + "os" + "time" +) + +const ( + PgNetworkErrorLogPrefix string = "PG_NETWORK_ERROR" + PgQueryFailLogPrefix string = "PG_QUERY_FAIL" + PgQuerySlowLogPrefix string = "PG_QUERY_SLOW" +) + +const ( + FAIL string = "FAIL" + SUCCESS string = "SUCCESS" +) + +type ErrorType string + +func (e ErrorType) String() string { + return string(e) +} + +const ( + NetworkErrorType ErrorType = "NETWORK_ERROR" + SyntaxErrorType ErrorType = "SYNTAX_ERROR" + TimeoutErrorType ErrorType = "TIMEOUT_ERROR" + NoErrorType ErrorType = "NA" +) + +func GetPGPostQueryProcessor(cfg bean.PgQueryMonitoringConfig) func(event *pg.QueryProcessedEvent) { + return func(event *pg.QueryProcessedEvent) { + query, err := event.FormattedQuery() + if err != nil { + log.Println("Error formatting query", "err", err) + return + } + ExecutePGQueryProcessor(cfg, bean.PgQueryEvent{ + StartTime: event.StartTime, + Error: event.Error, + Query: query, + FuncName: event.Func, + }) + } +} + +func ExecutePGQueryProcessor(cfg bean.PgQueryMonitoringConfig, event bean.PgQueryEvent) { + queryDuration := time.Since(event.StartTime) + var queryError bool + pgError := event.Error + if pgError != nil && !errors.Is(pgError, pg.ErrNoRows) && !isIntegrityViolationError(pgError) { + queryError = true + } + // Expose prom metrics + if cfg.ExportPromMetrics { + var status string + if queryError { + status = FAIL + } else { + status = SUCCESS + } + PgQueryDuration.WithLabelValues(status, cfg.ServiceName, event.FuncName, getErrorType(pgError).String()).Observe(queryDuration.Seconds()) + } + + // Log pg query if enabled + logThresholdQueries := cfg.LogSlowQuery && queryDuration.Milliseconds() > cfg.QueryDurationThreshold + logNetworkFailure := queryError && cfg.LogAllFailureQueries && isNetworkError(pgError) + if logNetworkFailure { + log.Println(fmt.Sprintf("%s - query time", PgNetworkErrorLogPrefix), "duration", queryDuration.Seconds(), "query", event.Query, "pgError", pgError) + } + logFailureQuery := queryError && cfg.LogAllFailureQueries && !isNetworkError(pgError) + if logFailureQuery { + log.Println(fmt.Sprintf("%s - query time", PgQueryFailLogPrefix), "duration", queryDuration.Seconds(), "query", event.Query, "pgError", pgError) + } + if logThresholdQueries { + log.Println(fmt.Sprintf("%s - query time", PgQuerySlowLogPrefix), "duration", queryDuration.Seconds(), "query", event.Query) + } + if cfg.LogAllQuery { + log.Println("query time", "duration", queryDuration.Seconds(), "query", event.Query) + } +} + +var PgQueryDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Name: "pg_query_duration_seconds", + Help: "Duration of PG queries", +}, []string{"status", "serviceName", "functionName", "errorType"}) + +func getErrorType(err error) ErrorType { + if err == nil { + return NoErrorType + } else if errors.Is(err, os.ErrDeadlineExceeded) { + return TimeoutErrorType + } else if isNetworkError(err) { + return NetworkErrorType + } + return SyntaxErrorType +} + +func isNetworkError(err error) bool { + if err == io.EOF { + return true + } + _, ok := err.(net.Error) + return ok +} + +func isIntegrityViolationError(err error) bool { + pgErr, ok := err.(pg.Error) + if !ok { + return false + } + return pgErr.IntegrityViolation() +} diff --git a/vendor/github.com/devtron-labs/common-lib/utils/bean/bean.go b/vendor/github.com/devtron-labs/common-lib/utils/bean/bean.go index 50b122e495..ea16a2f721 100644 --- a/vendor/github.com/devtron-labs/common-lib/utils/bean/bean.go +++ b/vendor/github.com/devtron-labs/common-lib/utils/bean/bean.go @@ -83,6 +83,7 @@ type PgQueryEvent struct { StartTime time.Time Error error Query string + FuncName string } type TargetPlatform struct { diff --git a/vendor/modules.txt b/vendor/modules.txt index 2db4b14bf8..fdb1c66d2e 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -472,7 +472,7 @@ github.com/davecgh/go-spew/spew # github.com/deckarep/golang-set v1.8.0 ## explicit; go 1.17 github.com/deckarep/golang-set -# github.com/devtron-labs/authenticator v0.4.35-0.20240809073103-6e11da8083f8 => github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250408072617-f637b61dcc6b +# github.com/devtron-labs/authenticator v0.4.35-0.20240809073103-6e11da8083f8 => github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250409180044-56ed13f8f5e5 ## explicit; go 1.21 github.com/devtron-labs/authenticator/apiToken github.com/devtron-labs/authenticator/client @@ -480,7 +480,7 @@ github.com/devtron-labs/authenticator/jwt github.com/devtron-labs/authenticator/middleware github.com/devtron-labs/authenticator/oidc github.com/devtron-labs/authenticator/password -# github.com/devtron-labs/common-lib v0.18.1-0.20241001061923-eda545dc839e => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250408072617-f637b61dcc6b +# github.com/devtron-labs/common-lib v0.18.1-0.20241001061923-eda545dc839e => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250409180044-56ed13f8f5e5 ## explicit; go 1.21 github.com/devtron-labs/common-lib/async github.com/devtron-labs/common-lib/blob-storage @@ -2350,8 +2350,8 @@ xorm.io/xorm/log xorm.io/xorm/names xorm.io/xorm/schemas xorm.io/xorm/tags -# github.com/devtron-labs/authenticator => github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250408072617-f637b61dcc6b -# github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250408072617-f637b61dcc6b +# github.com/devtron-labs/authenticator => github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250409180044-56ed13f8f5e5 +# github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250409180044-56ed13f8f5e5 # github.com/go-check/check => github.com/go-check/check v0.0.0-20180628173108-788fd7840127 # github.com/googleapis/gnostic => github.com/googleapis/gnostic v0.5.5 # k8s.io/api => k8s.io/api v0.29.7 From 8f9a83751105218c760d6b01ec188bd28e740382 Mon Sep 17 00:00:00 2001 From: prakhar katiyar Date: Thu, 10 Apr 2025 15:37:02 +0530 Subject: [PATCH 18/34] query fix --- internal/sql/repository/AppListingRepository.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/internal/sql/repository/AppListingRepository.go b/internal/sql/repository/AppListingRepository.go index 1866badee5..4b780c16a4 100644 --- a/internal/sql/repository/AppListingRepository.go +++ b/internal/sql/repository/AppListingRepository.go @@ -145,8 +145,7 @@ func (impl *AppListingRepositoryImpl) FetchAppsEnvContainers(envId int, appIds [ LEFT JOIN (SELECT pco.pipeline_id,MAX(pco.created_on) as last_deployed_time from pipeline_config_override pco GROUP BY pco.pipeline_id) ld ON ld.pipeline_id = p.id - WHERE a.active = true - ORDER BY a.app_name ` + WHERE a.active = true ` queryParams := []interface{}{envId, envId} @@ -156,6 +155,8 @@ func (impl *AppListingRepositoryImpl) FetchAppsEnvContainers(envId int, appIds [ queryParams = append(queryParams, pg.In(appIds)) } + query += " ORDER BY a.app_name " + if limit > 0 { query += fmt.Sprintf("LIMIT ? ") queryParams = append(queryParams, limit) From ed42b5d70c02e78ddce562b0c5198c094f119aa7 Mon Sep 17 00:00:00 2001 From: Neha130 Date: Thu, 10 Apr 2025 16:39:56 +0530 Subject: [PATCH 19/34] update the config file --- pkg/pipeline/PipelineBuilder.go | 4 +- pkg/pipeline/types/CiCdConfig.go | 174 +++++++++++++++---------------- 2 files changed, 89 insertions(+), 89 deletions(-) diff --git a/pkg/pipeline/PipelineBuilder.go b/pkg/pipeline/PipelineBuilder.go index 336dfd8cba..3ffa220179 100644 --- a/pkg/pipeline/PipelineBuilder.go +++ b/pkg/pipeline/PipelineBuilder.go @@ -40,7 +40,7 @@ const SECURITY_SCANNING = "FORCE_SECURITY_SCANNING" var DefaultPipelineValue = []byte(`{"ConfigMaps":{"enabled":false},"ConfigSecrets":{"enabled":false},"ContainerPort":[],"EnvVariables":[],"GracePeriod":30,"LivenessProbe":{},"MaxSurge":1,"MaxUnavailable":0,"MinReadySeconds":60,"ReadinessProbe":{},"Spec":{"Affinity":{"Values":"nodes","key":""}},"app":"13","appMetrics":false,"args":{},"autoscaling":{},"command":{"enabled":false,"value":[]},"containers":[],"dbMigrationConfig":{"enabled":false},"deployment":{"strategy":{"rolling":{"maxSurge":"25%","maxUnavailable":1}}},"deploymentType":"ROLLING","env":"1","envoyproxy":{"configMapName":"","image":"","resources":{"limits":{"cpu":"50m","memory":"50Mi"},"requests":{"cpu":"50m","memory":"50Mi"}}},"image":{"pullPolicy":"IfNotPresent"},"ingress":{},"ingressInternal":{"annotations":{},"enabled":false,"host":"","path":"","tls":[]},"initContainers":[],"pauseForSecondsBeforeSwitchActive":30,"pipelineName":"","prometheus":{"release":"monitoring"},"rawYaml":[],"releaseVersion":"1","replicaCount":1,"resources":{"limits":{"cpu":"0.05","memory":"50Mi"},"requests":{"cpu":"0.01","memory":"10Mi"}},"secret":{"data":{},"enabled":false},"server":{"deployment":{"image":"","image_tag":""}},"service":{"annotations":{},"type":"ClusterIP"},"servicemonitor":{"additionalLabels":{}},"tolerations":[],"volumeMounts":[],"volumes":[],"waitForSecondsBeforeScalingDown":30}`) type EcrConfig struct { - EcrPrefix string `env:"ECR_REPO_NAME_PREFIX" envDefault:"test/"` + EcrPrefix string `env:"ECR_REPO_NAME_PREFIX" envDefault:"test/" description:"Prefix for ECR repo to be created in does not exist"` } func GetEcrConfig() (*EcrConfig, error) { @@ -52,7 +52,7 @@ func GetEcrConfig() (*EcrConfig, error) { type SecurityConfig struct { //FORCE_SECURITY_SCANNING flag is being maintained in both dashboard and orchestrator CM's //TODO: rishabh will remove FORCE_SECURITY_SCANNING from dashboard's CM. - ForceSecurityScanning bool `env:"FORCE_SECURITY_SCANNING" envDefault:"false"` + ForceSecurityScanning bool `env:"FORCE_SECURITY_SCANNING" envDefault:"false" description: "By enabling this no one can disable image scaning on ci-pipeline from UI"` } type PipelineBuilder interface { diff --git a/pkg/pipeline/types/CiCdConfig.go b/pkg/pipeline/types/CiCdConfig.go index 961bd9f26f..83d9e97def 100644 --- a/pkg/pipeline/types/CiCdConfig.go +++ b/pkg/pipeline/types/CiCdConfig.go @@ -56,110 +56,110 @@ type CancelWfRequestDto struct { // CATEGORY=CI_RUNNER type CiCdConfig struct { // from ciConfig - DefaultCacheBucket string `env:"DEFAULT_CACHE_BUCKET" envDefault:"ci-caching"` - DefaultCacheBucketRegion string `env:"DEFAULT_CACHE_BUCKET_REGION" envDefault:"us-east-2"` - CiLogsKeyPrefix string `env:"CI_LOGS_KEY_PREFIX" envDxefault:"my-artifacts"` - CiDefaultImage string `env:"DEFAULT_CI_IMAGE" envDefault:"686244538589.dkr.ecr.us-east-2.amazonaws.com/cirunner:47"` - CiDefaultNamespace string `env:"DEFAULT_NAMESPACE" envDefault:"devtron-ci"` + DefaultCacheBucket string `env:"DEFAULT_CACHE_BUCKET" envDefault:"ci-caching" description:"Bucket name for build cache" ` + DefaultCacheBucketRegion string `env:"DEFAULT_CACHE_BUCKET_REGION" envDefault:"us-east-2" description:"Build Cache bucket region" ` + CiLogsKeyPrefix string `env:"CI_LOGS_KEY_PREFIX" envDxefault:"my-artifacts" description:"Prefix for build logs"` + CiDefaultImage string `env:"DEFAULT_CI_IMAGE" envDefault:"686244538589.dkr.ecr.us-east-2.amazonaws.com/cirunner:47" description:"To pass the ci-runner image"` + CiDefaultNamespace string `env:"DEFAULT_NAMESPACE" envDefault:"devtron-ci" description:"Timeout for CI to be completed"` CiDefaultBuildLogsBucket string `env:"DEFAULT_BUILD_LOGS_BUCKET" envDefault:"devtron-pro-ci-logs"` CiDefaultCdLogsBucketRegion string `env:"DEFAULT_CD_LOGS_BUCKET_REGION" envDefault:"us-east-2"` - CiTaintKey string `env:"CI_NODE_TAINTS_KEY" envDefault:""` - CiTaintValue string `env:"CI_NODE_TAINTS_VALUE" envDefault:""` - CiNodeLabelSelector []string `env:"CI_NODE_LABEL_SELECTOR"` - CacheLimit int64 `env:"CACHE_LIMIT" envDefault:"5000000000"` // TODO: Add to default db config also - CiDefaultBuildLogsKeyPrefix string `env:"DEFAULT_BUILD_LOGS_KEY_PREFIX" envDefault:"arsenal-v1"` - CiDefaultArtifactKeyPrefix string `env:"DEFAULT_ARTIFACT_KEY_LOCATION" envDefault:"arsenal-v1/ci-artifacts"` + CiTaintKey string `env:"CI_NODE_TAINTS_KEY" envDefault:"" description:"Toleration key for CI"` + CiTaintValue string `env:"CI_NODE_TAINTS_VALUE" envDefault:"" description:"Toleration value for CI" ` + CiNodeLabelSelector []string `env:"CI_NODE_LABEL_SELECTOR" description:"Node label selector for CI"` + CacheLimit int64 `env:"CACHE_LIMIT" envDefault:"5000000000" description:"Cache limit."` // TODO: Add to default db config also + CiDefaultBuildLogsKeyPrefix string `env:"DEFAULT_BUILD_LOGS_KEY_PREFIX" envDefault:"arsenal-v1" description:"Bucket prefix for build logs"` + CiDefaultArtifactKeyPrefix string `env:"DEFAULT_ARTIFACT_KEY_LOCATION" envDefault:"arsenal-v1/ci-artifacts" description:"Key location for artifacts being created"` CiWorkflowServiceAccount string `env:"WORKFLOW_SERVICE_ACCOUNT" envDefault:"ci-runner"` - ExternalCiApiSecret string `env:"EXTERNAL_CI_API_SECRET" envDefault:"devtroncd-secret"` - ExternalCiWebhookUrl string `env:"EXTERNAL_CI_WEB_HOOK_URL" envDefault:""` - ExternalCiPayload string `env:"EXTERNAL_CI_PAYLOAD" envDefault:"{\"ciProjectDetails\":[{\"gitRepository\":\"https://github.com/vikram1601/getting-started-nodejs.git\",\"checkoutPath\":\"./abc\",\"commitHash\":\"239077135f8cdeeccb7857e2851348f558cb53d3\",\"commitTime\":\"2022-10-30T20:00:00\",\"branch\":\"master\",\"message\":\"Update README.md\",\"author\":\"User Name \"}],\"dockerImage\":\"445808685819.dkr.ecr.us-east-2.amazonaws.com/orch:23907713-2\"}"` - ImageScannerEndpoint string `env:"IMAGE_SCANNER_ENDPOINT" envDefault:"http://image-scanner-new-demo-devtroncd-service.devtroncd:80"` - CiDefaultAddressPoolBaseCidr string `env:"CI_DEFAULT_ADDRESS_POOL_BASE_CIDR"` - CiDefaultAddressPoolSize int `env:"CI_DEFAULT_ADDRESS_POOL_SIZE"` - CiRunnerDockerMTUValue int `env:"CI_RUNNER_DOCKER_MTU_VALUE" envDefault:"-1"` + ExternalCiApiSecret string `env:"EXTERNAL_CI_API_SECRET" envDefault:"devtroncd-secret" description:"External CI API secret."` + ExternalCiWebhookUrl string `env:"EXTERNAL_CI_WEB_HOOK_URL" envDefault:"" description:"default is {{HOST_URL}}/orchestrator/webhook/ext-ci. It is used for external ci."` + ExternalCiPayload string `env:"EXTERNAL_CI_PAYLOAD" envDefault:"{\"ciProjectDetails\":[{\"gitRepository\":\"https://github.com/vikram1601/getting-started-nodejs.git\",\"checkoutPath\":\"./abc\",\"commitHash\":\"239077135f8cdeeccb7857e2851348f558cb53d3\",\"commitTime\":\"2022-10-30T20:00:00\",\"branch\":\"master\",\"message\":\"Update README.md\",\"author\":\"User Name \"}],\"dockerImage\":\"445808685819.dkr.ecr.us-east-2.amazonaws.com/orch:23907713-2\"}" description:"External CI payload with project details."` + ImageScannerEndpoint string `env:"IMAGE_SCANNER_ENDPOINT" envDefault:"http://image-scanner-new-demo-devtroncd-service.devtroncd:80" description:"Image-scanner micro-service URL"` + CiDefaultAddressPoolBaseCidr string `env:"CI_DEFAULT_ADDRESS_POOL_BASE_CIDR" description:"To pass the IP cidr for CI"` + CiDefaultAddressPoolSize int `env:"CI_DEFAULT_ADDRESS_POOL_SIZE" description:"The subnet size to allocate from the base pool for CI"` + CiRunnerDockerMTUValue int `env:"CI_RUNNER_DOCKER_MTU_VALUE" envDefault:"-1" description:"this is to control the bytes of inofrmation passed in a network packet in ci-runner. default is -1 (defaults to the underlying node mtu value)"` //Deprecated: use WorkflowCacheConfig instead - IgnoreDockerCacheForCI bool `env:"CI_IGNORE_DOCKER_CACHE"` - WorkflowCacheConfig string `env:"WORKFLOW_CACHE_CONFIG" envDefault:"{}"` - VolumeMountsForCiJson string `env:"CI_VOLUME_MOUNTS_JSON"` - BuildPvcCachePath string `env:"PRE_CI_CACHE_PATH" envDefault:"/devtroncd-cache"` - DefaultPvcCachePath string `env:"DOCKER_BUILD_CACHE_PATH" envDefault:"/var/lib/docker"` - BuildxPvcCachePath string `env:"BUILDX_CACHE_PATH" envDefault:"/var/lib/devtron/buildx"` - UseBlobStorageConfigInCiWorkflow bool `env:"USE_BLOB_STORAGE_CONFIG_IN_CI_WORKFLOW" envDefault:"true"` - DefaultTargetPlatform string `env:"DEFAULT_TARGET_PLATFORM" envDefault:""` - UseBuildx bool `env:"USE_BUILDX" envDefault:"false"` - EnableBuildContext bool `env:"ENABLE_BUILD_CONTEXT" envDefault:"false"` - ImageRetryCount int `env:"IMAGE_RETRY_COUNT" envDefault:"0"` - ImageRetryInterval int `env:"IMAGE_RETRY_INTERVAL" envDefault:"5"` // image retry interval takes value in seconds - CiWorkflowExecutorType cdWorkflow.WorkflowExecutorType `env:"CI_WORKFLOW_EXECUTOR_TYPE" envDefault:"AWF"` - BuildxK8sDriverOptions string `env:"BUILDX_K8S_DRIVER_OPTIONS" envDefault:""` - CIAutoTriggerBatchSize int `env:"CI_SUCCESS_AUTO_TRIGGER_BATCH_SIZE" envDefault:"1"` - SkipCreatingEcrRepo bool `env:"SKIP_CREATING_ECR_REPO" envDefault:"false"` - MaxCiWorkflowRetries int `env:"MAX_CI_WORKFLOW_RETRIES" envDefault:"0"` + IgnoreDockerCacheForCI bool `env:"CI_IGNORE_DOCKER_CACHE" description:"Ignoring docker cache "` + WorkflowCacheConfig string `env:"WORKFLOW_CACHE_CONFIG" envDefault:"{}" description:"flag is used to configure how Docker caches are handled during a CI/CD "` + VolumeMountsForCiJson string `env:"CI_VOLUME_MOUNTS_JSON" description:"additional volume mount data for CI and JOB"` + BuildPvcCachePath string `env:"PRE_CI_CACHE_PATH" envDefault:"/devtroncd-cache" description:"Cache path for Pre CI tasks"` + DefaultPvcCachePath string `env:"DOCKER_BUILD_CACHE_PATH" envDefault:"/var/lib/docker" description:"Path to store cache of docker build (/var/lib/docker-> for legacy docker build, /var/lib/devtron-> for buildx)"` + BuildxPvcCachePath string `env:"BUILDX_CACHE_PATH" envDefault:"/var/lib/devtron/buildx" description:"Path for the buildx cache"` + UseBlobStorageConfigInCiWorkflow bool `env:"USE_BLOB_STORAGE_CONFIG_IN_CI_WORKFLOW" envDefault:"true" description:"To enable blob storage in pre and post ci"` + DefaultTargetPlatform string `env:"DEFAULT_TARGET_PLATFORM" envDefault:"" description:"Default architecture for buildx"` + UseBuildx bool `env:"USE_BUILDX" envDefault:"false" description:"To enable buildx feature globally"` + EnableBuildContext bool `env:"ENABLE_BUILD_CONTEXT" envDefault:"false" description:"To Enable build context in Devtron."` + ImageRetryCount int `env:"IMAGE_RETRY_COUNT" envDefault:"0" description:"push artifact(image) in ci retry count "` + ImageRetryInterval int `env:"IMAGE_RETRY_INTERVAL" envDefault:"5" description:"image retry interval takes value in seconds"` // image retry interval takes value in seconds + CiWorkflowExecutorType cdWorkflow.WorkflowExecutorType `env:"CI_WORKFLOW_EXECUTOR_TYPE" envDefault:"AWF" description:"Executor type for CI(AWF,System)"` + BuildxK8sDriverOptions string `env:"BUILDX_K8S_DRIVER_OPTIONS" envDefault:"" description:"To enable the k8s driver and pass args for k8s driver in buildx"` + CIAutoTriggerBatchSize int `env:"CI_SUCCESS_AUTO_TRIGGER_BATCH_SIZE" envDefault:"1" description:"this is to control the no of linked pipelines should be hanled in one go when a ci-success event of an parent ci is received"` + SkipCreatingEcrRepo bool `env:"SKIP_CREATING_ECR_REPO" envDefault:"false" description:"By disabling this ECR repo won't get created if it's not available on ECR from build configuration"` + MaxCiWorkflowRetries int `env:"MAX_CI_WORKFLOW_RETRIES" envDefault:"0" description:"Maximum time CI-workflow create pod if it fails to complete"` NatsServerHost string `env:"NATS_SERVER_HOST" envDefault:"nats://devtron-nats.devtroncd:4222"` - ImageScanMaxRetries int `env:"IMAGE_SCAN_MAX_RETRIES" envDefault:"3"` - ImageScanRetryDelay int `env:"IMAGE_SCAN_RETRY_DELAY" envDefault:"5"` - ShowDockerBuildCmdInLogs bool `env:"SHOW_DOCKER_BUILD_ARGS" envDefault:"true"` - IgnoreCmCsInCiJob bool `env:"IGNORE_CM_CS_IN_CI_JOB" envDefault:"false"` + ImageScanMaxRetries int `env:"IMAGE_SCAN_MAX_RETRIES" envDefault:"3" description:"Max retry count for image-scanning"` + ImageScanRetryDelay int `env:"IMAGE_SCAN_RETRY_DELAY" envDefault:"5" description:"Delay for the image-scaning to start"` + ShowDockerBuildCmdInLogs bool `env:"SHOW_DOCKER_BUILD_ARGS" envDefault:"true" description:"To enable showing the args passed for CI in build logs"` + IgnoreCmCsInCiJob bool `env:"IGNORE_CM_CS_IN_CI_JOB" envDefault:"false" description:"Ignore CM/CS in CI-pipeline as Job"` //Deprecated: use WorkflowCacheConfig instead - SkipCiJobBuildCachePushPull bool `env:"SKIP_CI_JOB_BUILD_CACHE_PUSH_PULL" envDefault:"false"` + SkipCiJobBuildCachePushPull bool `env:"SKIP_CI_JOB_BUILD_CACHE_PUSH_PULL" envDefault:"false" description:"To skip cache Push/Pull for ci job"` // from CdConfig - CdLimitCpu string `env:"CD_LIMIT_CI_CPU" envDefault:"0.5"` - CdLimitMem string `env:"CD_LIMIT_CI_MEM" envDefault:"3G"` - CdReqCpu string `env:"CD_REQ_CI_CPU" envDefault:"0.5"` - CdReqMem string `env:"CD_REQ_CI_MEM" envDefault:"3G"` - CdTaintKey string `env:"CD_NODE_TAINTS_KEY" envDefault:"dedicated"` + CdLimitCpu string `env:"CD_LIMIT_CI_CPU" envDefault:"0.5" description:"CPU Resource Limit Pre/Post CD"` + CdLimitMem string `env:"CD_LIMIT_CI_MEM" envDefault:"3G" description:"Memory Resource Limit Pre/Post CD"` + CdReqCpu string `env:"CD_REQ_CI_CPU" envDefault:"0.5" description:"CPU Resource Rquest Pre/Post CD"` + CdReqMem string `env:"CD_REQ_CI_MEM" envDefault:"3G" description:"Memory Resource Rquest Pre/Post CD"` + CdTaintKey string `env:"CD_NODE_TAINTS_KEY" envDefault:"dedicated" description:"Toleration key for Pre/Post CD"` ExternalCdTaintKey string `env:"EXTERNAL_CD_NODE_TAINTS_KEY" envDefault:"dedicated"` - UseExternalNode bool `env:"USE_EXTERNAL_NODE" envDefault:"false"` - CdWorkflowServiceAccount string `env:"CD_WORKFLOW_SERVICE_ACCOUNT" envDefault:"cd-runner"` + UseExternalNode bool `env:"USE_EXTERNAL_NODE" envDefault:"false" description:"It is used in case of Pre/ Post Cd with run in application mode. If enabled the node lebels are read from EXTERNAL_CD_NODE_LABEL_SELECTOR else from CD_NODE_LABEL_SELECTOR MODE: if the vale is DEV, it will read the local kube config file or else from the cluser location."` + CdWorkflowServiceAccount string `env:"CD_WORKFLOW_SERVICE_ACCOUNT" envDefault:"cd-runner" description:"Service account to be used in Pre/Post CD pod"` CdDefaultBuildLogsKeyPrefix string `env:"DEFAULT_BUILD_LOGS_KEY_PREFIX" ` - CdDefaultArtifactKeyPrefix string `env:"DEFAULT_CD_ARTIFACT_KEY_LOCATION" ` - CdTaintValue string `env:"CD_NODE_TAINTS_VALUE" envDefault:"ci"` + CdDefaultArtifactKeyPrefix string `env:"DEFAULT_CD_ARTIFACT_KEY_LOCATION" description:"Bucket prefix for build cache"` + CdTaintValue string `env:"CD_NODE_TAINTS_VALUE" envDefault:"ci" description:"Toleration value for Pre/Post CD"` ExternalCdTaintValue string `env:"EXTERNAL_CD_NODE_TAINTS_VALUE" envDefault:"ci"` - CdDefaultBuildLogsBucket string `env:"DEFAULT_BUILD_LOGS_BUCKET" ` - CdNodeLabelSelector []string `env:"CD_NODE_LABEL_SELECTOR"` - ExternalCdNodeLabelSelector []string `env:"EXTERNAL_CD_NODE_LABEL_SELECTOR"` - CdDefaultNamespace string `env:"DEFAULT_CD_NAMESPACE"` + CdDefaultBuildLogsBucket string `env:"DEFAULT_BUILD_LOGS_BUCKET" description:"Bucket name for build logs"` + CdNodeLabelSelector []string `env:"CD_NODE_LABEL_SELECTOR" description:"Node label selector for Pre/Post CD"` + ExternalCdNodeLabelSelector []string `env:"EXTERNAL_CD_NODE_LABEL_SELECTOR" description:"This is an array of strings used when submitting a workflow for pre or post-CD execution. If the "Run in Target Env" option is selected (indicating execution in an external cluster) and the USE_EXTERNAL_NODE flag is set to true, the final node constraints will be determined by the values specified in this environment variable."` + CdDefaultNamespace string `env:"DEFAULT_CD_NAMESPACE" description:"Namespace for devtron stack"` CdDefaultImage string `env:"DEFAULT_CI_IMAGE"` - CdDefaultTimeout int64 `env:"DEFAULT_CD_TIMEOUT" envDefault:"3600"` + CdDefaultTimeout int64 `env:"DEFAULT_CD_TIMEOUT" envDefault:"3600" description:"Timeout for Pre/Post-Cd to be completed"` CdDefaultCdLogsBucketRegion string `env:"DEFAULT_CD_LOGS_BUCKET_REGION" ` - WfControllerInstanceID string `env:"WF_CONTROLLER_INSTANCE_ID" envDefault:"devtron-runner"` - CdDefaultAddressPoolBaseCidr string `env:"CD_DEFAULT_ADDRESS_POOL_BASE_CIDR"` - CdDefaultAddressPoolSize int `env:"CD_DEFAULT_ADDRESS_POOL_SIZE"` - ExposeCDMetrics bool `env:"EXPOSE_CD_METRICS" envDefault:"false"` - UseBlobStorageConfigInCdWorkflow bool `env:"USE_BLOB_STORAGE_CONFIG_IN_CD_WORKFLOW" envDefault:"true"` - CdWorkflowExecutorType cdWorkflow.WorkflowExecutorType `env:"CD_WORKFLOW_EXECUTOR_TYPE" envDefault:"AWF"` - TerminationGracePeriod int `env:"TERMINATION_GRACE_PERIOD_SECS" envDefault:"180"` - MaxCdWorkflowRunnerRetries int `env:"MAX_CD_WORKFLOW_RUNNER_RETRIES" envDefault:"0"` + WfControllerInstanceID string `env:"WF_CONTROLLER_INSTANCE_ID" envDefault:"devtron-runner" description:"Workflow controller instance ID."` + CdDefaultAddressPoolBaseCidr string `env:"CD_DEFAULT_ADDRESS_POOL_BASE_CIDR" description:"To pass the IP cidr for Pre/Post cd "` + CdDefaultAddressPoolSize int `env:"CD_DEFAULT_ADDRESS_POOL_SIZE" description:"The subnet size to allocate from the base pool for CD"` + ExposeCDMetrics bool `env:"EXPOSE_CD_METRICS" envDefault:"false" description:"To expose CD metrics"` + UseBlobStorageConfigInCdWorkflow bool `env:"USE_BLOB_STORAGE_CONFIG_IN_CD_WORKFLOW" envDefault:"true" description:"To enable blob storage in pre and post cd"` + CdWorkflowExecutorType cdWorkflow.WorkflowExecutorType `env:"CD_WORKFLOW_EXECUTOR_TYPE" envDefault:"AWF" description:"Executor type for Pre/Post CD(AWF,System)"` + TerminationGracePeriod int `env:"TERMINATION_GRACE_PERIOD_SECS" envDefault:"180" description:"this is the time given to workflow pods to shutdown. (grace full termination time)"` + MaxCdWorkflowRunnerRetries int `env:"MAX_CD_WORKFLOW_RUNNER_RETRIES" envDefault:"0" description:"Maximum time pre/post-cd-workflow create pod if it fails to complete"` // common in both ciconfig and cd config Type string Mode string `env:"MODE" envDefault:"DEV"` - OrchestratorHost string `env:"ORCH_HOST" envDefault:"http://devtroncd-orchestrator-service-prod.devtroncd/webhook/msg/nats"` - OrchestratorToken string `env:"ORCH_TOKEN" envDefault:""` + OrchestratorHost string `env:"ORCH_HOST" envDefault:"http://devtroncd-orchestrator-service-prod.devtroncd/webhook/msg/nats" description:"Orchestrator micro-service URL "` + OrchestratorToken string `env:"ORCH_TOKEN" envDefault:"" description:"Orchestrator token"` ClusterConfig *rest.Config - CloudProvider blob_storage.BlobStorageType `env:"BLOB_STORAGE_PROVIDER" envDefault:"S3"` - BlobStorageEnabled bool `env:"BLOB_STORAGE_ENABLED" envDefault:"false"` - BlobStorageS3AccessKey string `env:"BLOB_STORAGE_S3_ACCESS_KEY"` - BlobStorageS3SecretKey string `env:"BLOB_STORAGE_S3_SECRET_KEY"` - BlobStorageS3Endpoint string `env:"BLOB_STORAGE_S3_ENDPOINT"` - BlobStorageS3EndpointInsecure bool `env:"BLOB_STORAGE_S3_ENDPOINT_INSECURE" envDefault:"false"` - BlobStorageS3BucketVersioned bool `env:"BLOB_STORAGE_S3_BUCKET_VERSIONED" envDefault:"true"` - BlobStorageGcpCredentialJson string `env:"BLOB_STORAGE_GCP_CREDENTIALS_JSON"` - AzureAccountName string `env:"AZURE_ACCOUNT_NAME"` - AzureGatewayUrl string `env:"AZURE_GATEWAY_URL" envDefault:"http://devtron-minio.devtroncd:9000"` - AzureGatewayConnectionInsecure bool `env:"AZURE_GATEWAY_CONNECTION_INSECURE" envDefault:"true"` - AzureBlobContainerCiLog string `env:"AZURE_BLOB_CONTAINER_CI_LOG"` - AzureBlobContainerCiCache string `env:"AZURE_BLOB_CONTAINER_CI_CACHE"` - AzureAccountKey string `env:"AZURE_ACCOUNT_KEY"` - BuildLogTTLValue int `env:"BUILD_LOG_TTL_VALUE_IN_SECS" envDefault:"3600"` - BaseLogLocationPath string `env:"BASE_LOG_LOCATION_PATH" envDefault:"/home/devtron/"` - InAppLoggingEnabled bool `env:"IN_APP_LOGGING_ENABLED" envDefault:"false"` - BuildxProvenanceMode string `env:"BUILDX_PROVENANCE_MODE" envDefault:""` // provenance is set to false if this flag is not set - ExtBlobStorageCmName string `env:"EXTERNAL_BLOB_STORAGE_CM_NAME" envDefault:"blob-storage-cm"` - ExtBlobStorageSecretName string `env:"EXTERNAL_BLOB_STORAGE_SECRET_NAME" envDefault:"blob-storage-secret"` - UseArtifactListingQueryV2 bool `env:"USE_ARTIFACT_LISTING_QUERY_V2" envDefault:"true"` - UseImageTagFromGitProviderForTagBasedBuild bool `env:"USE_IMAGE_TAG_FROM_GIT_PROVIDER_FOR_TAG_BASED_BUILD" envDefault:"false"` // this is being done for https://github.com/devtron-labs/devtron/issues/4263 - UseDockerApiToGetDigest bool `env:"USE_DOCKER_API_TO_GET_DIGEST" envDefault:"false"` + CloudProvider blob_storage.BlobStorageType `env:"BLOB_STORAGE_PROVIDER" envDefault:"S3" description:"Blob storage provider name(AWS/GCP/Azure)"` + BlobStorageEnabled bool `env:"BLOB_STORAGE_ENABLED" envDefault:"false" description:"To enable blob storage"` + BlobStorageS3AccessKey string `env:"BLOB_STORAGE_S3_ACCESS_KEY" description:"S3 access key for s3 blob storage"` + BlobStorageS3SecretKey string `env:"BLOB_STORAGE_S3_SECRET_KEY" description:"Secret key for s3 blob storage"` + BlobStorageS3Endpoint string `env:"BLOB_STORAGE_S3_ENDPOINT" description:"S3 endpoint URL for s3 blob storage"` + BlobStorageS3EndpointInsecure bool `env:"BLOB_STORAGE_S3_ENDPOINT_INSECURE" envDefault:"false" description:"To use insecure s3 endpoint"` + BlobStorageS3BucketVersioned bool `env:"BLOB_STORAGE_S3_BUCKET_VERSIONED" envDefault:"true" description:"To enable buctet versioning for blob storage"` + BlobStorageGcpCredentialJson string `env:"BLOB_STORAGE_GCP_CREDENTIALS_JSON" description:"GCP cred json for GCS blob storage"` + AzureAccountName string `env:"AZURE_ACCOUNT_NAME" description:"Account name for azure blob storage"` + AzureGatewayUrl string `env:"AZURE_GATEWAY_URL" envDefault:"http://devtron-minio.devtroncd:9000" description:"Sent to CI runner for blob"` + AzureGatewayConnectionInsecure bool `env:"AZURE_GATEWAY_CONNECTION_INSECURE" envDefault:"true" description:"Azure gateway connection allows insecure if true"` + AzureBlobContainerCiLog string `env:"AZURE_BLOB_CONTAINER_CI_LOG" description:"Log bucket for azure blob storage"` + AzureBlobContainerCiCache string `env:"AZURE_BLOB_CONTAINER_CI_CACHE" description:"Cache bucket name for azure blob storage"` + AzureAccountKey string `env:"AZURE_ACCOUNT_KEY" description:"If blob storage is bieng used of azure then pass the secret key to access the bucket"` + BuildLogTTLValue int `env:"BUILD_LOG_TTL_VALUE_IN_SECS" envDefault:"3600" description:"This is the time that the pods of ci/pre-cd/post-cd live after completion state."` + BaseLogLocationPath string `env:"BASE_LOG_LOCATION_PATH" envDefault:"/home/devtron/" description:"Used to store, download logs of ci workflow, artifact"` + InAppLoggingEnabled bool `env:"IN_APP_LOGGING_ENABLED" envDefault:"false" description:"Used in case of argo workflow is enabled. If enabled logs push will be managed by us, else will be managed by argo workflow."` + BuildxProvenanceMode string `env:"BUILDX_PROVENANCE_MODE" envDefault:"" description:"provinance is set to true by default by docker. this will add some build related data in generated build manifest.it also adds some unknown:unknown key:value pair which may not be compatible by some container registries. with buildx k8s driver , provinenance=true is causing issue when push manifest to quay registry, so setting it to false"` // provenance is set to false if this flag is not set + ExtBlobStorageCmName string `env:"EXTERNAL_BLOB_STORAGE_CM_NAME" envDefault:"blob-storage-cm" description:"name of the config map(contains bucket name, etc.) in external cluster when there is some operation related to external cluster, for example:-downloading cd artifact pushed in external cluster's env and we need to download from there, downloads ci logs pushed in external cluster's blob"` + ExtBlobStorageSecretName string `env:"EXTERNAL_BLOB_STORAGE_SECRET_NAME" envDefault:"blob-storage-secret" description:"name of the secret(contains password, accessId,passKeys, etc.) in external cluster when there is some operation related to external cluster, for example:-downloading cd artifact pushed in external cluster's env and we need to download from there, downloads ci logs pushed in external cluster's blob"` + UseArtifactListingQueryV2 bool `env:"USE_ARTIFACT_LISTING_QUERY_V2" envDefault:"true" description:"To use the V2 query for listing artifacts"` + UseImageTagFromGitProviderForTagBasedBuild bool `env:"USE_IMAGE_TAG_FROM_GIT_PROVIDER_FOR_TAG_BASED_BUILD" envDefault:"false" description:"To use the same tag in container image as that of git tag"` // this is being done for https://github.com/devtron-labs/devtron/issues/4263 + UseDockerApiToGetDigest bool `env:"USE_DOCKER_API_TO_GET_DIGEST" envDefault:"false" description:"when user do not pass the digest then this flag controls , finding the image digest using docker API or not. if set to true we get the digest from docker API call else use docker pull command. [logic in ci-runner]"` EnableWorkflowExecutionStage bool `env:"ENABLE_WORKFLOW_EXECUTION_STAGE" envDefault:"true" description:"if enabled then we will display build stages separately for CI/Job/Pre-Post CD" example:"true"` } From 2665345fd31c1c58ebe911940b4a650be176b161 Mon Sep 17 00:00:00 2001 From: Neha130 Date: Thu, 10 Apr 2025 16:59:36 +0530 Subject: [PATCH 20/34] update the config file --- api/bean/AutocompleteConfig.go | 2 +- api/helm-app/gRPC/applicationClient.go | 2 +- api/helm-app/service/HelmAppService.go | 6 ++-- .../configure/PipelineConfigRestHandler.go | 2 +- .../argocdServer/ArgoClientWrapperService.go | 6 ++-- client/argocdServer/bean/connection.go | 4 +-- client/cron/CiStatusUpdateCron.go | 4 +-- client/cron/CiTriggerCron.go | 4 +-- client/dashboard/Config.go | 6 ++-- client/events/EventClient.go | 4 +-- client/gitSensor/GitSensorClient.go | 6 ++-- client/grafana/Config.go | 6 ++-- client/grafana/GrafanaClient.go | 8 ++--- client/lens/LensClient.go | 4 +-- client/proxy/ProxyRouter.go | 2 +- internal/sql/models/UserTerminalSession.go | 8 ++--- internal/util/BasicProviders.go | 2 +- otel/OtelTracingService.go | 2 +- pkg/app/AppCrudOperationService.go | 2 +- pkg/app/AppService.go | 18 +++++----- pkg/appStatus/AppStatusService_test.go | 14 ++++---- pkg/auth/authorisation/casbin/rbac.go | 8 ++--- pkg/auth/user/UserAuthService.go | 18 +++++----- pkg/auth/user/UserCommonService.go | 2 +- .../artifacts/imageTagging/read/Config.go | 2 +- .../git/commandManager/GitCommandManager.go | 4 +-- pkg/generateManifest/bean.go | 4 +-- pkg/infraConfig/bean/v1/infra_config.go | 2 +- pkg/k8s/K8sCommonService.go | 4 +-- pkg/k8s/application/k8sApplicationService.go | 2 +- pkg/module/bean/bean.go | 2 +- pkg/pipeline/CiService.go | 4 +-- pkg/pipeline/PipelineBuilder.go | 2 +- pkg/server/config/ServerEnvConfig.go | 34 +++++++++---------- pkg/sql/connection.go | 6 ++-- pkg/terminal/terminalSesion.go | 4 +-- pkg/util/TokenCache.go | 14 ++++---- pkg/variables/ScopedVariableService.go | 8 ++--- .../parsers/VariableTemplateParser.go | 6 ++-- util/GlobalConfig.go | 22 ++++++------ .../authenticator/client/k8sClient.go | 4 +-- .../authenticator/client/oidcClient.go | 10 +++--- .../common-lib/pubsub-lib/JetStreamUtil.go | 4 +-- .../common-lib/pubsub-lib/NatsClient.go | 12 +++---- .../common-lib/utils/bean/bean.go | 6 ++-- .../common-lib/utils/grpc/GrpcConfig.go | 4 +-- .../devtron-labs/common-lib/utils/k8s/bean.go | 12 +++---- 47 files changed, 156 insertions(+), 156 deletions(-) diff --git a/api/bean/AutocompleteConfig.go b/api/bean/AutocompleteConfig.go index 62d9191c0f..48e7c61337 100644 --- a/api/bean/AutocompleteConfig.go +++ b/api/bean/AutocompleteConfig.go @@ -17,5 +17,5 @@ package bean type Config struct { - IgnoreAuthCheck bool `env:"IGNORE_AUTOCOMPLETE_AUTH_CHECK" envDefault:"false" description: "flag for ignoring auth check in autocomplete apis."` + IgnoreAuthCheck bool `env:"IGNORE_AUTOCOMPLETE_AUTH_CHECK" envDefault:"false" description:"flag for ignoring auth check in autocomplete apis."` } diff --git a/api/helm-app/gRPC/applicationClient.go b/api/helm-app/gRPC/applicationClient.go index cabe07bd6b..5f79ebae40 100644 --- a/api/helm-app/gRPC/applicationClient.go +++ b/api/helm-app/gRPC/applicationClient.go @@ -75,7 +75,7 @@ func NewHelmAppClientImpl(logger *zap.SugaredLogger, // CATEGORY=INFRA_SETUP type HelmClientConfig struct { - Url string `env:"HELM_CLIENT_URL" envDefault:"127.0.0.1:50051" description: "Kubelink micro-service url "` + Url string `env:"HELM_CLIENT_URL" envDefault:"127.0.0.1:50051" description:"Kubelink micro-service url "` } func GetConfig() (*HelmClientConfig, error) { diff --git a/api/helm-app/service/HelmAppService.go b/api/helm-app/service/HelmAppService.go index 72e13fbcef..3e997ed451 100644 --- a/api/helm-app/service/HelmAppService.go +++ b/api/helm-app/service/HelmAppService.go @@ -159,9 +159,9 @@ func NewHelmAppServiceImpl(Logger *zap.SugaredLogger, clusterService cluster.Clu // CATEGORY=CD type HelmReleaseConfig struct { - RevisionHistoryLimitDevtronApp int `env:"REVISION_HISTORY_LIMIT_DEVTRON_APP" envDefault:"1" description: "Count for devtron application rivision history"` - RevisionHistoryLimitHelmApp int `env:"REVISION_HISTORY_LIMIT_HELM_APP" envDefault:"1" description: "To set the history limit for the helm app being deployed through devtron"` - RevisionHistoryLimitExternalHelmApp int `env:"REVISION_HISTORY_LIMIT_EXTERNAL_HELM_APP" envDefault:"0" description: "Count for external helm application rivision history"` + RevisionHistoryLimitDevtronApp int `env:"REVISION_HISTORY_LIMIT_DEVTRON_APP" envDefault:"1" description:"Count for devtron application rivision history"` + RevisionHistoryLimitHelmApp int `env:"REVISION_HISTORY_LIMIT_HELM_APP" envDefault:"1" description:"To set the history limit for the helm app being deployed through devtron"` + RevisionHistoryLimitExternalHelmApp int `env:"REVISION_HISTORY_LIMIT_EXTERNAL_HELM_APP" envDefault:"0" description:"Count for external helm application rivision history"` RevisionHistoryLimitLinkedHelmApp int `env:"REVISION_HISTORY_LIMIT_LINKED_HELM_APP" envDefault:"15"` } diff --git a/api/restHandler/app/pipeline/configure/PipelineConfigRestHandler.go b/api/restHandler/app/pipeline/configure/PipelineConfigRestHandler.go index 5711a35b4c..3db7029d0d 100644 --- a/api/restHandler/app/pipeline/configure/PipelineConfigRestHandler.go +++ b/api/restHandler/app/pipeline/configure/PipelineConfigRestHandler.go @@ -68,7 +68,7 @@ import ( ) type PipelineRestHandlerEnvConfig struct { - UseArtifactListApiV2 bool `env:"USE_ARTIFACT_LISTING_API_V2" envDefault:"true" description: "To use the V2 API for listing artifacts in Listing the images in pipeline"` //deprecated + UseArtifactListApiV2 bool `env:"USE_ARTIFACT_LISTING_API_V2" envDefault:"true" description:"To use the V2 API for listing artifacts in Listing the images in pipeline"` //deprecated } type DevtronAppRestHandler interface { diff --git a/client/argocdServer/ArgoClientWrapperService.go b/client/argocdServer/ArgoClientWrapperService.go index 4d19894aaf..9b501aef1c 100644 --- a/client/argocdServer/ArgoClientWrapperService.go +++ b/client/argocdServer/ArgoClientWrapperService.go @@ -51,9 +51,9 @@ import ( ) type ACDConfig struct { - ArgoCDAutoSyncEnabled bool `env:"ARGO_AUTO_SYNC_ENABLED" envDefault:"true" description: "If enabled all argocd application will have auto sync enabled"` // will gradually switch this flag to false in enterprise - RegisterRepoMaxRetryCount int `env:"ARGO_REPO_REGISTER_RETRY_COUNT" envDefault:"3" description: "Argo app registration in argo retries on deployment"` - RegisterRepoMaxRetryDelay int `env:"ARGO_REPO_REGISTER_RETRY_DELAY" envDefault:"10" description: "Argo app registration in argo cd on deployment delay between retry"` + ArgoCDAutoSyncEnabled bool `env:"ARGO_AUTO_SYNC_ENABLED" envDefault:"true" description:"If enabled all argocd application will have auto sync enabled"` // will gradually switch this flag to false in enterprise + RegisterRepoMaxRetryCount int `env:"ARGO_REPO_REGISTER_RETRY_COUNT" envDefault:"3" description:"Argo app registration in argo retries on deployment"` + RegisterRepoMaxRetryDelay int `env:"ARGO_REPO_REGISTER_RETRY_DELAY" envDefault:"10" description:"Argo app registration in argo cd on deployment delay between retry"` } func (config *ACDConfig) IsManualSyncEnabled() bool { diff --git a/client/argocdServer/bean/connection.go b/client/argocdServer/bean/connection.go index 7aaeb67f92..7b13819916 100644 --- a/client/argocdServer/bean/connection.go +++ b/client/argocdServer/bean/connection.go @@ -25,8 +25,8 @@ type AcdAuthConfig struct { } type Config struct { - Host string `env:"CD_HOST" envDefault:"localhost" description: "Host for the devtron stack"` - Port string `env:"CD_PORT" envDefault:"8000" description: "Port for pre/post-cd" ` + Host string `env:"CD_HOST" envDefault:"localhost" description:"Host for the devtron stack"` + Port string `env:"CD_PORT" envDefault:"8000" description:"Port for pre/post-cd" ` Namespace string `env:"CD_NAMESPACE" envDefault:"devtroncd"` } diff --git a/client/cron/CiStatusUpdateCron.go b/client/cron/CiStatusUpdateCron.go index 9ab20d1432..a62d4bc03f 100644 --- a/client/cron/CiStatusUpdateCron.go +++ b/client/cron/CiStatusUpdateCron.go @@ -66,8 +66,8 @@ func NewCiStatusUpdateCronImpl(logger *zap.SugaredLogger, appService app.AppServ } type CiWorkflowStatusUpdateConfig struct { - CiWorkflowStatusUpdateCron string `env:"CI_WORKFLOW_STATUS_UPDATE_CRON" envDefault:"*/5 * * * *" description: "Cron schedule for CI pipeline status"` - TimeoutForFailedCiBuild string `env:"TIMEOUT_FOR_FAILED_CI_BUILD" envDefault:"15" description: "Timeout for Failed CI build "` //in minutes + CiWorkflowStatusUpdateCron string `env:"CI_WORKFLOW_STATUS_UPDATE_CRON" envDefault:"*/5 * * * *" description:"Cron schedule for CI pipeline status"` + TimeoutForFailedCiBuild string `env:"TIMEOUT_FOR_FAILED_CI_BUILD" envDefault:"15" description:"Timeout for Failed CI build "` //in minutes } func GetCiWorkflowStatusUpdateConfig() (*CiWorkflowStatusUpdateConfig, error) { diff --git a/client/cron/CiTriggerCron.go b/client/cron/CiTriggerCron.go index fd958236c3..ad4ef91c99 100644 --- a/client/cron/CiTriggerCron.go +++ b/client/cron/CiTriggerCron.go @@ -68,8 +68,8 @@ func NewCiTriggerCronImpl(logger *zap.SugaredLogger, cfg *CiTriggerCronConfig, p } type CiTriggerCronConfig struct { - SourceControllerCronTime int `env:"CI_TRIGGER_CRON_TIME" envDefault:"2" description: "For image poll plugin"` - PluginName string `env:"PLUGIN_NAME" envDefault:"Pull images from container repository" description: "Handles image retrieval from a container repository and triggers subsequent CI processes upon detecting new images.Current default plugin name: Pull Images from Container Repository."` + SourceControllerCronTime int `env:"CI_TRIGGER_CRON_TIME" envDefault:"2" description:"For image poll plugin"` + PluginName string `env:"PLUGIN_NAME" envDefault:"Pull images from container repository" description:"Handles image retrieval from a container repository and triggers subsequent CI processes upon detecting new images.Current default plugin name: Pull Images from Container Repository."` } func GetCiTriggerCronConfig() (*CiTriggerCronConfig, error) { diff --git a/client/dashboard/Config.go b/client/dashboard/Config.go index 2c71b39982..1bc321d97a 100644 --- a/client/dashboard/Config.go +++ b/client/dashboard/Config.go @@ -22,9 +22,9 @@ import ( // CATEGORY=INFRA_SETUP type Config struct { - Host string `env:"DASHBOARD_HOST" envDefault:"localhost" description: "Dashboard micro-service URL"` - Port string `env:"DASHBOARD_PORT" envDefault:"3000" description: "Port for dashboard micro-service"` - Namespace string `env:"DASHBOARD_NAMESPACE" envDefault:"devtroncd" description: "Dashboard micro-service namespace"` + Host string `env:"DASHBOARD_HOST" envDefault:"localhost" description:"Dashboard micro-service URL"` + Port string `env:"DASHBOARD_PORT" envDefault:"3000" description:"Port for dashboard micro-service"` + Namespace string `env:"DASHBOARD_NAMESPACE" envDefault:"devtroncd" description:"Dashboard micro-service namespace"` } func GetConfig() (*Config, error) { diff --git a/client/events/EventClient.go b/client/events/EventClient.go index 2b472c1d77..5d7590f9b1 100644 --- a/client/events/EventClient.go +++ b/client/events/EventClient.go @@ -38,8 +38,8 @@ import ( ) type EventClientConfig struct { - DestinationURL string `env:"EVENT_URL" envDefault:"http://localhost:3000/notify" description: "Notifier service url"` - NotificationMedium NotificationMedium `env:"NOTIFICATION_MEDIUM" envDefault:"rest" description: "notification medium"` + DestinationURL string `env:"EVENT_URL" envDefault:"http://localhost:3000/notify" description:"Notifier service url"` + NotificationMedium NotificationMedium `env:"NOTIFICATION_MEDIUM" envDefault:"rest" description:"notification medium"` } type NotificationMedium string diff --git a/client/gitSensor/GitSensorClient.go b/client/gitSensor/GitSensorClient.go index f12b029f29..e36154beea 100644 --- a/client/gitSensor/GitSensorClient.go +++ b/client/gitSensor/GitSensorClient.go @@ -81,9 +81,9 @@ func NewGitSensorClient(logger *zap.SugaredLogger, config *ClientConfig) (*Clien // CATEGORY=INFRA_SETUP type ClientConfig struct { - Url string `env:"GIT_SENSOR_URL" envDefault:"127.0.0.1:7070" description: "git-sensor micro-service url "` - Protocol string `env:"GIT_SENSOR_PROTOCOL" envDefault:"REST" description: "Protocol to connect with git-sensor micro-service"` - Timeout int `env:"GIT_SENSOR_TIMEOUT" envDefault:"0" description: "Timeout for getting response from the git-sensor"` // in seconds + Url string `env:"GIT_SENSOR_URL" envDefault:"127.0.0.1:7070" description:"git-sensor micro-service url "` + Protocol string `env:"GIT_SENSOR_PROTOCOL" envDefault:"REST" description:"Protocol to connect with git-sensor micro-service"` + Timeout int `env:"GIT_SENSOR_TIMEOUT" envDefault:"0" description:"Timeout for getting response from the git-sensor"` // in seconds } func GetConfig() (*ClientConfig, error) { diff --git a/client/grafana/Config.go b/client/grafana/Config.go index 37dee8ef5d..01be9e1b3c 100644 --- a/client/grafana/Config.go +++ b/client/grafana/Config.go @@ -19,9 +19,9 @@ package grafana import "github.com/caarlos0/env" type Config struct { - Host string `env:"GRAFANA_HOST" envDefault:"localhost" description: "Host URL for the grafana dashboard"` - Port string `env:"GRAFANA_PORT" envDefault:"8090" description: "Port for grafana micro-service"` - Namespace string `env:"GRAFANA_NAMESPACE" envDefault:"devtroncd" description: "Namespace for grafana"` + Host string `env:"GRAFANA_HOST" envDefault:"localhost" description:"Host URL for the grafana dashboard"` + Port string `env:"GRAFANA_PORT" envDefault:"8090" description:"Port for grafana micro-service"` + Namespace string `env:"GRAFANA_NAMESPACE" envDefault:"devtroncd" description:"Namespace for grafana"` } func GetConfig() (*Config, error) { diff --git a/client/grafana/GrafanaClient.go b/client/grafana/GrafanaClient.go index bb2ff7e004..0503644adc 100644 --- a/client/grafana/GrafanaClient.go +++ b/client/grafana/GrafanaClient.go @@ -34,10 +34,10 @@ import ( ) type GrafanaClientConfig struct { - GrafanaUsername string `env:"GRAFANA_USERNAME" envDefault:"admin" description: "Username for grafana "` - GrafanaPassword string `env:"GRAFANA_PASSWORD" envDefault:"prom-operator" description: "Password for grafana dashboard"` - GrafanaOrgId int `env:"GRAFANA_ORG_ID" envDefault:"2" description: "Org ID for grafana for application metrics"` - DestinationURL string `env:"GRAFANA_URL" envDefault:"" description: "Host URL for the grafana dashboard"` + GrafanaUsername string `env:"GRAFANA_USERNAME" envDefault:"admin" description:"Username for grafana "` + GrafanaPassword string `env:"GRAFANA_PASSWORD" envDefault:"prom-operator" description:"Password for grafana dashboard"` + GrafanaOrgId int `env:"GRAFANA_ORG_ID" envDefault:"2" description:"Org ID for grafana for application metrics"` + DestinationURL string `env:"GRAFANA_URL" envDefault:"" description:"Host URL for the grafana dashboard"` } const PromDatasource = "/api/datasources" diff --git a/client/lens/LensClient.go b/client/lens/LensClient.go index c5c64ebd27..784c14598d 100644 --- a/client/lens/LensClient.go +++ b/client/lens/LensClient.go @@ -30,8 +30,8 @@ import ( ) type LensConfig struct { - Url string `env:"LENS_URL" envDefault:"http://lens-milandevtron-service:80" description: "Lens micro-service URL"` - Timeout int `env:"LENS_TIMEOUT" envDefault:"0" description: "Lens microservice timeout."` // in seconds + Url string `env:"LENS_URL" envDefault:"http://lens-milandevtron-service:80" description:"Lens micro-service URL"` + Timeout int `env:"LENS_TIMEOUT" envDefault:"0" description:"Lens microservice timeout."` // in seconds } type StatusCode int diff --git a/client/proxy/ProxyRouter.go b/client/proxy/ProxyRouter.go index 63ce696788..154710dc9a 100644 --- a/client/proxy/ProxyRouter.go +++ b/client/proxy/ProxyRouter.go @@ -38,7 +38,7 @@ type ProxyConnection struct { } type Config struct { - ProxyServiceConfig string `env:"PROXY_SERVICE_CONFIG" envDefault:"{}" description: "Proxy configuration for micro-service to be accessible on orhcestrator ingress"` + ProxyServiceConfig string `env:"PROXY_SERVICE_CONFIG" envDefault:"{}" description:"Proxy configuration for micro-service to be accessible on orhcestrator ingress"` } func GetProxyConfig() (*Config, error) { diff --git a/internal/sql/models/UserTerminalSession.go b/internal/sql/models/UserTerminalSession.go index 4db1013ea7..47e0c4bafb 100644 --- a/internal/sql/models/UserTerminalSession.go +++ b/internal/sql/models/UserTerminalSession.go @@ -55,10 +55,10 @@ type UserTerminalPodEvents struct { } type UserTerminalSessionConfig struct { - MaxSessionPerUser int `env:"MAX_SESSION_PER_USER" envDefault:"5" description: "max no of cluster terminal pods can be created by an user"` - TerminalPodStatusSyncTimeInSecs int `env:"TERMINAL_POD_STATUS_SYNC_In_SECS" envDefault:"600" description: "this is the time interval at which the status of the cluster terminal pod"` - TerminalPodDefaultNamespace string `env:"TERMINAL_POD_DEFAULT_NAMESPACE" envDefault:"default" description: "Cluster terminal default namespace"` - TerminalPodInActiveDurationInMins int `env:"TERMINAL_POD_INACTIVE_DURATION_IN_MINS" envDefault:"10" description: "Timeout for cluster terminal to be inactive"` + MaxSessionPerUser int `env:"MAX_SESSION_PER_USER" envDefault:"5" description:"max no of cluster terminal pods can be created by an user"` + TerminalPodStatusSyncTimeInSecs int `env:"TERMINAL_POD_STATUS_SYNC_In_SECS" envDefault:"600" description:"this is the time interval at which the status of the cluster terminal pod"` + TerminalPodDefaultNamespace string `env:"TERMINAL_POD_DEFAULT_NAMESPACE" envDefault:"default" description:"Cluster terminal default namespace"` + TerminalPodInActiveDurationInMins int `env:"TERMINAL_POD_INACTIVE_DURATION_IN_MINS" envDefault:"10" description:"Timeout for cluster terminal to be inactive"` } type UserTerminalSessionResponse struct { diff --git a/internal/util/BasicProviders.go b/internal/util/BasicProviders.go index 65e1b19c80..7f30ede6b5 100644 --- a/internal/util/BasicProviders.go +++ b/internal/util/BasicProviders.go @@ -39,7 +39,7 @@ func GetLogger() *zap.SugaredLogger { type LogConfig struct { Level int `env:"LOG_LEVEL" envDefault:"0"` // default info - DevMode bool `env:"LOGGER_DEV_MODE" envDefault:"false" description: "Enables a different logger theme."` + DevMode bool `env:"LOGGER_DEV_MODE" envDefault:"false" description:"Enables a different logger theme."` } func InitLogger() (*zap.SugaredLogger, error) { diff --git a/otel/OtelTracingService.go b/otel/OtelTracingService.go index 9457258462..1b296834c9 100644 --- a/otel/OtelTracingService.go +++ b/otel/OtelTracingService.go @@ -48,7 +48,7 @@ func NewOtelTracingServiceImpl(logger *zap.SugaredLogger) *OtelTracingServiceImp } type OtelConfig struct { - OtelCollectorUrl string `env:"OTEL_COLLECTOR_URL" envDefault:"" description: "Opentelemetry URL "` + OtelCollectorUrl string `env:"OTEL_COLLECTOR_URL" envDefault:"" description:"Opentelemetry URL "` } // Init configures an OpenTelemetry exporter and trace provider diff --git a/pkg/app/AppCrudOperationService.go b/pkg/app/AppCrudOperationService.go index e03b123c00..945e959ebf 100644 --- a/pkg/app/AppCrudOperationService.go +++ b/pkg/app/AppCrudOperationService.go @@ -53,7 +53,7 @@ const ( ) type CrudOperationServiceConfig struct { - PropagateExtraLabels bool `env:"PROPAGATE_EXTRA_LABELS" envDefault:"false" description: "Add additional propagate labels like api.devtron.ai/appName, api.devtron.ai/envName, api.devtron.ai/project along with the user defined ones."` + PropagateExtraLabels bool `env:"PROPAGATE_EXTRA_LABELS" envDefault:"false" description:"Add additional propagate labels like api.devtron.ai/appName, api.devtron.ai/envName, api.devtron.ai/project along with the user defined ones."` } type AppCrudOperationService interface { diff --git a/pkg/app/AppService.go b/pkg/app/AppService.go index 0d29403b1d..41c92e3424 100644 --- a/pkg/app/AppService.go +++ b/pkg/app/AppService.go @@ -72,16 +72,16 @@ import ( // CATEGORY=CD type AppServiceConfig struct { - CdPipelineStatusCronTime string `env:"CD_PIPELINE_STATUS_CRON_TIME" envDefault:"*/2 * * * *" description: "Cron time for CD pipeline status"` - CdHelmPipelineStatusCronTime string `env:"CD_HELM_PIPELINE_STATUS_CRON_TIME" envDefault:"*/2 * * * *" description: "Cron time to check the pipeline status "` - CdPipelineStatusTimeoutDuration string `env:"CD_PIPELINE_STATUS_TIMEOUT_DURATION" envDefault:"20" description: "Timeout for CD pipeline to get healthy" ` // in minutes - PipelineDegradedTime string `env:"PIPELINE_DEGRADED_TIME" envDefault:"10" description: "Time to mark a pipeline degraded if not healthy in defined time"` // in minutes - GetPipelineDeployedWithinHours int `env:"DEPLOY_STATUS_CRON_GET_PIPELINE_DEPLOYED_WITHIN_HOURS" envDefault:"12" description: "This flag is used to fetch the deployment status of the application. It retrieves the status of deployments that occurred between 12 hours and 10 minutes prior to the current time. It fetches non-terminal statuses."` // in hours - HelmPipelineStatusCheckEligibleTime string `env:"HELM_PIPELINE_STATUS_CHECK_ELIGIBLE_TIME" envDefault:"120" description: "eligible time for checking helm app status periodically and update in db, value is in seconds., default is 120, if wfr is updated within configured time i.e. HELM_PIPELINE_STATUS_CHECK_ELIGIBLE_TIME then do not include for this cron cycle."` // in seconds + CdPipelineStatusCronTime string `env:"CD_PIPELINE_STATUS_CRON_TIME" envDefault:"*/2 * * * *" description:"Cron time for CD pipeline status"` + CdHelmPipelineStatusCronTime string `env:"CD_HELM_PIPELINE_STATUS_CRON_TIME" envDefault:"*/2 * * * *" description:"Cron time to check the pipeline status "` + CdPipelineStatusTimeoutDuration string `env:"CD_PIPELINE_STATUS_TIMEOUT_DURATION" envDefault:"20" description:"Timeout for CD pipeline to get healthy" ` // in minutes + PipelineDegradedTime string `env:"PIPELINE_DEGRADED_TIME" envDefault:"10" description:"Time to mark a pipeline degraded if not healthy in defined time"` // in minutes + GetPipelineDeployedWithinHours int `env:"DEPLOY_STATUS_CRON_GET_PIPELINE_DEPLOYED_WITHIN_HOURS" envDefault:"12" description:"This flag is used to fetch the deployment status of the application. It retrieves the status of deployments that occurred between 12 hours and 10 minutes prior to the current time. It fetches non-terminal statuses."` // in hours + HelmPipelineStatusCheckEligibleTime string `env:"HELM_PIPELINE_STATUS_CHECK_ELIGIBLE_TIME" envDefault:"120" description:"eligible time for checking helm app status periodically and update in db, value is in seconds., default is 120, if wfr is updated within configured time i.e. HELM_PIPELINE_STATUS_CHECK_ELIGIBLE_TIME then do not include for this cron cycle."` // in seconds ExposeCDMetrics bool `env:"EXPOSE_CD_METRICS" envDefault:"false"` - DevtronChartHelmInstallRequestTimeout int `env:"DEVTRON_CHART_INSTALL_REQUEST_TIMEOUT" envDefault:"6" description: "Context timeout for no gitops concurrent async deployments"` // in minutes - DevtronChartArgoCdInstallRequestTimeout int `env:"DEVTRON_CHART_ARGO_CD_INSTALL_REQUEST_TIMEOUT" envDefault:"1" description: "Context timeout for gitops concurrent async deployments"` // in minutes - ArgoCdManualSyncCronPipelineDeployedBefore int `env:"ARGO_APP_MANUAL_SYNC_TIME" envDefault:"3" description: "retry argocd app manual sync if the timeline is stuck in ARGOCD_SYNC_INITIATED state for more than this defined time (in mins)"` // in minutes + DevtronChartHelmInstallRequestTimeout int `env:"DEVTRON_CHART_INSTALL_REQUEST_TIMEOUT" envDefault:"6" description:"Context timeout for no gitops concurrent async deployments"` // in minutes + DevtronChartArgoCdInstallRequestTimeout int `env:"DEVTRON_CHART_ARGO_CD_INSTALL_REQUEST_TIMEOUT" envDefault:"1" description:"Context timeout for gitops concurrent async deployments"` // in minutes + ArgoCdManualSyncCronPipelineDeployedBefore int `env:"ARGO_APP_MANUAL_SYNC_TIME" envDefault:"3" description:"retry argocd app manual sync if the timeline is stuck in ARGOCD_SYNC_INITIATED state for more than this defined time (in mins)"` // in minutes } func GetAppServiceConfig() (*AppServiceConfig, error) { diff --git a/pkg/appStatus/AppStatusService_test.go b/pkg/appStatus/AppStatusService_test.go index 537a1bce54..ab1ac4a677 100644 --- a/pkg/appStatus/AppStatusService_test.go +++ b/pkg/appStatus/AppStatusService_test.go @@ -189,13 +189,13 @@ func TestDeleteWithAppIdEnvId(t *testing.T) { } type Config struct { - Addr string `env:"TEST_PG_ADDR" envDefault:"127.0.0.1" description: "Postgres URL"` - Port string `env:"TEST_PG_PORT" envDefault:"55000" description: "Postgres port number"` - User string `env:"TEST_PG_USER" envDefault:"postgres"description: "Postgres user"` - Password string `env:"TEST_PG_PASSWORD" envDefault:"postgrespw" secretData:"-" description: "Postgres user password"` - Database string `env:"TEST_PG_DATABASE" envDefault:"orchestrator" description: "Postgres database name"` - ApplicationName string `env:"TEST_APP" envDefault:"orchestrator" description: "application name"` - LogQuery bool `env:"TEST_PG_LOG_QUERY" envDefault:"true" description: "Log PostgreSQL queries."` + Addr string `env:"TEST_PG_ADDR" envDefault:"127.0.0.1" description:"Postgres URL"` + Port string `env:"TEST_PG_PORT" envDefault:"55000" description:"Postgres port number"` + User string `env:"TEST_PG_USER" envDefault:"postgres"description:"Postgres user"` + Password string `env:"TEST_PG_PASSWORD" envDefault:"postgrespw" secretData:"-" description:"Postgres user password"` + Database string `env:"TEST_PG_DATABASE" envDefault:"orchestrator" description:"Postgres database name"` + ApplicationName string `env:"TEST_APP" envDefault:"orchestrator" description:"application name"` + LogQuery bool `env:"TEST_PG_LOG_QUERY" envDefault:"true" description:"Log PostgreSQL queries."` } func getDbConn() (*pg.DB, error) { diff --git a/pkg/auth/authorisation/casbin/rbac.go b/pkg/auth/authorisation/casbin/rbac.go index 1921e2500b..1782602ae0 100644 --- a/pkg/auth/authorisation/casbin/rbac.go +++ b/pkg/auth/authorisation/casbin/rbac.go @@ -74,10 +74,10 @@ type CacheData struct { // CATEGORY=RBAC type EnforcerConfig struct { - CacheEnabled bool `env:"ENFORCER_CACHE" envDefault:"false" description: "To Enable enforcer cache."` - CacheExpirationInSecs int `env:"ENFORCER_CACHE_EXPIRATION_IN_SEC" envDefault:"86400" description: "Expiration time (in seconds) for enforcer cache. "` - EnforcerBatchSize int `env:"ENFORCER_MAX_BATCH_SIZE" envDefault:"1" description: "Maximum batch size for the enforcer."` - UseCasbinV2 bool `env:"USE_CASBIN_V2" envDefault:"false" description: "To enable casbin V2 API"` + CacheEnabled bool `env:"ENFORCER_CACHE" envDefault:"false" description:"To Enable enforcer cache."` + CacheExpirationInSecs int `env:"ENFORCER_CACHE_EXPIRATION_IN_SEC" envDefault:"86400" description:"Expiration time (in seconds) for enforcer cache. "` + EnforcerBatchSize int `env:"ENFORCER_MAX_BATCH_SIZE" envDefault:"1" description:"Maximum batch size for the enforcer."` + UseCasbinV2 bool `env:"USE_CASBIN_V2" envDefault:"false" description:"To enable casbin V2 API"` } func getConfig() (*EnforcerConfig, error) { diff --git a/pkg/auth/user/UserAuthService.go b/pkg/auth/user/UserAuthService.go index e1e88a092f..243f47ade7 100644 --- a/pkg/auth/user/UserAuthService.go +++ b/pkg/auth/user/UserAuthService.go @@ -94,18 +94,18 @@ var Claims struct { } type DexConfig struct { - RedirectURL string `env:"DEX_RURL" envDefault:"http://127.0.0.1:8080/callback" description: "Dex redirect URL(http://argocd-dex-server.devtroncd:8080/callback)"` - ClientID string `env:"DEX_CID" envDefault:"example-app" description: "dex client id "` - ClientSecret string `env:"DEX_SECRET" description: "Dex secret"` - DexURL string `env:"DEX_URL" description: "Dex service endpoint with dex path(http://argocd-dex-server.devtroncd:5556/dex)"` - DexJwtKey string `env:"DEX_JWTKEY" description: "DEX JWT key. "` - CStoreKey string `env:"DEX_CSTOREKEY" description: "DEX CSTOREKEY."` - CookieExpirationTime int `env:"CExpirationTime" envDefault:"600" description: "Caching expiration time."` - JwtExpirationTime int `env:"JwtExpirationTime" envDefault:"120" description: "JWT expiration time."` + RedirectURL string `env:"DEX_RURL" envDefault:"http://127.0.0.1:8080/callback" description:"Dex redirect URL(http://argocd-dex-server.devtroncd:8080/callback)"` + ClientID string `env:"DEX_CID" envDefault:"example-app" description:"dex client id "` + ClientSecret string `env:"DEX_SECRET" description:"Dex secret"` + DexURL string `env:"DEX_URL" description:"Dex service endpoint with dex path(http://argocd-dex-server.devtroncd:5556/dex)"` + DexJwtKey string `env:"DEX_JWTKEY" description:"DEX JWT key. "` + CStoreKey string `env:"DEX_CSTOREKEY" description:"DEX CSTOREKEY."` + CookieExpirationTime int `env:"CExpirationTime" envDefault:"600" description:"Caching expiration time."` + JwtExpirationTime int `env:"JwtExpirationTime" envDefault:"120" description:"JWT expiration time."` } type WebhookToken struct { - WebhookToken string `env:"WEBHOOK_TOKEN" envDefault:"" description: "If you want to continue using jenkins for CI then please provide this for authentication of requests"` + WebhookToken string `env:"WEBHOOK_TOKEN" envDefault:"" description:"If you want to continue using jenkins for CI then please provide this for authentication of requests"` } func NewUserAuthServiceImpl(userAuthRepository repository.UserAuthRepository, sessionManager *middleware.SessionManager, diff --git a/pkg/auth/user/UserCommonService.go b/pkg/auth/user/UserCommonService.go index 5756eb9e4b..9f107d898b 100644 --- a/pkg/auth/user/UserCommonService.go +++ b/pkg/auth/user/UserCommonService.go @@ -105,7 +105,7 @@ func NewUserCommonServiceImpl(userAuthRepository repository.UserAuthRepository, } type UserRbacConfig struct { - UseRbacCreationV2 bool `env:"USE_RBAC_CREATION_V2" envDefault:"true" description: "To use the V2 for RBAC creation"` + UseRbacCreationV2 bool `env:"USE_RBAC_CREATION_V2" envDefault:"true" description:"To use the V2 for RBAC creation"` } func (impl UserCommonServiceImpl) CreateDefaultPoliciesForAllTypes(roleFieldsDto *bean.RoleModelFieldsDto, userId int32) (bool, error, []bean3.Policy) { diff --git a/pkg/build/artifacts/imageTagging/read/Config.go b/pkg/build/artifacts/imageTagging/read/Config.go index f29424e53e..a1a2056645 100644 --- a/pkg/build/artifacts/imageTagging/read/Config.go +++ b/pkg/build/artifacts/imageTagging/read/Config.go @@ -19,7 +19,7 @@ package read // ImageTaggingServiceConfig holds the configuration for the image tagging service type ImageTaggingServiceConfig struct { // HideImageTaggingHardDelete is a flag to hide the hard delete option in the image tagging service - HideImageTaggingHardDelete bool `env:"HIDE_IMAGE_TAGGING_HARD_DELETE" envDefault:"false" description: "Flag to hide the hard delete option in the image tagging service"` + HideImageTaggingHardDelete bool `env:"HIDE_IMAGE_TAGGING_HARD_DELETE" envDefault:"false" description:"Flag to hide the hard delete option in the image tagging service"` } func (c *ImageTaggingServiceConfig) IsHardDeleteHidden() bool { diff --git a/pkg/deployment/gitOps/git/commandManager/GitCommandManager.go b/pkg/deployment/gitOps/git/commandManager/GitCommandManager.go index 21fed3d942..0edaa7c8dd 100644 --- a/pkg/deployment/gitOps/git/commandManager/GitCommandManager.go +++ b/pkg/deployment/gitOps/git/commandManager/GitCommandManager.go @@ -45,8 +45,8 @@ func NewGitCommandManager(logger *zap.SugaredLogger) GitCommandManager { } type configuration struct { - UseGitCli bool `env:"USE_GIT_CLI" envDefault:"false" description: "To enable git cli"` - CliCmdTimeoutGlobal int `env:"CLI_CMD_TIMEOUT_GLOBAL_SECONDS" envDefault:"0" description: "Used in git cli opeartion timeout"` + UseGitCli bool `env:"USE_GIT_CLI" envDefault:"false" description:"To enable git cli"` + CliCmdTimeoutGlobal int `env:"CLI_CMD_TIMEOUT_GLOBAL_SECONDS" envDefault:"0" description:"Used in git cli opeartion timeout"` } func ParseConfiguration() (*configuration, error) { diff --git a/pkg/generateManifest/bean.go b/pkg/generateManifest/bean.go index a87317a914..50c31a8cbb 100644 --- a/pkg/generateManifest/bean.go +++ b/pkg/generateManifest/bean.go @@ -92,6 +92,6 @@ type ResourceMetadata struct { GroupVersionKind schema.GroupVersionKind `json:"groupVersionKind"` } type RestartWorkloadConfig struct { - WorkerPoolSize int `env:"FEATURE_RESTART_WORKLOAD_WORKER_POOL_SIZE" envDefault:"5" description: "restart workload retrieval pool size"` - RequestBatchSize int `env:"FEATURE_RESTART_WORKLOAD_BATCH_SIZE" envDefault:"1" description: "restart workload retrieval batch size "` + WorkerPoolSize int `env:"FEATURE_RESTART_WORKLOAD_WORKER_POOL_SIZE" envDefault:"5" description:"restart workload retrieval pool size"` + RequestBatchSize int `env:"FEATURE_RESTART_WORKLOAD_BATCH_SIZE" envDefault:"1" description:"restart workload retrieval batch size "` } diff --git a/pkg/infraConfig/bean/v1/infra_config.go b/pkg/infraConfig/bean/v1/infra_config.go index 9e5a75e3f6..de4760e9f4 100644 --- a/pkg/infraConfig/bean/v1/infra_config.go +++ b/pkg/infraConfig/bean/v1/infra_config.go @@ -31,7 +31,7 @@ type InfraConfig struct { CiReqMem string `env:"REQ_CI_MEM" envDefault:"3G"` // CiDefaultTimeout is the default timeout for CI jobs in seconds // Earlier it was in int64, but now it is in float64 - CiDefaultTimeout float64 `env:"DEFAULT_TIMEOUT" envDefault:"3600" description: "Timeout for CI to be completed"` + CiDefaultTimeout float64 `env:"DEFAULT_TIMEOUT" envDefault:"3600" description:"Timeout for CI to be completed"` // cm and cs ConfigMaps []bean.ConfigSecretMap `env:"-"` diff --git a/pkg/k8s/K8sCommonService.go b/pkg/k8s/K8sCommonService.go index cdca257bc5..360b8ab53d 100644 --- a/pkg/k8s/K8sCommonService.go +++ b/pkg/k8s/K8sCommonService.go @@ -77,8 +77,8 @@ type K8sCommonServiceImpl struct { ClusterReadService read.ClusterReadService } type K8sApplicationServiceConfig struct { - BatchSize int `env:"BATCH_SIZE" envDefault:"5" description: "there is feature to get URL's of services/ingresses. so to extract those, we need to parse all the servcie and ingress objects of the application. this BATCH_SIZE flag controls the no of these objects get parsed in one go."` - TimeOutInSeconds int `env:"TIMEOUT_IN_SECONDS" envDefault:"5" description: "timeout to compute the urls from services and ingress objects of an application"` + BatchSize int `env:"BATCH_SIZE" envDefault:"5" description:"there is feature to get URL's of services/ingresses. so to extract those, we need to parse all the servcie and ingress objects of the application. this BATCH_SIZE flag controls the no of these objects get parsed in one go."` + TimeOutInSeconds int `env:"TIMEOUT_IN_SECONDS" envDefault:"5" description:"timeout to compute the urls from services and ingress objects of an application"` } func NewK8sCommonServiceImpl(Logger *zap.SugaredLogger, k8sUtils *k8s.K8sServiceImpl, diff --git a/pkg/k8s/application/k8sApplicationService.go b/pkg/k8s/application/k8sApplicationService.go index c2648fffcc..0ba1876c94 100644 --- a/pkg/k8s/application/k8sApplicationService.go +++ b/pkg/k8s/application/k8sApplicationService.go @@ -145,7 +145,7 @@ func NewK8sApplicationServiceImpl(Logger *zap.SugaredLogger, clusterService clus } type EphemeralContainerConfig struct { - EphemeralServerVersionRegex string `env:"EPHEMERAL_SERVER_VERSION_REGEX" envDefault:"v[1-9]\\.\\b(2[3-9]|[3-9][0-9])\\b.*" description: "ephemeral containers support version regex that is compared with k8sServerVersion"` + EphemeralServerVersionRegex string `env:"EPHEMERAL_SERVER_VERSION_REGEX" envDefault:"v[1-9]\\.\\b(2[3-9]|[3-9][0-9])\\b.*" description:"ephemeral containers support version regex that is compared with k8sServerVersion"` } func (impl *K8sApplicationServiceImpl) ValidatePodLogsRequestQuery(r *http.Request) (*bean4.ResourceRequestBean, error) { diff --git a/pkg/module/bean/bean.go b/pkg/module/bean/bean.go index fd874781b3..724f39166e 100644 --- a/pkg/module/bean/bean.go +++ b/pkg/module/bean/bean.go @@ -52,7 +52,7 @@ type ActionResponse struct { } type ModuleEnvConfig struct { - ModuleStatusHandlingCronDurationInMin int `env:"MODULE_STATUS_HANDLING_CRON_DURATION_MIN" envDefault:"3" description: "Module status update cron, used in stack manager, to update the module installation status."` // default 3 minutes + ModuleStatusHandlingCronDurationInMin int `env:"MODULE_STATUS_HANDLING_CRON_DURATION_MIN" envDefault:"3" description:"Module status update cron, used in stack manager, to update the module installation status."` // default 3 minutes } type ModuleResourceStatusDto struct { diff --git a/pkg/pipeline/CiService.go b/pkg/pipeline/CiService.go index 0aea4c435a..3eb0faf6ee 100644 --- a/pkg/pipeline/CiService.go +++ b/pkg/pipeline/CiService.go @@ -83,8 +83,8 @@ type CiService interface { UpdateCiWorkflowWithStage(wf *pipelineConfig.CiWorkflow) error } type BuildxCacheFlags struct { - BuildxCacheModeMin bool `env:"BUILDX_CACHE_MODE_MIN" envDefault:"false" description: "To set build cache mode to minimum in buildx" ` - AsyncBuildxCacheExport bool `env:"ASYNC_BUILDX_CACHE_EXPORT" envDefault:"false" description: "To enable async container image cache export"` + BuildxCacheModeMin bool `env:"BUILDX_CACHE_MODE_MIN" envDefault:"false" description:"To set build cache mode to minimum in buildx" ` + AsyncBuildxCacheExport bool `env:"ASYNC_BUILDX_CACHE_EXPORT" envDefault:"false" description:"To enable async container image cache export"` } type CiServiceImpl struct { diff --git a/pkg/pipeline/PipelineBuilder.go b/pkg/pipeline/PipelineBuilder.go index 3ffa220179..cca170ebbc 100644 --- a/pkg/pipeline/PipelineBuilder.go +++ b/pkg/pipeline/PipelineBuilder.go @@ -52,7 +52,7 @@ func GetEcrConfig() (*EcrConfig, error) { type SecurityConfig struct { //FORCE_SECURITY_SCANNING flag is being maintained in both dashboard and orchestrator CM's //TODO: rishabh will remove FORCE_SECURITY_SCANNING from dashboard's CM. - ForceSecurityScanning bool `env:"FORCE_SECURITY_SCANNING" envDefault:"false" description: "By enabling this no one can disable image scaning on ci-pipeline from UI"` + ForceSecurityScanning bool `env:"FORCE_SECURITY_SCANNING" envDefault:"false" description:"By enabling this no one can disable image scaning on ci-pipeline from UI"` } type PipelineBuilder interface { diff --git a/pkg/server/config/ServerEnvConfig.go b/pkg/server/config/ServerEnvConfig.go index faeb8a8241..72f6e608ec 100644 --- a/pkg/server/config/ServerEnvConfig.go +++ b/pkg/server/config/ServerEnvConfig.go @@ -22,24 +22,24 @@ import ( ) type ServerEnvConfig struct { - DevtronInstallationType string `env:"DEVTRON_INSTALLATION_TYPE" description: "Devtron Installation type(EA/Full)"` - InstallerCrdObjectGroupName string `env:"INSTALLER_CRD_OBJECT_GROUP_NAME" envDefault:"installer.devtron.ai" description: "Devtron installer CRD group name, partially deprecated."` - InstallerCrdObjectVersion string `env:"INSTALLER_CRD_OBJECT_VERSION" envDefault:"v1alpha1" description: "version of the CRDs. default is v1alpha1"` - InstallerCrdObjectResource string `env:"INSTALLER_CRD_OBJECT_RESOURCE" envDefault:"installers" description: "Devtron installer CRD resource name, partially deprecated"` - InstallerCrdNamespace string `env:"INSTALLER_CRD_NAMESPACE" envDefault:"devtroncd" description: "namespace where Custom Resource Definitions get installed"` - DevtronHelmRepoName string `env:"DEVTRON_HELM_REPO_NAME" envDefault:"devtron" description: "Is used to install modules (stack manager)"` - DevtronHelmRepoUrl string `env:"DEVTRON_HELM_REPO_URL" envDefault:"https://helm.devtron.ai" description: "Is used to install modules (stack manager)"` - DevtronHelmReleaseName string `env:"DEVTRON_HELM_RELEASE_NAME" envDefault:"devtron" description: "Name of the Devtron Helm release. "` - DevtronHelmReleaseNamespace string `env:"DEVTRON_HELM_RELEASE_NAMESPACE" envDefault:"devtroncd" description: "Namespace of the Devtron Helm release"` - DevtronHelmReleaseChartName string `env:"DEVTRON_HELM_RELEASE_CHART_NAME" envDefault:"devtron-operator" description: ""` - DevtronVersionIdentifierInHelmValues string `env:"DEVTRON_VERSION_IDENTIFIER_IN_HELM_VALUES" envDefault:"installer.release" description: "devtron operator version identifier in helm values yaml"` + DevtronInstallationType string `env:"DEVTRON_INSTALLATION_TYPE" description:"Devtron Installation type(EA/Full)"` + InstallerCrdObjectGroupName string `env:"INSTALLER_CRD_OBJECT_GROUP_NAME" envDefault:"installer.devtron.ai" description:"Devtron installer CRD group name, partially deprecated."` + InstallerCrdObjectVersion string `env:"INSTALLER_CRD_OBJECT_VERSION" envDefault:"v1alpha1" description:"version of the CRDs. default is v1alpha1"` + InstallerCrdObjectResource string `env:"INSTALLER_CRD_OBJECT_RESOURCE" envDefault:"installers" description:"Devtron installer CRD resource name, partially deprecated"` + InstallerCrdNamespace string `env:"INSTALLER_CRD_NAMESPACE" envDefault:"devtroncd" description:"namespace where Custom Resource Definitions get installed"` + DevtronHelmRepoName string `env:"DEVTRON_HELM_REPO_NAME" envDefault:"devtron" description:"Is used to install modules (stack manager)"` + DevtronHelmRepoUrl string `env:"DEVTRON_HELM_REPO_URL" envDefault:"https://helm.devtron.ai" description:"Is used to install modules (stack manager)"` + DevtronHelmReleaseName string `env:"DEVTRON_HELM_RELEASE_NAME" envDefault:"devtron" description:"Name of the Devtron Helm release. "` + DevtronHelmReleaseNamespace string `env:"DEVTRON_HELM_RELEASE_NAMESPACE" envDefault:"devtroncd" description:"Namespace of the Devtron Helm release"` + DevtronHelmReleaseChartName string `env:"DEVTRON_HELM_RELEASE_CHART_NAME" envDefault:"devtron-operator" description:""` + DevtronVersionIdentifierInHelmValues string `env:"DEVTRON_VERSION_IDENTIFIER_IN_HELM_VALUES" envDefault:"installer.release" description:"devtron operator version identifier in helm values yaml"` DevtronModulesIdentifierInHelmValues string `env:"DEVTRON_MODULES_IDENTIFIER_IN_HELM_VALUES" envDefault:"installer.modules" ` - DevtronBomUrl string `env:"DEVTRON_BOM_URL" envDefault:"https://raw.githubusercontent.com/devtron-labs/devtron/%s/charts/devtron/devtron-bom.yaml" description: "Path to devtron-bom.yaml of devtron charts, used for module installation and devtron upgrade"` - AppSyncImage string `env:"APP_SYNC_IMAGE" envDefault:"quay.io/devtron/chart-sync:1227622d-132-3775" description: "For the app sync image, this image will be used in app-manual sync job"` - AppSyncServiceAccount string `env:"APP_SYNC_SERVICE_ACCOUNT" envDefault:"chart-sync" description: "Service account to be used in app sync Job"` - AppSyncJobResourcesObj string `env:"APP_SYNC_JOB_RESOURCES_OBJ" description: "To pass the resource of app sync"` - ModuleMetaDataApiUrl string `env:"MODULE_METADATA_API_URL" envDefault:"https://api.devtron.ai/module?name=%s" description: "Modules list and meta info will be fetched from this server, that is central api server of devtron."` - ParallelismLimitForTagProcessing int `env:"PARALLELISM_LIMIT_FOR_TAG_PROCESSING" description: "App manual sync job parallel tag processing count."` + DevtronBomUrl string `env:"DEVTRON_BOM_URL" envDefault:"https://raw.githubusercontent.com/devtron-labs/devtron/%s/charts/devtron/devtron-bom.yaml" description:"Path to devtron-bom.yaml of devtron charts, used for module installation and devtron upgrade"` + AppSyncImage string `env:"APP_SYNC_IMAGE" envDefault:"quay.io/devtron/chart-sync:1227622d-132-3775" description:"For the app sync image, this image will be used in app-manual sync job"` + AppSyncServiceAccount string `env:"APP_SYNC_SERVICE_ACCOUNT" envDefault:"chart-sync" description:"Service account to be used in app sync Job"` + AppSyncJobResourcesObj string `env:"APP_SYNC_JOB_RESOURCES_OBJ" description:"To pass the resource of app sync"` + ModuleMetaDataApiUrl string `env:"MODULE_METADATA_API_URL" envDefault:"https://api.devtron.ai/module?name=%s" description:"Modules list and meta info will be fetched from this server, that is central api server of devtron."` + ParallelismLimitForTagProcessing int `env:"PARALLELISM_LIMIT_FOR_TAG_PROCESSING" description:"App manual sync job parallel tag processing count."` AppSyncJobShutDownWaitDuration int `env:"APP_SYNC_SHUTDOWN_WAIT_DURATION" envDefault:"120"` ErrorEncounteredOnGettingDevtronHelmRelease error } diff --git a/pkg/sql/connection.go b/pkg/sql/connection.go index 478d327d23..8f3bec296b 100644 --- a/pkg/sql/connection.go +++ b/pkg/sql/connection.go @@ -34,10 +34,10 @@ type Config struct { User string `env:"PG_USER" envDefault:"" description:"user for postgres" example:"postgres"` Password string `env:"PG_PASSWORD" envDefault:"" secretData:"-" description:"password for postgres, associated with PG_USER" example:"confidential ;)"` Database string `env:"PG_DATABASE" envDefault:"orchestrator" description:"postgres database to be made connection with" example:"orchestrator, casbin, git_sensor, lens"` - CasbinDatabase string `env:"CASBIN_DATABASE" envDefault:"casbin" description: "Database for casbin"` + CasbinDatabase string `env:"CASBIN_DATABASE" envDefault:"casbin" description:"Database for casbin"` ApplicationName string `env:"APP" envDefault:"orchestrator" description:"Application name"` - ReadTimeout int64 `env:"PG_READ_TIMEOUT" envDefault:"30" description: "Time out for read operation in postgres"` - WriteTimeout int64 `env:"PG_WRITE_TIMEOUT" envDefault:"30" description: "Time out for write operation in postgres"` + ReadTimeout int64 `env:"PG_READ_TIMEOUT" envDefault:"30" description:"Time out for read operation in postgres"` + WriteTimeout int64 `env:"PG_WRITE_TIMEOUT" envDefault:"30" description:"Time out for write operation in postgres"` bean.PgQueryMonitoringConfig } diff --git a/pkg/terminal/terminalSesion.go b/pkg/terminal/terminalSesion.go index e6c87c5cd9..bc889b542c 100644 --- a/pkg/terminal/terminalSesion.go +++ b/pkg/terminal/terminalSesion.go @@ -282,8 +282,8 @@ func handleTerminalSession(session sockjs.Session) { } type SocketConfig struct { - SocketHeartbeatSeconds int `env:"SOCKET_HEARTBEAT_SECONDS" envDefault:"25" description: "In order to keep proxies and load balancers from closing long running http requests we need to pretend that the connection is active and send a heartbeat packet once in a while. This setting controls how often this is done. By default a heartbeat packet is sent every 25 seconds."` - SocketDisconnectDelay int `env:"SOCKET_DISCONNECT_DELAY_SECONDS" envDefault:"5" description: "The server closes a session when a client receiving connection have not been seen for a while.This delay is configured by this setting. By default the session is closed when a receiving connection wasn't seen for 5 seconds."` + SocketHeartbeatSeconds int `env:"SOCKET_HEARTBEAT_SECONDS" envDefault:"25" description:"In order to keep proxies and load balancers from closing long running http requests we need to pretend that the connection is active and send a heartbeat packet once in a while. This setting controls how often this is done. By default a heartbeat packet is sent every 25 seconds."` + SocketDisconnectDelay int `env:"SOCKET_DISCONNECT_DELAY_SECONDS" envDefault:"5" description:"The server closes a session when a client receiving connection have not been seen for a while.This delay is configured by this setting. By default the session is closed when a receiving connection wasn't seen for 5 seconds."` } var cfg *SocketConfig diff --git a/pkg/util/TokenCache.go b/pkg/util/TokenCache.go index bcae68cf82..0bfbc28311 100644 --- a/pkg/util/TokenCache.go +++ b/pkg/util/TokenCache.go @@ -62,13 +62,13 @@ func (impl *TokenCache) BuildACDSynchContext() (acdContext context.Context, err // CATEGORY=GITOPS type ACDAuthConfig struct { - ACDUsername string `env:"ACD_USERNAME" envDefault:"admin" description: "User name for argocd"` - ACDPassword string `env:"ACD_PASSWORD" description: "Password for the Argocd (deprecated)"` - ACDConfigMapName string `env:"ACD_CM" envDefault:"argocd-cm" description: "Name of the argocd CM"` - ACDConfigMapNamespace string `env:"ACD_NAMESPACE" envDefault:"devtroncd" description: "To pass the argocd namespace"` - GitOpsSecretName string `env:"GITOPS_SECRET_NAME" envDefault:"devtron-gitops-secret" description: "devtron-gitops-secret"` - ResourceListForReplicas string `env:"RESOURCE_LIST_FOR_REPLICAS" envDefault:"Deployment,Rollout,StatefulSet,ReplicaSet" description: "this holds the list of k8s resource names which support replicas key. this list used in hibernate/un hibernate process"` - ResourceListForReplicasBatchSize int `env:"RESOURCE_LIST_FOR_REPLICAS_BATCH_SIZE" envDefault:"5" description: "this the batch size to control no of above resources can be parsed in one go to determine hibernate status"` + ACDUsername string `env:"ACD_USERNAME" envDefault:"admin" description:"User name for argocd"` + ACDPassword string `env:"ACD_PASSWORD" description:"Password for the Argocd (deprecated)"` + ACDConfigMapName string `env:"ACD_CM" envDefault:"argocd-cm" description:"Name of the argocd CM"` + ACDConfigMapNamespace string `env:"ACD_NAMESPACE" envDefault:"devtroncd" description:"To pass the argocd namespace"` + GitOpsSecretName string `env:"GITOPS_SECRET_NAME" envDefault:"devtron-gitops-secret" description:"devtron-gitops-secret"` + ResourceListForReplicas string `env:"RESOURCE_LIST_FOR_REPLICAS" envDefault:"Deployment,Rollout,StatefulSet,ReplicaSet" description:"this holds the list of k8s resource names which support replicas key. this list used in hibernate/un hibernate process"` + ResourceListForReplicasBatchSize int `env:"RESOURCE_LIST_FOR_REPLICAS_BATCH_SIZE" envDefault:"5" description:"this the batch size to control no of above resources can be parsed in one go to determine hibernate status"` } func GetACDAuthConfig() (*ACDAuthConfig, error) { diff --git a/pkg/variables/ScopedVariableService.go b/pkg/variables/ScopedVariableService.go index b5361e4992..bf9e628e84 100644 --- a/pkg/variables/ScopedVariableService.go +++ b/pkg/variables/ScopedVariableService.go @@ -75,10 +75,10 @@ func NewScopedVariableServiceImpl(logger *zap.SugaredLogger, scopedVariableRepos } type VariableConfig struct { - VariableNameRegex string `env:"SCOPED_VARIABLE_NAME_REGEX" envDefault:"^[a-zA-Z][a-zA-Z0-9_-]{0,62}[a-zA-Z0-9]$" description: "Regex for scoped variable name that must passed this regex."` - VariableCacheEnabled bool `env:"VARIABLE_CACHE_ENABLED" envDefault:"true" description: "This is used to control caching of all the scope variables defined in the system."` - SystemVariablePrefix string `env:"SYSTEM_VAR_PREFIX" envDefault:"DEVTRON_" description: "Scoped variable prefix, variable name must have this prefix."` - ScopedVariableFormat string `env:"SCOPED_VARIABLE_FORMAT" envDefault:"@{{%s}}" description: "Its a scope format for varialbe name."` + VariableNameRegex string `env:"SCOPED_VARIABLE_NAME_REGEX" envDefault:"^[a-zA-Z][a-zA-Z0-9_-]{0,62}[a-zA-Z0-9]$" description:"Regex for scoped variable name that must passed this regex."` + VariableCacheEnabled bool `env:"VARIABLE_CACHE_ENABLED" envDefault:"true" description:"This is used to control caching of all the scope variables defined in the system."` + SystemVariablePrefix string `env:"SYSTEM_VAR_PREFIX" envDefault:"DEVTRON_" description:"Scoped variable prefix, variable name must have this prefix."` + ScopedVariableFormat string `env:"SCOPED_VARIABLE_FORMAT" envDefault:"@{{%s}}" description:"Its a scope format for varialbe name."` } func loadVariableCache(cfg *VariableConfig, service *ScopedVariableServiceImpl) { diff --git a/pkg/variables/parsers/VariableTemplateParser.go b/pkg/variables/parsers/VariableTemplateParser.go index fcfc023705..9a076f8eac 100644 --- a/pkg/variables/parsers/VariableTemplateParser.go +++ b/pkg/variables/parsers/VariableTemplateParser.go @@ -57,9 +57,9 @@ func NewVariableTemplateParserImpl(logger *zap.SugaredLogger) (*VariableTemplate } type VariableTemplateParserConfig struct { - ScopedVariableEnabled bool `env:"SCOPED_VARIABLE_ENABLED" envDefault:"false" description: "To enable scoped variable option"` - ScopedVariableHandlePrimitives bool `env:"SCOPED_VARIABLE_HANDLE_PRIMITIVES" envDefault:"false" description: "This describe should we handle primitives or not in scoped variable template parsing."` - VariableExpressionRegex string `env:"VARIABLE_EXPRESSION_REGEX" envDefault:"@{{([^}]+)}}" description: "Scoped variable expression regex"` + ScopedVariableEnabled bool `env:"SCOPED_VARIABLE_ENABLED" envDefault:"false" description:"To enable scoped variable option"` + ScopedVariableHandlePrimitives bool `env:"SCOPED_VARIABLE_HANDLE_PRIMITIVES" envDefault:"false" description:"This describe should we handle primitives or not in scoped variable template parsing."` + VariableExpressionRegex string `env:"VARIABLE_EXPRESSION_REGEX" envDefault:"@{{([^}]+)}}" description:"Scoped variable expression regex"` } func (cfg VariableTemplateParserConfig) isScopedVariablesDisabled() bool { diff --git a/util/GlobalConfig.go b/util/GlobalConfig.go index 1e2d02b3c8..926d4f43cd 100644 --- a/util/GlobalConfig.go +++ b/util/GlobalConfig.go @@ -31,7 +31,7 @@ type EnvironmentVariables struct { // CATEGORY=CD type DeploymentServiceTypeConfig struct { - ExternallyManagedDeploymentType bool `env:"IS_INTERNAL_USE" envDefault:"false" description: "If enabled then cd pipeline and helm apps will not need the deployment app type mandatorily. Couple this flag with HIDE_GITOPS_OR_HELM_OPTION (in Dashborad) and if gitops is configured and allowed for the env, pipeline/ helm app will gitops else no-gitops."` + ExternallyManagedDeploymentType bool `env:"IS_INTERNAL_USE" envDefault:"false" description:"If enabled then cd pipeline and helm apps will not need the deployment app type mandatorily. Couple this flag with HIDE_GITOPS_OR_HELM_OPTION (in Dashborad) and if gitops is configured and allowed for the env, pipeline/ helm app will gitops else no-gitops."` HelmInstallASyncMode bool `env:"RUN_HELM_INSTALL_IN_ASYNC_MODE_HELM_APPS" envDefault:"false"` UseDeploymentConfigData bool `env:"USE_DEPLOYMENT_CONFIG_DATA" envDefault:"false" description:"use deployment config data from deployment_config table" deprecated:"true"` MigrateDeploymentConfigData bool `env:"MIGRATE_DEPLOYMENT_CONFIG_DATA" envDefault:"false" description:"migrate deployment config data from charts table to deployment_config table" deprecated:"false"` @@ -47,27 +47,27 @@ func (d *DeploymentServiceTypeConfig) IsFeatureMigrateArgoCdApplicationEnable() } type GlobalEnvVariables struct { - GitOpsRepoPrefix string `env:"GITOPS_REPO_PREFIX" envDefault:"" description: "Prefix for Gitops repo being creation for argocd application"` - EnableAsyncHelmInstallDevtronChart bool `env:"ENABLE_ASYNC_INSTALL_DEVTRON_CHART" envDefault:"false" description: "To enable async installation of no-gitops application"` - EnableAsyncArgoCdInstallDevtronChart bool `env:"ENABLE_ASYNC_ARGO_CD_INSTALL_DEVTRON_CHART" envDefault:"false" description: "To enable async installation of gitops application"` - ArgoGitCommitRetryCountOnConflict int `env:"ARGO_GIT_COMMIT_RETRY_COUNT_ON_CONFLICT" envDefault:"3" description: "retry argocd app manual sync if the timeline is stuck in ARGOCD_SYNC_INITIATED state for more than this defined time (in mins)" + GitOpsRepoPrefix string `env:"GITOPS_REPO_PREFIX" envDefault:"" description:"Prefix for Gitops repo being creation for argocd application"` + EnableAsyncHelmInstallDevtronChart bool `env:"ENABLE_ASYNC_INSTALL_DEVTRON_CHART" envDefault:"false" description:"To enable async installation of no-gitops application"` + EnableAsyncArgoCdInstallDevtronChart bool `env:"ENABLE_ASYNC_ARGO_CD_INSTALL_DEVTRON_CHART" envDefault:"false" description:"To enable async installation of gitops application"` + ArgoGitCommitRetryCountOnConflict int `env:"ARGO_GIT_COMMIT_RETRY_COUNT_ON_CONFLICT" envDefault:"3" description:"retry argocd app manual sync if the timeline is stuck in ARGOCD_SYNC_INITIATED state for more than this defined time (in mins)" ` - ArgoGitCommitRetryDelayOnConflict int `env:"ARGO_GIT_COMMIT_RETRY_DELAY_ON_CONFLICT" envDefault:"1" description: "Delay on retrying the maifest commit the on gitops"` - ExposeCiMetrics bool `env:"EXPOSE_CI_METRICS" envDefault:"false" description: "To expose CI metrics"` - ExecuteWireNilChecker bool `env:"EXECUTE_WIRE_NIL_CHECKER" envDefault:"false" description: "checks for any nil pointer in wire.go"` + ArgoGitCommitRetryDelayOnConflict int `env:"ARGO_GIT_COMMIT_RETRY_DELAY_ON_CONFLICT" envDefault:"1" description:"Delay on retrying the maifest commit the on gitops"` + ExposeCiMetrics bool `env:"EXPOSE_CI_METRICS" envDefault:"false" description:"To expose CI metrics"` + ExecuteWireNilChecker bool `env:"EXECUTE_WIRE_NIL_CHECKER" envDefault:"false" description:"checks for any nil pointer in wire.go"` } type GlobalClusterConfig struct { - ClusterStatusCronTime int `env:"CLUSTER_STATUS_CRON_TIME" envDefault:"15" description: "Cron schedule for cluster status on resource browser"` + ClusterStatusCronTime int `env:"CLUSTER_STATUS_CRON_TIME" envDefault:"15" description:"Cron schedule for cluster status on resource browser"` } type DevtronSecretConfig struct { DevtronSecretName string `env:"DEVTRON_SECRET_NAME" envDefault:"devtron-secret"` - DevtronDexSecretNamespace string `env:"DEVTRON_DEX_SECRET_NAMESPACE" envDefault:"devtroncd" description: "Namespace of dex secret"` + DevtronDexSecretNamespace string `env:"DEVTRON_DEX_SECRET_NAMESPACE" envDefault:"devtroncd" description:"Namespace of dex secret"` } type TerminalEnvVariables struct { - RestrictTerminalAccessForNonSuperUser bool `env:"RESTRICT_TERMINAL_ACCESS_FOR_NON_SUPER_USER" envDefault:"false" description: "To restrict the cluster terminal from user having non-super admin acceess"` + RestrictTerminalAccessForNonSuperUser bool `env:"RESTRICT_TERMINAL_ACCESS_FOR_NON_SUPER_USER" envDefault:"false" description:"To restrict the cluster terminal from user having non-super admin acceess"` } type InternalEnvVariables struct { diff --git a/vendor/github.com/devtron-labs/authenticator/client/k8sClient.go b/vendor/github.com/devtron-labs/authenticator/client/k8sClient.go index abca34ff00..9d1bf6c311 100644 --- a/vendor/github.com/devtron-labs/authenticator/client/k8sClient.go +++ b/vendor/github.com/devtron-labs/authenticator/client/k8sClient.go @@ -47,8 +47,8 @@ type K8sClient struct { } type RuntimeConfig struct { - LocalDevMode LocalDevMode `env:"RUNTIME_CONFIG_LOCAL_DEV" envDefault:"false" description: "Used in local development process. Allows to read from local kube config file path for default cluster operations."` - DevtronDefaultNamespaceName string `env:"DEVTRON_DEFAULT_NAMESPACE" envDefault:"devtroncd" description: "default namespace for devtron"` + LocalDevMode LocalDevMode `env:"RUNTIME_CONFIG_LOCAL_DEV" envDefault:"false" description:"Used in local development process. Allows to read from local kube config file path for default cluster operations."` + DevtronDefaultNamespaceName string `env:"DEVTRON_DEFAULT_NAMESPACE" envDefault:"devtroncd" description:"default namespace for devtron"` } func GetRuntimeConfig() (*RuntimeConfig, error) { diff --git a/vendor/github.com/devtron-labs/authenticator/client/oidcClient.go b/vendor/github.com/devtron-labs/authenticator/client/oidcClient.go index 25bd1ab23c..75b35255ae 100644 --- a/vendor/github.com/devtron-labs/authenticator/client/oidcClient.go +++ b/vendor/github.com/devtron-labs/authenticator/client/oidcClient.go @@ -77,18 +77,18 @@ func getOidcClient(dexServerAddress string, settings *oidc.Settings, userVerifie const dexProxyUri = "api/dex" type DexConfig struct { - DexHost string `env:"DEX_HOST" envDefault:"http://localhost" description: "Dex service endpoint(http://argocd-dex-server.devtroncd)"` - DexPort string `env:"DEX_PORT" envDefault:"5556" description: "Dex service port"` - DexClientID string `env:"DEX_CLIENT_ID" envDefault:"argo-cd" description: "dex client id to be used when connecting to dex server"` + DexHost string `env:"DEX_HOST" envDefault:"http://localhost" description:"Dex service endpoint(http://argocd-dex-server.devtroncd)"` + DexPort string `env:"DEX_PORT" envDefault:"5556" description:"Dex service port"` + DexClientID string `env:"DEX_CLIENT_ID" envDefault:"argo-cd" description:"dex client id to be used when connecting to dex server"` DexServerAddress string Url string DexClientSecret string ServerSecret string // Specifies token expiration duration - UserSessionDurationSeconds int `env:"USER_SESSION_DURATION_SECONDS" envDefault:"86400" description: "Session time after which user have to reLogin"` + UserSessionDurationSeconds int `env:"USER_SESSION_DURATION_SECONDS" envDefault:"86400" description:"Session time after which user have to reLogin"` AdminPasswordMtime time.Time `json:"ADMIN_PASSWORD_MTIME"` DexConfigRaw string - DevtronSecretName string `env:"DEVTRON_SECRET_NAME" envDefault:"devtron-secret" description: "Secret name for orchestrator micro-services"` + DevtronSecretName string `env:"DEVTRON_SECRET_NAME" envDefault:"devtron-secret" description:"Secret name for orchestrator micro-services"` } func (c *DexConfig) GetDexProxyUrl() (string, error) { diff --git a/vendor/github.com/devtron-labs/common-lib/pubsub-lib/JetStreamUtil.go b/vendor/github.com/devtron-labs/common-lib/pubsub-lib/JetStreamUtil.go index d0114e1519..ae0ed13512 100644 --- a/vendor/github.com/devtron-labs/common-lib/pubsub-lib/JetStreamUtil.go +++ b/vendor/github.com/devtron-labs/common-lib/pubsub-lib/JetStreamUtil.go @@ -125,10 +125,10 @@ type NatsTopic struct { } type ConfigJson struct { // StreamConfigJson is a json string of map[string]NatsStreamConfig - StreamConfigJson string `env:"STREAM_CONFIG_JSON" description: "this is json map of nats stream configurations per stream. eg: {"ORCHESTRATOR":{"max_age":3600,"replicas":1}}"` + StreamConfigJson string `env:"STREAM_CONFIG_JSON" description:"this is json map of nats stream configurations per stream. eg: {"ORCHESTRATOR":{"max_age":3600,"replicas":1}}"` // ConsumerConfigJson is a json string of map[string]NatsConsumerConfig // eg: "{\"ARGO_PIPELINE_STATUS_UPDATE_DURABLE-1\" : \"{\"natsMsgProcessingBatchSize\" : 3, \"natsMsgBufferSize\" : 3, \"ackWaitInSecs\": 300}\"}" - ConsumerConfigJson string `env:"CONSUMER_CONFIG_JSON" description: "ConsumerConfigJson is a json string of map[string]NatsConsumerConfig eg: "{\"ARGO_PIPELINE_STATUS_UPDATE_DURABLE-1\" : \"{\"natsMsgProcessingBatchSize\" : 3, \"natsMsgBufferSize\" : 3, \"ackWaitInSecs\": 300}\"}""` + ConsumerConfigJson string `env:"CONSUMER_CONFIG_JSON" description:"ConsumerConfigJson is a json string of map[string]NatsConsumerConfig eg: "{\"ARGO_PIPELINE_STATUS_UPDATE_DURABLE-1\" : \"{\"natsMsgProcessingBatchSize\" : 3, \"natsMsgBufferSize\" : 3, \"ackWaitInSecs\": 300}\"}""` } var natsTopicMapping = map[string]NatsTopic{ diff --git a/vendor/github.com/devtron-labs/common-lib/pubsub-lib/NatsClient.go b/vendor/github.com/devtron-labs/common-lib/pubsub-lib/NatsClient.go index e82559c238..bd95783ff0 100644 --- a/vendor/github.com/devtron-labs/common-lib/pubsub-lib/NatsClient.go +++ b/vendor/github.com/devtron-labs/common-lib/pubsub-lib/NatsClient.go @@ -32,11 +32,11 @@ type NatsClient struct { } type NatsClientConfig struct { - NatsServerHost string `env:"NATS_SERVER_HOST" envDefault:"nats://devtron-nats.devtroncd:4222" description: "Nats micro-service URL "` + NatsServerHost string `env:"NATS_SERVER_HOST" envDefault:"nats://devtron-nats.devtroncd:4222" description:"Nats micro-service URL "` // consumer wise // NatsMsgProcessingBatchSize is the number of messages that will be processed in one go - NatsMsgProcessingBatchSize int `env:"NATS_MSG_PROCESSING_BATCH_SIZE" envDefault:"1" description: "NatsMsgProcessingBatchSize is the number of messages that will be processed in one go"` + NatsMsgProcessingBatchSize int `env:"NATS_MSG_PROCESSING_BATCH_SIZE" envDefault:"1" description:"NatsMsgProcessingBatchSize is the number of messages that will be processed in one go"` // NatsMsgBufferSize is the number of messages that will be buffered in memory (channel size) // it is recommended to set this value equal to NatsMsgProcessingBatchSize as we want to process maximum messages in the buffer in one go. @@ -45,10 +45,10 @@ type NatsClientConfig struct { // NatsMsgBufferSize can be configured independently of NatsMsgProcessingBatchSize if needed by setting its value to positive value in env. // if NatsMsgBufferSize set to a non-positive value then it will take the value of NatsMsgProcessingBatchSize. // Note: always get this value by calling GetNatsMsgBufferSize method - NatsMsgBufferSize int `env:"NATS_MSG_BUFFER_SIZE" envDefault:"-1" description: "NatsMsgBufferSize is the number of messages that will be buffered in memory (channel size)"` - NatsMsgMaxAge int `env:"NATS_MSG_MAX_AGE" envDefault:"86400" description: "Age for the message to persist "` - NatsMsgAckWaitInSecs int `env:"NATS_MSG_ACK_WAIT_IN_SECS" envDefault:"120" description: "Time to wait for acknowledging the message"` - NatsMsgReplicas int `env:"NATS_MSG_REPLICAS" envDefault:"0" description: "Replica count for runnings nats instance"` + NatsMsgBufferSize int `env:"NATS_MSG_BUFFER_SIZE" envDefault:"-1" description:"NatsMsgBufferSize is the number of messages that will be buffered in memory (channel size)"` + NatsMsgMaxAge int `env:"NATS_MSG_MAX_AGE" envDefault:"86400" description:"Age for the message to persist "` + NatsMsgAckWaitInSecs int `env:"NATS_MSG_ACK_WAIT_IN_SECS" envDefault:"120" description:"Time to wait for acknowledging the message"` + NatsMsgReplicas int `env:"NATS_MSG_REPLICAS" envDefault:"0" description:"Replica count for runnings nats instance"` } func (ncc NatsClientConfig) GetNatsMsgBufferSize() int { diff --git a/vendor/github.com/devtron-labs/common-lib/utils/bean/bean.go b/vendor/github.com/devtron-labs/common-lib/utils/bean/bean.go index ac75e25e90..6dff00ca6e 100644 --- a/vendor/github.com/devtron-labs/common-lib/utils/bean/bean.go +++ b/vendor/github.com/devtron-labs/common-lib/utils/bean/bean.go @@ -66,10 +66,10 @@ type DockerRegistryInfo struct { type PgQueryMonitoringConfig struct { LogSlowQuery bool `env:"PG_LOG_SLOW_QUERY" envDefault:"true"` - LogAllQuery bool `env:"PG_LOG_ALL_QUERY" envDefault:"false" description: "Log all query ran by orchestrator in logs"` + LogAllQuery bool `env:"PG_LOG_ALL_QUERY" envDefault:"false" description:"Log all query ran by orchestrator in logs"` LogAllFailureQueries bool `env:"PG_LOG_ALL_FAILURE_QUERIES" envDefault:"true"` - ExportPromMetrics bool `env:"PG_EXPORT_PROM_METRICS" envDefault:"true" description: "To expose postgres metrics"` - QueryDurationThreshold int64 `env:"PG_QUERY_DUR_THRESHOLD" envDefault:"5000" description: "if the particular querry execution should cross the threshold , then the querry should log."` + ExportPromMetrics bool `env:"PG_EXPORT_PROM_METRICS" envDefault:"true" description:"To expose postgres metrics"` + QueryDurationThreshold int64 `env:"PG_QUERY_DUR_THRESHOLD" envDefault:"5000" description:"if the particular querry execution should cross the threshold , then the querry should log."` ServiceName string } diff --git a/vendor/github.com/devtron-labs/common-lib/utils/grpc/GrpcConfig.go b/vendor/github.com/devtron-labs/common-lib/utils/grpc/GrpcConfig.go index 3e7a8df31d..603dc262ce 100644 --- a/vendor/github.com/devtron-labs/common-lib/utils/grpc/GrpcConfig.go +++ b/vendor/github.com/devtron-labs/common-lib/utils/grpc/GrpcConfig.go @@ -3,8 +3,8 @@ package grpc import "github.com/caarlos0/env" type Configuration struct { - KubelinkMaxRecvMsgSize int `env:"KUBELINK_GRPC_MAX_RECEIVE_MSG_SIZE" envDefault:"20" description: "Message size to receive from kubelink"` // In mb - KubelinkMaxSendMsgSize int `env:"KUBELINK_GRPC_MAX_SEND_MSG_SIZE" envDefault:"4" description: "Message size to send to kubelink"` // In mb + KubelinkMaxRecvMsgSize int `env:"KUBELINK_GRPC_MAX_RECEIVE_MSG_SIZE" envDefault:"20" description:"Message size to receive from kubelink"` // In mb + KubelinkMaxSendMsgSize int `env:"KUBELINK_GRPC_MAX_SEND_MSG_SIZE" envDefault:"4" description:"Message size to send to kubelink"` // In mb } func GetConfiguration() (*Configuration, error) { diff --git a/vendor/github.com/devtron-labs/common-lib/utils/k8s/bean.go b/vendor/github.com/devtron-labs/common-lib/utils/k8s/bean.go index a69121b368..7f83c24891 100644 --- a/vendor/github.com/devtron-labs/common-lib/utils/k8s/bean.go +++ b/vendor/github.com/devtron-labs/common-lib/utils/k8s/bean.go @@ -169,12 +169,12 @@ func GetResourceKey(obj *unstructured.Unstructured) ResourceKey { } type CustomK8sHttpTransportConfig struct { - UseCustomTransport bool `env:"USE_CUSTOM_HTTP_TRANSPORT" envDefault:"false" description: "There is an issue in updating cluster bearer token with same cluster url due to transport layer token caching in k8s client lib. so we added a custom transport while calling k8s api server. this flag controls the usage of this transport.(custom or k8s)"` - TimeOut int `env:"K8s_TCP_TIMEOUT" envDefault:"30" description: "is the maximum amount of time a dial will wait for a connect to complete, required only if USE_CUSTOM_HTTP_TRANSPORT is true."` - KeepAlive int `env:"K8s_TCP_KEEPALIVE" envDefault:"30" description: "specifies the interval between keep-alive probes for an active, required only if USE_CUSTOM_HTTP_TRANSPORT is true."` - TLSHandshakeTimeout int `env:"K8s_TLS_HANDSHAKE_TIMEOUT" envDefault:"10" description: "specifies the maximum amount of time to wait for a TLS handshake. Zero means no timeout, required only if USE_CUSTOM_HTTP_TRANSPORT is true."` - MaxIdleConnsPerHost int `env:"K8s_CLIENT_MAX_IDLE_CONNS_PER_HOST" envDefault:"25" description: "k8s client max idle connections per host"` - IdleConnTimeout int `env:"K8s_TCP_IDLE_CONN_TIMEOUT" envDefault:"300" description: "maximum amount of time an idle (keep-alive) connection will remain idle before closing itself, required only if USE_CUSTOM_HTTP_TRANSPORT is true."` + UseCustomTransport bool `env:"USE_CUSTOM_HTTP_TRANSPORT" envDefault:"false" description:"There is an issue in updating cluster bearer token with same cluster url due to transport layer token caching in k8s client lib. so we added a custom transport while calling k8s api server. this flag controls the usage of this transport.(custom or k8s)"` + TimeOut int `env:"K8s_TCP_TIMEOUT" envDefault:"30" description:"is the maximum amount of time a dial will wait for a connect to complete, required only if USE_CUSTOM_HTTP_TRANSPORT is true."` + KeepAlive int `env:"K8s_TCP_KEEPALIVE" envDefault:"30" description:"specifies the interval between keep-alive probes for an active, required only if USE_CUSTOM_HTTP_TRANSPORT is true."` + TLSHandshakeTimeout int `env:"K8s_TLS_HANDSHAKE_TIMEOUT" envDefault:"10" description:"specifies the maximum amount of time to wait for a TLS handshake. Zero means no timeout, required only if USE_CUSTOM_HTTP_TRANSPORT is true."` + MaxIdleConnsPerHost int `env:"K8s_CLIENT_MAX_IDLE_CONNS_PER_HOST" envDefault:"25" description:"k8s client max idle connections per host"` + IdleConnTimeout int `env:"K8s_TCP_IDLE_CONN_TIMEOUT" envDefault:"300" description:"maximum amount of time an idle (keep-alive) connection will remain idle before closing itself, required only if USE_CUSTOM_HTTP_TRANSPORT is true."` } type LocalDevMode bool From 1a4a38b84e515d9e13235c714332730600f5e050 Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Thu, 10 Apr 2025 17:10:44 +0530 Subject: [PATCH 21/34] env gen update --- env_gen.json | 2 +- env_gen.md | 413 +++++++++--------- go.mod | 4 +- go.sum | 8 +- .../authenticator/client/k8sClient.go | 4 +- .../authenticator/client/oidcClient.go | 11 +- .../common-lib/pubsub-lib/JetStreamUtil.go | 4 +- .../common-lib/pubsub-lib/NatsClient.go | 12 +- .../common-lib/utils/bean/bean.go | 6 +- .../common-lib/utils/grpc/GrpcConfig.go | 4 +- .../devtron-labs/common-lib/utils/k8s/bean.go | 12 +- vendor/modules.txt | 8 +- 12 files changed, 246 insertions(+), 242 deletions(-) diff --git a/env_gen.json b/env_gen.json index b4b889fc79..6fd874d675 100644 --- a/env_gen.json +++ b/env_gen.json @@ -1 +1 @@ -[{"Category":"CD","Fields":[{"Env":"ARGO_APP_MANUAL_SYNC_TIME","EnvType":"int","EnvValue":"3","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_HELM_PIPELINE_STATUS_CRON_TIME","EnvType":"string","EnvValue":"*/2 * * * *","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_PIPELINE_STATUS_CRON_TIME","EnvType":"string","EnvValue":"*/2 * * * *","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_PIPELINE_STATUS_TIMEOUT_DURATION","EnvType":"string","EnvValue":"20","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEPLOY_STATUS_CRON_GET_PIPELINE_DEPLOYED_WITHIN_HOURS","EnvType":"int","EnvValue":"12","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEVTRON_CHART_ARGO_CD_INSTALL_REQUEST_TIMEOUT","EnvType":"int","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEVTRON_CHART_INSTALL_REQUEST_TIMEOUT","EnvType":"int","EnvValue":"6","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"EXPOSE_CD_METRICS","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"FEATURE_MIGRATE_ARGOCD_APPLICATION_ENABLE","EnvType":"bool","EnvValue":"false","EnvDescription":"enable migration of external argocd application to devtron pipeline","Example":"","Deprecated":"false"},{"Env":"HELM_PIPELINE_STATUS_CHECK_ELIGIBLE_TIME","EnvType":"string","EnvValue":"120","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"IS_INTERNAL_USE","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"MIGRATE_DEPLOYMENT_CONFIG_DATA","EnvType":"bool","EnvValue":"false","EnvDescription":"migrate deployment config data from charts table to deployment_config table","Example":"","Deprecated":"false"},{"Env":"PIPELINE_DEGRADED_TIME","EnvType":"string","EnvValue":"10","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"REVISION_HISTORY_LIMIT_DEVTRON_APP","EnvType":"int","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"REVISION_HISTORY_LIMIT_EXTERNAL_HELM_APP","EnvType":"int","EnvValue":"0","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"REVISION_HISTORY_LIMIT_HELM_APP","EnvType":"int","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"REVISION_HISTORY_LIMIT_LINKED_HELM_APP","EnvType":"int","EnvValue":"15","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"RUN_HELM_INSTALL_IN_ASYNC_MODE_HELM_APPS","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"SHOULD_CHECK_NAMESPACE_ON_CLONE","EnvType":"bool","EnvValue":"false","EnvDescription":"should we check if namespace exists or not while cloning app","Example":"","Deprecated":"false"},{"Env":"USE_DEPLOYMENT_CONFIG_DATA","EnvType":"bool","EnvValue":"false","EnvDescription":"use deployment config data from deployment_config table","Example":"","Deprecated":"true"}]},{"Category":"CI_RUNNER","Fields":[{"Env":"AZURE_ACCOUNT_KEY","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"AZURE_ACCOUNT_NAME","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"AZURE_BLOB_CONTAINER_CI_CACHE","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"AZURE_BLOB_CONTAINER_CI_LOG","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"AZURE_GATEWAY_CONNECTION_INSECURE","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"AZURE_GATEWAY_URL","EnvType":"string","EnvValue":"http://devtron-minio.devtroncd:9000","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BASE_LOG_LOCATION_PATH","EnvType":"string","EnvValue":"/home/devtron/","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_GCP_CREDENTIALS_JSON","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_PROVIDER","EnvType":"","EnvValue":"S3","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_S3_ACCESS_KEY","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_S3_BUCKET_VERSIONED","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_S3_ENDPOINT","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_S3_ENDPOINT_INSECURE","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_S3_SECRET_KEY","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BUILDX_CACHE_PATH","EnvType":"string","EnvValue":"/var/lib/devtron/buildx","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BUILDX_K8S_DRIVER_OPTIONS","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BUILDX_PROVENANCE_MODE","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BUILD_LOG_TTL_VALUE_IN_SECS","EnvType":"int","EnvValue":"3600","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CACHE_LIMIT","EnvType":"int64","EnvValue":"5000000000","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_DEFAULT_ADDRESS_POOL_BASE_CIDR","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_DEFAULT_ADDRESS_POOL_SIZE","EnvType":"int","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_LIMIT_CI_CPU","EnvType":"string","EnvValue":"0.5","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_LIMIT_CI_MEM","EnvType":"string","EnvValue":"3G","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_NODE_LABEL_SELECTOR","EnvType":"","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_NODE_TAINTS_KEY","EnvType":"string","EnvValue":"dedicated","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_NODE_TAINTS_VALUE","EnvType":"string","EnvValue":"ci","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_REQ_CI_CPU","EnvType":"string","EnvValue":"0.5","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_REQ_CI_MEM","EnvType":"string","EnvValue":"3G","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_WORKFLOW_EXECUTOR_TYPE","EnvType":"","EnvValue":"AWF","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_WORKFLOW_SERVICE_ACCOUNT","EnvType":"string","EnvValue":"cd-runner","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CI_DEFAULT_ADDRESS_POOL_BASE_CIDR","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CI_DEFAULT_ADDRESS_POOL_SIZE","EnvType":"int","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CI_IGNORE_DOCKER_CACHE","EnvType":"bool","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CI_LOGS_KEY_PREFIX","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CI_NODE_LABEL_SELECTOR","EnvType":"","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CI_NODE_TAINTS_KEY","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CI_NODE_TAINTS_VALUE","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CI_RUNNER_DOCKER_MTU_VALUE","EnvType":"int","EnvValue":"-1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CI_SUCCESS_AUTO_TRIGGER_BATCH_SIZE","EnvType":"int","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CI_VOLUME_MOUNTS_JSON","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CI_WORKFLOW_EXECUTOR_TYPE","EnvType":"","EnvValue":"AWF","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_ARTIFACT_KEY_LOCATION","EnvType":"string","EnvValue":"arsenal-v1/ci-artifacts","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_BUILD_LOGS_BUCKET","EnvType":"string","EnvValue":"devtron-pro-ci-logs","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_BUILD_LOGS_KEY_PREFIX","EnvType":"string","EnvValue":"arsenal-v1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_CACHE_BUCKET","EnvType":"string","EnvValue":"ci-caching","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_CACHE_BUCKET_REGION","EnvType":"string","EnvValue":"us-east-2","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_CD_ARTIFACT_KEY_LOCATION","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_CD_LOGS_BUCKET_REGION","EnvType":"string","EnvValue":"us-east-2","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_CD_NAMESPACE","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_CD_TIMEOUT","EnvType":"int64","EnvValue":"3600","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_CI_IMAGE","EnvType":"string","EnvValue":"686244538589.dkr.ecr.us-east-2.amazonaws.com/cirunner:47","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_NAMESPACE","EnvType":"string","EnvValue":"devtron-ci","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_TARGET_PLATFORM","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DOCKER_BUILD_CACHE_PATH","EnvType":"string","EnvValue":"/var/lib/docker","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ENABLE_BUILD_CONTEXT","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ENABLE_WORKFLOW_EXECUTION_STAGE","EnvType":"bool","EnvValue":"true","EnvDescription":"if enabled then we will display build stages separately for CI/Job/Pre-Post CD","Example":"true","Deprecated":"false"},{"Env":"EXTERNAL_BLOB_STORAGE_CM_NAME","EnvType":"string","EnvValue":"blob-storage-cm","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"EXTERNAL_BLOB_STORAGE_SECRET_NAME","EnvType":"string","EnvValue":"blob-storage-secret","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"EXTERNAL_CD_NODE_LABEL_SELECTOR","EnvType":"","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"EXTERNAL_CD_NODE_TAINTS_KEY","EnvType":"string","EnvValue":"dedicated","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"EXTERNAL_CD_NODE_TAINTS_VALUE","EnvType":"string","EnvValue":"ci","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"EXTERNAL_CI_API_SECRET","EnvType":"string","EnvValue":"devtroncd-secret","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"EXTERNAL_CI_PAYLOAD","EnvType":"string","EnvValue":"{\"ciProjectDetails\":[{\"gitRepository\":\"https://github.com/vikram1601/getting-started-nodejs.git\",\"checkoutPath\":\"./abc\",\"commitHash\":\"239077135f8cdeeccb7857e2851348f558cb53d3\",\"commitTime\":\"2022-10-30T20:00:00\",\"branch\":\"master\",\"message\":\"Update README.md\",\"author\":\"User Name \"}],\"dockerImage\":\"445808685819.dkr.ecr.us-east-2.amazonaws.com/orch:23907713-2\"}","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"EXTERNAL_CI_WEB_HOOK_URL","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"IGNORE_CM_CS_IN_CI_JOB","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"IMAGE_RETRY_COUNT","EnvType":"int","EnvValue":"0","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"IMAGE_RETRY_INTERVAL","EnvType":"int","EnvValue":"5","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"IMAGE_SCANNER_ENDPOINT","EnvType":"string","EnvValue":"http://image-scanner-new-demo-devtroncd-service.devtroncd:80","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"IMAGE_SCAN_MAX_RETRIES","EnvType":"int","EnvValue":"3","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"IMAGE_SCAN_RETRY_DELAY","EnvType":"int","EnvValue":"5","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"IN_APP_LOGGING_ENABLED","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"MAX_CD_WORKFLOW_RUNNER_RETRIES","EnvType":"int","EnvValue":"0","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"MAX_CI_WORKFLOW_RETRIES","EnvType":"int","EnvValue":"0","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"MODE","EnvType":"string","EnvValue":"DEV","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_SERVER_HOST","EnvType":"string","EnvValue":"localhost:4222","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ORCH_HOST","EnvType":"string","EnvValue":"http://devtroncd-orchestrator-service-prod.devtroncd/webhook/msg/nats","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ORCH_TOKEN","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PRE_CI_CACHE_PATH","EnvType":"string","EnvValue":"/devtroncd-cache","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"SHOW_DOCKER_BUILD_ARGS","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"SKIP_CI_JOB_BUILD_CACHE_PUSH_PULL","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"SKIP_CREATING_ECR_REPO","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TERMINATION_GRACE_PERIOD_SECS","EnvType":"int","EnvValue":"180","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"USE_ARTIFACT_LISTING_QUERY_V2","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"USE_BLOB_STORAGE_CONFIG_IN_CD_WORKFLOW","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"USE_BLOB_STORAGE_CONFIG_IN_CI_WORKFLOW","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"USE_BUILDX","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"USE_DOCKER_API_TO_GET_DIGEST","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"USE_EXTERNAL_NODE","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"USE_IMAGE_TAG_FROM_GIT_PROVIDER_FOR_TAG_BASED_BUILD","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"WF_CONTROLLER_INSTANCE_ID","EnvType":"string","EnvValue":"devtron-runner","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"WORKFLOW_CACHE_CONFIG","EnvType":"string","EnvValue":"{}","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"WORKFLOW_SERVICE_ACCOUNT","EnvType":"string","EnvValue":"ci-runner","EnvDescription":"","Example":"","Deprecated":"false"}]},{"Category":"DEVTRON","Fields":[{"Env":"-","EnvType":"","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ADDITIONAL_NODE_GROUP_LABELS","EnvType":"","EnvValue":"","EnvDescription":"Add comma separated list of additional node group labels to default labels","Example":"karpenter.sh/nodepool,cloud.google.com/gke-nodepool","Deprecated":"false"},{"Env":"APP_SYNC_IMAGE","EnvType":"string","EnvValue":"quay.io/devtron/chart-sync:1227622d-132-3775","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"APP_SYNC_JOB_RESOURCES_OBJ","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"APP_SYNC_SERVICE_ACCOUNT","EnvType":"string","EnvValue":"chart-sync","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"APP_SYNC_SHUTDOWN_WAIT_DURATION","EnvType":"int","EnvValue":"120","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ARGO_AUTO_SYNC_ENABLED","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ARGO_GIT_COMMIT_RETRY_COUNT_ON_CONFLICT","EnvType":"int","EnvValue":"3","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ARGO_GIT_COMMIT_RETRY_DELAY_ON_CONFLICT","EnvType":"int","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ARGO_REPO_REGISTER_RETRY_COUNT","EnvType":"int","EnvValue":"3","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ARGO_REPO_REGISTER_RETRY_DELAY","EnvType":"int","EnvValue":"10","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ASYNC_BUILDX_CACHE_EXPORT","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BATCH_SIZE","EnvType":"int","EnvValue":"5","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_ENABLED","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BUILDX_CACHE_MODE_MIN","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_HOST","EnvType":"string","EnvValue":"localhost","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_NAMESPACE","EnvType":"string","EnvValue":"devtroncd","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_PORT","EnvType":"string","EnvValue":"8000","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CExpirationTime","EnvType":"int","EnvValue":"600","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CI_TRIGGER_CRON_TIME","EnvType":"int","EnvValue":"2","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CI_WORKFLOW_STATUS_UPDATE_CRON","EnvType":"string","EnvValue":"*/5 * * * *","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CLI_CMD_TIMEOUT_GLOBAL_SECONDS","EnvType":"int","EnvValue":"0","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CLUSTER_STATUS_CRON_TIME","EnvType":"int","EnvValue":"15","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CONSUMER_CONFIG_JSON","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_LOG_TIME_LIMIT","EnvType":"int64","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_TIMEOUT","EnvType":"float64","EnvValue":"3600","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEVTRON_BOM_URL","EnvType":"string","EnvValue":"https://raw.githubusercontent.com/devtron-labs/devtron/%s/charts/devtron/devtron-bom.yaml","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEVTRON_DEFAULT_NAMESPACE","EnvType":"string","EnvValue":"devtroncd","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEVTRON_DEX_SECRET_NAMESPACE","EnvType":"string","EnvValue":"devtroncd","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEVTRON_HELM_RELEASE_CHART_NAME","EnvType":"string","EnvValue":"devtron-operator","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEVTRON_HELM_RELEASE_NAME","EnvType":"string","EnvValue":"devtron","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEVTRON_HELM_RELEASE_NAMESPACE","EnvType":"string","EnvValue":"devtroncd","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEVTRON_HELM_REPO_NAME","EnvType":"string","EnvValue":"devtron","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEVTRON_HELM_REPO_URL","EnvType":"string","EnvValue":"https://helm.devtron.ai","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEVTRON_INSTALLATION_TYPE","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEVTRON_MODULES_IDENTIFIER_IN_HELM_VALUES","EnvType":"string","EnvValue":"installer.modules","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEVTRON_SECRET_NAME","EnvType":"string","EnvValue":"devtron-secret","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEVTRON_VERSION_IDENTIFIER_IN_HELM_VALUES","EnvType":"string","EnvValue":"installer.release","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEX_CID","EnvType":"string","EnvValue":"example-app","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEX_CLIENT_ID","EnvType":"string","EnvValue":"argo-cd","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEX_CSTOREKEY","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEX_JWTKEY","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEX_RURL","EnvType":"string","EnvValue":"http://127.0.0.1:8080/callback","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEX_SECRET","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEX_URL","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ECR_REPO_NAME_PREFIX","EnvType":"string","EnvValue":"test/","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ENABLE_ASYNC_ARGO_CD_INSTALL_DEVTRON_CHART","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ENABLE_ASYNC_INSTALL_DEVTRON_CHART","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"EPHEMERAL_SERVER_VERSION_REGEX","EnvType":"string","EnvValue":"v[1-9]\\.\\b(2[3-9]\\|[3-9][0-9])\\b.*","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"EVENT_URL","EnvType":"string","EnvValue":"http://localhost:3000/notify","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"EXECUTE_WIRE_NIL_CHECKER","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"EXPOSE_CI_METRICS","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"FEATURE_RESTART_WORKLOAD_BATCH_SIZE","EnvType":"int","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"FEATURE_RESTART_WORKLOAD_WORKER_POOL_SIZE","EnvType":"int","EnvValue":"5","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"FORCE_SECURITY_SCANNING","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"GITOPS_REPO_PREFIX","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"GO_RUNTIME_ENV","EnvType":"string","EnvValue":"production","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"GRAFANA_HOST","EnvType":"string","EnvValue":"localhost","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"GRAFANA_NAMESPACE","EnvType":"string","EnvValue":"devtroncd","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"GRAFANA_ORG_ID","EnvType":"int","EnvValue":"2","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"GRAFANA_PASSWORD","EnvType":"string","EnvValue":"prom-operator","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"GRAFANA_PORT","EnvType":"string","EnvValue":"8090","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"GRAFANA_URL","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"GRAFANA_USERNAME","EnvType":"string","EnvValue":"admin","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"HIDE_IMAGE_TAGGING_HARD_DELETE","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"IGNORE_AUTOCOMPLETE_AUTH_CHECK","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"INSTALLED_MODULES","EnvType":"","EnvValue":"","EnvDescription":"List of installed modules given in helm values/yaml are written in cm and used by devtron to know which modules are given","Example":"security.trivy,security.clair","Deprecated":"false"},{"Env":"INSTALLER_CRD_NAMESPACE","EnvType":"string","EnvValue":"devtroncd","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"INSTALLER_CRD_OBJECT_GROUP_NAME","EnvType":"string","EnvValue":"installer.devtron.ai","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"INSTALLER_CRD_OBJECT_RESOURCE","EnvType":"string","EnvValue":"installers","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"INSTALLER_CRD_OBJECT_VERSION","EnvType":"string","EnvValue":"v1alpha1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"IS_AIR_GAP_ENVIRONMENT","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"JwtExpirationTime","EnvType":"int","EnvValue":"120","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"K8s_CLIENT_MAX_IDLE_CONNS_PER_HOST","EnvType":"int","EnvValue":"25","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"K8s_TCP_IDLE_CONN_TIMEOUT","EnvType":"int","EnvValue":"300","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"K8s_TCP_KEEPALIVE","EnvType":"int","EnvValue":"30","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"K8s_TCP_TIMEOUT","EnvType":"int","EnvValue":"30","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"K8s_TLS_HANDSHAKE_TIMEOUT","EnvType":"int","EnvValue":"10","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"KUBELINK_GRPC_MAX_RECEIVE_MSG_SIZE","EnvType":"int","EnvValue":"20","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"KUBELINK_GRPC_MAX_SEND_MSG_SIZE","EnvType":"int","EnvValue":"4","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"LENS_TIMEOUT","EnvType":"int","EnvValue":"0","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"LENS_URL","EnvType":"string","EnvValue":"http://lens-milandevtron-service:80","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"LIMIT_CI_CPU","EnvType":"string","EnvValue":"0.5","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"LIMIT_CI_MEM","EnvType":"string","EnvValue":"3G","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"LOGGER_DEV_MODE","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"LOG_LEVEL","EnvType":"int","EnvValue":"-1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"MAX_SESSION_PER_USER","EnvType":"int","EnvValue":"5","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"MODULE_METADATA_API_URL","EnvType":"string","EnvValue":"https://api.devtron.ai/module?name=%s","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"MODULE_STATUS_HANDLING_CRON_DURATION_MIN","EnvType":"int","EnvValue":"3","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_ACK_WAIT_IN_SECS","EnvType":"int","EnvValue":"120","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_BUFFER_SIZE","EnvType":"int","EnvValue":"-1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_MAX_AGE","EnvType":"int","EnvValue":"86400","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_PROCESSING_BATCH_SIZE","EnvType":"int","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_REPLICAS","EnvType":"int","EnvValue":"0","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NOTIFICATION_MEDIUM","EnvType":"NotificationMedium","EnvValue":"rest","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"OTEL_COLLECTOR_URL","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PARALLELISM_LIMIT_FOR_TAG_PROCESSING","EnvType":"int","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_EXPORT_PROM_METRICS","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_LOG_ALL_FAILURE_QUERIES","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_LOG_ALL_QUERY","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_LOG_SLOW_QUERY","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_QUERY_DUR_THRESHOLD","EnvType":"int64","EnvValue":"5000","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PLUGIN_NAME","EnvType":"string","EnvValue":"Pull images from container repository","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PROPAGATE_EXTRA_LABELS","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PROXY_SERVICE_CONFIG","EnvType":"string","EnvValue":"{}","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"REQ_CI_CPU","EnvType":"string","EnvValue":"0.5","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"REQ_CI_MEM","EnvType":"string","EnvValue":"3G","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"RESTRICT_TERMINAL_ACCESS_FOR_NON_SUPER_USER","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"RUNTIME_CONFIG_LOCAL_DEV","EnvType":"LocalDevMode","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"SCOPED_VARIABLE_ENABLED","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"SCOPED_VARIABLE_FORMAT","EnvType":"string","EnvValue":"@{{%s}}","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"SCOPED_VARIABLE_HANDLE_PRIMITIVES","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"SCOPED_VARIABLE_NAME_REGEX","EnvType":"string","EnvValue":"^[a-zA-Z][a-zA-Z0-9_-]{0,62}[a-zA-Z0-9]$","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"SOCKET_DISCONNECT_DELAY_SECONDS","EnvType":"int","EnvValue":"5","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"SOCKET_HEARTBEAT_SECONDS","EnvType":"int","EnvValue":"25","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"STREAM_CONFIG_JSON","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"SYSTEM_VAR_PREFIX","EnvType":"string","EnvValue":"DEVTRON_","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TERMINAL_POD_DEFAULT_NAMESPACE","EnvType":"string","EnvValue":"default","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TERMINAL_POD_INACTIVE_DURATION_IN_MINS","EnvType":"int","EnvValue":"10","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TERMINAL_POD_STATUS_SYNC_In_SECS","EnvType":"int","EnvValue":"600","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TEST_APP","EnvType":"string","EnvValue":"orchestrator","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TEST_PG_ADDR","EnvType":"string","EnvValue":"127.0.0.1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TEST_PG_DATABASE","EnvType":"string","EnvValue":"orchestrator","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TEST_PG_LOG_QUERY","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TEST_PG_PASSWORD","EnvType":"string","EnvValue":"postgrespw","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TEST_PG_PORT","EnvType":"string","EnvValue":"55000","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TEST_PG_USER","EnvType":"string","EnvValue":"postgres","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TIMEOUT_FOR_FAILED_CI_BUILD","EnvType":"string","EnvValue":"15","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TIMEOUT_IN_SECONDS","EnvType":"int","EnvValue":"5","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"USER_SESSION_DURATION_SECONDS","EnvType":"int","EnvValue":"86400","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"USE_ARTIFACT_LISTING_API_V2","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"USE_CUSTOM_HTTP_TRANSPORT","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"USE_GIT_CLI","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"USE_RBAC_CREATION_V2","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"VARIABLE_CACHE_ENABLED","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"VARIABLE_EXPRESSION_REGEX","EnvType":"string","EnvValue":"@{{([^}]+)}}","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"WEBHOOK_TOKEN","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"}]},{"Category":"GITOPS","Fields":[{"Env":"ACD_CM","EnvType":"string","EnvValue":"argocd-cm","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ACD_NAMESPACE","EnvType":"string","EnvValue":"devtroncd","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ACD_PASSWORD","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ACD_USERNAME","EnvType":"string","EnvValue":"admin","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"GITOPS_SECRET_NAME","EnvType":"string","EnvValue":"devtron-gitops-secret","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"RESOURCE_LIST_FOR_REPLICAS","EnvType":"string","EnvValue":"Deployment,Rollout,StatefulSet,ReplicaSet","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"RESOURCE_LIST_FOR_REPLICAS_BATCH_SIZE","EnvType":"int","EnvValue":"5","EnvDescription":"","Example":"","Deprecated":"false"}]},{"Category":"INFRA_SETUP","Fields":[{"Env":"DASHBOARD_HOST","EnvType":"string","EnvValue":"localhost","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DASHBOARD_NAMESPACE","EnvType":"string","EnvValue":"devtroncd","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DASHBOARD_PORT","EnvType":"string","EnvValue":"3000","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEX_HOST","EnvType":"string","EnvValue":"http://localhost","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEX_PORT","EnvType":"string","EnvValue":"5556","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"GIT_SENSOR_PROTOCOL","EnvType":"string","EnvValue":"REST","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"GIT_SENSOR_TIMEOUT","EnvType":"int","EnvValue":"0","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"GIT_SENSOR_URL","EnvType":"string","EnvValue":"127.0.0.1:7070","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"HELM_CLIENT_URL","EnvType":"string","EnvValue":"127.0.0.1:50051","EnvDescription":"","Example":"","Deprecated":"false"}]},{"Category":"POSTGRES","Fields":[{"Env":"APP","EnvType":"string","EnvValue":"orchestrator","EnvDescription":"Application name","Example":"","Deprecated":"false"},{"Env":"CASBIN_DATABASE","EnvType":"string","EnvValue":"casbin","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_ADDR","EnvType":"string","EnvValue":"127.0.0.1","EnvDescription":"address of postgres service","Example":"postgresql-postgresql.devtroncd","Deprecated":"false"},{"Env":"PG_DATABASE","EnvType":"string","EnvValue":"orchestrator","EnvDescription":"postgres database to be made connection with","Example":"orchestrator, casbin, git_sensor, lens","Deprecated":"false"},{"Env":"PG_PASSWORD","EnvType":"string","EnvValue":"{password}","EnvDescription":"password for postgres, associated with PG_USER","Example":"confidential ;)","Deprecated":"false"},{"Env":"PG_PORT","EnvType":"string","EnvValue":"5432","EnvDescription":"port of postgresql service","Example":"5432","Deprecated":"false"},{"Env":"PG_READ_TIMEOUT","EnvType":"int64","EnvValue":"30","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_USER","EnvType":"string","EnvValue":"postgres","EnvDescription":"user for postgres","Example":"postgres","Deprecated":"false"},{"Env":"PG_WRITE_TIMEOUT","EnvType":"int64","EnvValue":"30","EnvDescription":"","Example":"","Deprecated":"false"}]},{"Category":"RBAC","Fields":[{"Env":"ENFORCER_CACHE","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ENFORCER_CACHE_EXPIRATION_IN_SEC","EnvType":"int","EnvValue":"86400","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ENFORCER_MAX_BATCH_SIZE","EnvType":"int","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"USE_CASBIN_V2","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"}]}] \ No newline at end of file +[{"Category":"CD","Fields":[{"Env":"ARGO_APP_MANUAL_SYNC_TIME","EnvType":"int","EnvValue":"3","EnvDescription":"retry argocd app manual sync if the timeline is stuck in ARGOCD_SYNC_INITIATED state for more than this defined time (in mins)","Example":"","Deprecated":"false"},{"Env":"CD_HELM_PIPELINE_STATUS_CRON_TIME","EnvType":"string","EnvValue":"*/2 * * * *","EnvDescription":"Cron time to check the pipeline status ","Example":"","Deprecated":"false"},{"Env":"CD_PIPELINE_STATUS_CRON_TIME","EnvType":"string","EnvValue":"*/2 * * * *","EnvDescription":"Cron time for CD pipeline status","Example":"","Deprecated":"false"},{"Env":"CD_PIPELINE_STATUS_TIMEOUT_DURATION","EnvType":"string","EnvValue":"20","EnvDescription":"Timeout for CD pipeline to get healthy","Example":"","Deprecated":"false"},{"Env":"DEPLOY_STATUS_CRON_GET_PIPELINE_DEPLOYED_WITHIN_HOURS","EnvType":"int","EnvValue":"12","EnvDescription":"This flag is used to fetch the deployment status of the application. It retrieves the status of deployments that occurred between 12 hours and 10 minutes prior to the current time. It fetches non-terminal statuses.","Example":"","Deprecated":"false"},{"Env":"DEVTRON_CHART_ARGO_CD_INSTALL_REQUEST_TIMEOUT","EnvType":"int","EnvValue":"1","EnvDescription":"Context timeout for gitops concurrent async deployments","Example":"","Deprecated":"false"},{"Env":"DEVTRON_CHART_INSTALL_REQUEST_TIMEOUT","EnvType":"int","EnvValue":"6","EnvDescription":"Context timeout for no gitops concurrent async deployments","Example":"","Deprecated":"false"},{"Env":"EXPOSE_CD_METRICS","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"FEATURE_MIGRATE_ARGOCD_APPLICATION_ENABLE","EnvType":"bool","EnvValue":"false","EnvDescription":"enable migration of external argocd application to devtron pipeline","Example":"","Deprecated":"false"},{"Env":"HELM_PIPELINE_STATUS_CHECK_ELIGIBLE_TIME","EnvType":"string","EnvValue":"120","EnvDescription":"eligible time for checking helm app status periodically and update in db, value is in seconds., default is 120, if wfr is updated within configured time i.e. HELM_PIPELINE_STATUS_CHECK_ELIGIBLE_TIME then do not include for this cron cycle.","Example":"","Deprecated":"false"},{"Env":"IS_INTERNAL_USE","EnvType":"bool","EnvValue":"true","EnvDescription":"If enabled then cd pipeline and helm apps will not need the deployment app type mandatorily. Couple this flag with HIDE_GITOPS_OR_HELM_OPTION (in Dashborad) and if gitops is configured and allowed for the env, pipeline/ helm app will gitops else no-gitops.","Example":"","Deprecated":"false"},{"Env":"MIGRATE_DEPLOYMENT_CONFIG_DATA","EnvType":"bool","EnvValue":"false","EnvDescription":"migrate deployment config data from charts table to deployment_config table","Example":"","Deprecated":"false"},{"Env":"PIPELINE_DEGRADED_TIME","EnvType":"string","EnvValue":"10","EnvDescription":"Time to mark a pipeline degraded if not healthy in defined time","Example":"","Deprecated":"false"},{"Env":"REVISION_HISTORY_LIMIT_DEVTRON_APP","EnvType":"int","EnvValue":"1","EnvDescription":"Count for devtron application rivision history","Example":"","Deprecated":"false"},{"Env":"REVISION_HISTORY_LIMIT_EXTERNAL_HELM_APP","EnvType":"int","EnvValue":"0","EnvDescription":"Count for external helm application rivision history","Example":"","Deprecated":"false"},{"Env":"REVISION_HISTORY_LIMIT_HELM_APP","EnvType":"int","EnvValue":"1","EnvDescription":"To set the history limit for the helm app being deployed through devtron","Example":"","Deprecated":"false"},{"Env":"REVISION_HISTORY_LIMIT_LINKED_HELM_APP","EnvType":"int","EnvValue":"15","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"RUN_HELM_INSTALL_IN_ASYNC_MODE_HELM_APPS","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"SHOULD_CHECK_NAMESPACE_ON_CLONE","EnvType":"bool","EnvValue":"false","EnvDescription":"should we check if namespace exists or not while cloning app","Example":"","Deprecated":"false"},{"Env":"USE_DEPLOYMENT_CONFIG_DATA","EnvType":"bool","EnvValue":"false","EnvDescription":"use deployment config data from deployment_config table","Example":"","Deprecated":"true"}]},{"Category":"CI_RUNNER","Fields":[{"Env":"AZURE_ACCOUNT_KEY","EnvType":"string","EnvValue":"","EnvDescription":"If blob storage is bieng used of azure then pass the secret key to access the bucket","Example":"","Deprecated":"false"},{"Env":"AZURE_ACCOUNT_NAME","EnvType":"string","EnvValue":"","EnvDescription":"Account name for azure blob storage","Example":"","Deprecated":"false"},{"Env":"AZURE_BLOB_CONTAINER_CI_CACHE","EnvType":"string","EnvValue":"","EnvDescription":"Cache bucket name for azure blob storage","Example":"","Deprecated":"false"},{"Env":"AZURE_BLOB_CONTAINER_CI_LOG","EnvType":"string","EnvValue":"","EnvDescription":"Log bucket for azure blob storage","Example":"","Deprecated":"false"},{"Env":"AZURE_GATEWAY_CONNECTION_INSECURE","EnvType":"bool","EnvValue":"true","EnvDescription":"Azure gateway connection allows insecure if true","Example":"","Deprecated":"false"},{"Env":"AZURE_GATEWAY_URL","EnvType":"string","EnvValue":"http://devtron-minio.devtroncd:9000","EnvDescription":"Sent to CI runner for blob","Example":"","Deprecated":"false"},{"Env":"BASE_LOG_LOCATION_PATH","EnvType":"string","EnvValue":"/home/devtron/","EnvDescription":"Used to store, download logs of ci workflow, artifact","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_GCP_CREDENTIALS_JSON","EnvType":"string","EnvValue":"","EnvDescription":"GCP cred json for GCS blob storage","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_PROVIDER","EnvType":"","EnvValue":"S3","EnvDescription":"Blob storage provider name(AWS/GCP/Azure)","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_S3_ACCESS_KEY","EnvType":"string","EnvValue":"","EnvDescription":"S3 access key for s3 blob storage","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_S3_BUCKET_VERSIONED","EnvType":"bool","EnvValue":"true","EnvDescription":"To enable buctet versioning for blob storage","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_S3_ENDPOINT","EnvType":"string","EnvValue":"","EnvDescription":"S3 endpoint URL for s3 blob storage","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_S3_ENDPOINT_INSECURE","EnvType":"bool","EnvValue":"false","EnvDescription":"To use insecure s3 endpoint","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_S3_SECRET_KEY","EnvType":"string","EnvValue":"","EnvDescription":"Secret key for s3 blob storage","Example":"","Deprecated":"false"},{"Env":"BUILDX_CACHE_PATH","EnvType":"string","EnvValue":"/var/lib/devtron/buildx","EnvDescription":"Path for the buildx cache","Example":"","Deprecated":"false"},{"Env":"BUILDX_K8S_DRIVER_OPTIONS","EnvType":"string","EnvValue":"","EnvDescription":"To enable the k8s driver and pass args for k8s driver in buildx","Example":"","Deprecated":"false"},{"Env":"BUILDX_PROVENANCE_MODE","EnvType":"string","EnvValue":"","EnvDescription":"provinance is set to true by default by docker. this will add some build related data in generated build manifest.it also adds some unknown:unknown key:value pair which may not be compatible by some container registries. with buildx k8s driver , provinenance=true is causing issue when push manifest to quay registry, so setting it to false","Example":"","Deprecated":"false"},{"Env":"BUILD_LOG_TTL_VALUE_IN_SECS","EnvType":"int","EnvValue":"3600","EnvDescription":"This is the time that the pods of ci/pre-cd/post-cd live after completion state.","Example":"","Deprecated":"false"},{"Env":"CACHE_LIMIT","EnvType":"int64","EnvValue":"5000000000","EnvDescription":"Cache limit.","Example":"","Deprecated":"false"},{"Env":"CD_DEFAULT_ADDRESS_POOL_BASE_CIDR","EnvType":"string","EnvValue":"","EnvDescription":"To pass the IP cidr for Pre/Post cd ","Example":"","Deprecated":"false"},{"Env":"CD_DEFAULT_ADDRESS_POOL_SIZE","EnvType":"int","EnvValue":"","EnvDescription":"The subnet size to allocate from the base pool for CD","Example":"","Deprecated":"false"},{"Env":"CD_LIMIT_CI_CPU","EnvType":"string","EnvValue":"0.5","EnvDescription":"CPU Resource Limit Pre/Post CD","Example":"","Deprecated":"false"},{"Env":"CD_LIMIT_CI_MEM","EnvType":"string","EnvValue":"3G","EnvDescription":"Memory Resource Limit Pre/Post CD","Example":"","Deprecated":"false"},{"Env":"CD_NODE_LABEL_SELECTOR","EnvType":"","EnvValue":"","EnvDescription":"Node label selector for Pre/Post CD","Example":"","Deprecated":"false"},{"Env":"CD_NODE_TAINTS_KEY","EnvType":"string","EnvValue":"dedicated","EnvDescription":"Toleration key for Pre/Post CD","Example":"","Deprecated":"false"},{"Env":"CD_NODE_TAINTS_VALUE","EnvType":"string","EnvValue":"ci","EnvDescription":"Toleration value for Pre/Post CD","Example":"","Deprecated":"false"},{"Env":"CD_REQ_CI_CPU","EnvType":"string","EnvValue":"0.5","EnvDescription":"CPU Resource Rquest Pre/Post CD","Example":"","Deprecated":"false"},{"Env":"CD_REQ_CI_MEM","EnvType":"string","EnvValue":"3G","EnvDescription":"Memory Resource Rquest Pre/Post CD","Example":"","Deprecated":"false"},{"Env":"CD_WORKFLOW_EXECUTOR_TYPE","EnvType":"","EnvValue":"AWF","EnvDescription":"Executor type for Pre/Post CD(AWF,System)","Example":"","Deprecated":"false"},{"Env":"CD_WORKFLOW_SERVICE_ACCOUNT","EnvType":"string","EnvValue":"cd-runner","EnvDescription":"Service account to be used in Pre/Post CD pod","Example":"","Deprecated":"false"},{"Env":"CI_DEFAULT_ADDRESS_POOL_BASE_CIDR","EnvType":"string","EnvValue":"","EnvDescription":"To pass the IP cidr for CI","Example":"","Deprecated":"false"},{"Env":"CI_DEFAULT_ADDRESS_POOL_SIZE","EnvType":"int","EnvValue":"","EnvDescription":"The subnet size to allocate from the base pool for CI","Example":"","Deprecated":"false"},{"Env":"CI_IGNORE_DOCKER_CACHE","EnvType":"bool","EnvValue":"","EnvDescription":"Ignoring docker cache ","Example":"","Deprecated":"false"},{"Env":"CI_LOGS_KEY_PREFIX","EnvType":"string","EnvValue":"","EnvDescription":"Prefix for build logs","Example":"","Deprecated":"false"},{"Env":"CI_NODE_LABEL_SELECTOR","EnvType":"","EnvValue":"","EnvDescription":"Node label selector for CI","Example":"","Deprecated":"false"},{"Env":"CI_NODE_TAINTS_KEY","EnvType":"string","EnvValue":"","EnvDescription":"Toleration key for CI","Example":"","Deprecated":"false"},{"Env":"CI_NODE_TAINTS_VALUE","EnvType":"string","EnvValue":"","EnvDescription":"Toleration value for CI","Example":"","Deprecated":"false"},{"Env":"CI_RUNNER_DOCKER_MTU_VALUE","EnvType":"int","EnvValue":"-1","EnvDescription":"this is to control the bytes of inofrmation passed in a network packet in ci-runner. default is -1 (defaults to the underlying node mtu value)","Example":"","Deprecated":"false"},{"Env":"CI_SUCCESS_AUTO_TRIGGER_BATCH_SIZE","EnvType":"int","EnvValue":"1","EnvDescription":"this is to control the no of linked pipelines should be hanled in one go when a ci-success event of an parent ci is received","Example":"","Deprecated":"false"},{"Env":"CI_VOLUME_MOUNTS_JSON","EnvType":"string","EnvValue":"","EnvDescription":"additional volume mount data for CI and JOB","Example":"","Deprecated":"false"},{"Env":"CI_WORKFLOW_EXECUTOR_TYPE","EnvType":"","EnvValue":"AWF","EnvDescription":"Executor type for CI(AWF,System)","Example":"","Deprecated":"false"},{"Env":"DEFAULT_ARTIFACT_KEY_LOCATION","EnvType":"string","EnvValue":"arsenal-v1/ci-artifacts","EnvDescription":"Key location for artifacts being created","Example":"","Deprecated":"false"},{"Env":"DEFAULT_BUILD_LOGS_BUCKET","EnvType":"string","EnvValue":"devtron-pro-ci-logs","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_BUILD_LOGS_KEY_PREFIX","EnvType":"string","EnvValue":"arsenal-v1","EnvDescription":"Bucket prefix for build logs","Example":"","Deprecated":"false"},{"Env":"DEFAULT_CACHE_BUCKET","EnvType":"string","EnvValue":"ci-caching","EnvDescription":"Bucket name for build cache","Example":"","Deprecated":"false"},{"Env":"DEFAULT_CACHE_BUCKET_REGION","EnvType":"string","EnvValue":"us-east-2","EnvDescription":"Build Cache bucket region","Example":"","Deprecated":"false"},{"Env":"DEFAULT_CD_ARTIFACT_KEY_LOCATION","EnvType":"string","EnvValue":"","EnvDescription":"Bucket prefix for build cache","Example":"","Deprecated":"false"},{"Env":"DEFAULT_CD_LOGS_BUCKET_REGION","EnvType":"string","EnvValue":"us-east-2","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_CD_NAMESPACE","EnvType":"string","EnvValue":"","EnvDescription":"Namespace for devtron stack","Example":"","Deprecated":"false"},{"Env":"DEFAULT_CD_TIMEOUT","EnvType":"int64","EnvValue":"3600","EnvDescription":"Timeout for Pre/Post-Cd to be completed","Example":"","Deprecated":"false"},{"Env":"DEFAULT_CI_IMAGE","EnvType":"string","EnvValue":"686244538589.dkr.ecr.us-east-2.amazonaws.com/cirunner:47","EnvDescription":"To pass the ci-runner image","Example":"","Deprecated":"false"},{"Env":"DEFAULT_NAMESPACE","EnvType":"string","EnvValue":"devtron-ci","EnvDescription":"Timeout for CI to be completed","Example":"","Deprecated":"false"},{"Env":"DEFAULT_TARGET_PLATFORM","EnvType":"string","EnvValue":"","EnvDescription":"Default architecture for buildx","Example":"","Deprecated":"false"},{"Env":"DOCKER_BUILD_CACHE_PATH","EnvType":"string","EnvValue":"/var/lib/docker","EnvDescription":"Path to store cache of docker build (/var/lib/docker-\u003e for legacy docker build, /var/lib/devtron-\u003e for buildx)","Example":"","Deprecated":"false"},{"Env":"ENABLE_BUILD_CONTEXT","EnvType":"bool","EnvValue":"false","EnvDescription":"To Enable build context in Devtron.","Example":"","Deprecated":"false"},{"Env":"ENABLE_WORKFLOW_EXECUTION_STAGE","EnvType":"bool","EnvValue":"true","EnvDescription":"if enabled then we will display build stages separately for CI/Job/Pre-Post CD","Example":"true","Deprecated":"false"},{"Env":"EXTERNAL_BLOB_STORAGE_CM_NAME","EnvType":"string","EnvValue":"blob-storage-cm","EnvDescription":"name of the config map(contains bucket name, etc.) in external cluster when there is some operation related to external cluster, for example:-downloading cd artifact pushed in external cluster's env and we need to download from there, downloads ci logs pushed in external cluster's blob","Example":"","Deprecated":"false"},{"Env":"EXTERNAL_BLOB_STORAGE_SECRET_NAME","EnvType":"string","EnvValue":"blob-storage-secret","EnvDescription":"name of the secret(contains password, accessId,passKeys, etc.) in external cluster when there is some operation related to external cluster, for example:-downloading cd artifact pushed in external cluster's env and we need to download from there, downloads ci logs pushed in external cluster's blob","Example":"","Deprecated":"false"},{"Env":"EXTERNAL_CD_NODE_LABEL_SELECTOR","EnvType":"","EnvValue":"","EnvDescription":"This is an array of strings used when submitting a workflow for pre or post-CD execution. If the ","Example":"","Deprecated":"false"},{"Env":"EXTERNAL_CD_NODE_TAINTS_KEY","EnvType":"string","EnvValue":"dedicated","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"EXTERNAL_CD_NODE_TAINTS_VALUE","EnvType":"string","EnvValue":"ci","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"EXTERNAL_CI_API_SECRET","EnvType":"string","EnvValue":"devtroncd-secret","EnvDescription":"External CI API secret.","Example":"","Deprecated":"false"},{"Env":"EXTERNAL_CI_PAYLOAD","EnvType":"string","EnvValue":"{\"ciProjectDetails\":[{\"gitRepository\":\"https://github.com/vikram1601/getting-started-nodejs.git\",\"checkoutPath\":\"./abc\",\"commitHash\":\"239077135f8cdeeccb7857e2851348f558cb53d3\",\"commitTime\":\"2022-10-30T20:00:00\",\"branch\":\"master\",\"message\":\"Update README.md\",\"author\":\"User Name \"}],\"dockerImage\":\"445808685819.dkr.ecr.us-east-2.amazonaws.com/orch:23907713-2\"}","EnvDescription":"External CI payload with project details.","Example":"","Deprecated":"false"},{"Env":"EXTERNAL_CI_WEB_HOOK_URL","EnvType":"string","EnvValue":"","EnvDescription":"default is {{HOST_URL}}/orchestrator/webhook/ext-ci. It is used for external ci.","Example":"","Deprecated":"false"},{"Env":"IGNORE_CM_CS_IN_CI_JOB","EnvType":"bool","EnvValue":"false","EnvDescription":"Ignore CM/CS in CI-pipeline as Job","Example":"","Deprecated":"false"},{"Env":"IMAGE_RETRY_COUNT","EnvType":"int","EnvValue":"0","EnvDescription":"push artifact(image) in ci retry count ","Example":"","Deprecated":"false"},{"Env":"IMAGE_RETRY_INTERVAL","EnvType":"int","EnvValue":"5","EnvDescription":"image retry interval takes value in seconds","Example":"","Deprecated":"false"},{"Env":"IMAGE_SCANNER_ENDPOINT","EnvType":"string","EnvValue":"http://image-scanner-new-demo-devtroncd-service.devtroncd:80","EnvDescription":"Image-scanner micro-service URL","Example":"","Deprecated":"false"},{"Env":"IMAGE_SCAN_MAX_RETRIES","EnvType":"int","EnvValue":"3","EnvDescription":"Max retry count for image-scanning","Example":"","Deprecated":"false"},{"Env":"IMAGE_SCAN_RETRY_DELAY","EnvType":"int","EnvValue":"5","EnvDescription":"Delay for the image-scaning to start","Example":"","Deprecated":"false"},{"Env":"IN_APP_LOGGING_ENABLED","EnvType":"bool","EnvValue":"false","EnvDescription":"Used in case of argo workflow is enabled. If enabled logs push will be managed by us, else will be managed by argo workflow.","Example":"","Deprecated":"false"},{"Env":"MAX_CD_WORKFLOW_RUNNER_RETRIES","EnvType":"int","EnvValue":"0","EnvDescription":"Maximum time pre/post-cd-workflow create pod if it fails to complete","Example":"","Deprecated":"false"},{"Env":"MAX_CI_WORKFLOW_RETRIES","EnvType":"int","EnvValue":"0","EnvDescription":"Maximum time CI-workflow create pod if it fails to complete","Example":"","Deprecated":"false"},{"Env":"MODE","EnvType":"string","EnvValue":"DEV","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_SERVER_HOST","EnvType":"string","EnvValue":"localhost:4222","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ORCH_HOST","EnvType":"string","EnvValue":"http://devtroncd-orchestrator-service-prod.devtroncd/webhook/msg/nats","EnvDescription":"Orchestrator micro-service URL ","Example":"","Deprecated":"false"},{"Env":"ORCH_TOKEN","EnvType":"string","EnvValue":"","EnvDescription":"Orchestrator token","Example":"","Deprecated":"false"},{"Env":"PRE_CI_CACHE_PATH","EnvType":"string","EnvValue":"/devtroncd-cache","EnvDescription":"Cache path for Pre CI tasks","Example":"","Deprecated":"false"},{"Env":"SHOW_DOCKER_BUILD_ARGS","EnvType":"bool","EnvValue":"true","EnvDescription":"To enable showing the args passed for CI in build logs","Example":"","Deprecated":"false"},{"Env":"SKIP_CI_JOB_BUILD_CACHE_PUSH_PULL","EnvType":"bool","EnvValue":"false","EnvDescription":"To skip cache Push/Pull for ci job","Example":"","Deprecated":"false"},{"Env":"SKIP_CREATING_ECR_REPO","EnvType":"bool","EnvValue":"false","EnvDescription":"By disabling this ECR repo won't get created if it's not available on ECR from build configuration","Example":"","Deprecated":"false"},{"Env":"TERMINATION_GRACE_PERIOD_SECS","EnvType":"int","EnvValue":"180","EnvDescription":"this is the time given to workflow pods to shutdown. (grace full termination time)","Example":"","Deprecated":"false"},{"Env":"USE_ARTIFACT_LISTING_QUERY_V2","EnvType":"bool","EnvValue":"true","EnvDescription":"To use the V2 query for listing artifacts","Example":"","Deprecated":"false"},{"Env":"USE_BLOB_STORAGE_CONFIG_IN_CD_WORKFLOW","EnvType":"bool","EnvValue":"true","EnvDescription":"To enable blob storage in pre and post cd","Example":"","Deprecated":"false"},{"Env":"USE_BLOB_STORAGE_CONFIG_IN_CI_WORKFLOW","EnvType":"bool","EnvValue":"true","EnvDescription":"To enable blob storage in pre and post ci","Example":"","Deprecated":"false"},{"Env":"USE_BUILDX","EnvType":"bool","EnvValue":"false","EnvDescription":"To enable buildx feature globally","Example":"","Deprecated":"false"},{"Env":"USE_DOCKER_API_TO_GET_DIGEST","EnvType":"bool","EnvValue":"false","EnvDescription":"when user do not pass the digest then this flag controls , finding the image digest using docker API or not. if set to true we get the digest from docker API call else use docker pull command. [logic in ci-runner]","Example":"","Deprecated":"false"},{"Env":"USE_EXTERNAL_NODE","EnvType":"bool","EnvValue":"false","EnvDescription":"It is used in case of Pre/ Post Cd with run in application mode. If enabled the node lebels are read from EXTERNAL_CD_NODE_LABEL_SELECTOR else from CD_NODE_LABEL_SELECTOR MODE: if the vale is DEV, it will read the local kube config file or else from the cluser location.","Example":"","Deprecated":"false"},{"Env":"USE_IMAGE_TAG_FROM_GIT_PROVIDER_FOR_TAG_BASED_BUILD","EnvType":"bool","EnvValue":"false","EnvDescription":"To use the same tag in container image as that of git tag","Example":"","Deprecated":"false"},{"Env":"WF_CONTROLLER_INSTANCE_ID","EnvType":"string","EnvValue":"devtron-runner","EnvDescription":"Workflow controller instance ID.","Example":"","Deprecated":"false"},{"Env":"WORKFLOW_CACHE_CONFIG","EnvType":"string","EnvValue":"{}","EnvDescription":"flag is used to configure how Docker caches are handled during a CI/CD ","Example":"","Deprecated":"false"},{"Env":"WORKFLOW_SERVICE_ACCOUNT","EnvType":"string","EnvValue":"ci-runner","EnvDescription":"","Example":"","Deprecated":"false"}]},{"Category":"DEVTRON","Fields":[{"Env":"-","EnvType":"","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ADDITIONAL_NODE_GROUP_LABELS","EnvType":"","EnvValue":"","EnvDescription":"Add comma separated list of additional node group labels to default labels","Example":"karpenter.sh/nodepool,cloud.google.com/gke-nodepool","Deprecated":"false"},{"Env":"APP_SYNC_IMAGE","EnvType":"string","EnvValue":"quay.io/devtron/chart-sync:1227622d-132-3775","EnvDescription":"For the app sync image, this image will be used in app-manual sync job","Example":"","Deprecated":"false"},{"Env":"APP_SYNC_JOB_RESOURCES_OBJ","EnvType":"string","EnvValue":"","EnvDescription":"To pass the resource of app sync","Example":"","Deprecated":"false"},{"Env":"APP_SYNC_SERVICE_ACCOUNT","EnvType":"string","EnvValue":"chart-sync","EnvDescription":"Service account to be used in app sync Job","Example":"","Deprecated":"false"},{"Env":"APP_SYNC_SHUTDOWN_WAIT_DURATION","EnvType":"int","EnvValue":"120","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ARGO_AUTO_SYNC_ENABLED","EnvType":"bool","EnvValue":"true","EnvDescription":"If enabled all argocd application will have auto sync enabled","Example":"","Deprecated":"false"},{"Env":"ARGO_GIT_COMMIT_RETRY_COUNT_ON_CONFLICT","EnvType":"int","EnvValue":"3","EnvDescription":"retry argocd app manual sync if the timeline is stuck in ARGOCD_SYNC_INITIATED state for more than this defined time (in mins)","Example":"","Deprecated":"false"},{"Env":"ARGO_GIT_COMMIT_RETRY_DELAY_ON_CONFLICT","EnvType":"int","EnvValue":"1","EnvDescription":"Delay on retrying the maifest commit the on gitops","Example":"","Deprecated":"false"},{"Env":"ARGO_REPO_REGISTER_RETRY_COUNT","EnvType":"int","EnvValue":"3","EnvDescription":"Argo app registration in argo retries on deployment","Example":"","Deprecated":"false"},{"Env":"ARGO_REPO_REGISTER_RETRY_DELAY","EnvType":"int","EnvValue":"10","EnvDescription":"Argo app registration in argo cd on deployment delay between retry","Example":"","Deprecated":"false"},{"Env":"ASYNC_BUILDX_CACHE_EXPORT","EnvType":"bool","EnvValue":"false","EnvDescription":"To enable async container image cache export","Example":"","Deprecated":"false"},{"Env":"BATCH_SIZE","EnvType":"int","EnvValue":"5","EnvDescription":"there is feature to get URL's of services/ingresses. so to extract those, we need to parse all the servcie and ingress objects of the application. this BATCH_SIZE flag controls the no of these objects get parsed in one go.","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_ENABLED","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BUILDX_CACHE_MODE_MIN","EnvType":"bool","EnvValue":"false","EnvDescription":"To set build cache mode to minimum in buildx","Example":"","Deprecated":"false"},{"Env":"CD_HOST","EnvType":"string","EnvValue":"localhost","EnvDescription":"Host for the devtron stack","Example":"","Deprecated":"false"},{"Env":"CD_NAMESPACE","EnvType":"string","EnvValue":"devtroncd","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_PORT","EnvType":"string","EnvValue":"8000","EnvDescription":"Port for pre/post-cd","Example":"","Deprecated":"false"},{"Env":"CExpirationTime","EnvType":"int","EnvValue":"600","EnvDescription":"Caching expiration time.","Example":"","Deprecated":"false"},{"Env":"CI_TRIGGER_CRON_TIME","EnvType":"int","EnvValue":"2","EnvDescription":"For image poll plugin","Example":"","Deprecated":"false"},{"Env":"CI_WORKFLOW_STATUS_UPDATE_CRON","EnvType":"string","EnvValue":"*/5 * * * *","EnvDescription":"Cron schedule for CI pipeline status","Example":"","Deprecated":"false"},{"Env":"CLI_CMD_TIMEOUT_GLOBAL_SECONDS","EnvType":"int","EnvValue":"0","EnvDescription":"Used in git cli opeartion timeout","Example":"","Deprecated":"false"},{"Env":"CLUSTER_STATUS_CRON_TIME","EnvType":"int","EnvValue":"15","EnvDescription":"Cron schedule for cluster status on resource browser","Example":"","Deprecated":"false"},{"Env":"CONSUMER_CONFIG_JSON","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_LOG_TIME_LIMIT","EnvType":"int64","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_TIMEOUT","EnvType":"float64","EnvValue":"3600","EnvDescription":"Timeout for CI to be completed","Example":"","Deprecated":"false"},{"Env":"DEVTRON_BOM_URL","EnvType":"string","EnvValue":"https://raw.githubusercontent.com/devtron-labs/devtron/%s/charts/devtron/devtron-bom.yaml","EnvDescription":"Path to devtron-bom.yaml of devtron charts, used for module installation and devtron upgrade","Example":"","Deprecated":"false"},{"Env":"DEVTRON_DEFAULT_NAMESPACE","EnvType":"string","EnvValue":"devtroncd","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEVTRON_DEX_SECRET_NAMESPACE","EnvType":"string","EnvValue":"devtroncd","EnvDescription":"Namespace of dex secret","Example":"","Deprecated":"false"},{"Env":"DEVTRON_HELM_RELEASE_CHART_NAME","EnvType":"string","EnvValue":"devtron-operator","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEVTRON_HELM_RELEASE_NAME","EnvType":"string","EnvValue":"devtron","EnvDescription":"Name of the Devtron Helm release. ","Example":"","Deprecated":"false"},{"Env":"DEVTRON_HELM_RELEASE_NAMESPACE","EnvType":"string","EnvValue":"devtroncd","EnvDescription":"Namespace of the Devtron Helm release","Example":"","Deprecated":"false"},{"Env":"DEVTRON_HELM_REPO_NAME","EnvType":"string","EnvValue":"devtron","EnvDescription":"Is used to install modules (stack manager)","Example":"","Deprecated":"false"},{"Env":"DEVTRON_HELM_REPO_URL","EnvType":"string","EnvValue":"https://helm.devtron.ai","EnvDescription":"Is used to install modules (stack manager)","Example":"","Deprecated":"false"},{"Env":"DEVTRON_INSTALLATION_TYPE","EnvType":"string","EnvValue":"","EnvDescription":"Devtron Installation type(EA/Full)","Example":"","Deprecated":"false"},{"Env":"DEVTRON_MODULES_IDENTIFIER_IN_HELM_VALUES","EnvType":"string","EnvValue":"installer.modules","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEVTRON_SECRET_NAME","EnvType":"string","EnvValue":"devtron-secret","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEVTRON_VERSION_IDENTIFIER_IN_HELM_VALUES","EnvType":"string","EnvValue":"installer.release","EnvDescription":"devtron operator version identifier in helm values yaml","Example":"","Deprecated":"false"},{"Env":"DEX_CID","EnvType":"string","EnvValue":"example-app","EnvDescription":"dex client id ","Example":"","Deprecated":"false"},{"Env":"DEX_CLIENT_ID","EnvType":"string","EnvValue":"argo-cd","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEX_CSTOREKEY","EnvType":"string","EnvValue":"","EnvDescription":"DEX CSTOREKEY.","Example":"","Deprecated":"false"},{"Env":"DEX_JWTKEY","EnvType":"string","EnvValue":"","EnvDescription":"DEX JWT key. ","Example":"","Deprecated":"false"},{"Env":"DEX_RURL","EnvType":"string","EnvValue":"http://127.0.0.1:8080/callback","EnvDescription":"Dex redirect URL(http://argocd-dex-server.devtroncd:8080/callback)","Example":"","Deprecated":"false"},{"Env":"DEX_SCOPES","EnvType":"","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEX_SECRET","EnvType":"string","EnvValue":"","EnvDescription":"Dex secret","Example":"","Deprecated":"false"},{"Env":"DEX_URL","EnvType":"string","EnvValue":"","EnvDescription":"Dex service endpoint with dex path(http://argocd-dex-server.devtroncd:5556/dex)","Example":"","Deprecated":"false"},{"Env":"ECR_REPO_NAME_PREFIX","EnvType":"string","EnvValue":"test/","EnvDescription":"Prefix for ECR repo to be created in does not exist","Example":"","Deprecated":"false"},{"Env":"ENABLE_ASYNC_ARGO_CD_INSTALL_DEVTRON_CHART","EnvType":"bool","EnvValue":"false","EnvDescription":"To enable async installation of gitops application","Example":"","Deprecated":"false"},{"Env":"ENABLE_ASYNC_INSTALL_DEVTRON_CHART","EnvType":"bool","EnvValue":"false","EnvDescription":"To enable async installation of no-gitops application","Example":"","Deprecated":"false"},{"Env":"EPHEMERAL_SERVER_VERSION_REGEX","EnvType":"string","EnvValue":"v[1-9]\\.\\b(2[3-9]\\|[3-9][0-9])\\b.*","EnvDescription":"ephemeral containers support version regex that is compared with k8sServerVersion","Example":"","Deprecated":"false"},{"Env":"EVENT_URL","EnvType":"string","EnvValue":"http://localhost:3000/notify","EnvDescription":"Notifier service url","Example":"","Deprecated":"false"},{"Env":"EXECUTE_WIRE_NIL_CHECKER","EnvType":"bool","EnvValue":"false","EnvDescription":"checks for any nil pointer in wire.go","Example":"","Deprecated":"false"},{"Env":"EXPOSE_CI_METRICS","EnvType":"bool","EnvValue":"false","EnvDescription":"To expose CI metrics","Example":"","Deprecated":"false"},{"Env":"FEATURE_RESTART_WORKLOAD_BATCH_SIZE","EnvType":"int","EnvValue":"1","EnvDescription":"restart workload retrieval batch size ","Example":"","Deprecated":"false"},{"Env":"FEATURE_RESTART_WORKLOAD_WORKER_POOL_SIZE","EnvType":"int","EnvValue":"5","EnvDescription":"restart workload retrieval pool size","Example":"","Deprecated":"false"},{"Env":"FORCE_SECURITY_SCANNING","EnvType":"bool","EnvValue":"false","EnvDescription":"By enabling this no one can disable image scaning on ci-pipeline from UI","Example":"","Deprecated":"false"},{"Env":"GITOPS_REPO_PREFIX","EnvType":"string","EnvValue":"","EnvDescription":"Prefix for Gitops repo being creation for argocd application","Example":"","Deprecated":"false"},{"Env":"GO_RUNTIME_ENV","EnvType":"string","EnvValue":"production","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"GRAFANA_HOST","EnvType":"string","EnvValue":"localhost","EnvDescription":"Host URL for the grafana dashboard","Example":"","Deprecated":"false"},{"Env":"GRAFANA_NAMESPACE","EnvType":"string","EnvValue":"devtroncd","EnvDescription":"Namespace for grafana","Example":"","Deprecated":"false"},{"Env":"GRAFANA_ORG_ID","EnvType":"int","EnvValue":"2","EnvDescription":"Org ID for grafana for application metrics","Example":"","Deprecated":"false"},{"Env":"GRAFANA_PASSWORD","EnvType":"string","EnvValue":"prom-operator","EnvDescription":"Password for grafana dashboard","Example":"","Deprecated":"false"},{"Env":"GRAFANA_PORT","EnvType":"string","EnvValue":"8090","EnvDescription":"Port for grafana micro-service","Example":"","Deprecated":"false"},{"Env":"GRAFANA_URL","EnvType":"string","EnvValue":"","EnvDescription":"Host URL for the grafana dashboard","Example":"","Deprecated":"false"},{"Env":"GRAFANA_USERNAME","EnvType":"string","EnvValue":"admin","EnvDescription":"Username for grafana ","Example":"","Deprecated":"false"},{"Env":"HIDE_IMAGE_TAGGING_HARD_DELETE","EnvType":"bool","EnvValue":"false","EnvDescription":"Flag to hide the hard delete option in the image tagging service","Example":"","Deprecated":"false"},{"Env":"IGNORE_AUTOCOMPLETE_AUTH_CHECK","EnvType":"bool","EnvValue":"false","EnvDescription":"flag for ignoring auth check in autocomplete apis.","Example":"","Deprecated":"false"},{"Env":"INSTALLED_MODULES","EnvType":"","EnvValue":"","EnvDescription":"List of installed modules given in helm values/yaml are written in cm and used by devtron to know which modules are given","Example":"security.trivy,security.clair","Deprecated":"false"},{"Env":"INSTALLER_CRD_NAMESPACE","EnvType":"string","EnvValue":"devtroncd","EnvDescription":"namespace where Custom Resource Definitions get installed","Example":"","Deprecated":"false"},{"Env":"INSTALLER_CRD_OBJECT_GROUP_NAME","EnvType":"string","EnvValue":"installer.devtron.ai","EnvDescription":"Devtron installer CRD group name, partially deprecated.","Example":"","Deprecated":"false"},{"Env":"INSTALLER_CRD_OBJECT_RESOURCE","EnvType":"string","EnvValue":"installers","EnvDescription":"Devtron installer CRD resource name, partially deprecated","Example":"","Deprecated":"false"},{"Env":"INSTALLER_CRD_OBJECT_VERSION","EnvType":"string","EnvValue":"v1alpha1","EnvDescription":"version of the CRDs. default is v1alpha1","Example":"","Deprecated":"false"},{"Env":"IS_AIR_GAP_ENVIRONMENT","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"JwtExpirationTime","EnvType":"int","EnvValue":"120","EnvDescription":"JWT expiration time.","Example":"","Deprecated":"false"},{"Env":"K8s_CLIENT_MAX_IDLE_CONNS_PER_HOST","EnvType":"int","EnvValue":"25","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"K8s_TCP_IDLE_CONN_TIMEOUT","EnvType":"int","EnvValue":"300","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"K8s_TCP_KEEPALIVE","EnvType":"int","EnvValue":"30","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"K8s_TCP_TIMEOUT","EnvType":"int","EnvValue":"30","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"K8s_TLS_HANDSHAKE_TIMEOUT","EnvType":"int","EnvValue":"10","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"KUBELINK_GRPC_MAX_RECEIVE_MSG_SIZE","EnvType":"int","EnvValue":"20","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"KUBELINK_GRPC_MAX_SEND_MSG_SIZE","EnvType":"int","EnvValue":"4","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"LENS_TIMEOUT","EnvType":"int","EnvValue":"0","EnvDescription":"Lens microservice timeout.","Example":"","Deprecated":"false"},{"Env":"LENS_URL","EnvType":"string","EnvValue":"http://lens-milandevtron-service:80","EnvDescription":"Lens micro-service URL","Example":"","Deprecated":"false"},{"Env":"LIMIT_CI_CPU","EnvType":"string","EnvValue":"0.5","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"LIMIT_CI_MEM","EnvType":"string","EnvValue":"3G","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"LOGGER_DEV_MODE","EnvType":"bool","EnvValue":"false","EnvDescription":"Enables a different logger theme.","Example":"","Deprecated":"false"},{"Env":"LOG_LEVEL","EnvType":"int","EnvValue":"-1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"MAX_SESSION_PER_USER","EnvType":"int","EnvValue":"5","EnvDescription":"max no of cluster terminal pods can be created by an user","Example":"","Deprecated":"false"},{"Env":"MODULE_METADATA_API_URL","EnvType":"string","EnvValue":"https://api.devtron.ai/module?name=%s","EnvDescription":"Modules list and meta info will be fetched from this server, that is central api server of devtron.","Example":"","Deprecated":"false"},{"Env":"MODULE_STATUS_HANDLING_CRON_DURATION_MIN","EnvType":"int","EnvValue":"3","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_ACK_WAIT_IN_SECS","EnvType":"int","EnvValue":"120","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_BUFFER_SIZE","EnvType":"int","EnvValue":"-1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_MAX_AGE","EnvType":"int","EnvValue":"86400","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_PROCESSING_BATCH_SIZE","EnvType":"int","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_REPLICAS","EnvType":"int","EnvValue":"0","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NOTIFICATION_MEDIUM","EnvType":"NotificationMedium","EnvValue":"rest","EnvDescription":"notification medium","Example":"","Deprecated":"false"},{"Env":"OTEL_COLLECTOR_URL","EnvType":"string","EnvValue":"","EnvDescription":"Opentelemetry URL ","Example":"","Deprecated":"false"},{"Env":"PARALLELISM_LIMIT_FOR_TAG_PROCESSING","EnvType":"int","EnvValue":"","EnvDescription":"App manual sync job parallel tag processing count.","Example":"","Deprecated":"false"},{"Env":"PG_EXPORT_PROM_METRICS","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_LOG_ALL_FAILURE_QUERIES","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_LOG_ALL_QUERY","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_LOG_SLOW_QUERY","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_QUERY_DUR_THRESHOLD","EnvType":"int64","EnvValue":"5000","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PLUGIN_NAME","EnvType":"string","EnvValue":"Pull images from container repository","EnvDescription":"Handles image retrieval from a container repository and triggers subsequent CI processes upon detecting new images.Current default plugin name: Pull Images from Container Repository.","Example":"","Deprecated":"false"},{"Env":"PROPAGATE_EXTRA_LABELS","EnvType":"bool","EnvValue":"false","EnvDescription":"Add additional propagate labels like api.devtron.ai/appName, api.devtron.ai/envName, api.devtron.ai/project along with the user defined ones.","Example":"","Deprecated":"false"},{"Env":"PROXY_SERVICE_CONFIG","EnvType":"string","EnvValue":"{}","EnvDescription":"Proxy configuration for micro-service to be accessible on orhcestrator ingress","Example":"","Deprecated":"false"},{"Env":"REQ_CI_CPU","EnvType":"string","EnvValue":"0.5","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"REQ_CI_MEM","EnvType":"string","EnvValue":"3G","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"RESTRICT_TERMINAL_ACCESS_FOR_NON_SUPER_USER","EnvType":"bool","EnvValue":"false","EnvDescription":"To restrict the cluster terminal from user having non-super admin acceess","Example":"","Deprecated":"false"},{"Env":"RUNTIME_CONFIG_LOCAL_DEV","EnvType":"LocalDevMode","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"SCOPED_VARIABLE_ENABLED","EnvType":"bool","EnvValue":"false","EnvDescription":"To enable scoped variable option","Example":"","Deprecated":"false"},{"Env":"SCOPED_VARIABLE_FORMAT","EnvType":"string","EnvValue":"@{{%s}}","EnvDescription":"Its a scope format for varialbe name.","Example":"","Deprecated":"false"},{"Env":"SCOPED_VARIABLE_HANDLE_PRIMITIVES","EnvType":"bool","EnvValue":"false","EnvDescription":"This describe should we handle primitives or not in scoped variable template parsing.","Example":"","Deprecated":"false"},{"Env":"SCOPED_VARIABLE_NAME_REGEX","EnvType":"string","EnvValue":"^[a-zA-Z][a-zA-Z0-9_-]{0,62}[a-zA-Z0-9]$","EnvDescription":"Regex for scoped variable name that must passed this regex.","Example":"","Deprecated":"false"},{"Env":"SOCKET_DISCONNECT_DELAY_SECONDS","EnvType":"int","EnvValue":"5","EnvDescription":"The server closes a session when a client receiving connection have not been seen for a while.This delay is configured by this setting. By default the session is closed when a receiving connection wasn't seen for 5 seconds.","Example":"","Deprecated":"false"},{"Env":"SOCKET_HEARTBEAT_SECONDS","EnvType":"int","EnvValue":"25","EnvDescription":"In order to keep proxies and load balancers from closing long running http requests we need to pretend that the connection is active and send a heartbeat packet once in a while. This setting controls how often this is done. By default a heartbeat packet is sent every 25 seconds.","Example":"","Deprecated":"false"},{"Env":"STREAM_CONFIG_JSON","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"SYSTEM_VAR_PREFIX","EnvType":"string","EnvValue":"DEVTRON_","EnvDescription":"Scoped variable prefix, variable name must have this prefix.","Example":"","Deprecated":"false"},{"Env":"TERMINAL_POD_DEFAULT_NAMESPACE","EnvType":"string","EnvValue":"default","EnvDescription":"Cluster terminal default namespace","Example":"","Deprecated":"false"},{"Env":"TERMINAL_POD_INACTIVE_DURATION_IN_MINS","EnvType":"int","EnvValue":"10","EnvDescription":"Timeout for cluster terminal to be inactive","Example":"","Deprecated":"false"},{"Env":"TERMINAL_POD_STATUS_SYNC_In_SECS","EnvType":"int","EnvValue":"600","EnvDescription":"this is the time interval at which the status of the cluster terminal pod","Example":"","Deprecated":"false"},{"Env":"TEST_APP","EnvType":"string","EnvValue":"orchestrator","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TEST_PG_ADDR","EnvType":"string","EnvValue":"127.0.0.1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TEST_PG_DATABASE","EnvType":"string","EnvValue":"orchestrator","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TEST_PG_LOG_QUERY","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TEST_PG_PASSWORD","EnvType":"string","EnvValue":"postgrespw","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TEST_PG_PORT","EnvType":"string","EnvValue":"55000","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TEST_PG_USER","EnvType":"string","EnvValue":"postgres","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TIMEOUT_FOR_FAILED_CI_BUILD","EnvType":"string","EnvValue":"15","EnvDescription":"Timeout for Failed CI build ","Example":"","Deprecated":"false"},{"Env":"TIMEOUT_IN_SECONDS","EnvType":"int","EnvValue":"5","EnvDescription":"timeout to compute the urls from services and ingress objects of an application","Example":"","Deprecated":"false"},{"Env":"USER_SESSION_DURATION_SECONDS","EnvType":"int","EnvValue":"86400","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"USE_ARTIFACT_LISTING_API_V2","EnvType":"bool","EnvValue":"true","EnvDescription":"To use the V2 API for listing artifacts in Listing the images in pipeline","Example":"","Deprecated":"false"},{"Env":"USE_CUSTOM_HTTP_TRANSPORT","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"USE_GIT_CLI","EnvType":"bool","EnvValue":"false","EnvDescription":"To enable git cli","Example":"","Deprecated":"false"},{"Env":"USE_RBAC_CREATION_V2","EnvType":"bool","EnvValue":"true","EnvDescription":"To use the V2 for RBAC creation","Example":"","Deprecated":"false"},{"Env":"VARIABLE_CACHE_ENABLED","EnvType":"bool","EnvValue":"true","EnvDescription":"This is used to control caching of all the scope variables defined in the system.","Example":"","Deprecated":"false"},{"Env":"VARIABLE_EXPRESSION_REGEX","EnvType":"string","EnvValue":"@{{([^}]+)}}","EnvDescription":"Scoped variable expression regex","Example":"","Deprecated":"false"},{"Env":"WEBHOOK_TOKEN","EnvType":"string","EnvValue":"","EnvDescription":"If you want to continue using jenkins for CI then please provide this for authentication of requests","Example":"","Deprecated":"false"}]},{"Category":"GITOPS","Fields":[{"Env":"ACD_CM","EnvType":"string","EnvValue":"argocd-cm","EnvDescription":"Name of the argocd CM","Example":"","Deprecated":"false"},{"Env":"ACD_NAMESPACE","EnvType":"string","EnvValue":"devtroncd","EnvDescription":"To pass the argocd namespace","Example":"","Deprecated":"false"},{"Env":"ACD_PASSWORD","EnvType":"string","EnvValue":"","EnvDescription":"Password for the Argocd (deprecated)","Example":"","Deprecated":"false"},{"Env":"ACD_USERNAME","EnvType":"string","EnvValue":"admin","EnvDescription":"User name for argocd","Example":"","Deprecated":"false"},{"Env":"GITOPS_SECRET_NAME","EnvType":"string","EnvValue":"devtron-gitops-secret","EnvDescription":"devtron-gitops-secret","Example":"","Deprecated":"false"},{"Env":"RESOURCE_LIST_FOR_REPLICAS","EnvType":"string","EnvValue":"Deployment,Rollout,StatefulSet,ReplicaSet","EnvDescription":"this holds the list of k8s resource names which support replicas key. this list used in hibernate/un hibernate process","Example":"","Deprecated":"false"},{"Env":"RESOURCE_LIST_FOR_REPLICAS_BATCH_SIZE","EnvType":"int","EnvValue":"5","EnvDescription":"this the batch size to control no of above resources can be parsed in one go to determine hibernate status","Example":"","Deprecated":"false"}]},{"Category":"INFRA_SETUP","Fields":[{"Env":"DASHBOARD_HOST","EnvType":"string","EnvValue":"localhost","EnvDescription":"Dashboard micro-service URL","Example":"","Deprecated":"false"},{"Env":"DASHBOARD_NAMESPACE","EnvType":"string","EnvValue":"devtroncd","EnvDescription":"Dashboard micro-service namespace","Example":"","Deprecated":"false"},{"Env":"DASHBOARD_PORT","EnvType":"string","EnvValue":"3000","EnvDescription":"Port for dashboard micro-service","Example":"","Deprecated":"false"},{"Env":"DEX_HOST","EnvType":"string","EnvValue":"http://localhost","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEX_PORT","EnvType":"string","EnvValue":"5556","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"GIT_SENSOR_PROTOCOL","EnvType":"string","EnvValue":"REST","EnvDescription":"Protocol to connect with git-sensor micro-service","Example":"","Deprecated":"false"},{"Env":"GIT_SENSOR_TIMEOUT","EnvType":"int","EnvValue":"0","EnvDescription":"Timeout for getting response from the git-sensor","Example":"","Deprecated":"false"},{"Env":"GIT_SENSOR_URL","EnvType":"string","EnvValue":"127.0.0.1:7070","EnvDescription":"git-sensor micro-service url ","Example":"","Deprecated":"false"},{"Env":"HELM_CLIENT_URL","EnvType":"string","EnvValue":"127.0.0.1:50051","EnvDescription":"Kubelink micro-service url ","Example":"","Deprecated":"false"}]},{"Category":"POSTGRES","Fields":[{"Env":"APP","EnvType":"string","EnvValue":"orchestrator","EnvDescription":"Application name","Example":"","Deprecated":"false"},{"Env":"CASBIN_DATABASE","EnvType":"string","EnvValue":"casbin","EnvDescription":"Database for casbin","Example":"","Deprecated":"false"},{"Env":"PG_ADDR","EnvType":"string","EnvValue":"127.0.0.1","EnvDescription":"address of postgres service","Example":"postgresql-postgresql.devtroncd","Deprecated":"false"},{"Env":"PG_DATABASE","EnvType":"string","EnvValue":"orchestrator","EnvDescription":"postgres database to be made connection with","Example":"orchestrator, casbin, git_sensor, lens","Deprecated":"false"},{"Env":"PG_PASSWORD","EnvType":"string","EnvValue":"{password}","EnvDescription":"password for postgres, associated with PG_USER","Example":"confidential ;)","Deprecated":"false"},{"Env":"PG_PORT","EnvType":"string","EnvValue":"5432","EnvDescription":"port of postgresql service","Example":"5432","Deprecated":"false"},{"Env":"PG_READ_TIMEOUT","EnvType":"int64","EnvValue":"30","EnvDescription":"Time out for read operation in postgres","Example":"","Deprecated":"false"},{"Env":"PG_USER","EnvType":"string","EnvValue":"postgres","EnvDescription":"user for postgres","Example":"postgres","Deprecated":"false"},{"Env":"PG_WRITE_TIMEOUT","EnvType":"int64","EnvValue":"30","EnvDescription":"Time out for write operation in postgres","Example":"","Deprecated":"false"}]},{"Category":"RBAC","Fields":[{"Env":"ENFORCER_CACHE","EnvType":"bool","EnvValue":"false","EnvDescription":"To Enable enforcer cache.","Example":"","Deprecated":"false"},{"Env":"ENFORCER_CACHE_EXPIRATION_IN_SEC","EnvType":"int","EnvValue":"86400","EnvDescription":"Expiration time (in seconds) for enforcer cache. ","Example":"","Deprecated":"false"},{"Env":"ENFORCER_MAX_BATCH_SIZE","EnvType":"int","EnvValue":"1","EnvDescription":"Maximum batch size for the enforcer.","Example":"","Deprecated":"false"},{"Env":"USE_CASBIN_V2","EnvType":"bool","EnvValue":"true","EnvDescription":"To enable casbin V2 API","Example":"","Deprecated":"false"}]}] \ No newline at end of file diff --git a/env_gen.md b/env_gen.md index f247bf8a1f..dcffdf8783 100644 --- a/env_gen.md +++ b/env_gen.md @@ -3,22 +3,22 @@ ## CD Related Environment Variables | Key | Type | Default Value | Description | Example | Deprecated | |-------|----------|-------------------|-------------------|-----------------------|------------------| - | ARGO_APP_MANUAL_SYNC_TIME | int |3 | | | false | - | CD_HELM_PIPELINE_STATUS_CRON_TIME | string |*/2 * * * * | | | false | - | CD_PIPELINE_STATUS_CRON_TIME | string |*/2 * * * * | | | false | - | CD_PIPELINE_STATUS_TIMEOUT_DURATION | string |20 | | | false | - | DEPLOY_STATUS_CRON_GET_PIPELINE_DEPLOYED_WITHIN_HOURS | int |12 | | | false | - | DEVTRON_CHART_ARGO_CD_INSTALL_REQUEST_TIMEOUT | int |1 | | | false | - | DEVTRON_CHART_INSTALL_REQUEST_TIMEOUT | int |6 | | | false | + | ARGO_APP_MANUAL_SYNC_TIME | int |3 | retry argocd app manual sync if the timeline is stuck in ARGOCD_SYNC_INITIATED state for more than this defined time (in mins) | | false | + | CD_HELM_PIPELINE_STATUS_CRON_TIME | string |*/2 * * * * | Cron time to check the pipeline status | | false | + | CD_PIPELINE_STATUS_CRON_TIME | string |*/2 * * * * | Cron time for CD pipeline status | | false | + | CD_PIPELINE_STATUS_TIMEOUT_DURATION | string |20 | Timeout for CD pipeline to get healthy | | false | + | DEPLOY_STATUS_CRON_GET_PIPELINE_DEPLOYED_WITHIN_HOURS | int |12 | This flag is used to fetch the deployment status of the application. It retrieves the status of deployments that occurred between 12 hours and 10 minutes prior to the current time. It fetches non-terminal statuses. | | false | + | DEVTRON_CHART_ARGO_CD_INSTALL_REQUEST_TIMEOUT | int |1 | Context timeout for gitops concurrent async deployments | | false | + | DEVTRON_CHART_INSTALL_REQUEST_TIMEOUT | int |6 | Context timeout for no gitops concurrent async deployments | | false | | EXPOSE_CD_METRICS | bool |false | | | false | | FEATURE_MIGRATE_ARGOCD_APPLICATION_ENABLE | bool |false | enable migration of external argocd application to devtron pipeline | | false | - | HELM_PIPELINE_STATUS_CHECK_ELIGIBLE_TIME | string |120 | | | false | - | IS_INTERNAL_USE | bool |true | | | false | + | HELM_PIPELINE_STATUS_CHECK_ELIGIBLE_TIME | string |120 | eligible time for checking helm app status periodically and update in db, value is in seconds., default is 120, if wfr is updated within configured time i.e. HELM_PIPELINE_STATUS_CHECK_ELIGIBLE_TIME then do not include for this cron cycle. | | false | + | IS_INTERNAL_USE | bool |true | If enabled then cd pipeline and helm apps will not need the deployment app type mandatorily. Couple this flag with HIDE_GITOPS_OR_HELM_OPTION (in Dashborad) and if gitops is configured and allowed for the env, pipeline/ helm app will gitops else no-gitops. | | false | | MIGRATE_DEPLOYMENT_CONFIG_DATA | bool |false | migrate deployment config data from charts table to deployment_config table | | false | - | PIPELINE_DEGRADED_TIME | string |10 | | | false | - | REVISION_HISTORY_LIMIT_DEVTRON_APP | int |1 | | | false | - | REVISION_HISTORY_LIMIT_EXTERNAL_HELM_APP | int |0 | | | false | - | REVISION_HISTORY_LIMIT_HELM_APP | int |1 | | | false | + | PIPELINE_DEGRADED_TIME | string |10 | Time to mark a pipeline degraded if not healthy in defined time | | false | + | REVISION_HISTORY_LIMIT_DEVTRON_APP | int |1 | Count for devtron application rivision history | | false | + | REVISION_HISTORY_LIMIT_EXTERNAL_HELM_APP | int |0 | Count for external helm application rivision history | | false | + | REVISION_HISTORY_LIMIT_HELM_APP | int |1 | To set the history limit for the helm app being deployed through devtron | | false | | REVISION_HISTORY_LIMIT_LINKED_HELM_APP | int |15 | | | false | | RUN_HELM_INSTALL_IN_ASYNC_MODE_HELM_APPS | bool |false | | | false | | SHOULD_CHECK_NAMESPACE_ON_CLONE | bool |false | should we check if namespace exists or not while cloning app | | false | @@ -28,97 +28,97 @@ ## CI_RUNNER Related Environment Variables | Key | Type | Default Value | Description | Example | Deprecated | |-------|----------|-------------------|-------------------|-----------------------|------------------| - | AZURE_ACCOUNT_KEY | string | | | | false | - | AZURE_ACCOUNT_NAME | string | | | | false | - | AZURE_BLOB_CONTAINER_CI_CACHE | string | | | | false | - | AZURE_BLOB_CONTAINER_CI_LOG | string | | | | false | - | AZURE_GATEWAY_CONNECTION_INSECURE | bool |true | | | false | - | AZURE_GATEWAY_URL | string |http://devtron-minio.devtroncd:9000 | | | false | - | BASE_LOG_LOCATION_PATH | string |/home/devtron/ | | | false | - | BLOB_STORAGE_GCP_CREDENTIALS_JSON | string | | | | false | - | BLOB_STORAGE_PROVIDER | |S3 | | | false | - | BLOB_STORAGE_S3_ACCESS_KEY | string | | | | false | - | BLOB_STORAGE_S3_BUCKET_VERSIONED | bool |true | | | false | - | BLOB_STORAGE_S3_ENDPOINT | string | | | | false | - | BLOB_STORAGE_S3_ENDPOINT_INSECURE | bool |false | | | false | - | BLOB_STORAGE_S3_SECRET_KEY | string | | | | false | - | BUILDX_CACHE_PATH | string |/var/lib/devtron/buildx | | | false | - | BUILDX_K8S_DRIVER_OPTIONS | string | | | | false | - | BUILDX_PROVENANCE_MODE | string | | | | false | - | BUILD_LOG_TTL_VALUE_IN_SECS | int |3600 | | | false | - | CACHE_LIMIT | int64 |5000000000 | | | false | - | CD_DEFAULT_ADDRESS_POOL_BASE_CIDR | string | | | | false | - | CD_DEFAULT_ADDRESS_POOL_SIZE | int | | | | false | - | CD_LIMIT_CI_CPU | string |0.5 | | | false | - | CD_LIMIT_CI_MEM | string |3G | | | false | - | CD_NODE_LABEL_SELECTOR | | | | | false | - | CD_NODE_TAINTS_KEY | string |dedicated | | | false | - | CD_NODE_TAINTS_VALUE | string |ci | | | false | - | CD_REQ_CI_CPU | string |0.5 | | | false | - | CD_REQ_CI_MEM | string |3G | | | false | - | CD_WORKFLOW_EXECUTOR_TYPE | |AWF | | | false | - | CD_WORKFLOW_SERVICE_ACCOUNT | string |cd-runner | | | false | - | CI_DEFAULT_ADDRESS_POOL_BASE_CIDR | string | | | | false | - | CI_DEFAULT_ADDRESS_POOL_SIZE | int | | | | false | - | CI_IGNORE_DOCKER_CACHE | bool | | | | false | - | CI_LOGS_KEY_PREFIX | string | | | | false | - | CI_NODE_LABEL_SELECTOR | | | | | false | - | CI_NODE_TAINTS_KEY | string | | | | false | - | CI_NODE_TAINTS_VALUE | string | | | | false | - | CI_RUNNER_DOCKER_MTU_VALUE | int |-1 | | | false | - | CI_SUCCESS_AUTO_TRIGGER_BATCH_SIZE | int |1 | | | false | - | CI_VOLUME_MOUNTS_JSON | string | | | | false | - | CI_WORKFLOW_EXECUTOR_TYPE | |AWF | | | false | - | DEFAULT_ARTIFACT_KEY_LOCATION | string |arsenal-v1/ci-artifacts | | | false | + | AZURE_ACCOUNT_KEY | string | | If blob storage is bieng used of azure then pass the secret key to access the bucket | | false | + | AZURE_ACCOUNT_NAME | string | | Account name for azure blob storage | | false | + | AZURE_BLOB_CONTAINER_CI_CACHE | string | | Cache bucket name for azure blob storage | | false | + | AZURE_BLOB_CONTAINER_CI_LOG | string | | Log bucket for azure blob storage | | false | + | AZURE_GATEWAY_CONNECTION_INSECURE | bool |true | Azure gateway connection allows insecure if true | | false | + | AZURE_GATEWAY_URL | string |http://devtron-minio.devtroncd:9000 | Sent to CI runner for blob | | false | + | BASE_LOG_LOCATION_PATH | string |/home/devtron/ | Used to store, download logs of ci workflow, artifact | | false | + | BLOB_STORAGE_GCP_CREDENTIALS_JSON | string | | GCP cred json for GCS blob storage | | false | + | BLOB_STORAGE_PROVIDER | |S3 | Blob storage provider name(AWS/GCP/Azure) | | false | + | BLOB_STORAGE_S3_ACCESS_KEY | string | | S3 access key for s3 blob storage | | false | + | BLOB_STORAGE_S3_BUCKET_VERSIONED | bool |true | To enable buctet versioning for blob storage | | false | + | BLOB_STORAGE_S3_ENDPOINT | string | | S3 endpoint URL for s3 blob storage | | false | + | BLOB_STORAGE_S3_ENDPOINT_INSECURE | bool |false | To use insecure s3 endpoint | | false | + | BLOB_STORAGE_S3_SECRET_KEY | string | | Secret key for s3 blob storage | | false | + | BUILDX_CACHE_PATH | string |/var/lib/devtron/buildx | Path for the buildx cache | | false | + | BUILDX_K8S_DRIVER_OPTIONS | string | | To enable the k8s driver and pass args for k8s driver in buildx | | false | + | BUILDX_PROVENANCE_MODE | string | | provinance is set to true by default by docker. this will add some build related data in generated build manifest.it also adds some unknown:unknown key:value pair which may not be compatible by some container registries. with buildx k8s driver , provinenance=true is causing issue when push manifest to quay registry, so setting it to false | | false | + | BUILD_LOG_TTL_VALUE_IN_SECS | int |3600 | This is the time that the pods of ci/pre-cd/post-cd live after completion state. | | false | + | CACHE_LIMIT | int64 |5000000000 | Cache limit. | | false | + | CD_DEFAULT_ADDRESS_POOL_BASE_CIDR | string | | To pass the IP cidr for Pre/Post cd | | false | + | CD_DEFAULT_ADDRESS_POOL_SIZE | int | | The subnet size to allocate from the base pool for CD | | false | + | CD_LIMIT_CI_CPU | string |0.5 | CPU Resource Limit Pre/Post CD | | false | + | CD_LIMIT_CI_MEM | string |3G | Memory Resource Limit Pre/Post CD | | false | + | CD_NODE_LABEL_SELECTOR | | | Node label selector for Pre/Post CD | | false | + | CD_NODE_TAINTS_KEY | string |dedicated | Toleration key for Pre/Post CD | | false | + | CD_NODE_TAINTS_VALUE | string |ci | Toleration value for Pre/Post CD | | false | + | CD_REQ_CI_CPU | string |0.5 | CPU Resource Rquest Pre/Post CD | | false | + | CD_REQ_CI_MEM | string |3G | Memory Resource Rquest Pre/Post CD | | false | + | CD_WORKFLOW_EXECUTOR_TYPE | |AWF | Executor type for Pre/Post CD(AWF,System) | | false | + | CD_WORKFLOW_SERVICE_ACCOUNT | string |cd-runner | Service account to be used in Pre/Post CD pod | | false | + | CI_DEFAULT_ADDRESS_POOL_BASE_CIDR | string | | To pass the IP cidr for CI | | false | + | CI_DEFAULT_ADDRESS_POOL_SIZE | int | | The subnet size to allocate from the base pool for CI | | false | + | CI_IGNORE_DOCKER_CACHE | bool | | Ignoring docker cache | | false | + | CI_LOGS_KEY_PREFIX | string | | Prefix for build logs | | false | + | CI_NODE_LABEL_SELECTOR | | | Node label selector for CI | | false | + | CI_NODE_TAINTS_KEY | string | | Toleration key for CI | | false | + | CI_NODE_TAINTS_VALUE | string | | Toleration value for CI | | false | + | CI_RUNNER_DOCKER_MTU_VALUE | int |-1 | this is to control the bytes of inofrmation passed in a network packet in ci-runner. default is -1 (defaults to the underlying node mtu value) | | false | + | CI_SUCCESS_AUTO_TRIGGER_BATCH_SIZE | int |1 | this is to control the no of linked pipelines should be hanled in one go when a ci-success event of an parent ci is received | | false | + | CI_VOLUME_MOUNTS_JSON | string | | additional volume mount data for CI and JOB | | false | + | CI_WORKFLOW_EXECUTOR_TYPE | |AWF | Executor type for CI(AWF,System) | | false | + | DEFAULT_ARTIFACT_KEY_LOCATION | string |arsenal-v1/ci-artifacts | Key location for artifacts being created | | false | | DEFAULT_BUILD_LOGS_BUCKET | string |devtron-pro-ci-logs | | | false | - | DEFAULT_BUILD_LOGS_KEY_PREFIX | string |arsenal-v1 | | | false | - | DEFAULT_CACHE_BUCKET | string |ci-caching | | | false | - | DEFAULT_CACHE_BUCKET_REGION | string |us-east-2 | | | false | - | DEFAULT_CD_ARTIFACT_KEY_LOCATION | string | | | | false | + | DEFAULT_BUILD_LOGS_KEY_PREFIX | string |arsenal-v1 | Bucket prefix for build logs | | false | + | DEFAULT_CACHE_BUCKET | string |ci-caching | Bucket name for build cache | | false | + | DEFAULT_CACHE_BUCKET_REGION | string |us-east-2 | Build Cache bucket region | | false | + | DEFAULT_CD_ARTIFACT_KEY_LOCATION | string | | Bucket prefix for build cache | | false | | DEFAULT_CD_LOGS_BUCKET_REGION | string |us-east-2 | | | false | - | DEFAULT_CD_NAMESPACE | string | | | | false | - | DEFAULT_CD_TIMEOUT | int64 |3600 | | | false | - | DEFAULT_CI_IMAGE | string |686244538589.dkr.ecr.us-east-2.amazonaws.com/cirunner:47 | | | false | - | DEFAULT_NAMESPACE | string |devtron-ci | | | false | - | DEFAULT_TARGET_PLATFORM | string | | | | false | - | DOCKER_BUILD_CACHE_PATH | string |/var/lib/docker | | | false | - | ENABLE_BUILD_CONTEXT | bool |false | | | false | + | DEFAULT_CD_NAMESPACE | string | | Namespace for devtron stack | | false | + | DEFAULT_CD_TIMEOUT | int64 |3600 | Timeout for Pre/Post-Cd to be completed | | false | + | DEFAULT_CI_IMAGE | string |686244538589.dkr.ecr.us-east-2.amazonaws.com/cirunner:47 | To pass the ci-runner image | | false | + | DEFAULT_NAMESPACE | string |devtron-ci | Timeout for CI to be completed | | false | + | DEFAULT_TARGET_PLATFORM | string | | Default architecture for buildx | | false | + | DOCKER_BUILD_CACHE_PATH | string |/var/lib/docker | Path to store cache of docker build (/var/lib/docker-> for legacy docker build, /var/lib/devtron-> for buildx) | | false | + | ENABLE_BUILD_CONTEXT | bool |false | To Enable build context in Devtron. | | false | | ENABLE_WORKFLOW_EXECUTION_STAGE | bool |true | if enabled then we will display build stages separately for CI/Job/Pre-Post CD | true | false | - | EXTERNAL_BLOB_STORAGE_CM_NAME | string |blob-storage-cm | | | false | - | EXTERNAL_BLOB_STORAGE_SECRET_NAME | string |blob-storage-secret | | | false | - | EXTERNAL_CD_NODE_LABEL_SELECTOR | | | | | false | + | EXTERNAL_BLOB_STORAGE_CM_NAME | string |blob-storage-cm | name of the config map(contains bucket name, etc.) in external cluster when there is some operation related to external cluster, for example:-downloading cd artifact pushed in external cluster's env and we need to download from there, downloads ci logs pushed in external cluster's blob | | false | + | EXTERNAL_BLOB_STORAGE_SECRET_NAME | string |blob-storage-secret | name of the secret(contains password, accessId,passKeys, etc.) in external cluster when there is some operation related to external cluster, for example:-downloading cd artifact pushed in external cluster's env and we need to download from there, downloads ci logs pushed in external cluster's blob | | false | + | EXTERNAL_CD_NODE_LABEL_SELECTOR | | | This is an array of strings used when submitting a workflow for pre or post-CD execution. If the | | false | | EXTERNAL_CD_NODE_TAINTS_KEY | string |dedicated | | | false | | EXTERNAL_CD_NODE_TAINTS_VALUE | string |ci | | | false | - | EXTERNAL_CI_API_SECRET | string |devtroncd-secret | | | false | - | EXTERNAL_CI_PAYLOAD | string |{"ciProjectDetails":[{"gitRepository":"https://github.com/vikram1601/getting-started-nodejs.git","checkoutPath":"./abc","commitHash":"239077135f8cdeeccb7857e2851348f558cb53d3","commitTime":"2022-10-30T20:00:00","branch":"master","message":"Update README.md","author":"User Name "}],"dockerImage":"445808685819.dkr.ecr.us-east-2.amazonaws.com/orch:23907713-2"} | | | false | - | EXTERNAL_CI_WEB_HOOK_URL | string | | | | false | - | IGNORE_CM_CS_IN_CI_JOB | bool |false | | | false | - | IMAGE_RETRY_COUNT | int |0 | | | false | - | IMAGE_RETRY_INTERVAL | int |5 | | | false | - | IMAGE_SCANNER_ENDPOINT | string |http://image-scanner-new-demo-devtroncd-service.devtroncd:80 | | | false | - | IMAGE_SCAN_MAX_RETRIES | int |3 | | | false | - | IMAGE_SCAN_RETRY_DELAY | int |5 | | | false | - | IN_APP_LOGGING_ENABLED | bool |false | | | false | - | MAX_CD_WORKFLOW_RUNNER_RETRIES | int |0 | | | false | - | MAX_CI_WORKFLOW_RETRIES | int |0 | | | false | + | EXTERNAL_CI_API_SECRET | string |devtroncd-secret | External CI API secret. | | false | + | EXTERNAL_CI_PAYLOAD | string |{"ciProjectDetails":[{"gitRepository":"https://github.com/vikram1601/getting-started-nodejs.git","checkoutPath":"./abc","commitHash":"239077135f8cdeeccb7857e2851348f558cb53d3","commitTime":"2022-10-30T20:00:00","branch":"master","message":"Update README.md","author":"User Name "}],"dockerImage":"445808685819.dkr.ecr.us-east-2.amazonaws.com/orch:23907713-2"} | External CI payload with project details. | | false | + | EXTERNAL_CI_WEB_HOOK_URL | string | | default is {{HOST_URL}}/orchestrator/webhook/ext-ci. It is used for external ci. | | false | + | IGNORE_CM_CS_IN_CI_JOB | bool |false | Ignore CM/CS in CI-pipeline as Job | | false | + | IMAGE_RETRY_COUNT | int |0 | push artifact(image) in ci retry count | | false | + | IMAGE_RETRY_INTERVAL | int |5 | image retry interval takes value in seconds | | false | + | IMAGE_SCANNER_ENDPOINT | string |http://image-scanner-new-demo-devtroncd-service.devtroncd:80 | Image-scanner micro-service URL | | false | + | IMAGE_SCAN_MAX_RETRIES | int |3 | Max retry count for image-scanning | | false | + | IMAGE_SCAN_RETRY_DELAY | int |5 | Delay for the image-scaning to start | | false | + | IN_APP_LOGGING_ENABLED | bool |false | Used in case of argo workflow is enabled. If enabled logs push will be managed by us, else will be managed by argo workflow. | | false | + | MAX_CD_WORKFLOW_RUNNER_RETRIES | int |0 | Maximum time pre/post-cd-workflow create pod if it fails to complete | | false | + | MAX_CI_WORKFLOW_RETRIES | int |0 | Maximum time CI-workflow create pod if it fails to complete | | false | | MODE | string |DEV | | | false | | NATS_SERVER_HOST | string |localhost:4222 | | | false | - | ORCH_HOST | string |http://devtroncd-orchestrator-service-prod.devtroncd/webhook/msg/nats | | | false | - | ORCH_TOKEN | string | | | | false | - | PRE_CI_CACHE_PATH | string |/devtroncd-cache | | | false | - | SHOW_DOCKER_BUILD_ARGS | bool |true | | | false | - | SKIP_CI_JOB_BUILD_CACHE_PUSH_PULL | bool |false | | | false | - | SKIP_CREATING_ECR_REPO | bool |false | | | false | - | TERMINATION_GRACE_PERIOD_SECS | int |180 | | | false | - | USE_ARTIFACT_LISTING_QUERY_V2 | bool |true | | | false | - | USE_BLOB_STORAGE_CONFIG_IN_CD_WORKFLOW | bool |true | | | false | - | USE_BLOB_STORAGE_CONFIG_IN_CI_WORKFLOW | bool |true | | | false | - | USE_BUILDX | bool |false | | | false | - | USE_DOCKER_API_TO_GET_DIGEST | bool |false | | | false | - | USE_EXTERNAL_NODE | bool |false | | | false | - | USE_IMAGE_TAG_FROM_GIT_PROVIDER_FOR_TAG_BASED_BUILD | bool |false | | | false | - | WF_CONTROLLER_INSTANCE_ID | string |devtron-runner | | | false | - | WORKFLOW_CACHE_CONFIG | string |{} | | | false | + | ORCH_HOST | string |http://devtroncd-orchestrator-service-prod.devtroncd/webhook/msg/nats | Orchestrator micro-service URL | | false | + | ORCH_TOKEN | string | | Orchestrator token | | false | + | PRE_CI_CACHE_PATH | string |/devtroncd-cache | Cache path for Pre CI tasks | | false | + | SHOW_DOCKER_BUILD_ARGS | bool |true | To enable showing the args passed for CI in build logs | | false | + | SKIP_CI_JOB_BUILD_CACHE_PUSH_PULL | bool |false | To skip cache Push/Pull for ci job | | false | + | SKIP_CREATING_ECR_REPO | bool |false | By disabling this ECR repo won't get created if it's not available on ECR from build configuration | | false | + | TERMINATION_GRACE_PERIOD_SECS | int |180 | this is the time given to workflow pods to shutdown. (grace full termination time) | | false | + | USE_ARTIFACT_LISTING_QUERY_V2 | bool |true | To use the V2 query for listing artifacts | | false | + | USE_BLOB_STORAGE_CONFIG_IN_CD_WORKFLOW | bool |true | To enable blob storage in pre and post cd | | false | + | USE_BLOB_STORAGE_CONFIG_IN_CI_WORKFLOW | bool |true | To enable blob storage in pre and post ci | | false | + | USE_BUILDX | bool |false | To enable buildx feature globally | | false | + | USE_DOCKER_API_TO_GET_DIGEST | bool |false | when user do not pass the digest then this flag controls , finding the image digest using docker API or not. if set to true we get the digest from docker API call else use docker pull command. [logic in ci-runner] | | false | + | USE_EXTERNAL_NODE | bool |false | It is used in case of Pre/ Post Cd with run in application mode. If enabled the node lebels are read from EXTERNAL_CD_NODE_LABEL_SELECTOR else from CD_NODE_LABEL_SELECTOR MODE: if the vale is DEV, it will read the local kube config file or else from the cluser location. | | false | + | USE_IMAGE_TAG_FROM_GIT_PROVIDER_FOR_TAG_BASED_BUILD | bool |false | To use the same tag in container image as that of git tag | | false | + | WF_CONTROLLER_INSTANCE_ID | string |devtron-runner | Workflow controller instance ID. | | false | + | WORKFLOW_CACHE_CONFIG | string |{} | flag is used to configure how Docker caches are handled during a CI/CD | | false | | WORKFLOW_SERVICE_ACCOUNT | string |ci-runner | | | false | @@ -127,77 +127,78 @@ |-------|----------|-------------------|-------------------|-----------------------|------------------| | - | | | | | false | | ADDITIONAL_NODE_GROUP_LABELS | | | Add comma separated list of additional node group labels to default labels | karpenter.sh/nodepool,cloud.google.com/gke-nodepool | false | - | APP_SYNC_IMAGE | string |quay.io/devtron/chart-sync:1227622d-132-3775 | | | false | - | APP_SYNC_JOB_RESOURCES_OBJ | string | | | | false | - | APP_SYNC_SERVICE_ACCOUNT | string |chart-sync | | | false | + | APP_SYNC_IMAGE | string |quay.io/devtron/chart-sync:1227622d-132-3775 | For the app sync image, this image will be used in app-manual sync job | | false | + | APP_SYNC_JOB_RESOURCES_OBJ | string | | To pass the resource of app sync | | false | + | APP_SYNC_SERVICE_ACCOUNT | string |chart-sync | Service account to be used in app sync Job | | false | | APP_SYNC_SHUTDOWN_WAIT_DURATION | int |120 | | | false | - | ARGO_AUTO_SYNC_ENABLED | bool |true | | | false | - | ARGO_GIT_COMMIT_RETRY_COUNT_ON_CONFLICT | int |3 | | | false | - | ARGO_GIT_COMMIT_RETRY_DELAY_ON_CONFLICT | int |1 | | | false | - | ARGO_REPO_REGISTER_RETRY_COUNT | int |3 | | | false | - | ARGO_REPO_REGISTER_RETRY_DELAY | int |10 | | | false | - | ASYNC_BUILDX_CACHE_EXPORT | bool |false | | | false | - | BATCH_SIZE | int |5 | | | false | + | ARGO_AUTO_SYNC_ENABLED | bool |true | If enabled all argocd application will have auto sync enabled | | false | + | ARGO_GIT_COMMIT_RETRY_COUNT_ON_CONFLICT | int |3 | retry argocd app manual sync if the timeline is stuck in ARGOCD_SYNC_INITIATED state for more than this defined time (in mins) | | false | + | ARGO_GIT_COMMIT_RETRY_DELAY_ON_CONFLICT | int |1 | Delay on retrying the maifest commit the on gitops | | false | + | ARGO_REPO_REGISTER_RETRY_COUNT | int |3 | Argo app registration in argo retries on deployment | | false | + | ARGO_REPO_REGISTER_RETRY_DELAY | int |10 | Argo app registration in argo cd on deployment delay between retry | | false | + | ASYNC_BUILDX_CACHE_EXPORT | bool |false | To enable async container image cache export | | false | + | BATCH_SIZE | int |5 | there is feature to get URL's of services/ingresses. so to extract those, we need to parse all the servcie and ingress objects of the application. this BATCH_SIZE flag controls the no of these objects get parsed in one go. | | false | | BLOB_STORAGE_ENABLED | bool |false | | | false | - | BUILDX_CACHE_MODE_MIN | bool |false | | | false | - | CD_HOST | string |localhost | | | false | + | BUILDX_CACHE_MODE_MIN | bool |false | To set build cache mode to minimum in buildx | | false | + | CD_HOST | string |localhost | Host for the devtron stack | | false | | CD_NAMESPACE | string |devtroncd | | | false | - | CD_PORT | string |8000 | | | false | - | CExpirationTime | int |600 | | | false | - | CI_TRIGGER_CRON_TIME | int |2 | | | false | - | CI_WORKFLOW_STATUS_UPDATE_CRON | string |*/5 * * * * | | | false | - | CLI_CMD_TIMEOUT_GLOBAL_SECONDS | int |0 | | | false | - | CLUSTER_STATUS_CRON_TIME | int |15 | | | false | + | CD_PORT | string |8000 | Port for pre/post-cd | | false | + | CExpirationTime | int |600 | Caching expiration time. | | false | + | CI_TRIGGER_CRON_TIME | int |2 | For image poll plugin | | false | + | CI_WORKFLOW_STATUS_UPDATE_CRON | string |*/5 * * * * | Cron schedule for CI pipeline status | | false | + | CLI_CMD_TIMEOUT_GLOBAL_SECONDS | int |0 | Used in git cli opeartion timeout | | false | + | CLUSTER_STATUS_CRON_TIME | int |15 | Cron schedule for cluster status on resource browser | | false | | CONSUMER_CONFIG_JSON | string | | | | false | | DEFAULT_LOG_TIME_LIMIT | int64 |1 | | | false | - | DEFAULT_TIMEOUT | float64 |3600 | | | false | - | DEVTRON_BOM_URL | string |https://raw.githubusercontent.com/devtron-labs/devtron/%s/charts/devtron/devtron-bom.yaml | | | false | + | DEFAULT_TIMEOUT | float64 |3600 | Timeout for CI to be completed | | false | + | DEVTRON_BOM_URL | string |https://raw.githubusercontent.com/devtron-labs/devtron/%s/charts/devtron/devtron-bom.yaml | Path to devtron-bom.yaml of devtron charts, used for module installation and devtron upgrade | | false | | DEVTRON_DEFAULT_NAMESPACE | string |devtroncd | | | false | - | DEVTRON_DEX_SECRET_NAMESPACE | string |devtroncd | | | false | + | DEVTRON_DEX_SECRET_NAMESPACE | string |devtroncd | Namespace of dex secret | | false | | DEVTRON_HELM_RELEASE_CHART_NAME | string |devtron-operator | | | false | - | DEVTRON_HELM_RELEASE_NAME | string |devtron | | | false | - | DEVTRON_HELM_RELEASE_NAMESPACE | string |devtroncd | | | false | - | DEVTRON_HELM_REPO_NAME | string |devtron | | | false | - | DEVTRON_HELM_REPO_URL | string |https://helm.devtron.ai | | | false | - | DEVTRON_INSTALLATION_TYPE | string | | | | false | + | DEVTRON_HELM_RELEASE_NAME | string |devtron | Name of the Devtron Helm release. | | false | + | DEVTRON_HELM_RELEASE_NAMESPACE | string |devtroncd | Namespace of the Devtron Helm release | | false | + | DEVTRON_HELM_REPO_NAME | string |devtron | Is used to install modules (stack manager) | | false | + | DEVTRON_HELM_REPO_URL | string |https://helm.devtron.ai | Is used to install modules (stack manager) | | false | + | DEVTRON_INSTALLATION_TYPE | string | | Devtron Installation type(EA/Full) | | false | | DEVTRON_MODULES_IDENTIFIER_IN_HELM_VALUES | string |installer.modules | | | false | | DEVTRON_SECRET_NAME | string |devtron-secret | | | false | - | DEVTRON_VERSION_IDENTIFIER_IN_HELM_VALUES | string |installer.release | | | false | - | DEX_CID | string |example-app | | | false | + | DEVTRON_VERSION_IDENTIFIER_IN_HELM_VALUES | string |installer.release | devtron operator version identifier in helm values yaml | | false | + | DEX_CID | string |example-app | dex client id | | false | | DEX_CLIENT_ID | string |argo-cd | | | false | - | DEX_CSTOREKEY | string | | | | false | - | DEX_JWTKEY | string | | | | false | - | DEX_RURL | string |http://127.0.0.1:8080/callback | | | false | - | DEX_SECRET | string | | | | false | - | DEX_URL | string | | | | false | - | ECR_REPO_NAME_PREFIX | string |test/ | | | false | - | ENABLE_ASYNC_ARGO_CD_INSTALL_DEVTRON_CHART | bool |false | | | false | - | ENABLE_ASYNC_INSTALL_DEVTRON_CHART | bool |false | | | false | - | EPHEMERAL_SERVER_VERSION_REGEX | string |v[1-9]\.\b(2[3-9]\|[3-9][0-9])\b.* | | | false | - | EVENT_URL | string |http://localhost:3000/notify | | | false | - | EXECUTE_WIRE_NIL_CHECKER | bool |false | | | false | - | EXPOSE_CI_METRICS | bool |false | | | false | - | FEATURE_RESTART_WORKLOAD_BATCH_SIZE | int |1 | | | false | - | FEATURE_RESTART_WORKLOAD_WORKER_POOL_SIZE | int |5 | | | false | - | FORCE_SECURITY_SCANNING | bool |false | | | false | - | GITOPS_REPO_PREFIX | string | | | | false | + | DEX_CSTOREKEY | string | | DEX CSTOREKEY. | | false | + | DEX_JWTKEY | string | | DEX JWT key. | | false | + | DEX_RURL | string |http://127.0.0.1:8080/callback | Dex redirect URL(http://argocd-dex-server.devtroncd:8080/callback) | | false | + | DEX_SCOPES | | | | | false | + | DEX_SECRET | string | | Dex secret | | false | + | DEX_URL | string | | Dex service endpoint with dex path(http://argocd-dex-server.devtroncd:5556/dex) | | false | + | ECR_REPO_NAME_PREFIX | string |test/ | Prefix for ECR repo to be created in does not exist | | false | + | ENABLE_ASYNC_ARGO_CD_INSTALL_DEVTRON_CHART | bool |false | To enable async installation of gitops application | | false | + | ENABLE_ASYNC_INSTALL_DEVTRON_CHART | bool |false | To enable async installation of no-gitops application | | false | + | EPHEMERAL_SERVER_VERSION_REGEX | string |v[1-9]\.\b(2[3-9]\|[3-9][0-9])\b.* | ephemeral containers support version regex that is compared with k8sServerVersion | | false | + | EVENT_URL | string |http://localhost:3000/notify | Notifier service url | | false | + | EXECUTE_WIRE_NIL_CHECKER | bool |false | checks for any nil pointer in wire.go | | false | + | EXPOSE_CI_METRICS | bool |false | To expose CI metrics | | false | + | FEATURE_RESTART_WORKLOAD_BATCH_SIZE | int |1 | restart workload retrieval batch size | | false | + | FEATURE_RESTART_WORKLOAD_WORKER_POOL_SIZE | int |5 | restart workload retrieval pool size | | false | + | FORCE_SECURITY_SCANNING | bool |false | By enabling this no one can disable image scaning on ci-pipeline from UI | | false | + | GITOPS_REPO_PREFIX | string | | Prefix for Gitops repo being creation for argocd application | | false | | GO_RUNTIME_ENV | string |production | | | false | - | GRAFANA_HOST | string |localhost | | | false | - | GRAFANA_NAMESPACE | string |devtroncd | | | false | - | GRAFANA_ORG_ID | int |2 | | | false | - | GRAFANA_PASSWORD | string |prom-operator | | | false | - | GRAFANA_PORT | string |8090 | | | false | - | GRAFANA_URL | string | | | | false | - | GRAFANA_USERNAME | string |admin | | | false | - | HIDE_IMAGE_TAGGING_HARD_DELETE | bool |false | | | false | - | IGNORE_AUTOCOMPLETE_AUTH_CHECK | bool |false | | | false | + | GRAFANA_HOST | string |localhost | Host URL for the grafana dashboard | | false | + | GRAFANA_NAMESPACE | string |devtroncd | Namespace for grafana | | false | + | GRAFANA_ORG_ID | int |2 | Org ID for grafana for application metrics | | false | + | GRAFANA_PASSWORD | string |prom-operator | Password for grafana dashboard | | false | + | GRAFANA_PORT | string |8090 | Port for grafana micro-service | | false | + | GRAFANA_URL | string | | Host URL for the grafana dashboard | | false | + | GRAFANA_USERNAME | string |admin | Username for grafana | | false | + | HIDE_IMAGE_TAGGING_HARD_DELETE | bool |false | Flag to hide the hard delete option in the image tagging service | | false | + | IGNORE_AUTOCOMPLETE_AUTH_CHECK | bool |false | flag for ignoring auth check in autocomplete apis. | | false | | INSTALLED_MODULES | | | List of installed modules given in helm values/yaml are written in cm and used by devtron to know which modules are given | security.trivy,security.clair | false | - | INSTALLER_CRD_NAMESPACE | string |devtroncd | | | false | - | INSTALLER_CRD_OBJECT_GROUP_NAME | string |installer.devtron.ai | | | false | - | INSTALLER_CRD_OBJECT_RESOURCE | string |installers | | | false | - | INSTALLER_CRD_OBJECT_VERSION | string |v1alpha1 | | | false | + | INSTALLER_CRD_NAMESPACE | string |devtroncd | namespace where Custom Resource Definitions get installed | | false | + | INSTALLER_CRD_OBJECT_GROUP_NAME | string |installer.devtron.ai | Devtron installer CRD group name, partially deprecated. | | false | + | INSTALLER_CRD_OBJECT_RESOURCE | string |installers | Devtron installer CRD resource name, partially deprecated | | false | + | INSTALLER_CRD_OBJECT_VERSION | string |v1alpha1 | version of the CRDs. default is v1alpha1 | | false | | IS_AIR_GAP_ENVIRONMENT | bool |false | | | false | - | JwtExpirationTime | int |120 | | | false | + | JwtExpirationTime | int |120 | JWT expiration time. | | false | | K8s_CLIENT_MAX_IDLE_CONNS_PER_HOST | int |25 | | | false | | K8s_TCP_IDLE_CONN_TIMEOUT | int |300 | | | false | | K8s_TCP_KEEPALIVE | int |30 | | | false | @@ -205,46 +206,46 @@ | K8s_TLS_HANDSHAKE_TIMEOUT | int |10 | | | false | | KUBELINK_GRPC_MAX_RECEIVE_MSG_SIZE | int |20 | | | false | | KUBELINK_GRPC_MAX_SEND_MSG_SIZE | int |4 | | | false | - | LENS_TIMEOUT | int |0 | | | false | - | LENS_URL | string |http://lens-milandevtron-service:80 | | | false | + | LENS_TIMEOUT | int |0 | Lens microservice timeout. | | false | + | LENS_URL | string |http://lens-milandevtron-service:80 | Lens micro-service URL | | false | | LIMIT_CI_CPU | string |0.5 | | | false | | LIMIT_CI_MEM | string |3G | | | false | - | LOGGER_DEV_MODE | bool |false | | | false | + | LOGGER_DEV_MODE | bool |false | Enables a different logger theme. | | false | | LOG_LEVEL | int |-1 | | | false | - | MAX_SESSION_PER_USER | int |5 | | | false | - | MODULE_METADATA_API_URL | string |https://api.devtron.ai/module?name=%s | | | false | + | MAX_SESSION_PER_USER | int |5 | max no of cluster terminal pods can be created by an user | | false | + | MODULE_METADATA_API_URL | string |https://api.devtron.ai/module?name=%s | Modules list and meta info will be fetched from this server, that is central api server of devtron. | | false | | MODULE_STATUS_HANDLING_CRON_DURATION_MIN | int |3 | | | false | | NATS_MSG_ACK_WAIT_IN_SECS | int |120 | | | false | | NATS_MSG_BUFFER_SIZE | int |-1 | | | false | | NATS_MSG_MAX_AGE | int |86400 | | | false | | NATS_MSG_PROCESSING_BATCH_SIZE | int |1 | | | false | | NATS_MSG_REPLICAS | int |0 | | | false | - | NOTIFICATION_MEDIUM | NotificationMedium |rest | | | false | - | OTEL_COLLECTOR_URL | string | | | | false | - | PARALLELISM_LIMIT_FOR_TAG_PROCESSING | int | | | | false | + | NOTIFICATION_MEDIUM | NotificationMedium |rest | notification medium | | false | + | OTEL_COLLECTOR_URL | string | | Opentelemetry URL | | false | + | PARALLELISM_LIMIT_FOR_TAG_PROCESSING | int | | App manual sync job parallel tag processing count. | | false | | PG_EXPORT_PROM_METRICS | bool |true | | | false | | PG_LOG_ALL_FAILURE_QUERIES | bool |true | | | false | | PG_LOG_ALL_QUERY | bool |false | | | false | | PG_LOG_SLOW_QUERY | bool |true | | | false | | PG_QUERY_DUR_THRESHOLD | int64 |5000 | | | false | - | PLUGIN_NAME | string |Pull images from container repository | | | false | - | PROPAGATE_EXTRA_LABELS | bool |false | | | false | - | PROXY_SERVICE_CONFIG | string |{} | | | false | + | PLUGIN_NAME | string |Pull images from container repository | Handles image retrieval from a container repository and triggers subsequent CI processes upon detecting new images.Current default plugin name: Pull Images from Container Repository. | | false | + | PROPAGATE_EXTRA_LABELS | bool |false | Add additional propagate labels like api.devtron.ai/appName, api.devtron.ai/envName, api.devtron.ai/project along with the user defined ones. | | false | + | PROXY_SERVICE_CONFIG | string |{} | Proxy configuration for micro-service to be accessible on orhcestrator ingress | | false | | REQ_CI_CPU | string |0.5 | | | false | | REQ_CI_MEM | string |3G | | | false | - | RESTRICT_TERMINAL_ACCESS_FOR_NON_SUPER_USER | bool |false | | | false | + | RESTRICT_TERMINAL_ACCESS_FOR_NON_SUPER_USER | bool |false | To restrict the cluster terminal from user having non-super admin acceess | | false | | RUNTIME_CONFIG_LOCAL_DEV | LocalDevMode |true | | | false | - | SCOPED_VARIABLE_ENABLED | bool |false | | | false | - | SCOPED_VARIABLE_FORMAT | string |@{{%s}} | | | false | - | SCOPED_VARIABLE_HANDLE_PRIMITIVES | bool |false | | | false | - | SCOPED_VARIABLE_NAME_REGEX | string |^[a-zA-Z][a-zA-Z0-9_-]{0,62}[a-zA-Z0-9]$ | | | false | - | SOCKET_DISCONNECT_DELAY_SECONDS | int |5 | | | false | - | SOCKET_HEARTBEAT_SECONDS | int |25 | | | false | + | SCOPED_VARIABLE_ENABLED | bool |false | To enable scoped variable option | | false | + | SCOPED_VARIABLE_FORMAT | string |@{{%s}} | Its a scope format for varialbe name. | | false | + | SCOPED_VARIABLE_HANDLE_PRIMITIVES | bool |false | This describe should we handle primitives or not in scoped variable template parsing. | | false | + | SCOPED_VARIABLE_NAME_REGEX | string |^[a-zA-Z][a-zA-Z0-9_-]{0,62}[a-zA-Z0-9]$ | Regex for scoped variable name that must passed this regex. | | false | + | SOCKET_DISCONNECT_DELAY_SECONDS | int |5 | The server closes a session when a client receiving connection have not been seen for a while.This delay is configured by this setting. By default the session is closed when a receiving connection wasn't seen for 5 seconds. | | false | + | SOCKET_HEARTBEAT_SECONDS | int |25 | In order to keep proxies and load balancers from closing long running http requests we need to pretend that the connection is active and send a heartbeat packet once in a while. This setting controls how often this is done. By default a heartbeat packet is sent every 25 seconds. | | false | | STREAM_CONFIG_JSON | string | | | | false | - | SYSTEM_VAR_PREFIX | string |DEVTRON_ | | | false | - | TERMINAL_POD_DEFAULT_NAMESPACE | string |default | | | false | - | TERMINAL_POD_INACTIVE_DURATION_IN_MINS | int |10 | | | false | - | TERMINAL_POD_STATUS_SYNC_In_SECS | int |600 | | | false | + | SYSTEM_VAR_PREFIX | string |DEVTRON_ | Scoped variable prefix, variable name must have this prefix. | | false | + | TERMINAL_POD_DEFAULT_NAMESPACE | string |default | Cluster terminal default namespace | | false | + | TERMINAL_POD_INACTIVE_DURATION_IN_MINS | int |10 | Timeout for cluster terminal to be inactive | | false | + | TERMINAL_POD_STATUS_SYNC_In_SECS | int |600 | this is the time interval at which the status of the cluster terminal pod | | false | | TEST_APP | string |orchestrator | | | false | | TEST_PG_ADDR | string |127.0.0.1 | | | false | | TEST_PG_DATABASE | string |orchestrator | | | false | @@ -252,63 +253,63 @@ | TEST_PG_PASSWORD | string |postgrespw | | | false | | TEST_PG_PORT | string |55000 | | | false | | TEST_PG_USER | string |postgres | | | false | - | TIMEOUT_FOR_FAILED_CI_BUILD | string |15 | | | false | - | TIMEOUT_IN_SECONDS | int |5 | | | false | + | TIMEOUT_FOR_FAILED_CI_BUILD | string |15 | Timeout for Failed CI build | | false | + | TIMEOUT_IN_SECONDS | int |5 | timeout to compute the urls from services and ingress objects of an application | | false | | USER_SESSION_DURATION_SECONDS | int |86400 | | | false | - | USE_ARTIFACT_LISTING_API_V2 | bool |true | | | false | + | USE_ARTIFACT_LISTING_API_V2 | bool |true | To use the V2 API for listing artifacts in Listing the images in pipeline | | false | | USE_CUSTOM_HTTP_TRANSPORT | bool |false | | | false | - | USE_GIT_CLI | bool |false | | | false | - | USE_RBAC_CREATION_V2 | bool |true | | | false | - | VARIABLE_CACHE_ENABLED | bool |true | | | false | - | VARIABLE_EXPRESSION_REGEX | string |@{{([^}]+)}} | | | false | - | WEBHOOK_TOKEN | string | | | | false | + | USE_GIT_CLI | bool |false | To enable git cli | | false | + | USE_RBAC_CREATION_V2 | bool |true | To use the V2 for RBAC creation | | false | + | VARIABLE_CACHE_ENABLED | bool |true | This is used to control caching of all the scope variables defined in the system. | | false | + | VARIABLE_EXPRESSION_REGEX | string |@{{([^}]+)}} | Scoped variable expression regex | | false | + | WEBHOOK_TOKEN | string | | If you want to continue using jenkins for CI then please provide this for authentication of requests | | false | ## GITOPS Related Environment Variables | Key | Type | Default Value | Description | Example | Deprecated | |-------|----------|-------------------|-------------------|-----------------------|------------------| - | ACD_CM | string |argocd-cm | | | false | - | ACD_NAMESPACE | string |devtroncd | | | false | - | ACD_PASSWORD | string | | | | false | - | ACD_USERNAME | string |admin | | | false | - | GITOPS_SECRET_NAME | string |devtron-gitops-secret | | | false | - | RESOURCE_LIST_FOR_REPLICAS | string |Deployment,Rollout,StatefulSet,ReplicaSet | | | false | - | RESOURCE_LIST_FOR_REPLICAS_BATCH_SIZE | int |5 | | | false | + | ACD_CM | string |argocd-cm | Name of the argocd CM | | false | + | ACD_NAMESPACE | string |devtroncd | To pass the argocd namespace | | false | + | ACD_PASSWORD | string | | Password for the Argocd (deprecated) | | false | + | ACD_USERNAME | string |admin | User name for argocd | | false | + | GITOPS_SECRET_NAME | string |devtron-gitops-secret | devtron-gitops-secret | | false | + | RESOURCE_LIST_FOR_REPLICAS | string |Deployment,Rollout,StatefulSet,ReplicaSet | this holds the list of k8s resource names which support replicas key. this list used in hibernate/un hibernate process | | false | + | RESOURCE_LIST_FOR_REPLICAS_BATCH_SIZE | int |5 | this the batch size to control no of above resources can be parsed in one go to determine hibernate status | | false | ## INFRA_SETUP Related Environment Variables | Key | Type | Default Value | Description | Example | Deprecated | |-------|----------|-------------------|-------------------|-----------------------|------------------| - | DASHBOARD_HOST | string |localhost | | | false | - | DASHBOARD_NAMESPACE | string |devtroncd | | | false | - | DASHBOARD_PORT | string |3000 | | | false | + | DASHBOARD_HOST | string |localhost | Dashboard micro-service URL | | false | + | DASHBOARD_NAMESPACE | string |devtroncd | Dashboard micro-service namespace | | false | + | DASHBOARD_PORT | string |3000 | Port for dashboard micro-service | | false | | DEX_HOST | string |http://localhost | | | false | | DEX_PORT | string |5556 | | | false | - | GIT_SENSOR_PROTOCOL | string |REST | | | false | - | GIT_SENSOR_TIMEOUT | int |0 | | | false | - | GIT_SENSOR_URL | string |127.0.0.1:7070 | | | false | - | HELM_CLIENT_URL | string |127.0.0.1:50051 | | | false | + | GIT_SENSOR_PROTOCOL | string |REST | Protocol to connect with git-sensor micro-service | | false | + | GIT_SENSOR_TIMEOUT | int |0 | Timeout for getting response from the git-sensor | | false | + | GIT_SENSOR_URL | string |127.0.0.1:7070 | git-sensor micro-service url | | false | + | HELM_CLIENT_URL | string |127.0.0.1:50051 | Kubelink micro-service url | | false | ## POSTGRES Related Environment Variables | Key | Type | Default Value | Description | Example | Deprecated | |-------|----------|-------------------|-------------------|-----------------------|------------------| | APP | string |orchestrator | Application name | | false | - | CASBIN_DATABASE | string |casbin | | | false | + | CASBIN_DATABASE | string |casbin | Database for casbin | | false | | PG_ADDR | string |127.0.0.1 | address of postgres service | postgresql-postgresql.devtroncd | false | | PG_DATABASE | string |orchestrator | postgres database to be made connection with | orchestrator, casbin, git_sensor, lens | false | | PG_PASSWORD | string |{password} | password for postgres, associated with PG_USER | confidential ;) | false | | PG_PORT | string |5432 | port of postgresql service | 5432 | false | - | PG_READ_TIMEOUT | int64 |30 | | | false | + | PG_READ_TIMEOUT | int64 |30 | Time out for read operation in postgres | | false | | PG_USER | string |postgres | user for postgres | postgres | false | - | PG_WRITE_TIMEOUT | int64 |30 | | | false | + | PG_WRITE_TIMEOUT | int64 |30 | Time out for write operation in postgres | | false | ## RBAC Related Environment Variables | Key | Type | Default Value | Description | Example | Deprecated | |-------|----------|-------------------|-------------------|-----------------------|------------------| - | ENFORCER_CACHE | bool |false | | | false | - | ENFORCER_CACHE_EXPIRATION_IN_SEC | int |86400 | | | false | - | ENFORCER_MAX_BATCH_SIZE | int |1 | | | false | - | USE_CASBIN_V2 | bool |true | | | false | + | ENFORCER_CACHE | bool |false | To Enable enforcer cache. | | false | + | ENFORCER_CACHE_EXPIRATION_IN_SEC | int |86400 | Expiration time (in seconds) for enforcer cache. | | false | + | ENFORCER_MAX_BATCH_SIZE | int |1 | Maximum batch size for the enforcer. | | false | + | USE_CASBIN_V2 | bool |true | To enable casbin V2 API | | false | diff --git a/go.mod b/go.mod index 392128e0b2..b9992c96fc 100644 --- a/go.mod +++ b/go.mod @@ -307,8 +307,8 @@ require ( replace ( github.com/argoproj/argo-workflows/v3 v3.5.13 => github.com/devtron-labs/argo-workflows/v3 v3.5.13 - github.com/devtron-labs/authenticator => github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250409180044-56ed13f8f5e5 - github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250409180044-56ed13f8f5e5 + github.com/devtron-labs/authenticator => github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250410055937-f65f6e252704 + github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250410055937-f65f6e252704 github.com/go-check/check => github.com/go-check/check v0.0.0-20180628173108-788fd7840127 github.com/googleapis/gnostic => github.com/googleapis/gnostic v0.5.5 k8s.io/api => k8s.io/api v0.29.7 diff --git a/go.sum b/go.sum index f0c1d38972..05a9382557 100644 --- a/go.sum +++ b/go.sum @@ -829,10 +829,10 @@ github.com/denisenkom/go-mssqldb v0.0.0-20200428022330-06a60b6afbbc h1:VRRKCwnzq github.com/denisenkom/go-mssqldb v0.0.0-20200428022330-06a60b6afbbc/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= github.com/devtron-labs/argo-workflows/v3 v3.5.13 h1:3pINq0gXOSeTw2z/vYe+j80lRpSN5Rp/8mfQORh8SmU= github.com/devtron-labs/argo-workflows/v3 v3.5.13/go.mod h1:/vqxcovDPT4zqr4DjR5v7CF8ggpY1l3TSa2CIG3jmjA= -github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250409180044-56ed13f8f5e5 h1:J/PT2HoFj3LtMJzMoUNDFFCGbPcVEt6X2SpEDgAqmFQ= -github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250409180044-56ed13f8f5e5/go.mod h1:5lv4Wfj5ERhhvDGXe2IeES6qxjvUVCcohaRwKnWBMNo= -github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250409180044-56ed13f8f5e5 h1:olj+0hnWMAg4lqqGAtX7sAdvAs6RbzQEcjHmER5MZTI= -github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250409180044-56ed13f8f5e5/go.mod h1:ceFKgQ2qm40PR95g5Xp2EClq7nDBKFTcglJ0JdsgClA= +github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250410055937-f65f6e252704 h1:xjTvh5f01PARce9TeKjpO6QRM1GaR2fh56/x6+HZcaI= +github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250410055937-f65f6e252704/go.mod h1:5lv4Wfj5ERhhvDGXe2IeES6qxjvUVCcohaRwKnWBMNo= +github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250410055937-f65f6e252704 h1:qA1Ec4flLuUozww0C3MnE3VGpuacBMyT6QxMyqECtrA= +github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250410055937-f65f6e252704/go.mod h1:ceFKgQ2qm40PR95g5Xp2EClq7nDBKFTcglJ0JdsgClA= github.com/devtron-labs/go-bitbucket v0.9.60-beta h1:VEx1jvDgdtDPS6A1uUFoaEi0l1/oLhbr+90xOwr6sDU= github.com/devtron-labs/go-bitbucket v0.9.60-beta/go.mod h1:GnuiCesvh8xyHeMCb+twm8lBR/kQzJYSKL28ZfObp1Y= github.com/devtron-labs/protos v0.0.3-0.20250323220609-ecf8a0f7305e h1:U6UdYbW8a7xn5IzFPd8cywjVVPfutGJCudjePAfL/Hs= diff --git a/vendor/github.com/devtron-labs/authenticator/client/k8sClient.go b/vendor/github.com/devtron-labs/authenticator/client/k8sClient.go index 9d1bf6c311..09f4b8171d 100644 --- a/vendor/github.com/devtron-labs/authenticator/client/k8sClient.go +++ b/vendor/github.com/devtron-labs/authenticator/client/k8sClient.go @@ -47,8 +47,8 @@ type K8sClient struct { } type RuntimeConfig struct { - LocalDevMode LocalDevMode `env:"RUNTIME_CONFIG_LOCAL_DEV" envDefault:"false" description:"Used in local development process. Allows to read from local kube config file path for default cluster operations."` - DevtronDefaultNamespaceName string `env:"DEVTRON_DEFAULT_NAMESPACE" envDefault:"devtroncd" description:"default namespace for devtron"` + LocalDevMode LocalDevMode `env:"RUNTIME_CONFIG_LOCAL_DEV" envDefault:"false"` + DevtronDefaultNamespaceName string `env:"DEVTRON_DEFAULT_NAMESPACE" envDefault:"devtroncd"` } func GetRuntimeConfig() (*RuntimeConfig, error) { diff --git a/vendor/github.com/devtron-labs/authenticator/client/oidcClient.go b/vendor/github.com/devtron-labs/authenticator/client/oidcClient.go index b7dcba2f99..551dec305e 100644 --- a/vendor/github.com/devtron-labs/authenticator/client/oidcClient.go +++ b/vendor/github.com/devtron-labs/authenticator/client/oidcClient.go @@ -44,9 +44,11 @@ func GetSettings(conf *DexConfig) (*oidc.Settings, error) { settings := &oidc.Settings{ URL: conf.Url, OIDCConfig: oidc.OIDCConfig{CLIClientID: conf.DexClientID, - ClientSecret: conf.DexClientSecret, - Issuer: proxyUrl, - ServerSecret: conf.ServerSecret}, + ClientSecret: conf.DexClientSecret, + Issuer: proxyUrl, + ServerSecret: conf.ServerSecret, + RequestedScopes: conf.DexScopes, + }, UserSessionDuration: time.Duration(conf.UserSessionDurationSeconds) * time.Second, AdminPasswordMtime: conf.AdminPasswordMtime, } @@ -88,7 +90,8 @@ type DexConfig struct { UserSessionDurationSeconds int `env:"USER_SESSION_DURATION_SECONDS" envDefault:"86400"` AdminPasswordMtime time.Time `json:"ADMIN_PASSWORD_MTIME"` DexConfigRaw string - DevtronSecretName string `env:"DEVTRON_SECRET_NAME" envDefault:"devtron-secret"` + DevtronSecretName string `env:"DEVTRON_SECRET_NAME" envDefault:"devtron-secret"` + DexScopes []string `env:"DEX_SCOPES" envDefault:"" envSeparator:","` } func (c *DexConfig) GetDexProxyUrl() (string, error) { diff --git a/vendor/github.com/devtron-labs/common-lib/pubsub-lib/JetStreamUtil.go b/vendor/github.com/devtron-labs/common-lib/pubsub-lib/JetStreamUtil.go index ae0ed13512..a4fb42eefa 100644 --- a/vendor/github.com/devtron-labs/common-lib/pubsub-lib/JetStreamUtil.go +++ b/vendor/github.com/devtron-labs/common-lib/pubsub-lib/JetStreamUtil.go @@ -125,10 +125,10 @@ type NatsTopic struct { } type ConfigJson struct { // StreamConfigJson is a json string of map[string]NatsStreamConfig - StreamConfigJson string `env:"STREAM_CONFIG_JSON" description:"this is json map of nats stream configurations per stream. eg: {"ORCHESTRATOR":{"max_age":3600,"replicas":1}}"` + StreamConfigJson string `env:"STREAM_CONFIG_JSON"` // ConsumerConfigJson is a json string of map[string]NatsConsumerConfig // eg: "{\"ARGO_PIPELINE_STATUS_UPDATE_DURABLE-1\" : \"{\"natsMsgProcessingBatchSize\" : 3, \"natsMsgBufferSize\" : 3, \"ackWaitInSecs\": 300}\"}" - ConsumerConfigJson string `env:"CONSUMER_CONFIG_JSON" description:"ConsumerConfigJson is a json string of map[string]NatsConsumerConfig eg: "{\"ARGO_PIPELINE_STATUS_UPDATE_DURABLE-1\" : \"{\"natsMsgProcessingBatchSize\" : 3, \"natsMsgBufferSize\" : 3, \"ackWaitInSecs\": 300}\"}""` + ConsumerConfigJson string `env:"CONSUMER_CONFIG_JSON"` } var natsTopicMapping = map[string]NatsTopic{ diff --git a/vendor/github.com/devtron-labs/common-lib/pubsub-lib/NatsClient.go b/vendor/github.com/devtron-labs/common-lib/pubsub-lib/NatsClient.go index bd95783ff0..1da3710cd4 100644 --- a/vendor/github.com/devtron-labs/common-lib/pubsub-lib/NatsClient.go +++ b/vendor/github.com/devtron-labs/common-lib/pubsub-lib/NatsClient.go @@ -32,11 +32,11 @@ type NatsClient struct { } type NatsClientConfig struct { - NatsServerHost string `env:"NATS_SERVER_HOST" envDefault:"nats://devtron-nats.devtroncd:4222" description:"Nats micro-service URL "` + NatsServerHost string `env:"NATS_SERVER_HOST" envDefault:"nats://devtron-nats.devtroncd:4222"` // consumer wise // NatsMsgProcessingBatchSize is the number of messages that will be processed in one go - NatsMsgProcessingBatchSize int `env:"NATS_MSG_PROCESSING_BATCH_SIZE" envDefault:"1" description:"NatsMsgProcessingBatchSize is the number of messages that will be processed in one go"` + NatsMsgProcessingBatchSize int `env:"NATS_MSG_PROCESSING_BATCH_SIZE" envDefault:"1"` // NatsMsgBufferSize is the number of messages that will be buffered in memory (channel size) // it is recommended to set this value equal to NatsMsgProcessingBatchSize as we want to process maximum messages in the buffer in one go. @@ -45,10 +45,10 @@ type NatsClientConfig struct { // NatsMsgBufferSize can be configured independently of NatsMsgProcessingBatchSize if needed by setting its value to positive value in env. // if NatsMsgBufferSize set to a non-positive value then it will take the value of NatsMsgProcessingBatchSize. // Note: always get this value by calling GetNatsMsgBufferSize method - NatsMsgBufferSize int `env:"NATS_MSG_BUFFER_SIZE" envDefault:"-1" description:"NatsMsgBufferSize is the number of messages that will be buffered in memory (channel size)"` - NatsMsgMaxAge int `env:"NATS_MSG_MAX_AGE" envDefault:"86400" description:"Age for the message to persist "` - NatsMsgAckWaitInSecs int `env:"NATS_MSG_ACK_WAIT_IN_SECS" envDefault:"120" description:"Time to wait for acknowledging the message"` - NatsMsgReplicas int `env:"NATS_MSG_REPLICAS" envDefault:"0" description:"Replica count for runnings nats instance"` + NatsMsgBufferSize int `env:"NATS_MSG_BUFFER_SIZE" envDefault:"-1"` + NatsMsgMaxAge int `env:"NATS_MSG_MAX_AGE" envDefault:"86400"` + NatsMsgAckWaitInSecs int `env:"NATS_MSG_ACK_WAIT_IN_SECS" envDefault:"120"` + NatsMsgReplicas int `env:"NATS_MSG_REPLICAS" envDefault:"0"` } func (ncc NatsClientConfig) GetNatsMsgBufferSize() int { diff --git a/vendor/github.com/devtron-labs/common-lib/utils/bean/bean.go b/vendor/github.com/devtron-labs/common-lib/utils/bean/bean.go index 3dcd80dcc5..ea16a2f721 100644 --- a/vendor/github.com/devtron-labs/common-lib/utils/bean/bean.go +++ b/vendor/github.com/devtron-labs/common-lib/utils/bean/bean.go @@ -66,10 +66,10 @@ type DockerRegistryInfo struct { type PgQueryMonitoringConfig struct { LogSlowQuery bool `env:"PG_LOG_SLOW_QUERY" envDefault:"true"` - LogAllQuery bool `env:"PG_LOG_ALL_QUERY" envDefault:"false" description:"Log all query ran by orchestrator in logs"` + LogAllQuery bool `env:"PG_LOG_ALL_QUERY" envDefault:"false"` LogAllFailureQueries bool `env:"PG_LOG_ALL_FAILURE_QUERIES" envDefault:"true"` - ExportPromMetrics bool `env:"PG_EXPORT_PROM_METRICS" envDefault:"true" description:"To expose postgres metrics"` - QueryDurationThreshold int64 `env:"PG_QUERY_DUR_THRESHOLD" envDefault:"5000" description:"if the particular querry execution should cross the threshold , then the querry should log."` + ExportPromMetrics bool `env:"PG_EXPORT_PROM_METRICS" envDefault:"true"` + QueryDurationThreshold int64 `env:"PG_QUERY_DUR_THRESHOLD" envDefault:"5000"` ServiceName string } diff --git a/vendor/github.com/devtron-labs/common-lib/utils/grpc/GrpcConfig.go b/vendor/github.com/devtron-labs/common-lib/utils/grpc/GrpcConfig.go index 603dc262ce..0ac82eedac 100644 --- a/vendor/github.com/devtron-labs/common-lib/utils/grpc/GrpcConfig.go +++ b/vendor/github.com/devtron-labs/common-lib/utils/grpc/GrpcConfig.go @@ -3,8 +3,8 @@ package grpc import "github.com/caarlos0/env" type Configuration struct { - KubelinkMaxRecvMsgSize int `env:"KUBELINK_GRPC_MAX_RECEIVE_MSG_SIZE" envDefault:"20" description:"Message size to receive from kubelink"` // In mb - KubelinkMaxSendMsgSize int `env:"KUBELINK_GRPC_MAX_SEND_MSG_SIZE" envDefault:"4" description:"Message size to send to kubelink"` // In mb + KubelinkMaxRecvMsgSize int `env:"KUBELINK_GRPC_MAX_RECEIVE_MSG_SIZE" envDefault:"20"` // In mb + KubelinkMaxSendMsgSize int `env:"KUBELINK_GRPC_MAX_SEND_MSG_SIZE" envDefault:"4"` // In mb } func GetConfiguration() (*Configuration, error) { diff --git a/vendor/github.com/devtron-labs/common-lib/utils/k8s/bean.go b/vendor/github.com/devtron-labs/common-lib/utils/k8s/bean.go index 7f83c24891..21f0a21be3 100644 --- a/vendor/github.com/devtron-labs/common-lib/utils/k8s/bean.go +++ b/vendor/github.com/devtron-labs/common-lib/utils/k8s/bean.go @@ -169,12 +169,12 @@ func GetResourceKey(obj *unstructured.Unstructured) ResourceKey { } type CustomK8sHttpTransportConfig struct { - UseCustomTransport bool `env:"USE_CUSTOM_HTTP_TRANSPORT" envDefault:"false" description:"There is an issue in updating cluster bearer token with same cluster url due to transport layer token caching in k8s client lib. so we added a custom transport while calling k8s api server. this flag controls the usage of this transport.(custom or k8s)"` - TimeOut int `env:"K8s_TCP_TIMEOUT" envDefault:"30" description:"is the maximum amount of time a dial will wait for a connect to complete, required only if USE_CUSTOM_HTTP_TRANSPORT is true."` - KeepAlive int `env:"K8s_TCP_KEEPALIVE" envDefault:"30" description:"specifies the interval between keep-alive probes for an active, required only if USE_CUSTOM_HTTP_TRANSPORT is true."` - TLSHandshakeTimeout int `env:"K8s_TLS_HANDSHAKE_TIMEOUT" envDefault:"10" description:"specifies the maximum amount of time to wait for a TLS handshake. Zero means no timeout, required only if USE_CUSTOM_HTTP_TRANSPORT is true."` - MaxIdleConnsPerHost int `env:"K8s_CLIENT_MAX_IDLE_CONNS_PER_HOST" envDefault:"25" description:"k8s client max idle connections per host"` - IdleConnTimeout int `env:"K8s_TCP_IDLE_CONN_TIMEOUT" envDefault:"300" description:"maximum amount of time an idle (keep-alive) connection will remain idle before closing itself, required only if USE_CUSTOM_HTTP_TRANSPORT is true."` + UseCustomTransport bool `env:"USE_CUSTOM_HTTP_TRANSPORT" envDefault:"false"` + TimeOut int `env:"K8s_TCP_TIMEOUT" envDefault:"30"` + KeepAlive int `env:"K8s_TCP_KEEPALIVE" envDefault:"30"` + TLSHandshakeTimeout int `env:"K8s_TLS_HANDSHAKE_TIMEOUT" envDefault:"10"` + MaxIdleConnsPerHost int `env:"K8s_CLIENT_MAX_IDLE_CONNS_PER_HOST" envDefault:"25"` + IdleConnTimeout int `env:"K8s_TCP_IDLE_CONN_TIMEOUT" envDefault:"300"` } type LocalDevMode bool diff --git a/vendor/modules.txt b/vendor/modules.txt index fdb1c66d2e..0203d2c3da 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -472,7 +472,7 @@ github.com/davecgh/go-spew/spew # github.com/deckarep/golang-set v1.8.0 ## explicit; go 1.17 github.com/deckarep/golang-set -# github.com/devtron-labs/authenticator v0.4.35-0.20240809073103-6e11da8083f8 => github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250409180044-56ed13f8f5e5 +# github.com/devtron-labs/authenticator v0.4.35-0.20240809073103-6e11da8083f8 => github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250410055937-f65f6e252704 ## explicit; go 1.21 github.com/devtron-labs/authenticator/apiToken github.com/devtron-labs/authenticator/client @@ -480,7 +480,7 @@ github.com/devtron-labs/authenticator/jwt github.com/devtron-labs/authenticator/middleware github.com/devtron-labs/authenticator/oidc github.com/devtron-labs/authenticator/password -# github.com/devtron-labs/common-lib v0.18.1-0.20241001061923-eda545dc839e => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250409180044-56ed13f8f5e5 +# github.com/devtron-labs/common-lib v0.18.1-0.20241001061923-eda545dc839e => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250410055937-f65f6e252704 ## explicit; go 1.21 github.com/devtron-labs/common-lib/async github.com/devtron-labs/common-lib/blob-storage @@ -2350,8 +2350,8 @@ xorm.io/xorm/log xorm.io/xorm/names xorm.io/xorm/schemas xorm.io/xorm/tags -# github.com/devtron-labs/authenticator => github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250409180044-56ed13f8f5e5 -# github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250409180044-56ed13f8f5e5 +# github.com/devtron-labs/authenticator => github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250410055937-f65f6e252704 +# github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250410055937-f65f6e252704 # github.com/go-check/check => github.com/go-check/check v0.0.0-20180628173108-788fd7840127 # github.com/googleapis/gnostic => github.com/googleapis/gnostic v0.5.5 # k8s.io/api => k8s.io/api v0.29.7 From 8cc6f73f8c1c70758c445e493336310f4ab6f9c2 Mon Sep 17 00:00:00 2001 From: prakhar katiyar Date: Fri, 11 Apr 2025 12:40:39 +0530 Subject: [PATCH 22/34] review commnets --- .../configure/BuildPipelineRestHandler.go | 77 ++++++++++++++++--- .../BuildPipelineRestHandlerHelper.go | 16 ---- 2 files changed, 66 insertions(+), 27 deletions(-) diff --git a/api/restHandler/app/pipeline/configure/BuildPipelineRestHandler.go b/api/restHandler/app/pipeline/configure/BuildPipelineRestHandler.go index 747d723588..15129f9028 100644 --- a/api/restHandler/app/pipeline/configure/BuildPipelineRestHandler.go +++ b/api/restHandler/app/pipeline/configure/BuildPipelineRestHandler.go @@ -144,7 +144,12 @@ func (handler *PipelineConfigRestHandlerImpl) CreateCiConfig(w http.ResponseWrit } createResp, err := handler.pipelineBuilder.CreateCiPipeline(&createRequest) - handler.handleServiceError(w, err, createResp, "create ci config", createRequest) + if err != nil { + handler.Logger.Errorw("service err, create", "err", err, "create request", createRequest) + common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) + return + } + common.WriteJsonResp(w, err, createResp, http.StatusOK) } func (handler *PipelineConfigRestHandlerImpl) UpdateCiTemplate(w http.ResponseWriter, r *http.Request) { @@ -172,7 +177,12 @@ func (handler *PipelineConfigRestHandlerImpl) UpdateCiTemplate(w http.ResponseWr } createResp, err := handler.pipelineBuilder.UpdateCiTemplate(&configRequest) - handler.handleServiceError(w, err, createResp, "UpdateCiTemplate", configRequest) + if err != nil { + handler.Logger.Errorw("service err", "err", err, "context", "UpdateCiTemplate", "data", configRequest) + common.WriteJsonResp(w, err, createResp, http.StatusInternalServerError) + return + } + common.WriteJsonResp(w, nil, createResp, http.StatusOK) } func (handler *PipelineConfigRestHandlerImpl) UpdateBranchCiPipelinesWithRegex(w http.ResponseWriter, r *http.Request) { @@ -218,7 +228,12 @@ func (handler *PipelineConfigRestHandlerImpl) UpdateBranchCiPipelinesWithRegex(w //if include/exclude configured showAll will include excluded materials also in list, if not configured it will ignore this flag resp, err := handler.ciHandler.FetchMaterialsByPipelineId(patchRequest.Id, false) - handler.handleServiceError(w, err, resp, "FetchMaterials", map[string]interface{}{"pipelineId": patchRequest.Id}) + if err != nil { + handler.Logger.Errorw("service err", "err", err, "context", "FetchMaterials", "data", map[string]interface{}{"pipelineId": patchRequest.Id}) + common.WriteJsonResp(w, err, resp, http.StatusInternalServerError) + return + } + common.WriteJsonResp(w, nil, resp, http.StatusOK) } func (handler *PipelineConfigRestHandlerImpl) parseSourceChangeRequest(w http.ResponseWriter, r *http.Request) (*bean.CiMaterialPatchRequest, int32, error) { @@ -525,7 +540,12 @@ func (handler *PipelineConfigRestHandlerImpl) GetCiPipeline(w http.ResponseWrite } ciConf, err := handler.pipelineBuilder.GetCiPipelineRespResolved(appId) - handler.handleServiceError(w, err, ciConf, "GetCiPipelineRespResolved", map[string]interface{}{"appId": appId}) + if err != nil { + handler.Logger.Errorw("service err", "err", err, "context", "GetCiPipelineRespResolved", "data", map[string]interface{}{"appId": appId}) + common.WriteJsonResp(w, err, ciConf, http.StatusInternalServerError) + return + } + common.WriteJsonResp(w, nil, ciConf, http.StatusOK) } func (handler *PipelineConfigRestHandlerImpl) GetExternalCi(w http.ResponseWriter, r *http.Request) { @@ -542,7 +562,12 @@ func (handler *PipelineConfigRestHandlerImpl) GetExternalCi(w http.ResponseWrite } ciConf, err := handler.pipelineBuilder.GetExternalCi(appId) - handler.handleServiceError(w, err, ciConf, "GetExternalCi", map[string]interface{}{"appId": appId}) + if err != nil { + handler.Logger.Errorw("service err", "err", err, "context", "GetExternalCi", "data", map[string]interface{}{"appId": appId}) + common.WriteJsonResp(w, err, ciConf, http.StatusInternalServerError) + return + } + common.WriteJsonResp(w, nil, ciConf, http.StatusOK) } func (handler *PipelineConfigRestHandlerImpl) GetExternalCiById(w http.ResponseWriter, r *http.Request) { @@ -564,7 +589,12 @@ func (handler *PipelineConfigRestHandlerImpl) GetExternalCiById(w http.ResponseW } ciConf, err := handler.pipelineBuilder.GetExternalCiById(appId, externalCiId) - handler.handleServiceError(w, err, ciConf, "GetExternalCiById", map[string]interface{}{"appId": appId, "externalCiId": externalCiId}) + if err != nil { + handler.Logger.Errorw("service err", "err", err, "context", "GetExternalCiById", "data", map[string]interface{}{"appId": appId, "externalCiId": externalCiId}) + common.WriteJsonResp(w, err, ciConf, http.StatusInternalServerError) + return + } + common.WriteJsonResp(w, nil, ciConf, http.StatusOK) } func (handler *PipelineConfigRestHandlerImpl) validateCiTriggerRBAC(token string, ciPipelineId, triggerEnvironmentId int) error { @@ -719,7 +749,12 @@ func (handler *PipelineConfigRestHandlerImpl) FetchMaterials(w http.ResponseWrit } resp, err := handler.ciHandler.FetchMaterialsByPipelineId(pipelineId, showAll) - handler.handleServiceError(w, err, resp, "FetchMaterials", map[string]interface{}{"pipelineId": pipelineId}) + if err != nil { + handler.Logger.Errorw("service err", "err", err, "context", "FetchMaterials", "data", map[string]interface{}{"pipelineId": pipelineId}) + common.WriteJsonResp(w, err, resp, http.StatusInternalServerError) + return + } + common.WriteJsonResp(w, nil, resp, http.StatusOK) } func (handler *PipelineConfigRestHandlerImpl) FetchMaterialsByMaterialId(w http.ResponseWriter, r *http.Request) { @@ -758,7 +793,12 @@ func (handler *PipelineConfigRestHandlerImpl) FetchMaterialsByMaterialId(w http. } resp, err := handler.ciHandler.FetchMaterialsByPipelineIdAndGitMaterialId(pipelineId, gitMaterialId, showAll) - handler.handleServiceError(w, err, resp, "FetchMaterialsByMaterialId", map[string]interface{}{"pipelineId": pipelineId, "gitMaterialId": gitMaterialId}) + if err != nil { + handler.Logger.Errorw("service err", "err", err, "context", "FetchMaterialsByMaterialId", "data", map[string]interface{}{"pipelineId": pipelineId, "gitMaterialId": gitMaterialId}) + common.WriteJsonResp(w, err, resp, http.StatusInternalServerError) + return + } + common.WriteJsonResp(w, nil, resp, http.StatusOK) } func (handler *PipelineConfigRestHandlerImpl) RefreshMaterials(w http.ResponseWriter, r *http.Request) { @@ -789,7 +829,12 @@ func (handler *PipelineConfigRestHandlerImpl) RefreshMaterials(w http.ResponseWr } resp, err := handler.ciHandler.RefreshMaterialByCiPipelineMaterialId(material.Id) - handler.handleServiceError(w, err, resp, "RefreshMaterials", map[string]interface{}{"gitMaterialId": gitMaterialId}) + if err != nil { + handler.Logger.Errorw("service err", "err", err, "context", "RefreshMaterials", "data", map[string]interface{}{"gitMaterialId": gitMaterialId}) + common.WriteJsonResp(w, err, resp, http.StatusInternalServerError) + return + } + common.WriteJsonResp(w, nil, resp, http.StatusOK) } func (handler *PipelineConfigRestHandlerImpl) GetCiPipelineMin(w http.ResponseWriter, r *http.Request) { @@ -826,7 +871,12 @@ func (handler *PipelineConfigRestHandlerImpl) GetCiPipelineMin(w http.ResponseWr } ciPipelines, err := handler.pipelineBuilder.GetCiPipelineMin(appId, envIds) - handler.handleServiceError(w, err, ciPipelines, "GetCiPipelineMin", map[string]interface{}{"appId": appId}) + if err != nil { + handler.Logger.Errorw("service err", "err", err, "context", "GetCiPipelineMin", "data", map[string]interface{}{"appId": appId}) + common.WriteJsonResp(w, err, ciPipelines, http.StatusInternalServerError) + return + } + common.WriteJsonResp(w, nil, ciPipelines, http.StatusOK) } func (handler *PipelineConfigRestHandlerImpl) DownloadCiWorkflowArtifacts(w http.ResponseWriter, r *http.Request) { @@ -921,7 +971,12 @@ func (handler *PipelineConfigRestHandlerImpl) GetHistoricBuildLogs(w http.Respon } resp, err := handler.ciHandler.GetHistoricBuildLogs(workflowId, nil) - handler.handleServiceError(w, err, resp, "GetHistoricBuildLogs", map[string]interface{}{"pipelineId": pipelineId, "workflowId": workflowId}) + if err != nil { + handler.Logger.Errorw("service err", "err", err, "context", "GetHistoricBuildLogs", "data", map[string]interface{}{"pipelineId": pipelineId, "workflowId": workflowId}) + common.WriteJsonResp(w, err, resp, http.StatusInternalServerError) + return + } + common.WriteJsonResp(w, nil, resp, http.StatusOK) } func (handler *PipelineConfigRestHandlerImpl) GetBuildHistory(w http.ResponseWriter, r *http.Request) { diff --git a/api/restHandler/app/pipeline/configure/BuildPipelineRestHandlerHelper.go b/api/restHandler/app/pipeline/configure/BuildPipelineRestHandlerHelper.go index f59829f57e..d3bbd75d0f 100644 --- a/api/restHandler/app/pipeline/configure/BuildPipelineRestHandlerHelper.go +++ b/api/restHandler/app/pipeline/configure/BuildPipelineRestHandlerHelper.go @@ -21,7 +21,6 @@ import ( "fmt" "github.com/devtron-labs/devtron/api/restHandler/common" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" - "github.com/devtron-labs/devtron/internal/util" "github.com/devtron-labs/devtron/pkg/auth/authorisation/casbin" "github.com/devtron-labs/devtron/pkg/bean" "net/http" @@ -122,21 +121,6 @@ func (handler *PipelineConfigRestHandlerImpl) getCiPipelineWithAuth(w http.Respo return ciPipeline, true } -// handleServiceError handles service errors and writes appropriate response -func (handler *PipelineConfigRestHandlerImpl) handleServiceError(w http.ResponseWriter, err error, resp interface{}, logContext string, logData interface{}) { - if err != nil { - if util.IsErrNoRows(err) { - err = &util.ApiError{Code: "404", HttpStatusCode: http.StatusNotFound, UserMessage: "no data found"} - common.WriteJsonResp(w, err, nil, http.StatusOK) - } else { - handler.Logger.Errorw("service err", "err", err, "context", logContext, "data", logData) - common.WriteJsonResp(w, err, resp, http.StatusInternalServerError) - } - return - } - common.WriteJsonResp(w, nil, resp, http.StatusOK) -} - // getQueryParamBool gets a boolean query parameter from the request func (handler *PipelineConfigRestHandlerImpl) getQueryParamBool(r *http.Request, paramName string, defaultValue bool) bool { v := r.URL.Query() From e114028fa99bd614865e1ec2684a758099122a1c Mon Sep 17 00:00:00 2001 From: prakhar katiyar Date: Fri, 11 Apr 2025 13:00:27 +0530 Subject: [PATCH 23/34] review comments --- .../configure/BuildPipelineRestHandler.go | 38 ++++++++++--------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/api/restHandler/app/pipeline/configure/BuildPipelineRestHandler.go b/api/restHandler/app/pipeline/configure/BuildPipelineRestHandler.go index 15129f9028..ec552d8e2b 100644 --- a/api/restHandler/app/pipeline/configure/BuildPipelineRestHandler.go +++ b/api/restHandler/app/pipeline/configure/BuildPipelineRestHandler.go @@ -178,8 +178,8 @@ func (handler *PipelineConfigRestHandlerImpl) UpdateCiTemplate(w http.ResponseWr createResp, err := handler.pipelineBuilder.UpdateCiTemplate(&configRequest) if err != nil { - handler.Logger.Errorw("service err", "err", err, "context", "UpdateCiTemplate", "data", configRequest) - common.WriteJsonResp(w, err, createResp, http.StatusInternalServerError) + handler.Logger.Errorw("service err, UpdateCiTemplate", "UpdateCiTemplate", configRequest, "err", err) + common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) return } common.WriteJsonResp(w, nil, createResp, http.StatusOK) @@ -229,7 +229,7 @@ func (handler *PipelineConfigRestHandlerImpl) UpdateBranchCiPipelinesWithRegex(w //if include/exclude configured showAll will include excluded materials also in list, if not configured it will ignore this flag resp, err := handler.ciHandler.FetchMaterialsByPipelineId(patchRequest.Id, false) if err != nil { - handler.Logger.Errorw("service err", "err", err, "context", "FetchMaterials", "data", map[string]interface{}{"pipelineId": patchRequest.Id}) + handler.Logger.Errorw("service err, FetchMaterials", "pipelineId", patchRequest.Id, "err", err) common.WriteJsonResp(w, err, resp, http.StatusInternalServerError) return } @@ -273,7 +273,7 @@ func (handler *PipelineConfigRestHandlerImpl) parseBulkSourceChangeRequest(w htt err = handler.validator.Struct(patchRequest) if err != nil { - handler.Logger.Errorw("request err, BulkPatchCiPipeline", "err", err, "BulkPatchCiPipeline", patchRequest) + handler.Logger.Errorw("request err, BulkPatchCiPipeline", "BulkPatchCiPipeline", patchRequest, "err", err) common.WriteJsonResp(w, err, nil, http.StatusBadRequest) return nil, 0, err } @@ -346,7 +346,7 @@ func (handler *PipelineConfigRestHandlerImpl) PatchCiMaterialSourceWithAppIdsAnd // Here passing the checkAppSpecificAccess func to check RBAC bulkPatchResponse, err := handler.pipelineBuilder.BulkPatchCiMaterialSource(bulkPatchRequest, userId, token, handler.checkAppSpecificAccess) if err != nil { - handler.Logger.Errorw("service err, BulkPatchCiPipelines", "err", err, "BulkPatchCiPipelines", bulkPatchRequest) + handler.Logger.Errorw("service err, BulkPatchCiPipelines", "BulkPatchCiPipelines", bulkPatchRequest, "err", err) common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) return } @@ -445,7 +445,7 @@ func (handler *PipelineConfigRestHandlerImpl) PatchCiPipelines(w http.ResponseWr common.WriteJsonResp(w, err, nil, http.StatusBadRequest) return } - handler.Logger.Errorw("service err, PatchCiPipelines", "err", err, "PatchCiPipelines", patchRequest) + handler.Logger.Errorw("service err, PatchCiPipelines", "PatchCiPipelines", patchRequest, "err", err) common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) return } @@ -541,7 +541,7 @@ func (handler *PipelineConfigRestHandlerImpl) GetCiPipeline(w http.ResponseWrite ciConf, err := handler.pipelineBuilder.GetCiPipelineRespResolved(appId) if err != nil { - handler.Logger.Errorw("service err", "err", err, "context", "GetCiPipelineRespResolved", "data", map[string]interface{}{"appId": appId}) + handler.Logger.Errorw("service err, GetCiPipelineRespResolved", "appId", appId, "err", err) common.WriteJsonResp(w, err, ciConf, http.StatusInternalServerError) return } @@ -563,8 +563,8 @@ func (handler *PipelineConfigRestHandlerImpl) GetExternalCi(w http.ResponseWrite ciConf, err := handler.pipelineBuilder.GetExternalCi(appId) if err != nil { - handler.Logger.Errorw("service err", "err", err, "context", "GetExternalCi", "data", map[string]interface{}{"appId": appId}) - common.WriteJsonResp(w, err, ciConf, http.StatusInternalServerError) + handler.Logger.Errorw("service err, GetExternalCi", "err", err, "appId", appId) + common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) return } common.WriteJsonResp(w, nil, ciConf, http.StatusOK) @@ -590,8 +590,8 @@ func (handler *PipelineConfigRestHandlerImpl) GetExternalCiById(w http.ResponseW ciConf, err := handler.pipelineBuilder.GetExternalCiById(appId, externalCiId) if err != nil { - handler.Logger.Errorw("service err", "err", err, "context", "GetExternalCiById", "data", map[string]interface{}{"appId": appId, "externalCiId": externalCiId}) - common.WriteJsonResp(w, err, ciConf, http.StatusInternalServerError) + handler.Logger.Errorw("service err, GetExternalCiById", "err", err, "appId", appId) + common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) return } common.WriteJsonResp(w, nil, ciConf, http.StatusOK) @@ -794,7 +794,7 @@ func (handler *PipelineConfigRestHandlerImpl) FetchMaterialsByMaterialId(w http. resp, err := handler.ciHandler.FetchMaterialsByPipelineIdAndGitMaterialId(pipelineId, gitMaterialId, showAll) if err != nil { - handler.Logger.Errorw("service err", "err", err, "context", "FetchMaterialsByMaterialId", "data", map[string]interface{}{"pipelineId": pipelineId, "gitMaterialId": gitMaterialId}) + handler.Logger.Errorw("service err, FetchMaterials", "err", err, "pipelineId", pipelineId) common.WriteJsonResp(w, err, resp, http.StatusInternalServerError) return } @@ -830,7 +830,7 @@ func (handler *PipelineConfigRestHandlerImpl) RefreshMaterials(w http.ResponseWr resp, err := handler.ciHandler.RefreshMaterialByCiPipelineMaterialId(material.Id) if err != nil { - handler.Logger.Errorw("service err", "err", err, "context", "RefreshMaterials", "data", map[string]interface{}{"gitMaterialId": gitMaterialId}) + handler.Logger.Errorw("service err, RefreshMaterials", "err", err, "gitMaterialId", gitMaterialId) common.WriteJsonResp(w, err, resp, http.StatusInternalServerError) return } @@ -872,9 +872,13 @@ func (handler *PipelineConfigRestHandlerImpl) GetCiPipelineMin(w http.ResponseWr ciPipelines, err := handler.pipelineBuilder.GetCiPipelineMin(appId, envIds) if err != nil { - handler.Logger.Errorw("service err", "err", err, "context", "GetCiPipelineMin", "data", map[string]interface{}{"appId": appId}) - common.WriteJsonResp(w, err, ciPipelines, http.StatusInternalServerError) - return + handler.Logger.Errorw("service err, GetCiPipelineMin", "err", err, "appId", appId) + if util.IsErrNoRows(err) { + err = &util.ApiError{Code: "404", HttpStatusCode: http.StatusNotFound, UserMessage: "no data found"} + common.WriteJsonResp(w, err, nil, http.StatusOK) + } else { + common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) + } } common.WriteJsonResp(w, nil, ciPipelines, http.StatusOK) } @@ -972,7 +976,7 @@ func (handler *PipelineConfigRestHandlerImpl) GetHistoricBuildLogs(w http.Respon resp, err := handler.ciHandler.GetHistoricBuildLogs(workflowId, nil) if err != nil { - handler.Logger.Errorw("service err", "err", err, "context", "GetHistoricBuildLogs", "data", map[string]interface{}{"pipelineId": pipelineId, "workflowId": workflowId}) + handler.Logger.Errorw("service err, GetHistoricBuildLogs", "err", err, "pipelineId", pipelineId, "workflowId", workflowId) common.WriteJsonResp(w, err, resp, http.StatusInternalServerError) return } From 09fc9f873cbe5e8c945badcbf01aa6a6a36ff941 Mon Sep 17 00:00:00 2001 From: Vikram Singh Date: Mon, 14 Apr 2025 13:28:02 +0530 Subject: [PATCH 24/34] vendor update --- go.mod | 4 ++-- go.sum | 8 ++++---- vendor/modules.txt | 8 ++++---- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/go.mod b/go.mod index b9992c96fc..eeb86de322 100644 --- a/go.mod +++ b/go.mod @@ -307,8 +307,8 @@ require ( replace ( github.com/argoproj/argo-workflows/v3 v3.5.13 => github.com/devtron-labs/argo-workflows/v3 v3.5.13 - github.com/devtron-labs/authenticator => github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250410055937-f65f6e252704 - github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250410055937-f65f6e252704 + github.com/devtron-labs/authenticator => github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250414075707-6076f6f97fb9 + github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250414075707-6076f6f97fb9 github.com/go-check/check => github.com/go-check/check v0.0.0-20180628173108-788fd7840127 github.com/googleapis/gnostic => github.com/googleapis/gnostic v0.5.5 k8s.io/api => k8s.io/api v0.29.7 diff --git a/go.sum b/go.sum index 05a9382557..1555df3ec7 100644 --- a/go.sum +++ b/go.sum @@ -829,10 +829,10 @@ github.com/denisenkom/go-mssqldb v0.0.0-20200428022330-06a60b6afbbc h1:VRRKCwnzq github.com/denisenkom/go-mssqldb v0.0.0-20200428022330-06a60b6afbbc/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= github.com/devtron-labs/argo-workflows/v3 v3.5.13 h1:3pINq0gXOSeTw2z/vYe+j80lRpSN5Rp/8mfQORh8SmU= github.com/devtron-labs/argo-workflows/v3 v3.5.13/go.mod h1:/vqxcovDPT4zqr4DjR5v7CF8ggpY1l3TSa2CIG3jmjA= -github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250410055937-f65f6e252704 h1:xjTvh5f01PARce9TeKjpO6QRM1GaR2fh56/x6+HZcaI= -github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250410055937-f65f6e252704/go.mod h1:5lv4Wfj5ERhhvDGXe2IeES6qxjvUVCcohaRwKnWBMNo= -github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250410055937-f65f6e252704 h1:qA1Ec4flLuUozww0C3MnE3VGpuacBMyT6QxMyqECtrA= -github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250410055937-f65f6e252704/go.mod h1:ceFKgQ2qm40PR95g5Xp2EClq7nDBKFTcglJ0JdsgClA= +github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250414075707-6076f6f97fb9 h1:z72S6KALLrQ2VmW785FuPZpYJTFSzq1QflZJcQv8/O0= +github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250414075707-6076f6f97fb9/go.mod h1:5lv4Wfj5ERhhvDGXe2IeES6qxjvUVCcohaRwKnWBMNo= +github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250414075707-6076f6f97fb9 h1:o/wKBkZ6aGrFhSsDGSyQVTD3uurGH3db4zdWjRVLkI8= +github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250414075707-6076f6f97fb9/go.mod h1:ceFKgQ2qm40PR95g5Xp2EClq7nDBKFTcglJ0JdsgClA= github.com/devtron-labs/go-bitbucket v0.9.60-beta h1:VEx1jvDgdtDPS6A1uUFoaEi0l1/oLhbr+90xOwr6sDU= github.com/devtron-labs/go-bitbucket v0.9.60-beta/go.mod h1:GnuiCesvh8xyHeMCb+twm8lBR/kQzJYSKL28ZfObp1Y= github.com/devtron-labs/protos v0.0.3-0.20250323220609-ecf8a0f7305e h1:U6UdYbW8a7xn5IzFPd8cywjVVPfutGJCudjePAfL/Hs= diff --git a/vendor/modules.txt b/vendor/modules.txt index 0203d2c3da..72d3319694 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -472,7 +472,7 @@ github.com/davecgh/go-spew/spew # github.com/deckarep/golang-set v1.8.0 ## explicit; go 1.17 github.com/deckarep/golang-set -# github.com/devtron-labs/authenticator v0.4.35-0.20240809073103-6e11da8083f8 => github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250410055937-f65f6e252704 +# github.com/devtron-labs/authenticator v0.4.35-0.20240809073103-6e11da8083f8 => github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250414075707-6076f6f97fb9 ## explicit; go 1.21 github.com/devtron-labs/authenticator/apiToken github.com/devtron-labs/authenticator/client @@ -480,7 +480,7 @@ github.com/devtron-labs/authenticator/jwt github.com/devtron-labs/authenticator/middleware github.com/devtron-labs/authenticator/oidc github.com/devtron-labs/authenticator/password -# github.com/devtron-labs/common-lib v0.18.1-0.20241001061923-eda545dc839e => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250410055937-f65f6e252704 +# github.com/devtron-labs/common-lib v0.18.1-0.20241001061923-eda545dc839e => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250414075707-6076f6f97fb9 ## explicit; go 1.21 github.com/devtron-labs/common-lib/async github.com/devtron-labs/common-lib/blob-storage @@ -2350,8 +2350,8 @@ xorm.io/xorm/log xorm.io/xorm/names xorm.io/xorm/schemas xorm.io/xorm/tags -# github.com/devtron-labs/authenticator => github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250410055937-f65f6e252704 -# github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250410055937-f65f6e252704 +# github.com/devtron-labs/authenticator => github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250414075707-6076f6f97fb9 +# github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250414075707-6076f6f97fb9 # github.com/go-check/check => github.com/go-check/check v0.0.0-20180628173108-788fd7840127 # github.com/googleapis/gnostic => github.com/googleapis/gnostic v0.5.5 # k8s.io/api => k8s.io/api v0.29.7 From 4edda50bcbe7c48c5d86d402a16d2548220f89cd Mon Sep 17 00:00:00 2001 From: prakhar katiyar Date: Tue, 15 Apr 2025 15:06:49 +0530 Subject: [PATCH 25/34] added more fields in webhook notification --- .gitignore | 2 ++ pkg/notifier/WebhookNotificationService.go | 4 +++- pkg/notifier/beans/beans.go | 2 ++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index c56fd8f053..f5ab61d6b1 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,5 @@ .env /cmd/external-app/devtron-ea devtron + +.qodo diff --git a/pkg/notifier/WebhookNotificationService.go b/pkg/notifier/WebhookNotificationService.go index 9251246f75..7fee81dd87 100644 --- a/pkg/notifier/WebhookNotificationService.go +++ b/pkg/notifier/WebhookNotificationService.go @@ -109,6 +109,8 @@ func (impl *WebhookNotificationServiceImpl) GetWebhookVariables() (map[string]be "devtronCiPipelineId": beans.DevtronCiPipelineId, "devtronCdPipelineId": beans.DevtronCdPipelineId, "devtronTriggeredByEmail": beans.DevtronTriggeredByEmail, + "devtronPipelineType": beans.DevtronPipelineType, + "devtronBuildGitCommitHash": beans.DevtronBuildGitCommitHash, "eventType": beans.EventType, } @@ -119,7 +121,7 @@ func (impl *WebhookNotificationServiceImpl) FetchAllWebhookNotificationConfig() var responseDto []*beans.WebhookConfigDto webhookConfigs, err := impl.webhookRepository.FindAll() if err != nil && !util.IsErrNoRows(err) { - impl.logger.Errorw("cannot find all webhoook config", "err", err) + impl.logger.Errorw("cannot find all webhook config", "err", err) return []*beans.WebhookConfigDto{}, err } for _, webhookConfig := range webhookConfigs { diff --git a/pkg/notifier/beans/beans.go b/pkg/notifier/beans/beans.go index 24f9b75c94..161735fa2c 100644 --- a/pkg/notifier/beans/beans.go +++ b/pkg/notifier/beans/beans.go @@ -23,6 +23,8 @@ const ( DevtronCiPipelineId WebhookVariable = "{{devtronCiPipelineId}}" DevtronCdPipelineId WebhookVariable = "{{devtronCdPipelineId}}" DevtronTriggeredByEmail WebhookVariable = "{{devtronTriggeredByEmail}}" + DevtronBuildGitCommitHash WebhookVariable = "{{devtronBuildGitCommitHash}}" + DevtronPipelineType WebhookVariable = "{{devtronPipelineType}}" EventType WebhookVariable = "{{eventType}}" ) From 56d412ee71058eeb29c49e9c362d60a1043bd787 Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Wed, 16 Apr 2025 10:44:20 +0530 Subject: [PATCH 26/34] ci dangling pod fix --- pkg/pipeline/executors/ArgoWorkflowExecutor.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/pipeline/executors/ArgoWorkflowExecutor.go b/pkg/pipeline/executors/ArgoWorkflowExecutor.go index c84a903e06..e54c04ac3b 100644 --- a/pkg/pipeline/executors/ArgoWorkflowExecutor.go +++ b/pkg/pipeline/executors/ArgoWorkflowExecutor.go @@ -152,6 +152,9 @@ func (impl *ArgoWorkflowExecutorImpl) ExecuteWorkflow(workflowTemplate bean.Work }, Templates: templates, Volumes: workflowTemplate.Volumes, + PodGC: &v1alpha1.PodGC{ + Strategy: v1alpha1.PodGCOnWorkflowCompletion, + }, }, } ) From e17fec4c872c7814fdb3b71351f7c07c7b38feaa Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Wed, 16 Apr 2025 11:08:38 +0530 Subject: [PATCH 27/34] delete delay duration update --- pkg/pipeline/bean/WorkflowTemplate.go | 39 ++++++++++--------- .../executors/ArgoWorkflowExecutor.go | 3 +- pkg/pipeline/types/CiCdConfig.go | 5 ++- pkg/pipeline/types/Workflow.go | 1 + 4 files changed, 26 insertions(+), 22 deletions(-) diff --git a/pkg/pipeline/bean/WorkflowTemplate.go b/pkg/pipeline/bean/WorkflowTemplate.go index 55e91e565c..c0d0fc933f 100644 --- a/pkg/pipeline/bean/WorkflowTemplate.go +++ b/pkg/pipeline/bean/WorkflowTemplate.go @@ -28,25 +28,26 @@ type WorkflowTemplate struct { WorkflowId int WorkflowRunnerId int v1.PodSpec - ConfigMaps []bean.ConfigSecretMap - Secrets []bean.ConfigSecretMap - TTLValue *int32 - WorkflowRequestJson string - WorkflowNamePrefix string - WfControllerInstanceID string - ClusterConfig *rest.Config - Namespace string - ArchiveLogs bool - BlobStorageConfigured bool - BlobStorageS3Config *blob_storage.BlobStorageS3Config - CloudProvider blob_storage.BlobStorageType - AzureBlobConfig *blob_storage.AzureBlobConfig - GcpBlobConfig *blob_storage.GcpBlobConfig - CloudStorageKey string - PrePostDeploySteps []*StepObject - RefPlugins []*RefPluginObject - TerminationGracePeriod int - WorkflowType string + ConfigMaps []bean.ConfigSecretMap + Secrets []bean.ConfigSecretMap + TTLValue *int32 + WorkflowRequestJson string + WorkflowNamePrefix string + WfControllerInstanceID string + ClusterConfig *rest.Config + Namespace string + ArchiveLogs bool + BlobStorageConfigured bool + BlobStorageS3Config *blob_storage.BlobStorageS3Config + CloudProvider blob_storage.BlobStorageType + AzureBlobConfig *blob_storage.AzureBlobConfig + GcpBlobConfig *blob_storage.GcpBlobConfig + CloudStorageKey string + PrePostDeploySteps []*StepObject + RefPlugins []*RefPluginObject + TerminationGracePeriod int + WorkflowType string + PodGCDeleteDelayDuration string } const ( diff --git a/pkg/pipeline/executors/ArgoWorkflowExecutor.go b/pkg/pipeline/executors/ArgoWorkflowExecutor.go index e54c04ac3b..c41e87f576 100644 --- a/pkg/pipeline/executors/ArgoWorkflowExecutor.go +++ b/pkg/pipeline/executors/ArgoWorkflowExecutor.go @@ -153,7 +153,8 @@ func (impl *ArgoWorkflowExecutorImpl) ExecuteWorkflow(workflowTemplate bean.Work Templates: templates, Volumes: workflowTemplate.Volumes, PodGC: &v1alpha1.PodGC{ - Strategy: v1alpha1.PodGCOnWorkflowCompletion, + Strategy: v1alpha1.PodGCOnWorkflowCompletion, + DeleteDelayDuration: workflowTemplate.PodGCDeleteDelayDuration, }, }, } diff --git a/pkg/pipeline/types/CiCdConfig.go b/pkg/pipeline/types/CiCdConfig.go index 83d9e97def..ce77dd1370 100644 --- a/pkg/pipeline/types/CiCdConfig.go +++ b/pkg/pipeline/types/CiCdConfig.go @@ -97,7 +97,7 @@ type CiCdConfig struct { MaxCiWorkflowRetries int `env:"MAX_CI_WORKFLOW_RETRIES" envDefault:"0" description:"Maximum time CI-workflow create pod if it fails to complete"` NatsServerHost string `env:"NATS_SERVER_HOST" envDefault:"nats://devtron-nats.devtroncd:4222"` ImageScanMaxRetries int `env:"IMAGE_SCAN_MAX_RETRIES" envDefault:"3" description:"Max retry count for image-scanning"` - ImageScanRetryDelay int `env:"IMAGE_SCAN_RETRY_DELAY" envDefault:"5" description:"Delay for the image-scaning to start"` + ImageScanRetryDelay int `env:"IMAGE_SCAN_RETRY_DELAY" envDefault:"5" description:"Delay for the image-scaning to start"` ShowDockerBuildCmdInLogs bool `env:"SHOW_DOCKER_BUILD_ARGS" envDefault:"true" description:"To enable showing the args passed for CI in build logs"` IgnoreCmCsInCiJob bool `env:"IGNORE_CM_CS_IN_CI_JOB" envDefault:"false" description:"Ignore CM/CS in CI-pipeline as Job"` //Deprecated: use WorkflowCacheConfig instead @@ -150,8 +150,9 @@ type CiCdConfig struct { AzureGatewayConnectionInsecure bool `env:"AZURE_GATEWAY_CONNECTION_INSECURE" envDefault:"true" description:"Azure gateway connection allows insecure if true"` AzureBlobContainerCiLog string `env:"AZURE_BLOB_CONTAINER_CI_LOG" description:"Log bucket for azure blob storage"` AzureBlobContainerCiCache string `env:"AZURE_BLOB_CONTAINER_CI_CACHE" description:"Cache bucket name for azure blob storage"` - AzureAccountKey string `env:"AZURE_ACCOUNT_KEY" description:"If blob storage is bieng used of azure then pass the secret key to access the bucket"` + AzureAccountKey string `env:"AZURE_ACCOUNT_KEY" description:"If blob storage is being used of azure then pass the secret key to access the bucket"` BuildLogTTLValue int `env:"BUILD_LOG_TTL_VALUE_IN_SECS" envDefault:"3600" description:"This is the time that the pods of ci/pre-cd/post-cd live after completion state."` + PodGCDeleteDelayDuration string `env:"POD_GC_DELETE_DELAY_DURATION" envDefault:"150s" description:"This is the time that pods will be delayed for deletion on workflow completion"` BaseLogLocationPath string `env:"BASE_LOG_LOCATION_PATH" envDefault:"/home/devtron/" description:"Used to store, download logs of ci workflow, artifact"` InAppLoggingEnabled bool `env:"IN_APP_LOGGING_ENABLED" envDefault:"false" description:"Used in case of argo workflow is enabled. If enabled logs push will be managed by us, else will be managed by argo workflow."` BuildxProvenanceMode string `env:"BUILDX_PROVENANCE_MODE" envDefault:"" description:"provinance is set to true by default by docker. this will add some build related data in generated build manifest.it also adds some unknown:unknown key:value pair which may not be compatible by some container registries. with buildx k8s driver , provinenance=true is causing issue when push manifest to quay registry, so setting it to false"` // provenance is set to false if this flag is not set diff --git a/pkg/pipeline/types/Workflow.go b/pkg/pipeline/types/Workflow.go index dca45ab8fc..b5bf066dc9 100644 --- a/pkg/pipeline/types/Workflow.go +++ b/pkg/pipeline/types/Workflow.go @@ -199,6 +199,7 @@ func (workflowRequest *WorkflowRequest) GetWorkflowTemplate(workflowJson []byte, workflowTemplate.WorkflowRunnerId = workflowRequest.WorkflowRunnerId workflowTemplate.PrePostDeploySteps = workflowRequest.PrePostDeploySteps } + workflowTemplate.PodGCDeleteDelayDuration = config.PodGCDeleteDelayDuration return workflowTemplate } From cd0ee66a0729689ca789d2ff89d5abc9327bcd28 Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Wed, 16 Apr 2025 12:03:00 +0530 Subject: [PATCH 28/34] removed delete delay seconds --- pkg/pipeline/bean/WorkflowTemplate.go | 39 +++++++++---------- .../executors/ArgoWorkflowExecutor.go | 3 +- pkg/pipeline/types/CiCdConfig.go | 1 - pkg/pipeline/types/Workflow.go | 1 - 4 files changed, 20 insertions(+), 24 deletions(-) diff --git a/pkg/pipeline/bean/WorkflowTemplate.go b/pkg/pipeline/bean/WorkflowTemplate.go index c0d0fc933f..55e91e565c 100644 --- a/pkg/pipeline/bean/WorkflowTemplate.go +++ b/pkg/pipeline/bean/WorkflowTemplate.go @@ -28,26 +28,25 @@ type WorkflowTemplate struct { WorkflowId int WorkflowRunnerId int v1.PodSpec - ConfigMaps []bean.ConfigSecretMap - Secrets []bean.ConfigSecretMap - TTLValue *int32 - WorkflowRequestJson string - WorkflowNamePrefix string - WfControllerInstanceID string - ClusterConfig *rest.Config - Namespace string - ArchiveLogs bool - BlobStorageConfigured bool - BlobStorageS3Config *blob_storage.BlobStorageS3Config - CloudProvider blob_storage.BlobStorageType - AzureBlobConfig *blob_storage.AzureBlobConfig - GcpBlobConfig *blob_storage.GcpBlobConfig - CloudStorageKey string - PrePostDeploySteps []*StepObject - RefPlugins []*RefPluginObject - TerminationGracePeriod int - WorkflowType string - PodGCDeleteDelayDuration string + ConfigMaps []bean.ConfigSecretMap + Secrets []bean.ConfigSecretMap + TTLValue *int32 + WorkflowRequestJson string + WorkflowNamePrefix string + WfControllerInstanceID string + ClusterConfig *rest.Config + Namespace string + ArchiveLogs bool + BlobStorageConfigured bool + BlobStorageS3Config *blob_storage.BlobStorageS3Config + CloudProvider blob_storage.BlobStorageType + AzureBlobConfig *blob_storage.AzureBlobConfig + GcpBlobConfig *blob_storage.GcpBlobConfig + CloudStorageKey string + PrePostDeploySteps []*StepObject + RefPlugins []*RefPluginObject + TerminationGracePeriod int + WorkflowType string } const ( diff --git a/pkg/pipeline/executors/ArgoWorkflowExecutor.go b/pkg/pipeline/executors/ArgoWorkflowExecutor.go index c41e87f576..e54c04ac3b 100644 --- a/pkg/pipeline/executors/ArgoWorkflowExecutor.go +++ b/pkg/pipeline/executors/ArgoWorkflowExecutor.go @@ -153,8 +153,7 @@ func (impl *ArgoWorkflowExecutorImpl) ExecuteWorkflow(workflowTemplate bean.Work Templates: templates, Volumes: workflowTemplate.Volumes, PodGC: &v1alpha1.PodGC{ - Strategy: v1alpha1.PodGCOnWorkflowCompletion, - DeleteDelayDuration: workflowTemplate.PodGCDeleteDelayDuration, + Strategy: v1alpha1.PodGCOnWorkflowCompletion, }, }, } diff --git a/pkg/pipeline/types/CiCdConfig.go b/pkg/pipeline/types/CiCdConfig.go index ce77dd1370..5d50162141 100644 --- a/pkg/pipeline/types/CiCdConfig.go +++ b/pkg/pipeline/types/CiCdConfig.go @@ -152,7 +152,6 @@ type CiCdConfig struct { AzureBlobContainerCiCache string `env:"AZURE_BLOB_CONTAINER_CI_CACHE" description:"Cache bucket name for azure blob storage"` AzureAccountKey string `env:"AZURE_ACCOUNT_KEY" description:"If blob storage is being used of azure then pass the secret key to access the bucket"` BuildLogTTLValue int `env:"BUILD_LOG_TTL_VALUE_IN_SECS" envDefault:"3600" description:"This is the time that the pods of ci/pre-cd/post-cd live after completion state."` - PodGCDeleteDelayDuration string `env:"POD_GC_DELETE_DELAY_DURATION" envDefault:"150s" description:"This is the time that pods will be delayed for deletion on workflow completion"` BaseLogLocationPath string `env:"BASE_LOG_LOCATION_PATH" envDefault:"/home/devtron/" description:"Used to store, download logs of ci workflow, artifact"` InAppLoggingEnabled bool `env:"IN_APP_LOGGING_ENABLED" envDefault:"false" description:"Used in case of argo workflow is enabled. If enabled logs push will be managed by us, else will be managed by argo workflow."` BuildxProvenanceMode string `env:"BUILDX_PROVENANCE_MODE" envDefault:"" description:"provinance is set to true by default by docker. this will add some build related data in generated build manifest.it also adds some unknown:unknown key:value pair which may not be compatible by some container registries. with buildx k8s driver , provinenance=true is causing issue when push manifest to quay registry, so setting it to false"` // provenance is set to false if this flag is not set diff --git a/pkg/pipeline/types/Workflow.go b/pkg/pipeline/types/Workflow.go index b5bf066dc9..dca45ab8fc 100644 --- a/pkg/pipeline/types/Workflow.go +++ b/pkg/pipeline/types/Workflow.go @@ -199,7 +199,6 @@ func (workflowRequest *WorkflowRequest) GetWorkflowTemplate(workflowJson []byte, workflowTemplate.WorkflowRunnerId = workflowRequest.WorkflowRunnerId workflowTemplate.PrePostDeploySteps = workflowRequest.PrePostDeploySteps } - workflowTemplate.PodGCDeleteDelayDuration = config.PodGCDeleteDelayDuration return workflowTemplate } From 8e3952ab86a1736216cde80b7e2f38aee62dfd41 Mon Sep 17 00:00:00 2001 From: iamayushm <32041961+iamayushm@users.noreply.github.com> Date: Mon, 21 Apr 2025 12:05:39 +0530 Subject: [PATCH 29/34] fix: issues in helm apps and git provide (#6523) * wip: adding code in oss * adding migration * fix for oci chart link * renaming migration --------- Co-authored-by: iamayushm --- .../configure/BuildPipelineRestHandler.go | 7 ++- .../configure/PipelineConfigRestHandler.go | 3 +- env_gen.json | 2 +- env_gen.md | 2 +- pkg/appStore/bean/bean.go | 1 + .../AppStoreApplicationVersionRepository.go | 2 +- .../discover/service/AppStoreService.go | 5 ++ .../service/AppStoreDeploymentService.go | 63 +++++++++++++++---- ..._docker_artifact_credentials_type.down.sql | 1 + ...00_docker_artifact_credentials_type.up.sql | 1 + wire_gen.go | 2 +- 11 files changed, 71 insertions(+), 18 deletions(-) create mode 100644 scripts/sql/32703400_docker_artifact_credentials_type.down.sql create mode 100644 scripts/sql/32703400_docker_artifact_credentials_type.up.sql diff --git a/api/restHandler/app/pipeline/configure/BuildPipelineRestHandler.go b/api/restHandler/app/pipeline/configure/BuildPipelineRestHandler.go index ec552d8e2b..0bb2a921b8 100644 --- a/api/restHandler/app/pipeline/configure/BuildPipelineRestHandler.go +++ b/api/restHandler/app/pipeline/configure/BuildPipelineRestHandler.go @@ -30,6 +30,7 @@ import ( "golang.org/x/exp/maps" "io" "net/http" + "regexp" "strconv" "strings" @@ -1479,10 +1480,12 @@ func (handler *PipelineConfigRestHandlerImpl) ValidateGitMaterialUrl(gitProvider return false, err } if gitProvider.AuthMode == constants.AUTH_MODE_SSH { - hasPrefixResult := strings.HasPrefix(url, SSH_URL_PREFIX) + // this regex is used to generic ssh providers like gogs where format is @:/.git + var scpLikeSSHRegex = regexp.MustCompile(`^[\w-]+@[\w.-]+:[\w./-]+\.git$`) + hasPrefixResult := strings.HasPrefix(url, SSH_URL_PREFIX) || scpLikeSSHRegex.MatchString(url) return hasPrefixResult, nil } - hasPrefixResult := strings.HasPrefix(url, HTTPS_URL_PREFIX) + hasPrefixResult := strings.HasPrefix(url, HTTPS_URL_PREFIX) || strings.HasPrefix(url, HTTP_URL_PREFIX) return hasPrefixResult, nil } diff --git a/api/restHandler/app/pipeline/configure/PipelineConfigRestHandler.go b/api/restHandler/app/pipeline/configure/PipelineConfigRestHandler.go index b7888d71c8..4d37e03579 100644 --- a/api/restHandler/app/pipeline/configure/PipelineConfigRestHandler.go +++ b/api/restHandler/app/pipeline/configure/PipelineConfigRestHandler.go @@ -31,8 +31,8 @@ import ( repository2 "github.com/devtron-labs/devtron/pkg/cluster/environment/repository" "github.com/devtron-labs/devtron/pkg/deployment/manifest/deployedAppMetrics" "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/chartRef" - "github.com/devtron-labs/devtron/pkg/pipeline/draftAwareConfigService" validator2 "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/validator" + "github.com/devtron-labs/devtron/pkg/pipeline/draftAwareConfigService" security2 "github.com/devtron-labs/devtron/pkg/policyGovernance/security/imageScanning" "github.com/devtron-labs/devtron/pkg/policyGovernance/security/imageScanning/read" read3 "github.com/devtron-labs/devtron/pkg/team/read" @@ -225,6 +225,7 @@ const ( devtron = "DEVTRON" SSH_URL_PREFIX = "git@" HTTPS_URL_PREFIX = "https://" + HTTP_URL_PREFIX = "http://" argoWFLogIdentifier = "argo=true" ) diff --git a/env_gen.json b/env_gen.json index 6fd874d675..2681e575b9 100644 --- a/env_gen.json +++ b/env_gen.json @@ -1 +1 @@ -[{"Category":"CD","Fields":[{"Env":"ARGO_APP_MANUAL_SYNC_TIME","EnvType":"int","EnvValue":"3","EnvDescription":"retry argocd app manual sync if the timeline is stuck in ARGOCD_SYNC_INITIATED state for more than this defined time (in mins)","Example":"","Deprecated":"false"},{"Env":"CD_HELM_PIPELINE_STATUS_CRON_TIME","EnvType":"string","EnvValue":"*/2 * * * *","EnvDescription":"Cron time to check the pipeline status ","Example":"","Deprecated":"false"},{"Env":"CD_PIPELINE_STATUS_CRON_TIME","EnvType":"string","EnvValue":"*/2 * * * *","EnvDescription":"Cron time for CD pipeline status","Example":"","Deprecated":"false"},{"Env":"CD_PIPELINE_STATUS_TIMEOUT_DURATION","EnvType":"string","EnvValue":"20","EnvDescription":"Timeout for CD pipeline to get healthy","Example":"","Deprecated":"false"},{"Env":"DEPLOY_STATUS_CRON_GET_PIPELINE_DEPLOYED_WITHIN_HOURS","EnvType":"int","EnvValue":"12","EnvDescription":"This flag is used to fetch the deployment status of the application. It retrieves the status of deployments that occurred between 12 hours and 10 minutes prior to the current time. It fetches non-terminal statuses.","Example":"","Deprecated":"false"},{"Env":"DEVTRON_CHART_ARGO_CD_INSTALL_REQUEST_TIMEOUT","EnvType":"int","EnvValue":"1","EnvDescription":"Context timeout for gitops concurrent async deployments","Example":"","Deprecated":"false"},{"Env":"DEVTRON_CHART_INSTALL_REQUEST_TIMEOUT","EnvType":"int","EnvValue":"6","EnvDescription":"Context timeout for no gitops concurrent async deployments","Example":"","Deprecated":"false"},{"Env":"EXPOSE_CD_METRICS","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"FEATURE_MIGRATE_ARGOCD_APPLICATION_ENABLE","EnvType":"bool","EnvValue":"false","EnvDescription":"enable migration of external argocd application to devtron pipeline","Example":"","Deprecated":"false"},{"Env":"HELM_PIPELINE_STATUS_CHECK_ELIGIBLE_TIME","EnvType":"string","EnvValue":"120","EnvDescription":"eligible time for checking helm app status periodically and update in db, value is in seconds., default is 120, if wfr is updated within configured time i.e. HELM_PIPELINE_STATUS_CHECK_ELIGIBLE_TIME then do not include for this cron cycle.","Example":"","Deprecated":"false"},{"Env":"IS_INTERNAL_USE","EnvType":"bool","EnvValue":"true","EnvDescription":"If enabled then cd pipeline and helm apps will not need the deployment app type mandatorily. Couple this flag with HIDE_GITOPS_OR_HELM_OPTION (in Dashborad) and if gitops is configured and allowed for the env, pipeline/ helm app will gitops else no-gitops.","Example":"","Deprecated":"false"},{"Env":"MIGRATE_DEPLOYMENT_CONFIG_DATA","EnvType":"bool","EnvValue":"false","EnvDescription":"migrate deployment config data from charts table to deployment_config table","Example":"","Deprecated":"false"},{"Env":"PIPELINE_DEGRADED_TIME","EnvType":"string","EnvValue":"10","EnvDescription":"Time to mark a pipeline degraded if not healthy in defined time","Example":"","Deprecated":"false"},{"Env":"REVISION_HISTORY_LIMIT_DEVTRON_APP","EnvType":"int","EnvValue":"1","EnvDescription":"Count for devtron application rivision history","Example":"","Deprecated":"false"},{"Env":"REVISION_HISTORY_LIMIT_EXTERNAL_HELM_APP","EnvType":"int","EnvValue":"0","EnvDescription":"Count for external helm application rivision history","Example":"","Deprecated":"false"},{"Env":"REVISION_HISTORY_LIMIT_HELM_APP","EnvType":"int","EnvValue":"1","EnvDescription":"To set the history limit for the helm app being deployed through devtron","Example":"","Deprecated":"false"},{"Env":"REVISION_HISTORY_LIMIT_LINKED_HELM_APP","EnvType":"int","EnvValue":"15","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"RUN_HELM_INSTALL_IN_ASYNC_MODE_HELM_APPS","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"SHOULD_CHECK_NAMESPACE_ON_CLONE","EnvType":"bool","EnvValue":"false","EnvDescription":"should we check if namespace exists or not while cloning app","Example":"","Deprecated":"false"},{"Env":"USE_DEPLOYMENT_CONFIG_DATA","EnvType":"bool","EnvValue":"false","EnvDescription":"use deployment config data from deployment_config table","Example":"","Deprecated":"true"}]},{"Category":"CI_RUNNER","Fields":[{"Env":"AZURE_ACCOUNT_KEY","EnvType":"string","EnvValue":"","EnvDescription":"If blob storage is bieng used of azure then pass the secret key to access the bucket","Example":"","Deprecated":"false"},{"Env":"AZURE_ACCOUNT_NAME","EnvType":"string","EnvValue":"","EnvDescription":"Account name for azure blob storage","Example":"","Deprecated":"false"},{"Env":"AZURE_BLOB_CONTAINER_CI_CACHE","EnvType":"string","EnvValue":"","EnvDescription":"Cache bucket name for azure blob storage","Example":"","Deprecated":"false"},{"Env":"AZURE_BLOB_CONTAINER_CI_LOG","EnvType":"string","EnvValue":"","EnvDescription":"Log bucket for azure blob storage","Example":"","Deprecated":"false"},{"Env":"AZURE_GATEWAY_CONNECTION_INSECURE","EnvType":"bool","EnvValue":"true","EnvDescription":"Azure gateway connection allows insecure if true","Example":"","Deprecated":"false"},{"Env":"AZURE_GATEWAY_URL","EnvType":"string","EnvValue":"http://devtron-minio.devtroncd:9000","EnvDescription":"Sent to CI runner for blob","Example":"","Deprecated":"false"},{"Env":"BASE_LOG_LOCATION_PATH","EnvType":"string","EnvValue":"/home/devtron/","EnvDescription":"Used to store, download logs of ci workflow, artifact","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_GCP_CREDENTIALS_JSON","EnvType":"string","EnvValue":"","EnvDescription":"GCP cred json for GCS blob storage","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_PROVIDER","EnvType":"","EnvValue":"S3","EnvDescription":"Blob storage provider name(AWS/GCP/Azure)","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_S3_ACCESS_KEY","EnvType":"string","EnvValue":"","EnvDescription":"S3 access key for s3 blob storage","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_S3_BUCKET_VERSIONED","EnvType":"bool","EnvValue":"true","EnvDescription":"To enable buctet versioning for blob storage","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_S3_ENDPOINT","EnvType":"string","EnvValue":"","EnvDescription":"S3 endpoint URL for s3 blob storage","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_S3_ENDPOINT_INSECURE","EnvType":"bool","EnvValue":"false","EnvDescription":"To use insecure s3 endpoint","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_S3_SECRET_KEY","EnvType":"string","EnvValue":"","EnvDescription":"Secret key for s3 blob storage","Example":"","Deprecated":"false"},{"Env":"BUILDX_CACHE_PATH","EnvType":"string","EnvValue":"/var/lib/devtron/buildx","EnvDescription":"Path for the buildx cache","Example":"","Deprecated":"false"},{"Env":"BUILDX_K8S_DRIVER_OPTIONS","EnvType":"string","EnvValue":"","EnvDescription":"To enable the k8s driver and pass args for k8s driver in buildx","Example":"","Deprecated":"false"},{"Env":"BUILDX_PROVENANCE_MODE","EnvType":"string","EnvValue":"","EnvDescription":"provinance is set to true by default by docker. this will add some build related data in generated build manifest.it also adds some unknown:unknown key:value pair which may not be compatible by some container registries. with buildx k8s driver , provinenance=true is causing issue when push manifest to quay registry, so setting it to false","Example":"","Deprecated":"false"},{"Env":"BUILD_LOG_TTL_VALUE_IN_SECS","EnvType":"int","EnvValue":"3600","EnvDescription":"This is the time that the pods of ci/pre-cd/post-cd live after completion state.","Example":"","Deprecated":"false"},{"Env":"CACHE_LIMIT","EnvType":"int64","EnvValue":"5000000000","EnvDescription":"Cache limit.","Example":"","Deprecated":"false"},{"Env":"CD_DEFAULT_ADDRESS_POOL_BASE_CIDR","EnvType":"string","EnvValue":"","EnvDescription":"To pass the IP cidr for Pre/Post cd ","Example":"","Deprecated":"false"},{"Env":"CD_DEFAULT_ADDRESS_POOL_SIZE","EnvType":"int","EnvValue":"","EnvDescription":"The subnet size to allocate from the base pool for CD","Example":"","Deprecated":"false"},{"Env":"CD_LIMIT_CI_CPU","EnvType":"string","EnvValue":"0.5","EnvDescription":"CPU Resource Limit Pre/Post CD","Example":"","Deprecated":"false"},{"Env":"CD_LIMIT_CI_MEM","EnvType":"string","EnvValue":"3G","EnvDescription":"Memory Resource Limit Pre/Post CD","Example":"","Deprecated":"false"},{"Env":"CD_NODE_LABEL_SELECTOR","EnvType":"","EnvValue":"","EnvDescription":"Node label selector for Pre/Post CD","Example":"","Deprecated":"false"},{"Env":"CD_NODE_TAINTS_KEY","EnvType":"string","EnvValue":"dedicated","EnvDescription":"Toleration key for Pre/Post CD","Example":"","Deprecated":"false"},{"Env":"CD_NODE_TAINTS_VALUE","EnvType":"string","EnvValue":"ci","EnvDescription":"Toleration value for Pre/Post CD","Example":"","Deprecated":"false"},{"Env":"CD_REQ_CI_CPU","EnvType":"string","EnvValue":"0.5","EnvDescription":"CPU Resource Rquest Pre/Post CD","Example":"","Deprecated":"false"},{"Env":"CD_REQ_CI_MEM","EnvType":"string","EnvValue":"3G","EnvDescription":"Memory Resource Rquest Pre/Post CD","Example":"","Deprecated":"false"},{"Env":"CD_WORKFLOW_EXECUTOR_TYPE","EnvType":"","EnvValue":"AWF","EnvDescription":"Executor type for Pre/Post CD(AWF,System)","Example":"","Deprecated":"false"},{"Env":"CD_WORKFLOW_SERVICE_ACCOUNT","EnvType":"string","EnvValue":"cd-runner","EnvDescription":"Service account to be used in Pre/Post CD pod","Example":"","Deprecated":"false"},{"Env":"CI_DEFAULT_ADDRESS_POOL_BASE_CIDR","EnvType":"string","EnvValue":"","EnvDescription":"To pass the IP cidr for CI","Example":"","Deprecated":"false"},{"Env":"CI_DEFAULT_ADDRESS_POOL_SIZE","EnvType":"int","EnvValue":"","EnvDescription":"The subnet size to allocate from the base pool for CI","Example":"","Deprecated":"false"},{"Env":"CI_IGNORE_DOCKER_CACHE","EnvType":"bool","EnvValue":"","EnvDescription":"Ignoring docker cache ","Example":"","Deprecated":"false"},{"Env":"CI_LOGS_KEY_PREFIX","EnvType":"string","EnvValue":"","EnvDescription":"Prefix for build logs","Example":"","Deprecated":"false"},{"Env":"CI_NODE_LABEL_SELECTOR","EnvType":"","EnvValue":"","EnvDescription":"Node label selector for CI","Example":"","Deprecated":"false"},{"Env":"CI_NODE_TAINTS_KEY","EnvType":"string","EnvValue":"","EnvDescription":"Toleration key for CI","Example":"","Deprecated":"false"},{"Env":"CI_NODE_TAINTS_VALUE","EnvType":"string","EnvValue":"","EnvDescription":"Toleration value for CI","Example":"","Deprecated":"false"},{"Env":"CI_RUNNER_DOCKER_MTU_VALUE","EnvType":"int","EnvValue":"-1","EnvDescription":"this is to control the bytes of inofrmation passed in a network packet in ci-runner. default is -1 (defaults to the underlying node mtu value)","Example":"","Deprecated":"false"},{"Env":"CI_SUCCESS_AUTO_TRIGGER_BATCH_SIZE","EnvType":"int","EnvValue":"1","EnvDescription":"this is to control the no of linked pipelines should be hanled in one go when a ci-success event of an parent ci is received","Example":"","Deprecated":"false"},{"Env":"CI_VOLUME_MOUNTS_JSON","EnvType":"string","EnvValue":"","EnvDescription":"additional volume mount data for CI and JOB","Example":"","Deprecated":"false"},{"Env":"CI_WORKFLOW_EXECUTOR_TYPE","EnvType":"","EnvValue":"AWF","EnvDescription":"Executor type for CI(AWF,System)","Example":"","Deprecated":"false"},{"Env":"DEFAULT_ARTIFACT_KEY_LOCATION","EnvType":"string","EnvValue":"arsenal-v1/ci-artifacts","EnvDescription":"Key location for artifacts being created","Example":"","Deprecated":"false"},{"Env":"DEFAULT_BUILD_LOGS_BUCKET","EnvType":"string","EnvValue":"devtron-pro-ci-logs","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_BUILD_LOGS_KEY_PREFIX","EnvType":"string","EnvValue":"arsenal-v1","EnvDescription":"Bucket prefix for build logs","Example":"","Deprecated":"false"},{"Env":"DEFAULT_CACHE_BUCKET","EnvType":"string","EnvValue":"ci-caching","EnvDescription":"Bucket name for build cache","Example":"","Deprecated":"false"},{"Env":"DEFAULT_CACHE_BUCKET_REGION","EnvType":"string","EnvValue":"us-east-2","EnvDescription":"Build Cache bucket region","Example":"","Deprecated":"false"},{"Env":"DEFAULT_CD_ARTIFACT_KEY_LOCATION","EnvType":"string","EnvValue":"","EnvDescription":"Bucket prefix for build cache","Example":"","Deprecated":"false"},{"Env":"DEFAULT_CD_LOGS_BUCKET_REGION","EnvType":"string","EnvValue":"us-east-2","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_CD_NAMESPACE","EnvType":"string","EnvValue":"","EnvDescription":"Namespace for devtron stack","Example":"","Deprecated":"false"},{"Env":"DEFAULT_CD_TIMEOUT","EnvType":"int64","EnvValue":"3600","EnvDescription":"Timeout for Pre/Post-Cd to be completed","Example":"","Deprecated":"false"},{"Env":"DEFAULT_CI_IMAGE","EnvType":"string","EnvValue":"686244538589.dkr.ecr.us-east-2.amazonaws.com/cirunner:47","EnvDescription":"To pass the ci-runner image","Example":"","Deprecated":"false"},{"Env":"DEFAULT_NAMESPACE","EnvType":"string","EnvValue":"devtron-ci","EnvDescription":"Timeout for CI to be completed","Example":"","Deprecated":"false"},{"Env":"DEFAULT_TARGET_PLATFORM","EnvType":"string","EnvValue":"","EnvDescription":"Default architecture for buildx","Example":"","Deprecated":"false"},{"Env":"DOCKER_BUILD_CACHE_PATH","EnvType":"string","EnvValue":"/var/lib/docker","EnvDescription":"Path to store cache of docker build (/var/lib/docker-\u003e for legacy docker build, /var/lib/devtron-\u003e for buildx)","Example":"","Deprecated":"false"},{"Env":"ENABLE_BUILD_CONTEXT","EnvType":"bool","EnvValue":"false","EnvDescription":"To Enable build context in Devtron.","Example":"","Deprecated":"false"},{"Env":"ENABLE_WORKFLOW_EXECUTION_STAGE","EnvType":"bool","EnvValue":"true","EnvDescription":"if enabled then we will display build stages separately for CI/Job/Pre-Post CD","Example":"true","Deprecated":"false"},{"Env":"EXTERNAL_BLOB_STORAGE_CM_NAME","EnvType":"string","EnvValue":"blob-storage-cm","EnvDescription":"name of the config map(contains bucket name, etc.) in external cluster when there is some operation related to external cluster, for example:-downloading cd artifact pushed in external cluster's env and we need to download from there, downloads ci logs pushed in external cluster's blob","Example":"","Deprecated":"false"},{"Env":"EXTERNAL_BLOB_STORAGE_SECRET_NAME","EnvType":"string","EnvValue":"blob-storage-secret","EnvDescription":"name of the secret(contains password, accessId,passKeys, etc.) in external cluster when there is some operation related to external cluster, for example:-downloading cd artifact pushed in external cluster's env and we need to download from there, downloads ci logs pushed in external cluster's blob","Example":"","Deprecated":"false"},{"Env":"EXTERNAL_CD_NODE_LABEL_SELECTOR","EnvType":"","EnvValue":"","EnvDescription":"This is an array of strings used when submitting a workflow for pre or post-CD execution. If the ","Example":"","Deprecated":"false"},{"Env":"EXTERNAL_CD_NODE_TAINTS_KEY","EnvType":"string","EnvValue":"dedicated","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"EXTERNAL_CD_NODE_TAINTS_VALUE","EnvType":"string","EnvValue":"ci","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"EXTERNAL_CI_API_SECRET","EnvType":"string","EnvValue":"devtroncd-secret","EnvDescription":"External CI API secret.","Example":"","Deprecated":"false"},{"Env":"EXTERNAL_CI_PAYLOAD","EnvType":"string","EnvValue":"{\"ciProjectDetails\":[{\"gitRepository\":\"https://github.com/vikram1601/getting-started-nodejs.git\",\"checkoutPath\":\"./abc\",\"commitHash\":\"239077135f8cdeeccb7857e2851348f558cb53d3\",\"commitTime\":\"2022-10-30T20:00:00\",\"branch\":\"master\",\"message\":\"Update README.md\",\"author\":\"User Name \"}],\"dockerImage\":\"445808685819.dkr.ecr.us-east-2.amazonaws.com/orch:23907713-2\"}","EnvDescription":"External CI payload with project details.","Example":"","Deprecated":"false"},{"Env":"EXTERNAL_CI_WEB_HOOK_URL","EnvType":"string","EnvValue":"","EnvDescription":"default is {{HOST_URL}}/orchestrator/webhook/ext-ci. It is used for external ci.","Example":"","Deprecated":"false"},{"Env":"IGNORE_CM_CS_IN_CI_JOB","EnvType":"bool","EnvValue":"false","EnvDescription":"Ignore CM/CS in CI-pipeline as Job","Example":"","Deprecated":"false"},{"Env":"IMAGE_RETRY_COUNT","EnvType":"int","EnvValue":"0","EnvDescription":"push artifact(image) in ci retry count ","Example":"","Deprecated":"false"},{"Env":"IMAGE_RETRY_INTERVAL","EnvType":"int","EnvValue":"5","EnvDescription":"image retry interval takes value in seconds","Example":"","Deprecated":"false"},{"Env":"IMAGE_SCANNER_ENDPOINT","EnvType":"string","EnvValue":"http://image-scanner-new-demo-devtroncd-service.devtroncd:80","EnvDescription":"Image-scanner micro-service URL","Example":"","Deprecated":"false"},{"Env":"IMAGE_SCAN_MAX_RETRIES","EnvType":"int","EnvValue":"3","EnvDescription":"Max retry count for image-scanning","Example":"","Deprecated":"false"},{"Env":"IMAGE_SCAN_RETRY_DELAY","EnvType":"int","EnvValue":"5","EnvDescription":"Delay for the image-scaning to start","Example":"","Deprecated":"false"},{"Env":"IN_APP_LOGGING_ENABLED","EnvType":"bool","EnvValue":"false","EnvDescription":"Used in case of argo workflow is enabled. If enabled logs push will be managed by us, else will be managed by argo workflow.","Example":"","Deprecated":"false"},{"Env":"MAX_CD_WORKFLOW_RUNNER_RETRIES","EnvType":"int","EnvValue":"0","EnvDescription":"Maximum time pre/post-cd-workflow create pod if it fails to complete","Example":"","Deprecated":"false"},{"Env":"MAX_CI_WORKFLOW_RETRIES","EnvType":"int","EnvValue":"0","EnvDescription":"Maximum time CI-workflow create pod if it fails to complete","Example":"","Deprecated":"false"},{"Env":"MODE","EnvType":"string","EnvValue":"DEV","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_SERVER_HOST","EnvType":"string","EnvValue":"localhost:4222","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ORCH_HOST","EnvType":"string","EnvValue":"http://devtroncd-orchestrator-service-prod.devtroncd/webhook/msg/nats","EnvDescription":"Orchestrator micro-service URL ","Example":"","Deprecated":"false"},{"Env":"ORCH_TOKEN","EnvType":"string","EnvValue":"","EnvDescription":"Orchestrator token","Example":"","Deprecated":"false"},{"Env":"PRE_CI_CACHE_PATH","EnvType":"string","EnvValue":"/devtroncd-cache","EnvDescription":"Cache path for Pre CI tasks","Example":"","Deprecated":"false"},{"Env":"SHOW_DOCKER_BUILD_ARGS","EnvType":"bool","EnvValue":"true","EnvDescription":"To enable showing the args passed for CI in build logs","Example":"","Deprecated":"false"},{"Env":"SKIP_CI_JOB_BUILD_CACHE_PUSH_PULL","EnvType":"bool","EnvValue":"false","EnvDescription":"To skip cache Push/Pull for ci job","Example":"","Deprecated":"false"},{"Env":"SKIP_CREATING_ECR_REPO","EnvType":"bool","EnvValue":"false","EnvDescription":"By disabling this ECR repo won't get created if it's not available on ECR from build configuration","Example":"","Deprecated":"false"},{"Env":"TERMINATION_GRACE_PERIOD_SECS","EnvType":"int","EnvValue":"180","EnvDescription":"this is the time given to workflow pods to shutdown. (grace full termination time)","Example":"","Deprecated":"false"},{"Env":"USE_ARTIFACT_LISTING_QUERY_V2","EnvType":"bool","EnvValue":"true","EnvDescription":"To use the V2 query for listing artifacts","Example":"","Deprecated":"false"},{"Env":"USE_BLOB_STORAGE_CONFIG_IN_CD_WORKFLOW","EnvType":"bool","EnvValue":"true","EnvDescription":"To enable blob storage in pre and post cd","Example":"","Deprecated":"false"},{"Env":"USE_BLOB_STORAGE_CONFIG_IN_CI_WORKFLOW","EnvType":"bool","EnvValue":"true","EnvDescription":"To enable blob storage in pre and post ci","Example":"","Deprecated":"false"},{"Env":"USE_BUILDX","EnvType":"bool","EnvValue":"false","EnvDescription":"To enable buildx feature globally","Example":"","Deprecated":"false"},{"Env":"USE_DOCKER_API_TO_GET_DIGEST","EnvType":"bool","EnvValue":"false","EnvDescription":"when user do not pass the digest then this flag controls , finding the image digest using docker API or not. if set to true we get the digest from docker API call else use docker pull command. [logic in ci-runner]","Example":"","Deprecated":"false"},{"Env":"USE_EXTERNAL_NODE","EnvType":"bool","EnvValue":"false","EnvDescription":"It is used in case of Pre/ Post Cd with run in application mode. If enabled the node lebels are read from EXTERNAL_CD_NODE_LABEL_SELECTOR else from CD_NODE_LABEL_SELECTOR MODE: if the vale is DEV, it will read the local kube config file or else from the cluser location.","Example":"","Deprecated":"false"},{"Env":"USE_IMAGE_TAG_FROM_GIT_PROVIDER_FOR_TAG_BASED_BUILD","EnvType":"bool","EnvValue":"false","EnvDescription":"To use the same tag in container image as that of git tag","Example":"","Deprecated":"false"},{"Env":"WF_CONTROLLER_INSTANCE_ID","EnvType":"string","EnvValue":"devtron-runner","EnvDescription":"Workflow controller instance ID.","Example":"","Deprecated":"false"},{"Env":"WORKFLOW_CACHE_CONFIG","EnvType":"string","EnvValue":"{}","EnvDescription":"flag is used to configure how Docker caches are handled during a CI/CD ","Example":"","Deprecated":"false"},{"Env":"WORKFLOW_SERVICE_ACCOUNT","EnvType":"string","EnvValue":"ci-runner","EnvDescription":"","Example":"","Deprecated":"false"}]},{"Category":"DEVTRON","Fields":[{"Env":"-","EnvType":"","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ADDITIONAL_NODE_GROUP_LABELS","EnvType":"","EnvValue":"","EnvDescription":"Add comma separated list of additional node group labels to default labels","Example":"karpenter.sh/nodepool,cloud.google.com/gke-nodepool","Deprecated":"false"},{"Env":"APP_SYNC_IMAGE","EnvType":"string","EnvValue":"quay.io/devtron/chart-sync:1227622d-132-3775","EnvDescription":"For the app sync image, this image will be used in app-manual sync job","Example":"","Deprecated":"false"},{"Env":"APP_SYNC_JOB_RESOURCES_OBJ","EnvType":"string","EnvValue":"","EnvDescription":"To pass the resource of app sync","Example":"","Deprecated":"false"},{"Env":"APP_SYNC_SERVICE_ACCOUNT","EnvType":"string","EnvValue":"chart-sync","EnvDescription":"Service account to be used in app sync Job","Example":"","Deprecated":"false"},{"Env":"APP_SYNC_SHUTDOWN_WAIT_DURATION","EnvType":"int","EnvValue":"120","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ARGO_AUTO_SYNC_ENABLED","EnvType":"bool","EnvValue":"true","EnvDescription":"If enabled all argocd application will have auto sync enabled","Example":"","Deprecated":"false"},{"Env":"ARGO_GIT_COMMIT_RETRY_COUNT_ON_CONFLICT","EnvType":"int","EnvValue":"3","EnvDescription":"retry argocd app manual sync if the timeline is stuck in ARGOCD_SYNC_INITIATED state for more than this defined time (in mins)","Example":"","Deprecated":"false"},{"Env":"ARGO_GIT_COMMIT_RETRY_DELAY_ON_CONFLICT","EnvType":"int","EnvValue":"1","EnvDescription":"Delay on retrying the maifest commit the on gitops","Example":"","Deprecated":"false"},{"Env":"ARGO_REPO_REGISTER_RETRY_COUNT","EnvType":"int","EnvValue":"3","EnvDescription":"Argo app registration in argo retries on deployment","Example":"","Deprecated":"false"},{"Env":"ARGO_REPO_REGISTER_RETRY_DELAY","EnvType":"int","EnvValue":"10","EnvDescription":"Argo app registration in argo cd on deployment delay between retry","Example":"","Deprecated":"false"},{"Env":"ASYNC_BUILDX_CACHE_EXPORT","EnvType":"bool","EnvValue":"false","EnvDescription":"To enable async container image cache export","Example":"","Deprecated":"false"},{"Env":"BATCH_SIZE","EnvType":"int","EnvValue":"5","EnvDescription":"there is feature to get URL's of services/ingresses. so to extract those, we need to parse all the servcie and ingress objects of the application. this BATCH_SIZE flag controls the no of these objects get parsed in one go.","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_ENABLED","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BUILDX_CACHE_MODE_MIN","EnvType":"bool","EnvValue":"false","EnvDescription":"To set build cache mode to minimum in buildx","Example":"","Deprecated":"false"},{"Env":"CD_HOST","EnvType":"string","EnvValue":"localhost","EnvDescription":"Host for the devtron stack","Example":"","Deprecated":"false"},{"Env":"CD_NAMESPACE","EnvType":"string","EnvValue":"devtroncd","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_PORT","EnvType":"string","EnvValue":"8000","EnvDescription":"Port for pre/post-cd","Example":"","Deprecated":"false"},{"Env":"CExpirationTime","EnvType":"int","EnvValue":"600","EnvDescription":"Caching expiration time.","Example":"","Deprecated":"false"},{"Env":"CI_TRIGGER_CRON_TIME","EnvType":"int","EnvValue":"2","EnvDescription":"For image poll plugin","Example":"","Deprecated":"false"},{"Env":"CI_WORKFLOW_STATUS_UPDATE_CRON","EnvType":"string","EnvValue":"*/5 * * * *","EnvDescription":"Cron schedule for CI pipeline status","Example":"","Deprecated":"false"},{"Env":"CLI_CMD_TIMEOUT_GLOBAL_SECONDS","EnvType":"int","EnvValue":"0","EnvDescription":"Used in git cli opeartion timeout","Example":"","Deprecated":"false"},{"Env":"CLUSTER_STATUS_CRON_TIME","EnvType":"int","EnvValue":"15","EnvDescription":"Cron schedule for cluster status on resource browser","Example":"","Deprecated":"false"},{"Env":"CONSUMER_CONFIG_JSON","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_LOG_TIME_LIMIT","EnvType":"int64","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_TIMEOUT","EnvType":"float64","EnvValue":"3600","EnvDescription":"Timeout for CI to be completed","Example":"","Deprecated":"false"},{"Env":"DEVTRON_BOM_URL","EnvType":"string","EnvValue":"https://raw.githubusercontent.com/devtron-labs/devtron/%s/charts/devtron/devtron-bom.yaml","EnvDescription":"Path to devtron-bom.yaml of devtron charts, used for module installation and devtron upgrade","Example":"","Deprecated":"false"},{"Env":"DEVTRON_DEFAULT_NAMESPACE","EnvType":"string","EnvValue":"devtroncd","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEVTRON_DEX_SECRET_NAMESPACE","EnvType":"string","EnvValue":"devtroncd","EnvDescription":"Namespace of dex secret","Example":"","Deprecated":"false"},{"Env":"DEVTRON_HELM_RELEASE_CHART_NAME","EnvType":"string","EnvValue":"devtron-operator","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEVTRON_HELM_RELEASE_NAME","EnvType":"string","EnvValue":"devtron","EnvDescription":"Name of the Devtron Helm release. ","Example":"","Deprecated":"false"},{"Env":"DEVTRON_HELM_RELEASE_NAMESPACE","EnvType":"string","EnvValue":"devtroncd","EnvDescription":"Namespace of the Devtron Helm release","Example":"","Deprecated":"false"},{"Env":"DEVTRON_HELM_REPO_NAME","EnvType":"string","EnvValue":"devtron","EnvDescription":"Is used to install modules (stack manager)","Example":"","Deprecated":"false"},{"Env":"DEVTRON_HELM_REPO_URL","EnvType":"string","EnvValue":"https://helm.devtron.ai","EnvDescription":"Is used to install modules (stack manager)","Example":"","Deprecated":"false"},{"Env":"DEVTRON_INSTALLATION_TYPE","EnvType":"string","EnvValue":"","EnvDescription":"Devtron Installation type(EA/Full)","Example":"","Deprecated":"false"},{"Env":"DEVTRON_MODULES_IDENTIFIER_IN_HELM_VALUES","EnvType":"string","EnvValue":"installer.modules","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEVTRON_SECRET_NAME","EnvType":"string","EnvValue":"devtron-secret","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEVTRON_VERSION_IDENTIFIER_IN_HELM_VALUES","EnvType":"string","EnvValue":"installer.release","EnvDescription":"devtron operator version identifier in helm values yaml","Example":"","Deprecated":"false"},{"Env":"DEX_CID","EnvType":"string","EnvValue":"example-app","EnvDescription":"dex client id ","Example":"","Deprecated":"false"},{"Env":"DEX_CLIENT_ID","EnvType":"string","EnvValue":"argo-cd","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEX_CSTOREKEY","EnvType":"string","EnvValue":"","EnvDescription":"DEX CSTOREKEY.","Example":"","Deprecated":"false"},{"Env":"DEX_JWTKEY","EnvType":"string","EnvValue":"","EnvDescription":"DEX JWT key. ","Example":"","Deprecated":"false"},{"Env":"DEX_RURL","EnvType":"string","EnvValue":"http://127.0.0.1:8080/callback","EnvDescription":"Dex redirect URL(http://argocd-dex-server.devtroncd:8080/callback)","Example":"","Deprecated":"false"},{"Env":"DEX_SCOPES","EnvType":"","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEX_SECRET","EnvType":"string","EnvValue":"","EnvDescription":"Dex secret","Example":"","Deprecated":"false"},{"Env":"DEX_URL","EnvType":"string","EnvValue":"","EnvDescription":"Dex service endpoint with dex path(http://argocd-dex-server.devtroncd:5556/dex)","Example":"","Deprecated":"false"},{"Env":"ECR_REPO_NAME_PREFIX","EnvType":"string","EnvValue":"test/","EnvDescription":"Prefix for ECR repo to be created in does not exist","Example":"","Deprecated":"false"},{"Env":"ENABLE_ASYNC_ARGO_CD_INSTALL_DEVTRON_CHART","EnvType":"bool","EnvValue":"false","EnvDescription":"To enable async installation of gitops application","Example":"","Deprecated":"false"},{"Env":"ENABLE_ASYNC_INSTALL_DEVTRON_CHART","EnvType":"bool","EnvValue":"false","EnvDescription":"To enable async installation of no-gitops application","Example":"","Deprecated":"false"},{"Env":"EPHEMERAL_SERVER_VERSION_REGEX","EnvType":"string","EnvValue":"v[1-9]\\.\\b(2[3-9]\\|[3-9][0-9])\\b.*","EnvDescription":"ephemeral containers support version regex that is compared with k8sServerVersion","Example":"","Deprecated":"false"},{"Env":"EVENT_URL","EnvType":"string","EnvValue":"http://localhost:3000/notify","EnvDescription":"Notifier service url","Example":"","Deprecated":"false"},{"Env":"EXECUTE_WIRE_NIL_CHECKER","EnvType":"bool","EnvValue":"false","EnvDescription":"checks for any nil pointer in wire.go","Example":"","Deprecated":"false"},{"Env":"EXPOSE_CI_METRICS","EnvType":"bool","EnvValue":"false","EnvDescription":"To expose CI metrics","Example":"","Deprecated":"false"},{"Env":"FEATURE_RESTART_WORKLOAD_BATCH_SIZE","EnvType":"int","EnvValue":"1","EnvDescription":"restart workload retrieval batch size ","Example":"","Deprecated":"false"},{"Env":"FEATURE_RESTART_WORKLOAD_WORKER_POOL_SIZE","EnvType":"int","EnvValue":"5","EnvDescription":"restart workload retrieval pool size","Example":"","Deprecated":"false"},{"Env":"FORCE_SECURITY_SCANNING","EnvType":"bool","EnvValue":"false","EnvDescription":"By enabling this no one can disable image scaning on ci-pipeline from UI","Example":"","Deprecated":"false"},{"Env":"GITOPS_REPO_PREFIX","EnvType":"string","EnvValue":"","EnvDescription":"Prefix for Gitops repo being creation for argocd application","Example":"","Deprecated":"false"},{"Env":"GO_RUNTIME_ENV","EnvType":"string","EnvValue":"production","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"GRAFANA_HOST","EnvType":"string","EnvValue":"localhost","EnvDescription":"Host URL for the grafana dashboard","Example":"","Deprecated":"false"},{"Env":"GRAFANA_NAMESPACE","EnvType":"string","EnvValue":"devtroncd","EnvDescription":"Namespace for grafana","Example":"","Deprecated":"false"},{"Env":"GRAFANA_ORG_ID","EnvType":"int","EnvValue":"2","EnvDescription":"Org ID for grafana for application metrics","Example":"","Deprecated":"false"},{"Env":"GRAFANA_PASSWORD","EnvType":"string","EnvValue":"prom-operator","EnvDescription":"Password for grafana dashboard","Example":"","Deprecated":"false"},{"Env":"GRAFANA_PORT","EnvType":"string","EnvValue":"8090","EnvDescription":"Port for grafana micro-service","Example":"","Deprecated":"false"},{"Env":"GRAFANA_URL","EnvType":"string","EnvValue":"","EnvDescription":"Host URL for the grafana dashboard","Example":"","Deprecated":"false"},{"Env":"GRAFANA_USERNAME","EnvType":"string","EnvValue":"admin","EnvDescription":"Username for grafana ","Example":"","Deprecated":"false"},{"Env":"HIDE_IMAGE_TAGGING_HARD_DELETE","EnvType":"bool","EnvValue":"false","EnvDescription":"Flag to hide the hard delete option in the image tagging service","Example":"","Deprecated":"false"},{"Env":"IGNORE_AUTOCOMPLETE_AUTH_CHECK","EnvType":"bool","EnvValue":"false","EnvDescription":"flag for ignoring auth check in autocomplete apis.","Example":"","Deprecated":"false"},{"Env":"INSTALLED_MODULES","EnvType":"","EnvValue":"","EnvDescription":"List of installed modules given in helm values/yaml are written in cm and used by devtron to know which modules are given","Example":"security.trivy,security.clair","Deprecated":"false"},{"Env":"INSTALLER_CRD_NAMESPACE","EnvType":"string","EnvValue":"devtroncd","EnvDescription":"namespace where Custom Resource Definitions get installed","Example":"","Deprecated":"false"},{"Env":"INSTALLER_CRD_OBJECT_GROUP_NAME","EnvType":"string","EnvValue":"installer.devtron.ai","EnvDescription":"Devtron installer CRD group name, partially deprecated.","Example":"","Deprecated":"false"},{"Env":"INSTALLER_CRD_OBJECT_RESOURCE","EnvType":"string","EnvValue":"installers","EnvDescription":"Devtron installer CRD resource name, partially deprecated","Example":"","Deprecated":"false"},{"Env":"INSTALLER_CRD_OBJECT_VERSION","EnvType":"string","EnvValue":"v1alpha1","EnvDescription":"version of the CRDs. default is v1alpha1","Example":"","Deprecated":"false"},{"Env":"IS_AIR_GAP_ENVIRONMENT","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"JwtExpirationTime","EnvType":"int","EnvValue":"120","EnvDescription":"JWT expiration time.","Example":"","Deprecated":"false"},{"Env":"K8s_CLIENT_MAX_IDLE_CONNS_PER_HOST","EnvType":"int","EnvValue":"25","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"K8s_TCP_IDLE_CONN_TIMEOUT","EnvType":"int","EnvValue":"300","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"K8s_TCP_KEEPALIVE","EnvType":"int","EnvValue":"30","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"K8s_TCP_TIMEOUT","EnvType":"int","EnvValue":"30","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"K8s_TLS_HANDSHAKE_TIMEOUT","EnvType":"int","EnvValue":"10","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"KUBELINK_GRPC_MAX_RECEIVE_MSG_SIZE","EnvType":"int","EnvValue":"20","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"KUBELINK_GRPC_MAX_SEND_MSG_SIZE","EnvType":"int","EnvValue":"4","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"LENS_TIMEOUT","EnvType":"int","EnvValue":"0","EnvDescription":"Lens microservice timeout.","Example":"","Deprecated":"false"},{"Env":"LENS_URL","EnvType":"string","EnvValue":"http://lens-milandevtron-service:80","EnvDescription":"Lens micro-service URL","Example":"","Deprecated":"false"},{"Env":"LIMIT_CI_CPU","EnvType":"string","EnvValue":"0.5","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"LIMIT_CI_MEM","EnvType":"string","EnvValue":"3G","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"LOGGER_DEV_MODE","EnvType":"bool","EnvValue":"false","EnvDescription":"Enables a different logger theme.","Example":"","Deprecated":"false"},{"Env":"LOG_LEVEL","EnvType":"int","EnvValue":"-1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"MAX_SESSION_PER_USER","EnvType":"int","EnvValue":"5","EnvDescription":"max no of cluster terminal pods can be created by an user","Example":"","Deprecated":"false"},{"Env":"MODULE_METADATA_API_URL","EnvType":"string","EnvValue":"https://api.devtron.ai/module?name=%s","EnvDescription":"Modules list and meta info will be fetched from this server, that is central api server of devtron.","Example":"","Deprecated":"false"},{"Env":"MODULE_STATUS_HANDLING_CRON_DURATION_MIN","EnvType":"int","EnvValue":"3","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_ACK_WAIT_IN_SECS","EnvType":"int","EnvValue":"120","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_BUFFER_SIZE","EnvType":"int","EnvValue":"-1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_MAX_AGE","EnvType":"int","EnvValue":"86400","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_PROCESSING_BATCH_SIZE","EnvType":"int","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_REPLICAS","EnvType":"int","EnvValue":"0","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NOTIFICATION_MEDIUM","EnvType":"NotificationMedium","EnvValue":"rest","EnvDescription":"notification medium","Example":"","Deprecated":"false"},{"Env":"OTEL_COLLECTOR_URL","EnvType":"string","EnvValue":"","EnvDescription":"Opentelemetry URL ","Example":"","Deprecated":"false"},{"Env":"PARALLELISM_LIMIT_FOR_TAG_PROCESSING","EnvType":"int","EnvValue":"","EnvDescription":"App manual sync job parallel tag processing count.","Example":"","Deprecated":"false"},{"Env":"PG_EXPORT_PROM_METRICS","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_LOG_ALL_FAILURE_QUERIES","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_LOG_ALL_QUERY","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_LOG_SLOW_QUERY","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_QUERY_DUR_THRESHOLD","EnvType":"int64","EnvValue":"5000","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PLUGIN_NAME","EnvType":"string","EnvValue":"Pull images from container repository","EnvDescription":"Handles image retrieval from a container repository and triggers subsequent CI processes upon detecting new images.Current default plugin name: Pull Images from Container Repository.","Example":"","Deprecated":"false"},{"Env":"PROPAGATE_EXTRA_LABELS","EnvType":"bool","EnvValue":"false","EnvDescription":"Add additional propagate labels like api.devtron.ai/appName, api.devtron.ai/envName, api.devtron.ai/project along with the user defined ones.","Example":"","Deprecated":"false"},{"Env":"PROXY_SERVICE_CONFIG","EnvType":"string","EnvValue":"{}","EnvDescription":"Proxy configuration for micro-service to be accessible on orhcestrator ingress","Example":"","Deprecated":"false"},{"Env":"REQ_CI_CPU","EnvType":"string","EnvValue":"0.5","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"REQ_CI_MEM","EnvType":"string","EnvValue":"3G","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"RESTRICT_TERMINAL_ACCESS_FOR_NON_SUPER_USER","EnvType":"bool","EnvValue":"false","EnvDescription":"To restrict the cluster terminal from user having non-super admin acceess","Example":"","Deprecated":"false"},{"Env":"RUNTIME_CONFIG_LOCAL_DEV","EnvType":"LocalDevMode","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"SCOPED_VARIABLE_ENABLED","EnvType":"bool","EnvValue":"false","EnvDescription":"To enable scoped variable option","Example":"","Deprecated":"false"},{"Env":"SCOPED_VARIABLE_FORMAT","EnvType":"string","EnvValue":"@{{%s}}","EnvDescription":"Its a scope format for varialbe name.","Example":"","Deprecated":"false"},{"Env":"SCOPED_VARIABLE_HANDLE_PRIMITIVES","EnvType":"bool","EnvValue":"false","EnvDescription":"This describe should we handle primitives or not in scoped variable template parsing.","Example":"","Deprecated":"false"},{"Env":"SCOPED_VARIABLE_NAME_REGEX","EnvType":"string","EnvValue":"^[a-zA-Z][a-zA-Z0-9_-]{0,62}[a-zA-Z0-9]$","EnvDescription":"Regex for scoped variable name that must passed this regex.","Example":"","Deprecated":"false"},{"Env":"SOCKET_DISCONNECT_DELAY_SECONDS","EnvType":"int","EnvValue":"5","EnvDescription":"The server closes a session when a client receiving connection have not been seen for a while.This delay is configured by this setting. By default the session is closed when a receiving connection wasn't seen for 5 seconds.","Example":"","Deprecated":"false"},{"Env":"SOCKET_HEARTBEAT_SECONDS","EnvType":"int","EnvValue":"25","EnvDescription":"In order to keep proxies and load balancers from closing long running http requests we need to pretend that the connection is active and send a heartbeat packet once in a while. This setting controls how often this is done. By default a heartbeat packet is sent every 25 seconds.","Example":"","Deprecated":"false"},{"Env":"STREAM_CONFIG_JSON","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"SYSTEM_VAR_PREFIX","EnvType":"string","EnvValue":"DEVTRON_","EnvDescription":"Scoped variable prefix, variable name must have this prefix.","Example":"","Deprecated":"false"},{"Env":"TERMINAL_POD_DEFAULT_NAMESPACE","EnvType":"string","EnvValue":"default","EnvDescription":"Cluster terminal default namespace","Example":"","Deprecated":"false"},{"Env":"TERMINAL_POD_INACTIVE_DURATION_IN_MINS","EnvType":"int","EnvValue":"10","EnvDescription":"Timeout for cluster terminal to be inactive","Example":"","Deprecated":"false"},{"Env":"TERMINAL_POD_STATUS_SYNC_In_SECS","EnvType":"int","EnvValue":"600","EnvDescription":"this is the time interval at which the status of the cluster terminal pod","Example":"","Deprecated":"false"},{"Env":"TEST_APP","EnvType":"string","EnvValue":"orchestrator","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TEST_PG_ADDR","EnvType":"string","EnvValue":"127.0.0.1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TEST_PG_DATABASE","EnvType":"string","EnvValue":"orchestrator","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TEST_PG_LOG_QUERY","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TEST_PG_PASSWORD","EnvType":"string","EnvValue":"postgrespw","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TEST_PG_PORT","EnvType":"string","EnvValue":"55000","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TEST_PG_USER","EnvType":"string","EnvValue":"postgres","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TIMEOUT_FOR_FAILED_CI_BUILD","EnvType":"string","EnvValue":"15","EnvDescription":"Timeout for Failed CI build ","Example":"","Deprecated":"false"},{"Env":"TIMEOUT_IN_SECONDS","EnvType":"int","EnvValue":"5","EnvDescription":"timeout to compute the urls from services and ingress objects of an application","Example":"","Deprecated":"false"},{"Env":"USER_SESSION_DURATION_SECONDS","EnvType":"int","EnvValue":"86400","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"USE_ARTIFACT_LISTING_API_V2","EnvType":"bool","EnvValue":"true","EnvDescription":"To use the V2 API for listing artifacts in Listing the images in pipeline","Example":"","Deprecated":"false"},{"Env":"USE_CUSTOM_HTTP_TRANSPORT","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"USE_GIT_CLI","EnvType":"bool","EnvValue":"false","EnvDescription":"To enable git cli","Example":"","Deprecated":"false"},{"Env":"USE_RBAC_CREATION_V2","EnvType":"bool","EnvValue":"true","EnvDescription":"To use the V2 for RBAC creation","Example":"","Deprecated":"false"},{"Env":"VARIABLE_CACHE_ENABLED","EnvType":"bool","EnvValue":"true","EnvDescription":"This is used to control caching of all the scope variables defined in the system.","Example":"","Deprecated":"false"},{"Env":"VARIABLE_EXPRESSION_REGEX","EnvType":"string","EnvValue":"@{{([^}]+)}}","EnvDescription":"Scoped variable expression regex","Example":"","Deprecated":"false"},{"Env":"WEBHOOK_TOKEN","EnvType":"string","EnvValue":"","EnvDescription":"If you want to continue using jenkins for CI then please provide this for authentication of requests","Example":"","Deprecated":"false"}]},{"Category":"GITOPS","Fields":[{"Env":"ACD_CM","EnvType":"string","EnvValue":"argocd-cm","EnvDescription":"Name of the argocd CM","Example":"","Deprecated":"false"},{"Env":"ACD_NAMESPACE","EnvType":"string","EnvValue":"devtroncd","EnvDescription":"To pass the argocd namespace","Example":"","Deprecated":"false"},{"Env":"ACD_PASSWORD","EnvType":"string","EnvValue":"","EnvDescription":"Password for the Argocd (deprecated)","Example":"","Deprecated":"false"},{"Env":"ACD_USERNAME","EnvType":"string","EnvValue":"admin","EnvDescription":"User name for argocd","Example":"","Deprecated":"false"},{"Env":"GITOPS_SECRET_NAME","EnvType":"string","EnvValue":"devtron-gitops-secret","EnvDescription":"devtron-gitops-secret","Example":"","Deprecated":"false"},{"Env":"RESOURCE_LIST_FOR_REPLICAS","EnvType":"string","EnvValue":"Deployment,Rollout,StatefulSet,ReplicaSet","EnvDescription":"this holds the list of k8s resource names which support replicas key. this list used in hibernate/un hibernate process","Example":"","Deprecated":"false"},{"Env":"RESOURCE_LIST_FOR_REPLICAS_BATCH_SIZE","EnvType":"int","EnvValue":"5","EnvDescription":"this the batch size to control no of above resources can be parsed in one go to determine hibernate status","Example":"","Deprecated":"false"}]},{"Category":"INFRA_SETUP","Fields":[{"Env":"DASHBOARD_HOST","EnvType":"string","EnvValue":"localhost","EnvDescription":"Dashboard micro-service URL","Example":"","Deprecated":"false"},{"Env":"DASHBOARD_NAMESPACE","EnvType":"string","EnvValue":"devtroncd","EnvDescription":"Dashboard micro-service namespace","Example":"","Deprecated":"false"},{"Env":"DASHBOARD_PORT","EnvType":"string","EnvValue":"3000","EnvDescription":"Port for dashboard micro-service","Example":"","Deprecated":"false"},{"Env":"DEX_HOST","EnvType":"string","EnvValue":"http://localhost","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEX_PORT","EnvType":"string","EnvValue":"5556","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"GIT_SENSOR_PROTOCOL","EnvType":"string","EnvValue":"REST","EnvDescription":"Protocol to connect with git-sensor micro-service","Example":"","Deprecated":"false"},{"Env":"GIT_SENSOR_TIMEOUT","EnvType":"int","EnvValue":"0","EnvDescription":"Timeout for getting response from the git-sensor","Example":"","Deprecated":"false"},{"Env":"GIT_SENSOR_URL","EnvType":"string","EnvValue":"127.0.0.1:7070","EnvDescription":"git-sensor micro-service url ","Example":"","Deprecated":"false"},{"Env":"HELM_CLIENT_URL","EnvType":"string","EnvValue":"127.0.0.1:50051","EnvDescription":"Kubelink micro-service url ","Example":"","Deprecated":"false"}]},{"Category":"POSTGRES","Fields":[{"Env":"APP","EnvType":"string","EnvValue":"orchestrator","EnvDescription":"Application name","Example":"","Deprecated":"false"},{"Env":"CASBIN_DATABASE","EnvType":"string","EnvValue":"casbin","EnvDescription":"Database for casbin","Example":"","Deprecated":"false"},{"Env":"PG_ADDR","EnvType":"string","EnvValue":"127.0.0.1","EnvDescription":"address of postgres service","Example":"postgresql-postgresql.devtroncd","Deprecated":"false"},{"Env":"PG_DATABASE","EnvType":"string","EnvValue":"orchestrator","EnvDescription":"postgres database to be made connection with","Example":"orchestrator, casbin, git_sensor, lens","Deprecated":"false"},{"Env":"PG_PASSWORD","EnvType":"string","EnvValue":"{password}","EnvDescription":"password for postgres, associated with PG_USER","Example":"confidential ;)","Deprecated":"false"},{"Env":"PG_PORT","EnvType":"string","EnvValue":"5432","EnvDescription":"port of postgresql service","Example":"5432","Deprecated":"false"},{"Env":"PG_READ_TIMEOUT","EnvType":"int64","EnvValue":"30","EnvDescription":"Time out for read operation in postgres","Example":"","Deprecated":"false"},{"Env":"PG_USER","EnvType":"string","EnvValue":"postgres","EnvDescription":"user for postgres","Example":"postgres","Deprecated":"false"},{"Env":"PG_WRITE_TIMEOUT","EnvType":"int64","EnvValue":"30","EnvDescription":"Time out for write operation in postgres","Example":"","Deprecated":"false"}]},{"Category":"RBAC","Fields":[{"Env":"ENFORCER_CACHE","EnvType":"bool","EnvValue":"false","EnvDescription":"To Enable enforcer cache.","Example":"","Deprecated":"false"},{"Env":"ENFORCER_CACHE_EXPIRATION_IN_SEC","EnvType":"int","EnvValue":"86400","EnvDescription":"Expiration time (in seconds) for enforcer cache. ","Example":"","Deprecated":"false"},{"Env":"ENFORCER_MAX_BATCH_SIZE","EnvType":"int","EnvValue":"1","EnvDescription":"Maximum batch size for the enforcer.","Example":"","Deprecated":"false"},{"Env":"USE_CASBIN_V2","EnvType":"bool","EnvValue":"true","EnvDescription":"To enable casbin V2 API","Example":"","Deprecated":"false"}]}] \ No newline at end of file +[{"Category":"CD","Fields":[{"Env":"ARGO_APP_MANUAL_SYNC_TIME","EnvType":"int","EnvValue":"3","EnvDescription":"retry argocd app manual sync if the timeline is stuck in ARGOCD_SYNC_INITIATED state for more than this defined time (in mins)","Example":"","Deprecated":"false"},{"Env":"CD_HELM_PIPELINE_STATUS_CRON_TIME","EnvType":"string","EnvValue":"*/2 * * * *","EnvDescription":"Cron time to check the pipeline status ","Example":"","Deprecated":"false"},{"Env":"CD_PIPELINE_STATUS_CRON_TIME","EnvType":"string","EnvValue":"*/2 * * * *","EnvDescription":"Cron time for CD pipeline status","Example":"","Deprecated":"false"},{"Env":"CD_PIPELINE_STATUS_TIMEOUT_DURATION","EnvType":"string","EnvValue":"20","EnvDescription":"Timeout for CD pipeline to get healthy","Example":"","Deprecated":"false"},{"Env":"DEPLOY_STATUS_CRON_GET_PIPELINE_DEPLOYED_WITHIN_HOURS","EnvType":"int","EnvValue":"12","EnvDescription":"This flag is used to fetch the deployment status of the application. It retrieves the status of deployments that occurred between 12 hours and 10 minutes prior to the current time. It fetches non-terminal statuses.","Example":"","Deprecated":"false"},{"Env":"DEVTRON_CHART_ARGO_CD_INSTALL_REQUEST_TIMEOUT","EnvType":"int","EnvValue":"1","EnvDescription":"Context timeout for gitops concurrent async deployments","Example":"","Deprecated":"false"},{"Env":"DEVTRON_CHART_INSTALL_REQUEST_TIMEOUT","EnvType":"int","EnvValue":"6","EnvDescription":"Context timeout for no gitops concurrent async deployments","Example":"","Deprecated":"false"},{"Env":"EXPOSE_CD_METRICS","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"FEATURE_MIGRATE_ARGOCD_APPLICATION_ENABLE","EnvType":"bool","EnvValue":"false","EnvDescription":"enable migration of external argocd application to devtron pipeline","Example":"","Deprecated":"false"},{"Env":"HELM_PIPELINE_STATUS_CHECK_ELIGIBLE_TIME","EnvType":"string","EnvValue":"120","EnvDescription":"eligible time for checking helm app status periodically and update in db, value is in seconds., default is 120, if wfr is updated within configured time i.e. HELM_PIPELINE_STATUS_CHECK_ELIGIBLE_TIME then do not include for this cron cycle.","Example":"","Deprecated":"false"},{"Env":"IS_INTERNAL_USE","EnvType":"bool","EnvValue":"true","EnvDescription":"If enabled then cd pipeline and helm apps will not need the deployment app type mandatorily. Couple this flag with HIDE_GITOPS_OR_HELM_OPTION (in Dashborad) and if gitops is configured and allowed for the env, pipeline/ helm app will gitops else no-gitops.","Example":"","Deprecated":"false"},{"Env":"MIGRATE_DEPLOYMENT_CONFIG_DATA","EnvType":"bool","EnvValue":"false","EnvDescription":"migrate deployment config data from charts table to deployment_config table","Example":"","Deprecated":"false"},{"Env":"PIPELINE_DEGRADED_TIME","EnvType":"string","EnvValue":"10","EnvDescription":"Time to mark a pipeline degraded if not healthy in defined time","Example":"","Deprecated":"false"},{"Env":"REVISION_HISTORY_LIMIT_DEVTRON_APP","EnvType":"int","EnvValue":"1","EnvDescription":"Count for devtron application rivision history","Example":"","Deprecated":"false"},{"Env":"REVISION_HISTORY_LIMIT_EXTERNAL_HELM_APP","EnvType":"int","EnvValue":"0","EnvDescription":"Count for external helm application rivision history","Example":"","Deprecated":"false"},{"Env":"REVISION_HISTORY_LIMIT_HELM_APP","EnvType":"int","EnvValue":"1","EnvDescription":"To set the history limit for the helm app being deployed through devtron","Example":"","Deprecated":"false"},{"Env":"REVISION_HISTORY_LIMIT_LINKED_HELM_APP","EnvType":"int","EnvValue":"15","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"RUN_HELM_INSTALL_IN_ASYNC_MODE_HELM_APPS","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"SHOULD_CHECK_NAMESPACE_ON_CLONE","EnvType":"bool","EnvValue":"false","EnvDescription":"should we check if namespace exists or not while cloning app","Example":"","Deprecated":"false"},{"Env":"USE_DEPLOYMENT_CONFIG_DATA","EnvType":"bool","EnvValue":"false","EnvDescription":"use deployment config data from deployment_config table","Example":"","Deprecated":"true"}]},{"Category":"CI_RUNNER","Fields":[{"Env":"AZURE_ACCOUNT_KEY","EnvType":"string","EnvValue":"","EnvDescription":"If blob storage is being used of azure then pass the secret key to access the bucket","Example":"","Deprecated":"false"},{"Env":"AZURE_ACCOUNT_NAME","EnvType":"string","EnvValue":"","EnvDescription":"Account name for azure blob storage","Example":"","Deprecated":"false"},{"Env":"AZURE_BLOB_CONTAINER_CI_CACHE","EnvType":"string","EnvValue":"","EnvDescription":"Cache bucket name for azure blob storage","Example":"","Deprecated":"false"},{"Env":"AZURE_BLOB_CONTAINER_CI_LOG","EnvType":"string","EnvValue":"","EnvDescription":"Log bucket for azure blob storage","Example":"","Deprecated":"false"},{"Env":"AZURE_GATEWAY_CONNECTION_INSECURE","EnvType":"bool","EnvValue":"true","EnvDescription":"Azure gateway connection allows insecure if true","Example":"","Deprecated":"false"},{"Env":"AZURE_GATEWAY_URL","EnvType":"string","EnvValue":"http://devtron-minio.devtroncd:9000","EnvDescription":"Sent to CI runner for blob","Example":"","Deprecated":"false"},{"Env":"BASE_LOG_LOCATION_PATH","EnvType":"string","EnvValue":"/home/devtron/","EnvDescription":"Used to store, download logs of ci workflow, artifact","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_GCP_CREDENTIALS_JSON","EnvType":"string","EnvValue":"","EnvDescription":"GCP cred json for GCS blob storage","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_PROVIDER","EnvType":"","EnvValue":"S3","EnvDescription":"Blob storage provider name(AWS/GCP/Azure)","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_S3_ACCESS_KEY","EnvType":"string","EnvValue":"","EnvDescription":"S3 access key for s3 blob storage","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_S3_BUCKET_VERSIONED","EnvType":"bool","EnvValue":"true","EnvDescription":"To enable buctet versioning for blob storage","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_S3_ENDPOINT","EnvType":"string","EnvValue":"","EnvDescription":"S3 endpoint URL for s3 blob storage","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_S3_ENDPOINT_INSECURE","EnvType":"bool","EnvValue":"false","EnvDescription":"To use insecure s3 endpoint","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_S3_SECRET_KEY","EnvType":"string","EnvValue":"","EnvDescription":"Secret key for s3 blob storage","Example":"","Deprecated":"false"},{"Env":"BUILDX_CACHE_PATH","EnvType":"string","EnvValue":"/var/lib/devtron/buildx","EnvDescription":"Path for the buildx cache","Example":"","Deprecated":"false"},{"Env":"BUILDX_K8S_DRIVER_OPTIONS","EnvType":"string","EnvValue":"","EnvDescription":"To enable the k8s driver and pass args for k8s driver in buildx","Example":"","Deprecated":"false"},{"Env":"BUILDX_PROVENANCE_MODE","EnvType":"string","EnvValue":"","EnvDescription":"provinance is set to true by default by docker. this will add some build related data in generated build manifest.it also adds some unknown:unknown key:value pair which may not be compatible by some container registries. with buildx k8s driver , provinenance=true is causing issue when push manifest to quay registry, so setting it to false","Example":"","Deprecated":"false"},{"Env":"BUILD_LOG_TTL_VALUE_IN_SECS","EnvType":"int","EnvValue":"3600","EnvDescription":"This is the time that the pods of ci/pre-cd/post-cd live after completion state.","Example":"","Deprecated":"false"},{"Env":"CACHE_LIMIT","EnvType":"int64","EnvValue":"5000000000","EnvDescription":"Cache limit.","Example":"","Deprecated":"false"},{"Env":"CD_DEFAULT_ADDRESS_POOL_BASE_CIDR","EnvType":"string","EnvValue":"","EnvDescription":"To pass the IP cidr for Pre/Post cd ","Example":"","Deprecated":"false"},{"Env":"CD_DEFAULT_ADDRESS_POOL_SIZE","EnvType":"int","EnvValue":"","EnvDescription":"The subnet size to allocate from the base pool for CD","Example":"","Deprecated":"false"},{"Env":"CD_LIMIT_CI_CPU","EnvType":"string","EnvValue":"0.5","EnvDescription":"CPU Resource Limit Pre/Post CD","Example":"","Deprecated":"false"},{"Env":"CD_LIMIT_CI_MEM","EnvType":"string","EnvValue":"3G","EnvDescription":"Memory Resource Limit Pre/Post CD","Example":"","Deprecated":"false"},{"Env":"CD_NODE_LABEL_SELECTOR","EnvType":"","EnvValue":"","EnvDescription":"Node label selector for Pre/Post CD","Example":"","Deprecated":"false"},{"Env":"CD_NODE_TAINTS_KEY","EnvType":"string","EnvValue":"dedicated","EnvDescription":"Toleration key for Pre/Post CD","Example":"","Deprecated":"false"},{"Env":"CD_NODE_TAINTS_VALUE","EnvType":"string","EnvValue":"ci","EnvDescription":"Toleration value for Pre/Post CD","Example":"","Deprecated":"false"},{"Env":"CD_REQ_CI_CPU","EnvType":"string","EnvValue":"0.5","EnvDescription":"CPU Resource Rquest Pre/Post CD","Example":"","Deprecated":"false"},{"Env":"CD_REQ_CI_MEM","EnvType":"string","EnvValue":"3G","EnvDescription":"Memory Resource Rquest Pre/Post CD","Example":"","Deprecated":"false"},{"Env":"CD_WORKFLOW_EXECUTOR_TYPE","EnvType":"","EnvValue":"AWF","EnvDescription":"Executor type for Pre/Post CD(AWF,System)","Example":"","Deprecated":"false"},{"Env":"CD_WORKFLOW_SERVICE_ACCOUNT","EnvType":"string","EnvValue":"cd-runner","EnvDescription":"Service account to be used in Pre/Post CD pod","Example":"","Deprecated":"false"},{"Env":"CI_DEFAULT_ADDRESS_POOL_BASE_CIDR","EnvType":"string","EnvValue":"","EnvDescription":"To pass the IP cidr for CI","Example":"","Deprecated":"false"},{"Env":"CI_DEFAULT_ADDRESS_POOL_SIZE","EnvType":"int","EnvValue":"","EnvDescription":"The subnet size to allocate from the base pool for CI","Example":"","Deprecated":"false"},{"Env":"CI_IGNORE_DOCKER_CACHE","EnvType":"bool","EnvValue":"","EnvDescription":"Ignoring docker cache ","Example":"","Deprecated":"false"},{"Env":"CI_LOGS_KEY_PREFIX","EnvType":"string","EnvValue":"","EnvDescription":"Prefix for build logs","Example":"","Deprecated":"false"},{"Env":"CI_NODE_LABEL_SELECTOR","EnvType":"","EnvValue":"","EnvDescription":"Node label selector for CI","Example":"","Deprecated":"false"},{"Env":"CI_NODE_TAINTS_KEY","EnvType":"string","EnvValue":"","EnvDescription":"Toleration key for CI","Example":"","Deprecated":"false"},{"Env":"CI_NODE_TAINTS_VALUE","EnvType":"string","EnvValue":"","EnvDescription":"Toleration value for CI","Example":"","Deprecated":"false"},{"Env":"CI_RUNNER_DOCKER_MTU_VALUE","EnvType":"int","EnvValue":"-1","EnvDescription":"this is to control the bytes of inofrmation passed in a network packet in ci-runner. default is -1 (defaults to the underlying node mtu value)","Example":"","Deprecated":"false"},{"Env":"CI_SUCCESS_AUTO_TRIGGER_BATCH_SIZE","EnvType":"int","EnvValue":"1","EnvDescription":"this is to control the no of linked pipelines should be hanled in one go when a ci-success event of an parent ci is received","Example":"","Deprecated":"false"},{"Env":"CI_VOLUME_MOUNTS_JSON","EnvType":"string","EnvValue":"","EnvDescription":"additional volume mount data for CI and JOB","Example":"","Deprecated":"false"},{"Env":"CI_WORKFLOW_EXECUTOR_TYPE","EnvType":"","EnvValue":"AWF","EnvDescription":"Executor type for CI(AWF,System)","Example":"","Deprecated":"false"},{"Env":"DEFAULT_ARTIFACT_KEY_LOCATION","EnvType":"string","EnvValue":"arsenal-v1/ci-artifacts","EnvDescription":"Key location for artifacts being created","Example":"","Deprecated":"false"},{"Env":"DEFAULT_BUILD_LOGS_BUCKET","EnvType":"string","EnvValue":"devtron-pro-ci-logs","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_BUILD_LOGS_KEY_PREFIX","EnvType":"string","EnvValue":"arsenal-v1","EnvDescription":"Bucket prefix for build logs","Example":"","Deprecated":"false"},{"Env":"DEFAULT_CACHE_BUCKET","EnvType":"string","EnvValue":"ci-caching","EnvDescription":"Bucket name for build cache","Example":"","Deprecated":"false"},{"Env":"DEFAULT_CACHE_BUCKET_REGION","EnvType":"string","EnvValue":"us-east-2","EnvDescription":"Build Cache bucket region","Example":"","Deprecated":"false"},{"Env":"DEFAULT_CD_ARTIFACT_KEY_LOCATION","EnvType":"string","EnvValue":"","EnvDescription":"Bucket prefix for build cache","Example":"","Deprecated":"false"},{"Env":"DEFAULT_CD_LOGS_BUCKET_REGION","EnvType":"string","EnvValue":"us-east-2","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_CD_NAMESPACE","EnvType":"string","EnvValue":"","EnvDescription":"Namespace for devtron stack","Example":"","Deprecated":"false"},{"Env":"DEFAULT_CD_TIMEOUT","EnvType":"int64","EnvValue":"3600","EnvDescription":"Timeout for Pre/Post-Cd to be completed","Example":"","Deprecated":"false"},{"Env":"DEFAULT_CI_IMAGE","EnvType":"string","EnvValue":"686244538589.dkr.ecr.us-east-2.amazonaws.com/cirunner:47","EnvDescription":"To pass the ci-runner image","Example":"","Deprecated":"false"},{"Env":"DEFAULT_NAMESPACE","EnvType":"string","EnvValue":"devtron-ci","EnvDescription":"Timeout for CI to be completed","Example":"","Deprecated":"false"},{"Env":"DEFAULT_TARGET_PLATFORM","EnvType":"string","EnvValue":"","EnvDescription":"Default architecture for buildx","Example":"","Deprecated":"false"},{"Env":"DOCKER_BUILD_CACHE_PATH","EnvType":"string","EnvValue":"/var/lib/docker","EnvDescription":"Path to store cache of docker build (/var/lib/docker-\u003e for legacy docker build, /var/lib/devtron-\u003e for buildx)","Example":"","Deprecated":"false"},{"Env":"ENABLE_BUILD_CONTEXT","EnvType":"bool","EnvValue":"false","EnvDescription":"To Enable build context in Devtron.","Example":"","Deprecated":"false"},{"Env":"ENABLE_WORKFLOW_EXECUTION_STAGE","EnvType":"bool","EnvValue":"true","EnvDescription":"if enabled then we will display build stages separately for CI/Job/Pre-Post CD","Example":"true","Deprecated":"false"},{"Env":"EXTERNAL_BLOB_STORAGE_CM_NAME","EnvType":"string","EnvValue":"blob-storage-cm","EnvDescription":"name of the config map(contains bucket name, etc.) in external cluster when there is some operation related to external cluster, for example:-downloading cd artifact pushed in external cluster's env and we need to download from there, downloads ci logs pushed in external cluster's blob","Example":"","Deprecated":"false"},{"Env":"EXTERNAL_BLOB_STORAGE_SECRET_NAME","EnvType":"string","EnvValue":"blob-storage-secret","EnvDescription":"name of the secret(contains password, accessId,passKeys, etc.) in external cluster when there is some operation related to external cluster, for example:-downloading cd artifact pushed in external cluster's env and we need to download from there, downloads ci logs pushed in external cluster's blob","Example":"","Deprecated":"false"},{"Env":"EXTERNAL_CD_NODE_LABEL_SELECTOR","EnvType":"","EnvValue":"","EnvDescription":"This is an array of strings used when submitting a workflow for pre or post-CD execution. If the ","Example":"","Deprecated":"false"},{"Env":"EXTERNAL_CD_NODE_TAINTS_KEY","EnvType":"string","EnvValue":"dedicated","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"EXTERNAL_CD_NODE_TAINTS_VALUE","EnvType":"string","EnvValue":"ci","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"EXTERNAL_CI_API_SECRET","EnvType":"string","EnvValue":"devtroncd-secret","EnvDescription":"External CI API secret.","Example":"","Deprecated":"false"},{"Env":"EXTERNAL_CI_PAYLOAD","EnvType":"string","EnvValue":"{\"ciProjectDetails\":[{\"gitRepository\":\"https://github.com/vikram1601/getting-started-nodejs.git\",\"checkoutPath\":\"./abc\",\"commitHash\":\"239077135f8cdeeccb7857e2851348f558cb53d3\",\"commitTime\":\"2022-10-30T20:00:00\",\"branch\":\"master\",\"message\":\"Update README.md\",\"author\":\"User Name \"}],\"dockerImage\":\"445808685819.dkr.ecr.us-east-2.amazonaws.com/orch:23907713-2\"}","EnvDescription":"External CI payload with project details.","Example":"","Deprecated":"false"},{"Env":"EXTERNAL_CI_WEB_HOOK_URL","EnvType":"string","EnvValue":"","EnvDescription":"default is {{HOST_URL}}/orchestrator/webhook/ext-ci. It is used for external ci.","Example":"","Deprecated":"false"},{"Env":"IGNORE_CM_CS_IN_CI_JOB","EnvType":"bool","EnvValue":"false","EnvDescription":"Ignore CM/CS in CI-pipeline as Job","Example":"","Deprecated":"false"},{"Env":"IMAGE_RETRY_COUNT","EnvType":"int","EnvValue":"0","EnvDescription":"push artifact(image) in ci retry count ","Example":"","Deprecated":"false"},{"Env":"IMAGE_RETRY_INTERVAL","EnvType":"int","EnvValue":"5","EnvDescription":"image retry interval takes value in seconds","Example":"","Deprecated":"false"},{"Env":"IMAGE_SCANNER_ENDPOINT","EnvType":"string","EnvValue":"http://image-scanner-new-demo-devtroncd-service.devtroncd:80","EnvDescription":"Image-scanner micro-service URL","Example":"","Deprecated":"false"},{"Env":"IMAGE_SCAN_MAX_RETRIES","EnvType":"int","EnvValue":"3","EnvDescription":"Max retry count for image-scanning","Example":"","Deprecated":"false"},{"Env":"IMAGE_SCAN_RETRY_DELAY","EnvType":"int","EnvValue":"5","EnvDescription":"Delay for the image-scaning to start","Example":"","Deprecated":"false"},{"Env":"IN_APP_LOGGING_ENABLED","EnvType":"bool","EnvValue":"false","EnvDescription":"Used in case of argo workflow is enabled. If enabled logs push will be managed by us, else will be managed by argo workflow.","Example":"","Deprecated":"false"},{"Env":"MAX_CD_WORKFLOW_RUNNER_RETRIES","EnvType":"int","EnvValue":"0","EnvDescription":"Maximum time pre/post-cd-workflow create pod if it fails to complete","Example":"","Deprecated":"false"},{"Env":"MAX_CI_WORKFLOW_RETRIES","EnvType":"int","EnvValue":"0","EnvDescription":"Maximum time CI-workflow create pod if it fails to complete","Example":"","Deprecated":"false"},{"Env":"MODE","EnvType":"string","EnvValue":"DEV","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_SERVER_HOST","EnvType":"string","EnvValue":"localhost:4222","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ORCH_HOST","EnvType":"string","EnvValue":"http://devtroncd-orchestrator-service-prod.devtroncd/webhook/msg/nats","EnvDescription":"Orchestrator micro-service URL ","Example":"","Deprecated":"false"},{"Env":"ORCH_TOKEN","EnvType":"string","EnvValue":"","EnvDescription":"Orchestrator token","Example":"","Deprecated":"false"},{"Env":"PRE_CI_CACHE_PATH","EnvType":"string","EnvValue":"/devtroncd-cache","EnvDescription":"Cache path for Pre CI tasks","Example":"","Deprecated":"false"},{"Env":"SHOW_DOCKER_BUILD_ARGS","EnvType":"bool","EnvValue":"true","EnvDescription":"To enable showing the args passed for CI in build logs","Example":"","Deprecated":"false"},{"Env":"SKIP_CI_JOB_BUILD_CACHE_PUSH_PULL","EnvType":"bool","EnvValue":"false","EnvDescription":"To skip cache Push/Pull for ci job","Example":"","Deprecated":"false"},{"Env":"SKIP_CREATING_ECR_REPO","EnvType":"bool","EnvValue":"false","EnvDescription":"By disabling this ECR repo won't get created if it's not available on ECR from build configuration","Example":"","Deprecated":"false"},{"Env":"TERMINATION_GRACE_PERIOD_SECS","EnvType":"int","EnvValue":"180","EnvDescription":"this is the time given to workflow pods to shutdown. (grace full termination time)","Example":"","Deprecated":"false"},{"Env":"USE_ARTIFACT_LISTING_QUERY_V2","EnvType":"bool","EnvValue":"true","EnvDescription":"To use the V2 query for listing artifacts","Example":"","Deprecated":"false"},{"Env":"USE_BLOB_STORAGE_CONFIG_IN_CD_WORKFLOW","EnvType":"bool","EnvValue":"true","EnvDescription":"To enable blob storage in pre and post cd","Example":"","Deprecated":"false"},{"Env":"USE_BLOB_STORAGE_CONFIG_IN_CI_WORKFLOW","EnvType":"bool","EnvValue":"true","EnvDescription":"To enable blob storage in pre and post ci","Example":"","Deprecated":"false"},{"Env":"USE_BUILDX","EnvType":"bool","EnvValue":"false","EnvDescription":"To enable buildx feature globally","Example":"","Deprecated":"false"},{"Env":"USE_DOCKER_API_TO_GET_DIGEST","EnvType":"bool","EnvValue":"false","EnvDescription":"when user do not pass the digest then this flag controls , finding the image digest using docker API or not. if set to true we get the digest from docker API call else use docker pull command. [logic in ci-runner]","Example":"","Deprecated":"false"},{"Env":"USE_EXTERNAL_NODE","EnvType":"bool","EnvValue":"false","EnvDescription":"It is used in case of Pre/ Post Cd with run in application mode. If enabled the node lebels are read from EXTERNAL_CD_NODE_LABEL_SELECTOR else from CD_NODE_LABEL_SELECTOR MODE: if the vale is DEV, it will read the local kube config file or else from the cluser location.","Example":"","Deprecated":"false"},{"Env":"USE_IMAGE_TAG_FROM_GIT_PROVIDER_FOR_TAG_BASED_BUILD","EnvType":"bool","EnvValue":"false","EnvDescription":"To use the same tag in container image as that of git tag","Example":"","Deprecated":"false"},{"Env":"WF_CONTROLLER_INSTANCE_ID","EnvType":"string","EnvValue":"devtron-runner","EnvDescription":"Workflow controller instance ID.","Example":"","Deprecated":"false"},{"Env":"WORKFLOW_CACHE_CONFIG","EnvType":"string","EnvValue":"{}","EnvDescription":"flag is used to configure how Docker caches are handled during a CI/CD ","Example":"","Deprecated":"false"},{"Env":"WORKFLOW_SERVICE_ACCOUNT","EnvType":"string","EnvValue":"ci-runner","EnvDescription":"","Example":"","Deprecated":"false"}]},{"Category":"DEVTRON","Fields":[{"Env":"-","EnvType":"","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ADDITIONAL_NODE_GROUP_LABELS","EnvType":"","EnvValue":"","EnvDescription":"Add comma separated list of additional node group labels to default labels","Example":"karpenter.sh/nodepool,cloud.google.com/gke-nodepool","Deprecated":"false"},{"Env":"APP_SYNC_IMAGE","EnvType":"string","EnvValue":"quay.io/devtron/chart-sync:1227622d-132-3775","EnvDescription":"For the app sync image, this image will be used in app-manual sync job","Example":"","Deprecated":"false"},{"Env":"APP_SYNC_JOB_RESOURCES_OBJ","EnvType":"string","EnvValue":"","EnvDescription":"To pass the resource of app sync","Example":"","Deprecated":"false"},{"Env":"APP_SYNC_SERVICE_ACCOUNT","EnvType":"string","EnvValue":"chart-sync","EnvDescription":"Service account to be used in app sync Job","Example":"","Deprecated":"false"},{"Env":"APP_SYNC_SHUTDOWN_WAIT_DURATION","EnvType":"int","EnvValue":"120","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ARGO_AUTO_SYNC_ENABLED","EnvType":"bool","EnvValue":"true","EnvDescription":"If enabled all argocd application will have auto sync enabled","Example":"","Deprecated":"false"},{"Env":"ARGO_GIT_COMMIT_RETRY_COUNT_ON_CONFLICT","EnvType":"int","EnvValue":"3","EnvDescription":"retry argocd app manual sync if the timeline is stuck in ARGOCD_SYNC_INITIATED state for more than this defined time (in mins)","Example":"","Deprecated":"false"},{"Env":"ARGO_GIT_COMMIT_RETRY_DELAY_ON_CONFLICT","EnvType":"int","EnvValue":"1","EnvDescription":"Delay on retrying the maifest commit the on gitops","Example":"","Deprecated":"false"},{"Env":"ARGO_REPO_REGISTER_RETRY_COUNT","EnvType":"int","EnvValue":"3","EnvDescription":"Argo app registration in argo retries on deployment","Example":"","Deprecated":"false"},{"Env":"ARGO_REPO_REGISTER_RETRY_DELAY","EnvType":"int","EnvValue":"10","EnvDescription":"Argo app registration in argo cd on deployment delay between retry","Example":"","Deprecated":"false"},{"Env":"ASYNC_BUILDX_CACHE_EXPORT","EnvType":"bool","EnvValue":"false","EnvDescription":"To enable async container image cache export","Example":"","Deprecated":"false"},{"Env":"BATCH_SIZE","EnvType":"int","EnvValue":"5","EnvDescription":"there is feature to get URL's of services/ingresses. so to extract those, we need to parse all the servcie and ingress objects of the application. this BATCH_SIZE flag controls the no of these objects get parsed in one go.","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_ENABLED","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"BUILDX_CACHE_MODE_MIN","EnvType":"bool","EnvValue":"false","EnvDescription":"To set build cache mode to minimum in buildx","Example":"","Deprecated":"false"},{"Env":"CD_HOST","EnvType":"string","EnvValue":"localhost","EnvDescription":"Host for the devtron stack","Example":"","Deprecated":"false"},{"Env":"CD_NAMESPACE","EnvType":"string","EnvValue":"devtroncd","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_PORT","EnvType":"string","EnvValue":"8000","EnvDescription":"Port for pre/post-cd","Example":"","Deprecated":"false"},{"Env":"CExpirationTime","EnvType":"int","EnvValue":"600","EnvDescription":"Caching expiration time.","Example":"","Deprecated":"false"},{"Env":"CI_TRIGGER_CRON_TIME","EnvType":"int","EnvValue":"2","EnvDescription":"For image poll plugin","Example":"","Deprecated":"false"},{"Env":"CI_WORKFLOW_STATUS_UPDATE_CRON","EnvType":"string","EnvValue":"*/5 * * * *","EnvDescription":"Cron schedule for CI pipeline status","Example":"","Deprecated":"false"},{"Env":"CLI_CMD_TIMEOUT_GLOBAL_SECONDS","EnvType":"int","EnvValue":"0","EnvDescription":"Used in git cli opeartion timeout","Example":"","Deprecated":"false"},{"Env":"CLUSTER_STATUS_CRON_TIME","EnvType":"int","EnvValue":"15","EnvDescription":"Cron schedule for cluster status on resource browser","Example":"","Deprecated":"false"},{"Env":"CONSUMER_CONFIG_JSON","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_LOG_TIME_LIMIT","EnvType":"int64","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_TIMEOUT","EnvType":"float64","EnvValue":"3600","EnvDescription":"Timeout for CI to be completed","Example":"","Deprecated":"false"},{"Env":"DEVTRON_BOM_URL","EnvType":"string","EnvValue":"https://raw.githubusercontent.com/devtron-labs/devtron/%s/charts/devtron/devtron-bom.yaml","EnvDescription":"Path to devtron-bom.yaml of devtron charts, used for module installation and devtron upgrade","Example":"","Deprecated":"false"},{"Env":"DEVTRON_DEFAULT_NAMESPACE","EnvType":"string","EnvValue":"devtroncd","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEVTRON_DEX_SECRET_NAMESPACE","EnvType":"string","EnvValue":"devtroncd","EnvDescription":"Namespace of dex secret","Example":"","Deprecated":"false"},{"Env":"DEVTRON_HELM_RELEASE_CHART_NAME","EnvType":"string","EnvValue":"devtron-operator","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEVTRON_HELM_RELEASE_NAME","EnvType":"string","EnvValue":"devtron","EnvDescription":"Name of the Devtron Helm release. ","Example":"","Deprecated":"false"},{"Env":"DEVTRON_HELM_RELEASE_NAMESPACE","EnvType":"string","EnvValue":"devtroncd","EnvDescription":"Namespace of the Devtron Helm release","Example":"","Deprecated":"false"},{"Env":"DEVTRON_HELM_REPO_NAME","EnvType":"string","EnvValue":"devtron","EnvDescription":"Is used to install modules (stack manager)","Example":"","Deprecated":"false"},{"Env":"DEVTRON_HELM_REPO_URL","EnvType":"string","EnvValue":"https://helm.devtron.ai","EnvDescription":"Is used to install modules (stack manager)","Example":"","Deprecated":"false"},{"Env":"DEVTRON_INSTALLATION_TYPE","EnvType":"string","EnvValue":"","EnvDescription":"Devtron Installation type(EA/Full)","Example":"","Deprecated":"false"},{"Env":"DEVTRON_MODULES_IDENTIFIER_IN_HELM_VALUES","EnvType":"string","EnvValue":"installer.modules","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEVTRON_SECRET_NAME","EnvType":"string","EnvValue":"devtron-secret","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEVTRON_VERSION_IDENTIFIER_IN_HELM_VALUES","EnvType":"string","EnvValue":"installer.release","EnvDescription":"devtron operator version identifier in helm values yaml","Example":"","Deprecated":"false"},{"Env":"DEX_CID","EnvType":"string","EnvValue":"example-app","EnvDescription":"dex client id ","Example":"","Deprecated":"false"},{"Env":"DEX_CLIENT_ID","EnvType":"string","EnvValue":"argo-cd","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEX_CSTOREKEY","EnvType":"string","EnvValue":"","EnvDescription":"DEX CSTOREKEY.","Example":"","Deprecated":"false"},{"Env":"DEX_JWTKEY","EnvType":"string","EnvValue":"","EnvDescription":"DEX JWT key. ","Example":"","Deprecated":"false"},{"Env":"DEX_RURL","EnvType":"string","EnvValue":"http://127.0.0.1:8080/callback","EnvDescription":"Dex redirect URL(http://argocd-dex-server.devtroncd:8080/callback)","Example":"","Deprecated":"false"},{"Env":"DEX_SCOPES","EnvType":"","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEX_SECRET","EnvType":"string","EnvValue":"","EnvDescription":"Dex secret","Example":"","Deprecated":"false"},{"Env":"DEX_URL","EnvType":"string","EnvValue":"","EnvDescription":"Dex service endpoint with dex path(http://argocd-dex-server.devtroncd:5556/dex)","Example":"","Deprecated":"false"},{"Env":"ECR_REPO_NAME_PREFIX","EnvType":"string","EnvValue":"test/","EnvDescription":"Prefix for ECR repo to be created in does not exist","Example":"","Deprecated":"false"},{"Env":"ENABLE_ASYNC_ARGO_CD_INSTALL_DEVTRON_CHART","EnvType":"bool","EnvValue":"false","EnvDescription":"To enable async installation of gitops application","Example":"","Deprecated":"false"},{"Env":"ENABLE_ASYNC_INSTALL_DEVTRON_CHART","EnvType":"bool","EnvValue":"false","EnvDescription":"To enable async installation of no-gitops application","Example":"","Deprecated":"false"},{"Env":"EPHEMERAL_SERVER_VERSION_REGEX","EnvType":"string","EnvValue":"v[1-9]\\.\\b(2[3-9]\\|[3-9][0-9])\\b.*","EnvDescription":"ephemeral containers support version regex that is compared with k8sServerVersion","Example":"","Deprecated":"false"},{"Env":"EVENT_URL","EnvType":"string","EnvValue":"http://localhost:3000/notify","EnvDescription":"Notifier service url","Example":"","Deprecated":"false"},{"Env":"EXECUTE_WIRE_NIL_CHECKER","EnvType":"bool","EnvValue":"false","EnvDescription":"checks for any nil pointer in wire.go","Example":"","Deprecated":"false"},{"Env":"EXPOSE_CI_METRICS","EnvType":"bool","EnvValue":"false","EnvDescription":"To expose CI metrics","Example":"","Deprecated":"false"},{"Env":"FEATURE_RESTART_WORKLOAD_BATCH_SIZE","EnvType":"int","EnvValue":"1","EnvDescription":"restart workload retrieval batch size ","Example":"","Deprecated":"false"},{"Env":"FEATURE_RESTART_WORKLOAD_WORKER_POOL_SIZE","EnvType":"int","EnvValue":"5","EnvDescription":"restart workload retrieval pool size","Example":"","Deprecated":"false"},{"Env":"FORCE_SECURITY_SCANNING","EnvType":"bool","EnvValue":"false","EnvDescription":"By enabling this no one can disable image scaning on ci-pipeline from UI","Example":"","Deprecated":"false"},{"Env":"GITOPS_REPO_PREFIX","EnvType":"string","EnvValue":"","EnvDescription":"Prefix for Gitops repo being creation for argocd application","Example":"","Deprecated":"false"},{"Env":"GO_RUNTIME_ENV","EnvType":"string","EnvValue":"production","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"GRAFANA_HOST","EnvType":"string","EnvValue":"localhost","EnvDescription":"Host URL for the grafana dashboard","Example":"","Deprecated":"false"},{"Env":"GRAFANA_NAMESPACE","EnvType":"string","EnvValue":"devtroncd","EnvDescription":"Namespace for grafana","Example":"","Deprecated":"false"},{"Env":"GRAFANA_ORG_ID","EnvType":"int","EnvValue":"2","EnvDescription":"Org ID for grafana for application metrics","Example":"","Deprecated":"false"},{"Env":"GRAFANA_PASSWORD","EnvType":"string","EnvValue":"prom-operator","EnvDescription":"Password for grafana dashboard","Example":"","Deprecated":"false"},{"Env":"GRAFANA_PORT","EnvType":"string","EnvValue":"8090","EnvDescription":"Port for grafana micro-service","Example":"","Deprecated":"false"},{"Env":"GRAFANA_URL","EnvType":"string","EnvValue":"","EnvDescription":"Host URL for the grafana dashboard","Example":"","Deprecated":"false"},{"Env":"GRAFANA_USERNAME","EnvType":"string","EnvValue":"admin","EnvDescription":"Username for grafana ","Example":"","Deprecated":"false"},{"Env":"HIDE_IMAGE_TAGGING_HARD_DELETE","EnvType":"bool","EnvValue":"false","EnvDescription":"Flag to hide the hard delete option in the image tagging service","Example":"","Deprecated":"false"},{"Env":"IGNORE_AUTOCOMPLETE_AUTH_CHECK","EnvType":"bool","EnvValue":"false","EnvDescription":"flag for ignoring auth check in autocomplete apis.","Example":"","Deprecated":"false"},{"Env":"INSTALLED_MODULES","EnvType":"","EnvValue":"","EnvDescription":"List of installed modules given in helm values/yaml are written in cm and used by devtron to know which modules are given","Example":"security.trivy,security.clair","Deprecated":"false"},{"Env":"INSTALLER_CRD_NAMESPACE","EnvType":"string","EnvValue":"devtroncd","EnvDescription":"namespace where Custom Resource Definitions get installed","Example":"","Deprecated":"false"},{"Env":"INSTALLER_CRD_OBJECT_GROUP_NAME","EnvType":"string","EnvValue":"installer.devtron.ai","EnvDescription":"Devtron installer CRD group name, partially deprecated.","Example":"","Deprecated":"false"},{"Env":"INSTALLER_CRD_OBJECT_RESOURCE","EnvType":"string","EnvValue":"installers","EnvDescription":"Devtron installer CRD resource name, partially deprecated","Example":"","Deprecated":"false"},{"Env":"INSTALLER_CRD_OBJECT_VERSION","EnvType":"string","EnvValue":"v1alpha1","EnvDescription":"version of the CRDs. default is v1alpha1","Example":"","Deprecated":"false"},{"Env":"IS_AIR_GAP_ENVIRONMENT","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"JwtExpirationTime","EnvType":"int","EnvValue":"120","EnvDescription":"JWT expiration time.","Example":"","Deprecated":"false"},{"Env":"K8s_CLIENT_MAX_IDLE_CONNS_PER_HOST","EnvType":"int","EnvValue":"25","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"K8s_TCP_IDLE_CONN_TIMEOUT","EnvType":"int","EnvValue":"300","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"K8s_TCP_KEEPALIVE","EnvType":"int","EnvValue":"30","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"K8s_TCP_TIMEOUT","EnvType":"int","EnvValue":"30","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"K8s_TLS_HANDSHAKE_TIMEOUT","EnvType":"int","EnvValue":"10","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"KUBELINK_GRPC_MAX_RECEIVE_MSG_SIZE","EnvType":"int","EnvValue":"20","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"KUBELINK_GRPC_MAX_SEND_MSG_SIZE","EnvType":"int","EnvValue":"4","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"LENS_TIMEOUT","EnvType":"int","EnvValue":"0","EnvDescription":"Lens microservice timeout.","Example":"","Deprecated":"false"},{"Env":"LENS_URL","EnvType":"string","EnvValue":"http://lens-milandevtron-service:80","EnvDescription":"Lens micro-service URL","Example":"","Deprecated":"false"},{"Env":"LIMIT_CI_CPU","EnvType":"string","EnvValue":"0.5","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"LIMIT_CI_MEM","EnvType":"string","EnvValue":"3G","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"LOGGER_DEV_MODE","EnvType":"bool","EnvValue":"false","EnvDescription":"Enables a different logger theme.","Example":"","Deprecated":"false"},{"Env":"LOG_LEVEL","EnvType":"int","EnvValue":"-1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"MAX_SESSION_PER_USER","EnvType":"int","EnvValue":"5","EnvDescription":"max no of cluster terminal pods can be created by an user","Example":"","Deprecated":"false"},{"Env":"MODULE_METADATA_API_URL","EnvType":"string","EnvValue":"https://api.devtron.ai/module?name=%s","EnvDescription":"Modules list and meta info will be fetched from this server, that is central api server of devtron.","Example":"","Deprecated":"false"},{"Env":"MODULE_STATUS_HANDLING_CRON_DURATION_MIN","EnvType":"int","EnvValue":"3","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_ACK_WAIT_IN_SECS","EnvType":"int","EnvValue":"120","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_BUFFER_SIZE","EnvType":"int","EnvValue":"-1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_MAX_AGE","EnvType":"int","EnvValue":"86400","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_PROCESSING_BATCH_SIZE","EnvType":"int","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_REPLICAS","EnvType":"int","EnvValue":"0","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NOTIFICATION_MEDIUM","EnvType":"NotificationMedium","EnvValue":"rest","EnvDescription":"notification medium","Example":"","Deprecated":"false"},{"Env":"OTEL_COLLECTOR_URL","EnvType":"string","EnvValue":"","EnvDescription":"Opentelemetry URL ","Example":"","Deprecated":"false"},{"Env":"PARALLELISM_LIMIT_FOR_TAG_PROCESSING","EnvType":"int","EnvValue":"","EnvDescription":"App manual sync job parallel tag processing count.","Example":"","Deprecated":"false"},{"Env":"PG_EXPORT_PROM_METRICS","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_LOG_ALL_FAILURE_QUERIES","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_LOG_ALL_QUERY","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_LOG_SLOW_QUERY","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_QUERY_DUR_THRESHOLD","EnvType":"int64","EnvValue":"5000","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PLUGIN_NAME","EnvType":"string","EnvValue":"Pull images from container repository","EnvDescription":"Handles image retrieval from a container repository and triggers subsequent CI processes upon detecting new images.Current default plugin name: Pull Images from Container Repository.","Example":"","Deprecated":"false"},{"Env":"PROPAGATE_EXTRA_LABELS","EnvType":"bool","EnvValue":"false","EnvDescription":"Add additional propagate labels like api.devtron.ai/appName, api.devtron.ai/envName, api.devtron.ai/project along with the user defined ones.","Example":"","Deprecated":"false"},{"Env":"PROXY_SERVICE_CONFIG","EnvType":"string","EnvValue":"{}","EnvDescription":"Proxy configuration for micro-service to be accessible on orhcestrator ingress","Example":"","Deprecated":"false"},{"Env":"REQ_CI_CPU","EnvType":"string","EnvValue":"0.5","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"REQ_CI_MEM","EnvType":"string","EnvValue":"3G","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"RESTRICT_TERMINAL_ACCESS_FOR_NON_SUPER_USER","EnvType":"bool","EnvValue":"false","EnvDescription":"To restrict the cluster terminal from user having non-super admin acceess","Example":"","Deprecated":"false"},{"Env":"RUNTIME_CONFIG_LOCAL_DEV","EnvType":"LocalDevMode","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"SCOPED_VARIABLE_ENABLED","EnvType":"bool","EnvValue":"false","EnvDescription":"To enable scoped variable option","Example":"","Deprecated":"false"},{"Env":"SCOPED_VARIABLE_FORMAT","EnvType":"string","EnvValue":"@{{%s}}","EnvDescription":"Its a scope format for varialbe name.","Example":"","Deprecated":"false"},{"Env":"SCOPED_VARIABLE_HANDLE_PRIMITIVES","EnvType":"bool","EnvValue":"false","EnvDescription":"This describe should we handle primitives or not in scoped variable template parsing.","Example":"","Deprecated":"false"},{"Env":"SCOPED_VARIABLE_NAME_REGEX","EnvType":"string","EnvValue":"^[a-zA-Z][a-zA-Z0-9_-]{0,62}[a-zA-Z0-9]$","EnvDescription":"Regex for scoped variable name that must passed this regex.","Example":"","Deprecated":"false"},{"Env":"SOCKET_DISCONNECT_DELAY_SECONDS","EnvType":"int","EnvValue":"5","EnvDescription":"The server closes a session when a client receiving connection have not been seen for a while.This delay is configured by this setting. By default the session is closed when a receiving connection wasn't seen for 5 seconds.","Example":"","Deprecated":"false"},{"Env":"SOCKET_HEARTBEAT_SECONDS","EnvType":"int","EnvValue":"25","EnvDescription":"In order to keep proxies and load balancers from closing long running http requests we need to pretend that the connection is active and send a heartbeat packet once in a while. This setting controls how often this is done. By default a heartbeat packet is sent every 25 seconds.","Example":"","Deprecated":"false"},{"Env":"STREAM_CONFIG_JSON","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"SYSTEM_VAR_PREFIX","EnvType":"string","EnvValue":"DEVTRON_","EnvDescription":"Scoped variable prefix, variable name must have this prefix.","Example":"","Deprecated":"false"},{"Env":"TERMINAL_POD_DEFAULT_NAMESPACE","EnvType":"string","EnvValue":"default","EnvDescription":"Cluster terminal default namespace","Example":"","Deprecated":"false"},{"Env":"TERMINAL_POD_INACTIVE_DURATION_IN_MINS","EnvType":"int","EnvValue":"10","EnvDescription":"Timeout for cluster terminal to be inactive","Example":"","Deprecated":"false"},{"Env":"TERMINAL_POD_STATUS_SYNC_In_SECS","EnvType":"int","EnvValue":"600","EnvDescription":"this is the time interval at which the status of the cluster terminal pod","Example":"","Deprecated":"false"},{"Env":"TEST_APP","EnvType":"string","EnvValue":"orchestrator","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TEST_PG_ADDR","EnvType":"string","EnvValue":"127.0.0.1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TEST_PG_DATABASE","EnvType":"string","EnvValue":"orchestrator","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TEST_PG_LOG_QUERY","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TEST_PG_PASSWORD","EnvType":"string","EnvValue":"postgrespw","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TEST_PG_PORT","EnvType":"string","EnvValue":"55000","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TEST_PG_USER","EnvType":"string","EnvValue":"postgres","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TIMEOUT_FOR_FAILED_CI_BUILD","EnvType":"string","EnvValue":"15","EnvDescription":"Timeout for Failed CI build ","Example":"","Deprecated":"false"},{"Env":"TIMEOUT_IN_SECONDS","EnvType":"int","EnvValue":"5","EnvDescription":"timeout to compute the urls from services and ingress objects of an application","Example":"","Deprecated":"false"},{"Env":"USER_SESSION_DURATION_SECONDS","EnvType":"int","EnvValue":"86400","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"USE_ARTIFACT_LISTING_API_V2","EnvType":"bool","EnvValue":"true","EnvDescription":"To use the V2 API for listing artifacts in Listing the images in pipeline","Example":"","Deprecated":"false"},{"Env":"USE_CUSTOM_HTTP_TRANSPORT","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"USE_GIT_CLI","EnvType":"bool","EnvValue":"false","EnvDescription":"To enable git cli","Example":"","Deprecated":"false"},{"Env":"USE_RBAC_CREATION_V2","EnvType":"bool","EnvValue":"true","EnvDescription":"To use the V2 for RBAC creation","Example":"","Deprecated":"false"},{"Env":"VARIABLE_CACHE_ENABLED","EnvType":"bool","EnvValue":"true","EnvDescription":"This is used to control caching of all the scope variables defined in the system.","Example":"","Deprecated":"false"},{"Env":"VARIABLE_EXPRESSION_REGEX","EnvType":"string","EnvValue":"@{{([^}]+)}}","EnvDescription":"Scoped variable expression regex","Example":"","Deprecated":"false"},{"Env":"WEBHOOK_TOKEN","EnvType":"string","EnvValue":"","EnvDescription":"If you want to continue using jenkins for CI then please provide this for authentication of requests","Example":"","Deprecated":"false"}]},{"Category":"GITOPS","Fields":[{"Env":"ACD_CM","EnvType":"string","EnvValue":"argocd-cm","EnvDescription":"Name of the argocd CM","Example":"","Deprecated":"false"},{"Env":"ACD_NAMESPACE","EnvType":"string","EnvValue":"devtroncd","EnvDescription":"To pass the argocd namespace","Example":"","Deprecated":"false"},{"Env":"ACD_PASSWORD","EnvType":"string","EnvValue":"","EnvDescription":"Password for the Argocd (deprecated)","Example":"","Deprecated":"false"},{"Env":"ACD_USERNAME","EnvType":"string","EnvValue":"admin","EnvDescription":"User name for argocd","Example":"","Deprecated":"false"},{"Env":"GITOPS_SECRET_NAME","EnvType":"string","EnvValue":"devtron-gitops-secret","EnvDescription":"devtron-gitops-secret","Example":"","Deprecated":"false"},{"Env":"RESOURCE_LIST_FOR_REPLICAS","EnvType":"string","EnvValue":"Deployment,Rollout,StatefulSet,ReplicaSet","EnvDescription":"this holds the list of k8s resource names which support replicas key. this list used in hibernate/un hibernate process","Example":"","Deprecated":"false"},{"Env":"RESOURCE_LIST_FOR_REPLICAS_BATCH_SIZE","EnvType":"int","EnvValue":"5","EnvDescription":"this the batch size to control no of above resources can be parsed in one go to determine hibernate status","Example":"","Deprecated":"false"}]},{"Category":"INFRA_SETUP","Fields":[{"Env":"DASHBOARD_HOST","EnvType":"string","EnvValue":"localhost","EnvDescription":"Dashboard micro-service URL","Example":"","Deprecated":"false"},{"Env":"DASHBOARD_NAMESPACE","EnvType":"string","EnvValue":"devtroncd","EnvDescription":"Dashboard micro-service namespace","Example":"","Deprecated":"false"},{"Env":"DASHBOARD_PORT","EnvType":"string","EnvValue":"3000","EnvDescription":"Port for dashboard micro-service","Example":"","Deprecated":"false"},{"Env":"DEX_HOST","EnvType":"string","EnvValue":"http://localhost","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEX_PORT","EnvType":"string","EnvValue":"5556","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"GIT_SENSOR_PROTOCOL","EnvType":"string","EnvValue":"REST","EnvDescription":"Protocol to connect with git-sensor micro-service","Example":"","Deprecated":"false"},{"Env":"GIT_SENSOR_TIMEOUT","EnvType":"int","EnvValue":"0","EnvDescription":"Timeout for getting response from the git-sensor","Example":"","Deprecated":"false"},{"Env":"GIT_SENSOR_URL","EnvType":"string","EnvValue":"127.0.0.1:7070","EnvDescription":"git-sensor micro-service url ","Example":"","Deprecated":"false"},{"Env":"HELM_CLIENT_URL","EnvType":"string","EnvValue":"127.0.0.1:50051","EnvDescription":"Kubelink micro-service url ","Example":"","Deprecated":"false"}]},{"Category":"POSTGRES","Fields":[{"Env":"APP","EnvType":"string","EnvValue":"orchestrator","EnvDescription":"Application name","Example":"","Deprecated":"false"},{"Env":"CASBIN_DATABASE","EnvType":"string","EnvValue":"casbin","EnvDescription":"Database for casbin","Example":"","Deprecated":"false"},{"Env":"PG_ADDR","EnvType":"string","EnvValue":"127.0.0.1","EnvDescription":"address of postgres service","Example":"postgresql-postgresql.devtroncd","Deprecated":"false"},{"Env":"PG_DATABASE","EnvType":"string","EnvValue":"orchestrator","EnvDescription":"postgres database to be made connection with","Example":"orchestrator, casbin, git_sensor, lens","Deprecated":"false"},{"Env":"PG_PASSWORD","EnvType":"string","EnvValue":"{password}","EnvDescription":"password for postgres, associated with PG_USER","Example":"confidential ;)","Deprecated":"false"},{"Env":"PG_PORT","EnvType":"string","EnvValue":"5432","EnvDescription":"port of postgresql service","Example":"5432","Deprecated":"false"},{"Env":"PG_READ_TIMEOUT","EnvType":"int64","EnvValue":"30","EnvDescription":"Time out for read operation in postgres","Example":"","Deprecated":"false"},{"Env":"PG_USER","EnvType":"string","EnvValue":"postgres","EnvDescription":"user for postgres","Example":"postgres","Deprecated":"false"},{"Env":"PG_WRITE_TIMEOUT","EnvType":"int64","EnvValue":"30","EnvDescription":"Time out for write operation in postgres","Example":"","Deprecated":"false"}]},{"Category":"RBAC","Fields":[{"Env":"ENFORCER_CACHE","EnvType":"bool","EnvValue":"false","EnvDescription":"To Enable enforcer cache.","Example":"","Deprecated":"false"},{"Env":"ENFORCER_CACHE_EXPIRATION_IN_SEC","EnvType":"int","EnvValue":"86400","EnvDescription":"Expiration time (in seconds) for enforcer cache. ","Example":"","Deprecated":"false"},{"Env":"ENFORCER_MAX_BATCH_SIZE","EnvType":"int","EnvValue":"1","EnvDescription":"Maximum batch size for the enforcer.","Example":"","Deprecated":"false"},{"Env":"USE_CASBIN_V2","EnvType":"bool","EnvValue":"true","EnvDescription":"To enable casbin V2 API","Example":"","Deprecated":"false"}]}] \ No newline at end of file diff --git a/env_gen.md b/env_gen.md index dcffdf8783..0f6f039a1c 100644 --- a/env_gen.md +++ b/env_gen.md @@ -28,7 +28,7 @@ ## CI_RUNNER Related Environment Variables | Key | Type | Default Value | Description | Example | Deprecated | |-------|----------|-------------------|-------------------|-----------------------|------------------| - | AZURE_ACCOUNT_KEY | string | | If blob storage is bieng used of azure then pass the secret key to access the bucket | | false | + | AZURE_ACCOUNT_KEY | string | | If blob storage is being used of azure then pass the secret key to access the bucket | | false | | AZURE_ACCOUNT_NAME | string | | Account name for azure blob storage | | false | | AZURE_BLOB_CONTAINER_CI_CACHE | string | | Cache bucket name for azure blob storage | | false | | AZURE_BLOB_CONTAINER_CI_LOG | string | | Log bucket for azure blob storage | | false | diff --git a/pkg/appStore/bean/bean.go b/pkg/appStore/bean/bean.go index 6644ef5bc1..d5967d892c 100644 --- a/pkg/appStore/bean/bean.go +++ b/pkg/appStore/bean/bean.go @@ -433,6 +433,7 @@ type ChartRepoSearch struct { ChartId int `json:"chartId"` ChartName string `json:"chartName"` ChartRepoId int `json:"chartRepoId"` + DockerArtifactStoreId string `json:"DockerArtifactStoreId"` ChartRepoName string `json:"chartRepoName"` Version string `json:"version"` Deprecated bool `json:"deprecated"` diff --git a/pkg/appStore/discover/repository/AppStoreApplicationVersionRepository.go b/pkg/appStore/discover/repository/AppStoreApplicationVersionRepository.go index 19a34631c6..c68d67c1cf 100644 --- a/pkg/appStore/discover/repository/AppStoreApplicationVersionRepository.go +++ b/pkg/appStore/discover/repository/AppStoreApplicationVersionRepository.go @@ -273,7 +273,7 @@ func (impl *AppStoreApplicationVersionRepositoryImpl) SearchAppStoreChartByName( var chartRepos []*appStoreBean.ChartRepoSearch //for chart repos, created (derived through index.yaml) column of app_store_application_version is used for finding latest version and for oci repo id is used (because created is null) queryTemp := `select asv.id as app_store_application_version_id, asv.version, asv.deprecated, aps.id as chart_id, - aps.name as chart_name, chr.id as chart_repo_id, chr.name as chart_repo_name + aps.name as chart_name, chr.id as chart_repo_id, chr.name as chart_repo_name , das.id as docker_artifact_store_id from app_store_application_version asv inner join app_store aps on asv.app_store_id = aps.id left join chart_repo chr on aps.chart_repo_id = chr.id diff --git a/pkg/appStore/discover/service/AppStoreService.go b/pkg/appStore/discover/service/AppStoreService.go index 9eb0e06018..7430cfdf15 100644 --- a/pkg/appStore/discover/service/AppStoreService.go +++ b/pkg/appStore/discover/service/AppStoreService.go @@ -130,5 +130,10 @@ func (impl *AppStoreServiceImpl) SearchAppStoreChartByName(chartName string) ([] if err != nil && !util.IsErrNoRows(err) { return nil, err } + for _, appStore := range appStoreApplications { + if len(appStore.ChartRepoName) == 0 && len(appStore.DockerArtifactStoreId) != 0 { + appStore.ChartRepoName = appStore.DockerArtifactStoreId + } + } return appStoreApplications, nil } diff --git a/pkg/appStore/installedApp/service/AppStoreDeploymentService.go b/pkg/appStore/installedApp/service/AppStoreDeploymentService.go index 210d5739a9..7a9fd2e40b 100644 --- a/pkg/appStore/installedApp/service/AppStoreDeploymentService.go +++ b/pkg/appStore/installedApp/service/AppStoreDeploymentService.go @@ -29,6 +29,7 @@ import ( openapi2 "github.com/devtron-labs/devtron/api/openapi/openapiClient" "github.com/devtron-labs/devtron/client/argocdServer" "github.com/devtron-labs/devtron/internal/sql/repository/app" + repository2 "github.com/devtron-labs/devtron/internal/sql/repository/dockerRegistry" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/timelineStatus" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow/cdWorkflow" "github.com/devtron-labs/devtron/internal/util" @@ -90,6 +91,7 @@ type AppStoreDeploymentServiceImpl struct { deletePostProcessor DeletePostProcessor appStoreValidator AppStoreValidator deploymentConfigService common.DeploymentConfigService + OCIRegistryConfigRepository repository2.OCIRegistryConfigRepository } func NewAppStoreDeploymentServiceImpl(logger *zap.SugaredLogger, @@ -109,7 +111,7 @@ func NewAppStoreDeploymentServiceImpl(logger *zap.SugaredLogger, gitOpsConfigReadService config.GitOpsConfigReadService, deletePostProcessor DeletePostProcessor, appStoreValidator AppStoreValidator, deploymentConfigService common.DeploymentConfigService, -) *AppStoreDeploymentServiceImpl { + OCIRegistryConfigRepository repository2.OCIRegistryConfigRepository) *AppStoreDeploymentServiceImpl { return &AppStoreDeploymentServiceImpl{ logger: logger, @@ -130,6 +132,7 @@ func NewAppStoreDeploymentServiceImpl(logger *zap.SugaredLogger, deletePostProcessor: deletePostProcessor, appStoreValidator: appStoreValidator, deploymentConfigService: deploymentConfigService, + OCIRegistryConfigRepository: OCIRegistryConfigRepository, } } @@ -951,7 +954,49 @@ func (impl *AppStoreDeploymentServiceImpl) linkHelmApplicationToChartStore(insta } // STEP-2 update APP with chart info - chartRepoInfo := appStoreAppVersion.AppStore.ChartRepo + //TODO: below code is duplicated + var IsOCIRepo bool + var registryCredential *bean4.RegistryCredential + var chartRepository *bean4.ChartRepository + dockerRegistryId := appStoreAppVersion.AppStore.DockerArtifactStoreId + if dockerRegistryId != "" { + ociRegistryConfigs, err := impl.OCIRegistryConfigRepository.FindByDockerRegistryId(dockerRegistryId) + if err != nil { + impl.logger.Errorw("error in fetching oci registry config", "err", err) + return nil, err + } + var ociRegistryConfig *repository2.OCIRegistryConfig + for _, config := range ociRegistryConfigs { + if config.RepositoryAction == repository2.STORAGE_ACTION_TYPE_PULL || config.RepositoryAction == repository2.STORAGE_ACTION_TYPE_PULL_AND_PUSH { + ociRegistryConfig = config + break + } + } + IsOCIRepo = true + registryCredential = &bean4.RegistryCredential{ + RegistryUrl: appStoreAppVersion.AppStore.DockerArtifactStore.RegistryURL, + Username: appStoreAppVersion.AppStore.DockerArtifactStore.Username, + Password: appStoreAppVersion.AppStore.DockerArtifactStore.Password, + AwsRegion: appStoreAppVersion.AppStore.DockerArtifactStore.AWSRegion, + AccessKey: appStoreAppVersion.AppStore.DockerArtifactStore.AWSAccessKeyId, + SecretKey: appStoreAppVersion.AppStore.DockerArtifactStore.AWSSecretAccessKey, + RegistryType: string(appStoreAppVersion.AppStore.DockerArtifactStore.RegistryType), + RepoName: appStoreAppVersion.AppStore.Name, + IsPublic: ociRegistryConfig.IsPublic, + Connection: appStoreAppVersion.AppStore.DockerArtifactStore.Connection, + RegistryName: appStoreAppVersion.AppStore.DockerArtifactStore.Id, + RegistryCertificate: appStoreAppVersion.AppStore.DockerArtifactStore.Cert, + } + } else { + chartRepository = &bean4.ChartRepository{ + Name: appStoreAppVersion.AppStore.ChartRepo.Name, + Url: appStoreAppVersion.AppStore.ChartRepo.Url, + Username: appStoreAppVersion.AppStore.ChartRepo.UserName, + Password: appStoreAppVersion.AppStore.ChartRepo.Password, + AllowInsecureConnection: appStoreAppVersion.AppStore.ChartRepo.AllowInsecureConnection, + } + } + updateReleaseRequest := &bean3.UpdateApplicationWithChartInfoRequestDto{ InstallReleaseRequest: &bean4.InstallReleaseRequest{ ValuesYaml: installAppVersionRequest.ValuesOverrideYaml, @@ -961,18 +1006,14 @@ func (impl *AppStoreDeploymentServiceImpl) linkHelmApplicationToChartStore(insta ReleaseNamespace: installAppVersionRequest.Namespace, ReleaseName: installAppVersionRequest.DisplayName, }, + RegistryCredential: registryCredential, + ChartRepository: chartRepository, + IsOCIRepo: IsOCIRepo, + InstallAppVersionHistoryId: 0, }, SourceAppType: bean3.SOURCE_HELM_APP, } - if chartRepoInfo != nil { - updateReleaseRequest.ChartRepository = &bean4.ChartRepository{ - Name: chartRepoInfo.Name, - Url: chartRepoInfo.Url, - Username: chartRepoInfo.UserName, - Password: chartRepoInfo.Password, - AllowInsecureConnection: chartRepoInfo.AllowInsecureConnection, - } - } + res, err := impl.helmAppService.UpdateApplicationWithChartInfo(ctx, installAppVersionRequest.ClusterId, updateReleaseRequest) if err != nil { return nil, err diff --git a/scripts/sql/32703400_docker_artifact_credentials_type.down.sql b/scripts/sql/32703400_docker_artifact_credentials_type.down.sql new file mode 100644 index 0000000000..65d77135df --- /dev/null +++ b/scripts/sql/32703400_docker_artifact_credentials_type.down.sql @@ -0,0 +1 @@ +ALTER TABLE docker_artifact_store DROP COLUMN IF EXISTS credentials_type; \ No newline at end of file diff --git a/scripts/sql/32703400_docker_artifact_credentials_type.up.sql b/scripts/sql/32703400_docker_artifact_credentials_type.up.sql new file mode 100644 index 0000000000..2f918878f1 --- /dev/null +++ b/scripts/sql/32703400_docker_artifact_credentials_type.up.sql @@ -0,0 +1 @@ +ALTER TABLE docker_artifact_store ADD COLUMN IF NOT EXISTS credentials_type VARCHAR(124); \ No newline at end of file diff --git a/wire_gen.go b/wire_gen.go index f4510a2d24..78a257534d 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -842,7 +842,7 @@ func InitializeApp() (*App, error) { appStoreDeploymentDBServiceImpl := service5.NewAppStoreDeploymentDBServiceImpl(sugaredLogger, installedAppRepositoryImpl, appStoreApplicationVersionRepositoryImpl, appRepositoryImpl, environmentServiceImpl, installedAppVersionHistoryRepositoryImpl, environmentVariables, gitOpsConfigReadServiceImpl, deploymentTypeOverrideServiceImpl, fullModeDeploymentServiceImpl, appStoreValidatorImpl, installedAppDBServiceImpl, deploymentConfigServiceImpl, clusterReadServiceImpl) eaModeDeploymentServiceImpl := deployment2.NewEAModeDeploymentServiceImpl(sugaredLogger, helmAppServiceImpl, appStoreApplicationVersionRepositoryImpl, helmAppClientImpl, installedAppRepositoryImpl, ociRegistryConfigRepositoryImpl, appStoreDeploymentCommonServiceImpl, helmAppReadServiceImpl) deletePostProcessorImpl := service5.NewDeletePostProcessorImpl(sugaredLogger) - appStoreDeploymentServiceImpl := service5.NewAppStoreDeploymentServiceImpl(sugaredLogger, installedAppRepositoryImpl, installedAppDBServiceImpl, appStoreDeploymentDBServiceImpl, chartGroupDeploymentRepositoryImpl, appStoreApplicationVersionRepositoryImpl, appRepositoryImpl, eaModeDeploymentServiceImpl, fullModeDeploymentServiceImpl, environmentServiceImpl, helmAppServiceImpl, installedAppVersionHistoryRepositoryImpl, environmentVariables, acdConfig, gitOpsConfigReadServiceImpl, deletePostProcessorImpl, appStoreValidatorImpl, deploymentConfigServiceImpl) + appStoreDeploymentServiceImpl := service5.NewAppStoreDeploymentServiceImpl(sugaredLogger, installedAppRepositoryImpl, installedAppDBServiceImpl, appStoreDeploymentDBServiceImpl, chartGroupDeploymentRepositoryImpl, appStoreApplicationVersionRepositoryImpl, appRepositoryImpl, eaModeDeploymentServiceImpl, fullModeDeploymentServiceImpl, environmentServiceImpl, helmAppServiceImpl, installedAppVersionHistoryRepositoryImpl, environmentVariables, acdConfig, gitOpsConfigReadServiceImpl, deletePostProcessorImpl, appStoreValidatorImpl, deploymentConfigServiceImpl, ociRegistryConfigRepositoryImpl) appStoreAppsEventPublishServiceImpl := out.NewAppStoreAppsEventPublishServiceImpl(sugaredLogger, pubSubClientServiceImpl) chartGroupServiceImpl, err := chartGroup.NewChartGroupServiceImpl(sugaredLogger, chartGroupEntriesRepositoryImpl, chartGroupReposotoryImpl, chartGroupDeploymentRepositoryImpl, installedAppRepositoryImpl, appStoreVersionValuesRepositoryImpl, appStoreRepositoryImpl, userAuthServiceImpl, appStoreApplicationVersionRepositoryImpl, environmentServiceImpl, teamRepositoryImpl, clusterInstalledAppsRepositoryImpl, appStoreValuesServiceImpl, appStoreDeploymentServiceImpl, appStoreDeploymentDBServiceImpl, pipelineStatusTimelineServiceImpl, acdConfig, fullModeDeploymentServiceImpl, gitOperationServiceImpl, installedAppDBExtendedServiceImpl, appStoreAppsEventPublishServiceImpl, teamReadServiceImpl) if err != nil { From 7e2cda099819d53c0145e9a17a9de86248e99c33 Mon Sep 17 00:00:00 2001 From: Vikram Singh Date: Mon, 21 Apr 2025 19:03:21 +0530 Subject: [PATCH 30/34] vendor update in develop --- go.mod | 4 ++-- go.sum | 8 ++++---- .../devtron-labs/common-lib/constants/constants.go | 7 +++++++ vendor/modules.txt | 8 ++++---- 4 files changed, 17 insertions(+), 10 deletions(-) diff --git a/go.mod b/go.mod index eeb86de322..7b92d82d38 100644 --- a/go.mod +++ b/go.mod @@ -307,8 +307,8 @@ require ( replace ( github.com/argoproj/argo-workflows/v3 v3.5.13 => github.com/devtron-labs/argo-workflows/v3 v3.5.13 - github.com/devtron-labs/authenticator => github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250414075707-6076f6f97fb9 - github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250414075707-6076f6f97fb9 + github.com/devtron-labs/authenticator => github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250421131910-ad3aa9bb920e + github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250421131910-ad3aa9bb920e github.com/go-check/check => github.com/go-check/check v0.0.0-20180628173108-788fd7840127 github.com/googleapis/gnostic => github.com/googleapis/gnostic v0.5.5 k8s.io/api => k8s.io/api v0.29.7 diff --git a/go.sum b/go.sum index 1555df3ec7..328d6622fa 100644 --- a/go.sum +++ b/go.sum @@ -829,10 +829,10 @@ github.com/denisenkom/go-mssqldb v0.0.0-20200428022330-06a60b6afbbc h1:VRRKCwnzq github.com/denisenkom/go-mssqldb v0.0.0-20200428022330-06a60b6afbbc/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= github.com/devtron-labs/argo-workflows/v3 v3.5.13 h1:3pINq0gXOSeTw2z/vYe+j80lRpSN5Rp/8mfQORh8SmU= github.com/devtron-labs/argo-workflows/v3 v3.5.13/go.mod h1:/vqxcovDPT4zqr4DjR5v7CF8ggpY1l3TSa2CIG3jmjA= -github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250414075707-6076f6f97fb9 h1:z72S6KALLrQ2VmW785FuPZpYJTFSzq1QflZJcQv8/O0= -github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250414075707-6076f6f97fb9/go.mod h1:5lv4Wfj5ERhhvDGXe2IeES6qxjvUVCcohaRwKnWBMNo= -github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250414075707-6076f6f97fb9 h1:o/wKBkZ6aGrFhSsDGSyQVTD3uurGH3db4zdWjRVLkI8= -github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250414075707-6076f6f97fb9/go.mod h1:ceFKgQ2qm40PR95g5Xp2EClq7nDBKFTcglJ0JdsgClA= +github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250421131910-ad3aa9bb920e h1:6/RNcoG39xH35ZHb024ERluk2HH3sf+uNWDR5AHb4xs= +github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250421131910-ad3aa9bb920e/go.mod h1:5lv4Wfj5ERhhvDGXe2IeES6qxjvUVCcohaRwKnWBMNo= +github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250421131910-ad3aa9bb920e h1:YkUbwbsa5phIo0ikGvkCOHigF6IQZPfUTE9WcaF4vaM= +github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250421131910-ad3aa9bb920e/go.mod h1:ceFKgQ2qm40PR95g5Xp2EClq7nDBKFTcglJ0JdsgClA= github.com/devtron-labs/go-bitbucket v0.9.60-beta h1:VEx1jvDgdtDPS6A1uUFoaEi0l1/oLhbr+90xOwr6sDU= github.com/devtron-labs/go-bitbucket v0.9.60-beta/go.mod h1:GnuiCesvh8xyHeMCb+twm8lBR/kQzJYSKL28ZfObp1Y= github.com/devtron-labs/protos v0.0.3-0.20250323220609-ecf8a0f7305e h1:U6UdYbW8a7xn5IzFPd8cywjVVPfutGJCudjePAfL/Hs= diff --git a/vendor/github.com/devtron-labs/common-lib/constants/constants.go b/vendor/github.com/devtron-labs/common-lib/constants/constants.go index 26f918e230..f327d712ba 100644 --- a/vendor/github.com/devtron-labs/common-lib/constants/constants.go +++ b/vendor/github.com/devtron-labs/common-lib/constants/constants.go @@ -73,3 +73,10 @@ const ( SourceSubTypeCi SourceSubType = 1 // relevant for ci code(2,1) or ci built image(1,1) SourceSubTypeManifest SourceSubType = 2 // relevant for devtron app deployment manifest/helm app manifest(2,2) or images retrieved from manifest(1,2)) ) + +type CredentialsType string + +const ( + CredentialsTypeAnonymous CredentialsType = "anonymous" + CredentialsTypeUsernamePassword CredentialsType = "username_password" +) diff --git a/vendor/modules.txt b/vendor/modules.txt index 72d3319694..4a1cd3503d 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -472,7 +472,7 @@ github.com/davecgh/go-spew/spew # github.com/deckarep/golang-set v1.8.0 ## explicit; go 1.17 github.com/deckarep/golang-set -# github.com/devtron-labs/authenticator v0.4.35-0.20240809073103-6e11da8083f8 => github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250414075707-6076f6f97fb9 +# github.com/devtron-labs/authenticator v0.4.35-0.20240809073103-6e11da8083f8 => github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250421131910-ad3aa9bb920e ## explicit; go 1.21 github.com/devtron-labs/authenticator/apiToken github.com/devtron-labs/authenticator/client @@ -480,7 +480,7 @@ github.com/devtron-labs/authenticator/jwt github.com/devtron-labs/authenticator/middleware github.com/devtron-labs/authenticator/oidc github.com/devtron-labs/authenticator/password -# github.com/devtron-labs/common-lib v0.18.1-0.20241001061923-eda545dc839e => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250414075707-6076f6f97fb9 +# github.com/devtron-labs/common-lib v0.18.1-0.20241001061923-eda545dc839e => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250421131910-ad3aa9bb920e ## explicit; go 1.21 github.com/devtron-labs/common-lib/async github.com/devtron-labs/common-lib/blob-storage @@ -2350,8 +2350,8 @@ xorm.io/xorm/log xorm.io/xorm/names xorm.io/xorm/schemas xorm.io/xorm/tags -# github.com/devtron-labs/authenticator => github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250414075707-6076f6f97fb9 -# github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250414075707-6076f6f97fb9 +# github.com/devtron-labs/authenticator => github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250421131910-ad3aa9bb920e +# github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250421131910-ad3aa9bb920e # github.com/go-check/check => github.com/go-check/check v0.0.0-20180628173108-788fd7840127 # github.com/googleapis/gnostic => github.com/googleapis/gnostic v0.5.5 # k8s.io/api => k8s.io/api v0.29.7 From fc44e3c246c0cc5ace1e88ae72095f2e82d20eca Mon Sep 17 00:00:00 2001 From: kartik-579 <84493919+kartik-579@users.noreply.github.com> Date: Tue, 22 Apr 2025 14:40:39 +0530 Subject: [PATCH 31/34] refactor: Refactor ci and multi cluster cicd (#6505) * changes around ciHandler * common constants create * ci service and trigger service creation * wire file missed * trigger service moved completely * feat: added ucid validation for ci/cd workflow status update event * telemetry svc refactoring * updated wire gen files * fix: CheckIfReTriggerRequired condition and CdHandlerImpl.UpdateWorkflow method * fix: workflow message update * updated import alias * fix: ci/ cd status update inconsistencies * fix: checkForOptOut method error handling * fix: panic handling * fix: updated DevtronAdministratorInstanceLabelKey for system executor * fix: reverted base64 encoding * fix: ci/ cd workflow msg update * fix: ci/ cd workflow msg update * matrics for retrigger failed event * updated logger * ci service ent oss completed * added app to subdirectory for categorising entity * cd handler and ci handle extracted, executor workflow service executed * workflow service ent change * workflow service ent change * Redundant import removed * rename pkg/ * reverted pkg categorisation * chore: posthug refactoring and const renamed * chore: kubewatch const renamed * chore: removed posthug module * chore: ci/cd workflow message datatype updated * wip * file renaming * deployStage unstacked file * file renamed * updated BuildxCacheFlags description --------- Co-authored-by: Ash-exp --- App.go | 6 +- Wire.go | 15 +- api/argoApplication/wire_argoApplication.go | 12 +- api/restHandler/CoreAppRestHandler.go | 3 +- .../configure/BuildPipelineRestHandler.go | 20 +- .../DeploymentPipelineRestHandler.go | 6 +- .../configure/PipelineConfigRestHandler.go | 8 + .../trigger/PipelineTriggerRestHandler.go | 8 +- client/argocdServer/config/Config.go | 2 +- client/cron/CiStatusUpdateCron.go | 11 +- client/cron/CiTriggerCron.go | 15 +- client/events/EventBuilder.go | 13 +- client/events/EventClient.go | 60 +- client/telemetry/TelemetryEventClient.go | 211 +- .../telemetry/TelemetryEventClientExtended.go | 77 +- client/telemetry/bean.go | 139 ++ cmd/external-app/externalApp.go | 5 +- cmd/external-app/wire.go | 5 +- cmd/external-app/wire_gen.go | 11 +- go.mod | 6 +- go.sum | 12 +- internal/middleware/instrument.go | 5 + .../CIPipelineRepository_ent.go | 8 + .../pipelineConfig/CdWorfkflowRepository.go | 31 +- .../pipelineConfig/CiPipelineRepository.go | 8 +- .../pipelineConfig/bean/cdWorkflow/bean.go | 54 - pkg/app/AppListingService.go | 4 +- .../deployment/InstalledAppArgoCdService.go | 4 +- pkg/bean/app.go | 101 +- pkg/build/git/gitWebhook/GitWebhookService.go | 11 +- .../pipeline/bean/CIPipelineMaterialBean.go | 30 + pkg/build/pipeline/bean/CiBuildConfig.go | 26 - .../pipeline/bean/common/CommonBuildBean.go | 40 + pkg/build/trigger/HandlerService.go | 1957 +++++++++++++++++ pkg/build/trigger/HandlerService_ent.go | 104 + pkg/build/trigger/wire_trigger.go | 10 + pkg/build/wire_build.go | 2 + pkg/bulkAction/service/BulkUpdateService.go | 8 +- pkg/chartRepo/ChartRepositoryService.go | 2 +- pkg/cluster/ClusterService.go | 113 +- pkg/cluster/bean/bean.go | 7 +- pkg/cluster/clusterUtil.go | 4 +- pkg/delete/DeleteService.go | 2 +- .../deployedApp/DeployedAppService.go | 8 +- .../trigger/devtronApps/HandlerService.go | 345 +++ ...erService.go => deployStageHandlerCode.go} | 378 +--- ..._ent1.go => deployStageHandlerCode_ent.go} | 38 +- .../trigger/devtronApps/feasibilityChecker.go | 2 +- ...ggerService.go => postStageHandlerCode.go} | 6 +- ...ice_ent.go => postStageHandlerCode_ent.go} | 4 +- .../devtronApps/prePostWfAndLogsCode.go | 337 +++ ...iggerService.go => preStageHandlerCode.go} | 41 +- ...vice_ent.go => preStageHandlerCode_ent.go} | 12 +- .../wire_devtronAppsDeployTrigger.go | 4 +- pkg/eventProcessor/bean/workflowEventBean.go | 12 + .../in/CDPipelineEventProcessorService.go | 8 +- .../in/WorkflowEventProcessorService.go | 98 +- pkg/{pipeline => executor}/WorkflowService.go | 52 +- .../WorkflowServiceIT_test.go | 7 +- pkg/executor/WorkflowService_ent.go | 30 + .../WorkflowService_test.go | 2 +- pkg/executor/wire_executor.go | 8 + pkg/gitops/GitOpsConfigService.go | 4 +- pkg/pipeline/BlobStorageConfigService.go | 7 +- pkg/pipeline/BuildPipelineConfigService.go | 37 +- pkg/pipeline/BuildPipelineSwitchService.go | 20 +- pkg/pipeline/CIHandler_ent.go | 21 + pkg/pipeline/CdHandler.go | 402 +--- pkg/pipeline/CiCdPipelineOrchestrator.go | 11 +- pkg/pipeline/CiHandler.go | 1134 +--------- pkg/pipeline/CiLogService.go | 4 +- pkg/pipeline/CiService.go | 1100 +-------- pkg/pipeline/CiService_ent.go | 16 - pkg/pipeline/PipelineStageService.go | 5 + pkg/pipeline/adapter/adapter.go | 7 +- pkg/pipeline/bean/CdHandlerBean.go | 57 +- pkg/pipeline/bean/WorkflowTemplate.go | 27 +- pkg/pipeline/constants/constants.go | 32 - pkg/pipeline/executors/WorkflowUtils.go | 17 +- pkg/pipeline/pipelineStageVariableParser.go | 6 +- pkg/pipeline/types/CiCdConfig.go | 24 +- pkg/pipeline/types/Workflow.go | 63 +- pkg/pipeline/types/Workflow_ent.go | 2 +- pkg/ucid/bean.go | 24 + pkg/ucid/ucid.go | 96 + pkg/ucid/wire_ucid.go | 8 + pkg/workflow/dag/WorkflowDagExecutor.go | 177 +- .../sql/32803400_workflow_message.down.sql | 1 + scripts/sql/32803400_workflow_message.up.sql | 11 + util/helper.go | 10 +- .../devtron-labs/common-lib/informer/bean.go | 40 + .../common-lib}/telemetry/PosthogClient.go | 33 +- .../devtron-labs/common-lib/telemetry/bean.go | 34 + .../common-lib/utils/http/HttpUtil.go | 12 +- vendor/github.com/golang-jwt/jwt/v4/parser.go | 36 +- vendor/modules.txt | 12 +- wire_gen.go | 146 +- 97 files changed, 4434 insertions(+), 3770 deletions(-) create mode 100644 client/telemetry/bean.go create mode 100644 internal/sql/repository/pipelineConfig/CIPipelineRepository_ent.go delete mode 100644 internal/sql/repository/pipelineConfig/bean/cdWorkflow/bean.go create mode 100644 pkg/build/pipeline/bean/CIPipelineMaterialBean.go create mode 100644 pkg/build/pipeline/bean/common/CommonBuildBean.go create mode 100644 pkg/build/trigger/HandlerService.go create mode 100644 pkg/build/trigger/HandlerService_ent.go create mode 100644 pkg/build/trigger/wire_trigger.go create mode 100644 pkg/deployment/trigger/devtronApps/HandlerService.go rename pkg/deployment/trigger/devtronApps/{TriggerService.go => deployStageHandlerCode.go} (77%) rename pkg/deployment/trigger/devtronApps/{TriggerService_ent1.go => deployStageHandlerCode_ent.go} (67%) rename pkg/deployment/trigger/devtronApps/{PostStageTriggerService.go => postStageHandlerCode.go} (97%) rename pkg/deployment/trigger/devtronApps/{PostStageTriggerService_ent.go => postStageHandlerCode_ent.go} (92%) create mode 100644 pkg/deployment/trigger/devtronApps/prePostWfAndLogsCode.go rename pkg/deployment/trigger/devtronApps/{PreStageTriggerService.go => preStageHandlerCode.go} (97%) rename pkg/deployment/trigger/devtronApps/{PreStageTriggerService_ent.go => preStageHandlerCode_ent.go} (82%) rename pkg/{pipeline => executor}/WorkflowService.go (93%) rename pkg/{pipeline => executor}/WorkflowServiceIT_test.go (99%) create mode 100644 pkg/executor/WorkflowService_ent.go rename pkg/{pipeline => executor}/WorkflowService_test.go (99%) create mode 100644 pkg/executor/wire_executor.go create mode 100644 pkg/pipeline/CIHandler_ent.go delete mode 100644 pkg/pipeline/CiService_ent.go create mode 100644 pkg/ucid/bean.go create mode 100644 pkg/ucid/ucid.go create mode 100644 pkg/ucid/wire_ucid.go create mode 100644 scripts/sql/32803400_workflow_message.down.sql create mode 100644 scripts/sql/32803400_workflow_message.up.sql create mode 100644 vendor/github.com/devtron-labs/common-lib/informer/bean.go rename {client => vendor/github.com/devtron-labs/common-lib}/telemetry/PosthogClient.go (70%) create mode 100644 vendor/github.com/devtron-labs/common-lib/telemetry/bean.go diff --git a/App.go b/App.go index 08b240a8b7..649c319298 100644 --- a/App.go +++ b/App.go @@ -23,6 +23,7 @@ import ( "fmt" "github.com/devtron-labs/common-lib/middlewares" pubsub "github.com/devtron-labs/common-lib/pubsub-lib" + posthogTelemetry "github.com/devtron-labs/common-lib/telemetry" "github.com/devtron-labs/devtron/pkg/eventProcessor" "github.com/devtron-labs/devtron/pkg/eventProcessor/in" "log" @@ -31,7 +32,6 @@ import ( "time" "github.com/devtron-labs/devtron/api/util" - "github.com/devtron-labs/devtron/client/telemetry" "github.com/devtron-labs/devtron/otel" "github.com/devtron-labs/devtron/pkg/auth/user" @@ -55,7 +55,7 @@ type App struct { EnforcerV2 *casbinv2.SyncedEnforcer server *http.Server db *pg.DB - posthogClient *telemetry.PosthogClient + posthogClient *posthogTelemetry.PosthogClient // eventProcessor.CentralEventProcessor is used to register event processors centralEventProcessor *eventProcessor.CentralEventProcessor // do not remove this. // used for local dev only @@ -73,7 +73,7 @@ func NewApp(router *router.MuxRouter, enforcer *casbin.SyncedEnforcer, db *pg.DB, sessionManager2 *authMiddleware.SessionManager, - posthogClient *telemetry.PosthogClient, + posthogClient *posthogTelemetry.PosthogClient, loggingMiddleware util.LoggingMiddleware, centralEventProcessor *eventProcessor.CentralEventProcessor, pubSubClient *pubsub.PubSubClientServiceImpl, diff --git a/Wire.go b/Wire.go index ebfb6a943d..f239eb7d06 100644 --- a/Wire.go +++ b/Wire.go @@ -22,7 +22,8 @@ package main import ( "github.com/devtron-labs/authenticator/middleware" cloudProviderIdentifier "github.com/devtron-labs/common-lib/cloud-provider-identifier" - pubsub1 "github.com/devtron-labs/common-lib/pubsub-lib" + pubSub "github.com/devtron-labs/common-lib/pubsub-lib" + posthogTelemetry "github.com/devtron-labs/common-lib/telemetry" util4 "github.com/devtron-labs/common-lib/utils/k8s" "github.com/devtron-labs/devtron/api/apiToken" appStoreRestHandler "github.com/devtron-labs/devtron/api/appStore" @@ -144,6 +145,7 @@ import ( "github.com/devtron-labs/devtron/pkg/deploymentGroup" "github.com/devtron-labs/devtron/pkg/dockerRegistry" "github.com/devtron-labs/devtron/pkg/eventProcessor" + "github.com/devtron-labs/devtron/pkg/executor" "github.com/devtron-labs/devtron/pkg/generateManifest" "github.com/devtron-labs/devtron/pkg/gitops" "github.com/devtron-labs/devtron/pkg/imageDigestPolicy" @@ -165,6 +167,7 @@ import ( resourceGroup2 "github.com/devtron-labs/devtron/pkg/resourceGroup" "github.com/devtron-labs/devtron/pkg/resourceQualifiers" "github.com/devtron-labs/devtron/pkg/sql" + "github.com/devtron-labs/devtron/pkg/ucid" util3 "github.com/devtron-labs/devtron/pkg/util" "github.com/devtron-labs/devtron/pkg/variables" "github.com/devtron-labs/devtron/pkg/variables/parsers" @@ -219,7 +222,7 @@ func InitializeApp() (*App, error) { userResource.UserResourceWireSet, policyGovernance.PolicyGovernanceWireSet, resourceScan.ScanningResultWireSet, - + executor.ExecutorWireSet, // -------wireset end ---------- // ------- gitSensor.GetConfig, @@ -478,9 +481,6 @@ func InitializeApp() (*App, error) { util.IntValidator, types.GetCiCdConfig, - pipeline.NewWorkflowServiceImpl, - wire.Bind(new(pipeline.WorkflowService), new(*pipeline.WorkflowServiceImpl)), - pipeline.NewCiServiceImpl, wire.Bind(new(pipeline.CiService), new(*pipeline.CiServiceImpl)), @@ -501,7 +501,7 @@ func InitializeApp() (*App, error) { pipeline.NewCiLogServiceImpl, wire.Bind(new(pipeline.CiLogService), new(*pipeline.CiLogServiceImpl)), - pubsub1.NewPubSubClientServiceImpl, + pubSub.NewPubSubClientServiceImpl, rbac.NewEnforcerUtilImpl, wire.Bind(new(rbac.EnforcerUtil), new(*rbac.EnforcerUtilImpl)), @@ -696,7 +696,8 @@ func InitializeApp() (*App, error) { wire.Bind(new(router.TelemetryRouter), new(*router.TelemetryRouterImpl)), restHandler.NewTelemetryRestHandlerImpl, wire.Bind(new(restHandler.TelemetryRestHandler), new(*restHandler.TelemetryRestHandlerImpl)), - telemetry.NewPosthogClient, + posthogTelemetry.NewPosthogClient, + ucid.WireSet, cloudProviderIdentifier.NewProviderIdentifierServiceImpl, wire.Bind(new(cloudProviderIdentifier.ProviderIdentifierService), new(*cloudProviderIdentifier.ProviderIdentifierServiceImpl)), diff --git a/api/argoApplication/wire_argoApplication.go b/api/argoApplication/wire_argoApplication.go index e3c7381725..45ef8139db 100644 --- a/api/argoApplication/wire_argoApplication.go +++ b/api/argoApplication/wire_argoApplication.go @@ -17,7 +17,7 @@ package argoApplication import ( - "github.com/devtron-labs/devtron/pkg/argoApplication" + argoApplication3 "github.com/devtron-labs/devtron/pkg/argoApplication" "github.com/devtron-labs/devtron/pkg/argoApplication/read" "github.com/devtron-labs/devtron/pkg/argoApplication/read/config" "github.com/google/wire" @@ -30,9 +30,9 @@ var ArgoApplicationWireSetFull = wire.NewSet( config.NewArgoApplicationConfigServiceImpl, wire.Bind(new(config.ArgoApplicationConfigService), new(*config.ArgoApplicationConfigServiceImpl)), - argoApplication.NewArgoApplicationServiceImpl, - argoApplication.NewArgoApplicationServiceExtendedServiceImpl, - wire.Bind(new(argoApplication.ArgoApplicationService), new(*argoApplication.ArgoApplicationServiceExtendedImpl)), + argoApplication3.NewArgoApplicationServiceImpl, + argoApplication3.NewArgoApplicationServiceExtendedServiceImpl, + wire.Bind(new(argoApplication3.ArgoApplicationService), new(*argoApplication3.ArgoApplicationServiceExtendedImpl)), NewArgoApplicationRestHandlerImpl, wire.Bind(new(ArgoApplicationRestHandler), new(*ArgoApplicationRestHandlerImpl)), @@ -48,8 +48,8 @@ var ArgoApplicationWireSetEA = wire.NewSet( config.NewArgoApplicationConfigServiceImpl, wire.Bind(new(config.ArgoApplicationConfigService), new(*config.ArgoApplicationConfigServiceImpl)), - argoApplication.NewArgoApplicationServiceImpl, - wire.Bind(new(argoApplication.ArgoApplicationService), new(*argoApplication.ArgoApplicationServiceImpl)), + argoApplication3.NewArgoApplicationServiceImpl, + wire.Bind(new(argoApplication3.ArgoApplicationService), new(*argoApplication3.ArgoApplicationServiceImpl)), NewArgoApplicationRestHandlerImpl, wire.Bind(new(ArgoApplicationRestHandler), new(*ArgoApplicationRestHandlerImpl)), diff --git a/api/restHandler/CoreAppRestHandler.go b/api/restHandler/CoreAppRestHandler.go index eff6fba1c0..277405a08b 100644 --- a/api/restHandler/CoreAppRestHandler.go +++ b/api/restHandler/CoreAppRestHandler.go @@ -29,6 +29,7 @@ import ( "github.com/devtron-labs/devtron/pkg/build/git/gitProvider" "github.com/devtron-labs/devtron/pkg/build/git/gitProvider/read" pipelineBean "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" + common2 "github.com/devtron-labs/devtron/pkg/build/pipeline/bean/common" bean3 "github.com/devtron-labs/devtron/pkg/chart/bean" read5 "github.com/devtron-labs/devtron/pkg/chart/read" "github.com/devtron-labs/devtron/pkg/cluster/environment/repository" @@ -1688,7 +1689,7 @@ func (handler CoreAppRestHandlerImpl) createCiPipeline(appId int, userId int32, ParentCiPipeline: ciPipelineData.ParentCiPipeline, ParentAppId: ciPipelineData.ParentAppId, LinkedCount: ciPipelineData.LinkedCount, - PipelineType: pipelineBean.PipelineType(ciPipelineData.PipelineType), + PipelineType: common2.PipelineType(ciPipelineData.PipelineType), }, } diff --git a/api/restHandler/app/pipeline/configure/BuildPipelineRestHandler.go b/api/restHandler/app/pipeline/configure/BuildPipelineRestHandler.go index 0bb2a921b8..13b61730e3 100644 --- a/api/restHandler/app/pipeline/configure/BuildPipelineRestHandler.go +++ b/api/restHandler/app/pipeline/configure/BuildPipelineRestHandler.go @@ -25,6 +25,7 @@ import ( "github.com/devtron-labs/devtron/internal/sql/constants" "github.com/devtron-labs/devtron/pkg/build/artifacts/imageTagging" bean2 "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" + eventProcessorBean "github.com/devtron-labs/devtron/pkg/eventProcessor/bean" constants2 "github.com/devtron-labs/devtron/pkg/pipeline/constants" "github.com/devtron-labs/devtron/util/stringsUtil" "golang.org/x/exp/maps" @@ -37,7 +38,6 @@ import ( "github.com/devtron-labs/devtron/util/response/pagination" "github.com/gorilla/schema" - "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" "github.com/devtron-labs/devtron/api/restHandler/common" "github.com/devtron-labs/devtron/client/gitSensor" "github.com/devtron-labs/devtron/internal/sql/repository" @@ -701,8 +701,7 @@ func (handler *PipelineConfigRestHandlerImpl) TriggerCiPipeline(w http.ResponseW handler.Logger.Infow("request payload, TriggerCiPipeline", "payload", ciTriggerRequest) response := make(map[string]string) - resp, err := handler.ciHandler.HandleCIManual(ciTriggerRequest) - + resp, err := handler.ciHandlerService.HandleCIManual(ciTriggerRequest) if errors.Is(err, bean1.ErrImagePathInUse) { handler.Logger.Errorw("service err duplicate image tag, TriggerCiPipeline", "err", err, "payload", ciTriggerRequest) common.WriteJsonResp(w, err, err, http.StatusConflict) @@ -918,7 +917,8 @@ func (handler *PipelineConfigRestHandlerImpl) DownloadCiWorkflowArtifacts(w http return } - file, err := handler.ciHandler.DownloadCiWorkflowArtifacts(pipelineId, buildId) + file, err := handler.ciHandlerService.DownloadCiWorkflowArtifacts(pipelineId, buildId) + defer file.Close() if err != nil { handler.Logger.Errorw("service err, DownloadCiWorkflowArtifacts", "err", err, "pipelineId", pipelineId, "buildId", buildId) if util.IsErrNoRows(err) { @@ -974,8 +974,8 @@ func (handler *PipelineConfigRestHandlerImpl) GetHistoricBuildLogs(w http.Respon common.WriteJsonResp(w, nil, "Unauthorized User", http.StatusForbidden) return } - - resp, err := handler.ciHandler.GetHistoricBuildLogs(workflowId, nil) + // RBAC + resp, err := handler.ciHandlerService.GetHistoricBuildLogs(workflowId, nil) if err != nil { handler.Logger.Errorw("service err, GetHistoricBuildLogs", "err", err, "pipelineId", pipelineId, "workflowId", workflowId) common.WriteJsonResp(w, err, resp, http.StatusInternalServerError) @@ -1114,7 +1114,7 @@ func (handler *PipelineConfigRestHandlerImpl) GetBuildLogs(w http.ResponseWriter return } } - logsReader, cleanUp, err := handler.ciHandler.GetRunningWorkflowLogs(workflowId) + logsReader, cleanUp, err := handler.ciHandlerService.GetRunningWorkflowLogs(workflowId) if err != nil { handler.Logger.Errorw("service err, GetBuildLogs", "err", err, "pipelineId", pipelineId, "workflowId", workflowId, "lastEventId", lastEventId) common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) @@ -1446,7 +1446,7 @@ func (handler *PipelineConfigRestHandlerImpl) DeleteMaterial(w http.ResponseWrit func (handler *PipelineConfigRestHandlerImpl) HandleWorkflowWebhook(w http.ResponseWriter, r *http.Request) { decoder := json.NewDecoder(r.Body) - var wfUpdateReq v1alpha1.WorkflowStatus + var wfUpdateReq eventProcessorBean.CiCdStatus err := decoder.Decode(&wfUpdateReq) if err != nil { handler.Logger.Errorw("request err, HandleWorkflowWebhook", "err", err, "payload", wfUpdateReq) @@ -1454,7 +1454,7 @@ func (handler *PipelineConfigRestHandlerImpl) HandleWorkflowWebhook(w http.Respo return } handler.Logger.Infow("request payload, HandleWorkflowWebhook", "payload", wfUpdateReq) - resp, err := handler.ciHandler.UpdateWorkflow(wfUpdateReq) + resp, _, err := handler.ciHandler.UpdateWorkflow(wfUpdateReq) if err != nil { handler.Logger.Errorw("service err, HandleWorkflowWebhook", "err", err, "payload", wfUpdateReq) common.WriteJsonResp(w, err, resp, http.StatusInternalServerError) @@ -1559,7 +1559,7 @@ func (handler *PipelineConfigRestHandlerImpl) CancelWorkflow(w http.ResponseWrit //RBAC - resp, err := handler.ciHandler.CancelBuild(workflowId, forceAbort) + resp, err := handler.ciHandlerService.CancelBuild(workflowId, forceAbort) if err != nil { handler.Logger.Errorw("service err, CancelWorkflow", "err", err, "workflowId", workflowId, "pipelineId", pipelineId) if util.IsErrNoRows(err) { diff --git a/api/restHandler/app/pipeline/configure/DeploymentPipelineRestHandler.go b/api/restHandler/app/pipeline/configure/DeploymentPipelineRestHandler.go index 8437aa7ec4..67e78a7ccf 100644 --- a/api/restHandler/app/pipeline/configure/DeploymentPipelineRestHandler.go +++ b/api/restHandler/app/pipeline/configure/DeploymentPipelineRestHandler.go @@ -1620,7 +1620,7 @@ func (handler *PipelineConfigRestHandlerImpl) GetPrePostDeploymentLogs(w http.Re } // RBAC CHECK - logsReader, cleanUp, err := handler.cdHandler.GetRunningWorkflowLogs(environmentId, pipelineId, workflowId) + logsReader, cleanUp, err := handler.cdHandlerService.GetRunningWorkflowLogs(environmentId, pipelineId, workflowId) if err != nil { handler.Logger.Errorw("service err, GetPrePostDeploymentLogs", "err", err, "appId", appId, "environmentId", environmentId, "pipelineId", pipelineId, "workflowId", workflowId) common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) @@ -1745,7 +1745,7 @@ func (handler *PipelineConfigRestHandlerImpl) DownloadArtifacts(w http.ResponseW } // RBAC CHECK - file, err := handler.cdHandler.DownloadCdWorkflowArtifacts(buildId) + file, err := handler.cdHandlerService.DownloadCdWorkflowArtifacts(buildId) defer file.Close() if err != nil { @@ -1929,7 +1929,7 @@ func (handler *PipelineConfigRestHandlerImpl) CancelStage(w http.ResponseWriter, } // RBAC - resp, err := handler.cdHandler.CancelStage(workflowRunnerId, forceAbort, userId) + resp, err := handler.cdHandlerService.CancelStage(workflowRunnerId, forceAbort, userId) if err != nil { handler.Logger.Errorw("service err, CancelStage", "err", err, "pipelineId", pipelineId, "workflowRunnerId", workflowRunnerId) if util.IsErrNoRows(err) { diff --git a/api/restHandler/app/pipeline/configure/PipelineConfigRestHandler.go b/api/restHandler/app/pipeline/configure/PipelineConfigRestHandler.go index 4d37e03579..74ecde9387 100644 --- a/api/restHandler/app/pipeline/configure/PipelineConfigRestHandler.go +++ b/api/restHandler/app/pipeline/configure/PipelineConfigRestHandler.go @@ -26,12 +26,14 @@ import ( read2 "github.com/devtron-labs/devtron/pkg/build/git/gitMaterial/read" gitProviderRead "github.com/devtron-labs/devtron/pkg/build/git/gitProvider/read" bean3 "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" + "github.com/devtron-labs/devtron/pkg/build/trigger" "github.com/devtron-labs/devtron/pkg/chart/gitOpsConfig" read5 "github.com/devtron-labs/devtron/pkg/chart/read" repository2 "github.com/devtron-labs/devtron/pkg/cluster/environment/repository" "github.com/devtron-labs/devtron/pkg/deployment/manifest/deployedAppMetrics" "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/chartRef" validator2 "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/validator" + "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps" "github.com/devtron-labs/devtron/pkg/pipeline/draftAwareConfigService" security2 "github.com/devtron-labs/devtron/pkg/policyGovernance/security/imageScanning" "github.com/devtron-labs/devtron/pkg/policyGovernance/security/imageScanning/read" @@ -140,6 +142,8 @@ type PipelineConfigRestHandlerImpl struct { environmentRepository repository2.EnvironmentRepository chartReadService read5.ChartReadService draftAwareResourceService draftAwareConfigService.DraftAwareConfigService + ciHandlerService trigger.HandlerService + cdHandlerService devtronApps.HandlerService } func NewPipelineRestHandlerImpl(pipelineBuilder pipeline.PipelineBuilder, Logger *zap.SugaredLogger, @@ -175,6 +179,8 @@ func NewPipelineRestHandlerImpl(pipelineBuilder pipeline.PipelineBuilder, Logger EnvironmentRepository repository2.EnvironmentRepository, chartReadService read5.ChartReadService, draftAwareResourceService draftAwareConfigService.DraftAwareConfigService, + ciHandlerService trigger.HandlerService, + cdHandlerService devtronApps.HandlerService, ) *PipelineConfigRestHandlerImpl { envConfig := &PipelineRestHandlerEnvConfig{} err := env.Parse(envConfig) @@ -218,6 +224,8 @@ func NewPipelineRestHandlerImpl(pipelineBuilder pipeline.PipelineBuilder, Logger environmentRepository: EnvironmentRepository, chartReadService: chartReadService, draftAwareResourceService: draftAwareResourceService, + ciHandlerService: ciHandlerService, + cdHandlerService: cdHandlerService, } } diff --git a/api/restHandler/app/pipeline/trigger/PipelineTriggerRestHandler.go b/api/restHandler/app/pipeline/trigger/PipelineTriggerRestHandler.go index 08b43625b1..8fab862852 100644 --- a/api/restHandler/app/pipeline/trigger/PipelineTriggerRestHandler.go +++ b/api/restHandler/app/pipeline/trigger/PipelineTriggerRestHandler.go @@ -66,7 +66,7 @@ type PipelineTriggerRestHandlerImpl struct { deploymentGroupService deploymentGroup.DeploymentGroupService deploymentConfigService pipeline.PipelineDeploymentConfigService deployedAppService deployedApp.DeployedAppService - cdTriggerService devtronApps.TriggerService + cdHandlerService devtronApps.HandlerService workflowEventPublishService out.WorkflowEventPublishService } @@ -75,7 +75,7 @@ func NewPipelineRestHandler(appService app.AppService, userAuthService user.User deploymentGroupService deploymentGroup.DeploymentGroupService, deploymentConfigService pipeline.PipelineDeploymentConfigService, deployedAppService deployedApp.DeployedAppService, - cdTriggerService devtronApps.TriggerService, + cdHandlerService devtronApps.HandlerService, workflowEventPublishService out.WorkflowEventPublishService) *PipelineTriggerRestHandlerImpl { pipelineHandler := &PipelineTriggerRestHandlerImpl{ appService: appService, @@ -88,7 +88,7 @@ func NewPipelineRestHandler(appService app.AppService, userAuthService user.User deploymentGroupService: deploymentGroupService, deploymentConfigService: deploymentConfigService, deployedAppService: deployedAppService, - cdTriggerService: cdTriggerService, + cdHandlerService: cdHandlerService, workflowEventPublishService: workflowEventPublishService, } return pipelineHandler @@ -140,7 +140,7 @@ func (handler PipelineTriggerRestHandlerImpl) OverrideConfig(w http.ResponseWrit triggerContext := bean3.TriggerContext{ Context: ctx, } - mergeResp, helmPackageName, _, err := handler.cdTriggerService.ManualCdTrigger(triggerContext, &overrideRequest) + mergeResp, helmPackageName, _, err := handler.cdHandlerService.ManualCdTrigger(triggerContext, &overrideRequest) span.End() if err != nil { handler.logger.Errorw("request err, OverrideConfig", "err", err, "payload", overrideRequest) diff --git a/client/argocdServer/config/Config.go b/client/argocdServer/config/Config.go index 20b622c38e..2027a2257f 100644 --- a/client/argocdServer/config/Config.go +++ b/client/argocdServer/config/Config.go @@ -71,7 +71,7 @@ func (impl *ArgoCDConfigGetterImpl) GetGRPCConfig() (*bean.ArgoGRPCConfig, error } func (impl *ArgoCDConfigGetterImpl) GetK8sConfig() (*bean.ArgoK8sConfig, error) { - clusterBean, err := impl.clusterReadService.FindOne(bean2.DEFAULT_CLUSTER) + clusterBean, err := impl.clusterReadService.FindOne(bean2.DefaultCluster) if err != nil { impl.logger.Errorw("error in fetching cluster bean from db", "err", err) return nil, err diff --git a/client/cron/CiStatusUpdateCron.go b/client/cron/CiStatusUpdateCron.go index a62d4bc03f..4dbdd7ece5 100644 --- a/client/cron/CiStatusUpdateCron.go +++ b/client/cron/CiStatusUpdateCron.go @@ -21,7 +21,7 @@ import ( "github.com/caarlos0/env" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" "github.com/devtron-labs/devtron/pkg/app" - "github.com/devtron-labs/devtron/pkg/pipeline" + "github.com/devtron-labs/devtron/pkg/workflow/dag" cron2 "github.com/devtron-labs/devtron/util/cron" "github.com/robfig/cron/v3" "go.uber.org/zap" @@ -38,12 +38,13 @@ type CiStatusUpdateCronImpl struct { appService app.AppService ciWorkflowStatusUpdateConfig *CiWorkflowStatusUpdateConfig ciPipelineRepository pipelineConfig.CiPipelineRepository - ciHandler pipeline.CiHandler + workflowDagExecutor dag.WorkflowDagExecutor } func NewCiStatusUpdateCronImpl(logger *zap.SugaredLogger, appService app.AppService, ciWorkflowStatusUpdateConfig *CiWorkflowStatusUpdateConfig, ciPipelineRepository pipelineConfig.CiPipelineRepository, - ciHandler pipeline.CiHandler, cronLogger *cron2.CronLoggerImpl) *CiStatusUpdateCronImpl { + cronLogger *cron2.CronLoggerImpl, + workflowDagExecutor dag.WorkflowDagExecutor) *CiStatusUpdateCronImpl { cron := cron.New( cron.WithChain(cron.Recover(cronLogger))) cron.Start() @@ -53,7 +54,7 @@ func NewCiStatusUpdateCronImpl(logger *zap.SugaredLogger, appService app.AppServ appService: appService, ciWorkflowStatusUpdateConfig: ciWorkflowStatusUpdateConfig, ciPipelineRepository: ciPipelineRepository, - ciHandler: ciHandler, + workflowDagExecutor: workflowDagExecutor, } // execute periodically, update ci workflow status for failed process @@ -87,7 +88,7 @@ func (impl *CiStatusUpdateCronImpl) UpdateCiWorkflowStatusFailedCron() { impl.logger.Errorw("error in converting string to int", "err", err) return } - err = impl.ciHandler.UpdateCiWorkflowStatusFailure(timeoutForFailureCiBuild) + err = impl.workflowDagExecutor.UpdateCiWorkflowStatusFailure(timeoutForFailureCiBuild) if err != nil { impl.logger.Errorw("error in updating ci workflow status for failed workflows", "err", err) return diff --git a/client/cron/CiTriggerCron.go b/client/cron/CiTriggerCron.go index 147015a840..ddc84da2c7 100644 --- a/client/cron/CiTriggerCron.go +++ b/client/cron/CiTriggerCron.go @@ -22,8 +22,8 @@ import ( repository2 "github.com/devtron-labs/devtron/internal/sql/repository" bean2 "github.com/devtron-labs/devtron/pkg/auth/user/bean" "github.com/devtron-labs/devtron/pkg/bean" - pipelineConfigBean "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" - "github.com/devtron-labs/devtron/pkg/pipeline" + "github.com/devtron-labs/devtron/pkg/build/pipeline/bean/common" + "github.com/devtron-labs/devtron/pkg/build/trigger" "github.com/devtron-labs/devtron/pkg/pipeline/repository" repository3 "github.com/devtron-labs/devtron/pkg/plugin/repository" cron2 "github.com/devtron-labs/devtron/util/cron" @@ -40,13 +40,14 @@ type CiTriggerCronImpl struct { cron *cron.Cron cfg *CiTriggerCronConfig pipelineStageRepository repository.PipelineStageRepository - ciHandler pipeline.CiHandler ciArtifactRepository repository2.CiArtifactRepository globalPluginRepository repository3.GlobalPluginRepository + ciHandlerService trigger.HandlerService } func NewCiTriggerCronImpl(logger *zap.SugaredLogger, cfg *CiTriggerCronConfig, pipelineStageRepository repository.PipelineStageRepository, - ciHandler pipeline.CiHandler, ciArtifactRepository repository2.CiArtifactRepository, globalPluginRepository repository3.GlobalPluginRepository, cronLogger *cron2.CronLoggerImpl) *CiTriggerCronImpl { + ciArtifactRepository repository2.CiArtifactRepository, globalPluginRepository repository3.GlobalPluginRepository, cronLogger *cron2.CronLoggerImpl, + ciHandlerService trigger.HandlerService) *CiTriggerCronImpl { cron := cron.New( cron.WithChain(cron.Recover(cronLogger))) cron.Start() @@ -54,10 +55,10 @@ func NewCiTriggerCronImpl(logger *zap.SugaredLogger, cfg *CiTriggerCronConfig, p logger: logger, cron: cron, pipelineStageRepository: pipelineStageRepository, - ciHandler: ciHandler, cfg: cfg, ciArtifactRepository: ciArtifactRepository, globalPluginRepository: globalPluginRepository, + ciHandlerService: ciHandlerService, } _, err := cron.AddFunc(fmt.Sprintf("@every %dm", cfg.SourceControllerCronTime), impl.TriggerCiCron) @@ -101,9 +102,9 @@ func (impl *CiTriggerCronImpl) TriggerCiCron() { CiPipelineMaterial: ciPipelineMaterials, TriggeredBy: bean2.SYSTEM_USER_ID, InvalidateCache: false, - PipelineType: string(pipelineConfigBean.CI_JOB), + PipelineType: string(common.CI_JOB), } - _, err = impl.ciHandler.HandleCIManual(ciTriggerRequest) + _, err = impl.ciHandlerService.HandleCIManual(ciTriggerRequest) if err != nil { return } diff --git a/client/events/EventBuilder.go b/client/events/EventBuilder.go index 83f7186811..4464d7aa98 100644 --- a/client/events/EventBuilder.go +++ b/client/events/EventBuilder.go @@ -19,6 +19,7 @@ package client import ( "context" "fmt" + buildBean "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" repository4 "github.com/devtron-labs/devtron/pkg/cluster/environment/repository" "strings" "time" @@ -38,7 +39,7 @@ import ( type EventFactory interface { Build(eventType util.EventType, sourceId *int, appId int, envId *int, pipelineType util.PipelineType) (Event, error) BuildExtraCDData(event Event, wfr *pipelineConfig.CdWorkflowRunner, pipelineOverrideId int, stage bean2.WorkflowType) Event - BuildExtraCIData(event Event, material *MaterialTriggerInfo) Event + BuildExtraCIData(event Event, material *buildBean.MaterialTriggerInfo) Event //BuildFinalData(event Event) *Payload } @@ -173,7 +174,7 @@ func (impl *EventSimpleFactoryImpl) BuildExtraCDData(event Event, wfr *pipelineC return event } -func (impl *EventSimpleFactoryImpl) BuildExtraCIData(event Event, material *MaterialTriggerInfo) Event { +func (impl *EventSimpleFactoryImpl) BuildExtraCIData(event Event, material *buildBean.MaterialTriggerInfo) Event { if material == nil { materialInfo, err := impl.getCiMaterialInfo(event.PipelineId, event.CiArtifactId) if err != nil { @@ -207,8 +208,8 @@ func (impl *EventSimpleFactoryImpl) BuildExtraCIData(event Event, material *Mate return event } -func (impl *EventSimpleFactoryImpl) getCiMaterialInfo(ciPipelineId int, ciArtifactId int) (*MaterialTriggerInfo, error) { - materialTriggerInfo := &MaterialTriggerInfo{} +func (impl *EventSimpleFactoryImpl) getCiMaterialInfo(ciPipelineId int, ciArtifactId int) (*buildBean.MaterialTriggerInfo, error) { + materialTriggerInfo := &buildBean.MaterialTriggerInfo{} if ciPipelineId > 0 { ciMaterials, err := impl.ciPipelineMaterialRepository.GetByPipelineId(ciPipelineId) if err != nil { @@ -216,13 +217,13 @@ func (impl *EventSimpleFactoryImpl) getCiMaterialInfo(ciPipelineId int, ciArtifa return nil, err } - var ciMaterialsArr []CiPipelineMaterialResponse + var ciMaterialsArr []buildBean.CiPipelineMaterialResponse for _, m := range ciMaterials { if m.GitMaterial == nil { impl.logger.Warnw("git material are empty", "material", m) continue } - res := CiPipelineMaterialResponse{ + res := buildBean.CiPipelineMaterialResponse{ Id: m.Id, GitMaterialId: m.GitMaterialId, GitMaterialName: m.GitMaterial.Name[strings.Index(m.GitMaterial.Name, "-")+1:], diff --git a/client/events/EventClient.go b/client/events/EventClient.go index 5d7590f9b1..b15c23b513 100644 --- a/client/events/EventClient.go +++ b/client/events/EventClient.go @@ -21,20 +21,18 @@ import ( "encoding/json" "errors" "fmt" - bean2 "github.com/devtron-labs/devtron/pkg/attributes/bean" - "github.com/devtron-labs/devtron/pkg/module" - bean3 "github.com/devtron-labs/devtron/pkg/module/bean" - "net/http" - "time" - "github.com/caarlos0/env" pubsub "github.com/devtron-labs/common-lib/pubsub-lib" "github.com/devtron-labs/devtron/api/bean" - "github.com/devtron-labs/devtron/client/gitSensor" "github.com/devtron-labs/devtron/internal/sql/repository" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" + bean2 "github.com/devtron-labs/devtron/pkg/attributes/bean" + buildBean "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" + "github.com/devtron-labs/devtron/pkg/module" + bean3 "github.com/devtron-labs/devtron/pkg/module/bean" util "github.com/devtron-labs/devtron/util/event" "go.uber.org/zap" + "net/http" ) type EventClientConfig struct { @@ -81,41 +79,19 @@ type Event struct { } type Payload struct { - AppName string `json:"appName"` - EnvName string `json:"envName"` - PipelineName string `json:"pipelineName"` - Source string `json:"source"` - DockerImageUrl string `json:"dockerImageUrl"` - TriggeredBy string `json:"triggeredBy"` - Stage string `json:"stage"` - DeploymentHistoryLink string `json:"deploymentHistoryLink"` - AppDetailLink string `json:"appDetailLink"` - DownloadLink string `json:"downloadLink"` - BuildHistoryLink string `json:"buildHistoryLink"` - MaterialTriggerInfo *MaterialTriggerInfo `json:"material"` - FailureReason string `json:"failureReason"` -} - -type CiPipelineMaterialResponse struct { - Id int `json:"id"` - GitMaterialId int `json:"gitMaterialId"` - GitMaterialUrl string `json:"gitMaterialUrl"` - GitMaterialName string `json:"gitMaterialName"` - Type string `json:"type"` - Value string `json:"value"` - Active bool `json:"active"` - History []*gitSensor.GitCommit `json:"history,omitempty"` - LastFetchTime time.Time `json:"lastFetchTime"` - IsRepoError bool `json:"isRepoError"` - RepoErrorMsg string `json:"repoErrorMsg"` - IsBranchError bool `json:"isBranchError"` - BranchErrorMsg string `json:"branchErrorMsg"` - Url string `json:"url"` -} - -type MaterialTriggerInfo struct { - GitTriggers map[int]pipelineConfig.GitCommit `json:"gitTriggers"` - CiMaterials []CiPipelineMaterialResponse `json:"ciMaterials"` + AppName string `json:"appName"` + EnvName string `json:"envName"` + PipelineName string `json:"pipelineName"` + Source string `json:"source"` + DockerImageUrl string `json:"dockerImageUrl"` + TriggeredBy string `json:"triggeredBy"` + Stage string `json:"stage"` + DeploymentHistoryLink string `json:"deploymentHistoryLink"` + AppDetailLink string `json:"appDetailLink"` + DownloadLink string `json:"downloadLink"` + BuildHistoryLink string `json:"buildHistoryLink"` + MaterialTriggerInfo *buildBean.MaterialTriggerInfo `json:"material"` + FailureReason string `json:"failureReason"` } type EventRESTClientImpl struct { diff --git a/client/telemetry/TelemetryEventClient.go b/client/telemetry/TelemetryEventClient.go index b6c1979825..bfa4bf15c2 100644 --- a/client/telemetry/TelemetryEventClient.go +++ b/client/telemetry/TelemetryEventClient.go @@ -22,6 +22,7 @@ import ( "encoding/json" "fmt" cloudProviderIdentifier "github.com/devtron-labs/common-lib/cloud-provider-identifier" + posthogTelemetry "github.com/devtron-labs/common-lib/telemetry" "github.com/devtron-labs/common-lib/utils/k8s/commonBean" "github.com/devtron-labs/devtron/api/helm-app/gRPC" installedAppReader "github.com/devtron-labs/devtron/pkg/appStore/installedApp/read" @@ -29,7 +30,9 @@ import ( "github.com/devtron-labs/devtron/pkg/auth/user/bean" bean3 "github.com/devtron-labs/devtron/pkg/cluster/bean" module2 "github.com/devtron-labs/devtron/pkg/module/bean" + ucidService "github.com/devtron-labs/devtron/pkg/ucid" cron3 "github.com/devtron-labs/devtron/util/cron" + "go.opentelemetry.io/otel" "net/http" "time" @@ -43,22 +46,13 @@ import ( util3 "github.com/devtron-labs/devtron/pkg/util" "github.com/devtron-labs/devtron/util" "github.com/go-pg/pg" - "github.com/patrickmn/go-cache" "github.com/posthog/posthog-go" "github.com/robfig/cron/v3" "github.com/tidwall/gjson" "go.uber.org/zap" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" - "k8s.io/api/core/v1" - v12 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/version" ) -const LOGIN_COUNT_CONST = "login-count" -const SKIPPED_ONBOARDING_CONST = "SkippedOnboarding" -const ADMIN_EMAIL_ID_CONST = "admin" - type TelemetryEventClientImpl struct { cron *cron.Cron logger *zap.SugaredLogger @@ -69,7 +63,8 @@ type TelemetryEventClientImpl struct { userService user2.UserService attributeRepo repository.AttributesRepository ssoLoginService sso.SSOLoginService - PosthogClient *PosthogClient + posthogClient *posthogTelemetry.PosthogClient + ucid ucidService.Service moduleRepository moduleRepo.ModuleRepository serverDataStore *serverDataStore.ServerDataStore userAuditService user2.UserAuditService @@ -93,9 +88,9 @@ type TelemetryEventClient interface { func NewTelemetryEventClientImpl(logger *zap.SugaredLogger, client *http.Client, clusterService cluster.ClusterService, K8sUtil *k8s.K8sServiceImpl, aCDAuthConfig *util3.ACDAuthConfig, userService user2.UserService, attributeRepo repository.AttributesRepository, ssoLoginService sso.SSOLoginService, - PosthogClient *PosthogClient, moduleRepository moduleRepo.ModuleRepository, - serverDataStore *serverDataStore.ServerDataStore, userAuditService user2.UserAuditService, - helmAppClient gRPC.HelmAppClient, + posthog *posthogTelemetry.PosthogClient, ucid ucidService.Service, + moduleRepository moduleRepo.ModuleRepository, serverDataStore *serverDataStore.ServerDataStore, + userAuditService user2.UserAuditService, helmAppClient gRPC.HelmAppClient, cloudProviderIdentifierService cloudProviderIdentifier.ProviderIdentifierService, cronLogger *cron3.CronLoggerImpl, installedAppReadService installedAppReader.InstalledAppReadServiceEA, envVariables *util.EnvironmentVariables, @@ -113,7 +108,8 @@ func NewTelemetryEventClientImpl(logger *zap.SugaredLogger, client *http.Client, userService: userService, attributeRepo: attributeRepo, ssoLoginService: ssoLoginService, - PosthogClient: PosthogClient, + posthogClient: posthog, + ucid: ucid, moduleRepository: moduleRepository, serverDataStore: serverDataStore, userAuditService: userAuditService, @@ -126,13 +122,13 @@ func NewTelemetryEventClientImpl(logger *zap.SugaredLogger, client *http.Client, } watcher.HeartbeatEventForTelemetry() - _, err := cron.AddFunc(SummaryCronExpr, watcher.SummaryEventForTelemetryEA) + _, err := cron.AddFunc(posthogTelemetry.SummaryCronExpr, watcher.SummaryEventForTelemetryEA) if err != nil { logger.Errorw("error in starting summery event", "err", err) return nil, err } - _, err = cron.AddFunc(HeartbeatCronExpr, watcher.HeartbeatEventForTelemetry) + _, err = cron.AddFunc(posthogTelemetry.HeartbeatCronExpr, watcher.HeartbeatEventForTelemetry) if err != nil { logger.Errorw("error in starting heartbeat event", "err", err) return nil, err @@ -157,59 +153,6 @@ func (impl *TelemetryEventClientImpl) StopCron() { impl.cron.Stop() } -type TelemetryEventEA struct { - UCID string `json:"ucid"` //unique client id - Timestamp time.Time `json:"timestamp"` - EventMessage string `json:"eventMessage,omitempty"` - EventType TelemetryEventType `json:"eventType"` - ServerVersion string `json:"serverVersion,omitempty"` - UserCount int `json:"userCount,omitempty"` - ClusterCount int `json:"clusterCount,omitempty"` - HostURL bool `json:"hostURL,omitempty"` - SSOLogin bool `json:"ssoLogin,omitempty"` - DevtronVersion string `json:"devtronVersion,omitempty"` - DevtronMode string `json:"devtronMode,omitempty"` - InstalledIntegrations []string `json:"installedIntegrations,omitempty"` - InstallFailedIntegrations []string `json:"installFailedIntegrations,omitempty"` - InstallTimedOutIntegrations []string `json:"installTimedOutIntegrations,omitempty"` - LastLoginTime time.Time `json:"LastLoginTime,omitempty"` - InstallingIntegrations []string `json:"installingIntegrations,omitempty"` - DevtronReleaseVersion string `json:"devtronReleaseVersion,omitempty"` - HelmAppAccessCounter string `json:"HelmAppAccessCounter,omitempty"` - HelmAppUpdateCounter string `json:"HelmAppUpdateCounter,omitempty"` - ChartStoreVisitCount string `json:"ChartStoreVisitCount,omitempty"` - SkippedOnboarding bool `json:"SkippedOnboarding"` - HelmChartSuccessfulDeploymentCount int `json:"helmChartSuccessfulDeploymentCount,omitempty"` - ExternalHelmAppClusterCount map[int32]int `json:"ExternalHelmAppClusterCount,omitempty"` - ClusterProvider string `json:"clusterProvider,omitempty"` -} - -const DevtronUniqueClientIdConfigMap = "devtron-ucid" -const DevtronUniqueClientIdConfigMapKey = "UCID" -const InstallEventKey = "installEvent" -const UIEventKey = "uiEventKey" - -type TelemetryEventType string - -const ( - Heartbeat TelemetryEventType = "Heartbeat" - InstallationStart TelemetryEventType = "InstallationStart" - InstallationInProgress TelemetryEventType = "InstallationInProgress" - InstallationInterrupt TelemetryEventType = "InstallationInterrupt" - InstallationSuccess TelemetryEventType = "InstallationSuccess" - InstallationFailure TelemetryEventType = "InstallationFailure" - UpgradeStart TelemetryEventType = "UpgradeStart" - UpgradeInProgress TelemetryEventType = "UpgradeInProgress" - UpgradeInterrupt TelemetryEventType = "UpgradeInterrupt" - UpgradeSuccess TelemetryEventType = "UpgradeSuccess" - UpgradeFailure TelemetryEventType = "UpgradeFailure" - Summary TelemetryEventType = "Summary" - InstallationApplicationError TelemetryEventType = "InstallationApplicationError" - DashboardAccessed TelemetryEventType = "DashboardAccessed" - DashboardLoggedIn TelemetryEventType = "DashboardLoggedIn" - SIG_TERM TelemetryEventType = "SIG_TERM" -) - func (impl *TelemetryEventClientImpl) SummaryDetailsForTelemetry() (cluster []bean3.ClusterBean, user []bean.UserInfo, k8sServerVersion *version.Info, hostURL bool, ssoSetup bool, HelmAppAccessCount string, ChartStoreVisitCount string, SkippedOnboarding bool, HelmAppUpdateCounter string, helmChartSuccessfulDeploymentCount int, ExternalHelmAppClusterCount map[int32]int) { @@ -299,9 +242,9 @@ func (impl *TelemetryEventClientImpl) SummaryDetailsForTelemetry() (cluster []be } //getting userData from emailId - userData, err := impl.userAttributesRepository.GetUserDataByEmailId(ADMIN_EMAIL_ID_CONST) + userData, err := impl.userAttributesRepository.GetUserDataByEmailId(AdminEmailIdConst) - SkippedOnboardingValue := gjson.Get(userData, SKIPPED_ONBOARDING_CONST).Str + SkippedOnboardingValue := gjson.Get(userData, SkippedOnboardingConst).Str if SkippedOnboardingValue == "true" { SkippedOnboarding = true @@ -328,13 +271,13 @@ func (impl *TelemetryEventClientImpl) SummaryEventForTelemetryEA() { func (impl *TelemetryEventClientImpl) SendSummaryEvent(eventType string) error { impl.logger.Infow("sending summary event", "eventType", eventType) - ucid, err := impl.getUCID() + ucid, err := impl.getUCIDAndCheckIsOptedOut(context.Background()) if err != nil { impl.logger.Errorw("exception caught inside telemetry summary event", "err", err) return err } - if IsOptOut { + if posthogTelemetry.IsOptOut { impl.logger.Warnw("client is opt-out for telemetry, there will be no events capture", "ucid", ucid) return err } @@ -406,13 +349,13 @@ func (impl *TelemetryEventClientImpl) EnqueuePostHog(ucid string, eventType Tele } func (impl *TelemetryEventClientImpl) SendGenericTelemetryEvent(eventType string, prop map[string]interface{}) error { - ucid, err := impl.getUCID() + ucid, err := impl.getUCIDAndCheckIsOptedOut(context.Background()) if err != nil { impl.logger.Errorw("exception caught inside telemetry generic event", "err", err) return nil } - if IsOptOut { + if posthogTelemetry.IsOptOut { impl.logger.Warnw("client is opt-out for telemetry, there will be no events capture", "ucid", ucid) return nil } @@ -421,15 +364,15 @@ func (impl *TelemetryEventClientImpl) SendGenericTelemetryEvent(eventType string } func (impl *TelemetryEventClientImpl) EnqueueGenericPostHogEvent(ucid string, eventType string, prop map[string]interface{}) error { - if impl.PosthogClient.Client == nil { + if impl.posthogClient.Client == nil { impl.logger.Warn("no posthog client found, creating new") - client, err := impl.retryPosthogClient(PosthogApiKey, PosthogEndpoint) + client, err := impl.retryPosthogClient(posthogTelemetry.PosthogApiKey, posthogTelemetry.PosthogEndpoint) if err == nil { - impl.PosthogClient.Client = client + impl.posthogClient.Client = client } } - if impl.PosthogClient.Client != nil && !impl.globalEnvVariables.IsAirGapEnvironment { - err := impl.PosthogClient.Client.Enqueue(posthog.Capture{ + if impl.posthogClient.Client != nil && !impl.globalEnvVariables.IsAirGapEnvironment { + err := impl.posthogClient.Client.Enqueue(posthog.Capture{ DistinctId: ucid, Event: eventType, Properties: prop, @@ -443,12 +386,12 @@ func (impl *TelemetryEventClientImpl) EnqueueGenericPostHogEvent(ucid string, ev } func (impl *TelemetryEventClientImpl) HeartbeatEventForTelemetry() { - ucid, err := impl.getUCID() + ucid, err := impl.getUCIDAndCheckIsOptedOut(context.Background()) if err != nil { impl.logger.Errorw("exception caught inside telemetry heartbeat event", "err", err) return } - if IsOptOut { + if posthogTelemetry.IsOptOut { impl.logger.Warnw("client is opt-out for telemetry, there will be no events capture", "ucid", ucid) return } @@ -490,21 +433,21 @@ func (impl *TelemetryEventClientImpl) HeartbeatEventForTelemetry() { } func (impl *TelemetryEventClientImpl) GetTelemetryMetaInfo() (*TelemetryMetaInfo, error) { - ucid, err := impl.getUCID() + ucid, err := impl.getUCIDAndCheckIsOptedOut(context.Background()) if err != nil { impl.logger.Errorw("exception while getting unique client id", "error", err) return nil, err } data := &TelemetryMetaInfo{ - Url: PosthogEndpoint, + Url: posthogTelemetry.PosthogEndpoint, UCID: ucid, - ApiKey: PosthogEncodedApiKey, + ApiKey: posthogTelemetry.PosthogEncodedApiKey, } return data, err } func (impl *TelemetryEventClientImpl) SendTelemetryInstallEventEA() (*TelemetryEventType, error) { - ucid, err := impl.getUCID() + ucid, err := impl.getUCIDAndCheckIsOptedOut(context.Background()) if err != nil { impl.logger.Errorw("exception while getting unique client id", "error", err) return nil, err @@ -548,15 +491,19 @@ func (impl *TelemetryEventClientImpl) SendTelemetryInstallEventEA() (*TelemetryE impl.logger.Errorw("Installation EventForTelemetry EA Mode, payload unmarshal error", "error", err) return nil, nil } - cm, err := impl.K8sUtil.GetConfigMap(impl.aCDAuthConfig.ACDConfigMapNamespace, DevtronUniqueClientIdConfigMap, client) + cm, err := impl.K8sUtil.GetConfigMap(impl.aCDAuthConfig.ACDConfigMapNamespace, ucidService.DevtronUniqueClientIdConfigMap, client) + if err != nil { + impl.logger.Errorw("Installation EventForTelemetry EA Mode, failed to get DevtronUniqueClientIdConfigMap", "error", err) + return nil, err + } datamap := cm.Data - installEventValue, installEventKeyExists := datamap[InstallEventKey] + installEventValue, installEventKeyExists := datamap[ucidService.InstallEventKey] if installEventKeyExists == false || installEventValue == "1" { err = impl.EnqueuePostHog(ucid, InstallationSuccess, prop) if err == nil { - datamap[InstallEventKey] = "2" + datamap[ucidService.InstallEventKey] = "2" cm.Data = datamap _, err = impl.K8sUtil.UpdateConfigMap(impl.aCDAuthConfig.ACDConfigMapNamespace, cm, client) if err != nil { @@ -570,7 +517,7 @@ func (impl *TelemetryEventClientImpl) SendTelemetryInstallEventEA() (*TelemetryE } func (impl *TelemetryEventClientImpl) SendTelemetryDashboardAccessEvent() error { - ucid, err := impl.getUCID() + ucid, err := impl.getUCIDAndCheckIsOptedOut(context.Background()) if err != nil { impl.logger.Errorw("exception while getting unique client id", "error", err) return err @@ -614,19 +561,19 @@ func (impl *TelemetryEventClientImpl) SendTelemetryDashboardAccessEvent() error impl.logger.Errorw("DashboardAccessed EventForTelemetry, payload unmarshal error", "error", err) return err } - cm, err := impl.K8sUtil.GetConfigMap(impl.aCDAuthConfig.ACDConfigMapNamespace, DevtronUniqueClientIdConfigMap, client) + cm, err := impl.K8sUtil.GetConfigMap(impl.aCDAuthConfig.ACDConfigMapNamespace, ucidService.DevtronUniqueClientIdConfigMap, client) if err != nil { impl.logger.Errorw("DashboardAccessed EventForTelemetry,failed to get DevtronUniqueClientIdConfigMap", "error", err) return err } datamap := cm.Data - accessEventValue, installEventKeyExists := datamap[UIEventKey] + accessEventValue, installEventKeyExists := datamap[ucidService.UIEventKey] if installEventKeyExists == false || accessEventValue <= "1" { err = impl.EnqueuePostHog(ucid, DashboardAccessed, prop) if err == nil { - datamap[UIEventKey] = "2" + datamap[ucidService.UIEventKey] = "2" cm.Data = datamap _, err = impl.K8sUtil.UpdateConfigMap(impl.aCDAuthConfig.ACDConfigMapNamespace, cm, client) if err != nil { @@ -640,7 +587,7 @@ func (impl *TelemetryEventClientImpl) SendTelemetryDashboardAccessEvent() error } func (impl *TelemetryEventClientImpl) SendTelemetryDashboardLoggedInEvent() error { - ucid, err := impl.getUCID() + ucid, err := impl.getUCIDAndCheckIsOptedOut(context.Background()) if err != nil { impl.logger.Errorw("exception while getting unique client id", "error", err) return err @@ -684,19 +631,19 @@ func (impl *TelemetryEventClientImpl) SendTelemetryDashboardLoggedInEvent() erro impl.logger.Errorw("DashboardLoggedIn EventForTelemetry, payload unmarshal error", "error", err) return err } - cm, err := impl.K8sUtil.GetConfigMap(impl.aCDAuthConfig.ACDConfigMapNamespace, DevtronUniqueClientIdConfigMap, client) + cm, err := impl.K8sUtil.GetConfigMap(impl.aCDAuthConfig.ACDConfigMapNamespace, ucidService.DevtronUniqueClientIdConfigMap, client) if err != nil { impl.logger.Errorw("DashboardLoggedIn EventForTelemetry,failed to get DevtronUniqueClientIdConfigMap", "error", err) return err } datamap := cm.Data - accessEventValue, installEventKeyExists := datamap[UIEventKey] + accessEventValue, installEventKeyExists := datamap[ucidService.UIEventKey] if installEventKeyExists == false || accessEventValue != "3" { err = impl.EnqueuePostHog(ucid, DashboardLoggedIn, prop) if err == nil { - datamap[UIEventKey] = "3" + datamap[ucidService.UIEventKey] = "3" cm.Data = datamap _, err = impl.K8sUtil.UpdateConfigMap(impl.aCDAuthConfig.ACDConfigMapNamespace, cm, client) if err != nil { @@ -709,57 +656,29 @@ func (impl *TelemetryEventClientImpl) SendTelemetryDashboardLoggedInEvent() erro return nil } -type TelemetryMetaInfo struct { - Url string `json:"url,omitempty"` - UCID string `json:"ucid,omitempty"` - ApiKey string `json:"apiKey,omitempty"` -} - -func (impl *TelemetryEventClientImpl) getUCID() (string, error) { - ucid, found := impl.PosthogClient.cache.Get(DevtronUniqueClientIdConfigMapKey) - if found { - return ucid.(string), nil - } else { - client, err := impl.K8sUtil.GetClientForInCluster() - if err != nil { - impl.logger.Errorw("exception while getting unique client id", "error", err) - return "", err - } - - cm, err := impl.K8sUtil.GetConfigMap(impl.aCDAuthConfig.ACDConfigMapNamespace, DevtronUniqueClientIdConfigMap, client) - if errStatus, ok := status.FromError(err); !ok || errStatus.Code() == codes.NotFound || errStatus.Code() == codes.Unknown { - // if not found, create new cm - cm = &v1.ConfigMap{ObjectMeta: v12.ObjectMeta{Name: DevtronUniqueClientIdConfigMap}} - data := map[string]string{} - data[DevtronUniqueClientIdConfigMapKey] = util.Generate(16) // generate unique random number - data[InstallEventKey] = "1" // used in operator to detect event is install or upgrade - data[UIEventKey] = "1" - cm.Data = data - _, err = impl.K8sUtil.CreateConfigMap(impl.aCDAuthConfig.ACDConfigMapNamespace, cm, client) - if err != nil { - impl.logger.Errorw("exception while getting unique client id", "error", err) - return "", err - } - } - dataMap := cm.Data - ucid = dataMap[DevtronUniqueClientIdConfigMapKey] - impl.PosthogClient.cache.Set(DevtronUniqueClientIdConfigMapKey, ucid, cache.DefaultExpiration) - if cm == nil { - impl.logger.Errorw("configmap not found while getting unique client id", "cm", cm) - return ucid.(string), err - } - flag, err := impl.checkForOptOut(ucid.(string)) +func (impl *TelemetryEventClientImpl) getUCIDAndCheckIsOptedOut(ctx context.Context) (string, error) { + newCtx, span := otel.Tracer("orchestrator").Start(ctx, "TelemetryEventClientImpl.getUCIDAndCheckIsOptedOut") + defer span.End() + ucid, found, err := impl.ucid.GetUCIDWithCache(impl.posthogClient.GetCache()) + if err != nil { + impl.logger.Errorw("exception while getting unique client id", "error", err) + return "", err + } + if !found { + flag, err := impl.checkForOptOut(newCtx, ucid) if err != nil { impl.logger.Errorw("error sending event to posthog, failed check for opt-out", "err", err) return "", err } - IsOptOut = flag + posthogTelemetry.IsOptOut = flag } - return ucid.(string), nil + return ucid, nil } -func (impl *TelemetryEventClientImpl) checkForOptOut(UCID string) (bool, error) { - decodedUrl, err := base64.StdEncoding.DecodeString(TelemetryOptOutApiBaseUrl) +func (impl *TelemetryEventClientImpl) checkForOptOut(ctx context.Context, UCID string) (bool, error) { + newCtx, span := otel.Tracer("orchestrator").Start(ctx, "TelemetryEventClientImpl.checkForOptOut") + defer span.End() + decodedUrl, err := base64.StdEncoding.DecodeString(posthogTelemetry.TelemetryOptOutApiBaseUrl) if err != nil { impl.logger.Errorw("check opt-out list failed, decode error", "err", err) return false, err @@ -767,13 +686,13 @@ func (impl *TelemetryEventClientImpl) checkForOptOut(UCID string) (bool, error) encodedUrl := string(decodedUrl) url := fmt.Sprintf("%s/%s", encodedUrl, UCID) - response, err := util.HttpRequest(url) + response, err := util.HttpRequest(newCtx, url) if err != nil { - impl.logger.Errorw("check opt-out list failed, rest api error", "err", err) - return false, err + // this should be non-blocking call and should not fail the request for ucid getting + impl.logger.Errorw("check opt-out list failed, rest api error", "ucid", UCID, "err", err) } flag := response["result"].(bool) - return flag, err + return flag, nil } func (impl *TelemetryEventClientImpl) retryPosthogClient(PosthogApiKey string, PosthogEndpoint string) (posthog.Client, error) { @@ -785,7 +704,7 @@ func (impl *TelemetryEventClientImpl) retryPosthogClient(PosthogApiKey string, P return client, err } -// returns installedIntegrations, installFailedIntegrations, installTimedOutIntegrations, installingIntegrations +// buildIntegrationsList - returns installedIntegrations, installFailedIntegrations, installTimedOutIntegrations, installingIntegrations func (impl *TelemetryEventClientImpl) buildIntegrationsList() ([]string, []string, []string, []string, error) { impl.logger.Info("building integrations list for telemetry") diff --git a/client/telemetry/TelemetryEventClientExtended.go b/client/telemetry/TelemetryEventClientExtended.go index d185789fef..87de1d237f 100644 --- a/client/telemetry/TelemetryEventClientExtended.go +++ b/client/telemetry/TelemetryEventClientExtended.go @@ -17,8 +17,10 @@ package telemetry import ( + "context" "encoding/json" cloudProviderIdentifier "github.com/devtron-labs/common-lib/cloud-provider-identifier" + posthogTelemetry "github.com/devtron-labs/common-lib/telemetry" util2 "github.com/devtron-labs/common-lib/utils/k8s" client "github.com/devtron-labs/devtron/api/helm-app/gRPC" installedAppReader "github.com/devtron-labs/devtron/pkg/appStore/installedApp/read" @@ -33,6 +35,7 @@ import ( "github.com/devtron-labs/devtron/pkg/deployment/gitOps/config" moduleRepo "github.com/devtron-labs/devtron/pkg/module/repo" serverDataStore "github.com/devtron-labs/devtron/pkg/server/store" + ucidService "github.com/devtron-labs/devtron/pkg/ucid" util3 "github.com/devtron-labs/devtron/pkg/util" cron3 "github.com/devtron-labs/devtron/util/cron" "net/http" @@ -50,8 +53,6 @@ import ( "go.uber.org/zap" ) -const AppsCount int = 50 - type TelemetryEventClientImplExtended struct { environmentService environment.EnvironmentService appListingRepository repository.AppListingRepository @@ -73,7 +74,7 @@ type TelemetryEventClientImplExtended struct { func NewTelemetryEventClientImplExtended(logger *zap.SugaredLogger, client *http.Client, clusterService cluster.ClusterService, K8sUtil *util2.K8sServiceImpl, aCDAuthConfig *util3.ACDAuthConfig, environmentService environment.EnvironmentService, userService user2.UserService, - appListingRepository repository.AppListingRepository, PosthogClient *PosthogClient, + appListingRepository repository.AppListingRepository, posthog *posthogTelemetry.PosthogClient, ucid ucidService.Service, ciPipelineConfigReadService ciConfig.CiPipelineConfigReadService, pipelineRepository pipelineConfig.PipelineRepository, gitProviderRepository repository3.GitProviderRepository, attributeRepo repository.AttributesRepository, ssoLoginService sso.SSOLoginService, appRepository app.AppRepository, @@ -114,7 +115,8 @@ func NewTelemetryEventClientImplExtended(logger *zap.SugaredLogger, client *http userService: userService, attributeRepo: attributeRepo, ssoLoginService: ssoLoginService, - PosthogClient: PosthogClient, + posthogClient: posthog, + ucid: ucid, moduleRepository: moduleRepository, serverDataStore: serverDataStore, userAuditService: userAuditService, @@ -128,13 +130,13 @@ func NewTelemetryEventClientImplExtended(logger *zap.SugaredLogger, client *http } watcher.HeartbeatEventForTelemetry() - _, err := cron.AddFunc(SummaryCronExpr, watcher.SummaryEventForTelemetry) + _, err := cron.AddFunc(posthogTelemetry.SummaryCronExpr, watcher.SummaryEventForTelemetry) if err != nil { logger.Errorw("error in starting summery event", "err", err) return nil, err } - _, err = cron.AddFunc(HeartbeatCronExpr, watcher.HeartbeatEventForTelemetry) + _, err = cron.AddFunc(posthogTelemetry.HeartbeatCronExpr, watcher.HeartbeatEventForTelemetry) if err != nil { logger.Errorw("error in starting heartbeat event", "err", err) return nil, err @@ -142,65 +144,6 @@ func NewTelemetryEventClientImplExtended(logger *zap.SugaredLogger, client *http return watcher, err } -type TelemetryEventDto struct { - UCID string `json:"ucid"` //unique client id - Timestamp time.Time `json:"timestamp"` - EventMessage string `json:"eventMessage,omitempty"` - EventType TelemetryEventType `json:"eventType"` - ProdAppCount int `json:"prodAppCount,omitempty"` - NonProdAppCount int `json:"nonProdAppCount,omitempty"` - UserCount int `json:"userCount,omitempty"` - EnvironmentCount int `json:"environmentCount,omitempty"` - ClusterCount int `json:"clusterCount,omitempty"` - CiCreatedPerDay int `json:"ciCreatedPerDay"` - CdCreatedPerDay int `json:"cdCreatedPerDay"` - CiDeletedPerDay int `json:"ciDeletedPerDay"` - CdDeletedPerDay int `json:"cdDeletedPerDay"` - CiTriggeredPerDay int `json:"ciTriggeredPerDay"` - CdTriggeredPerDay int `json:"cdTriggeredPerDay"` - HelmChartCount int `json:"helmChartCount,omitempty"` - SecurityScanCountPerDay int `json:"securityScanCountPerDay,omitempty"` - GitAccountsCount int `json:"gitAccountsCount,omitempty"` - GitOpsCount int `json:"gitOpsCount,omitempty"` - RegistryCount int `json:"registryCount,omitempty"` - HostURL bool `json:"hostURL,omitempty"` - SSOLogin bool `json:"ssoLogin,omitempty"` - AppCount int `json:"appCount,omitempty"` - AppsWithGitRepoConfigured int `json:"appsWithGitRepoConfigured,omitempty"` - AppsWithDockerConfigured int `json:"appsWithDockerConfigured,omitempty"` - AppsWithDeploymentTemplateConfigured int `json:"appsWithDeploymentTemplateConfigured,omitempty"` - AppsWithCiPipelineConfigured int `json:"appsWithCiPipelineConfigured,omitempty"` - AppsWithCdPipelineConfigured int `json:"appsWithCdPipelineConfigured,omitempty"` - Build bool `json:"build,omitempty"` - Deployment bool `json:"deployment,omitempty"` - ServerVersion string `json:"serverVersion,omitempty"` - DevtronGitVersion string `json:"devtronGitVersion,omitempty"` - DevtronVersion string `json:"devtronVersion,omitempty"` - DevtronMode string `json:"devtronMode,omitempty"` - InstalledIntegrations []string `json:"installedIntegrations,omitempty"` - InstallFailedIntegrations []string `json:"installFailedIntegrations,omitempty"` - InstallTimedOutIntegrations []string `json:"installTimedOutIntegrations,omitempty"` - InstallingIntegrations []string `json:"installingIntegrations,omitempty"` - DevtronReleaseVersion string `json:"devtronReleaseVersion,omitempty"` - LastLoginTime time.Time `json:"LastLoginTime,omitempty"` - SelfDockerfileCount int `json:"selfDockerfileCount"` - ManagedDockerfileCount int `json:"managedDockerfileCount"` - BuildPackCount int `json:"buildPackCount"` - SelfDockerfileSuccessCount int `json:"selfDockerfileSuccessCount"` - SelfDockerfileFailureCount int `json:"selfDockerfileFailureCount"` - ManagedDockerfileSuccessCount int `json:"managedDockerfileSuccessCount"` - ManagedDockerfileFailureCount int `json:"managedDockerfileFailureCount"` - BuildPackSuccessCount int `json:"buildPackSuccessCount"` - BuildPackFailureCount int `json:"buildPackFailureCount"` - HelmAppAccessCounter string `json:"HelmAppAccessCounter,omitempty"` - ChartStoreVisitCount string `json:"ChartStoreVisitCount,omitempty"` - SkippedOnboarding bool `json:"SkippedOnboarding"` - HelmAppUpdateCounter string `json:"HelmAppUpdateCounter,omitempty"` - HelmChartSuccessfulDeploymentCount int `json:"helmChartSuccessfulDeploymentCount,omitempty"` - ExternalHelmAppClusterCount map[int32]int `json:"ExternalHelmAppClusterCount"` - ClusterProvider string `json:"clusterProvider,omitempty"` -} - func (impl *TelemetryEventClientImplExtended) SummaryEventForTelemetry() { err := impl.SendSummaryEvent(string(Summary)) if err != nil { @@ -210,13 +153,13 @@ func (impl *TelemetryEventClientImplExtended) SummaryEventForTelemetry() { func (impl *TelemetryEventClientImplExtended) SendSummaryEvent(eventType string) error { impl.logger.Infow("sending summary event", "eventType", eventType) - ucid, err := impl.getUCID() + ucid, err := impl.getUCIDAndCheckIsOptedOut(context.Background()) if err != nil { impl.logger.Errorw("exception caught inside telemetry summary event while retrieving ucid", "err", err) return err } - if IsOptOut { + if posthogTelemetry.IsOptOut { impl.logger.Warnw("client is opt-out for telemetry, there will be no events capture", "ucid", ucid) return err } diff --git a/client/telemetry/bean.go b/client/telemetry/bean.go new file mode 100644 index 0000000000..f3e95dd0e7 --- /dev/null +++ b/client/telemetry/bean.go @@ -0,0 +1,139 @@ +/* + * Copyright (c) 2020-2024. Devtron Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package telemetry + +import "time" + +type TelemetryMetaInfo struct { + Url string `json:"url,omitempty"` + UCID string `json:"ucid,omitempty"` + ApiKey string `json:"apiKey,omitempty"` +} + +const ( + SkippedOnboardingConst = "SkippedOnboarding" + AdminEmailIdConst = "admin" +) + +type TelemetryEventType string + +const ( + Heartbeat TelemetryEventType = "Heartbeat" + InstallationStart TelemetryEventType = "InstallationStart" + InstallationInProgress TelemetryEventType = "InstallationInProgress" + InstallationInterrupt TelemetryEventType = "InstallationInterrupt" + InstallationSuccess TelemetryEventType = "InstallationSuccess" + InstallationFailure TelemetryEventType = "InstallationFailure" + UpgradeStart TelemetryEventType = "UpgradeStart" + UpgradeInProgress TelemetryEventType = "UpgradeInProgress" + UpgradeInterrupt TelemetryEventType = "UpgradeInterrupt" + UpgradeSuccess TelemetryEventType = "UpgradeSuccess" + UpgradeFailure TelemetryEventType = "UpgradeFailure" + Summary TelemetryEventType = "Summary" + InstallationApplicationError TelemetryEventType = "InstallationApplicationError" + DashboardAccessed TelemetryEventType = "DashboardAccessed" + DashboardLoggedIn TelemetryEventType = "DashboardLoggedIn" + SIG_TERM TelemetryEventType = "SIG_TERM" +) + +type TelemetryEventEA struct { + UCID string `json:"ucid"` //unique client id + Timestamp time.Time `json:"timestamp"` + EventMessage string `json:"eventMessage,omitempty"` + EventType TelemetryEventType `json:"eventType"` + ServerVersion string `json:"serverVersion,omitempty"` + UserCount int `json:"userCount,omitempty"` + ClusterCount int `json:"clusterCount,omitempty"` + HostURL bool `json:"hostURL,omitempty"` + SSOLogin bool `json:"ssoLogin,omitempty"` + DevtronVersion string `json:"devtronVersion,omitempty"` + DevtronMode string `json:"devtronMode,omitempty"` + InstalledIntegrations []string `json:"installedIntegrations,omitempty"` + InstallFailedIntegrations []string `json:"installFailedIntegrations,omitempty"` + InstallTimedOutIntegrations []string `json:"installTimedOutIntegrations,omitempty"` + LastLoginTime time.Time `json:"LastLoginTime,omitempty"` + InstallingIntegrations []string `json:"installingIntegrations,omitempty"` + DevtronReleaseVersion string `json:"devtronReleaseVersion,omitempty"` + HelmAppAccessCounter string `json:"HelmAppAccessCounter,omitempty"` + HelmAppUpdateCounter string `json:"HelmAppUpdateCounter,omitempty"` + ChartStoreVisitCount string `json:"ChartStoreVisitCount,omitempty"` + SkippedOnboarding bool `json:"SkippedOnboarding"` + HelmChartSuccessfulDeploymentCount int `json:"helmChartSuccessfulDeploymentCount,omitempty"` + ExternalHelmAppClusterCount map[int32]int `json:"ExternalHelmAppClusterCount,omitempty"` + ClusterProvider string `json:"clusterProvider,omitempty"` +} + +const AppsCount int = 50 + +type TelemetryEventDto struct { + UCID string `json:"ucid"` //unique client id + Timestamp time.Time `json:"timestamp"` + EventMessage string `json:"eventMessage,omitempty"` + EventType TelemetryEventType `json:"eventType"` + ProdAppCount int `json:"prodAppCount,omitempty"` + NonProdAppCount int `json:"nonProdAppCount,omitempty"` + UserCount int `json:"userCount,omitempty"` + EnvironmentCount int `json:"environmentCount,omitempty"` + ClusterCount int `json:"clusterCount,omitempty"` + CiCreatedPerDay int `json:"ciCreatedPerDay"` + CdCreatedPerDay int `json:"cdCreatedPerDay"` + CiDeletedPerDay int `json:"ciDeletedPerDay"` + CdDeletedPerDay int `json:"cdDeletedPerDay"` + CiTriggeredPerDay int `json:"ciTriggeredPerDay"` + CdTriggeredPerDay int `json:"cdTriggeredPerDay"` + HelmChartCount int `json:"helmChartCount,omitempty"` + SecurityScanCountPerDay int `json:"securityScanCountPerDay,omitempty"` + GitAccountsCount int `json:"gitAccountsCount,omitempty"` + GitOpsCount int `json:"gitOpsCount,omitempty"` + RegistryCount int `json:"registryCount,omitempty"` + HostURL bool `json:"hostURL,omitempty"` + SSOLogin bool `json:"ssoLogin,omitempty"` + AppCount int `json:"appCount,omitempty"` + AppsWithGitRepoConfigured int `json:"appsWithGitRepoConfigured,omitempty"` + AppsWithDockerConfigured int `json:"appsWithDockerConfigured,omitempty"` + AppsWithDeploymentTemplateConfigured int `json:"appsWithDeploymentTemplateConfigured,omitempty"` + AppsWithCiPipelineConfigured int `json:"appsWithCiPipelineConfigured,omitempty"` + AppsWithCdPipelineConfigured int `json:"appsWithCdPipelineConfigured,omitempty"` + Build bool `json:"build,omitempty"` + Deployment bool `json:"deployment,omitempty"` + ServerVersion string `json:"serverVersion,omitempty"` + DevtronGitVersion string `json:"devtronGitVersion,omitempty"` + DevtronVersion string `json:"devtronVersion,omitempty"` + DevtronMode string `json:"devtronMode,omitempty"` + InstalledIntegrations []string `json:"installedIntegrations,omitempty"` + InstallFailedIntegrations []string `json:"installFailedIntegrations,omitempty"` + InstallTimedOutIntegrations []string `json:"installTimedOutIntegrations,omitempty"` + InstallingIntegrations []string `json:"installingIntegrations,omitempty"` + DevtronReleaseVersion string `json:"devtronReleaseVersion,omitempty"` + LastLoginTime time.Time `json:"LastLoginTime,omitempty"` + SelfDockerfileCount int `json:"selfDockerfileCount"` + ManagedDockerfileCount int `json:"managedDockerfileCount"` + BuildPackCount int `json:"buildPackCount"` + SelfDockerfileSuccessCount int `json:"selfDockerfileSuccessCount"` + SelfDockerfileFailureCount int `json:"selfDockerfileFailureCount"` + ManagedDockerfileSuccessCount int `json:"managedDockerfileSuccessCount"` + ManagedDockerfileFailureCount int `json:"managedDockerfileFailureCount"` + BuildPackSuccessCount int `json:"buildPackSuccessCount"` + BuildPackFailureCount int `json:"buildPackFailureCount"` + HelmAppAccessCounter string `json:"HelmAppAccessCounter,omitempty"` + ChartStoreVisitCount string `json:"ChartStoreVisitCount,omitempty"` + SkippedOnboarding bool `json:"SkippedOnboarding"` + HelmAppUpdateCounter string `json:"HelmAppUpdateCounter,omitempty"` + HelmChartSuccessfulDeploymentCount int `json:"helmChartSuccessfulDeploymentCount,omitempty"` + ExternalHelmAppClusterCount map[int32]int `json:"ExternalHelmAppClusterCount"` + ClusterProvider string `json:"clusterProvider,omitempty"` +} diff --git a/cmd/external-app/externalApp.go b/cmd/external-app/externalApp.go index b7f90fb2dd..dba06f4eea 100644 --- a/cmd/external-app/externalApp.go +++ b/cmd/external-app/externalApp.go @@ -20,6 +20,7 @@ import ( "context" "errors" "fmt" + posthogTelemetry "github.com/devtron-labs/common-lib/telemetry" "net/http" "os" "time" @@ -40,14 +41,14 @@ type App struct { Logger *zap.SugaredLogger server *http.Server telemetry telemetry.TelemetryEventClient - posthogClient *telemetry.PosthogClient + posthogClient *posthogTelemetry.PosthogClient } func NewApp(db *pg.DB, sessionManager *authMiddleware.SessionManager, MuxRouter *MuxRouter, telemetry telemetry.TelemetryEventClient, - posthogClient *telemetry.PosthogClient, + posthogClient *posthogTelemetry.PosthogClient, Logger *zap.SugaredLogger) *App { return &App{ db: db, diff --git a/cmd/external-app/wire.go b/cmd/external-app/wire.go index 8e633f9983..8986381ea6 100644 --- a/cmd/external-app/wire.go +++ b/cmd/external-app/wire.go @@ -22,6 +22,7 @@ package main import ( "github.com/devtron-labs/authenticator/middleware" cloudProviderIdentifier "github.com/devtron-labs/common-lib/cloud-provider-identifier" + posthogTelemetry "github.com/devtron-labs/common-lib/telemetry" util4 "github.com/devtron-labs/common-lib/utils/k8s" "github.com/devtron-labs/devtron/api/apiToken" "github.com/devtron-labs/devtron/api/appStore" @@ -87,6 +88,7 @@ import ( "github.com/devtron-labs/devtron/pkg/policyGovernance/security/scanTool" security2 "github.com/devtron-labs/devtron/pkg/policyGovernance/security/scanTool/repository" "github.com/devtron-labs/devtron/pkg/sql" + "github.com/devtron-labs/devtron/pkg/ucid" util2 "github.com/devtron-labs/devtron/pkg/util" util3 "github.com/devtron-labs/devtron/util" "github.com/devtron-labs/devtron/util/commonEnforcementFunctionsUtil" @@ -132,7 +134,8 @@ func InitializeApp() (*App, error) { util.NewSugardLogger, util.IntValidator, util2.GetACDAuthConfig, - telemetry.NewPosthogClient, + posthogTelemetry.NewPosthogClient, + ucid.WireSet, delete2.NewDeleteServiceImpl, gitMaterial.GitMaterialWireSet, scanTool.ScanToolWireSet, diff --git a/cmd/external-app/wire_gen.go b/cmd/external-app/wire_gen.go index d690e0d96c..43cd02248a 100644 --- a/cmd/external-app/wire_gen.go +++ b/cmd/external-app/wire_gen.go @@ -1,6 +1,6 @@ // Code generated by Wire. DO NOT EDIT. -//go:generate go run -mod=mod github.com/google/wire/cmd/wire +//go:generate go run github.com/google/wire/cmd/wire //go:build !wireinject // +build !wireinject @@ -11,6 +11,7 @@ import ( "github.com/devtron-labs/authenticator/client" "github.com/devtron-labs/authenticator/middleware" "github.com/devtron-labs/common-lib/cloud-provider-identifier" + "github.com/devtron-labs/common-lib/telemetry" "github.com/devtron-labs/common-lib/utils/grpc" "github.com/devtron-labs/common-lib/utils/k8s" apiToken2 "github.com/devtron-labs/devtron/api/apiToken" @@ -52,7 +53,7 @@ import ( "github.com/devtron-labs/devtron/client/argocdServer/repoCredsK8sClient" "github.com/devtron-labs/devtron/client/dashboard" "github.com/devtron-labs/devtron/client/grafana" - "github.com/devtron-labs/devtron/client/telemetry" + telemetry2 "github.com/devtron-labs/devtron/client/telemetry" repository5 "github.com/devtron-labs/devtron/internal/sql/repository" "github.com/devtron-labs/devtron/internal/sql/repository/app" "github.com/devtron-labs/devtron/internal/sql/repository/appStatus" @@ -130,6 +131,7 @@ import ( "github.com/devtron-labs/devtron/pkg/team/read" repository2 "github.com/devtron-labs/devtron/pkg/team/repository" "github.com/devtron-labs/devtron/pkg/terminal" + "github.com/devtron-labs/devtron/pkg/ucid" "github.com/devtron-labs/devtron/pkg/userResource" util3 "github.com/devtron-labs/devtron/pkg/util" "github.com/devtron-labs/devtron/pkg/webhook/helm" @@ -352,7 +354,7 @@ func InitializeApp() (*App, error) { return nil, err } deletePostProcessorImpl := service2.NewDeletePostProcessorImpl(sugaredLogger) - appStoreDeploymentServiceImpl := service2.NewAppStoreDeploymentServiceImpl(sugaredLogger, installedAppRepositoryImpl, installedAppDBServiceImpl, appStoreDeploymentDBServiceImpl, chartGroupDeploymentRepositoryImpl, appStoreApplicationVersionRepositoryImpl, appRepositoryImpl, eaModeDeploymentServiceImpl, eaModeDeploymentServiceImpl, environmentServiceImpl, helmAppServiceImpl, installedAppVersionHistoryRepositoryImpl, environmentVariables, acdConfig, gitOpsConfigReadServiceImpl, deletePostProcessorImpl, appStoreValidatorImpl, deploymentConfigServiceImpl) + appStoreDeploymentServiceImpl := service2.NewAppStoreDeploymentServiceImpl(sugaredLogger, installedAppRepositoryImpl, installedAppDBServiceImpl, appStoreDeploymentDBServiceImpl, chartGroupDeploymentRepositoryImpl, appStoreApplicationVersionRepositoryImpl, appRepositoryImpl, eaModeDeploymentServiceImpl, eaModeDeploymentServiceImpl, environmentServiceImpl, helmAppServiceImpl, installedAppVersionHistoryRepositoryImpl, environmentVariables, acdConfig, gitOpsConfigReadServiceImpl, deletePostProcessorImpl, appStoreValidatorImpl, deploymentConfigServiceImpl, ociRegistryConfigRepositoryImpl) fluxApplicationServiceImpl := fluxApplication.NewFluxApplicationServiceImpl(sugaredLogger, helmAppReadServiceImpl, clusterServiceImpl, helmAppClientImpl, pumpImpl) k8sResourceHistoryRepositoryImpl := repository10.NewK8sResourceHistoryRepositoryImpl(db, sugaredLogger) k8sResourceHistoryServiceImpl := kubernetesResourceAuditLogs.Newk8sResourceHistoryServiceImpl(k8sResourceHistoryRepositoryImpl, sugaredLogger, appRepositoryImpl, environmentRepositoryImpl) @@ -394,9 +396,10 @@ func InitializeApp() (*App, error) { if err != nil { return nil, err } + serviceImpl := ucid.NewServiceImpl(sugaredLogger, k8sServiceImpl, acdAuthConfig) providerIdentifierServiceImpl := providerIdentifier.NewProviderIdentifierServiceImpl(sugaredLogger) userAttributesRepositoryImpl := repository5.NewUserAttributesRepositoryImpl(db) - telemetryEventClientImpl, err := telemetry.NewTelemetryEventClientImpl(sugaredLogger, httpClient, clusterServiceImpl, k8sServiceImpl, acdAuthConfig, userServiceImpl, attributesRepositoryImpl, ssoLoginServiceImpl, posthogClient, moduleRepositoryImpl, serverDataStoreServerDataStore, userAuditServiceImpl, helmAppClientImpl, providerIdentifierServiceImpl, cronLoggerImpl, installedAppReadServiceEAImpl, environmentVariables, userAttributesRepositoryImpl) + telemetryEventClientImpl, err := telemetry2.NewTelemetryEventClientImpl(sugaredLogger, httpClient, clusterServiceImpl, k8sServiceImpl, acdAuthConfig, userServiceImpl, attributesRepositoryImpl, ssoLoginServiceImpl, posthogClient, serviceImpl, moduleRepositoryImpl, serverDataStoreServerDataStore, userAuditServiceImpl, helmAppClientImpl, providerIdentifierServiceImpl, cronLoggerImpl, installedAppReadServiceEAImpl, environmentVariables, userAttributesRepositoryImpl) if err != nil { return nil, err } diff --git a/go.mod b/go.mod index 7b92d82d38..0e23cf5e03 100644 --- a/go.mod +++ b/go.mod @@ -32,7 +32,7 @@ require ( github.com/go-pg/pg v6.15.1+incompatible github.com/go-resty/resty/v2 v2.7.0 github.com/gogo/protobuf v1.3.2 - github.com/golang-jwt/jwt/v4 v4.5.1 + github.com/golang-jwt/jwt/v4 v4.5.2 github.com/golang/mock v1.6.0 github.com/golang/protobuf v1.5.4 github.com/google/cel-go v0.17.8 @@ -307,8 +307,8 @@ require ( replace ( github.com/argoproj/argo-workflows/v3 v3.5.13 => github.com/devtron-labs/argo-workflows/v3 v3.5.13 - github.com/devtron-labs/authenticator => github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250421131910-ad3aa9bb920e - github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250421131910-ad3aa9bb920e + github.com/devtron-labs/authenticator => github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250422075757-fe1e5dd1b92b + github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250422075757-fe1e5dd1b92b github.com/go-check/check => github.com/go-check/check v0.0.0-20180628173108-788fd7840127 github.com/googleapis/gnostic => github.com/googleapis/gnostic v0.5.5 k8s.io/api => k8s.io/api v0.29.7 diff --git a/go.sum b/go.sum index 328d6622fa..2824784a2a 100644 --- a/go.sum +++ b/go.sum @@ -829,10 +829,10 @@ github.com/denisenkom/go-mssqldb v0.0.0-20200428022330-06a60b6afbbc h1:VRRKCwnzq github.com/denisenkom/go-mssqldb v0.0.0-20200428022330-06a60b6afbbc/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= github.com/devtron-labs/argo-workflows/v3 v3.5.13 h1:3pINq0gXOSeTw2z/vYe+j80lRpSN5Rp/8mfQORh8SmU= github.com/devtron-labs/argo-workflows/v3 v3.5.13/go.mod h1:/vqxcovDPT4zqr4DjR5v7CF8ggpY1l3TSa2CIG3jmjA= -github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250421131910-ad3aa9bb920e h1:6/RNcoG39xH35ZHb024ERluk2HH3sf+uNWDR5AHb4xs= -github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250421131910-ad3aa9bb920e/go.mod h1:5lv4Wfj5ERhhvDGXe2IeES6qxjvUVCcohaRwKnWBMNo= -github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250421131910-ad3aa9bb920e h1:YkUbwbsa5phIo0ikGvkCOHigF6IQZPfUTE9WcaF4vaM= -github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250421131910-ad3aa9bb920e/go.mod h1:ceFKgQ2qm40PR95g5Xp2EClq7nDBKFTcglJ0JdsgClA= +github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250422075757-fe1e5dd1b92b h1:b6AkVK1t2NNJcCusTF3w5UN9OwFMS6XEn7MUsSkraTY= +github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250422075757-fe1e5dd1b92b/go.mod h1:FfaLDXN1ZXxyRpnskBqVIYkpkWDCzBmDgIO9xqLnxdQ= +github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250422075757-fe1e5dd1b92b h1:PyI1pxuq7KiSbM2ObApJBcMSRkjXkMfNBZGzy9EJGQI= +github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250422075757-fe1e5dd1b92b/go.mod h1:zkNShlkcHxsmnL0gKNbs0uyRL8lZonGKr5Km63uTLI0= github.com/devtron-labs/go-bitbucket v0.9.60-beta h1:VEx1jvDgdtDPS6A1uUFoaEi0l1/oLhbr+90xOwr6sDU= github.com/devtron-labs/go-bitbucket v0.9.60-beta/go.mod h1:GnuiCesvh8xyHeMCb+twm8lBR/kQzJYSKL28ZfObp1Y= github.com/devtron-labs/protos v0.0.3-0.20250323220609-ecf8a0f7305e h1:U6UdYbW8a7xn5IzFPd8cywjVVPfutGJCudjePAfL/Hs= @@ -993,8 +993,8 @@ github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= -github.com/golang-jwt/jwt/v4 v4.5.1 h1:JdqV9zKUdtaa9gdPlywC3aeoEsR681PlKC+4F5gQgeo= -github.com/golang-jwt/jwt/v4 v4.5.1/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= +github.com/golang-jwt/jwt/v4 v4.5.2 h1:YtQM7lnr8iZ+j5q71MGKkNw9Mn7AjHM68uc9g5fXeUI= +github.com/golang-jwt/jwt/v4 v4.5.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe h1:lXe2qZdvpiX5WZkZR4hgp4KJVfY3nMkvmwbVkpv1rVY= github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= diff --git a/internal/middleware/instrument.go b/internal/middleware/instrument.go index 0bbb8693e3..7fcc0d9ebf 100644 --- a/internal/middleware/instrument.go +++ b/internal/middleware/instrument.go @@ -117,6 +117,11 @@ var TerminalSessionDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ Help: "duration of each terminal session", }, []string{"podName", "namespace", "clusterId"}) +var ReTriggerFailedCounter = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "re_trigger_failed_counter", + Help: "ci/ pre cd/ post cd workflow re-trigger failed counter", +}, []string{"workflowType", "workflowId"}) + // prometheusMiddleware implements mux.MiddlewareFunc. func PrometheusMiddleware(next http.Handler) http.Handler { // prometheus.MustRegister(requestCounter) diff --git a/internal/sql/repository/pipelineConfig/CIPipelineRepository_ent.go b/internal/sql/repository/pipelineConfig/CIPipelineRepository_ent.go new file mode 100644 index 0000000000..3a1538c1f8 --- /dev/null +++ b/internal/sql/repository/pipelineConfig/CIPipelineRepository_ent.go @@ -0,0 +1,8 @@ +package pipelineConfig + +import "github.com/devtron-labs/devtron/pkg/bean/common" + +func (p *CiPipeline) GetWorkflowCacheConfig() common.WorkflowCacheConfigType { + //in oss, there is no pipeline level workflow cache config, so we pass inherit to get the app level config + return common.WorkflowCacheConfigInherit +} diff --git a/internal/sql/repository/pipelineConfig/CdWorfkflowRepository.go b/internal/sql/repository/pipelineConfig/CdWorfkflowRepository.go index a737a7076a..9569be90c6 100644 --- a/internal/sql/repository/pipelineConfig/CdWorfkflowRepository.go +++ b/internal/sql/repository/pipelineConfig/CdWorfkflowRepository.go @@ -20,7 +20,6 @@ import ( "context" "errors" apiBean "github.com/devtron-labs/devtron/api/bean" - "github.com/devtron-labs/devtron/client/gitSensor" "github.com/devtron-labs/devtron/internal/sql/repository" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow/cdWorkflow" @@ -50,6 +49,7 @@ type CdWorkflowRepository interface { FindWorkflowRunnerByCdWorkflowId(wfIds []int) ([]*CdWorkflowRunner, error) FindPreviousCdWfRunnerByStatus(pipelineId int, currentWFRunnerId int, status []string) ([]*CdWorkflowRunner, error) FindWorkflowRunnerById(wfrId int) (*CdWorkflowRunner, error) + FindPreOrPostCdWorkflowRunnerById(wfrId int) (*CdWorkflowRunner, error) FindBasicWorkflowRunnerById(wfrId int) (*CdWorkflowRunner, error) FindRetriedWorkflowCountByReferenceId(wfrId int) (int, error) FindLatestWfrByAppIdAndEnvironmentId(appId int, environmentId int) (*CdWorkflowRunner, error) @@ -149,24 +149,6 @@ func (c *CdWorkflowRunner) IsExternalRun() bool { return isExtCluster } -type CiPipelineMaterialResponse struct { - Id int `json:"id"` - GitMaterialId int `json:"gitMaterialId"` - GitMaterialUrl string `json:"gitMaterialUrl"` - GitMaterialName string `json:"gitMaterialName"` - Type string `json:"type"` - Value string `json:"value"` - Active bool `json:"active"` - History []*gitSensor.GitCommit `json:"history,omitempty"` - LastFetchTime time.Time `json:"lastFetchTime"` - IsRepoError bool `json:"isRepoError"` - RepoErrorMsg string `json:"repoErrorMsg"` - IsBranchError bool `json:"isBranchError"` - BranchErrorMsg string `json:"branchErrorMsg"` - Url string `json:"url"` - Regex string `json:"regex"` -} - type TriggerWorkflowStatus struct { CdWorkflowStatus []*CdWorkflowStatus `json:"cdWorkflowStatus"` CiWorkflowStatus []*CiWorkflowStatus `json:"ciWorkflowStatus"` @@ -457,7 +439,6 @@ func (impl *CdWorkflowRepositoryImpl) SaveWorkFlowRunnerWithTx(wfr *CdWorkflowRu } func (impl *CdWorkflowRepositoryImpl) UpdateWorkFlowRunnerWithTx(wfr *CdWorkflowRunner, tx *pg.Tx) error { - wfr.Message = util.GetTruncatedMessage(wfr.Message, 1000) err := tx.Update(wfr) return err } @@ -528,6 +509,16 @@ func (impl *CdWorkflowRepositoryImpl) FindWorkflowRunnerById(wfrId int) (*CdWork return wfr, err } +func (impl *CdWorkflowRepositoryImpl) FindPreOrPostCdWorkflowRunnerById(wfrId int) (*CdWorkflowRunner, error) { + wfr := &CdWorkflowRunner{} + err := impl.dbConnection.Model(wfr). + Column("cd_workflow_runner.*", "CdWorkflow", "CdWorkflow.Pipeline", "CdWorkflow.CiArtifact", "CdWorkflow.Pipeline.Environment"). + Where("cd_workflow_runner.id = ?", wfrId). + Where("cd_workflow_runner.workflow_type != ?", apiBean.CD_WORKFLOW_TYPE_DEPLOY). + Select() + return wfr, err +} + func (impl *CdWorkflowRepositoryImpl) FindBasicWorkflowRunnerById(wfrId int) (*CdWorkflowRunner, error) { wfr := &CdWorkflowRunner{} err := impl.dbConnection.Model(wfr). diff --git a/internal/sql/repository/pipelineConfig/CiPipelineRepository.go b/internal/sql/repository/pipelineConfig/CiPipelineRepository.go index fb90f43255..c6adb8b7a8 100644 --- a/internal/sql/repository/pipelineConfig/CiPipelineRepository.go +++ b/internal/sql/repository/pipelineConfig/CiPipelineRepository.go @@ -21,8 +21,8 @@ import ( "fmt" "github.com/devtron-labs/devtron/internal/sql/repository/app" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/ciPipeline" + buildCommonBean "github.com/devtron-labs/devtron/pkg/build/pipeline/bean/common" repository2 "github.com/devtron-labs/devtron/pkg/cluster/environment/repository" - "github.com/devtron-labs/devtron/pkg/pipeline/constants" "github.com/devtron-labs/devtron/pkg/sql" "github.com/devtron-labs/devtron/util/response/pagination" "github.com/go-pg/pg" @@ -641,7 +641,7 @@ func (impl *CiPipelineRepositoryImpl) FindLinkedCiCount(ciPipelineId int) (int, pipeline := &CiPipeline{} cnt, err := impl.dbConnection.Model(pipeline). Where("parent_ci_pipeline = ?", ciPipelineId). - Where("ci_pipeline_type != ?", constants.LINKED_CD). + Where("ci_pipeline_type != ?", buildCommonBean.LINKED_CD). Where("deleted = ?", false). Count() if err == pg.ErrNoRows { @@ -656,7 +656,7 @@ func (impl *CiPipelineRepositoryImpl) GetLinkedCiPipelines(ctx context.Context, var linkedCIPipelines []*CiPipeline err := impl.dbConnection.Model(&linkedCIPipelines). Where("parent_ci_pipeline = ?", ciPipelineId). - Where("ci_pipeline_type != ?", constants.LINKED_CD). + Where("ci_pipeline_type != ?", buildCommonBean.LINKED_CD). Where("deleted = ?", false). Select() if err != nil { @@ -694,7 +694,7 @@ func (impl *CiPipelineRepositoryImpl) GetDownStreamInfo(ctx context.Context, sou JoinOn("e.active = ?", true). // constrains Where("ci_pipeline.parent_ci_pipeline = ?", sourceCiPipelineId). - Where("ci_pipeline.ci_pipeline_type != ?", constants.LINKED_CD). + Where("ci_pipeline.ci_pipeline_type != ?", buildCommonBean.LINKED_CD). Where("ci_pipeline.deleted = ?", false) // app name filtering with lower case if len(appNameMatch) != 0 { diff --git a/internal/sql/repository/pipelineConfig/bean/cdWorkflow/bean.go b/internal/sql/repository/pipelineConfig/bean/cdWorkflow/bean.go deleted file mode 100644 index 645ef52eb2..0000000000 --- a/internal/sql/repository/pipelineConfig/bean/cdWorkflow/bean.go +++ /dev/null @@ -1,54 +0,0 @@ -package cdWorkflow - -import ( - "errors" - "github.com/devtron-labs/common-lib/utils/k8s/health" - "github.com/devtron-labs/devtron/client/argocdServer/bean" -) - -var WfrTerminalStatusList = []string{WorkflowAborted, WorkflowFailed, WorkflowSucceeded, bean.HIBERNATING, string(health.HealthStatusHealthy), string(health.HealthStatusDegraded)} - -type WorkflowStatus int - -const ( - WF_UNKNOWN WorkflowStatus = iota - REQUEST_ACCEPTED - ENQUEUED - QUE_ERROR - WF_STARTED - DROPPED_STALE - DEQUE_ERROR - TRIGGER_ERROR -) - -const ( - WorkflowStarting = "Starting" - WorkflowInQueue = "Queued" - WorkflowInitiated = "Initiating" - WorkflowInProgress = "Progressing" - WorkflowAborted = "Aborted" - WorkflowFailed = "Failed" - WorkflowSucceeded = "Succeeded" - WorkflowTimedOut = "TimedOut" - WorkflowUnableToFetchState = "UnableToFetch" - WorkflowTypeDeploy = "DEPLOY" - WorkflowTypePre = "PRE" - WorkflowTypePost = "POST" -) - -func (a WorkflowStatus) String() string { - return [...]string{"WF_UNKNOWN", "REQUEST_ACCEPTED", "ENQUEUED", "QUE_ERROR", "WF_STARTED", "DROPPED_STALE", "DEQUE_ERROR", "TRIGGER_ERROR"}[a] -} - -var ErrorDeploymentSuperseded = errors.New(NEW_DEPLOYMENT_INITIATED) - -const ( - WORKFLOW_EXECUTOR_TYPE_AWF = "AWF" - WORKFLOW_EXECUTOR_TYPE_SYSTEM = "SYSTEM" - NEW_DEPLOYMENT_INITIATED = "A new deployment was initiated before this deployment completed!" - PIPELINE_DELETED = "The pipeline has been deleted!" - FOUND_VULNERABILITY = "Found vulnerability on image" - GITOPS_REPO_NOT_CONFIGURED = "GitOps repository is not configured for the app" -) - -type WorkflowExecutorType string diff --git a/pkg/app/AppListingService.go b/pkg/app/AppListingService.go index 48c5728c91..a5d5654ff2 100644 --- a/pkg/app/AppListingService.go +++ b/pkg/app/AppListingService.go @@ -26,13 +26,13 @@ import ( "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow/cdWorkflow" read4 "github.com/devtron-labs/devtron/pkg/app/appDetails/read" userrepository "github.com/devtron-labs/devtron/pkg/auth/user/repository" + buildCommonBean "github.com/devtron-labs/devtron/pkg/build/pipeline/bean/common" ciConfig "github.com/devtron-labs/devtron/pkg/build/pipeline/read" chartRepoRepository "github.com/devtron-labs/devtron/pkg/chartRepo/repository" repository2 "github.com/devtron-labs/devtron/pkg/cluster/environment/repository" "github.com/devtron-labs/devtron/pkg/deployment/manifest/deployedAppMetrics" "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/read" "github.com/devtron-labs/devtron/pkg/dockerRegistry" - "github.com/devtron-labs/devtron/pkg/pipeline/constants" errors2 "github.com/juju/errors" "go.opentelemetry.io/otel" "golang.org/x/exp/slices" @@ -652,7 +652,7 @@ func (impl AppListingServiceImpl) setIpAccessProvidedData(ctx context.Context, a } if ciPipeline != nil && ciPipeline.CiTemplate != nil && len(*ciPipeline.CiTemplate.DockerRegistryId) > 0 { - if !ciPipeline.IsExternal || ciPipeline.ParentCiPipeline != 0 && ciPipeline.PipelineType != string(constants.LINKED_CD) { + if !ciPipeline.IsExternal || ciPipeline.ParentCiPipeline != 0 && ciPipeline.PipelineType != string(buildCommonBean.LINKED_CD) { appDetailContainer.IsExternalCi = false } // get dockerRegistryId starts diff --git a/pkg/appStore/installedApp/service/FullMode/deployment/InstalledAppArgoCdService.go b/pkg/appStore/installedApp/service/FullMode/deployment/InstalledAppArgoCdService.go index 9b637360f0..05c6b07ef2 100644 --- a/pkg/appStore/installedApp/service/FullMode/deployment/InstalledAppArgoCdService.go +++ b/pkg/appStore/installedApp/service/FullMode/deployment/InstalledAppArgoCdService.go @@ -25,7 +25,7 @@ import ( "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/timelineStatus" "github.com/devtron-labs/devtron/internal/util" appStoreBean "github.com/devtron-labs/devtron/pkg/appStore/bean" - cluster2 "github.com/devtron-labs/devtron/pkg/cluster" + bean2 "github.com/devtron-labs/devtron/pkg/cluster/bean" "github.com/devtron-labs/devtron/pkg/cluster/environment/bean" commonBean "github.com/devtron-labs/devtron/pkg/deployment/gitOps/common/bean" util2 "github.com/devtron-labs/devtron/util" @@ -167,7 +167,7 @@ func (impl *FullModeDeploymentServiceImpl) DeleteACD(acdAppName string, ctx cont func (impl *FullModeDeploymentServiceImpl) createInArgo(ctx context.Context, chartGitAttribute *commonBean.ChartGitAttribute, envModel bean.EnvironmentBean, argocdAppName string) error { appNamespace := envModel.Namespace if appNamespace == "" { - appNamespace = cluster2.DEFAULT_NAMESPACE + appNamespace = bean2.DefaultNamespace } appReq := &argocdServer.AppTemplate{ ApplicationName: argocdAppName, diff --git a/pkg/bean/app.go b/pkg/bean/app.go index 96c581b5f6..83c6717093 100644 --- a/pkg/bean/app.go +++ b/pkg/bean/app.go @@ -29,6 +29,7 @@ import ( "github.com/devtron-labs/devtron/internal/util" "github.com/devtron-labs/devtron/pkg/bean/common" CiPipeline2 "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" + common2 "github.com/devtron-labs/devtron/pkg/build/pipeline/bean/common" "github.com/devtron-labs/devtron/pkg/chartRepo/repository" bean3 "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/bean" "github.com/devtron-labs/devtron/pkg/pipeline/bean" @@ -117,39 +118,39 @@ type CiMaterial struct { } type CiPipeline struct { - IsManual bool `json:"isManual"` - DockerArgs map[string]string `json:"dockerArgs"` - IsExternal bool `json:"isExternal"` - ParentCiPipeline int `json:"parentCiPipeline"` - ParentAppId int `json:"parentAppId"` - AppId int `json:"appId"` - AppName string `json:"appName,omitempty"` - AppType helper.AppType `json:"appType,omitempty"` - ExternalCiConfig ExternalCiConfig `json:"externalCiConfig"` - CiMaterial []*CiMaterial `json:"ciMaterial,omitempty" validate:"dive,min=1"` - Name string `json:"name,omitempty" validate:"name-component,max=100"` // name suffix of corresponding pipeline. required, unique, validation corresponding to gocd pipelineName will be applicable - Id int `json:"id,omitempty" ` - Version string `json:"version,omitempty"` // matchIf token version in gocd . used for update request - Active bool `json:"active,omitempty"` // pipeline is active or not - Deleted bool `json:"deleted,omitempty"` - BeforeDockerBuild []*Task `json:"beforeDockerBuild,omitempty" validate:"dive"` - AfterDockerBuild []*Task `json:"afterDockerBuild,omitempty" validate:"dive"` - BeforeDockerBuildScripts []*CiScript `json:"beforeDockerBuildScripts,omitempty" validate:"dive"` - AfterDockerBuildScripts []*CiScript `json:"afterDockerBuildScripts,omitempty" validate:"dive"` - LinkedCount int `json:"linkedCount"` - PipelineType CiPipeline2.PipelineType `json:"pipelineType,omitempty"` - ScanEnabled bool `json:"scanEnabled,notnull"` - AppWorkflowId int `json:"appWorkflowId,omitempty"` - PreBuildStage *bean.PipelineStageDto `json:"preBuildStage,omitempty" validate:"omitempty,dive"` - PostBuildStage *bean.PipelineStageDto `json:"postBuildStage,omitempty" validate:"omitempty,dive"` - TargetPlatform string `json:"targetPlatform,omitempty"` - IsDockerConfigOverridden bool `json:"isDockerConfigOverridden"` - DockerConfigOverride DockerConfigOverride `json:"dockerConfigOverride,omitempty"` - EnvironmentId int `json:"environmentId,omitempty"` - LastTriggeredEnvId int `json:"lastTriggeredEnvId"` - CustomTagObject *CustomTagData `json:"customTag,omitempty"` - DefaultTag []string `json:"defaultTag,omitempty"` - EnableCustomTag bool `json:"enableCustomTag"` + IsManual bool `json:"isManual"` + DockerArgs map[string]string `json:"dockerArgs"` + IsExternal bool `json:"isExternal"` + ParentCiPipeline int `json:"parentCiPipeline"` + ParentAppId int `json:"parentAppId"` + AppId int `json:"appId"` + AppName string `json:"appName,omitempty"` + AppType helper.AppType `json:"appType,omitempty"` + ExternalCiConfig ExternalCiConfig `json:"externalCiConfig"` + CiMaterial []*CiMaterial `json:"ciMaterial,omitempty" validate:"dive,min=1"` + Name string `json:"name,omitempty" validate:"name-component,max=100"` // name suffix of corresponding pipeline. required, unique, validation corresponding to gocd pipelineName will be applicable + Id int `json:"id,omitempty" ` + Version string `json:"version,omitempty"` // matchIf token version in gocd . used for update request + Active bool `json:"active,omitempty"` // pipeline is active or not + Deleted bool `json:"deleted,omitempty"` + BeforeDockerBuild []*Task `json:"beforeDockerBuild,omitempty" validate:"dive"` + AfterDockerBuild []*Task `json:"afterDockerBuild,omitempty" validate:"dive"` + BeforeDockerBuildScripts []*CiScript `json:"beforeDockerBuildScripts,omitempty" validate:"dive"` + AfterDockerBuildScripts []*CiScript `json:"afterDockerBuildScripts,omitempty" validate:"dive"` + LinkedCount int `json:"linkedCount"` + PipelineType common2.PipelineType `json:"pipelineType,omitempty"` + ScanEnabled bool `json:"scanEnabled,notnull"` + AppWorkflowId int `json:"appWorkflowId,omitempty"` + PreBuildStage *bean.PipelineStageDto `json:"preBuildStage,omitempty" validate:"omitempty,dive"` + PostBuildStage *bean.PipelineStageDto `json:"postBuildStage,omitempty" validate:"omitempty,dive"` + TargetPlatform string `json:"targetPlatform,omitempty"` + IsDockerConfigOverridden bool `json:"isDockerConfigOverridden"` + DockerConfigOverride DockerConfigOverride `json:"dockerConfigOverride,omitempty"` + EnvironmentId int `json:"environmentId,omitempty"` + LastTriggeredEnvId int `json:"lastTriggeredEnvId"` + CustomTagObject *CustomTagData `json:"customTag,omitempty"` + DefaultTag []string `json:"defaultTag,omitempty"` + EnableCustomTag bool `json:"enableCustomTag"` } func (ciPipeline *CiPipeline) IsLinkedCi() bool { @@ -164,14 +165,14 @@ type DockerConfigOverride struct { } type CiPipelineMin struct { - Name string `json:"name,omitempty" validate:"name-component,max=100"` //name suffix of corresponding pipeline. required, unique, validation corresponding to gocd pipelineName will be applicable - Id int `json:"id,omitempty" ` - Version string `json:"version,omitempty"` //matchIf token version in gocd . used for update request - IsExternal bool `json:"isExternal,omitempty"` - ParentCiPipeline int `json:"parentCiPipeline"` - ParentAppId int `json:"parentAppId"` - PipelineType CiPipeline2.PipelineType `json:"pipelineType,omitempty"` - ScanEnabled bool `json:"scanEnabled,notnull"` + Name string `json:"name,omitempty" validate:"name-component,max=100"` //name suffix of corresponding pipeline. required, unique, validation corresponding to gocd pipelineName will be applicable + Id int `json:"id,omitempty" ` + Version string `json:"version,omitempty"` //matchIf token version in gocd . used for update request + IsExternal bool `json:"isExternal,omitempty"` + ParentCiPipeline int `json:"parentCiPipeline"` + ParentAppId int `json:"parentAppId"` + PipelineType common2.PipelineType `json:"pipelineType,omitempty"` + ScanEnabled bool `json:"scanEnabled,notnull"` } type CiScript struct { @@ -303,20 +304,20 @@ type CiPatchRequest struct { IsJob bool `json:"-"` IsCloneJob bool `json:"isCloneJob,omitempty"` - ParentCDPipeline int `json:"parentCDPipeline"` - DeployEnvId int `json:"deployEnvId"` - SwitchFromCiPipelineId int `json:"switchFromCiPipelineId"` - SwitchFromExternalCiPipelineId int `json:"switchFromExternalCiPipelineId"` - SwitchFromCiPipelineType CiPipeline2.PipelineType `json:"-"` - SwitchToCiPipelineType CiPipeline2.PipelineType `json:"-"` + ParentCDPipeline int `json:"parentCDPipeline"` + DeployEnvId int `json:"deployEnvId"` + SwitchFromCiPipelineId int `json:"switchFromCiPipelineId"` + SwitchFromExternalCiPipelineId int `json:"switchFromExternalCiPipelineId"` + SwitchFromCiPipelineType common2.PipelineType `json:"-"` + SwitchToCiPipelineType common2.PipelineType `json:"-"` } -func (ciPatchRequest CiPatchRequest) SwitchSourceInfo() (int, CiPipeline2.PipelineType) { +func (ciPatchRequest CiPatchRequest) SwitchSourceInfo() (int, common2.PipelineType) { // get the ciPipeline - var switchFromType CiPipeline2.PipelineType + var switchFromType common2.PipelineType var switchFromPipelineId int if ciPatchRequest.SwitchFromExternalCiPipelineId != 0 { - switchFromType = CiPipeline2.EXTERNAL + switchFromType = common2.EXTERNAL switchFromPipelineId = ciPatchRequest.SwitchFromExternalCiPipelineId } else { switchFromPipelineId = ciPatchRequest.SwitchFromCiPipelineId diff --git a/pkg/build/git/gitWebhook/GitWebhookService.go b/pkg/build/git/gitWebhook/GitWebhookService.go index 3f301db311..7ef9e0edc6 100644 --- a/pkg/build/git/gitWebhook/GitWebhookService.go +++ b/pkg/build/git/gitWebhook/GitWebhookService.go @@ -23,7 +23,7 @@ import ( bean2 "github.com/devtron-labs/devtron/pkg/auth/user/bean" "github.com/devtron-labs/devtron/pkg/bean" "github.com/devtron-labs/devtron/pkg/build/git/gitWebhook/repository" - "github.com/devtron-labs/devtron/pkg/pipeline" + "github.com/devtron-labs/devtron/pkg/build/trigger" "go.uber.org/zap" ) @@ -33,15 +33,16 @@ type GitWebhookService interface { type GitWebhookServiceImpl struct { logger *zap.SugaredLogger - ciHandler pipeline.CiHandler gitWebhookRepository repository.GitWebhookRepository + ciHandlerService trigger.HandlerService } -func NewGitWebhookServiceImpl(Logger *zap.SugaredLogger, ciHandler pipeline.CiHandler, gitWebhookRepository repository.GitWebhookRepository) *GitWebhookServiceImpl { +func NewGitWebhookServiceImpl(Logger *zap.SugaredLogger, gitWebhookRepository repository.GitWebhookRepository, + ciHandlerService trigger.HandlerService) *GitWebhookServiceImpl { return &GitWebhookServiceImpl{ logger: Logger, - ciHandler: ciHandler, gitWebhookRepository: gitWebhookRepository, + ciHandlerService: ciHandlerService, } } @@ -70,7 +71,7 @@ func (impl *GitWebhookServiceImpl) HandleGitWebhook(gitWebhookRequest gitSensor. } } - resp, err := impl.ciHandler.HandleCIWebhook(bean.GitCiTriggerRequest{ + resp, err := impl.ciHandlerService.HandleCIWebhook(bean.GitCiTriggerRequest{ CiPipelineMaterial: ciPipelineMaterial, TriggeredBy: bean2.SYSTEM_USER_ID, // Automatic trigger, system user ExtraEnvironmentVariables: gitWebhookRequest.ExtraEnvironmentVariables, diff --git a/pkg/build/pipeline/bean/CIPipelineMaterialBean.go b/pkg/build/pipeline/bean/CIPipelineMaterialBean.go new file mode 100644 index 0000000000..8f09c47ebe --- /dev/null +++ b/pkg/build/pipeline/bean/CIPipelineMaterialBean.go @@ -0,0 +1,30 @@ +package bean + +import ( + "github.com/devtron-labs/devtron/client/gitSensor" + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" + "time" +) + +type CiPipelineMaterialResponse struct { + Id int `json:"id"` + GitMaterialId int `json:"gitMaterialId"` + GitMaterialUrl string `json:"gitMaterialUrl"` + GitMaterialName string `json:"gitMaterialName"` + Type string `json:"type"` + Value string `json:"value"` + Active bool `json:"active"` + History []*gitSensor.GitCommit `json:"history,omitempty"` + LastFetchTime time.Time `json:"lastFetchTime"` + IsRepoError bool `json:"isRepoError"` + RepoErrorMsg string `json:"repoErrorMsg"` + IsBranchError bool `json:"isBranchError"` + BranchErrorMsg string `json:"branchErrorMsg"` + Url string `json:"url"` + Regex string `json:"regex"` +} + +type MaterialTriggerInfo struct { + GitTriggers map[int]pipelineConfig.GitCommit `json:"gitTriggers"` + CiMaterials []CiPipelineMaterialResponse `json:"ciMaterials"` +} diff --git a/pkg/build/pipeline/bean/CiBuildConfig.go b/pkg/build/pipeline/bean/CiBuildConfig.go index 34357f5e68..39d8671a35 100644 --- a/pkg/build/pipeline/bean/CiBuildConfig.go +++ b/pkg/build/pipeline/bean/CiBuildConfig.go @@ -30,23 +30,6 @@ const UniquePlaceHolderForAppName = "$etron" const PIPELINE_NAME_ALREADY_EXISTS_ERROR = "pipeline name already exist" const PIPELINE_TYPE_IS_NOT_VALID = "PipelineType is not valid for pipeline %s" -type PipelineType string - -// default PipelineType -const DefaultPipelineType = CI_BUILD - -const ( - CI_BUILD PipelineType = "CI_BUILD" - LINKED PipelineType = "LINKED" - EXTERNAL PipelineType = "EXTERNAL" - CI_JOB PipelineType = "CI_JOB" - LINKED_CD PipelineType = "LINKED_CD" -) - -func (pType PipelineType) ToString() string { - return string(pType) -} - type CiBuildConfigBean struct { Id int `json:"id"` GitMaterialId int `json:"gitMaterialId,omitempty" validate:"required"` @@ -81,15 +64,6 @@ type BuildPackConfig struct { ProjectPath string `json:"projectPath,omitempty"` } -func (pType PipelineType) IsValidPipelineType() bool { - switch pType { - case CI_BUILD, LINKED, EXTERNAL, CI_JOB, LINKED_CD: - return true - default: - return false - } -} - const ( ExtraEnvVarExternalCiArtifactKey = "externalCiArtifact" ExtraEnvVarImageDigestKey = "imageDigest" diff --git a/pkg/build/pipeline/bean/common/CommonBuildBean.go b/pkg/build/pipeline/bean/common/CommonBuildBean.go new file mode 100644 index 0000000000..81692c377a --- /dev/null +++ b/pkg/build/pipeline/bean/common/CommonBuildBean.go @@ -0,0 +1,40 @@ +package common + +type PipelineType string + +// default PipelineType +const DefaultPipelineType = CI_BUILD + +const ( + CI_BUILD PipelineType = "CI_BUILD" + LINKED PipelineType = "LINKED" + EXTERNAL PipelineType = "EXTERNAL" + CI_JOB PipelineType = "CI_JOB" + LINKED_CD PipelineType = "LINKED_CD" +) + +func (pType PipelineType) ToString() string { + return string(pType) +} + +func (pType PipelineType) IsValidPipelineType() bool { + switch pType { + case CI_BUILD, LINKED, EXTERNAL, CI_JOB, LINKED_CD: + return true + default: + return false + } +} + +const ( + BEFORE_DOCKER_BUILD = "BEFORE_DOCKER_BUILD" + AFTER_DOCKER_BUILD = "AFTER_DOCKER_BUILD" +) + +type RefPluginName = string + +const ( + COPY_CONTAINER_IMAGE RefPluginName = "Copy container image" + COPY_CONTAINER_IMAGE_VERSION_V1 = "1.0.0" + COPY_CONTAINER_IMAGE_VERSION_V2 = "2.0.0" +) diff --git a/pkg/build/trigger/HandlerService.go b/pkg/build/trigger/HandlerService.go new file mode 100644 index 0000000000..7b5832ffa4 --- /dev/null +++ b/pkg/build/trigger/HandlerService.go @@ -0,0 +1,1957 @@ +package trigger + +import ( + "bufio" + "context" + "errors" + "fmt" + "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" + "github.com/caarlos0/env" + blob_storage "github.com/devtron-labs/common-lib/blob-storage" + "github.com/devtron-labs/common-lib/utils" + bean4 "github.com/devtron-labs/common-lib/utils/bean" + "github.com/devtron-labs/common-lib/utils/k8s" + commonBean "github.com/devtron-labs/common-lib/workflow" + client "github.com/devtron-labs/devtron/client/events" + "github.com/devtron-labs/devtron/client/gitSensor" + "github.com/devtron-labs/devtron/internal/middleware" + "github.com/devtron-labs/devtron/internal/sql/constants" + repository5 "github.com/devtron-labs/devtron/internal/sql/repository" + appRepository "github.com/devtron-labs/devtron/internal/sql/repository/app" + repository3 "github.com/devtron-labs/devtron/internal/sql/repository/dockerRegistry" + "github.com/devtron-labs/devtron/internal/sql/repository/helper" + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow/cdWorkflow" + "github.com/devtron-labs/devtron/internal/util" + "github.com/devtron-labs/devtron/pkg/app" + "github.com/devtron-labs/devtron/pkg/attributes" + bean3 "github.com/devtron-labs/devtron/pkg/attributes/bean" + "github.com/devtron-labs/devtron/pkg/auth/user" + bean6 "github.com/devtron-labs/devtron/pkg/auth/user/bean" + "github.com/devtron-labs/devtron/pkg/bean" + "github.com/devtron-labs/devtron/pkg/bean/common" + "github.com/devtron-labs/devtron/pkg/build/pipeline" + buildBean "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" + buildCommonBean "github.com/devtron-labs/devtron/pkg/build/pipeline/bean/common" + "github.com/devtron-labs/devtron/pkg/cluster" + adapter2 "github.com/devtron-labs/devtron/pkg/cluster/adapter" + clusterBean "github.com/devtron-labs/devtron/pkg/cluster/bean" + "github.com/devtron-labs/devtron/pkg/cluster/environment" + repository6 "github.com/devtron-labs/devtron/pkg/cluster/environment/repository" + eventProcessorBean "github.com/devtron-labs/devtron/pkg/eventProcessor/bean" + "github.com/devtron-labs/devtron/pkg/executor" + pipeline2 "github.com/devtron-labs/devtron/pkg/pipeline" + "github.com/devtron-labs/devtron/pkg/pipeline/adapter" + pipelineConfigBean "github.com/devtron-labs/devtron/pkg/pipeline/bean" + constants2 "github.com/devtron-labs/devtron/pkg/pipeline/constants" + "github.com/devtron-labs/devtron/pkg/pipeline/executors" + "github.com/devtron-labs/devtron/pkg/pipeline/repository" + "github.com/devtron-labs/devtron/pkg/pipeline/types" + util2 "github.com/devtron-labs/devtron/pkg/pipeline/util" + "github.com/devtron-labs/devtron/pkg/plugin" + bean2 "github.com/devtron-labs/devtron/pkg/plugin/bean" + repository2 "github.com/devtron-labs/devtron/pkg/plugin/repository" + "github.com/devtron-labs/devtron/pkg/resourceQualifiers" + "github.com/devtron-labs/devtron/pkg/variables" + repository4 "github.com/devtron-labs/devtron/pkg/variables/repository" + "github.com/devtron-labs/devtron/util/sliceUtil" + "github.com/go-pg/pg" + "go.uber.org/zap" + "io/ioutil" + "k8s.io/client-go/rest" + "net/http" + "os" + "path" + "path/filepath" + "slices" + "strconv" + "strings" + "time" +) + +type HandlerService interface { + HandlePodDeleted(ciWorkflow *pipelineConfig.CiWorkflow) + CheckAndReTriggerCI(workflowStatus eventProcessorBean.CiCdStatus) error + HandleCIManual(ciTriggerRequest bean.CiTriggerRequest) (int, error) + HandleCIWebhook(gitCiTriggerRequest bean.GitCiTriggerRequest) (int, error) + + StartCiWorkflowAndPrepareWfRequest(trigger types.Trigger) (*pipelineConfig.CiPipeline, map[string]string, *pipelineConfig.CiWorkflow, *types.WorkflowRequest, error) + + CancelBuild(workflowId int, forceAbort bool) (int, error) + GetRunningWorkflowLogs(workflowId int) (*bufio.Reader, func() error, error) + GetHistoricBuildLogs(workflowId int, ciWorkflow *pipelineConfig.CiWorkflow) (map[string]string, error) + DownloadCiWorkflowArtifacts(pipelineId int, buildId int) (*os.File, error) +} + +type BuildxCacheFlags struct { + BuildxCacheModeMin bool `env:"BUILDX_CACHE_MODE_MIN" envDefault:"false" description:"To set build cache mode to minimum in buildx" ` + AsyncBuildxCacheExport bool `env:"ASYNC_BUILDX_CACHE_EXPORT" envDefault:"false" description:"To enable async container image cache export"` +} + +type HandlerServiceImpl struct { + Logger *zap.SugaredLogger + workflowService executor.WorkflowService + ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository + ciPipelineRepository pipelineConfig.CiPipelineRepository + ciArtifactRepository repository5.CiArtifactRepository + pipelineStageService pipeline2.PipelineStageService + userService user.UserService + ciTemplateService pipeline.CiTemplateReadService + appCrudOperationService app.AppCrudOperationService + envRepository repository6.EnvironmentRepository + appRepository appRepository.AppRepository + customTagService pipeline2.CustomTagService + config *types.CiConfig + scopedVariableManager variables.ScopedVariableManager + ciCdPipelineOrchestrator pipeline2.CiCdPipelineOrchestrator + buildxCacheFlags *BuildxCacheFlags + attributeService attributes.AttributesService + pluginInputVariableParser pipeline2.PluginInputVariableParser + globalPluginService plugin.GlobalPluginService + ciService pipeline2.CiService + ciWorkflowRepository pipelineConfig.CiWorkflowRepository + gitSensorClient gitSensor.Client + ciLogService pipeline2.CiLogService + blobConfigStorageService pipeline2.BlobStorageConfigService + clusterService cluster.ClusterService + envService environment.EnvironmentService + K8sUtil *k8s.K8sServiceImpl +} + +func NewHandlerServiceImpl(Logger *zap.SugaredLogger, workflowService executor.WorkflowService, + ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository, + ciPipelineRepository pipelineConfig.CiPipelineRepository, + ciArtifactRepository repository5.CiArtifactRepository, + pipelineStageService pipeline2.PipelineStageService, + userService user.UserService, + ciTemplateService pipeline.CiTemplateReadService, + appCrudOperationService app.AppCrudOperationService, + envRepository repository6.EnvironmentRepository, + appRepository appRepository.AppRepository, + scopedVariableManager variables.ScopedVariableManager, + customTagService pipeline2.CustomTagService, + ciCdPipelineOrchestrator pipeline2.CiCdPipelineOrchestrator, attributeService attributes.AttributesService, + pluginInputVariableParser pipeline2.PluginInputVariableParser, + globalPluginService plugin.GlobalPluginService, + ciService pipeline2.CiService, + ciWorkflowRepository pipelineConfig.CiWorkflowRepository, + gitSensorClient gitSensor.Client, + ciLogService pipeline2.CiLogService, + blobConfigStorageService pipeline2.BlobStorageConfigService, + clusterService cluster.ClusterService, + envService environment.EnvironmentService, + K8sUtil *k8s.K8sServiceImpl, +) *HandlerServiceImpl { + buildxCacheFlags := &BuildxCacheFlags{} + err := env.Parse(buildxCacheFlags) + if err != nil { + Logger.Infow("error occurred while parsing BuildxCacheFlags env,so setting BuildxCacheModeMin and AsyncBuildxCacheExport to default value", "err", err) + } + cis := &HandlerServiceImpl{ + Logger: Logger, + workflowService: workflowService, + ciPipelineMaterialRepository: ciPipelineMaterialRepository, + ciPipelineRepository: ciPipelineRepository, + ciArtifactRepository: ciArtifactRepository, + pipelineStageService: pipelineStageService, + userService: userService, + ciTemplateService: ciTemplateService, + appCrudOperationService: appCrudOperationService, + envRepository: envRepository, + appRepository: appRepository, + scopedVariableManager: scopedVariableManager, + customTagService: customTagService, + ciCdPipelineOrchestrator: ciCdPipelineOrchestrator, + buildxCacheFlags: buildxCacheFlags, + attributeService: attributeService, + pluginInputVariableParser: pluginInputVariableParser, + globalPluginService: globalPluginService, + ciService: ciService, + ciWorkflowRepository: ciWorkflowRepository, + gitSensorClient: gitSensorClient, + ciLogService: ciLogService, + blobConfigStorageService: blobConfigStorageService, + clusterService: clusterService, + envService: envService, + K8sUtil: K8sUtil, + } + config, err := types.GetCiConfig() + if err != nil { + return nil + } + cis.config = config + return cis +} + +func (impl *HandlerServiceImpl) HandlePodDeleted(ciWorkflow *pipelineConfig.CiWorkflow) { + if !impl.config.WorkflowRetriesEnabled() { + impl.Logger.Debug("ci workflow retry feature disabled") + return + } + retryCount, refCiWorkflow, err := impl.getRefWorkflowAndCiRetryCount(ciWorkflow) + if err != nil { + impl.Logger.Errorw("error in getRefWorkflowAndCiRetryCount", "ciWorkflowId", ciWorkflow.Id, "err", err) + } + impl.Logger.Infow("re-triggering ci by UpdateCiWorkflowStatusFailedCron", "refCiWorkflowId", refCiWorkflow.Id, "ciWorkflow.Status", ciWorkflow.Status, "ciWorkflow.Message", ciWorkflow.Message, "retryCount", retryCount) + err = impl.reTriggerCi(retryCount, refCiWorkflow) + if err != nil { + impl.Logger.Errorw("error in reTriggerCi", "ciWorkflowId", refCiWorkflow.Id, "workflowStatus", ciWorkflow.Status, "ciWorkflowMessage", "ciWorkflow.Message", "retryCount", retryCount, "err", err) + } +} + +func (impl *HandlerServiceImpl) CheckAndReTriggerCI(workflowStatus eventProcessorBean.CiCdStatus) error { + + // return if re-trigger feature is disabled + if !impl.config.WorkflowRetriesEnabled() { + impl.Logger.Debug("CI re-trigger is disabled") + return nil + } + + status, message, ciWorkFlow, err := impl.extractPodStatusAndWorkflow(workflowStatus) + if err != nil { + impl.Logger.Errorw("error in extractPodStatusAndWorkflow", "err", err) + return err + } + + if !executors.CheckIfReTriggerRequired(status, message, ciWorkFlow.Status) { + impl.Logger.Debugw("not re-triggering ci", "status", status, "message", message, "ciWorkflowStatus", ciWorkFlow.Status) + return nil + } + + impl.Logger.Debugw("re-triggering ci", "status", status, "message", message, "ciWorkflowStatus", ciWorkFlow.Status, "ciWorkFlowId", ciWorkFlow.Id) + + retryCount, refCiWorkflow, err := impl.getRefWorkflowAndCiRetryCount(ciWorkFlow) + if err != nil { + impl.Logger.Errorw("error while getting retry count value for a ciWorkflow", "ciWorkFlowId", ciWorkFlow.Id) + return err + } + + err = impl.reTriggerCi(retryCount, refCiWorkflow) + if err != nil { + impl.Logger.Errorw("error in reTriggerCi", "err", err, "status", status, "message", message, "retryCount", retryCount, "ciWorkFlowId", ciWorkFlow.Id) + } + return err +} + +func (impl *HandlerServiceImpl) reTriggerCi(retryCount int, refCiWorkflow *pipelineConfig.CiWorkflow) error { + if retryCount >= impl.config.MaxCiWorkflowRetries { + impl.Logger.Infow("maximum retries exhausted for this ciWorkflow", "ciWorkflowId", refCiWorkflow.Id, "retries", retryCount, "configuredRetries", impl.config.MaxCiWorkflowRetries) + return nil + } + impl.Logger.Infow("re-triggering ci for a ci workflow", "ReferenceCiWorkflowId", refCiWorkflow.Id) + ciPipelineMaterialIds := make([]int, 0, len(refCiWorkflow.GitTriggers)) + for id, _ := range refCiWorkflow.GitTriggers { + ciPipelineMaterialIds = append(ciPipelineMaterialIds, id) + } + ciMaterials, err := impl.ciPipelineMaterialRepository.GetByIdsIncludeDeleted(ciPipelineMaterialIds) + if err != nil { + impl.Logger.Errorw("error in getting ci Pipeline Materials using ciPipeline Material Ids", "ciPipelineMaterialIds", ciPipelineMaterialIds, "err", err) + return err + } + + trigger := types.Trigger{} + trigger.BuildTriggerObject(refCiWorkflow, ciMaterials, bean6.SYSTEM_USER_ID, true, nil, "") + + // updating runtime params + trigger.RuntimeParameters, err = impl.updateRuntimeParamsForAutoCI(trigger.PipelineId, trigger.RuntimeParameters) + if err != nil { + impl.Logger.Errorw("err, updateRuntimeParamsForAutoCI", "ciPipelineId", trigger.PipelineId, + "runtimeParameters", trigger.RuntimeParameters, "err", err) + return err + } + _, err = impl.triggerCiPipeline(trigger) + + if err != nil { + impl.Logger.Errorw("error occurred in re-triggering ciWorkflow", "triggerDetails", trigger, "err", err) + return err + } + return nil +} + +func (impl *HandlerServiceImpl) HandleCIManual(ciTriggerRequest bean.CiTriggerRequest) (int, error) { + impl.Logger.Debugw("HandleCIManual for pipeline ", "PipelineId", ciTriggerRequest.PipelineId) + commitHashes, runtimeParams, err := impl.buildManualTriggerCommitHashes(ciTriggerRequest) + if err != nil { + return 0, err + } + + ciArtifact, err := impl.ciArtifactRepository.GetLatestArtifactTimeByCiPipelineId(ciTriggerRequest.PipelineId) + if err != nil && err != pg.ErrNoRows { + impl.Logger.Errorw("Error in GetLatestArtifactTimeByCiPipelineId", "err", err, "pipelineId", ciTriggerRequest.PipelineId) + return 0, err + } + + createdOn := time.Time{} + if err != pg.ErrNoRows { + createdOn = ciArtifact.CreatedOn + } + + trigger := types.Trigger{ + PipelineId: ciTriggerRequest.PipelineId, + CommitHashes: commitHashes, + CiMaterials: nil, + TriggeredBy: ciTriggerRequest.TriggeredBy, + InvalidateCache: ciTriggerRequest.InvalidateCache, + RuntimeParameters: runtimeParams, + EnvironmentId: ciTriggerRequest.EnvironmentId, + PipelineType: ciTriggerRequest.PipelineType, + CiArtifactLastFetch: createdOn, + } + id, err := impl.triggerCiPipeline(trigger) + + if err != nil { + return 0, err + } + return id, nil +} + +func (impl *HandlerServiceImpl) HandleCIWebhook(gitCiTriggerRequest bean.GitCiTriggerRequest) (int, error) { + impl.Logger.Debugw("HandleCIWebhook for material ", "material", gitCiTriggerRequest.CiPipelineMaterial) + ciPipeline, err := impl.GetCiPipeline(gitCiTriggerRequest.CiPipelineMaterial.Id) + if err != nil { + impl.Logger.Errorw("err in getting ci_pipeline by ciPipelineMaterialId", "ciPipelineMaterialId", gitCiTriggerRequest.CiPipelineMaterial.Id, "err", err) + return 0, err + } + if ciPipeline.IsManual || ciPipeline.PipelineType == buildCommonBean.LINKED_CD.ToString() { + impl.Logger.Debugw("not handling for manual pipeline or in case of linked cd", "pipelineId", ciPipeline.Id) + return 0, err + } + + ciMaterials, err := impl.ciPipelineMaterialRepository.GetByPipelineId(ciPipeline.Id) + if err != nil { + impl.Logger.Errorw("err", "err", err) + return 0, err + } + isValidBuildSequence, err := impl.validateBuildSequence(gitCiTriggerRequest, ciPipeline.Id) + if !isValidBuildSequence { + return 0, errors.New("ignoring older build for ciMaterial " + strconv.Itoa(gitCiTriggerRequest.CiPipelineMaterial.Id) + + " commit " + gitCiTriggerRequest.CiPipelineMaterial.GitCommit.Commit) + } + // updating runtime params + runtimeParams := common.NewRuntimeParameters() + for k, v := range gitCiTriggerRequest.ExtraEnvironmentVariables { + runtimeParams = runtimeParams.AddSystemVariable(k, v) + } + runtimeParams, err = impl.updateRuntimeParamsForAutoCI(ciPipeline.Id, runtimeParams) + if err != nil { + impl.Logger.Errorw("err, updateRuntimeParamsForAutoCI", "ciPipelineId", ciPipeline.Id, + "runtimeParameters", runtimeParams, "err", err) + return 0, err + } + commitHashes, err := impl.buildAutomaticTriggerCommitHashes(ciMaterials, gitCiTriggerRequest) + if err != nil { + return 0, err + } + + trigger := types.Trigger{ + PipelineId: ciPipeline.Id, + CommitHashes: commitHashes, + CiMaterials: ciMaterials, + TriggeredBy: gitCiTriggerRequest.TriggeredBy, + RuntimeParameters: runtimeParams, + } + id, err := impl.triggerCiPipeline(trigger) + if err != nil { + return 0, err + } + return id, nil +} + +func (impl *HandlerServiceImpl) extractPodStatusAndWorkflow(workflowStatus eventProcessorBean.CiCdStatus) (string, string, *pipelineConfig.CiWorkflow, error) { + workflowName, status, _, message, _, _ := pipeline2.ExtractWorkflowStatus(workflowStatus) + if workflowName == "" { + impl.Logger.Errorw("extract workflow status, invalid wf name", "workflowName", workflowName, "status", status, "message", message) + return status, message, nil, errors.New("invalid wf name") + } + workflowId, err := strconv.Atoi(workflowName[:strings.Index(workflowName, "-")]) + if err != nil { + impl.Logger.Errorw("extract workflowId, invalid wf name", "workflowName", workflowName, "err", err) + return status, message, nil, err + } + + savedWorkflow, err := impl.ciWorkflowRepository.FindById(workflowId) + if err != nil { + impl.Logger.Errorw("cannot get saved wf", "workflowId", workflowId, "err", err) + return status, message, nil, err + } + + return status, message, savedWorkflow, err + +} + +func (impl *HandlerServiceImpl) getRefWorkflowAndCiRetryCount(savedWorkflow *pipelineConfig.CiWorkflow) (int, *pipelineConfig.CiWorkflow, error) { + var err error + + if savedWorkflow.ReferenceCiWorkflowId != 0 { + savedWorkflow, err = impl.ciWorkflowRepository.FindById(savedWorkflow.ReferenceCiWorkflowId) + if err != nil { + impl.Logger.Errorw("cannot get saved wf", "err", err) + return 0, savedWorkflow, err + } + } + retryCount, err := impl.ciWorkflowRepository.FindRetriedWorkflowCountByReferenceId(savedWorkflow.Id) + return retryCount, savedWorkflow, err +} + +func (impl *HandlerServiceImpl) validateBuildSequence(gitCiTriggerRequest bean.GitCiTriggerRequest, pipelineId int) (bool, error) { + isValid := true + lastTriggeredBuild, err := impl.ciWorkflowRepository.FindLastTriggeredWorkflow(pipelineId) + if !(lastTriggeredBuild.Status == string(v1alpha1.NodePending) || lastTriggeredBuild.Status == string(v1alpha1.NodeRunning)) { + return true, nil + } + if err != nil && !util.IsErrNoRows(err) { + impl.Logger.Errorw("cannot get last build for pipeline", "pipelineId", pipelineId) + return false, err + } + + ciPipelineMaterial := gitCiTriggerRequest.CiPipelineMaterial + + if ciPipelineMaterial.Type == string(constants.SOURCE_TYPE_BRANCH_FIXED) { + if ciPipelineMaterial.GitCommit.Date.Before(lastTriggeredBuild.GitTriggers[ciPipelineMaterial.Id].Date) { + impl.Logger.Warnw("older commit cannot be built for pipeline", "pipelineId", pipelineId, "ciMaterial", gitCiTriggerRequest.CiPipelineMaterial.Id) + isValid = false + } + } + + return isValid, nil +} + +func (impl *HandlerServiceImpl) buildAutomaticTriggerCommitHashes(ciMaterials []*pipelineConfig.CiPipelineMaterial, request bean.GitCiTriggerRequest) (map[int]pipelineConfig.GitCommit, error) { + commitHashes := map[int]pipelineConfig.GitCommit{} + for _, ciMaterial := range ciMaterials { + if ciMaterial.Id == request.CiPipelineMaterial.Id || len(ciMaterials) == 1 { + request.CiPipelineMaterial.GitCommit = SetGitCommitValuesForBuildingCommitHash(ciMaterial, request.CiPipelineMaterial.GitCommit) + commitHashes[ciMaterial.Id] = request.CiPipelineMaterial.GitCommit + } else { + // this is possible in case of non Webhook, as there would be only one pipeline material per git material in case of PR + lastCommit, err := impl.getLastSeenCommit(ciMaterial.Id) + if err != nil { + return map[int]pipelineConfig.GitCommit{}, err + } + lastCommit = SetGitCommitValuesForBuildingCommitHash(ciMaterial, lastCommit) + commitHashes[ciMaterial.Id] = lastCommit + } + } + return commitHashes, nil +} + +func (impl *HandlerServiceImpl) GetCiPipeline(ciMaterialId int) (*pipelineConfig.CiPipeline, error) { + ciMaterial, err := impl.ciPipelineMaterialRepository.GetById(ciMaterialId) + if err != nil { + return nil, err + } + ciPipeline := ciMaterial.CiPipeline + return ciPipeline, nil +} + +func (impl *HandlerServiceImpl) buildManualTriggerCommitHashes(ciTriggerRequest bean.CiTriggerRequest) (map[int]pipelineConfig.GitCommit, *common.RuntimeParameters, error) { + commitHashes := map[int]pipelineConfig.GitCommit{} + runtimeParams := impl.getRuntimeParamsForBuildingManualTriggerHashes(ciTriggerRequest) + for _, ciPipelineMaterial := range ciTriggerRequest.CiPipelineMaterial { + + pipeLineMaterialFromDb, err := impl.ciPipelineMaterialRepository.GetById(ciPipelineMaterial.Id) + if err != nil { + impl.Logger.Errorw("err in fetching pipeline material by id", "err", err) + return map[int]pipelineConfig.GitCommit{}, nil, err + } + + pipelineType := pipeLineMaterialFromDb.Type + if pipelineType == constants.SOURCE_TYPE_BRANCH_FIXED { + gitCommit, err := impl.BuildManualTriggerCommitHashesForSourceTypeBranchFix(ciPipelineMaterial, pipeLineMaterialFromDb) + if err != nil { + impl.Logger.Errorw("err", "err", err) + return map[int]pipelineConfig.GitCommit{}, nil, err + } + commitHashes[ciPipelineMaterial.Id] = gitCommit + + } else if pipelineType == constants.SOURCE_TYPE_WEBHOOK { + gitCommit, extraEnvVariables, err := impl.BuildManualTriggerCommitHashesForSourceTypeWebhook(ciPipelineMaterial, pipeLineMaterialFromDb) + if err != nil { + impl.Logger.Errorw("err", "err", err) + return map[int]pipelineConfig.GitCommit{}, nil, err + } + commitHashes[ciPipelineMaterial.Id] = gitCommit + for key, value := range extraEnvVariables { + runtimeParams = runtimeParams.AddSystemVariable(key, value) + } + } + } + return commitHashes, runtimeParams, nil +} + +func (impl *HandlerServiceImpl) BuildManualTriggerCommitHashesForSourceTypeBranchFix(ciPipelineMaterial bean.CiPipelineMaterial, pipeLineMaterialFromDb *pipelineConfig.CiPipelineMaterial) (pipelineConfig.GitCommit, error) { + commitMetadataRequest := &gitSensor.CommitMetadataRequest{ + PipelineMaterialId: ciPipelineMaterial.Id, + GitHash: ciPipelineMaterial.GitCommit.Commit, + GitTag: ciPipelineMaterial.GitTag, + } + gitCommitResponse, err := impl.gitSensorClient.GetCommitMetadataForPipelineMaterial(context.Background(), commitMetadataRequest) + if err != nil { + impl.Logger.Errorw("err in fetching commit metadata", "commitMetadataRequest", commitMetadataRequest, "err", err) + return pipelineConfig.GitCommit{}, err + } + if gitCommitResponse == nil { + return pipelineConfig.GitCommit{}, errors.New("commit not found") + } + + gitCommit := pipelineConfig.GitCommit{ + Commit: gitCommitResponse.Commit, + Author: gitCommitResponse.Author, + Date: gitCommitResponse.Date, + Message: gitCommitResponse.Message, + Changes: gitCommitResponse.Changes, + GitRepoName: pipeLineMaterialFromDb.GitMaterial.Name[strings.Index(pipeLineMaterialFromDb.GitMaterial.Name, "-")+1:], + GitRepoUrl: pipeLineMaterialFromDb.GitMaterial.Url, + CiConfigureSourceValue: pipeLineMaterialFromDb.Value, + CiConfigureSourceType: pipeLineMaterialFromDb.Type, + } + + return gitCommit, nil +} + +func (impl *HandlerServiceImpl) BuildManualTriggerCommitHashesForSourceTypeWebhook(ciPipelineMaterial bean.CiPipelineMaterial, pipeLineMaterialFromDb *pipelineConfig.CiPipelineMaterial) (pipelineConfig.GitCommit, map[string]string, error) { + webhookDataInput := ciPipelineMaterial.GitCommit.WebhookData + + // fetch webhook data on the basis of Id + webhookDataRequest := &gitSensor.WebhookDataRequest{ + Id: webhookDataInput.Id, + CiPipelineMaterialId: ciPipelineMaterial.Id, + } + + webhookAndCiData, err := impl.gitSensorClient.GetWebhookData(context.Background(), webhookDataRequest) + if err != nil { + impl.Logger.Errorw("err", "err", err) + return pipelineConfig.GitCommit{}, nil, err + } + webhookData := webhookAndCiData.WebhookData + + // if webhook event is of merged type, then fetch latest commit for target branch + if webhookData.EventActionType == bean.WEBHOOK_EVENT_MERGED_ACTION_TYPE { + + // get target branch name from webhook + targetBranchName := webhookData.Data[bean.WEBHOOK_SELECTOR_TARGET_BRANCH_NAME_NAME] + if targetBranchName == "" { + impl.Logger.Error("target branch not found from webhook data") + return pipelineConfig.GitCommit{}, nil, err + } + + // get latest commit hash for target branch + latestCommitMetadataRequest := &gitSensor.CommitMetadataRequest{ + PipelineMaterialId: ciPipelineMaterial.Id, + BranchName: targetBranchName, + } + + latestCommit, err := impl.gitSensorClient.GetCommitMetadata(context.Background(), latestCommitMetadataRequest) + + if err != nil { + impl.Logger.Errorw("err", "err", err) + return pipelineConfig.GitCommit{}, nil, err + } + + // update webhookData (local) with target latest hash + webhookData.Data[bean.WEBHOOK_SELECTOR_TARGET_CHECKOUT_NAME] = latestCommit.Commit + + } + + // build git commit + gitCommit := pipelineConfig.GitCommit{ + GitRepoName: pipeLineMaterialFromDb.GitMaterial.Name[strings.Index(pipeLineMaterialFromDb.GitMaterial.Name, "-")+1:], + GitRepoUrl: pipeLineMaterialFromDb.GitMaterial.Url, + CiConfigureSourceValue: pipeLineMaterialFromDb.Value, + CiConfigureSourceType: pipeLineMaterialFromDb.Type, + WebhookData: pipelineConfig.WebhookData{ + Id: int(webhookData.Id), + EventActionType: webhookData.EventActionType, + Data: webhookData.Data, + }, + } + + return gitCommit, webhookAndCiData.ExtraEnvironmentVariables, nil +} + +func (impl *HandlerServiceImpl) getLastSeenCommit(ciMaterialId int) (pipelineConfig.GitCommit, error) { + var materialIds []int + materialIds = append(materialIds, ciMaterialId) + headReq := &gitSensor.HeadRequest{ + MaterialIds: materialIds, + } + res, err := impl.gitSensorClient.GetHeadForPipelineMaterials(context.Background(), headReq) + if err != nil { + return pipelineConfig.GitCommit{}, err + } + if len(res) == 0 { + return pipelineConfig.GitCommit{}, errors.New("received empty response") + } + gitCommit := pipelineConfig.GitCommit{ + Commit: res[0].GitCommit.Commit, + Author: res[0].GitCommit.Author, + Date: res[0].GitCommit.Date, + Message: res[0].GitCommit.Message, + Changes: res[0].GitCommit.Changes, + } + return gitCommit, nil +} + +func SetGitCommitValuesForBuildingCommitHash(ciMaterial *pipelineConfig.CiPipelineMaterial, oldGitCommit pipelineConfig.GitCommit) pipelineConfig.GitCommit { + newGitCommit := oldGitCommit + newGitCommit.CiConfigureSourceType = ciMaterial.Type + newGitCommit.CiConfigureSourceValue = ciMaterial.Value + newGitCommit.GitRepoUrl = ciMaterial.GitMaterial.Url + newGitCommit.GitRepoName = ciMaterial.GitMaterial.Name[strings.Index(ciMaterial.GitMaterial.Name, "-")+1:] + return newGitCommit +} + +func (impl *HandlerServiceImpl) triggerCiPipeline(trigger types.Trigger) (int, error) { + pipeline, variableSnapshot, savedCiWf, workflowRequest, err := impl.StartCiWorkflowAndPrepareWfRequest(trigger) + if err != nil { + return 0, err + } + workflowRequest.CiPipelineType = trigger.PipelineType + err = impl.executeCiPipeline(workflowRequest) + if err != nil { + impl.Logger.Errorw("error in executing ci pipeline", "err", err) + dbErr := impl.markCurrentCiWorkflowFailed(savedCiWf, err) + if dbErr != nil { + impl.Logger.Errorw("update ci workflow error", "err", dbErr) + } + return 0, err + } + impl.Logger.Debugw("ci triggered", " pipeline ", trigger.PipelineId) + + var variableSnapshotHistories = sliceUtil.GetBeansPtr( + repository4.GetSnapshotBean(savedCiWf.Id, repository4.HistoryReferenceTypeCIWORKFLOW, variableSnapshot)) + if len(variableSnapshotHistories) > 0 { + err = impl.scopedVariableManager.SaveVariableHistoriesForTrigger(variableSnapshotHistories, trigger.TriggeredBy) + if err != nil { + impl.Logger.Errorf("Not able to save variable snapshot for CI trigger %s", err) + } + } + + middleware.CiTriggerCounter.WithLabelValues(pipeline.App.AppName, pipeline.Name).Inc() + go impl.ciService.WriteCITriggerEvent(trigger, pipeline, workflowRequest) + return savedCiWf.Id, err +} + +func (impl *HandlerServiceImpl) GetCiMaterials(pipelineId int, ciMaterials []*pipelineConfig.CiPipelineMaterial) ([]*pipelineConfig.CiPipelineMaterial, error) { + if !(len(ciMaterials) == 0) { + return ciMaterials, nil + } else { + ciMaterials, err := impl.ciPipelineMaterialRepository.GetByPipelineId(pipelineId) + if err != nil { + impl.Logger.Errorw("err", "err", err) + return nil, err + } + impl.Logger.Debug("ciMaterials for pipeline trigger ", ciMaterials) + return ciMaterials, nil + } +} + +func (impl *HandlerServiceImpl) StartCiWorkflowAndPrepareWfRequest(trigger types.Trigger) (*pipelineConfig.CiPipeline, map[string]string, *pipelineConfig.CiWorkflow, *types.WorkflowRequest, error) { + impl.Logger.Debugw("ci pipeline manual trigger", "request", trigger) + ciMaterials, err := impl.GetCiMaterials(trigger.PipelineId, trigger.CiMaterials) + if err != nil { + return nil, nil, nil, nil, err + } + + ciPipelineScripts, err := impl.ciPipelineRepository.FindCiScriptsByCiPipelineId(trigger.PipelineId) + if err != nil && !util.IsErrNoRows(err) { + return nil, nil, nil, nil, err + } + + var pipeline *pipelineConfig.CiPipeline + for _, m := range ciMaterials { + pipeline = m.CiPipeline + break + } + + scope := resourceQualifiers.Scope{ + AppId: pipeline.App.Id, + } + ciWorkflowConfigNamespace := impl.config.GetDefaultNamespace() + envModal, isJob, err := impl.getEnvironmentForJob(pipeline, trigger) + if err != nil { + return nil, nil, nil, nil, err + } + if isJob && envModal != nil { + + err = impl.checkArgoSetupRequirement(envModal) + if err != nil { + return nil, nil, nil, nil, err + } + + ciWorkflowConfigNamespace = envModal.Namespace + + // This will be populated for jobs running in selected environment + scope.EnvId = envModal.Id + scope.ClusterId = envModal.ClusterId + + scope.SystemMetadata = &resourceQualifiers.SystemMetadata{ + EnvironmentName: envModal.Name, + ClusterName: envModal.Cluster.ClusterName, + Namespace: envModal.Namespace, + } + } + if scope.SystemMetadata == nil { + scope.SystemMetadata = &resourceQualifiers.SystemMetadata{ + Namespace: ciWorkflowConfigNamespace, + AppName: pipeline.App.AppName, + } + } + savedCiWf, err := impl.saveNewWorkflowForCITrigger(pipeline, ciWorkflowConfigNamespace, trigger.CommitHashes, trigger.TriggeredBy, ciMaterials, trigger.EnvironmentId, isJob, trigger.ReferenceCiWorkflowId) + if err != nil { + impl.Logger.Errorw("could not save new workflow", "err", err) + return nil, nil, nil, nil, err + } + // preCiSteps, postCiSteps, refPluginsData, err := impl.pipelineStageService.BuildPrePostAndRefPluginStepsDataForWfRequest(pipeline.Id, ciEvent) + request := pipelineConfigBean.NewBuildPrePostStepDataReq(pipeline.Id, pipelineConfigBean.CiStage, scope) + request = updateBuildPrePostStepDataReq(request, trigger) + prePostAndRefPluginResponse, err := impl.pipelineStageService.BuildPrePostAndRefPluginStepsDataForWfRequest(request) + if err != nil { + impl.Logger.Errorw("error in getting pre steps data for wf request", "err", err, "ciPipelineId", pipeline.Id) + dbErr := impl.markCurrentCiWorkflowFailed(savedCiWf, err) + if dbErr != nil { + impl.Logger.Errorw("saving workflow error", "err", dbErr) + } + return nil, nil, nil, nil, err + } + preCiSteps := prePostAndRefPluginResponse.PreStageSteps + postCiSteps := prePostAndRefPluginResponse.PostStageSteps + refPluginsData := prePostAndRefPluginResponse.RefPluginData + variableSnapshot := prePostAndRefPluginResponse.VariableSnapshot + + if len(preCiSteps) == 0 && isJob { + errMsg := fmt.Sprintf("No tasks are configured in this job pipeline") + validationErr := util.NewApiError(http.StatusNotFound, errMsg, errMsg) + return nil, nil, nil, nil, validationErr + } + + // get env variables of git trigger data and add it in the extraEnvVariables + gitTriggerEnvVariables, _, err := impl.ciCdPipelineOrchestrator.GetGitCommitEnvVarDataForCICDStage(savedCiWf.GitTriggers) + if err != nil { + impl.Logger.Errorw("error in getting gitTrigger env data for stage", "gitTriggers", savedCiWf.GitTriggers, "err", err) + return nil, nil, nil, nil, err + } + + for k, v := range gitTriggerEnvVariables { + trigger.RuntimeParameters = trigger.RuntimeParameters.AddSystemVariable(k, v) + } + + workflowRequest, err := impl.buildWfRequestForCiPipeline(pipeline, trigger, ciMaterials, savedCiWf, ciWorkflowConfigNamespace, ciPipelineScripts, preCiSteps, postCiSteps, refPluginsData, isJob) + if err != nil { + impl.Logger.Errorw("make workflow req", "err", err) + return nil, nil, nil, nil, err + } + err = impl.handleRuntimeParamsValidations(trigger, ciMaterials, workflowRequest) + if err != nil { + savedCiWf.Status = cdWorkflow.WorkflowAborted + savedCiWf.Message = err.Error() + err1 := impl.ciService.UpdateCiWorkflowWithStage(savedCiWf) + if err1 != nil { + impl.Logger.Errorw("could not save workflow, after failing due to conflicting image tag") + } + return nil, nil, nil, nil, err + } + + workflowRequest.Scope = scope + workflowRequest.AppId = pipeline.AppId + workflowRequest.Env = envModal + if isJob { + workflowRequest.Type = pipelineConfigBean.JOB_WORKFLOW_PIPELINE_TYPE + } else { + workflowRequest.Type = pipelineConfigBean.CI_WORKFLOW_PIPELINE_TYPE + } + workflowRequest, err = impl.updateWorkflowRequestWithBuildCacheData(workflowRequest, scope) + if err != nil { + impl.Logger.Errorw("error, updateWorkflowRequestWithBuildCacheData", "workflowRequest", workflowRequest, "err", err) + return nil, nil, nil, nil, err + } + if impl.canSetK8sDriverData(workflowRequest) { + err = impl.setBuildxK8sDriverData(workflowRequest) + if err != nil { + impl.Logger.Errorw("error in setBuildxK8sDriverData", "BUILDX_K8S_DRIVER_OPTIONS", impl.config.BuildxK8sDriverOptions, "err", err) + return nil, nil, nil, nil, err + } + } + savedCiWf.LogLocation = fmt.Sprintf("%s/%s/main.log", impl.config.GetDefaultBuildLogsKeyPrefix(), workflowRequest.WorkflowNamePrefix) + err = impl.updateCiWorkflow(workflowRequest, savedCiWf) + appLabels, err := impl.appCrudOperationService.GetLabelsByAppId(pipeline.AppId) + if err != nil { + return nil, nil, nil, nil, err + } + workflowRequest.AppLabels = appLabels + workflowRequest = impl.updateWorkflowRequestWithEntSupportData(workflowRequest) + return pipeline, variableSnapshot, savedCiWf, workflowRequest, nil +} + +func (impl *HandlerServiceImpl) setBuildxK8sDriverData(workflowRequest *types.WorkflowRequest) error { + dockerBuildConfig := workflowRequest.CiBuildConfig.DockerBuildConfig + k8sDriverOptions, err := impl.getK8sDriverOptions(workflowRequest, dockerBuildConfig.TargetPlatform) + if err != nil { + impl.Logger.Errorw("error in parsing BUILDX_K8S_DRIVER_OPTIONS from the devtron-cm", "err", err) + } + dockerBuildConfig.BuildxK8sDriverOptions = k8sDriverOptions + return nil +} + +func (impl *HandlerServiceImpl) getEnvironmentForJob(pipeline *pipelineConfig.CiPipeline, trigger types.Trigger) (*repository6.Environment, bool, error) { + app, err := impl.appRepository.FindById(pipeline.AppId) + if err != nil { + impl.Logger.Errorw("could not find app", "err", err) + return nil, false, err + } + + var env *repository6.Environment + isJob := false + if app.AppType == helper.Job { + isJob = true + if trigger.EnvironmentId != 0 { + env, err = impl.envRepository.FindById(trigger.EnvironmentId) + if err != nil { + impl.Logger.Errorw("could not find environment", "err", err) + return nil, isJob, err + } + return env, isJob, nil + } + } + return nil, isJob, nil +} + +// TODO: Send all trigger data +func (impl *HandlerServiceImpl) BuildPayload(trigger types.Trigger, pipeline *pipelineConfig.CiPipeline) *client.Payload { + payload := &client.Payload{} + payload.AppName = pipeline.App.AppName + payload.PipelineName = pipeline.Name + return payload +} + +func (impl *HandlerServiceImpl) saveNewWorkflowForCITrigger(pipeline *pipelineConfig.CiPipeline, ciWorkflowConfigNamespace string, + commitHashes map[int]pipelineConfig.GitCommit, userId int32, ciMaterials []*pipelineConfig.CiPipelineMaterial, EnvironmentId int, isJob bool, refCiWorkflowId int) (*pipelineConfig.CiWorkflow, error) { + + isCiTriggerBlocked, err := impl.checkIfCITriggerIsBlocked(pipeline, ciMaterials, isJob) + if err != nil { + impl.Logger.Errorw("error, checkIfCITriggerIsBlocked", "pipelineId", pipeline.Id, "err", err) + return &pipelineConfig.CiWorkflow{}, err + } + ciWorkflow := &pipelineConfig.CiWorkflow{ + Name: pipeline.Name + "-" + strconv.Itoa(pipeline.Id), + Status: cdWorkflow.WorkflowStarting, // starting CIStage + Message: "", + StartedOn: time.Now(), + CiPipelineId: pipeline.Id, + Namespace: impl.config.GetDefaultNamespace(), + BlobStorageEnabled: impl.config.BlobStorageEnabled, + GitTriggers: commitHashes, + LogLocation: "", + TriggeredBy: userId, + ReferenceCiWorkflowId: refCiWorkflowId, + ExecutorType: impl.config.GetWorkflowExecutorType(), + } + if isJob { + ciWorkflow.Namespace = ciWorkflowConfigNamespace + ciWorkflow.EnvironmentId = EnvironmentId + } + if isCiTriggerBlocked { + return impl.handleWFIfCITriggerIsBlocked(ciWorkflow) + } + err = impl.ciService.SaveCiWorkflowWithStage(ciWorkflow) + if err != nil { + impl.Logger.Errorw("saving workflow error", "err", err) + return &pipelineConfig.CiWorkflow{}, err + } + impl.Logger.Debugw("workflow saved ", "id", ciWorkflow.Id) + return ciWorkflow, nil +} + +func (impl *HandlerServiceImpl) executeCiPipeline(workflowRequest *types.WorkflowRequest) error { + _, _, err := impl.workflowService.SubmitWorkflow(workflowRequest) + if err != nil { + impl.Logger.Errorw("workflow error", "err", err) + return err + } + return nil +} + +func (impl *HandlerServiceImpl) buildS3ArtifactLocation(ciWorkflowConfigLogsBucket string, savedWf *pipelineConfig.CiWorkflow) (string, string, string) { + ciArtifactLocationFormat := impl.config.GetArtifactLocationFormat() + ArtifactLocation := fmt.Sprintf("s3://"+path.Join(ciWorkflowConfigLogsBucket, ciArtifactLocationFormat), savedWf.Id, savedWf.Id) + artifactFileName := fmt.Sprintf(ciArtifactLocationFormat, savedWf.Id, savedWf.Id) + return ArtifactLocation, ciWorkflowConfigLogsBucket, artifactFileName +} + +func (impl *HandlerServiceImpl) buildDefaultArtifactLocation(savedWf *pipelineConfig.CiWorkflow) string { + ciArtifactLocationFormat := impl.config.GetArtifactLocationFormat() + ArtifactLocation := fmt.Sprintf(ciArtifactLocationFormat, savedWf.Id, savedWf.Id) + return ArtifactLocation +} + +func (impl *HandlerServiceImpl) buildWfRequestForCiPipeline(pipeline *pipelineConfig.CiPipeline, trigger types.Trigger, ciMaterials []*pipelineConfig.CiPipelineMaterial, savedWf *pipelineConfig.CiWorkflow, ciWorkflowConfigNamespace string, ciPipelineScripts []*pipelineConfig.CiPipelineScript, preCiSteps []*pipelineConfigBean.StepObject, postCiSteps []*pipelineConfigBean.StepObject, refPluginsData []*pipelineConfigBean.RefPluginObject, isJob bool) (*types.WorkflowRequest, error) { + var ciProjectDetails []pipelineConfigBean.CiProjectDetails + commitHashes := trigger.CommitHashes + for _, ciMaterial := range ciMaterials { + // ignore those materials which have inactive git material + if ciMaterial == nil || ciMaterial.GitMaterial == nil || !ciMaterial.GitMaterial.Active { + continue + } + commitHashForPipelineId := commitHashes[ciMaterial.Id] + ciProjectDetail := pipelineConfigBean.CiProjectDetails{ + GitRepository: ciMaterial.GitMaterial.Url, + MaterialName: ciMaterial.GitMaterial.Name, + CheckoutPath: ciMaterial.GitMaterial.CheckoutPath, + FetchSubmodules: ciMaterial.GitMaterial.FetchSubmodules, + CommitHash: commitHashForPipelineId.Commit, + Author: commitHashForPipelineId.Author, + SourceType: ciMaterial.Type, + SourceValue: ciMaterial.Value, + GitTag: ciMaterial.GitTag, + Message: commitHashForPipelineId.Message, + Type: string(ciMaterial.Type), + CommitTime: commitHashForPipelineId.Date.Format(bean.LayoutRFC3339), + GitOptions: pipelineConfigBean.GitOptions{ + UserName: ciMaterial.GitMaterial.GitProvider.UserName, + Password: ciMaterial.GitMaterial.GitProvider.Password, + SshPrivateKey: ciMaterial.GitMaterial.GitProvider.SshPrivateKey, + AccessToken: ciMaterial.GitMaterial.GitProvider.AccessToken, + AuthMode: ciMaterial.GitMaterial.GitProvider.AuthMode, + EnableTLSVerification: ciMaterial.GitMaterial.GitProvider.EnableTLSVerification, + TlsKey: ciMaterial.GitMaterial.GitProvider.TlsKey, + TlsCert: ciMaterial.GitMaterial.GitProvider.TlsCert, + CaCert: ciMaterial.GitMaterial.GitProvider.CaCert, + }, + } + var err error + ciProjectDetail, err = impl.updateCIProjectDetailWithCloningMode(pipeline.AppId, ciMaterial, ciProjectDetail) + if err != nil { + impl.Logger.Errorw("error, updateCIProjectDetailWithCloningMode", "pipelineId", pipeline.Id, "err", err) + return nil, err + } + if ciMaterial.Type == constants.SOURCE_TYPE_WEBHOOK { + webhookData := commitHashForPipelineId.WebhookData + ciProjectDetail.WebhookData = pipelineConfig.WebhookData{ + Id: webhookData.Id, + EventActionType: webhookData.EventActionType, + Data: webhookData.Data, + } + } + + ciProjectDetails = append(ciProjectDetails, ciProjectDetail) + } + + var beforeDockerBuildScripts []*bean.CiScript + var afterDockerBuildScripts []*bean.CiScript + for _, ciPipelineScript := range ciPipelineScripts { + ciTask := &bean.CiScript{ + Id: ciPipelineScript.Id, + Index: ciPipelineScript.Index, + Name: ciPipelineScript.Name, + Script: ciPipelineScript.Script, + OutputLocation: ciPipelineScript.OutputLocation, + } + + if ciPipelineScript.Stage == buildCommonBean.BEFORE_DOCKER_BUILD { + beforeDockerBuildScripts = append(beforeDockerBuildScripts, ciTask) + } else if ciPipelineScript.Stage == buildCommonBean.AFTER_DOCKER_BUILD { + afterDockerBuildScripts = append(afterDockerBuildScripts, ciTask) + } + } + + if !(len(beforeDockerBuildScripts) == 0 && len(afterDockerBuildScripts) == 0) { + // found beforeDockerBuildScripts/afterDockerBuildScripts + // building preCiSteps & postCiSteps from them, refPluginsData not needed + preCiSteps = buildCiStepsDataFromDockerBuildScripts(beforeDockerBuildScripts) + postCiSteps = buildCiStepsDataFromDockerBuildScripts(afterDockerBuildScripts) + refPluginsData = []*pipelineConfigBean.RefPluginObject{} + } + + host, err := impl.attributeService.GetByKey(bean3.HostUrlKey) + if err != nil { + impl.Logger.Errorw("error in getting host url", "err", err, "hostUrl", host.Value) + return nil, err + } + ciWorkflowConfigCiCacheBucket := impl.config.DefaultCacheBucket + + ciWorkflowConfigCiCacheRegion := impl.config.DefaultCacheBucketRegion + + ciWorkflowConfigCiImage := impl.config.GetDefaultImage() + + ciTemplate := pipeline.CiTemplate + ciLevelArgs := pipeline.DockerArgs + + if ciLevelArgs == "" { + ciLevelArgs = "{}" + } + + if pipeline.CiTemplate.DockerBuildOptions == "" { + pipeline.CiTemplate.DockerBuildOptions = "{}" + } + userEmailId, err := impl.userService.GetActiveEmailById(trigger.TriggeredBy) + if err != nil { + impl.Logger.Errorw("unable to find user email by id", "err", err, "id", trigger.TriggeredBy) + return nil, err + } + var dockerfilePath string + var dockerRepository string + var checkoutPath string + var ciBuildConfigBean *buildBean.CiBuildConfigBean + dockerRegistry := &repository3.DockerArtifactStore{} + ciBaseBuildConfigEntity := ciTemplate.CiBuildConfig + ciBaseBuildConfigBean, err := adapter.ConvertDbBuildConfigToBean(ciBaseBuildConfigEntity) + if err != nil { + impl.Logger.Errorw("error occurred while converting buildconfig dbEntity to configBean", "ciBuildConfigEntity", ciBaseBuildConfigEntity, "err", err) + return nil, errors.New("error while parsing ci build config") + } + if !pipeline.IsExternal && pipeline.IsDockerConfigOverridden { + templateOverrideBean, err := impl.ciTemplateService.FindTemplateOverrideByCiPipelineId(pipeline.Id) + if err != nil { + return nil, err + } + ciBuildConfigBean = templateOverrideBean.CiBuildConfig + // updating args coming from ciBaseBuildConfigEntity because it is not part of Ci override + if ciBuildConfigBean != nil && ciBuildConfigBean.DockerBuildConfig != nil && ciBaseBuildConfigBean != nil && ciBaseBuildConfigBean.DockerBuildConfig != nil { + ciBuildConfigBean.DockerBuildConfig.Args = ciBaseBuildConfigBean.DockerBuildConfig.Args + } + templateOverride := templateOverrideBean.CiTemplateOverride + checkoutPath = templateOverride.GitMaterial.CheckoutPath + dockerfilePath = templateOverride.DockerfilePath + dockerRepository = templateOverride.DockerRepository + dockerRegistry = templateOverride.DockerRegistry + } else { + checkoutPath = ciTemplate.GitMaterial.CheckoutPath + dockerfilePath = ciTemplate.DockerfilePath + dockerRegistry = ciTemplate.DockerRegistry + dockerRepository = ciTemplate.DockerRepository + ciBuildConfigBean = ciBaseBuildConfigBean + if ciBuildConfigBean != nil { + ciBuildConfigBean.BuildContextGitMaterialId = ciTemplate.BuildContextGitMaterialId + } + + } + if checkoutPath == "" { + checkoutPath = "./" + } + var dockerImageTag string + customTag, err := impl.customTagService.GetActiveCustomTagByEntityKeyAndValue(pipelineConfigBean.EntityTypeCiPipelineId, strconv.Itoa(pipeline.Id)) + if err != nil && err != pg.ErrNoRows { + return nil, err + } + if customTag.Id != 0 && customTag.Enabled == true { + imagePathReservation, err := impl.customTagService.GenerateImagePath(pipelineConfigBean.EntityTypeCiPipelineId, strconv.Itoa(pipeline.Id), dockerRegistry.RegistryURL, dockerRepository) + if err != nil { + if errors.Is(err, pipelineConfigBean.ErrImagePathInUse) { + errMsg := pipelineConfigBean.ImageTagUnavailableMessage + validationErr := util.NewApiError(http.StatusConflict, errMsg, errMsg) + dbErr := impl.markCurrentCiWorkflowFailed(savedWf, validationErr) + if dbErr != nil { + impl.Logger.Errorw("could not save workflow, after failing due to conflicting image tag", "err", dbErr, "savedWf", savedWf.Id) + } + return nil, err + } + return nil, err + } + savedWf.ImagePathReservationIds = []int{imagePathReservation.Id} + // imagePath = docker.io/avd0/dashboard:fd23414b + imagePathSplit := strings.Split(imagePathReservation.ImagePath, ":") + if len(imagePathSplit) >= 1 { + dockerImageTag = imagePathSplit[len(imagePathSplit)-1] + } + } else { + dockerImageTag = impl.buildImageTag(commitHashes, pipeline.Id, savedWf.Id) + } + + // copyContainerImage plugin specific logic + var registryCredentialMap map[string]bean2.RegistryCredentials + var pluginArtifactStage string + var imageReservationIds []int + var registryDestinationImageMap map[string][]string + if !isJob { + registryDestinationImageMap, registryCredentialMap, pluginArtifactStage, imageReservationIds, err = impl.GetWorkflowRequestVariablesForCopyContainerImagePlugin(preCiSteps, postCiSteps, dockerImageTag, customTag.Id, + fmt.Sprintf(pipelineConfigBean.ImagePathPattern, + dockerRegistry.RegistryURL, + dockerRepository, + dockerImageTag), + dockerRegistry.Id) + if err != nil { + impl.Logger.Errorw("error in getting env variables for copyContainerImage plugin") + dbErr := impl.markCurrentCiWorkflowFailed(savedWf, err) + if dbErr != nil { + impl.Logger.Errorw("could not save workflow, after failing due to conflicting image tag", "err", dbErr, "savedWf", savedWf.Id) + } + return nil, err + } + + savedWf.ImagePathReservationIds = append(savedWf.ImagePathReservationIds, imageReservationIds...) + } + // mergedArgs := string(merged) + oldArgs := ciTemplate.Args + ciBuildConfigBean, err = adapter.OverrideCiBuildConfig(dockerfilePath, oldArgs, ciLevelArgs, ciTemplate.DockerBuildOptions, ciTemplate.TargetPlatform, ciBuildConfigBean) + if err != nil { + impl.Logger.Errorw("error occurred while overriding ci build config", "oldArgs", oldArgs, "ciLevelArgs", ciLevelArgs, "error", err) + return nil, errors.New("error while parsing ci build config") + } + buildContextCheckoutPath, err := impl.ciPipelineMaterialRepository.GetCheckoutPath(ciBuildConfigBean.BuildContextGitMaterialId) + if err != nil && err != pg.ErrNoRows { + impl.Logger.Errorw("error occurred while getting checkout path from git material", "gitMaterialId", ciBuildConfigBean.BuildContextGitMaterialId, "error", err) + return nil, err + } + if buildContextCheckoutPath == "" { + buildContextCheckoutPath = checkoutPath + } + if ciBuildConfigBean.UseRootBuildContext { + // use root build context i.e '.' + buildContextCheckoutPath = "." + } + + ciBuildConfigBean.PipelineType = trigger.PipelineType + + if ciBuildConfigBean.CiBuildType == buildBean.SELF_DOCKERFILE_BUILD_TYPE || ciBuildConfigBean.CiBuildType == buildBean.MANAGED_DOCKERFILE_BUILD_TYPE { + ciBuildConfigBean.DockerBuildConfig.BuildContext = filepath.Join(buildContextCheckoutPath, ciBuildConfigBean.DockerBuildConfig.BuildContext) + dockerBuildConfig := ciBuildConfigBean.DockerBuildConfig + dockerfilePath = filepath.Join(checkoutPath, dockerBuildConfig.DockerfilePath) + dockerBuildConfig.DockerfilePath = dockerfilePath + checkoutPath = dockerfilePath[:strings.LastIndex(dockerfilePath, "/")+1] + } else if ciBuildConfigBean.CiBuildType == buildBean.BUILDPACK_BUILD_TYPE { + buildPackConfig := ciBuildConfigBean.BuildPackConfig + checkoutPath = filepath.Join(checkoutPath, buildPackConfig.ProjectPath) + } + + if ciBuildConfigBean.DockerBuildConfig != nil { + ciBuildConfigBean = impl.updateCIBuildConfig(ciBuildConfigBean) + } + + workflowRequest := &types.WorkflowRequest{ + WorkflowNamePrefix: strconv.Itoa(savedWf.Id) + "-" + savedWf.Name, + PipelineName: pipeline.Name, + PipelineId: pipeline.Id, + CiCacheFileName: pipeline.Name + "-" + strconv.Itoa(pipeline.Id) + ".tar.gz", + CiProjectDetails: ciProjectDetails, + Namespace: ciWorkflowConfigNamespace, + BlobStorageConfigured: savedWf.BlobStorageEnabled, + CiImage: ciWorkflowConfigCiImage, + WorkflowId: savedWf.Id, + TriggeredBy: savedWf.TriggeredBy, + CacheLimit: impl.config.CacheLimit, + ScanEnabled: pipeline.ScanEnabled, + CloudProvider: impl.config.CloudProvider, + DefaultAddressPoolBaseCidr: impl.config.GetDefaultAddressPoolBaseCidr(), + DefaultAddressPoolSize: impl.config.GetDefaultAddressPoolSize(), + PreCiSteps: preCiSteps, + PostCiSteps: postCiSteps, + RefPlugins: refPluginsData, + AppName: pipeline.App.AppName, + TriggerByAuthor: userEmailId, + CiBuildConfig: ciBuildConfigBean, + CiBuildDockerMtuValue: impl.config.CiRunnerDockerMTUValue, + CacheInvalidate: trigger.InvalidateCache, + SystemEnvironmentVariables: trigger.RuntimeParameters.GetSystemVariables(), + EnableBuildContext: impl.config.EnableBuildContext, + OrchestratorHost: impl.config.OrchestratorHost, + HostUrl: host.Value, + OrchestratorToken: impl.config.OrchestratorToken, + ImageRetryCount: impl.config.ImageRetryCount, + ImageRetryInterval: impl.config.ImageRetryInterval, + WorkflowExecutor: impl.config.GetWorkflowExecutorType(), + Type: pipelineConfigBean.CI_WORKFLOW_PIPELINE_TYPE, + CiArtifactLastFetch: trigger.CiArtifactLastFetch, + RegistryCredentialMap: registryCredentialMap, + PluginArtifactStage: pluginArtifactStage, + ImageScanMaxRetries: impl.config.ImageScanMaxRetries, + ImageScanRetryDelay: impl.config.ImageScanRetryDelay, + UseDockerApiToGetDigest: impl.config.UseDockerApiToGetDigest, + RegistryDestinationImageMap: registryDestinationImageMap, + } + workflowRequest.SetEntOnlyFields(trigger, impl.config) + workflowCacheConfig := impl.ciCdPipelineOrchestrator.GetWorkflowCacheConfig(pipeline.App.AppType, trigger.PipelineType, pipeline.GetWorkflowCacheConfig()) + workflowRequest.IgnoreDockerCachePush = !workflowCacheConfig.Value + workflowRequest.IgnoreDockerCachePull = !workflowCacheConfig.Value + impl.Logger.Debugw("Ignore Cache values", "IgnoreDockerCachePush", workflowRequest.IgnoreDockerCachePush, "IgnoreDockerCachePull", workflowRequest.IgnoreDockerCachePull) + if pipeline.App.AppType == helper.Job { + workflowRequest.AppName = pipeline.App.DisplayName + } + if pipeline.ScanEnabled { + scanToolMetadata, scanVia, err := impl.fetchImageScanExecutionMedium() + if err != nil { + impl.Logger.Errorw("error occurred getting scanned via", "err", err) + return nil, err + } + workflowRequest.SetExecuteImageScanningVia(scanVia) + if scanVia.IsScanMediumExternal() { + imageScanExecutionSteps, refPlugins, err := impl.fetchImageScanExecutionStepsForWfRequest(scanToolMetadata) + if err != nil { + impl.Logger.Errorw("error occurred, fetchImageScanExecutionStepsForWfRequest", "scanToolMetadata", scanToolMetadata, "err", err) + return nil, err + } + workflowRequest.SetImageScanningSteps(imageScanExecutionSteps) + workflowRequest.RefPlugins = append(workflowRequest.RefPlugins, refPlugins...) + } + } + if dockerRegistry != nil { + workflowRequest, err = impl.updateWorkflowRequestWithRemoteConnConf(dockerRegistry, workflowRequest) + if err != nil { + impl.Logger.Errorw("error occurred updating workflow request", "dockerRegistryId", dockerRegistry.Id, "err", err) + return nil, err + } + workflowRequest.DockerRegistryId = dockerRegistry.Id + workflowRequest.DockerRegistryType = string(dockerRegistry.RegistryType) + workflowRequest.DockerImageTag = dockerImageTag + workflowRequest.DockerRegistryURL = dockerRegistry.RegistryURL + workflowRequest.DockerRepository = dockerRepository + workflowRequest.CheckoutPath = checkoutPath + workflowRequest.DockerUsername = dockerRegistry.Username + workflowRequest.DockerPassword = dockerRegistry.Password + workflowRequest.AwsRegion = dockerRegistry.AWSRegion + workflowRequest.AccessKey = dockerRegistry.AWSAccessKeyId + workflowRequest.SecretKey = dockerRegistry.AWSSecretAccessKey + workflowRequest.DockerConnection = dockerRegistry.Connection + workflowRequest.DockerCert = dockerRegistry.Cert + + } + ciWorkflowConfigLogsBucket := impl.config.GetDefaultBuildLogsBucket() + + switch workflowRequest.CloudProvider { + case types.BLOB_STORAGE_S3: + // No AccessKey is used for uploading artifacts, instead IAM based auth is used + workflowRequest.CiCacheRegion = ciWorkflowConfigCiCacheRegion + workflowRequest.CiCacheLocation = ciWorkflowConfigCiCacheBucket + workflowRequest.CiArtifactLocation, workflowRequest.CiArtifactBucket, workflowRequest.CiArtifactFileName = impl.buildS3ArtifactLocation(ciWorkflowConfigLogsBucket, savedWf) + workflowRequest.BlobStorageS3Config = &blob_storage.BlobStorageS3Config{ + AccessKey: impl.config.BlobStorageS3AccessKey, + Passkey: impl.config.BlobStorageS3SecretKey, + EndpointUrl: impl.config.BlobStorageS3Endpoint, + IsInSecure: impl.config.BlobStorageS3EndpointInsecure, + CiCacheBucketName: ciWorkflowConfigCiCacheBucket, + CiCacheRegion: ciWorkflowConfigCiCacheRegion, + CiCacheBucketVersioning: impl.config.BlobStorageS3BucketVersioned, + CiArtifactBucketName: workflowRequest.CiArtifactBucket, + CiArtifactRegion: impl.config.GetDefaultCdLogsBucketRegion(), + CiArtifactBucketVersioning: impl.config.BlobStorageS3BucketVersioned, + CiLogBucketName: impl.config.GetDefaultBuildLogsBucket(), + CiLogRegion: impl.config.GetDefaultCdLogsBucketRegion(), + CiLogBucketVersioning: impl.config.BlobStorageS3BucketVersioned, + } + case types.BLOB_STORAGE_GCP: + workflowRequest.GcpBlobConfig = &blob_storage.GcpBlobConfig{ + CredentialFileJsonData: impl.config.BlobStorageGcpCredentialJson, + CacheBucketName: ciWorkflowConfigCiCacheBucket, + LogBucketName: ciWorkflowConfigLogsBucket, + ArtifactBucketName: ciWorkflowConfigLogsBucket, + } + workflowRequest.CiArtifactLocation = impl.buildDefaultArtifactLocation(savedWf) + workflowRequest.CiArtifactFileName = workflowRequest.CiArtifactLocation + case types.BLOB_STORAGE_AZURE: + workflowRequest.AzureBlobConfig = &blob_storage.AzureBlobConfig{ + Enabled: impl.config.CloudProvider == types.BLOB_STORAGE_AZURE, + AccountName: impl.config.AzureAccountName, + BlobContainerCiCache: impl.config.AzureBlobContainerCiCache, + AccountKey: impl.config.AzureAccountKey, + BlobContainerCiLog: impl.config.AzureBlobContainerCiLog, + BlobContainerArtifact: impl.config.AzureBlobContainerCiLog, + } + workflowRequest.BlobStorageS3Config = &blob_storage.BlobStorageS3Config{ + EndpointUrl: impl.config.AzureGatewayUrl, + IsInSecure: impl.config.AzureGatewayConnectionInsecure, + CiLogBucketName: impl.config.AzureBlobContainerCiLog, + CiLogRegion: impl.config.DefaultCacheBucketRegion, + CiLogBucketVersioning: impl.config.BlobStorageS3BucketVersioned, + AccessKey: impl.config.AzureAccountName, + } + workflowRequest.CiArtifactLocation = impl.buildDefaultArtifactLocation(savedWf) + workflowRequest.CiArtifactFileName = workflowRequest.CiArtifactLocation + default: + if impl.config.BlobStorageEnabled { + return nil, fmt.Errorf("blob storage %s not supported", workflowRequest.CloudProvider) + } + } + return workflowRequest, nil +} + +func (impl *HandlerServiceImpl) GetWorkflowRequestVariablesForCopyContainerImagePlugin(preCiSteps []*pipelineConfigBean.StepObject, postCiSteps []*pipelineConfigBean.StepObject, customTag string, customTagId int, buildImagePath string, buildImagedockerRegistryId string) (map[string][]string, map[string]bean2.RegistryCredentials, string, []int, error) { + + copyContainerImagePluginDetail, err := impl.globalPluginService.GetRefPluginIdByRefPluginName(buildCommonBean.COPY_CONTAINER_IMAGE) + if err != nil && err != pg.ErrNoRows { + impl.Logger.Errorw("error in getting copyContainerImage plugin id", "err", err) + return nil, nil, "", nil, err + } + + pluginIdToVersionMap := make(map[int]string) + for _, p := range copyContainerImagePluginDetail { + pluginIdToVersionMap[p.Id] = p.Version + } + + for _, step := range preCiSteps { + if _, ok := pluginIdToVersionMap[step.RefPluginId]; ok { + // for copyContainerImage plugin parse destination images and save its data in image path reservation table + return nil, nil, "", nil, errors.New("copyContainerImage plugin not allowed in pre-ci step, please remove it and try again") + } + } + + registryCredentialMap := make(map[string]bean2.RegistryCredentials) + registryDestinationImageMap := make(map[string][]string) + var allDestinationImages []string //saving all images to be reserved in this array + + for _, step := range postCiSteps { + if version, ok := pluginIdToVersionMap[step.RefPluginId]; ok { + destinationImageMap, credentialMap, err := impl.pluginInputVariableParser.HandleCopyContainerImagePluginInputVariables(step.InputVars, customTag, buildImagePath, buildImagedockerRegistryId) + if err != nil { + impl.Logger.Errorw("error in parsing copyContainerImage input variable", "err", err) + return nil, nil, "", nil, err + } + if version == buildCommonBean.COPY_CONTAINER_IMAGE_VERSION_V1 { + // this is needed in ci runner only for v1 + registryDestinationImageMap = destinationImageMap + } + for _, images := range destinationImageMap { + allDestinationImages = append(allDestinationImages, images...) + } + for k, v := range credentialMap { + registryCredentialMap[k] = v + } + } + } + + pluginArtifactStage := repository5.POST_CI + for _, image := range allDestinationImages { + if image == buildImagePath { + return nil, registryCredentialMap, pluginArtifactStage, nil, + pipelineConfigBean.ErrImagePathInUse + } + } + + var imagePathReservationIds []int + if len(allDestinationImages) > 0 { + savedCIArtifacts, err := impl.ciArtifactRepository.FindCiArtifactByImagePaths(allDestinationImages) + if err != nil { + impl.Logger.Errorw("error in fetching artifacts by image path", "err", err) + return nil, nil, pluginArtifactStage, nil, err + } + if len(savedCIArtifacts) > 0 { + // if already present in ci artifact, return "image path already in use error" + return nil, nil, pluginArtifactStage, nil, pipelineConfigBean.ErrImagePathInUse + } + imagePathReservationIds, err = impl.ReserveImagesGeneratedAtPlugin(customTagId, allDestinationImages) + if err != nil { + return nil, nil, pluginArtifactStage, imagePathReservationIds, err + } + } + return registryDestinationImageMap, registryCredentialMap, pluginArtifactStage, imagePathReservationIds, nil +} + +func (impl *HandlerServiceImpl) ReserveImagesGeneratedAtPlugin(customTagId int, destinationImages []string) ([]int, error) { + var imagePathReservationIds []int + for _, image := range destinationImages { + imagePathReservationData, err := impl.customTagService.ReserveImagePath(image, customTagId) + if err != nil { + impl.Logger.Errorw("Error in marking custom tag reserved", "err", err) + return imagePathReservationIds, err + } + imagePathReservationIds = append(imagePathReservationIds, imagePathReservationData.Id) + } + return imagePathReservationIds, nil +} + +func buildCiStepsDataFromDockerBuildScripts(dockerBuildScripts []*bean.CiScript) []*pipelineConfigBean.StepObject { + // before plugin support, few variables were set as env vars in ci-runner + // these variables are now moved to global vars in plugin steps, but to avoid error in old scripts adding those variables in payload + inputVars := []*commonBean.VariableObject{ + { + Name: "DOCKER_IMAGE_TAG", + Format: "STRING", + VariableType: commonBean.VariableTypeRefGlobal, + ReferenceVariableName: "DOCKER_IMAGE_TAG", + }, + { + Name: "DOCKER_REPOSITORY", + Format: "STRING", + VariableType: commonBean.VariableTypeRefGlobal, + ReferenceVariableName: "DOCKER_REPOSITORY", + }, + { + Name: "DOCKER_REGISTRY_URL", + Format: "STRING", + VariableType: commonBean.VariableTypeRefGlobal, + ReferenceVariableName: "DOCKER_REGISTRY_URL", + }, + { + Name: "DOCKER_IMAGE", + Format: "STRING", + VariableType: commonBean.VariableTypeRefGlobal, + ReferenceVariableName: "DOCKER_IMAGE", + }, + } + var ciSteps []*pipelineConfigBean.StepObject + for _, dockerBuildScript := range dockerBuildScripts { + ciStep := &pipelineConfigBean.StepObject{ + Name: dockerBuildScript.Name, + Index: dockerBuildScript.Index, + Script: dockerBuildScript.Script, + ArtifactPaths: []string{dockerBuildScript.OutputLocation}, + StepType: string(repository.PIPELINE_STEP_TYPE_INLINE), + ExecutorType: string(repository2.SCRIPT_TYPE_SHELL), + InputVars: inputVars, + } + ciSteps = append(ciSteps, ciStep) + } + return ciSteps +} + +func (impl *HandlerServiceImpl) buildImageTag(commitHashes map[int]pipelineConfig.GitCommit, id int, wfId int) string { + dockerImageTag := "" + toAppendDevtronParamInTag := true + for _, v := range commitHashes { + if v.WebhookData.Id == 0 { + if v.Commit == "" { + continue + } + dockerImageTag = getUpdatedDockerImageTagWithCommitOrCheckOutData(dockerImageTag, _getTruncatedImageTag(v.Commit)) + } else { + _targetCheckout := v.WebhookData.Data[bean.WEBHOOK_SELECTOR_TARGET_CHECKOUT_NAME] + if _targetCheckout == "" { + continue + } + // if not PR based then meaning tag based + isPRBasedEvent := v.WebhookData.EventActionType == bean.WEBHOOK_EVENT_MERGED_ACTION_TYPE + if !isPRBasedEvent && impl.config.CiCdConfig.UseImageTagFromGitProviderForTagBasedBuild { + dockerImageTag = getUpdatedDockerImageTagWithCommitOrCheckOutData(dockerImageTag, _targetCheckout) + } else { + dockerImageTag = getUpdatedDockerImageTagWithCommitOrCheckOutData(dockerImageTag, _getTruncatedImageTag(_targetCheckout)) + } + if isPRBasedEvent { + _sourceCheckout := v.WebhookData.Data[bean.WEBHOOK_SELECTOR_SOURCE_CHECKOUT_NAME] + dockerImageTag = getUpdatedDockerImageTagWithCommitOrCheckOutData(dockerImageTag, _getTruncatedImageTag(_sourceCheckout)) + } else { + toAppendDevtronParamInTag = !impl.config.CiCdConfig.UseImageTagFromGitProviderForTagBasedBuild + } + } + } + toAppendDevtronParamInTag = toAppendDevtronParamInTag && dockerImageTag != "" + if toAppendDevtronParamInTag { + dockerImageTag = fmt.Sprintf("%s-%d-%d", dockerImageTag, id, wfId) + } + // replace / with underscore, as docker image tag doesn't support slash. it gives error + dockerImageTag = strings.ReplaceAll(dockerImageTag, "/", "_") + return dockerImageTag +} + +func getUpdatedDockerImageTagWithCommitOrCheckOutData(dockerImageTag, commitOrCheckOutData string) string { + if dockerImageTag == "" { + dockerImageTag = commitOrCheckOutData + } else { + if commitOrCheckOutData != "" { + dockerImageTag = fmt.Sprintf("%s-%s", dockerImageTag, commitOrCheckOutData) + } + } + return dockerImageTag +} + +func (impl *HandlerServiceImpl) updateCiWorkflow(request *types.WorkflowRequest, savedWf *pipelineConfig.CiWorkflow) error { + ciBuildConfig := request.CiBuildConfig + ciBuildType := string(ciBuildConfig.CiBuildType) + savedWf.CiBuildType = ciBuildType + return impl.ciService.UpdateCiWorkflowWithStage(savedWf) +} + +func (impl *HandlerServiceImpl) handleRuntimeParamsValidations(trigger types.Trigger, ciMaterials []*pipelineConfig.CiPipelineMaterial, workflowRequest *types.WorkflowRequest) error { + // externalCi artifact is meant only for CI_JOB + if trigger.PipelineType != string(buildCommonBean.CI_JOB) { + return nil + } + + // checking if user has given run time parameters for externalCiArtifact, if given then sending git material to Ci-Runner + externalCiArtifact, exists := trigger.RuntimeParameters.GetGlobalRuntimeVariables()[buildBean.ExtraEnvVarExternalCiArtifactKey] + // validate externalCiArtifact as docker image + if exists { + externalCiArtifact = strings.TrimSpace(externalCiArtifact) + if !strings.Contains(externalCiArtifact, ":") { + if utils.IsValidDockerTagName(externalCiArtifact) { + fullImageUrl, err := utils.BuildDockerImagePath(bean4.DockerRegistryInfo{ + DockerImageTag: externalCiArtifact, + DockerRegistryId: workflowRequest.DockerRegistryId, + DockerRegistryType: workflowRequest.DockerRegistryType, + DockerRegistryURL: workflowRequest.DockerRegistryURL, + DockerRepository: workflowRequest.DockerRepository, + }) + if err != nil { + impl.Logger.Errorw("Error in building docker image", "err", err) + return err + } + externalCiArtifact = fullImageUrl + } else { + impl.Logger.Errorw("validation error", "externalCiArtifact", externalCiArtifact) + return fmt.Errorf("invalid image name or url given in externalCiArtifact") + } + + } + // This will overwrite the existing runtime parameters value for constants.externalCiArtifact + trigger.RuntimeParameters = trigger.RuntimeParameters.AddRuntimeGlobalVariable(buildBean.ExtraEnvVarExternalCiArtifactKey, externalCiArtifact) + var artifactExists bool + var err error + + imageDigest, ok := trigger.RuntimeParameters.GetGlobalRuntimeVariables()[buildBean.ExtraEnvVarImageDigestKey] + if !ok || len(imageDigest) == 0 { + artifactExists, err = impl.ciArtifactRepository.IfArtifactExistByImage(externalCiArtifact, trigger.PipelineId) + if err != nil { + impl.Logger.Errorw("error in fetching ci artifact", "err", err) + return err + } + if artifactExists { + impl.Logger.Errorw("ci artifact already exists with same image name", "artifact", externalCiArtifact) + return fmt.Errorf("ci artifact already exists with same image name") + } + workflowRequest, err = impl.updateWorkflowRequestForDigestPull(trigger.PipelineId, workflowRequest) + if err != nil { + impl.Logger.Errorw("error in updating workflow request", "err", err) + return err + } + } else { + artifactExists, err = impl.ciArtifactRepository.IfArtifactExistByImageDigest(imageDigest, externalCiArtifact, trigger.PipelineId) + if err != nil { + impl.Logger.Errorw("error in fetching ci artifact", "err", err, "imageDigest", imageDigest) + return err + } + if artifactExists { + impl.Logger.Errorw("ci artifact already exists with same digest", "artifact", externalCiArtifact) + return fmt.Errorf("ci artifact already exists with same digest") + } + } + } + if trigger.PipelineType == string(buildCommonBean.CI_JOB) && len(ciMaterials) != 0 && !exists && externalCiArtifact == "" { + ciMaterials[0].GitMaterial = nil + ciMaterials[0].GitMaterialId = 0 + } + return nil +} + +func _getTruncatedImageTag(imageTag string) string { + _length := len(imageTag) + if _length == 0 { + return imageTag + } + + _truncatedLength := 8 + + if _length < _truncatedLength { + return imageTag + } else { + return imageTag[:_truncatedLength] + } +} + +func (impl *HandlerServiceImpl) markCurrentCiWorkflowFailed(savedCiWf *pipelineConfig.CiWorkflow, validationErr error) error { + // currently such requirement is not there + if savedCiWf == nil { + return nil + } + if savedCiWf.Id != 0 && slices.Contains(cdWorkflow.WfrTerminalStatusList, savedCiWf.Status) { + impl.Logger.Debug("workflow is already in terminal state", "status", savedCiWf.Status, "workflowId", savedCiWf.Id, "message", savedCiWf.Message) + return nil + } + + savedCiWf.Status = cdWorkflow.WorkflowFailed + savedCiWf.Message = validationErr.Error() + savedCiWf.FinishedOn = time.Now() + + var dbErr error + if savedCiWf.Id == 0 { + dbErr = impl.ciService.SaveCiWorkflowWithStage(savedCiWf) + } else { + dbErr = impl.ciService.UpdateCiWorkflowWithStage(savedCiWf) + } + + if dbErr != nil { + impl.Logger.Errorw("save/update workflow error", "err", dbErr) + return dbErr + } + return nil +} + +func (impl *HandlerServiceImpl) CancelBuild(workflowId int, forceAbort bool) (int, error) { + workflow, err := impl.ciWorkflowRepository.FindById(workflowId) + if err != nil { + impl.Logger.Errorw("error in finding ci-workflow by workflow id", "ciWorkflowId", workflowId, "err", err) + return 0, err + } + isExt := workflow.Namespace != constants2.DefaultCiWorkflowNamespace + var env *repository6.Environment + var restConfig *rest.Config + if isExt { + restConfig, err = impl.getRestConfig(workflow) + if err != nil { + return 0, err + } + } + // Terminate workflow + cancelWfDtoRequest := &types.CancelWfRequestDto{ + ExecutorType: workflow.ExecutorType, + WorkflowName: workflow.Name, + Namespace: workflow.Namespace, + RestConfig: restConfig, + IsExt: isExt, + Environment: env, + } + // Terminate workflow + err = impl.workflowService.TerminateWorkflow(cancelWfDtoRequest) + if err != nil && forceAbort { + impl.Logger.Errorw("error in terminating workflow, with force abort flag flag as true", "workflowName", workflow.Name, "err", err) + + cancelWfDtoRequest.WorkflowGenerateName = fmt.Sprintf("%d-%s", workflowId, workflow.Name) + err1 := impl.workflowService.TerminateDanglingWorkflows(cancelWfDtoRequest) + if err1 != nil { + impl.Logger.Errorw("error in terminating dangling workflows", "cancelWfDtoRequest", cancelWfDtoRequest, "err", err) + // ignoring error here in case of force abort, confirmed from product + } + } else if err != nil && strings.Contains(err.Error(), "cannot find workflow") { + return 0, &util.ApiError{Code: "200", HttpStatusCode: http.StatusBadRequest, UserMessage: err.Error()} + } else if err != nil { + impl.Logger.Errorw("cannot terminate wf", "err", err) + return 0, err + } + if forceAbort { + err = impl.handleForceAbortCaseForCi(workflow, forceAbort) + if err != nil { + impl.Logger.Errorw("error in handleForceAbortCaseForCi", "forceAbortFlag", forceAbort, "workflow", workflow, "err", err) + return 0, err + } + return workflow.Id, nil + } + + workflow.Status = cdWorkflow.WorkflowCancel + if workflow.ExecutorType == cdWorkflow.WORKFLOW_EXECUTOR_TYPE_SYSTEM { + workflow.PodStatus = "Failed" + workflow.Message = constants2.TERMINATE_MESSAGE + } + err = impl.ciService.UpdateCiWorkflowWithStage(workflow) + if err != nil { + impl.Logger.Errorw("cannot update deleted workflow status, but wf deleted", "err", err) + return 0, err + } + imagePathReservationId := workflow.ImagePathReservationId + err = impl.customTagService.DeactivateImagePathReservation(imagePathReservationId) + if err != nil { + impl.Logger.Errorw("error in marking image tag unreserved", "err", err) + return 0, err + } + imagePathReservationIds := workflow.ImagePathReservationIds + if len(imagePathReservationIds) > 0 { + err = impl.customTagService.DeactivateImagePathReservationByImageIds(imagePathReservationIds) + if err != nil { + impl.Logger.Errorw("error in marking image tag unreserved", "err", err) + return 0, err + } + } + return workflow.Id, nil +} + +func (impl *HandlerServiceImpl) handleForceAbortCaseForCi(workflow *pipelineConfig.CiWorkflow, forceAbort bool) error { + isWorkflowInNonTerminalStage := workflow.Status == string(v1alpha1.NodePending) || workflow.Status == string(v1alpha1.NodeRunning) + if !isWorkflowInNonTerminalStage { + if forceAbort { + return impl.updateWorkflowForForceAbort(workflow) + } else { + return &util.ApiError{Code: "200", HttpStatusCode: 400, UserMessage: "cannot cancel build, build not in progress"} + } + } + //this arises when someone deletes the workflow in resource browser and wants to force abort a ci + if workflow.Status == string(v1alpha1.NodeRunning) && forceAbort { + return impl.updateWorkflowForForceAbort(workflow) + } + return nil +} + +func (impl *HandlerServiceImpl) updateWorkflowForForceAbort(workflow *pipelineConfig.CiWorkflow) error { + workflow.Status = cdWorkflow.WorkflowCancel + workflow.PodStatus = string(bean.Failed) + workflow.Message = constants2.FORCE_ABORT_MESSAGE_AFTER_STARTING_STAGE + err := impl.ciService.UpdateCiWorkflowWithStage(workflow) + if err != nil { + impl.Logger.Errorw("error in updating workflow status", "err", err) + return err + } + return nil +} + +func (impl *HandlerServiceImpl) getRestConfig(workflow *pipelineConfig.CiWorkflow) (*rest.Config, error) { + env, err := impl.envRepository.FindById(workflow.EnvironmentId) + if err != nil { + impl.Logger.Errorw("could not fetch stage env", "err", err) + return nil, err + } + + clusterBean := adapter2.GetClusterBean(*env.Cluster) + + clusterConfig := clusterBean.GetClusterConfig() + restConfig, err := impl.K8sUtil.GetRestConfigByCluster(clusterConfig) + if err != nil { + impl.Logger.Errorw("error in getting rest config by cluster id", "err", err) + return nil, err + } + return restConfig, nil +} + +func (impl *HandlerServiceImpl) GetRunningWorkflowLogs(workflowId int) (*bufio.Reader, func() error, error) { + ciWorkflow, err := impl.ciWorkflowRepository.FindById(workflowId) + if err != nil { + impl.Logger.Errorw("err", "err", err) + return nil, nil, err + } + return impl.getWorkflowLogs(ciWorkflow) +} + +func (impl *HandlerServiceImpl) getWorkflowLogs(ciWorkflow *pipelineConfig.CiWorkflow) (*bufio.Reader, func() error, error) { + if string(v1alpha1.NodePending) == ciWorkflow.PodStatus { + return bufio.NewReader(strings.NewReader("")), func() error { return nil }, nil + } + ciLogRequest := types.BuildLogRequest{ + PodName: ciWorkflow.PodName, + Namespace: ciWorkflow.Namespace, + } + isExt := false + clusterConfig := &k8s.ClusterConfig{} + if ciWorkflow.EnvironmentId != 0 { + env, err := impl.envRepository.FindById(ciWorkflow.EnvironmentId) + if err != nil { + return nil, nil, err + } + var clusterBean clusterBean.ClusterBean + if env != nil && env.Cluster != nil { + clusterBean = adapter2.GetClusterBean(*env.Cluster) + } + clusterConfig = clusterBean.GetClusterConfig() + isExt = true + } + + logStream, cleanUp, err := impl.ciLogService.FetchRunningWorkflowLogs(ciLogRequest, clusterConfig, isExt) + if logStream == nil || err != nil { + if !ciWorkflow.BlobStorageEnabled { + return nil, nil, &util.ApiError{Code: "200", HttpStatusCode: 400, UserMessage: "logs-not-stored-in-repository"} + } else if string(v1alpha1.NodeSucceeded) == ciWorkflow.Status || string(v1alpha1.NodeError) == ciWorkflow.Status || string(v1alpha1.NodeFailed) == ciWorkflow.Status || ciWorkflow.Status == cdWorkflow.WorkflowCancel { + impl.Logger.Debugw("pod is not live", "podName", ciWorkflow.PodName, "err", err) + return impl.getLogsFromRepository(ciWorkflow, clusterConfig, isExt) + } + if err != nil { + impl.Logger.Errorw("err on fetch workflow logs", "err", err) + return nil, nil, &util.ApiError{Code: "200", HttpStatusCode: 400, UserMessage: err.Error()} + } else if logStream == nil { + return nil, cleanUp, fmt.Errorf("no logs found for pod %s", ciWorkflow.PodName) + } + } + logReader := bufio.NewReader(logStream) + return logReader, cleanUp, err +} + +func (impl *HandlerServiceImpl) getLogsFromRepository(ciWorkflow *pipelineConfig.CiWorkflow, clusterConfig *k8s.ClusterConfig, isExt bool) (*bufio.Reader, func() error, error) { + impl.Logger.Debug("getting historic logs", "ciWorkflowId", ciWorkflow.Id) + ciConfigLogsBucket := impl.config.GetDefaultBuildLogsBucket() + ciConfigCiCacheRegion := impl.config.DefaultCacheBucketRegion + logsFilePath := impl.config.GetDefaultBuildLogsKeyPrefix() + "/" + ciWorkflow.Name + "/main.log" // this is for backward compatibilty + if strings.Contains(ciWorkflow.LogLocation, "main.log") { + logsFilePath = ciWorkflow.LogLocation + } + ciLogRequest := types.BuildLogRequest{ + PipelineId: ciWorkflow.CiPipelineId, + WorkflowId: ciWorkflow.Id, + PodName: ciWorkflow.PodName, + LogsFilePath: logsFilePath, + CloudProvider: impl.config.CloudProvider, + AzureBlobConfig: &blob_storage.AzureBlobBaseConfig{ + Enabled: impl.config.CloudProvider == types.BLOB_STORAGE_AZURE, + AccountName: impl.config.AzureAccountName, + BlobContainerName: impl.config.AzureBlobContainerCiLog, + AccountKey: impl.config.AzureAccountKey, + }, + AwsS3BaseConfig: &blob_storage.AwsS3BaseConfig{ + AccessKey: impl.config.BlobStorageS3AccessKey, + Passkey: impl.config.BlobStorageS3SecretKey, + EndpointUrl: impl.config.BlobStorageS3Endpoint, + IsInSecure: impl.config.BlobStorageS3EndpointInsecure, + BucketName: ciConfigLogsBucket, + Region: ciConfigCiCacheRegion, + VersioningEnabled: impl.config.BlobStorageS3BucketVersioned, + }, + GcpBlobBaseConfig: &blob_storage.GcpBlobBaseConfig{ + BucketName: ciConfigLogsBucket, + CredentialFileJsonData: impl.config.BlobStorageGcpCredentialJson, + }, + } + useExternalBlobStorage := pipeline2.IsExternalBlobStorageEnabled(isExt, impl.config.UseBlobStorageConfigInCiWorkflow) + if useExternalBlobStorage { + // fetch extClusterBlob cm and cs from k8s client, if they are present then read creds + // from them else return. + cmConfig, secretConfig, err := impl.blobConfigStorageService.FetchCmAndSecretBlobConfigFromExternalCluster(clusterConfig, ciWorkflow.Namespace) + if err != nil { + impl.Logger.Errorw("error in fetching config map and secret from external cluster", "err", err, "clusterConfig", clusterConfig) + return nil, nil, err + } + rq := &ciLogRequest + rq.SetBuildLogRequest(cmConfig, secretConfig) + } + oldLogsStream, cleanUp, err := impl.ciLogService.FetchLogs(impl.config.BaseLogLocationPath, ciLogRequest) + if err != nil { + impl.Logger.Errorw("err", "err", err) + return nil, nil, err + } + logReader := bufio.NewReader(oldLogsStream) + return logReader, cleanUp, err +} + +func (impl *HandlerServiceImpl) GetHistoricBuildLogs(workflowId int, ciWorkflow *pipelineConfig.CiWorkflow) (map[string]string, error) { + var err error + if ciWorkflow == nil { + ciWorkflow, err = impl.ciWorkflowRepository.FindById(workflowId) + if err != nil { + impl.Logger.Errorw("err", "err", err) + return nil, err + } + } + ciConfigLogsBucket := impl.config.GetDefaultBuildLogsBucket() + ciConfigCiCacheRegion := impl.config.DefaultCacheBucketRegion + ciLogRequest := types.BuildLogRequest{ + PipelineId: ciWorkflow.CiPipelineId, + WorkflowId: ciWorkflow.Id, + PodName: ciWorkflow.PodName, + LogsFilePath: ciWorkflow.LogLocation, + CloudProvider: impl.config.CloudProvider, + AzureBlobConfig: &blob_storage.AzureBlobBaseConfig{ + Enabled: impl.config.CloudProvider == types.BLOB_STORAGE_AZURE, + AccountName: impl.config.AzureAccountName, + BlobContainerName: impl.config.AzureBlobContainerCiLog, + AccountKey: impl.config.AzureAccountKey, + }, + AwsS3BaseConfig: &blob_storage.AwsS3BaseConfig{ + AccessKey: impl.config.BlobStorageS3AccessKey, + Passkey: impl.config.BlobStorageS3SecretKey, + EndpointUrl: impl.config.BlobStorageS3Endpoint, + IsInSecure: impl.config.BlobStorageS3EndpointInsecure, + BucketName: ciConfigLogsBucket, + Region: ciConfigCiCacheRegion, + VersioningEnabled: impl.config.BlobStorageS3BucketVersioned, + }, + GcpBlobBaseConfig: &blob_storage.GcpBlobBaseConfig{ + BucketName: ciConfigLogsBucket, + CredentialFileJsonData: impl.config.BlobStorageGcpCredentialJson, + }, + } + useExternalBlobStorage := pipeline2.IsExternalBlobStorageEnabled(ciWorkflow.IsExternalRunInJobType(), impl.config.UseBlobStorageConfigInCiWorkflow) + if useExternalBlobStorage { + envBean, err := impl.envService.FindById(ciWorkflow.EnvironmentId) + if err != nil { + impl.Logger.Errorw("error in getting envBean by envId", "err", err, "envId", ciWorkflow.EnvironmentId) + return nil, err + } + clusterConfig, err := impl.clusterService.GetClusterConfigByClusterId(envBean.ClusterId) + if err != nil { + impl.Logger.Errorw("GetClusterConfigByClusterId, error in fetching clusterConfig by clusterId", "err", err, "clusterId", envBean.ClusterId) + return nil, err + } + // fetch extClusterBlob cm and cs from k8s client, if they are present then read creds + // from them else return. + cmConfig, secretConfig, err := impl.blobConfigStorageService.FetchCmAndSecretBlobConfigFromExternalCluster(clusterConfig, ciWorkflow.Namespace) + if err != nil { + impl.Logger.Errorw("error in fetching config map and secret from external cluster", "err", err, "clusterConfig", clusterConfig) + return nil, err + } + rq := &ciLogRequest + rq.SetBuildLogRequest(cmConfig, secretConfig) + } + logsFile, cleanUp, err := impl.ciLogService.FetchLogs(impl.config.BaseLogLocationPath, ciLogRequest) + logs, err := ioutil.ReadFile(logsFile.Name()) + if err != nil { + impl.Logger.Errorw("err", "err", err) + return map[string]string{}, err + } + logStr := string(logs) + resp := make(map[string]string) + resp["logs"] = logStr + defer cleanUp() + return resp, err +} + +func (impl *HandlerServiceImpl) DownloadCiWorkflowArtifacts(pipelineId int, buildId int) (*os.File, error) { + ciWorkflow, err := impl.ciWorkflowRepository.FindById(buildId) + if err != nil { + impl.Logger.Errorw("unable to fetch ciWorkflow", "err", err) + return nil, err + } + useExternalBlobStorage := pipeline2.IsExternalBlobStorageEnabled(ciWorkflow.IsExternalRunInJobType(), impl.config.UseBlobStorageConfigInCiWorkflow) + if !ciWorkflow.BlobStorageEnabled { + return nil, errors.New("logs-not-stored-in-repository") + } + + if ciWorkflow.CiPipelineId != pipelineId { + impl.Logger.Error("invalid request, wf not in pipeline") + return nil, errors.New("invalid request, wf not in pipeline") + } + + ciConfigLogsBucket := impl.config.GetDefaultBuildLogsBucket() + item := strconv.Itoa(ciWorkflow.Id) + ciConfigCiCacheRegion := impl.config.DefaultCacheBucketRegion + azureBlobConfig := &blob_storage.AzureBlobBaseConfig{ + Enabled: impl.config.CloudProvider == types.BLOB_STORAGE_AZURE, + AccountName: impl.config.AzureAccountName, + BlobContainerName: impl.config.AzureBlobContainerCiLog, + AccountKey: impl.config.AzureAccountKey, + } + awsS3BaseConfig := &blob_storage.AwsS3BaseConfig{ + AccessKey: impl.config.BlobStorageS3AccessKey, + Passkey: impl.config.BlobStorageS3SecretKey, + EndpointUrl: impl.config.BlobStorageS3Endpoint, + IsInSecure: impl.config.BlobStorageS3EndpointInsecure, + BucketName: ciConfigLogsBucket, + Region: ciConfigCiCacheRegion, + VersioningEnabled: impl.config.BlobStorageS3BucketVersioned, + } + gcpBlobBaseConfig := &blob_storage.GcpBlobBaseConfig{ + BucketName: ciConfigLogsBucket, + CredentialFileJsonData: impl.config.BlobStorageGcpCredentialJson, + } + + ciArtifactLocationFormat := impl.config.GetArtifactLocationFormat() + key := fmt.Sprintf(ciArtifactLocationFormat, ciWorkflow.Id, ciWorkflow.Id) + if len(ciWorkflow.CiArtifactLocation) != 0 && util2.IsValidUrlSubPath(ciWorkflow.CiArtifactLocation) { + key = ciWorkflow.CiArtifactLocation + } else if util2.IsValidUrlSubPath(key) { + impl.ciWorkflowRepository.MigrateCiArtifactLocation(ciWorkflow.Id, key) + } + baseLogLocationPathConfig := impl.config.BaseLogLocationPath + blobStorageService := blob_storage.NewBlobStorageServiceImpl(nil) + destinationKey := filepath.Clean(filepath.Join(baseLogLocationPathConfig, item)) + request := &blob_storage.BlobStorageRequest{ + StorageType: impl.config.CloudProvider, + SourceKey: key, + DestinationKey: destinationKey, + AzureBlobBaseConfig: azureBlobConfig, + AwsS3BaseConfig: awsS3BaseConfig, + GcpBlobBaseConfig: gcpBlobBaseConfig, + } + if useExternalBlobStorage { + envBean, err := impl.envService.FindById(ciWorkflow.EnvironmentId) + if err != nil { + impl.Logger.Errorw("error in getting envBean by envId", "err", err, "envId", ciWorkflow.EnvironmentId) + return nil, err + } + clusterConfig, err := impl.clusterService.GetClusterConfigByClusterId(envBean.ClusterId) + if err != nil { + impl.Logger.Errorw("GetClusterConfigByClusterId, error in fetching clusterConfig by clusterId", "err", err, "clusterId", envBean.ClusterId) + return nil, err + } + // fetch extClusterBlob cm and cs from k8s client, if they are present then read creds + // from them else return. + cmConfig, secretConfig, err := impl.blobConfigStorageService.FetchCmAndSecretBlobConfigFromExternalCluster(clusterConfig, ciWorkflow.Namespace) + if err != nil { + impl.Logger.Errorw("error in fetching config map and secret from external cluster", "err", err, "clusterConfig", clusterConfig) + return nil, err + } + request = pipeline2.UpdateRequestWithExtClusterCmAndSecret(request, cmConfig, secretConfig) + } + _, numBytes, err := blobStorageService.Get(request) + if err != nil { + impl.Logger.Errorw("error occurred while downloading file", "request", request, "error", err) + return nil, errors.New("failed to download resource") + } + + file, err := os.Open(destinationKey) + if err != nil { + impl.Logger.Errorw("unable to open file", "file", item, "err", err) + return nil, errors.New("unable to open file") + } + + impl.Logger.Infow("Downloaded ", "filename", file.Name(), "bytes", numBytes) + return file, nil +} diff --git a/pkg/build/trigger/HandlerService_ent.go b/pkg/build/trigger/HandlerService_ent.go new file mode 100644 index 0000000000..3938288091 --- /dev/null +++ b/pkg/build/trigger/HandlerService_ent.go @@ -0,0 +1,104 @@ +package trigger + +import ( + "encoding/json" + "github.com/devtron-labs/common-lib/imageScan/bean" + repository3 "github.com/devtron-labs/devtron/internal/sql/repository/dockerRegistry" + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" + "github.com/devtron-labs/devtron/internal/util" + bean3 "github.com/devtron-labs/devtron/pkg/bean" + "github.com/devtron-labs/devtron/pkg/bean/common" + bean2 "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" + repository2 "github.com/devtron-labs/devtron/pkg/cluster/environment/repository" + pipelineConfigBean "github.com/devtron-labs/devtron/pkg/pipeline/bean" + "github.com/devtron-labs/devtron/pkg/pipeline/types" + "github.com/devtron-labs/devtron/pkg/policyGovernance/security/scanTool/repository" + "github.com/devtron-labs/devtron/pkg/resourceQualifiers" + "net/http" +) + +func (impl *HandlerServiceImpl) updateRuntimeParamsForAutoCI(ciPipelineId int, runtimeParameters *common.RuntimeParameters) (*common.RuntimeParameters, error) { + return runtimeParameters, nil +} + +func (impl *HandlerServiceImpl) getRuntimeParamsForBuildingManualTriggerHashes(ciTriggerRequest bean3.CiTriggerRequest) *common.RuntimeParameters { + return common.NewRuntimeParameters() +} + +func (impl *HandlerServiceImpl) fetchImageScanExecutionMedium() (*repository.ScanToolMetadata, bean.ScanExecutionMedium, error) { + return &repository.ScanToolMetadata{}, "", nil +} + +func (impl *HandlerServiceImpl) fetchImageScanExecutionStepsForWfRequest(scanToolMetadata *repository.ScanToolMetadata) ([]*types.ImageScanningSteps, []*pipelineConfigBean.RefPluginObject, error) { + return nil, nil, nil +} + +func (impl *HandlerServiceImpl) checkIfCITriggerIsBlocked(pipeline *pipelineConfig.CiPipeline, + ciMaterials []*pipelineConfig.CiPipelineMaterial, isJob bool) (bool, error) { + return false, nil +} + +func (impl *HandlerServiceImpl) handleWFIfCITriggerIsBlocked(ciWorkflow *pipelineConfig.CiWorkflow) (*pipelineConfig.CiWorkflow, error) { + impl.Logger.Errorw("cannot trigger pipeline, blocked by mandatory plugin policy", "ciPipelineId", ciWorkflow.CiPipelineId) + return &pipelineConfig.CiWorkflow{}, util.GetApiErrorAdapter(http.StatusInternalServerError, "500", "Invalid flow access, corrupt data possibility", "Invalid flow access, corrupt data possibility") +} + +func (impl *HandlerServiceImpl) checkArgoSetupRequirement(envModal *repository2.Environment) error { + return nil +} + +func (impl *HandlerServiceImpl) updateWorkflowRequestForDigestPull(pipelineId int, workflowRequest *types.WorkflowRequest) (*types.WorkflowRequest, error) { + return workflowRequest, nil +} + +func (impl *HandlerServiceImpl) updateCIProjectDetailWithCloningMode(appId int, ciMaterial *pipelineConfig.CiPipelineMaterial, + ciProjectDetail pipelineConfigBean.CiProjectDetails) (pipelineConfigBean.CiProjectDetails, error) { + return ciProjectDetail, nil +} + +func (impl *HandlerServiceImpl) updateWorkflowRequestWithRemoteConnConf(dockerRegistry *repository3.DockerArtifactStore, + workflowRequest *types.WorkflowRequest) (*types.WorkflowRequest, error) { + return workflowRequest, nil +} + +func (impl *HandlerServiceImpl) updateWorkflowRequestWithEntSupportData(workflowRequest *types.WorkflowRequest) *types.WorkflowRequest { + return workflowRequest +} + +func (impl *HandlerServiceImpl) updateWorkflowRequestWithBuildCacheData(workflowRequest *types.WorkflowRequest, + scope resourceQualifiers.Scope) (*types.WorkflowRequest, error) { + workflowRequest.BuildxCacheModeMin = impl.buildxCacheFlags.BuildxCacheModeMin + workflowRequest.AsyncBuildxCacheExport = impl.buildxCacheFlags.AsyncBuildxCacheExport + return workflowRequest, nil +} + +func (impl *HandlerServiceImpl) canSetK8sDriverData(workflowRequest *types.WorkflowRequest) bool { + return impl.config != nil && impl.config.BuildxK8sDriverOptions != "" && workflowRequest.CiBuildConfig != nil && + workflowRequest.CiBuildConfig.DockerBuildConfig != nil +} + +func (impl *HandlerServiceImpl) getK8sDriverOptions(workflowRequest *types.WorkflowRequest, targetPlatforms string) ([]map[string]string, error) { + buildxK8sDriverOptions := make([]map[string]string, 0) + err := json.Unmarshal([]byte(impl.config.BuildxK8sDriverOptions), &buildxK8sDriverOptions) + if err != nil { + return nil, err + } + return buildxK8sDriverOptions, nil +} + +func (impl *HandlerServiceImpl) updateCIBuildConfig(ciBuildConfigBean *bean2.CiBuildConfigBean) *bean2.CiBuildConfigBean { + defaultTargetPlatform := impl.config.DefaultTargetPlatform + useBuildx := impl.config.UseBuildx + if ciBuildConfigBean.DockerBuildConfig != nil { + if ciBuildConfigBean.DockerBuildConfig.TargetPlatform == "" && useBuildx { + ciBuildConfigBean.DockerBuildConfig.TargetPlatform = defaultTargetPlatform + ciBuildConfigBean.DockerBuildConfig.UseBuildx = useBuildx + } + ciBuildConfigBean.DockerBuildConfig.BuildxProvenanceMode = impl.config.BuildxProvenanceMode + } + return ciBuildConfigBean +} + +func updateBuildPrePostStepDataReq(req *pipelineConfigBean.BuildPrePostStepDataRequest, trigger types.Trigger) *pipelineConfigBean.BuildPrePostStepDataRequest { + return req +} diff --git a/pkg/build/trigger/wire_trigger.go b/pkg/build/trigger/wire_trigger.go new file mode 100644 index 0000000000..aca718e3cd --- /dev/null +++ b/pkg/build/trigger/wire_trigger.go @@ -0,0 +1,10 @@ +package trigger + +import ( + "github.com/google/wire" +) + +var WireSet = wire.NewSet( + NewHandlerServiceImpl, + wire.Bind(new(HandlerService), new(*HandlerServiceImpl)), +) diff --git a/pkg/build/wire_build.go b/pkg/build/wire_build.go index 6610dd24cc..32c2fc0b0b 100644 --- a/pkg/build/wire_build.go +++ b/pkg/build/wire_build.go @@ -20,6 +20,7 @@ import ( "github.com/devtron-labs/devtron/pkg/build/artifacts" "github.com/devtron-labs/devtron/pkg/build/git" "github.com/devtron-labs/devtron/pkg/build/pipeline" + "github.com/devtron-labs/devtron/pkg/build/trigger" "github.com/google/wire" ) @@ -27,4 +28,5 @@ var WireSet = wire.NewSet( artifacts.WireSet, pipeline.WireSet, git.GitWireSet, + trigger.WireSet, ) diff --git a/pkg/bulkAction/service/BulkUpdateService.go b/pkg/bulkAction/service/BulkUpdateService.go index 3b94648b69..3e6c0f7f78 100644 --- a/pkg/bulkAction/service/BulkUpdateService.go +++ b/pkg/bulkAction/service/BulkUpdateService.go @@ -35,6 +35,7 @@ import ( "github.com/devtron-labs/devtron/internal/util" appWorkflow2 "github.com/devtron-labs/devtron/pkg/appWorkflow" bean2 "github.com/devtron-labs/devtron/pkg/bean" + "github.com/devtron-labs/devtron/pkg/build/trigger" bean4 "github.com/devtron-labs/devtron/pkg/bulkAction/bean" "github.com/devtron-labs/devtron/pkg/bulkAction/utils" chartRepoRepository "github.com/devtron-labs/devtron/pkg/chartRepo/repository" @@ -102,6 +103,7 @@ type BulkUpdateServiceImpl struct { chartRefService chartRef.ChartRefService deployedAppService deployedApp.DeployedAppService cdPipelineEventPublishService out.CDPipelineEventPublishService + ciHandlerService trigger.HandlerService } func NewBulkUpdateServiceImpl(bulkUpdateRepository bulkUpdate.BulkUpdateRepository, @@ -121,7 +123,8 @@ func NewBulkUpdateServiceImpl(bulkUpdateRepository bulkUpdate.BulkUpdateReposito deployedAppMetricsService deployedAppMetrics.DeployedAppMetricsService, chartRefService chartRef.ChartRefService, deployedAppService deployedApp.DeployedAppService, - cdPipelineEventPublishService out.CDPipelineEventPublishService) *BulkUpdateServiceImpl { + cdPipelineEventPublishService out.CDPipelineEventPublishService, + ciHandlerService trigger.HandlerService) *BulkUpdateServiceImpl { return &BulkUpdateServiceImpl{ bulkUpdateRepository: bulkUpdateRepository, logger: logger, @@ -141,6 +144,7 @@ func NewBulkUpdateServiceImpl(bulkUpdateRepository bulkUpdate.BulkUpdateReposito chartRefService: chartRefService, deployedAppService: deployedAppService, cdPipelineEventPublishService: cdPipelineEventPublishService, + ciHandlerService: ciHandlerService, } } @@ -1467,7 +1471,7 @@ func (impl BulkUpdateServiceImpl) BulkBuildTrigger(request *bean4.BulkApplicatio } ciTriggerRequest := latestCommitsMap[pipeline.CiPipelineId] - _, err = impl.ciHandler.HandleCIManual(ciTriggerRequest) + _, err = impl.ciHandlerService.HandleCIManual(ciTriggerRequest) if err != nil { impl.logger.Errorw("service err, HandleCIManual", "err", err, "ciTriggerRequest", ciTriggerRequest) //return nil, err diff --git a/pkg/chartRepo/ChartRepositoryService.go b/pkg/chartRepo/ChartRepositoryService.go index 1df2c48e43..8cdef0a9fd 100644 --- a/pkg/chartRepo/ChartRepositoryService.go +++ b/pkg/chartRepo/ChartRepositoryService.go @@ -501,7 +501,7 @@ func (impl *ChartRepositoryServiceImpl) ValidateAndUpdateChartRepo(request *Char } func (impl *ChartRepositoryServiceImpl) TriggerChartSyncManual(chartProviderConfig *ChartProviderConfig) error { - defaultClusterBean, err := impl.clusterReadService.FindOne(bean2.DEFAULT_CLUSTER) + defaultClusterBean, err := impl.clusterReadService.FindOne(bean2.DefaultCluster) if err != nil { impl.logger.Errorw("defaultClusterBean err, TriggerChartSyncManual", "err", err) return err diff --git a/pkg/cluster/ClusterService.go b/pkg/cluster/ClusterService.go index 9bab50066a..1b00c2f84f 100644 --- a/pkg/cluster/ClusterService.go +++ b/pkg/cluster/ClusterService.go @@ -20,6 +20,7 @@ import ( "context" "encoding/json" "fmt" + informerBean "github.com/devtron-labs/common-lib/informer" "github.com/devtron-labs/common-lib/utils/k8s/commonBean" "github.com/devtron-labs/devtron/pkg/cluster/adapter" "github.com/devtron-labs/devtron/pkg/cluster/bean" @@ -28,7 +29,6 @@ import ( cronUtil "github.com/devtron-labs/devtron/util/cron" "github.com/robfig/cron/v3" "log" - "net/http" "net/url" "sync" "time" @@ -38,8 +38,8 @@ import ( casbin2 "github.com/devtron-labs/devtron/pkg/auth/authorisation/casbin" repository3 "github.com/devtron-labs/devtron/pkg/auth/user/repository" "github.com/devtron-labs/devtron/pkg/k8s/informer" - errors1 "github.com/juju/errors" - "k8s.io/apimachinery/pkg/api/errors" + customErr "github.com/juju/errors" + k8sError "k8s.io/apimachinery/pkg/api/errors" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/kubernetes" @@ -56,18 +56,6 @@ import ( "go.uber.org/zap" ) -const ( - DEFAULT_CLUSTER = "default_cluster" - DEFAULT_NAMESPACE = "default" - CLUSTER_MODIFY_EVENT_SECRET_TYPE = "cluster.request/modify" - CLUSTER_ACTION_ADD = "add" - CLUSTER_ACTION_UPDATE = "update" - SECRET_FIELD_CLUSTER_ID = "cluster_id" - SECRET_FIELD_UPDATED_ON = "updated_on" - SECRET_FIELD_ACTION = "action" - TokenFilePath = "/var/run/secrets/kubernetes.io/serviceaccount/token" -) - type ClusterService interface { Save(parent context.Context, bean *bean.ClusterBean, userId int32) (*bean.ClusterBean, error) UpdateClusterDescription(bean *bean.ClusterBean, userId int32) error @@ -195,7 +183,7 @@ func (impl *ClusterServiceImpl) Save(parent context.Context, bean *bean.ClusterB if err != nil { if len(err.Error()) > 2000 { - err = errors.NewBadRequest("unable to connect to cluster") + err = k8sError.NewBadRequest("unable to connect to cluster") } return nil, err } @@ -248,16 +236,16 @@ func (impl *ClusterServiceImpl) Save(parent context.Context, bean *bean.ClusterB secretName := ParseSecretNameForKubelinkInformer(bean.Id) data := make(map[string][]byte) - data[SECRET_FIELD_CLUSTER_ID] = []byte(fmt.Sprintf("%v", bean.Id)) - data[SECRET_FIELD_ACTION] = []byte(CLUSTER_ACTION_ADD) - data[SECRET_FIELD_UPDATED_ON] = []byte(time.Now().String()) // this field will ensure that informer detects change as other fields can be constant even if cluster config changes - _, err = impl.K8sUtil.CreateSecret(DEFAULT_NAMESPACE, data, secretName, CLUSTER_MODIFY_EVENT_SECRET_TYPE, k8sClient, nil, nil) + data[informerBean.SecretFieldClusterId] = []byte(fmt.Sprintf("%v", bean.Id)) + data[informerBean.SecretFieldAction] = []byte(informerBean.ClusterActionAdd) + data[clusterBean.SecretFieldUpdatedOn] = []byte(time.Now().String()) // this field will ensure that informer detects change as other fields can be constant even if cluster config changes + // TODO Asutosh: Why not UPSERT ?? + _, err = impl.K8sUtil.CreateSecret(clusterBean.DefaultNamespace, data, secretName, informerBean.ClusterModifyEventSecretType, k8sClient, nil, nil) if err != nil { - impl.logger.Errorw("error in updating secret for informers") + impl.logger.Errorw("error in creating secret for informers", "secretName", secretName, "err", err) return bean, nil } - - return bean, err + return bean, nil } // UpdateClusterDescription is new api service logic to only update description, this should be done in cluster update operation only @@ -415,7 +403,7 @@ func (impl *ClusterServiceImpl) Update(ctx context.Context, bean *bean.ClusterBe } if bean.ServerUrl != model.ServerUrl || bean.InsecureSkipTLSVerify != model.InsecureSkipTlsVerify || dbConfigBearerToken != requestConfigBearerToken || dbConfigTlsKey != requestConfigTlsKey || dbConfigCertData != requestConfigCertData || dbConfigCAData != requestConfigCAData { - if bean.ClusterName == clusterBean.DEFAULT_CLUSTER { + if bean.ClusterName == clusterBean.DefaultCluster { impl.logger.Errorw("default_cluster is reserved by the system and cannot be updated, default_cluster", "name", bean.ClusterName) return nil, fmt.Errorf("default_cluster is reserved by the system and cannot be updated") } @@ -480,37 +468,47 @@ func (impl *ClusterServiceImpl) Update(ctx context.Context, bean *bean.ClusterBe impl.SyncNsInformer(bean) } impl.logger.Infow("saving secret for cluster informer") + if bean.HasConfigOrUrlChanged { + data := make(map[string][]byte) + data[informerBean.SecretFieldClusterId] = []byte(fmt.Sprintf("%v", bean.Id)) + data[informerBean.SecretFieldAction] = []byte(informerBean.ClusterActionUpdate) + data[clusterBean.SecretFieldUpdatedOn] = []byte(time.Now().String()) // this field will ensure that informer detects change as other fields can be constant even if cluster config changes + if err = impl.upsertClusterSecret(bean, data); err != nil { + impl.logger.Errorw("error upserting cluster secret", "data", data, "err", err) + // TODO Asutosh: why error is not propagated ?? + return bean, nil + } + } + return bean, nil +} + +func (impl *ClusterServiceImpl) upsertClusterSecret(bean *bean.ClusterBean, data map[string][]byte) error { k8sClient, err := impl.K8sUtil.GetCoreV1ClientInCluster() if err != nil { - return bean, nil + impl.logger.Errorw("error in getting k8s client", "err", err) + return err } - // below secret will act as an event for informer running on secret object in kubelink - if bean.HasConfigOrUrlChanged { - secretName := ParseSecretNameForKubelinkInformer(bean.Id) - secret, err := impl.K8sUtil.GetSecret(DEFAULT_NAMESPACE, secretName, k8sClient) - statusError, _ := err.(*errors.StatusError) - if err != nil && statusError.Status().Code != http.StatusNotFound { - impl.logger.Errorw("secret not found", "err", err) - return bean, nil + // below secret will act as an event for informer running on a secret object in kubelink and kubewatch + secretName := ParseSecretNameForKubelinkInformer(bean.Id) + secret, err := impl.K8sUtil.GetSecret(clusterBean.DefaultNamespace, secretName, k8sClient) + if err != nil && !k8sError.IsNotFound(err) { + impl.logger.Errorw("error in getting cluster secret", "secretName", secretName, "err", err) + return err + } else if k8sError.IsNotFound(err) { + _, err = impl.K8sUtil.CreateSecret(clusterBean.DefaultNamespace, data, secretName, informerBean.ClusterModifyEventSecretType, k8sClient, nil, nil) + if err != nil { + impl.logger.Errorw("error in creating secret for informers", "secretName", secretName, "err", err) + return err } - data := make(map[string][]byte) - data[SECRET_FIELD_CLUSTER_ID] = []byte(fmt.Sprintf("%v", bean.Id)) - data[SECRET_FIELD_ACTION] = []byte(CLUSTER_ACTION_UPDATE) - data[SECRET_FIELD_UPDATED_ON] = []byte(time.Now().String()) // this field will ensure that informer detects change as other fields can be constant even if cluster config changes - if secret == nil { - _, err = impl.K8sUtil.CreateSecret(DEFAULT_NAMESPACE, data, secretName, CLUSTER_MODIFY_EVENT_SECRET_TYPE, k8sClient, nil, nil) - if err != nil { - impl.logger.Errorw("error in creating secret for informers") - } - } else { - secret.Data = data - secret, err = impl.K8sUtil.UpdateSecret(DEFAULT_NAMESPACE, secret, k8sClient) - if err != nil { - impl.logger.Errorw("error in updating secret for informers") - } + } else { + secret.Data = data + secret, err = impl.K8sUtil.UpdateSecret(clusterBean.DefaultNamespace, secret, k8sClient) + if err != nil { + impl.logger.Errorw("error in updating secret for informers", "secretName", secretName, "err", err) + return err } } - return bean, nil + return nil } func (impl *ClusterServiceImpl) SyncNsInformer(bean *bean.ClusterBean) { @@ -609,8 +607,9 @@ func (impl *ClusterServiceImpl) DeleteFromDb(bean *bean.ClusterBean, userId int3 impl.logger.Errorw("error in getting in cluster k8s client", "err", err, "clusterName", bean.ClusterName) return "", nil } + // TODO Asutosh: why we maintain this duplicate code ?? secretName := ParseSecretNameForKubelinkInformer(bean.Id) - err = impl.K8sUtil.DeleteSecret(DEFAULT_NAMESPACE, secretName, k8sClient) + err = impl.K8sUtil.DeleteSecret(clusterBean.DefaultNamespace, secretName, k8sClient) impl.logger.Errorw("error in deleting secret", "error", err) return existingCluster.ClusterName, nil } @@ -621,7 +620,7 @@ func (impl *ClusterServiceImpl) CheckIfConfigIsValid(cluster *bean.ClusterBean) if err != nil { if _, ok := err.(*url.Error); ok { return fmt.Errorf("Incorrect server url : %v", err) - } else if statusError, ok := err.(*errors.StatusError); ok { + } else if statusError, ok := err.(*k8sError.StatusError); ok { if statusError != nil { errReason := statusError.ErrStatus.Reason var errMsg string @@ -850,16 +849,16 @@ func (impl *ClusterServiceImpl) ValidateKubeconfig(kubeConfig string) (map[strin err := json.Unmarshal([]byte(kubeConfig), &kubeConfigDataMap) if err != nil { impl.logger.Errorw("error in unmarshalling kubeConfig") - return nil, errors1.New("invalid kubeConfig found , " + err.Error()) + return nil, customErr.New("invalid kubeConfig found , " + err.Error()) } if kubeConfigDataMap["apiVersion"] == nil { impl.logger.Errorw("api version missing from kubeConfig") - return nil, errors1.New("api version missing from kubeConfig") + return nil, customErr.New("api version missing from kubeConfig") } if kubeConfigDataMap["kind"] == nil { impl.logger.Errorw("kind missing from kubeConfig") - return nil, errors1.New("kind missing from kubeConfig") + return nil, customErr.New("kind missing from kubeConfig") } gvk.Version = kubeConfigDataMap["apiVersion"].(string) @@ -909,7 +908,7 @@ func (impl *ClusterServiceImpl) ValidateKubeconfig(kubeConfig string) (map[strin clusterBeanObject.ErrorInConnecting = "cluster name missing from kubeconfig" } - if clusterBeanObject.ClusterName == clusterBean.DEFAULT_CLUSTER { + if clusterBeanObject.ClusterName == clusterBean.DefaultCluster { clusterBeanObject.ErrorInConnecting = "default_cluster is reserved by the system and cannot be updated" } @@ -1000,7 +999,7 @@ func (impl *ClusterServiceImpl) ValidateKubeconfig(kubeConfig string) (map[strin if len(ValidateObjects) == 0 { impl.logger.Errorw("No valid cluster object provided in kubeconfig for context", "context", kubeConfig) - return nil, errors1.New("No valid cluster object provided in kubeconfig for context") + return nil, customErr.New("No valid cluster object provided in kubeconfig for context") } else { return ValidateObjects, nil } @@ -1014,7 +1013,7 @@ func (impl *ClusterServiceImpl) GetAndUpdateConnectionStatusForOneCluster(k8sCli if err != nil { if _, ok := err.(*url.Error); ok { err = fmt.Errorf("Incorrect server url : %v", err) - } else if statusError, ok := err.(*errors.StatusError); ok { + } else if statusError, ok := err.(*k8sError.StatusError); ok { if statusError != nil { errReason := statusError.ErrStatus.Reason var errMsg string diff --git a/pkg/cluster/bean/bean.go b/pkg/cluster/bean/bean.go index 6548613adb..a2438a2b44 100644 --- a/pkg/cluster/bean/bean.go +++ b/pkg/cluster/bean/bean.go @@ -7,7 +7,7 @@ import ( const ( DefaultClusterId = 1 - DEFAULT_CLUSTER = "default_cluster" + DefaultCluster = "default_cluster" ) type PrometheusAuth struct { @@ -81,3 +81,8 @@ type DefaultClusterComponent struct { EnvName string `json:"envName"` Status string `json:"status"` } + +const ( + DefaultNamespace = "default" + SecretFieldUpdatedOn = "updated_on" +) diff --git a/pkg/cluster/clusterUtil.go b/pkg/cluster/clusterUtil.go index 102519ff19..48ed681400 100644 --- a/pkg/cluster/clusterUtil.go +++ b/pkg/cluster/clusterUtil.go @@ -3,9 +3,9 @@ package cluster import "fmt" const ( - SECRET_NAME = "cluster-event" + SecretName = "cluster-event" ) func ParseSecretNameForKubelinkInformer(clusterId int) string { - return fmt.Sprintf("%s-%d", SECRET_NAME, clusterId) + return fmt.Sprintf("%s-%d", SecretName, clusterId) } diff --git a/pkg/delete/DeleteService.go b/pkg/delete/DeleteService.go index e9a53b3922..278d4182d0 100644 --- a/pkg/delete/DeleteService.go +++ b/pkg/delete/DeleteService.go @@ -108,7 +108,7 @@ func (impl DeleteServiceImpl) DeleteClusterSecret(deleteRequest *bean2.ClusterBe return nil } secretName := cluster.ParseSecretNameForKubelinkInformer(deleteRequest.Id) - err = impl.K8sUtil.DeleteSecret(cluster.DEFAULT_NAMESPACE, secretName, k8sClient) + err = impl.K8sUtil.DeleteSecret(bean2.DefaultNamespace, secretName, k8sClient) return err } diff --git a/pkg/deployment/deployedApp/DeployedAppService.go b/pkg/deployment/deployedApp/DeployedAppService.go index f8c65d2373..83107b22fa 100644 --- a/pkg/deployment/deployedApp/DeployedAppService.go +++ b/pkg/deployment/deployedApp/DeployedAppService.go @@ -46,7 +46,7 @@ type DeployedAppService interface { type DeployedAppServiceImpl struct { logger *zap.SugaredLogger k8sCommonService k8s.K8sCommonService - cdTriggerService devtronApps.TriggerService + cdHandlerService devtronApps.HandlerService envRepository repository.EnvironmentRepository pipelineRepository pipelineConfig.PipelineRepository cdWorkflowRepository pipelineConfig.CdWorkflowRepository @@ -54,14 +54,14 @@ type DeployedAppServiceImpl struct { func NewDeployedAppServiceImpl(logger *zap.SugaredLogger, k8sCommonService k8s.K8sCommonService, - cdTriggerService devtronApps.TriggerService, + cdHandlerService devtronApps.HandlerService, envRepository repository.EnvironmentRepository, pipelineRepository pipelineConfig.PipelineRepository, cdWorkflowRepository pipelineConfig.CdWorkflowRepository) *DeployedAppServiceImpl { return &DeployedAppServiceImpl{ logger: logger, k8sCommonService: k8sCommonService, - cdTriggerService: cdTriggerService, + cdHandlerService: cdHandlerService, envRepository: envRepository, pipelineRepository: pipelineRepository, cdWorkflowRepository: cdWorkflowRepository, @@ -127,7 +127,7 @@ func (impl *DeployedAppServiceImpl) stopStartApp(ctx context.Context, stopReques Context: ctx, ReferenceId: stopRequest.ReferenceId, } - id, _, _, err := impl.cdTriggerService.ManualCdTrigger(triggerContext, overrideRequest) + id, _, _, err := impl.cdHandlerService.ManualCdTrigger(triggerContext, overrideRequest) if err != nil { impl.logger.Errorw("error in stopping app", "err", err, "appId", stopRequest.AppId, "envId", stopRequest.EnvironmentId) return 0, err diff --git a/pkg/deployment/trigger/devtronApps/HandlerService.go b/pkg/deployment/trigger/devtronApps/HandlerService.go new file mode 100644 index 0000000000..c92a009134 --- /dev/null +++ b/pkg/deployment/trigger/devtronApps/HandlerService.go @@ -0,0 +1,345 @@ +/* + * Copyright (c) 2024. Devtron Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package devtronApps + +import ( + "bufio" + "context" + pubsub "github.com/devtron-labs/common-lib/pubsub-lib" + util5 "github.com/devtron-labs/common-lib/utils/k8s" + bean3 "github.com/devtron-labs/devtron/api/bean" + "github.com/devtron-labs/devtron/api/helm-app/gRPC" + client2 "github.com/devtron-labs/devtron/api/helm-app/service" + "github.com/devtron-labs/devtron/client/argocdServer" + client "github.com/devtron-labs/devtron/client/events" + repository3 "github.com/devtron-labs/devtron/internal/sql/repository" + appRepository "github.com/devtron-labs/devtron/internal/sql/repository/app" + "github.com/devtron-labs/devtron/internal/sql/repository/appWorkflow" + "github.com/devtron-labs/devtron/internal/sql/repository/chartConfig" + repository4 "github.com/devtron-labs/devtron/internal/sql/repository/dockerRegistry" + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" + "github.com/devtron-labs/devtron/internal/util" + "github.com/devtron-labs/devtron/pkg/app" + bean4 "github.com/devtron-labs/devtron/pkg/app/bean" + "github.com/devtron-labs/devtron/pkg/app/status" + "github.com/devtron-labs/devtron/pkg/attributes" + "github.com/devtron-labs/devtron/pkg/auth/user" + "github.com/devtron-labs/devtron/pkg/build/git/gitMaterial/read" + pipeline2 "github.com/devtron-labs/devtron/pkg/build/pipeline" + chartRepoRepository "github.com/devtron-labs/devtron/pkg/chartRepo/repository" + "github.com/devtron-labs/devtron/pkg/cluster" + repository2 "github.com/devtron-labs/devtron/pkg/cluster/environment/repository" + repository5 "github.com/devtron-labs/devtron/pkg/cluster/repository" + "github.com/devtron-labs/devtron/pkg/deployment/common" + bean9 "github.com/devtron-labs/devtron/pkg/deployment/common/bean" + "github.com/devtron-labs/devtron/pkg/deployment/gitOps/config" + "github.com/devtron-labs/devtron/pkg/deployment/gitOps/git" + "github.com/devtron-labs/devtron/pkg/deployment/manifest" + "github.com/devtron-labs/devtron/pkg/deployment/manifest/publish" + "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/bean" + "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/userDeploymentRequest/service" + "github.com/devtron-labs/devtron/pkg/eventProcessor/out" + "github.com/devtron-labs/devtron/pkg/executor" + "github.com/devtron-labs/devtron/pkg/imageDigestPolicy" + "github.com/devtron-labs/devtron/pkg/pipeline" + "github.com/devtron-labs/devtron/pkg/pipeline/history" + "github.com/devtron-labs/devtron/pkg/pipeline/repository" + "github.com/devtron-labs/devtron/pkg/pipeline/types" + "github.com/devtron-labs/devtron/pkg/plugin" + security2 "github.com/devtron-labs/devtron/pkg/policyGovernance/security/imageScanning" + read2 "github.com/devtron-labs/devtron/pkg/policyGovernance/security/imageScanning/read" + "github.com/devtron-labs/devtron/pkg/sql" + "github.com/devtron-labs/devtron/pkg/variables" + "github.com/devtron-labs/devtron/pkg/workflow/cd" + globalUtil "github.com/devtron-labs/devtron/util" + util2 "github.com/devtron-labs/devtron/util/event" + "github.com/devtron-labs/devtron/util/rbac" + "go.uber.org/zap" + "os" + "time" +) + +/* +files in this package are - +HandlerService.go - containing If and impl with common used code +HandlerService_ent.go - containing ent If and impl with common used code +deployStageHandlerCode.go - code related to deploy stage trigger +deployStageHandlerCode_ent.go - ent code related to deploy stage trigger +preStageHandlerCode.go - code related to pre stage trigger +preStageHandlerCode_ent.go - ent code related to pre stage trigger +postStageHandlerCode.go - code related to post stage trigger +postStageHandlerCode_ent.go - ent code related to post stage trigger +prePostWfAndLogsCode.go - code containing pre/post wf handling(abort) and logs related code +*/ + +type HandlerService interface { + TriggerPostStage(request bean.TriggerRequest) (*bean4.ManifestPushTemplate, error) + TriggerPreStage(request bean.TriggerRequest) (*bean4.ManifestPushTemplate, error) + + TriggerAutoCDOnPreStageSuccess(triggerContext bean.TriggerContext, cdPipelineId, ciArtifactId, workflowId int) error + + TriggerStageForBulk(triggerRequest bean.TriggerRequest) error + + ManualCdTrigger(triggerContext bean.TriggerContext, overrideRequest *bean3.ValuesOverrideRequest) (int, string, *bean4.ManifestPushTemplate, error) + TriggerAutomaticDeployment(request bean.TriggerRequest) error + + TriggerRelease(ctx context.Context, overrideRequest *bean3.ValuesOverrideRequest, envDeploymentConfig *bean9.DeploymentConfig, triggeredAt time.Time, triggeredBy int32) (releaseNo int, manifestPushTemplate *bean4.ManifestPushTemplate, err error) + + CancelStage(workflowRunnerId int, forceAbort bool, userId int32) (int, error) + DownloadCdWorkflowArtifacts(buildId int) (*os.File, error) + GetRunningWorkflowLogs(environmentId int, pipelineId int, workflowId int) (*bufio.Reader, func() error, error) +} + +type HandlerServiceImpl struct { + logger *zap.SugaredLogger + cdWorkflowCommonService cd.CdWorkflowCommonService + gitOpsManifestPushService publish.GitOpsPushService + gitOpsConfigReadService config.GitOpsConfigReadService + argoK8sClient argocdServer.ArgoK8sClient + ACDConfig *argocdServer.ACDConfig + argoClientWrapperService argocdServer.ArgoClientWrapperService + pipelineStatusTimelineService status.PipelineStatusTimelineService + chartTemplateService util.ChartTemplateService + eventFactory client.EventFactory + eventClient client.EventClient + globalEnvVariables *globalUtil.GlobalEnvVariables + workflowEventPublishService out.WorkflowEventPublishService + manifestCreationService manifest.ManifestCreationService + deployedConfigurationHistoryService history.DeployedConfigurationHistoryService + pipelineStageService pipeline.PipelineStageService + globalPluginService plugin.GlobalPluginService + customTagService pipeline.CustomTagService + pluginInputVariableParser pipeline.PluginInputVariableParser + prePostCdScriptHistoryService history.PrePostCdScriptHistoryService + scopedVariableManager variables.ScopedVariableCMCSManager + imageDigestPolicyService imageDigestPolicy.ImageDigestPolicyService + userService user.UserService + config *types.CdConfig + helmAppService client2.HelmAppService + imageScanService security2.ImageScanService + enforcerUtil rbac.EnforcerUtil + userDeploymentRequestService service.UserDeploymentRequestService + helmAppClient gRPC.HelmAppClient //TODO refactoring: use helm app service instead + appRepository appRepository.AppRepository + ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository + imageScanHistoryReadService read2.ImageScanHistoryReadService + imageScanDeployInfoService security2.ImageScanDeployInfoService + imageScanDeployInfoReadService read2.ImageScanDeployInfoReadService + pipelineRepository pipelineConfig.PipelineRepository + pipelineOverrideRepository chartConfig.PipelineOverrideRepository + manifestPushConfigRepository repository.ManifestPushConfigRepository + chartRepository chartRepoRepository.ChartRepository + envRepository repository2.EnvironmentRepository + cdWorkflowRepository pipelineConfig.CdWorkflowRepository + ciWorkflowRepository pipelineConfig.CiWorkflowRepository + ciArtifactRepository repository3.CiArtifactRepository + ciTemplateService pipeline2.CiTemplateReadService + gitMaterialReadService read.GitMaterialReadService + appLabelRepository pipelineConfig.AppLabelRepository + ciPipelineRepository pipelineConfig.CiPipelineRepository + appWorkflowRepository appWorkflow.AppWorkflowRepository + dockerArtifactStoreRepository repository4.DockerArtifactStoreRepository + K8sUtil *util5.K8sServiceImpl + transactionUtilImpl *sql.TransactionUtilImpl + deploymentConfigService common.DeploymentConfigService + ciCdPipelineOrchestrator pipeline.CiCdPipelineOrchestrator + gitOperationService git.GitOperationService + attributeService attributes.AttributesService + clusterRepository repository5.ClusterRepository + cdWorkflowRunnerService cd.CdWorkflowRunnerService + clusterService cluster.ClusterService + ciLogService pipeline.CiLogService + workflowService executor.WorkflowService + blobConfigStorageService pipeline.BlobStorageConfigService +} + +func NewHandlerServiceImpl(logger *zap.SugaredLogger, + cdWorkflowCommonService cd.CdWorkflowCommonService, + gitOpsManifestPushService publish.GitOpsPushService, + gitOpsConfigReadService config.GitOpsConfigReadService, + argoK8sClient argocdServer.ArgoK8sClient, + ACDConfig *argocdServer.ACDConfig, + argoClientWrapperService argocdServer.ArgoClientWrapperService, + pipelineStatusTimelineService status.PipelineStatusTimelineService, + chartTemplateService util.ChartTemplateService, + workflowEventPublishService out.WorkflowEventPublishService, + manifestCreationService manifest.ManifestCreationService, + deployedConfigurationHistoryService history.DeployedConfigurationHistoryService, + pipelineStageService pipeline.PipelineStageService, + globalPluginService plugin.GlobalPluginService, + customTagService pipeline.CustomTagService, + pluginInputVariableParser pipeline.PluginInputVariableParser, + prePostCdScriptHistoryService history.PrePostCdScriptHistoryService, + scopedVariableManager variables.ScopedVariableCMCSManager, + imageDigestPolicyService imageDigestPolicy.ImageDigestPolicyService, + userService user.UserService, + helmAppService client2.HelmAppService, + enforcerUtil rbac.EnforcerUtil, + userDeploymentRequestService service.UserDeploymentRequestService, + helmAppClient gRPC.HelmAppClient, + eventFactory client.EventFactory, + eventClient client.EventClient, + envVariables *globalUtil.EnvironmentVariables, + appRepository appRepository.AppRepository, + ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository, + imageScanHistoryReadService read2.ImageScanHistoryReadService, + imageScanDeployInfoReadService read2.ImageScanDeployInfoReadService, + imageScanDeployInfoService security2.ImageScanDeployInfoService, + pipelineRepository pipelineConfig.PipelineRepository, + pipelineOverrideRepository chartConfig.PipelineOverrideRepository, + manifestPushConfigRepository repository.ManifestPushConfigRepository, + chartRepository chartRepoRepository.ChartRepository, + envRepository repository2.EnvironmentRepository, + cdWorkflowRepository pipelineConfig.CdWorkflowRepository, + ciWorkflowRepository pipelineConfig.CiWorkflowRepository, + ciArtifactRepository repository3.CiArtifactRepository, + ciTemplateService pipeline2.CiTemplateReadService, + gitMaterialReadService read.GitMaterialReadService, + appLabelRepository pipelineConfig.AppLabelRepository, + ciPipelineRepository pipelineConfig.CiPipelineRepository, + appWorkflowRepository appWorkflow.AppWorkflowRepository, + dockerArtifactStoreRepository repository4.DockerArtifactStoreRepository, + imageScanService security2.ImageScanService, + K8sUtil *util5.K8sServiceImpl, + transactionUtilImpl *sql.TransactionUtilImpl, + deploymentConfigService common.DeploymentConfigService, + ciCdPipelineOrchestrator pipeline.CiCdPipelineOrchestrator, + gitOperationService git.GitOperationService, + attributeService attributes.AttributesService, + clusterRepository repository5.ClusterRepository, + cdWorkflowRunnerService cd.CdWorkflowRunnerService, + clusterService cluster.ClusterService, + ciLogService pipeline.CiLogService, + workflowService executor.WorkflowService, + blobConfigStorageService pipeline.BlobStorageConfigService, +) (*HandlerServiceImpl, error) { + impl := &HandlerServiceImpl{ + logger: logger, + cdWorkflowCommonService: cdWorkflowCommonService, + gitOpsManifestPushService: gitOpsManifestPushService, + gitOpsConfigReadService: gitOpsConfigReadService, + argoK8sClient: argoK8sClient, + ACDConfig: ACDConfig, + argoClientWrapperService: argoClientWrapperService, + pipelineStatusTimelineService: pipelineStatusTimelineService, + chartTemplateService: chartTemplateService, + workflowEventPublishService: workflowEventPublishService, + manifestCreationService: manifestCreationService, + deployedConfigurationHistoryService: deployedConfigurationHistoryService, + pipelineStageService: pipelineStageService, + globalPluginService: globalPluginService, + customTagService: customTagService, + pluginInputVariableParser: pluginInputVariableParser, + prePostCdScriptHistoryService: prePostCdScriptHistoryService, + scopedVariableManager: scopedVariableManager, + imageDigestPolicyService: imageDigestPolicyService, + userService: userService, + helmAppService: helmAppService, + enforcerUtil: enforcerUtil, + eventFactory: eventFactory, + eventClient: eventClient, + + globalEnvVariables: envVariables.GlobalEnvVariables, + userDeploymentRequestService: userDeploymentRequestService, + helmAppClient: helmAppClient, + appRepository: appRepository, + ciPipelineMaterialRepository: ciPipelineMaterialRepository, + imageScanHistoryReadService: imageScanHistoryReadService, + imageScanDeployInfoReadService: imageScanDeployInfoReadService, + imageScanDeployInfoService: imageScanDeployInfoService, + pipelineRepository: pipelineRepository, + pipelineOverrideRepository: pipelineOverrideRepository, + manifestPushConfigRepository: manifestPushConfigRepository, + chartRepository: chartRepository, + envRepository: envRepository, + cdWorkflowRepository: cdWorkflowRepository, + ciWorkflowRepository: ciWorkflowRepository, + ciArtifactRepository: ciArtifactRepository, + ciTemplateService: ciTemplateService, + gitMaterialReadService: gitMaterialReadService, + appLabelRepository: appLabelRepository, + ciPipelineRepository: ciPipelineRepository, + appWorkflowRepository: appWorkflowRepository, + dockerArtifactStoreRepository: dockerArtifactStoreRepository, + + imageScanService: imageScanService, + K8sUtil: K8sUtil, + + transactionUtilImpl: transactionUtilImpl, + + deploymentConfigService: deploymentConfigService, + ciCdPipelineOrchestrator: ciCdPipelineOrchestrator, + gitOperationService: gitOperationService, + attributeService: attributeService, + cdWorkflowRunnerService: cdWorkflowRunnerService, + + clusterRepository: clusterRepository, + clusterService: clusterService, + ciLogService: ciLogService, + workflowService: workflowService, + blobConfigStorageService: blobConfigStorageService, + } + config, err := types.GetCdConfig() + if err != nil { + return nil, err + } + impl.config = config + return impl, nil +} + +func (impl *HandlerServiceImpl) writeCDTriggerEvent(overrideRequest *bean3.ValuesOverrideRequest, artifact *repository3.CiArtifact, releaseId, pipelineOverrideId, wfrId int) { + + event, err := impl.eventFactory.Build(util2.Trigger, &overrideRequest.PipelineId, overrideRequest.AppId, &overrideRequest.EnvId, util2.CD) + if err != nil { + impl.logger.Errorw("error in building cd trigger event", "cdPipelineId", overrideRequest.PipelineId, "err", err) + } + impl.logger.Debugw("event WriteCDTriggerEvent", "event", event) + wfr := impl.getEnrichedWorkflowRunner(overrideRequest, artifact, wfrId) + event = impl.eventFactory.BuildExtraCDData(event, wfr, pipelineOverrideId, bean3.CD_WORKFLOW_TYPE_DEPLOY) + _, evtErr := impl.eventClient.WriteNotificationEvent(event) + if evtErr != nil { + impl.logger.Errorw("CD trigger event not sent", "error", evtErr) + } + deploymentEvent := app.DeploymentEvent{ + ApplicationId: overrideRequest.AppId, + EnvironmentId: overrideRequest.EnvId, // check for production Environment + ReleaseId: releaseId, + PipelineOverrideId: pipelineOverrideId, + TriggerTime: time.Now(), + CiArtifactId: overrideRequest.CiArtifactId, + } + + ciPipelineMaterials, err := impl.ciPipelineMaterialRepository.GetByPipelineId(artifact.PipelineId) + if err != nil { + impl.logger.Errorw("error in ") + } + materialInfoMap, mErr := artifact.ParseMaterialInfo() + if mErr != nil { + impl.logger.Errorw("material info map error", mErr) + return + } + for _, ciPipelineMaterial := range ciPipelineMaterials { + hash := materialInfoMap[ciPipelineMaterial.GitMaterial.Url] + pipelineMaterialInfo := &app.PipelineMaterialInfo{PipelineMaterialId: ciPipelineMaterial.Id, CommitHash: hash} + deploymentEvent.PipelineMaterials = append(deploymentEvent.PipelineMaterials, pipelineMaterialInfo) + } + impl.logger.Infow("triggering deployment event", "event", deploymentEvent) + err = impl.eventClient.WriteNatsEvent(pubsub.CD_SUCCESS, deploymentEvent) + if err != nil { + impl.logger.Errorw("error in writing cd trigger event", "err", err) + } +} diff --git a/pkg/deployment/trigger/devtronApps/TriggerService.go b/pkg/deployment/trigger/devtronApps/deployStageHandlerCode.go similarity index 77% rename from pkg/deployment/trigger/devtronApps/TriggerService.go rename to pkg/deployment/trigger/devtronApps/deployStageHandlerCode.go index 7a5d135045..c0d8a204a1 100644 --- a/pkg/deployment/trigger/devtronApps/TriggerService.go +++ b/pkg/deployment/trigger/devtronApps/deployStageHandlerCode.go @@ -20,74 +20,38 @@ import ( "context" "errors" "fmt" - pubsub "github.com/devtron-labs/common-lib/pubsub-lib" - util5 "github.com/devtron-labs/common-lib/utils/k8s" bean3 "github.com/devtron-labs/devtron/api/bean" "github.com/devtron-labs/devtron/api/bean/gitOps" bean6 "github.com/devtron-labs/devtron/api/helm-app/bean" "github.com/devtron-labs/devtron/api/helm-app/gRPC" - client2 "github.com/devtron-labs/devtron/api/helm-app/service" "github.com/devtron-labs/devtron/client/argocdServer" bean7 "github.com/devtron-labs/devtron/client/argocdServer/bean" - client "github.com/devtron-labs/devtron/client/events" - gitSensorClient "github.com/devtron-labs/devtron/client/gitSensor" "github.com/devtron-labs/devtron/internal/middleware" "github.com/devtron-labs/devtron/internal/sql/models" repository3 "github.com/devtron-labs/devtron/internal/sql/repository" - appRepository "github.com/devtron-labs/devtron/internal/sql/repository/app" - "github.com/devtron-labs/devtron/internal/sql/repository/appWorkflow" - "github.com/devtron-labs/devtron/internal/sql/repository/chartConfig" - repository4 "github.com/devtron-labs/devtron/internal/sql/repository/dockerRegistry" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/timelineStatus" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow/cdWorkflow" "github.com/devtron-labs/devtron/internal/util" "github.com/devtron-labs/devtron/pkg/app" bean4 "github.com/devtron-labs/devtron/pkg/app/bean" - "github.com/devtron-labs/devtron/pkg/app/status" statusBean "github.com/devtron-labs/devtron/pkg/app/status/bean" - "github.com/devtron-labs/devtron/pkg/attributes" - "github.com/devtron-labs/devtron/pkg/auth/user" bean2 "github.com/devtron-labs/devtron/pkg/bean" - "github.com/devtron-labs/devtron/pkg/build/git/gitMaterial/read" - pipeline2 "github.com/devtron-labs/devtron/pkg/build/pipeline" - chartRepoRepository "github.com/devtron-labs/devtron/pkg/chartRepo/repository" - repository2 "github.com/devtron-labs/devtron/pkg/cluster/environment/repository" - repository5 "github.com/devtron-labs/devtron/pkg/cluster/repository" "github.com/devtron-labs/devtron/pkg/deployment/common" bean9 "github.com/devtron-labs/devtron/pkg/deployment/common/bean" - "github.com/devtron-labs/devtron/pkg/deployment/gitOps/config" - "github.com/devtron-labs/devtron/pkg/deployment/gitOps/git" - "github.com/devtron-labs/devtron/pkg/deployment/manifest" bean10 "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/bean" bean5 "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/chartRef/bean" - "github.com/devtron-labs/devtron/pkg/deployment/manifest/publish" "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/adapter" "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/bean" "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/helper" - "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/userDeploymentRequest/service" clientErrors "github.com/devtron-labs/devtron/pkg/errors" - "github.com/devtron-labs/devtron/pkg/eventProcessor/out" - "github.com/devtron-labs/devtron/pkg/imageDigestPolicy" k8s2 "github.com/devtron-labs/devtron/pkg/k8s" - "github.com/devtron-labs/devtron/pkg/pipeline" bean8 "github.com/devtron-labs/devtron/pkg/pipeline/bean" - "github.com/devtron-labs/devtron/pkg/pipeline/history" "github.com/devtron-labs/devtron/pkg/pipeline/repository" - "github.com/devtron-labs/devtron/pkg/pipeline/types" - "github.com/devtron-labs/devtron/pkg/plugin" - security2 "github.com/devtron-labs/devtron/pkg/policyGovernance/security/imageScanning" - read2 "github.com/devtron-labs/devtron/pkg/policyGovernance/security/imageScanning/read" repository6 "github.com/devtron-labs/devtron/pkg/policyGovernance/security/imageScanning/repository" "github.com/devtron-labs/devtron/pkg/sql" - "github.com/devtron-labs/devtron/pkg/variables" - "github.com/devtron-labs/devtron/pkg/workflow/cd" - globalUtil "github.com/devtron-labs/devtron/util" - util2 "github.com/devtron-labs/devtron/util/event" - "github.com/devtron-labs/devtron/util/rbac" "github.com/go-pg/pg" "go.opentelemetry.io/otel" - "go.uber.org/zap" "golang.org/x/exp/slices" "google.golang.org/grpc/codes" status2 "google.golang.org/grpc/status" @@ -100,214 +64,7 @@ import ( "time" ) -type TriggerService interface { - TriggerPostStage(request bean.TriggerRequest) (*bean4.ManifestPushTemplate, error) - TriggerPreStage(request bean.TriggerRequest) (*bean4.ManifestPushTemplate, error) - - TriggerAutoCDOnPreStageSuccess(triggerContext bean.TriggerContext, cdPipelineId, ciArtifactId, workflowId int) error - - TriggerStageForBulk(triggerRequest bean.TriggerRequest) error - - ManualCdTrigger(triggerContext bean.TriggerContext, overrideRequest *bean3.ValuesOverrideRequest) (int, string, *bean4.ManifestPushTemplate, error) - TriggerAutomaticDeployment(request bean.TriggerRequest) error - - TriggerRelease(ctx context.Context, overrideRequest *bean3.ValuesOverrideRequest, envDeploymentConfig *bean9.DeploymentConfig, triggeredAt time.Time, triggeredBy int32) (releaseNo int, manifestPushTemplate *bean4.ManifestPushTemplate, err error) -} - -type TriggerServiceImpl struct { - logger *zap.SugaredLogger - cdWorkflowCommonService cd.CdWorkflowCommonService - gitOpsManifestPushService publish.GitOpsPushService - gitOpsConfigReadService config.GitOpsConfigReadService - argoK8sClient argocdServer.ArgoK8sClient - ACDConfig *argocdServer.ACDConfig - argoClientWrapperService argocdServer.ArgoClientWrapperService - pipelineStatusTimelineService status.PipelineStatusTimelineService - chartTemplateService util.ChartTemplateService - eventFactory client.EventFactory - eventClient client.EventClient - globalEnvVariables *globalUtil.GlobalEnvVariables - workflowEventPublishService out.WorkflowEventPublishService - manifestCreationService manifest.ManifestCreationService - deployedConfigurationHistoryService history.DeployedConfigurationHistoryService - pipelineStageService pipeline.PipelineStageService - globalPluginService plugin.GlobalPluginService - customTagService pipeline.CustomTagService - pluginInputVariableParser pipeline.PluginInputVariableParser - prePostCdScriptHistoryService history.PrePostCdScriptHistoryService - scopedVariableManager variables.ScopedVariableCMCSManager - cdWorkflowService pipeline.WorkflowService - imageDigestPolicyService imageDigestPolicy.ImageDigestPolicyService - userService user.UserService - gitSensorClient gitSensorClient.Client - config *types.CdConfig - helmAppService client2.HelmAppService - imageScanService security2.ImageScanService - enforcerUtil rbac.EnforcerUtil - userDeploymentRequestService service.UserDeploymentRequestService - helmAppClient gRPC.HelmAppClient //TODO refactoring: use helm app service instead - appRepository appRepository.AppRepository - ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository - imageScanHistoryReadService read2.ImageScanHistoryReadService - imageScanDeployInfoService security2.ImageScanDeployInfoService - imageScanDeployInfoReadService read2.ImageScanDeployInfoReadService - pipelineRepository pipelineConfig.PipelineRepository - pipelineOverrideRepository chartConfig.PipelineOverrideRepository - manifestPushConfigRepository repository.ManifestPushConfigRepository - chartRepository chartRepoRepository.ChartRepository - envRepository repository2.EnvironmentRepository - cdWorkflowRepository pipelineConfig.CdWorkflowRepository - ciWorkflowRepository pipelineConfig.CiWorkflowRepository - ciArtifactRepository repository3.CiArtifactRepository - ciTemplateService pipeline2.CiTemplateReadService - gitMaterialReadService read.GitMaterialReadService - appLabelRepository pipelineConfig.AppLabelRepository - ciPipelineRepository pipelineConfig.CiPipelineRepository - appWorkflowRepository appWorkflow.AppWorkflowRepository - dockerArtifactStoreRepository repository4.DockerArtifactStoreRepository - K8sUtil *util5.K8sServiceImpl - transactionUtilImpl *sql.TransactionUtilImpl - deploymentConfigService common.DeploymentConfigService - deploymentServiceTypeConfig *globalUtil.DeploymentServiceTypeConfig - ciCdPipelineOrchestrator pipeline.CiCdPipelineOrchestrator - gitOperationService git.GitOperationService - attributeService attributes.AttributesService - clusterRepository repository5.ClusterRepository - cdWorkflowRunnerService cd.CdWorkflowRunnerService -} - -func NewTriggerServiceImpl(logger *zap.SugaredLogger, - cdWorkflowCommonService cd.CdWorkflowCommonService, - gitOpsManifestPushService publish.GitOpsPushService, - gitOpsConfigReadService config.GitOpsConfigReadService, - argoK8sClient argocdServer.ArgoK8sClient, - ACDConfig *argocdServer.ACDConfig, - argoClientWrapperService argocdServer.ArgoClientWrapperService, - pipelineStatusTimelineService status.PipelineStatusTimelineService, - chartTemplateService util.ChartTemplateService, - workflowEventPublishService out.WorkflowEventPublishService, - manifestCreationService manifest.ManifestCreationService, - deployedConfigurationHistoryService history.DeployedConfigurationHistoryService, - pipelineStageService pipeline.PipelineStageService, - globalPluginService plugin.GlobalPluginService, - customTagService pipeline.CustomTagService, - pluginInputVariableParser pipeline.PluginInputVariableParser, - prePostCdScriptHistoryService history.PrePostCdScriptHistoryService, - scopedVariableManager variables.ScopedVariableCMCSManager, - cdWorkflowService pipeline.WorkflowService, - imageDigestPolicyService imageDigestPolicy.ImageDigestPolicyService, - userService user.UserService, - gitSensorClient gitSensorClient.Client, - helmAppService client2.HelmAppService, - enforcerUtil rbac.EnforcerUtil, - userDeploymentRequestService service.UserDeploymentRequestService, - helmAppClient gRPC.HelmAppClient, - eventFactory client.EventFactory, - eventClient client.EventClient, - envVariables *globalUtil.EnvironmentVariables, - appRepository appRepository.AppRepository, - ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository, - imageScanHistoryReadService read2.ImageScanHistoryReadService, - imageScanDeployInfoReadService read2.ImageScanDeployInfoReadService, - imageScanDeployInfoService security2.ImageScanDeployInfoService, - pipelineRepository pipelineConfig.PipelineRepository, - pipelineOverrideRepository chartConfig.PipelineOverrideRepository, - manifestPushConfigRepository repository.ManifestPushConfigRepository, - chartRepository chartRepoRepository.ChartRepository, - envRepository repository2.EnvironmentRepository, - cdWorkflowRepository pipelineConfig.CdWorkflowRepository, - ciWorkflowRepository pipelineConfig.CiWorkflowRepository, - ciArtifactRepository repository3.CiArtifactRepository, - ciTemplateService pipeline2.CiTemplateReadService, - gitMaterialReadService read.GitMaterialReadService, - appLabelRepository pipelineConfig.AppLabelRepository, - ciPipelineRepository pipelineConfig.CiPipelineRepository, - appWorkflowRepository appWorkflow.AppWorkflowRepository, - dockerArtifactStoreRepository repository4.DockerArtifactStoreRepository, - imageScanService security2.ImageScanService, - K8sUtil *util5.K8sServiceImpl, - transactionUtilImpl *sql.TransactionUtilImpl, - deploymentConfigService common.DeploymentConfigService, - ciCdPipelineOrchestrator pipeline.CiCdPipelineOrchestrator, - gitOperationService git.GitOperationService, - attributeService attributes.AttributesService, - clusterRepository repository5.ClusterRepository, - cdWorkflowRunnerService cd.CdWorkflowRunnerService, -) (*TriggerServiceImpl, error) { - impl := &TriggerServiceImpl{ - logger: logger, - cdWorkflowCommonService: cdWorkflowCommonService, - gitOpsManifestPushService: gitOpsManifestPushService, - gitOpsConfigReadService: gitOpsConfigReadService, - argoK8sClient: argoK8sClient, - ACDConfig: ACDConfig, - argoClientWrapperService: argoClientWrapperService, - pipelineStatusTimelineService: pipelineStatusTimelineService, - chartTemplateService: chartTemplateService, - workflowEventPublishService: workflowEventPublishService, - manifestCreationService: manifestCreationService, - deployedConfigurationHistoryService: deployedConfigurationHistoryService, - pipelineStageService: pipelineStageService, - globalPluginService: globalPluginService, - customTagService: customTagService, - pluginInputVariableParser: pluginInputVariableParser, - prePostCdScriptHistoryService: prePostCdScriptHistoryService, - scopedVariableManager: scopedVariableManager, - cdWorkflowService: cdWorkflowService, - imageDigestPolicyService: imageDigestPolicyService, - userService: userService, - gitSensorClient: gitSensorClient, - helmAppService: helmAppService, - enforcerUtil: enforcerUtil, - eventFactory: eventFactory, - eventClient: eventClient, - - globalEnvVariables: envVariables.GlobalEnvVariables, - userDeploymentRequestService: userDeploymentRequestService, - helmAppClient: helmAppClient, - appRepository: appRepository, - ciPipelineMaterialRepository: ciPipelineMaterialRepository, - imageScanHistoryReadService: imageScanHistoryReadService, - imageScanDeployInfoReadService: imageScanDeployInfoReadService, - imageScanDeployInfoService: imageScanDeployInfoService, - pipelineRepository: pipelineRepository, - pipelineOverrideRepository: pipelineOverrideRepository, - manifestPushConfigRepository: manifestPushConfigRepository, - chartRepository: chartRepository, - envRepository: envRepository, - cdWorkflowRepository: cdWorkflowRepository, - ciWorkflowRepository: ciWorkflowRepository, - ciArtifactRepository: ciArtifactRepository, - ciTemplateService: ciTemplateService, - gitMaterialReadService: gitMaterialReadService, - appLabelRepository: appLabelRepository, - ciPipelineRepository: ciPipelineRepository, - appWorkflowRepository: appWorkflowRepository, - dockerArtifactStoreRepository: dockerArtifactStoreRepository, - - imageScanService: imageScanService, - K8sUtil: K8sUtil, - - transactionUtilImpl: transactionUtilImpl, - - deploymentConfigService: deploymentConfigService, - deploymentServiceTypeConfig: envVariables.DeploymentServiceTypeConfig, - ciCdPipelineOrchestrator: ciCdPipelineOrchestrator, - gitOperationService: gitOperationService, - attributeService: attributeService, - cdWorkflowRunnerService: cdWorkflowRunnerService, - - clusterRepository: clusterRepository, - } - config, err := types.GetCdConfig() - if err != nil { - return nil, err - } - impl.config = config - return impl, nil -} - -func (impl *TriggerServiceImpl) TriggerStageForBulk(triggerRequest bean.TriggerRequest) error { +func (impl *HandlerServiceImpl) TriggerStageForBulk(triggerRequest bean.TriggerRequest) error { preStage, err := impl.pipelineStageService.GetCdStageByCdPipelineIdAndStageType(triggerRequest.Pipeline.Id, repository.PIPELINE_STAGE_TYPE_PRE_CD, false) if err != nil && err != pg.ErrNoRows { @@ -341,8 +98,8 @@ func (impl *TriggerServiceImpl) TriggerStageForBulk(triggerRequest bean.TriggerR } } -func (impl *TriggerServiceImpl) getCdPipelineForManualCdTrigger(ctx context.Context, pipelineId int) (*pipelineConfig.Pipeline, error) { - _, span := otel.Tracer("TriggerService").Start(ctx, "getCdPipelineForManualCdTrigger") +func (impl *HandlerServiceImpl) getCdPipelineForManualCdTrigger(ctx context.Context, pipelineId int) (*pipelineConfig.Pipeline, error) { + _, span := otel.Tracer("HandlerService").Start(ctx, "getCdPipelineForManualCdTrigger") defer span.End() cdPipeline, err := impl.pipelineRepository.FindById(pipelineId) if err != nil { @@ -362,8 +119,8 @@ func (impl *TriggerServiceImpl) getCdPipelineForManualCdTrigger(ctx context.Cont return cdPipeline, nil } -func (impl *TriggerServiceImpl) validateDeploymentTriggerRequest(ctx context.Context, validateDeploymentTriggerObj *bean.ValidateDeploymentTriggerObj) error { - newCtx, span := otel.Tracer("orchestrator").Start(ctx, "TriggerServiceImpl.validateDeploymentTriggerRequest") +func (impl *HandlerServiceImpl) validateDeploymentTriggerRequest(ctx context.Context, validateDeploymentTriggerObj *bean.ValidateDeploymentTriggerObj) error { + newCtx, span := otel.Tracer("orchestrator").Start(ctx, "HandlerServiceImpl.validateDeploymentTriggerRequest") defer span.End() // custom GitOps repo url validation --> Start err := impl.handleCustomGitOpsRepoValidation(validateDeploymentTriggerObj.Runner, validateDeploymentTriggerObj.CdPipeline, validateDeploymentTriggerObj.DeploymentConfig, validateDeploymentTriggerObj.TriggeredBy) @@ -395,7 +152,7 @@ func (impl *TriggerServiceImpl) validateDeploymentTriggerRequest(ctx context.Con } // TODO: write a wrapper to handle auto and manual trigger -func (impl *TriggerServiceImpl) ManualCdTrigger(triggerContext bean.TriggerContext, overrideRequest *bean3.ValuesOverrideRequest) (int, string, *bean4.ManifestPushTemplate, error) { +func (impl *HandlerServiceImpl) ManualCdTrigger(triggerContext bean.TriggerContext, overrideRequest *bean3.ValuesOverrideRequest) (int, string, *bean4.ManifestPushTemplate, error) { triggerContext.TriggerType = bean.Manual // setting triggeredAt variable to have consistent data for various audit log places in db for deployment time @@ -617,7 +374,7 @@ func isNotHibernateRequest(deploymentType models.DeploymentType) bool { } // TODO: write a wrapper to handle auto and manual trigger -func (impl *TriggerServiceImpl) TriggerAutomaticDeployment(request bean.TriggerRequest) error { +func (impl *HandlerServiceImpl) TriggerAutomaticDeployment(request bean.TriggerRequest) error { // in case of manual trigger auth is already applied and for auto triggers there is no need for auth check here triggeredBy := request.TriggeredBy pipeline := request.Pipeline @@ -694,7 +451,7 @@ func (impl *TriggerServiceImpl) TriggerAutomaticDeployment(request bean.TriggerR return nil } -func (impl *TriggerServiceImpl) TriggerCD(ctx context.Context, artifact *repository3.CiArtifact, cdWorkflowId, wfrId int, pipeline *pipelineConfig.Pipeline, envDeploymentConfig *bean9.DeploymentConfig, triggeredAt time.Time, triggeredBy int32) error { +func (impl *HandlerServiceImpl) TriggerCD(ctx context.Context, artifact *repository3.CiArtifact, cdWorkflowId, wfrId int, pipeline *pipelineConfig.Pipeline, envDeploymentConfig *bean9.DeploymentConfig, triggeredAt time.Time, triggeredBy int32) error { impl.logger.Debugw("automatic pipeline trigger attempt async", "artifactId", artifact.Id) err := impl.triggerReleaseAsync(ctx, artifact, cdWorkflowId, wfrId, pipeline, envDeploymentConfig, triggeredAt, triggeredBy) if err != nil { @@ -704,7 +461,7 @@ func (impl *TriggerServiceImpl) TriggerCD(ctx context.Context, artifact *reposit return err } -func (impl *TriggerServiceImpl) triggerReleaseAsync(ctx context.Context, artifact *repository3.CiArtifact, cdWorkflowId, wfrId int, pipeline *pipelineConfig.Pipeline, envDeploymentConfig *bean9.DeploymentConfig, triggeredAt time.Time, triggeredBy int32) error { +func (impl *HandlerServiceImpl) triggerReleaseAsync(ctx context.Context, artifact *repository3.CiArtifact, cdWorkflowId, wfrId int, pipeline *pipelineConfig.Pipeline, envDeploymentConfig *bean9.DeploymentConfig, triggeredAt time.Time, triggeredBy int32) error { err := impl.validateAndTrigger(ctx, pipeline, envDeploymentConfig, artifact, cdWorkflowId, wfrId, triggeredAt, triggeredBy) if err != nil { impl.logger.Errorw("error in trigger for pipeline", "pipelineId", strconv.Itoa(pipeline.Id)) @@ -713,7 +470,7 @@ func (impl *TriggerServiceImpl) triggerReleaseAsync(ctx context.Context, artifac return err } -func (impl *TriggerServiceImpl) validateAndTrigger(ctx context.Context, p *pipelineConfig.Pipeline, envDeploymentConfig *bean9.DeploymentConfig, artifact *repository3.CiArtifact, cdWorkflowId, wfrId int, triggeredAt time.Time, triggeredBy int32) error { +func (impl *HandlerServiceImpl) validateAndTrigger(ctx context.Context, p *pipelineConfig.Pipeline, envDeploymentConfig *bean9.DeploymentConfig, artifact *repository3.CiArtifact, cdWorkflowId, wfrId int, triggeredAt time.Time, triggeredBy int32) error { //TODO: verify this logic object := impl.enforcerUtil.GetAppRBACNameByAppId(p.AppId) envApp := strings.Split(object, "/") @@ -725,7 +482,7 @@ func (impl *TriggerServiceImpl) validateAndTrigger(ctx context.Context, p *pipel return err } -func (impl *TriggerServiceImpl) releasePipeline(ctx context.Context, pipeline *pipelineConfig.Pipeline, envDeploymentConfig *bean9.DeploymentConfig, artifact *repository3.CiArtifact, cdWorkflowId, wfrId int, triggeredAt time.Time, triggeredBy int32) error { +func (impl *HandlerServiceImpl) releasePipeline(ctx context.Context, pipeline *pipelineConfig.Pipeline, envDeploymentConfig *bean9.DeploymentConfig, artifact *repository3.CiArtifact, cdWorkflowId, wfrId int, triggeredAt time.Time, triggeredBy int32) error { startTime := time.Now() defer func() { impl.logger.Debugw("auto trigger release process completed", "timeTaken", time.Since(startTime), "cdPipelineId", pipeline.Id, "artifactId", artifact.Id, "wfrId", wfrId) @@ -759,9 +516,9 @@ func (impl *TriggerServiceImpl) releasePipeline(ctx context.Context, pipeline *p return err } -func (impl *TriggerServiceImpl) triggerAsyncRelease(ctx context.Context, overrideRequest *bean3.ValuesOverrideRequest, +func (impl *HandlerServiceImpl) triggerAsyncRelease(ctx context.Context, overrideRequest *bean3.ValuesOverrideRequest, envDeploymentConfig *bean9.DeploymentConfig, userDeploymentRequestId int, triggeredAt time.Time, deployedBy int32) (releaseNo int, manifestPushTemplate *bean4.ManifestPushTemplate, err error) { - newCtx, span := otel.Tracer("orchestrator").Start(ctx, "TriggerServiceImpl.triggerAsyncRelease") + newCtx, span := otel.Tracer("orchestrator").Start(ctx, "HandlerServiceImpl.triggerAsyncRelease") defer span.End() // build merged values and save PCO history for the release valuesOverrideResponse, err := impl.manifestCreationService.GetValuesOverrideForTrigger(newCtx, overrideRequest, envDeploymentConfig, triggeredAt) @@ -780,8 +537,8 @@ func (impl *TriggerServiceImpl) triggerAsyncRelease(ctx context.Context, overrid return impl.workflowEventPublishService.TriggerAsyncRelease(userDeploymentRequestId, overrideRequest, valuesOverrideResponse, newCtx, deployedBy) } -func (impl *TriggerServiceImpl) handleCDTriggerRelease(ctx context.Context, overrideRequest *bean3.ValuesOverrideRequest, envDeploymentConfig *bean9.DeploymentConfig, triggeredAt time.Time, deployedBy int32) (releaseNo int, manifestPushTemplate *bean4.ManifestPushTemplate, err error) { - newCtx, span := otel.Tracer("orchestrator").Start(ctx, "TriggerServiceImpl.handleCDTriggerRelease") +func (impl *HandlerServiceImpl) handleCDTriggerRelease(ctx context.Context, overrideRequest *bean3.ValuesOverrideRequest, envDeploymentConfig *bean9.DeploymentConfig, triggeredAt time.Time, deployedBy int32) (releaseNo int, manifestPushTemplate *bean4.ManifestPushTemplate, err error) { + newCtx, span := otel.Tracer("orchestrator").Start(ctx, "HandlerServiceImpl.handleCDTriggerRelease") defer span.End() // Handling for auto trigger if overrideRequest.UserId == 0 { @@ -829,7 +586,7 @@ func (impl *TriggerServiceImpl) handleCDTriggerRelease(ctx context.Context, over return impl.TriggerRelease(newCtx, overrideRequest, envDeploymentConfig, triggeredAt, deployedBy) } -func (impl *TriggerServiceImpl) auditDeploymentTriggerHistory(cdWfrId int, valuesOverrideResponse *app.ValuesOverrideResponse, ctx context.Context, triggeredAt time.Time, triggeredBy int32) (err error) { +func (impl *HandlerServiceImpl) auditDeploymentTriggerHistory(cdWfrId int, valuesOverrideResponse *app.ValuesOverrideResponse, ctx context.Context, triggeredAt time.Time, triggeredBy int32) (err error) { if valuesOverrideResponse.Pipeline == nil || valuesOverrideResponse.EnvOverride == nil { impl.logger.Warnw("unable to save histories for deployment trigger, invalid valuesOverrideResponse received", "cdWfrId", cdWfrId) return nil @@ -843,9 +600,9 @@ func (impl *TriggerServiceImpl) auditDeploymentTriggerHistory(cdWfrId int, value } // TriggerRelease will trigger Install/Upgrade request for Devtron App releases synchronously -func (impl *TriggerServiceImpl) TriggerRelease(ctx context.Context, overrideRequest *bean3.ValuesOverrideRequest, +func (impl *HandlerServiceImpl) TriggerRelease(ctx context.Context, overrideRequest *bean3.ValuesOverrideRequest, envDeploymentConfig *bean9.DeploymentConfig, triggeredAt time.Time, triggeredBy int32) (releaseNo int, manifestPushTemplate *bean4.ManifestPushTemplate, err error) { - newCtx, span := otel.Tracer("orchestrator").Start(ctx, "TriggerServiceImpl.TriggerRelease") + newCtx, span := otel.Tracer("orchestrator").Start(ctx, "HandlerServiceImpl.TriggerRelease") defer span.End() triggerEvent, skipRequest, err := impl.buildTriggerEventForOverrideRequest(overrideRequest, triggeredAt) if err != nil { @@ -897,10 +654,10 @@ func (impl *TriggerServiceImpl) TriggerRelease(ctx context.Context, overrideRequ return releaseNo, valuesOverrideResponse.ManifestPushTemplate, nil } -func (impl *TriggerServiceImpl) performGitOps(ctx context.Context, +func (impl *HandlerServiceImpl) performGitOps(ctx context.Context, overrideRequest *bean3.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse, builtChartPath string, triggerEvent bean.TriggerEvent) error { - newCtx, span := otel.Tracer("orchestrator").Start(ctx, "TriggerServiceImpl.performGitOps") + newCtx, span := otel.Tracer("orchestrator").Start(ctx, "HandlerServiceImpl.performGitOps") defer span.End() // update workflow runner status, used in app workflow view err := impl.cdWorkflowCommonService.UpdateNonTerminalStatusInRunner(newCtx, overrideRequest.WfrId, overrideRequest.UserId, cdWorkflow.WorkflowInProgress) @@ -927,7 +684,7 @@ func (impl *TriggerServiceImpl) performGitOps(ctx context.Context, return nil } -func (impl *TriggerServiceImpl) buildTriggerEventForOverrideRequest(overrideRequest *bean3.ValuesOverrideRequest, triggeredAt time.Time) (triggerEvent bean.TriggerEvent, skipRequest bool, err error) { +func (impl *HandlerServiceImpl) buildTriggerEventForOverrideRequest(overrideRequest *bean3.ValuesOverrideRequest, triggeredAt time.Time) (triggerEvent bean.TriggerEvent, skipRequest bool, err error) { triggerEvent = helper.NewTriggerEvent(overrideRequest.DeploymentAppType, triggeredAt, overrideRequest.UserId) request := statusBean.NewTimelineGetRequest(). WithCdWfrId(overrideRequest.WfrId). @@ -962,8 +719,8 @@ func (impl *TriggerServiceImpl) buildTriggerEventForOverrideRequest(overrideRequ return triggerEvent, skipRequest, nil } -func (impl *TriggerServiceImpl) triggerPipeline(overrideRequest *bean3.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse, builtChartPath string, triggerEvent bean.TriggerEvent, ctx context.Context) (releaseNo int, err error) { - newCtx, span := otel.Tracer("orchestrator").Start(ctx, "TriggerServiceImpl.triggerPipeline") +func (impl *HandlerServiceImpl) triggerPipeline(overrideRequest *bean3.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse, builtChartPath string, triggerEvent bean.TriggerEvent, ctx context.Context) (releaseNo int, err error) { + newCtx, span := otel.Tracer("orchestrator").Start(ctx, "HandlerServiceImpl.triggerPipeline") defer span.End() if triggerEvent.PerformChartPush { impl.logger.Debugw("performing chart push operation in triggerPipeline", "cdWfrId", overrideRequest.WfrId) @@ -999,7 +756,7 @@ func (impl *TriggerServiceImpl) triggerPipeline(overrideRequest *bean3.ValuesOve return valuesOverrideResponse.PipelineOverride.PipelineReleaseCounter, nil } -func (impl *TriggerServiceImpl) buildManifestPushTemplate(overrideRequest *bean3.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse, builtChartPath string) (*bean4.ManifestPushTemplate, error) { +func (impl *HandlerServiceImpl) buildManifestPushTemplate(overrideRequest *bean3.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse, builtChartPath string) (*bean4.ManifestPushTemplate, error) { manifestPushTemplate := &bean4.ManifestPushTemplate{ WorkflowRunnerId: overrideRequest.WfrId, @@ -1046,8 +803,8 @@ func (impl *TriggerServiceImpl) buildManifestPushTemplate(overrideRequest *bean3 return manifestPushTemplate, nil } -func (impl *TriggerServiceImpl) deployApp(ctx context.Context, overrideRequest *bean3.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse, triggerEvent bean.TriggerEvent) error { - newCtx, span := otel.Tracer("orchestrator").Start(ctx, "TriggerServiceImpl.deployApp") +func (impl *HandlerServiceImpl) deployApp(ctx context.Context, overrideRequest *bean3.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse, triggerEvent bean.TriggerEvent) error { + newCtx, span := otel.Tracer("orchestrator").Start(ctx, "HandlerServiceImpl.deployApp") defer span.End() var referenceChartByte []byte var err error @@ -1069,8 +826,8 @@ func (impl *TriggerServiceImpl) deployApp(ctx context.Context, overrideRequest * return nil } -func (impl *TriggerServiceImpl) createHelmAppForCdPipeline(ctx context.Context, overrideRequest *bean3.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse) (bool, []byte, error) { - newCtx, span := otel.Tracer("orchestrator").Start(ctx, "TriggerServiceImpl.createHelmAppForCdPipeline") +func (impl *HandlerServiceImpl) createHelmAppForCdPipeline(ctx context.Context, overrideRequest *bean3.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse) (bool, []byte, error) { + newCtx, span := otel.Tracer("orchestrator").Start(ctx, "HandlerServiceImpl.createHelmAppForCdPipeline") defer span.End() pipelineModel := valuesOverrideResponse.Pipeline envOverride := valuesOverrideResponse.EnvOverride @@ -1175,7 +932,7 @@ func (impl *TriggerServiceImpl) createHelmAppForCdPipeline(ctx context.Context, return true, referenceChartByte, nil } -func (impl *TriggerServiceImpl) getHelmHistoryLimitAndChartMetadataForHelmAppCreation(ctx context.Context, +func (impl *HandlerServiceImpl) getHelmHistoryLimitAndChartMetadataForHelmAppCreation(ctx context.Context, valuesOverrideResponse *app.ValuesOverrideResponse) (*chart.Metadata, int32, *gRPC.ReleaseIdentifier, error) { pipelineModel := valuesOverrideResponse.Pipeline envOverride := valuesOverrideResponse.EnvOverride @@ -1224,9 +981,9 @@ func (impl *TriggerServiceImpl) getHelmHistoryLimitAndChartMetadataForHelmAppCre return chartMetaData, helmRevisionHistory, releaseIdentifier, nil } -func (impl *TriggerServiceImpl) deployArgoCdApp(ctx context.Context, overrideRequest *bean3.ValuesOverrideRequest, +func (impl *HandlerServiceImpl) deployArgoCdApp(ctx context.Context, overrideRequest *bean3.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse) error { - newCtx, span := otel.Tracer("orchestrator").Start(ctx, "TriggerServiceImpl.deployArgoCdApp") + newCtx, span := otel.Tracer("orchestrator").Start(ctx, "HandlerServiceImpl.deployArgoCdApp") defer span.End() name, err := impl.createArgoApplicationIfRequired(newCtx, valuesOverrideResponse.EnvOverride, valuesOverrideResponse.Pipeline, valuesOverrideResponse.DeploymentConfig, overrideRequest.UserId) if err != nil { @@ -1270,7 +1027,7 @@ func (impl *TriggerServiceImpl) deployArgoCdApp(ctx context.Context, overrideReq } // update repoUrl, revision and argo app sync mode (auto/manual) if needed -func (impl *TriggerServiceImpl) updateArgoPipeline(ctx context.Context, pipeline *pipelineConfig.Pipeline, envOverride *bean10.EnvConfigOverride, deploymentConfig *bean9.DeploymentConfig) (bool, error) { +func (impl *HandlerServiceImpl) updateArgoPipeline(ctx context.Context, pipeline *pipelineConfig.Pipeline, envOverride *bean10.EnvConfigOverride, deploymentConfig *bean9.DeploymentConfig) (bool, error) { if !deploymentConfig.IsArgoAppPatchSupported() { impl.logger.Infow("argo app patch not supported", "pipelineId", pipeline.Id, "pipelineName", pipeline.Name) return false, nil @@ -1279,7 +1036,7 @@ func (impl *TriggerServiceImpl) updateArgoPipeline(ctx context.Context, pipeline impl.logger.Errorw("err in syncing ACD, ctx is NULL", "pipelineId", pipeline.Id, "pipelineName", pipeline.Name) return false, nil } - newCtx, span := otel.Tracer("orchestrator").Start(ctx, "TriggerServiceImpl.updateArgoPipeline") + newCtx, span := otel.Tracer("orchestrator").Start(ctx, "HandlerServiceImpl.updateArgoPipeline") defer span.End() argoAppName := pipeline.DeploymentAppName impl.logger.Infow("received payload, updateArgoPipeline", "appId", pipeline.AppId, "pipelineName", pipeline.Name, "envId", envOverride.TargetEnvironment, "argoAppName", argoAppName) @@ -1335,9 +1092,9 @@ func (impl *TriggerServiceImpl) updateArgoPipeline(ctx context.Context, pipeline } } -func (impl *TriggerServiceImpl) createArgoApplicationIfRequired(ctx context.Context, envConfigOverride *bean10.EnvConfigOverride, +func (impl *HandlerServiceImpl) createArgoApplicationIfRequired(ctx context.Context, envConfigOverride *bean10.EnvConfigOverride, pipeline *pipelineConfig.Pipeline, deploymentConfig *bean9.DeploymentConfig, userId int32) (string, error) { - newCtx, span := otel.Tracer("orchestrator").Start(ctx, "TriggerServiceImpl.createArgoApplicationIfRequired") + newCtx, span := otel.Tracer("orchestrator").Start(ctx, "HandlerServiceImpl.createArgoApplicationIfRequired") defer span.End() envModel, err := impl.envRepository.FindById(envConfigOverride.TargetEnvironment) if err != nil { @@ -1384,7 +1141,7 @@ func (impl *TriggerServiceImpl) createArgoApplicationIfRequired(ctx context.Cont } } -func (impl *TriggerServiceImpl) updatePipeline(pipeline *pipelineConfig.Pipeline, userId int32) (bool, error) { +func (impl *HandlerServiceImpl) updatePipeline(pipeline *pipelineConfig.Pipeline, userId int32) (bool, error) { err := impl.pipelineRepository.SetDeploymentAppCreatedInPipeline(true, pipeline.Id, userId) if err != nil { impl.logger.Errorw("error on updating cd pipeline for setting deployment app created", "err", err) @@ -1394,9 +1151,9 @@ func (impl *TriggerServiceImpl) updatePipeline(pipeline *pipelineConfig.Pipeline } // helmInstallReleaseWithCustomChart performs helm install with custom chart -func (impl *TriggerServiceImpl) helmInstallReleaseWithCustomChart(ctx context.Context, releaseIdentifier *gRPC.ReleaseIdentifier, +func (impl *HandlerServiceImpl) helmInstallReleaseWithCustomChart(ctx context.Context, releaseIdentifier *gRPC.ReleaseIdentifier, referenceChartByte []byte, valuesYaml, k8sServerVersion string, forceSync bool) (*gRPC.HelmInstallCustomResponse, error) { - newCtx, span := otel.Tracer("orchestrator").Start(ctx, "TriggerServiceImpl.helmInstallReleaseWithCustomChart") + newCtx, span := otel.Tracer("orchestrator").Start(ctx, "HandlerServiceImpl.helmInstallReleaseWithCustomChart") defer span.End() helmInstallRequest := gRPC.HelmInstallCustomRequest{ ValuesYaml: valuesYaml, @@ -1413,52 +1170,9 @@ func (impl *TriggerServiceImpl) helmInstallReleaseWithCustomChart(ctx context.Co return impl.helmAppClient.InstallReleaseWithCustomChart(newCtx, &helmInstallRequest) } -func (impl *TriggerServiceImpl) writeCDTriggerEvent(overrideRequest *bean3.ValuesOverrideRequest, artifact *repository3.CiArtifact, releaseId, pipelineOverrideId, wfrId int) { - - event, err := impl.eventFactory.Build(util2.Trigger, &overrideRequest.PipelineId, overrideRequest.AppId, &overrideRequest.EnvId, util2.CD) - if err != nil { - impl.logger.Errorw("error in building cd trigger event", "cdPipelineId", overrideRequest.PipelineId, "err", err) - } - impl.logger.Debugw("event WriteCDTriggerEvent", "event", event) - wfr := impl.getEnrichedWorkflowRunner(overrideRequest, artifact, wfrId) - event = impl.eventFactory.BuildExtraCDData(event, wfr, pipelineOverrideId, bean3.CD_WORKFLOW_TYPE_DEPLOY) - _, evtErr := impl.eventClient.WriteNotificationEvent(event) - if evtErr != nil { - impl.logger.Errorw("CD trigger event not sent", "error", evtErr) - } - deploymentEvent := app.DeploymentEvent{ - ApplicationId: overrideRequest.AppId, - EnvironmentId: overrideRequest.EnvId, // check for production Environment - ReleaseId: releaseId, - PipelineOverrideId: pipelineOverrideId, - TriggerTime: time.Now(), - CiArtifactId: overrideRequest.CiArtifactId, - } - - ciPipelineMaterials, err := impl.ciPipelineMaterialRepository.GetByPipelineId(artifact.PipelineId) - if err != nil { - impl.logger.Errorw("error in ") - } - materialInfoMap, mErr := artifact.ParseMaterialInfo() - if mErr != nil { - impl.logger.Errorw("material info map error", mErr) - return - } - for _, ciPipelineMaterial := range ciPipelineMaterials { - hash := materialInfoMap[ciPipelineMaterial.GitMaterial.Url] - pipelineMaterialInfo := &app.PipelineMaterialInfo{PipelineMaterialId: ciPipelineMaterial.Id, CommitHash: hash} - deploymentEvent.PipelineMaterials = append(deploymentEvent.PipelineMaterials, pipelineMaterialInfo) - } - impl.logger.Infow("triggering deployment event", "event", deploymentEvent) - err = impl.eventClient.WriteNatsEvent(pubsub.CD_SUCCESS, deploymentEvent) - if err != nil { - impl.logger.Errorw("error in writing cd trigger event", "err", err) - } -} - -func (impl *TriggerServiceImpl) markImageScanDeployed(ctx context.Context, appId, envId, clusterId int, +func (impl *HandlerServiceImpl) markImageScanDeployed(ctx context.Context, appId, envId, clusterId int, imageDigest string, isScanEnabled bool, image string) error { - _, span := otel.Tracer("orchestrator").Start(ctx, "TriggerServiceImpl.markImageScanDeployed") + _, span := otel.Tracer("orchestrator").Start(ctx, "HandlerServiceImpl.markImageScanDeployed") defer span.End() // TODO KB: send NATS event for self consumption impl.logger.Debugw("mark image scan deployed for devtron app, from cd auto or manual trigger", "imageDigest", imageDigest) @@ -1526,11 +1240,11 @@ func (impl *TriggerServiceImpl) markImageScanDeployed(ctx context.Context, appId return err } -func (impl *TriggerServiceImpl) isDevtronAsyncHelmInstallModeEnabled(forceSync bool) bool { +func (impl *HandlerServiceImpl) isDevtronAsyncHelmInstallModeEnabled(forceSync bool) bool { return impl.globalEnvVariables.EnableAsyncHelmInstallDevtronChart && !forceSync } -func (impl *TriggerServiceImpl) isDevtronAsyncInstallModeEnabled(overrideRequest *bean3.ValuesOverrideRequest) (bool, error) { +func (impl *HandlerServiceImpl) isDevtronAsyncInstallModeEnabled(overrideRequest *bean3.ValuesOverrideRequest) (bool, error) { if util.IsHelmApp(overrideRequest.DeploymentAppType) { return impl.isDevtronAsyncHelmInstallModeEnabled(overrideRequest.ForceSyncDeployment), nil } else if util.IsAcdApp(overrideRequest.DeploymentAppType) { @@ -1540,7 +1254,7 @@ func (impl *TriggerServiceImpl) isDevtronAsyncInstallModeEnabled(overrideRequest return false, nil } -func (impl *TriggerServiceImpl) deleteCorruptedPipelineStage(pipelineStage *repository.PipelineStage, triggeredBy int32) (error, bool) { +func (impl *HandlerServiceImpl) deleteCorruptedPipelineStage(pipelineStage *repository.PipelineStage, triggeredBy int32) (error, bool) { if pipelineStage != nil { stageReq := &bean8.PipelineStageDto{ Id: pipelineStage.Id, @@ -1556,7 +1270,7 @@ func (impl *TriggerServiceImpl) deleteCorruptedPipelineStage(pipelineStage *repo return nil, false } -func (impl *TriggerServiceImpl) handleCustomGitOpsRepoValidation(runner *pipelineConfig.CdWorkflowRunner, pipeline *pipelineConfig.Pipeline, envDeploymentConfig *bean9.DeploymentConfig, triggeredBy int32) error { +func (impl *HandlerServiceImpl) handleCustomGitOpsRepoValidation(runner *pipelineConfig.CdWorkflowRunner, pipeline *pipelineConfig.Pipeline, envDeploymentConfig *bean9.DeploymentConfig, triggeredBy int32) error { if !util.IsAcdApp(pipeline.DeploymentAppName) { return nil } @@ -1589,7 +1303,7 @@ func (impl *TriggerServiceImpl) handleCustomGitOpsRepoValidation(runner *pipelin return nil } -func (impl *TriggerServiceImpl) getSanitizedK8sVersion(referenceTemplate string) (string, error) { +func (impl *HandlerServiceImpl) getSanitizedK8sVersion(referenceTemplate string) (string, error) { var sanitizedK8sVersion string //handle specific case for all cronjob charts from cronjob-chart_1-2-0 to cronjob-chart_1-5-0 where semverCompare //comparison func has wrong api version mentioned, so for already installed charts via these charts that comparison @@ -1606,7 +1320,7 @@ func (impl *TriggerServiceImpl) getSanitizedK8sVersion(referenceTemplate string) return sanitizedK8sVersion, nil } -func (impl *TriggerServiceImpl) getReferenceChartByteForHelmTypeApp(envOverride *bean10.EnvConfigOverride, +func (impl *HandlerServiceImpl) getReferenceChartByteForHelmTypeApp(envOverride *bean10.EnvConfigOverride, chartMetaData *chart.Metadata, referenceTemplatePath string, overrideRequest *bean3.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse) ([]byte, error) { referenceChartByte := envOverride.Chart.ReferenceChart diff --git a/pkg/deployment/trigger/devtronApps/TriggerService_ent1.go b/pkg/deployment/trigger/devtronApps/deployStageHandlerCode_ent.go similarity index 67% rename from pkg/deployment/trigger/devtronApps/TriggerService_ent1.go rename to pkg/deployment/trigger/devtronApps/deployStageHandlerCode_ent.go index f008f72adc..347f980a07 100644 --- a/pkg/deployment/trigger/devtronApps/TriggerService_ent1.go +++ b/pkg/deployment/trigger/devtronApps/deployStageHandlerCode_ent.go @@ -1,3 +1,19 @@ +/* + * Copyright (c) 2024. Devtron Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package devtronApps import ( @@ -17,29 +33,29 @@ import ( "helm.sh/helm/v3/pkg/chart" ) -func (impl *TriggerServiceImpl) getEnrichedWorkflowRunner(overrideRequest *bean3.ValuesOverrideRequest, artifact *repository3.CiArtifact, wfrId int) *pipelineConfig.CdWorkflowRunner { +func (impl *HandlerServiceImpl) getEnrichedWorkflowRunner(overrideRequest *bean3.ValuesOverrideRequest, artifact *repository3.CiArtifact, wfrId int) *pipelineConfig.CdWorkflowRunner { return nil } -func (impl *TriggerServiceImpl) postDeployHook(overrideRequest *bean3.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse, referenceChartByte []byte, err error) { +func (impl *HandlerServiceImpl) postDeployHook(overrideRequest *bean3.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse, referenceChartByte []byte, err error) { impl.logger.Debugw("no post deploy hook registered") } -func (impl *TriggerServiceImpl) isDevtronAsyncArgoCdInstallModeEnabledForApp(appId, envId int, forceSync bool) (bool, error) { +func (impl *HandlerServiceImpl) isDevtronAsyncArgoCdInstallModeEnabledForApp(appId, envId int, forceSync bool) (bool, error) { return impl.globalEnvVariables.EnableAsyncArgoCdInstallDevtronChart && !forceSync, nil } -func (impl *TriggerServiceImpl) getClusterGRPCConfig(cluster repository2.Cluster) *gRPC.ClusterConfig { +func (impl *HandlerServiceImpl) getClusterGRPCConfig(cluster repository2.Cluster) *gRPC.ClusterConfig { clusterConfig := helper.ConvertClusterBeanToGrpcConfig(cluster) return clusterConfig } -func (impl *TriggerServiceImpl) overrideReferenceChartByteForHelmTypeApp(valuesOverrideResponse *app.ValuesOverrideResponse, +func (impl *HandlerServiceImpl) overrideReferenceChartByteForHelmTypeApp(valuesOverrideResponse *app.ValuesOverrideResponse, chartMetaData *chart.Metadata, referenceTemplatePath string, referenceChartByte []byte) ([]byte, error) { return referenceChartByte, nil } -func (impl *TriggerServiceImpl) getManifestPushService(storageType string) publish.ManifestPushService { +func (impl *HandlerServiceImpl) getManifestPushService(storageType string) publish.ManifestPushService { var manifestPushService publish.ManifestPushService if storageType == bean2.ManifestStorageGit { manifestPushService = impl.gitOpsManifestPushService @@ -47,25 +63,25 @@ func (impl *TriggerServiceImpl) getManifestPushService(storageType string) publi return manifestPushService } -func (impl *TriggerServiceImpl) preStageHandlingForTriggerStageInBulk(triggerRequest *bean.TriggerRequest) error { +func (impl *HandlerServiceImpl) preStageHandlingForTriggerStageInBulk(triggerRequest *bean.TriggerRequest) error { return nil } -func (impl *TriggerServiceImpl) manifestGenerationFailedTimelineHandling(triggerEvent bean.TriggerEvent, overrideRequest *bean3.ValuesOverrideRequest, err error) { +func (impl *HandlerServiceImpl) manifestGenerationFailedTimelineHandling(triggerEvent bean.TriggerEvent, overrideRequest *bean3.ValuesOverrideRequest, err error) { } -func (impl *TriggerServiceImpl) getHelmManifestForTriggerRelease(ctx context.Context, triggerEvent bean.TriggerEvent, overrideRequest *bean3.ValuesOverrideRequest, +func (impl *HandlerServiceImpl) getHelmManifestForTriggerRelease(ctx context.Context, triggerEvent bean.TriggerEvent, overrideRequest *bean3.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse, builtChartPath string) ([]byte, error) { return nil, nil } -func (impl *TriggerServiceImpl) buildManifestPushTemplateForNonGitStorageType(overrideRequest *bean3.ValuesOverrideRequest, +func (impl *HandlerServiceImpl) buildManifestPushTemplateForNonGitStorageType(overrideRequest *bean3.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse, builtChartPath string, err error, manifestPushConfig *repository.ManifestPushConfig, manifestPushTemplate *bean4.ManifestPushTemplate) error { return nil } -func (impl *TriggerServiceImpl) triggerReleaseSuccessHandling(triggerEvent bean.TriggerEvent, overrideRequest *bean3.ValuesOverrideRequest, +func (impl *HandlerServiceImpl) triggerReleaseSuccessHandling(triggerEvent bean.TriggerEvent, overrideRequest *bean3.ValuesOverrideRequest, valuesOverrideResponse *app.ValuesOverrideResponse, helmManifest []byte) error { return nil } diff --git a/pkg/deployment/trigger/devtronApps/feasibilityChecker.go b/pkg/deployment/trigger/devtronApps/feasibilityChecker.go index 43926bb86a..c03968d5de 100644 --- a/pkg/deployment/trigger/devtronApps/feasibilityChecker.go +++ b/pkg/deployment/trigger/devtronApps/feasibilityChecker.go @@ -24,7 +24,7 @@ type FeasibilityManager interface { CheckFeasibility(triggerRequirementRequest *bean.TriggerRequirementRequestDto) error } -func (impl *TriggerServiceImpl) CheckFeasibility(triggerRequirementRequest *bean.TriggerRequirementRequestDto) error { +func (impl *HandlerServiceImpl) CheckFeasibility(triggerRequirementRequest *bean.TriggerRequirementRequestDto) error { // have not implemented right now, will be implemented in future for security vulnerability return nil } diff --git a/pkg/deployment/trigger/devtronApps/PostStageTriggerService.go b/pkg/deployment/trigger/devtronApps/postStageHandlerCode.go similarity index 97% rename from pkg/deployment/trigger/devtronApps/PostStageTriggerService.go rename to pkg/deployment/trigger/devtronApps/postStageHandlerCode.go index 8a11b06a6b..1bf2399ffc 100644 --- a/pkg/deployment/trigger/devtronApps/PostStageTriggerService.go +++ b/pkg/deployment/trigger/devtronApps/postStageHandlerCode.go @@ -30,7 +30,7 @@ import ( "time" ) -func (impl *TriggerServiceImpl) TriggerPostStage(request bean.TriggerRequest) (*bean4.ManifestPushTemplate, error) { +func (impl *HandlerServiceImpl) TriggerPostStage(request bean.TriggerRequest) (*bean4.ManifestPushTemplate, error) { request.WorkflowType = bean2.CD_WORKFLOW_TYPE_POST // setting triggeredAt variable to have consistent data for various audit log places in db for deployment time triggeredAt := time.Now() @@ -123,7 +123,7 @@ func (impl *TriggerServiceImpl) TriggerPostStage(request bean.TriggerRequest) (* return nil, err } - _, jobHelmPackagePath, err := impl.cdWorkflowService.SubmitWorkflow(cdStageWorkflowRequest) + _, jobHelmPackagePath, err := impl.workflowService.SubmitWorkflow(cdStageWorkflowRequest) if err != nil { impl.logger.Errorw("error in submitting workflow", "err", err, "workflowId", cdStageWorkflowRequest.WorkflowId, "pipeline", pipeline, "env", env) runner.Status = cdWorkflow.WorkflowFailed @@ -164,7 +164,7 @@ func (impl *TriggerServiceImpl) TriggerPostStage(request bean.TriggerRequest) (* return manifestPushTempate, nil } -func (impl *TriggerServiceImpl) buildWfRequestErrorHandler(runner *pipelineConfig.CdWorkflowRunner, err error, triggeredBy int32) (*bean4.ManifestPushTemplate, error) { +func (impl *HandlerServiceImpl) buildWfRequestErrorHandler(runner *pipelineConfig.CdWorkflowRunner, err error, triggeredBy int32) (*bean4.ManifestPushTemplate, error) { dbErr := impl.cdWorkflowCommonService.MarkCurrentDeploymentFailed(runner, err, triggeredBy) if dbErr != nil { impl.logger.Errorw("error while updating current runner status to failed, buildWfRequestErrorHandler", "runner", runner.Id, "err", dbErr, "releaseErr", err) diff --git a/pkg/deployment/trigger/devtronApps/PostStageTriggerService_ent.go b/pkg/deployment/trigger/devtronApps/postStageHandlerCode_ent.go similarity index 92% rename from pkg/deployment/trigger/devtronApps/PostStageTriggerService_ent.go rename to pkg/deployment/trigger/devtronApps/postStageHandlerCode_ent.go index d99b813b1d..197aecd22d 100644 --- a/pkg/deployment/trigger/devtronApps/PostStageTriggerService_ent.go +++ b/pkg/deployment/trigger/devtronApps/postStageHandlerCode_ent.go @@ -26,13 +26,13 @@ import ( "time" ) -func (impl *TriggerServiceImpl) checkFeasibilityForPostStage(pipeline *pipelineConfig.Pipeline, request *bean.TriggerRequest, +func (impl *HandlerServiceImpl) checkFeasibilityForPostStage(pipeline *pipelineConfig.Pipeline, request *bean.TriggerRequest, env *repository.Environment, cdWf *pipelineConfig.CdWorkflow, triggeredBy int32) (interface{}, error) { //here return type is interface as ResourceFilterEvaluationAudit is not present in this version return nil, nil } -func (impl *TriggerServiceImpl) getManifestPushTemplateForPostStage(request bean.TriggerRequest, envDevploymentConfig *bean5.DeploymentConfig, +func (impl *HandlerServiceImpl) getManifestPushTemplateForPostStage(request bean.TriggerRequest, envDevploymentConfig *bean5.DeploymentConfig, jobHelmPackagePath string, cdStageWorkflowRequest *types.WorkflowRequest, cdWf *pipelineConfig.CdWorkflow, runner *pipelineConfig.CdWorkflowRunner, pipeline *pipelineConfig.Pipeline, triggeredBy int32, triggeredAt time.Time) (*bean4.ManifestPushTemplate, error) { return nil, nil diff --git a/pkg/deployment/trigger/devtronApps/prePostWfAndLogsCode.go b/pkg/deployment/trigger/devtronApps/prePostWfAndLogsCode.go new file mode 100644 index 0000000000..d3ff8b5e5f --- /dev/null +++ b/pkg/deployment/trigger/devtronApps/prePostWfAndLogsCode.go @@ -0,0 +1,337 @@ +package devtronApps + +import ( + "bufio" + "errors" + "fmt" + "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" + blob_storage "github.com/devtron-labs/common-lib/blob-storage" + "github.com/devtron-labs/common-lib/utils/k8s" + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" + cdWorkflow2 "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow/cdWorkflow" + "github.com/devtron-labs/devtron/internal/util" + bean2 "github.com/devtron-labs/devtron/pkg/bean" + "github.com/devtron-labs/devtron/pkg/cluster/adapter" + bean3 "github.com/devtron-labs/devtron/pkg/cluster/bean" + "github.com/devtron-labs/devtron/pkg/pipeline" + "github.com/devtron-labs/devtron/pkg/pipeline/constants" + "github.com/devtron-labs/devtron/pkg/pipeline/types" + util2 "github.com/devtron-labs/devtron/pkg/pipeline/util" + "k8s.io/client-go/rest" + "net/http" + "os" + "path/filepath" + "strconv" + "strings" + "time" +) + +func (impl *HandlerServiceImpl) CancelStage(workflowRunnerId int, forceAbort bool, userId int32) (int, error) { + workflowRunner, err := impl.cdWorkflowRepository.FindWorkflowRunnerById(workflowRunnerId) + if err != nil { + impl.logger.Errorw("err", "err", err) + return 0, err + } + pipeline, err := impl.pipelineRepository.FindById(workflowRunner.CdWorkflow.PipelineId) + if err != nil { + impl.logger.Errorw("error while fetching cd pipeline", "err", err) + return 0, err + } + + env, err := impl.envRepository.FindById(pipeline.EnvironmentId) + if err != nil { + impl.logger.Errorw("could not fetch stage env", "err", err) + return 0, err + } + + var clusterBean bean3.ClusterBean + if env != nil && env.Cluster != nil { + clusterBean = adapter.GetClusterBean(*env.Cluster) + } + clusterConfig := clusterBean.GetClusterConfig() + var isExtCluster bool + if workflowRunner.WorkflowType == types.PRE { + isExtCluster = pipeline.RunPreStageInEnv + } else if workflowRunner.WorkflowType == types.POST { + isExtCluster = pipeline.RunPostStageInEnv + } + var restConfig *rest.Config + if isExtCluster { + restConfig, err = impl.K8sUtil.GetRestConfigByCluster(clusterConfig) + if err != nil { + impl.logger.Errorw("error in getting rest config by cluster id", "err", err) + return 0, err + } + } + // Terminate workflow + cancelWfDtoRequest := &types.CancelWfRequestDto{ + ExecutorType: workflowRunner.ExecutorType, + WorkflowName: workflowRunner.Name, + Namespace: workflowRunner.Namespace, + RestConfig: restConfig, + IsExt: isExtCluster, + Environment: nil, + } + err = impl.workflowService.TerminateWorkflow(cancelWfDtoRequest) + if err != nil && forceAbort { + impl.logger.Errorw("error in terminating workflow, with force abort flag as true", "workflowName", workflowRunner.Name, "err", err) + cancelWfDtoRequest.WorkflowGenerateName = fmt.Sprintf("%d-%s", workflowRunnerId, workflowRunner.Name) + err1 := impl.workflowService.TerminateDanglingWorkflows(cancelWfDtoRequest) + if err1 != nil { + impl.logger.Errorw("error in terminating dangling workflows", "cancelWfDtoRequest", cancelWfDtoRequest, "err", err) + // ignoring error here in case of force abort, confirmed from product + } + } else if err != nil && strings.Contains(err.Error(), "cannot find workflow") { + return 0, &util.ApiError{Code: "200", HttpStatusCode: http.StatusBadRequest, UserMessage: err.Error()} + } else if err != nil { + impl.logger.Error("cannot terminate wf runner", "err", err) + return 0, err + } + if forceAbort { + err = impl.handleForceAbortCaseForCdStage(workflowRunner, forceAbort) + if err != nil { + impl.logger.Errorw("error in handleForceAbortCaseForCdStage", "forceAbortFlag", forceAbort, "workflowRunner", workflowRunner, "err", err) + return 0, err + } + return workflowRunner.Id, nil + } + if len(workflowRunner.ImagePathReservationIds) > 0 { + err := impl.customTagService.DeactivateImagePathReservationByImageIds(workflowRunner.ImagePathReservationIds) + if err != nil { + impl.logger.Errorw("error in deactivating image path reservation ids", "err", err) + return 0, err + } + } + workflowRunner.Status = cdWorkflow2.WorkflowCancel + workflowRunner.UpdatedOn = time.Now() + workflowRunner.UpdatedBy = userId + err = impl.cdWorkflowRunnerService.UpdateCdWorkflowRunnerWithStage(workflowRunner) + if err != nil { + impl.logger.Error("cannot update deleted workflow runner status, but wf deleted", "err", err) + return 0, err + } + return workflowRunner.Id, nil +} + +func (impl *HandlerServiceImpl) updateWorkflowRunnerForForceAbort(workflowRunner *pipelineConfig.CdWorkflowRunner) error { + workflowRunner.Status = cdWorkflow2.WorkflowCancel + workflowRunner.PodStatus = string(bean2.Failed) + workflowRunner.Message = constants.FORCE_ABORT_MESSAGE_AFTER_STARTING_STAGE + err := impl.cdWorkflowRunnerService.UpdateCdWorkflowRunnerWithStage(workflowRunner) + if err != nil { + impl.logger.Errorw("error in updating workflow status in cd workflow runner in force abort case", "err", err) + return err + } + return nil +} + +func (impl *HandlerServiceImpl) handleForceAbortCaseForCdStage(workflowRunner *pipelineConfig.CdWorkflowRunner, forceAbort bool) error { + isWorkflowInNonTerminalStage := workflowRunner.Status == string(v1alpha1.NodePending) || workflowRunner.Status == string(v1alpha1.NodeRunning) + if !isWorkflowInNonTerminalStage { + if forceAbort { + return impl.updateWorkflowRunnerForForceAbort(workflowRunner) + } else { + return &util.ApiError{Code: "200", HttpStatusCode: 400, UserMessage: "cannot cancel stage, stage not in progress"} + } + } + //this arises when someone deletes the workflow in resource browser and wants to force abort a cd stage(pre/post) + if workflowRunner.Status == string(v1alpha1.NodeRunning) && forceAbort { + return impl.updateWorkflowRunnerForForceAbort(workflowRunner) + } + return nil +} + +func (impl *HandlerServiceImpl) DownloadCdWorkflowArtifacts(buildId int) (*os.File, error) { + wfr, err := impl.cdWorkflowRepository.FindWorkflowRunnerById(buildId) + if err != nil { + impl.logger.Errorw("unable to fetch ciWorkflow", "err", err) + return nil, err + } + useExternalBlobStorage := pipeline.IsExternalBlobStorageEnabled(wfr.IsExternalRun(), impl.config.UseBlobStorageConfigInCdWorkflow) + if !wfr.BlobStorageEnabled { + return nil, errors.New("logs-not-stored-in-repository") + } + + cdConfigLogsBucket := impl.config.GetDefaultBuildLogsBucket() + cdConfigCdCacheRegion := impl.config.GetDefaultCdLogsBucketRegion() + + item := strconv.Itoa(wfr.Id) + awsS3BaseConfig := &blob_storage.AwsS3BaseConfig{ + AccessKey: impl.config.BlobStorageS3AccessKey, + Passkey: impl.config.BlobStorageS3SecretKey, + EndpointUrl: impl.config.BlobStorageS3Endpoint, + IsInSecure: impl.config.BlobStorageS3EndpointInsecure, + BucketName: cdConfigLogsBucket, + Region: cdConfigCdCacheRegion, + VersioningEnabled: impl.config.BlobStorageS3BucketVersioned, + } + azureBlobBaseConfig := &blob_storage.AzureBlobBaseConfig{ + Enabled: impl.config.CloudProvider == types.BLOB_STORAGE_AZURE, + AccountKey: impl.config.AzureAccountKey, + AccountName: impl.config.AzureAccountName, + BlobContainerName: impl.config.AzureBlobContainerCiLog, + } + gcpBlobBaseConfig := &blob_storage.GcpBlobBaseConfig{ + BucketName: cdConfigLogsBucket, + CredentialFileJsonData: impl.config.BlobStorageGcpCredentialJson, + } + cdArtifactLocationFormat := impl.config.GetArtifactLocationFormat() + key := fmt.Sprintf(cdArtifactLocationFormat, wfr.CdWorkflow.Id, wfr.Id) + if len(wfr.CdArtifactLocation) != 0 && util2.IsValidUrlSubPath(wfr.CdArtifactLocation) { + key = wfr.CdArtifactLocation + } else if util2.IsValidUrlSubPath(key) { + impl.cdWorkflowRepository.MigrateCdArtifactLocation(wfr.Id, key) + } + baseLogLocationPathConfig := impl.config.BaseLogLocationPath + blobStorageService := blob_storage.NewBlobStorageServiceImpl(nil) + destinationKey := filepath.Clean(filepath.Join(baseLogLocationPathConfig, item)) + request := &blob_storage.BlobStorageRequest{ + StorageType: impl.config.CloudProvider, + SourceKey: key, + DestinationKey: destinationKey, + AzureBlobBaseConfig: azureBlobBaseConfig, + AwsS3BaseConfig: awsS3BaseConfig, + GcpBlobBaseConfig: gcpBlobBaseConfig, + } + if useExternalBlobStorage { + clusterConfig, err := impl.clusterService.GetClusterConfigByClusterId(wfr.CdWorkflow.Pipeline.Environment.ClusterId) + if err != nil { + impl.logger.Errorw("GetClusterConfigByClusterId, error in fetching clusterConfig", "err", err, "clusterId", wfr.CdWorkflow.Pipeline.Environment.ClusterId) + return nil, err + } + // fetch extClusterBlob cm and cs from k8s client, if they are present then read creds + // from them else return. + cmConfig, secretConfig, err := impl.blobConfigStorageService.FetchCmAndSecretBlobConfigFromExternalCluster(clusterConfig, wfr.Namespace) + if err != nil { + impl.logger.Errorw("error in fetching config map and secret from external cluster", "err", err, "clusterConfig", clusterConfig) + return nil, err + } + request = pipeline.UpdateRequestWithExtClusterCmAndSecret(request, cmConfig, secretConfig) + } + _, numBytes, err := blobStorageService.Get(request) + if err != nil { + impl.logger.Errorw("error occurred while downloading file", "request", request, "error", err) + return nil, errors.New("failed to download resource") + } + + file, err := os.Open(destinationKey) + if err != nil { + impl.logger.Errorw("unable to open file", "file", item, "err", err) + return nil, errors.New("unable to open file") + } + + impl.logger.Infow("Downloaded ", "name", file.Name(), "bytes", numBytes) + return file, nil +} + +func (impl *HandlerServiceImpl) GetRunningWorkflowLogs(environmentId int, pipelineId int, wfrId int) (*bufio.Reader, func() error, error) { + cdWorkflow, err := impl.cdWorkflowRepository.FindWorkflowRunnerById(wfrId) + if err != nil { + impl.logger.Errorw("error on fetch wf runner", "err", err) + return nil, nil, err + } + + env, err := impl.envRepository.FindById(environmentId) + if err != nil { + impl.logger.Errorw("could not fetch stage env", "err", err) + return nil, nil, err + } + + pipeline, err := impl.pipelineRepository.FindById(cdWorkflow.CdWorkflow.PipelineId) + if err != nil { + impl.logger.Errorw("error while fetching cd pipeline", "err", err) + return nil, nil, err + } + var clusterBean bean3.ClusterBean + if env != nil && env.Cluster != nil { + clusterBean = adapter.GetClusterBean(*env.Cluster) + } + clusterConfig := clusterBean.GetClusterConfig() + var isExtCluster bool + if cdWorkflow.WorkflowType == types.PRE { + isExtCluster = pipeline.RunPreStageInEnv + } else if cdWorkflow.WorkflowType == types.POST { + isExtCluster = pipeline.RunPostStageInEnv + } + return impl.getWorkflowLogs(pipelineId, cdWorkflow, clusterConfig, isExtCluster) +} + +func (impl *HandlerServiceImpl) getWorkflowLogs(pipelineId int, cdWorkflow *pipelineConfig.CdWorkflowRunner, clusterConfig *k8s.ClusterConfig, runStageInEnv bool) (*bufio.Reader, func() error, error) { + cdLogRequest := types.BuildLogRequest{ + PodName: cdWorkflow.PodName, + Namespace: cdWorkflow.Namespace, + } + + logStream, cleanUp, err := impl.ciLogService.FetchRunningWorkflowLogs(cdLogRequest, clusterConfig, runStageInEnv) + if logStream == nil || err != nil { + if !cdWorkflow.BlobStorageEnabled { + return nil, nil, errors.New("logs-not-stored-in-repository") + } else if string(v1alpha1.NodeSucceeded) == cdWorkflow.Status || string(v1alpha1.NodeError) == cdWorkflow.Status || string(v1alpha1.NodeFailed) == cdWorkflow.Status || cdWorkflow.Status == cdWorkflow2.WorkflowCancel { + impl.logger.Debugw("pod is not live", "podName", cdWorkflow.PodName, "err", err) + return impl.getLogsFromRepository(pipelineId, cdWorkflow, clusterConfig, runStageInEnv) + } + if err != nil { + impl.logger.Errorw("err on fetch workflow logs", "err", err) + return nil, nil, err + } else if logStream == nil { + return nil, cleanUp, fmt.Errorf("no logs found for pod %s", cdWorkflow.PodName) + } + } + logReader := bufio.NewReader(logStream) + return logReader, cleanUp, err +} + +func (impl *HandlerServiceImpl) getLogsFromRepository(pipelineId int, cdWorkflow *pipelineConfig.CdWorkflowRunner, clusterConfig *k8s.ClusterConfig, isExt bool) (*bufio.Reader, func() error, error) { + impl.logger.Debug("getting historic logs", "pipelineId", pipelineId) + + cdConfigLogsBucket := impl.config.GetDefaultBuildLogsBucket() // TODO -fixme + cdConfigCdCacheRegion := impl.config.GetDefaultCdLogsBucketRegion() + + cdLogRequest := types.BuildLogRequest{ + PipelineId: cdWorkflow.CdWorkflow.PipelineId, + WorkflowId: cdWorkflow.Id, + PodName: cdWorkflow.PodName, + LogsFilePath: cdWorkflow.LogLocation, // impl.ciCdConfig.CiDefaultBuildLogsKeyPrefix + "/" + cdWorkflow.Name + "/main.log", //TODO - fixme + CloudProvider: impl.config.CloudProvider, + AzureBlobConfig: &blob_storage.AzureBlobBaseConfig{ + Enabled: impl.config.CloudProvider == types.BLOB_STORAGE_AZURE, + AccountName: impl.config.AzureAccountName, + BlobContainerName: impl.config.AzureBlobContainerCiLog, + AccountKey: impl.config.AzureAccountKey, + }, + AwsS3BaseConfig: &blob_storage.AwsS3BaseConfig{ + AccessKey: impl.config.BlobStorageS3AccessKey, + Passkey: impl.config.BlobStorageS3SecretKey, + EndpointUrl: impl.config.BlobStorageS3Endpoint, + IsInSecure: impl.config.BlobStorageS3EndpointInsecure, + BucketName: cdConfigLogsBucket, + Region: cdConfigCdCacheRegion, + VersioningEnabled: impl.config.BlobStorageS3BucketVersioned, + }, + GcpBlobBaseConfig: &blob_storage.GcpBlobBaseConfig{ + BucketName: cdConfigLogsBucket, + CredentialFileJsonData: impl.config.BlobStorageGcpCredentialJson, + }, + } + useExternalBlobStorage := pipeline.IsExternalBlobStorageEnabled(isExt, impl.config.UseBlobStorageConfigInCdWorkflow) + if useExternalBlobStorage { + // fetch extClusterBlob cm and cs from k8s client, if they are present then read creds + // from them else return. + cmConfig, secretConfig, err := impl.blobConfigStorageService.FetchCmAndSecretBlobConfigFromExternalCluster(clusterConfig, cdWorkflow.Namespace) + if err != nil { + impl.logger.Errorw("error in fetching config map and secret from external cluster", "err", err, "clusterConfig", clusterConfig) + return nil, nil, err + } + rq := &cdLogRequest + rq.SetBuildLogRequest(cmConfig, secretConfig) + } + + impl.logger.Debugw("s3 log req ", "pipelineId", pipelineId, "runnerId", cdWorkflow.Id) + oldLogsStream, cleanUp, err := impl.ciLogService.FetchLogs(impl.config.BaseLogLocationPath, cdLogRequest) + if err != nil { + impl.logger.Errorw("err", err) + return nil, nil, err + } + logReader := bufio.NewReader(oldLogsStream) + return logReader, cleanUp, err +} diff --git a/pkg/deployment/trigger/devtronApps/PreStageTriggerService.go b/pkg/deployment/trigger/devtronApps/preStageHandlerCode.go similarity index 97% rename from pkg/deployment/trigger/devtronApps/PreStageTriggerService.go rename to pkg/deployment/trigger/devtronApps/preStageHandlerCode.go index 580bbf4da9..ad0051e76d 100644 --- a/pkg/deployment/trigger/devtronApps/PreStageTriggerService.go +++ b/pkg/deployment/trigger/devtronApps/preStageHandlerCode.go @@ -35,15 +35,14 @@ import ( bean7 "github.com/devtron-labs/devtron/pkg/auth/user/bean" bean4 "github.com/devtron-labs/devtron/pkg/bean" "github.com/devtron-labs/devtron/pkg/bean/common" + buildCommonBean "github.com/devtron-labs/devtron/pkg/build/pipeline/bean/common" repository4 "github.com/devtron-labs/devtron/pkg/cluster/environment/repository" bean5 "github.com/devtron-labs/devtron/pkg/deployment/common/bean" adapter2 "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/adapter" "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/bean" "github.com/devtron-labs/devtron/pkg/imageDigestPolicy" - "github.com/devtron-labs/devtron/pkg/pipeline" "github.com/devtron-labs/devtron/pkg/pipeline/adapter" pipelineConfigBean "github.com/devtron-labs/devtron/pkg/pipeline/bean" - "github.com/devtron-labs/devtron/pkg/pipeline/constants" repository3 "github.com/devtron-labs/devtron/pkg/pipeline/history/repository" "github.com/devtron-labs/devtron/pkg/pipeline/types" "github.com/devtron-labs/devtron/pkg/plugin" @@ -62,7 +61,7 @@ import ( "time" ) -func (impl *TriggerServiceImpl) TriggerPreStage(request bean.TriggerRequest) (*bean6.ManifestPushTemplate, error) { +func (impl *HandlerServiceImpl) TriggerPreStage(request bean.TriggerRequest) (*bean6.ManifestPushTemplate, error) { request.WorkflowType = bean2.CD_WORKFLOW_TYPE_PRE // setting triggeredAt variable to have consistent data for various audit log places in db for deployment time triggeredAt := time.Now() @@ -148,7 +147,7 @@ func (impl *TriggerServiceImpl) TriggerPreStage(request bean.TriggerRequest) (*b cdStageWorkflowRequest.Pipeline = pipeline cdStageWorkflowRequest.Env = env cdStageWorkflowRequest.Type = pipelineConfigBean.CD_WORKFLOW_PIPELINE_TYPE - _, jobHelmPackagePath, err := impl.cdWorkflowService.SubmitWorkflow(cdStageWorkflowRequest) + _, jobHelmPackagePath, err := impl.workflowService.SubmitWorkflow(cdStageWorkflowRequest) span.End() if err != nil { runner.Status = cdWorkflow.WorkflowFailed @@ -178,7 +177,7 @@ func (impl *TriggerServiceImpl) TriggerPreStage(request bean.TriggerRequest) (*b return manifestPushTemplate, nil } -func (impl *TriggerServiceImpl) TriggerAutoCDOnPreStageSuccess(triggerContext bean.TriggerContext, cdPipelineId, ciArtifactId, workflowId int) error { +func (impl *HandlerServiceImpl) TriggerAutoCDOnPreStageSuccess(triggerContext bean.TriggerContext, cdPipelineId, ciArtifactId, workflowId int) error { pipeline, err := impl.pipelineRepository.FindById(cdPipelineId) if err != nil { return err @@ -217,7 +216,7 @@ func (impl *TriggerServiceImpl) TriggerAutoCDOnPreStageSuccess(triggerContext be } return nil } -func (impl *TriggerServiceImpl) checkDeploymentTriggeredAlready(wfId int) bool { +func (impl *HandlerServiceImpl) checkDeploymentTriggeredAlready(wfId int) bool { deploymentTriggeredAlready := false // TODO : need to check this logic for status check in case of multiple deployments requirement for same workflow workflowRunner, err := impl.cdWorkflowRepository.FindByWorkflowIdAndRunnerType(context.Background(), wfId, bean2.CD_WORKFLOW_TYPE_DEPLOY) @@ -229,7 +228,7 @@ func (impl *TriggerServiceImpl) checkDeploymentTriggeredAlready(wfId int) bool { return deploymentTriggeredAlready } -func (impl *TriggerServiceImpl) createStartingWfAndRunner(request bean.TriggerRequest, triggeredAt time.Time) (*pipelineConfig.CdWorkflow, *pipelineConfig.CdWorkflowRunner, error) { +func (impl *HandlerServiceImpl) createStartingWfAndRunner(request bean.TriggerRequest, triggeredAt time.Time) (*pipelineConfig.CdWorkflow, *pipelineConfig.CdWorkflowRunner, error) { triggeredBy := request.TriggeredBy artifact := request.Artifact pipeline := request.Pipeline @@ -273,7 +272,7 @@ func (impl *TriggerServiceImpl) createStartingWfAndRunner(request bean.TriggerRe return cdWf, runner, nil } -func (impl *TriggerServiceImpl) getEnvAndNsIfRunStageInEnv(ctx context.Context, request bean.TriggerRequest) (*repository4.Environment, string, error) { +func (impl *HandlerServiceImpl) getEnvAndNsIfRunStageInEnv(ctx context.Context, request bean.TriggerRequest) (*repository4.Environment, string, error) { workflowStage := request.WorkflowType pipeline := request.Pipeline var env *repository4.Environment @@ -298,7 +297,7 @@ func (impl *TriggerServiceImpl) getEnvAndNsIfRunStageInEnv(ctx context.Context, return env, namespace, nil } -func (impl *TriggerServiceImpl) checkVulnerabilityStatusAndFailWfIfNeeded(ctx context.Context, artifact *repository.CiArtifact, +func (impl *HandlerServiceImpl) checkVulnerabilityStatusAndFailWfIfNeeded(ctx context.Context, artifact *repository.CiArtifact, cdPipeline *pipelineConfig.Pipeline, runner *pipelineConfig.CdWorkflowRunner, triggeredBy int32) error { //checking vulnerability for the selected image vulnerabilityCheckRequest := adapter2.GetVulnerabilityCheckRequest(cdPipeline, artifact.ImageDigest) @@ -325,9 +324,9 @@ func (impl *TriggerServiceImpl) checkVulnerabilityStatusAndFailWfIfNeeded(ctx co } // setCopyContainerImagePluginDataAndReserveImages sets required fields in cdStageWorkflowRequest and reserve images generated by plugin -func (impl *TriggerServiceImpl) setCopyContainerImagePluginDataAndReserveImages(cdStageWorkflowRequest *types.WorkflowRequest, pipelineId int, pipelineStage string, artifact *repository.CiArtifact) ([]int, error) { +func (impl *HandlerServiceImpl) setCopyContainerImagePluginDataAndReserveImages(cdStageWorkflowRequest *types.WorkflowRequest, pipelineId int, pipelineStage string, artifact *repository.CiArtifact) ([]int, error) { - copyContainerImagePluginDetail, err := impl.globalPluginService.GetRefPluginIdByRefPluginName(pipeline.COPY_CONTAINER_IMAGE) + copyContainerImagePluginDetail, err := impl.globalPluginService.GetRefPluginIdByRefPluginName(buildCommonBean.COPY_CONTAINER_IMAGE) if err != nil && err != pg.ErrNoRows { impl.logger.Errorw("error in getting copyContainerImage plugin id", "err", err) return nil, err @@ -363,7 +362,7 @@ func (impl *TriggerServiceImpl) setCopyContainerImagePluginDataAndReserveImages( impl.logger.Errorw("error in parsing copyContainerImage input variable", "err", err) return nil, err } - if version == pipeline.COPY_CONTAINER_IMAGE_VERSION_V1 { + if version == buildCommonBean.COPY_CONTAINER_IMAGE_VERSION_V1 { // this is needed in ci runner only for v1 cdStageWorkflowRequest.RegistryDestinationImageMap = registryDestinationImageMap } @@ -408,7 +407,7 @@ func (impl *TriggerServiceImpl) setCopyContainerImagePluginDataAndReserveImages( return imagePathReservationIds, nil } -func (impl *TriggerServiceImpl) getDockerTagAndCustomTagIdForPlugin(pipelineStage string, pipelineId int, artifact *repository.CiArtifact) (string, int, error) { +func (impl *HandlerServiceImpl) getDockerTagAndCustomTagIdForPlugin(pipelineStage string, pipelineId int, artifact *repository.CiArtifact) (string, int, error) { var pipelineStageEntityType int if pipelineStage == types.PRE { pipelineStageEntityType = pipelineConfigBean.EntityTypePreCD @@ -443,7 +442,7 @@ func (impl *TriggerServiceImpl) getDockerTagAndCustomTagIdForPlugin(pipelineStag return DockerImageTag, customTagId, nil } -func (impl *TriggerServiceImpl) buildWFRequest(runner *pipelineConfig.CdWorkflowRunner, cdWf *pipelineConfig.CdWorkflow, cdPipeline *pipelineConfig.Pipeline, envDeploymentConfig *bean5.DeploymentConfig, triggeredBy int32) (*types.WorkflowRequest, error) { +func (impl *HandlerServiceImpl) buildWFRequest(runner *pipelineConfig.CdWorkflowRunner, cdWf *pipelineConfig.CdWorkflow, cdPipeline *pipelineConfig.Pipeline, envDeploymentConfig *bean5.DeploymentConfig, triggeredBy int32) (*types.WorkflowRequest, error) { if cdPipeline.App.Id == 0 { appModel, err := impl.appRepository.FindById(cdPipeline.AppId) if err != nil { @@ -532,7 +531,7 @@ func (impl *TriggerServiceImpl) buildWFRequest(runner *pipelineConfig.CdWorkflow return nil, err } ciProjectDetail.CommitTime = commitTime.Format(bean4.LayoutRFC3339) - } else if ciPipeline.PipelineType == string(constants.CI_JOB) { + } else if ciPipeline.PipelineType == string(buildCommonBean.CI_JOB) { // This has been done to resolve unmarshalling issue in ci-runner, in case of no commit time(eg- polling container images) ciProjectDetail.CommitTime = time.Time{}.Format(bean4.LayoutRFC3339) } else { @@ -911,7 +910,7 @@ getBuildRegistryConfigForArtifact performs the following logic to get Pre/Post C If the ci_pipeline_type type is CI_JOB We will always fetch the registry credentials from the ci_template_override table */ -func (impl *TriggerServiceImpl) getBuildRegistryConfigForArtifact(sourceCiPipeline *pipelineConfig.CiPipeline, artifact *repository.CiArtifact, appId int) (*types.DockerArtifactStoreBean, error) { +func (impl *HandlerServiceImpl) getBuildRegistryConfigForArtifact(sourceCiPipeline *pipelineConfig.CiPipeline, artifact *repository.CiArtifact, appId int) (*types.DockerArtifactStoreBean, error) { var buildRegistryConfig *types.DockerArtifactStoreBean var err error @@ -971,7 +970,7 @@ func (impl *TriggerServiceImpl) getBuildRegistryConfigForArtifact(sourceCiPipeli return buildRegistryConfig, nil } -func (impl *TriggerServiceImpl) ReserveImagesGeneratedAtPlugin(customTagId int, destinationImages []string) ([]int, error) { +func (impl *HandlerServiceImpl) ReserveImagesGeneratedAtPlugin(customTagId int, destinationImages []string) ([]int, error) { var imagePathReservationIds []int for _, image := range destinationImages { @@ -1005,7 +1004,7 @@ func setExtraEnvVariableInDeployStep(deploySteps []*pipelineConfigBean.StepObjec } } -func (impl *TriggerServiceImpl) getDeployStageDetails(pipelineId int) (pipelineConfig.CdWorkflowRunner, string, int, error) { +func (impl *HandlerServiceImpl) getDeployStageDetails(pipelineId int) (pipelineConfig.CdWorkflowRunner, string, int, error) { deployStageWfr := pipelineConfig.CdWorkflowRunner{} //getting deployment pipeline latest wfr by pipelineId deployStageWfr, err := impl.cdWorkflowRepository.FindLatestByPipelineIdAndRunnerType(pipelineId, bean2.CD_WORKFLOW_TYPE_DEPLOY) @@ -1026,7 +1025,7 @@ func (impl *TriggerServiceImpl) getDeployStageDetails(pipelineId int) (pipelineC return deployStageWfr, deployStageTriggeredByUserEmail, pipelineReleaseCounter, nil } -func (impl *TriggerServiceImpl) buildArtifactLocationForS3(cdWf *pipelineConfig.CdWorkflow, runner *pipelineConfig.CdWorkflowRunner) (string, string, string) { +func (impl *HandlerServiceImpl) buildArtifactLocationForS3(cdWf *pipelineConfig.CdWorkflow, runner *pipelineConfig.CdWorkflowRunner) (string, string, string) { cdArtifactLocationFormat := impl.config.GetArtifactLocationFormat() cdWorkflowConfigLogsBucket := impl.config.GetDefaultBuildLogsBucket() ArtifactLocation := fmt.Sprintf("s3://"+path.Join(cdWorkflowConfigLogsBucket, cdArtifactLocationFormat), cdWf.Id, runner.Id) @@ -1034,7 +1033,7 @@ func (impl *TriggerServiceImpl) buildArtifactLocationForS3(cdWf *pipelineConfig. return ArtifactLocation, cdWorkflowConfigLogsBucket, artifactFileName } -func (impl *TriggerServiceImpl) buildDefaultArtifactLocation(savedWf *pipelineConfig.CdWorkflow, runner *pipelineConfig.CdWorkflowRunner) string { +func (impl *HandlerServiceImpl) buildDefaultArtifactLocation(savedWf *pipelineConfig.CdWorkflow, runner *pipelineConfig.CdWorkflowRunner) string { cdArtifactLocationFormat := impl.config.GetArtifactLocationFormat() ArtifactLocation := fmt.Sprintf(cdArtifactLocationFormat, savedWf.Id, runner.Id) return ArtifactLocation @@ -1046,7 +1045,7 @@ func ReplaceImageTagWithDigest(image, digest string) string { return imageWithDigest } -func (impl *TriggerServiceImpl) sendPreStageNotification(ctx context.Context, cdWf *pipelineConfig.CdWorkflow, pipeline *pipelineConfig.Pipeline) error { +func (impl *HandlerServiceImpl) sendPreStageNotification(ctx context.Context, cdWf *pipelineConfig.CdWorkflow, pipeline *pipelineConfig.Pipeline) error { wfr, err := impl.cdWorkflowRepository.FindByWorkflowIdAndRunnerType(ctx, cdWf.Id, bean2.CD_WORKFLOW_TYPE_PRE) if err != nil { return err diff --git a/pkg/deployment/trigger/devtronApps/PreStageTriggerService_ent.go b/pkg/deployment/trigger/devtronApps/preStageHandlerCode_ent.go similarity index 82% rename from pkg/deployment/trigger/devtronApps/PreStageTriggerService_ent.go rename to pkg/deployment/trigger/devtronApps/preStageHandlerCode_ent.go index 8f5b3a5c39..d285ba8224 100644 --- a/pkg/deployment/trigger/devtronApps/PreStageTriggerService_ent.go +++ b/pkg/deployment/trigger/devtronApps/preStageHandlerCode_ent.go @@ -14,33 +14,33 @@ import ( "time" ) -func (impl *TriggerServiceImpl) checkFeasibilityForPreStage(pipeline *pipelineConfig.Pipeline, request *bean2.TriggerRequest, +func (impl *HandlerServiceImpl) checkFeasibilityForPreStage(pipeline *pipelineConfig.Pipeline, request *bean2.TriggerRequest, env *repository.Environment, artifact *repository2.CiArtifact, triggeredBy int32) (interface{}, error) { //here return type is interface as ResourceFilterEvaluationAudit is not present in this version return nil, nil } -func (impl *TriggerServiceImpl) createAuditDataForDeploymentWindowBypass(request bean2.TriggerRequest, wfrId int) error { +func (impl *HandlerServiceImpl) createAuditDataForDeploymentWindowBypass(request bean2.TriggerRequest, wfrId int) error { return nil } -func (impl *TriggerServiceImpl) getManifestPushTemplateForPreStage(ctx context.Context, envDeploymentConfig *bean3.DeploymentConfig, +func (impl *HandlerServiceImpl) getManifestPushTemplateForPreStage(ctx context.Context, envDeploymentConfig *bean3.DeploymentConfig, pipeline *pipelineConfig.Pipeline, artifact *repository2.CiArtifact, jobHelmPackagePath string, cdWf *pipelineConfig.CdWorkflow, runner *pipelineConfig.CdWorkflowRunner, triggeredBy int32, triggeredAt time.Time, request bean2.TriggerRequest) (*bean6.ManifestPushTemplate, error) { return nil, nil } -func (impl *TriggerServiceImpl) setCloningModeInCIProjectDetail(ciProjectDetail *bean.CiProjectDetails, appId int, +func (impl *HandlerServiceImpl) setCloningModeInCIProjectDetail(ciProjectDetail *bean.CiProjectDetails, appId int, m *pipelineConfig.CiPipelineMaterial) error { return nil } -func (impl *TriggerServiceImpl) getPreStageBuildRegistryConfigIfSourcePipelineNotPresent(appId int) (*types.DockerArtifactStoreBean, error) { +func (impl *HandlerServiceImpl) getPreStageBuildRegistryConfigIfSourcePipelineNotPresent(appId int) (*types.DockerArtifactStoreBean, error) { return nil, fmt.Errorf("soucePipeline is mandatory, corrupt data") } -func (impl *TriggerServiceImpl) handlerFilterEvaluationAudit(filterEvaluationAudit interface{}, +func (impl *HandlerServiceImpl) handlerFilterEvaluationAudit(filterEvaluationAudit interface{}, runner *pipelineConfig.CdWorkflowRunner) error { //here ip type of filterEvaluationAudit is interface as ResourceFilterEvaluationAudit is not present in this version return nil diff --git a/pkg/deployment/trigger/devtronApps/wire_devtronAppsDeployTrigger.go b/pkg/deployment/trigger/devtronApps/wire_devtronAppsDeployTrigger.go index 4bf4174cfe..953b46422f 100644 --- a/pkg/deployment/trigger/devtronApps/wire_devtronAppsDeployTrigger.go +++ b/pkg/deployment/trigger/devtronApps/wire_devtronAppsDeployTrigger.go @@ -23,6 +23,6 @@ import ( var DevtronAppsDeployTriggerWireSet = wire.NewSet( userDeploymentRequest.WireSet, - NewTriggerServiceImpl, - wire.Bind(new(TriggerService), new(*TriggerServiceImpl)), + NewHandlerServiceImpl, + wire.Bind(new(HandlerService), new(*HandlerServiceImpl)), ) diff --git a/pkg/eventProcessor/bean/workflowEventBean.go b/pkg/eventProcessor/bean/workflowEventBean.go index 4b33b8cba4..ad150af539 100644 --- a/pkg/eventProcessor/bean/workflowEventBean.go +++ b/pkg/eventProcessor/bean/workflowEventBean.go @@ -19,6 +19,7 @@ package bean import ( "context" "encoding/json" + "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" "github.com/devtron-labs/common-lib/utils/registry" "github.com/devtron-labs/devtron/api/bean" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" @@ -114,3 +115,14 @@ type DevtronAppReleaseContextType struct { CancelContext context.CancelCauseFunc RunnerId int } + +type CiCdStatus struct { + DevtronOwnerInstance string `json:"devtronOwnerInstance"` + *v1alpha1.WorkflowStatus +} + +func NewCiCdStatus() CiCdStatus { + return CiCdStatus{ + WorkflowStatus: &v1alpha1.WorkflowStatus{}, + } +} diff --git a/pkg/eventProcessor/in/CDPipelineEventProcessorService.go b/pkg/eventProcessor/in/CDPipelineEventProcessorService.go index 5a54759107..cb18e00066 100644 --- a/pkg/eventProcessor/in/CDPipelineEventProcessorService.go +++ b/pkg/eventProcessor/in/CDPipelineEventProcessorService.go @@ -38,7 +38,7 @@ type CDPipelineEventProcessorImpl struct { pubSubClient *pubsub.PubSubClientServiceImpl cdWorkflowCommonService cd.CdWorkflowCommonService workflowStatusService status.WorkflowStatusService - cdTriggerService devtronApps.TriggerService + cdHandlerService devtronApps.HandlerService pipelineRepository pipelineConfig.PipelineRepository installedAppReadService installedAppReader.InstalledAppReadService } @@ -47,7 +47,7 @@ func NewCDPipelineEventProcessorImpl(logger *zap.SugaredLogger, pubSubClient *pubsub.PubSubClientServiceImpl, cdWorkflowCommonService cd.CdWorkflowCommonService, workflowStatusService status.WorkflowStatusService, - cdTriggerService devtronApps.TriggerService, + cdHandlerService devtronApps.HandlerService, pipelineRepository pipelineConfig.PipelineRepository, installedAppReadService installedAppReader.InstalledAppReadService) *CDPipelineEventProcessorImpl { cdPipelineEventProcessorImpl := &CDPipelineEventProcessorImpl{ @@ -55,7 +55,7 @@ func NewCDPipelineEventProcessorImpl(logger *zap.SugaredLogger, pubSubClient: pubSubClient, cdWorkflowCommonService: cdWorkflowCommonService, workflowStatusService: workflowStatusService, - cdTriggerService: cdTriggerService, + cdHandlerService: cdHandlerService, pipelineRepository: pipelineRepository, installedAppReadService: installedAppReadService, } @@ -78,7 +78,7 @@ func (impl *CDPipelineEventProcessorImpl) SubscribeCDBulkTriggerTopic() error { ReferenceId: pointer.String(msg.MsgId), Context: context2.Background(), } - _, _, _, err = impl.cdTriggerService.ManualCdTrigger(triggerContext, event.ValuesOverrideRequest) + _, _, _, err = impl.cdHandlerService.ManualCdTrigger(triggerContext, event.ValuesOverrideRequest) if err != nil { impl.logger.Errorw("Error triggering CD", "topic", pubsub.CD_BULK_DEPLOY_TRIGGER_TOPIC, "msg", msg.Data, "err", err) } diff --git a/pkg/eventProcessor/in/WorkflowEventProcessorService.go b/pkg/eventProcessor/in/WorkflowEventProcessorService.go index 2800e9a3bb..cb34134dd1 100644 --- a/pkg/eventProcessor/in/WorkflowEventProcessorService.go +++ b/pkg/eventProcessor/in/WorkflowEventProcessorService.go @@ -27,6 +27,7 @@ import ( "github.com/devtron-labs/common-lib/utils/registry" apiBean "github.com/devtron-labs/devtron/api/bean" client "github.com/devtron-labs/devtron/client/events" + "github.com/devtron-labs/devtron/internal/middleware" "github.com/devtron-labs/devtron/internal/sql/constants" "github.com/devtron-labs/devtron/internal/sql/models" "github.com/devtron-labs/devtron/internal/sql/repository" @@ -35,6 +36,7 @@ import ( util3 "github.com/devtron-labs/devtron/internal/util" "github.com/devtron-labs/devtron/pkg/app" userBean "github.com/devtron-labs/devtron/pkg/auth/user/bean" + "github.com/devtron-labs/devtron/pkg/build/trigger" "github.com/devtron-labs/devtron/pkg/deployment/common" "github.com/devtron-labs/devtron/pkg/deployment/deployedApp" deploymentBean "github.com/devtron-labs/devtron/pkg/deployment/deployedApp/bean" @@ -46,6 +48,7 @@ import ( eventProcessorBean "github.com/devtron-labs/devtron/pkg/eventProcessor/out/bean" "github.com/devtron-labs/devtron/pkg/pipeline" "github.com/devtron-labs/devtron/pkg/pipeline/executors" + "github.com/devtron-labs/devtron/pkg/ucid" "github.com/devtron-labs/devtron/pkg/workflow/cd" "github.com/devtron-labs/devtron/pkg/workflow/cd/adapter" cdWorkflowBean "github.com/devtron-labs/devtron/pkg/workflow/cd/bean" @@ -78,7 +81,7 @@ type WorkflowEventProcessorImpl struct { cdHandler pipeline.CdHandler eventFactory client.EventFactory eventClient client.EventClient - cdTriggerService devtronApps.TriggerService + cdHandlerService devtronApps.HandlerService deployedAppService deployedApp.DeployedAppService webhookService pipeline.WebhookService validator *validator.Validate @@ -86,11 +89,15 @@ type WorkflowEventProcessorImpl struct { cdWorkflowCommonService cd.CdWorkflowCommonService cdPipelineConfigService pipeline.CdPipelineConfigService userDeploymentRequestService service.UserDeploymentRequestService + ucid ucid.Service devtronAppReleaseContextMap map[int]bean.DevtronAppReleaseContextType devtronAppReleaseContextMapLock *sync.Mutex appServiceConfig *app.AppServiceConfig + //ent only + ciHandlerService trigger.HandlerService + // repositories import to be removed pipelineRepository pipelineConfig.PipelineRepository ciArtifactRepository repository.CiArtifactRepository @@ -107,7 +114,7 @@ func NewWorkflowEventProcessorImpl(logger *zap.SugaredLogger, workflowDagExecutor dag.WorkflowDagExecutor, ciHandler pipeline.CiHandler, cdHandler pipeline.CdHandler, eventFactory client.EventFactory, eventClient client.EventClient, - cdTriggerService devtronApps.TriggerService, + cdHandlerService devtronApps.HandlerService, deployedAppService deployedApp.DeployedAppService, webhookService pipeline.WebhookService, validator *validator.Validate, @@ -115,10 +122,12 @@ func NewWorkflowEventProcessorImpl(logger *zap.SugaredLogger, cdWorkflowCommonService cd.CdWorkflowCommonService, cdPipelineConfigService pipeline.CdPipelineConfigService, userDeploymentRequestService service.UserDeploymentRequestService, + ucid ucid.Service, pipelineRepository pipelineConfig.PipelineRepository, ciArtifactRepository repository.CiArtifactRepository, cdWorkflowRepository pipelineConfig.CdWorkflowRepository, - deploymentConfigService common.DeploymentConfigService) (*WorkflowEventProcessorImpl, error) { + deploymentConfigService common.DeploymentConfigService, + ciHandlerService trigger.HandlerService) (*WorkflowEventProcessorImpl, error) { impl := &WorkflowEventProcessorImpl{ logger: logger, pubSubClient: pubSubClient, @@ -131,7 +140,7 @@ func NewWorkflowEventProcessorImpl(logger *zap.SugaredLogger, eventFactory: eventFactory, eventClient: eventClient, workflowDagExecutor: workflowDagExecutor, - cdTriggerService: cdTriggerService, + cdHandlerService: cdHandlerService, deployedAppService: deployedAppService, webhookService: webhookService, validator: validator, @@ -139,12 +148,14 @@ func NewWorkflowEventProcessorImpl(logger *zap.SugaredLogger, cdWorkflowCommonService: cdWorkflowCommonService, cdPipelineConfigService: cdPipelineConfigService, userDeploymentRequestService: userDeploymentRequestService, + ucid: ucid, devtronAppReleaseContextMap: make(map[int]bean.DevtronAppReleaseContextType), devtronAppReleaseContextMapLock: &sync.Mutex{}, pipelineRepository: pipelineRepository, ciArtifactRepository: ciArtifactRepository, cdWorkflowRepository: cdWorkflowRepository, deploymentConfigService: deploymentConfigService, + ciHandlerService: ciHandlerService, } appServiceConfig, err := app.GetAppServiceConfig() if err != nil { @@ -320,7 +331,7 @@ func (impl *WorkflowEventProcessorImpl) SubscribeTriggerBulkAction() error { ApplyAuth: false, TriggerContext: triggerContext, } - err = impl.cdTriggerService.TriggerStageForBulk(triggerRequest) + err = impl.cdHandlerService.TriggerStageForBulk(triggerRequest) if err != nil { impl.logger.Errorw("error in cd trigger ", "err", err) wf.WorkflowStatus = cdWorkflowModelBean.TRIGGER_ERROR @@ -387,30 +398,45 @@ func (impl *WorkflowEventProcessorImpl) SubscribeHibernateBulkAction() error { func (impl *WorkflowEventProcessorImpl) SubscribeCIWorkflowStatusUpdate() error { callback := func(msg *model.PubSubMsg) { - wfStatus := v1alpha1.WorkflowStatus{} + wfStatus := bean.NewCiCdStatus() err := json.Unmarshal([]byte(msg.Data), &wfStatus) if err != nil { impl.logger.Errorw("error while unmarshalling wf status update", "err", err, "msg", msg.Data) return } - - err = impl.ciHandler.CheckAndReTriggerCI(wfStatus) - if err != nil { - impl.logger.Errorw("error in checking and re triggering ci", "err", err) - //don't return as we have to update the workflow status + if len(wfStatus.DevtronOwnerInstance) != 0 { + devtronUCID, _, err := impl.ucid.GetUCIDWithOutCache() + if err != nil { + impl.logger.Errorw("error in getting UCID", "err", err) + return + } + if wfStatus.DevtronOwnerInstance != devtronUCID { + impl.logger.Warnw("mis match in UCID. skipping...", "devtronAdministratorInstance", wfStatus.DevtronOwnerInstance, "devtronUCID", devtronUCID) + return + } } - - _, err = impl.ciHandler.UpdateWorkflow(wfStatus) + // update the ci workflow status + ciWfId, stateChanged, err := impl.ciHandler.UpdateWorkflow(wfStatus) if err != nil { - impl.logger.Errorw("error on update workflow status", "err", err, "msg", msg.Data) + impl.logger.Errorw("error on update workflow status", "msg", msg.Data, "err", err) return } - + if stateChanged { + // check if we need to re-trigger the ci + err = impl.ciHandlerService.CheckAndReTriggerCI(wfStatus) + if err != nil { + middleware.ReTriggerFailedCounter.WithLabelValues("CI", strconv.Itoa(ciWfId)).Inc() + impl.logger.Errorw("error in checking and re triggering ci", "wfStatus", wfStatus, "err", err) + return + } + } else { + impl.logger.Debugw("no state change detected for the ci workflow status update, ignoring this event", "workflowRunnerId", ciWfId, "wfStatus", wfStatus) + } } // add required logging here var loggerFunc pubsub.LoggerFunc = func(msg model.PubSubMsg) (string, []interface{}) { - wfStatus := v1alpha1.WorkflowStatus{} + wfStatus := bean.NewCiCdStatus() err := json.Unmarshal([]byte(msg.Data), &wfStatus) if err != nil { return "error while unmarshalling wf status update", []interface{}{"err", err, "msg", msg.Data} @@ -423,7 +449,7 @@ func (impl *WorkflowEventProcessorImpl) SubscribeCIWorkflowStatusUpdate() error err := impl.pubSubClient.Subscribe(pubsub.WORKFLOW_STATUS_UPDATE_TOPIC, callback, loggerFunc) if err != nil { - impl.logger.Error("err", err) + impl.logger.Error("error in subscribing to ci workflow status update topic", "err", err) return err } return nil @@ -431,17 +457,27 @@ func (impl *WorkflowEventProcessorImpl) SubscribeCIWorkflowStatusUpdate() error func (impl *WorkflowEventProcessorImpl) SubscribeCDWorkflowStatusUpdate() error { callback := func(msg *model.PubSubMsg) { - wfStatus := v1alpha1.WorkflowStatus{} + wfStatus := bean.NewCiCdStatus() err := json.Unmarshal([]byte(msg.Data), &wfStatus) if err != nil { impl.logger.Error("Error while unmarshalling wfStatus json object", "error", err) return } - - wfrId, wfrStatus, stateChanged, err := impl.cdHandler.UpdateWorkflow(wfStatus) - impl.logger.Debugw("UpdateWorkflow", "wfrId", wfrId, "wfrStatus", wfrStatus) + if len(wfStatus.DevtronOwnerInstance) != 0 { + devtronUCID, _, err := impl.ucid.GetUCIDWithOutCache() + if err != nil { + impl.logger.Errorw("error in getting UCID", "err", err) + return + } + if wfStatus.DevtronOwnerInstance != devtronUCID { + impl.logger.Warnw("mis match in UCID. skipping...", "devtronAdministratorInstance", wfStatus.DevtronOwnerInstance, "devtronUCID", devtronUCID) + return + } + } + wfrId, status, stateChanged, err := impl.cdHandler.UpdateWorkflow(wfStatus) + impl.logger.Debugw("cd UpdateWorkflow for wfStatus", "wfrId", wfrId, "status", status, "wfStatus", wfStatus) if err != nil { - impl.logger.Error("err", err) + impl.logger.Errorw("error in cd workflow status update", "wfrId", wfrId, "status", status, "wfStatus", wfStatus, "err", err) return } @@ -452,11 +488,12 @@ func (impl *WorkflowEventProcessorImpl) SubscribeCDWorkflowStatusUpdate() error return } - if wfrStatus == string(v1alpha1.NodeFailed) || wfrStatus == string(v1alpha1.NodeError) { + if wfr.Status == string(v1alpha1.NodeFailed) || wfr.Status == string(v1alpha1.NodeError) { if len(wfr.ImagePathReservationIds) > 0 { err := impl.cdHandler.DeactivateImageReservationPathsOnFailure(wfr.ImagePathReservationIds) if err != nil { impl.logger.Errorw("error in removing image path reservation ", "imagePathReservationIds", wfr.ImagePathReservationIds, "err", err) + // not returning here as we need to send the notification event and re-trigger the cd stage (if required) } } } @@ -464,13 +501,16 @@ func (impl *WorkflowEventProcessorImpl) SubscribeCDWorkflowStatusUpdate() error wfStatusInEvent := string(wfStatus.Phase) if wfStatusInEvent == string(v1alpha1.NodeSucceeded) || wfStatusInEvent == string(v1alpha1.NodeFailed) || wfStatusInEvent == string(v1alpha1.NodeError) { // the re-trigger should only happen when we get a pod deleted event. - if wfr != nil && executors.CheckIfReTriggerRequired(wfrStatus, wfStatus.Message, wfr.Status) { + if executors.CheckIfReTriggerRequired(status, wfStatus.Message, wfr.Status) { err = impl.workflowDagExecutor.HandleCdStageReTrigger(wfr) if err != nil { // check if this log required or not - impl.logger.Errorw("error in HandleCdStageReTrigger", "workflowRunnerId", wfr.Id, "workflowStatus", wfrStatus, "workflowStatusMessage", wfStatus.Message, "error", err) + workflowType := fmt.Sprintf("%s-CD", wfr.WorkflowType) + middleware.ReTriggerFailedCounter.WithLabelValues(workflowType, strconv.Itoa(wfrId)).Inc() + impl.logger.Errorw("error in HandleCdStageReTrigger", "workflowRunnerId", wfr.Id, "status", status, "message", wfStatus.Message, "error", err) + return } - impl.logger.Debugw("re-triggered cd stage", "workflowRunnerId", wfr.Id, "workflowStatus", wfrStatus, "workflowStatusMessage", wfStatus.Message) + impl.logger.Infow("re-triggered cd stage", "workflowRunnerId", wfr.Id, "status", status, "message", wfStatus.Message) } else { // we send this notification on *workflow_runner* status, both success and failure // during workflow runner failure, particularly when failure occurred due to pod deletion , we get two events from kubewatch. @@ -486,13 +526,13 @@ func (impl *WorkflowEventProcessorImpl) SubscribeCDWorkflowStatusUpdate() error } } } else { - impl.logger.Debugw("no state change detected for the cd workflow status update, ignoring this event", "workflowRunnerId", wfrId, "wfrStatus", wfrStatus) + impl.logger.Debugw("no state change detected for the cd workflow status update, ignoring this event", "workflowRunnerId", wfrId, "status", status) } } // add required logging here var loggerFunc pubsub.LoggerFunc = func(msg model.PubSubMsg) (string, []interface{}) { - wfStatus := v1alpha1.WorkflowStatus{} + wfStatus := bean.NewCiCdStatus() err := json.Unmarshal([]byte(msg.Data), &wfStatus) if err != nil { return "error while unmarshalling wfStatus json object", []interface{}{"error", err} @@ -503,7 +543,7 @@ func (impl *WorkflowEventProcessorImpl) SubscribeCDWorkflowStatusUpdate() error err := impl.pubSubClient.Subscribe(pubsub.CD_WORKFLOW_STATUS_UPDATE, callback, loggerFunc) if err != nil { - impl.logger.Error("err", err) + impl.logger.Error("error in subscribing to cd workflow status update topic", "err", err) return err } return nil diff --git a/pkg/pipeline/WorkflowService.go b/pkg/executor/WorkflowService.go similarity index 93% rename from pkg/pipeline/WorkflowService.go rename to pkg/executor/WorkflowService.go index 3cc2215867..14ff02b5b8 100644 --- a/pkg/pipeline/WorkflowService.go +++ b/pkg/executor/WorkflowService.go @@ -14,7 +14,7 @@ * limitations under the License. */ -package pipeline +package executor import ( "context" @@ -28,16 +28,18 @@ import ( "github.com/devtron-labs/common-lib/utils/k8s/commonBean" "github.com/devtron-labs/devtron/api/bean" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow/cdWorkflow" - bean2 "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" + "github.com/devtron-labs/devtron/pkg/build/pipeline/bean/common" repository2 "github.com/devtron-labs/devtron/pkg/cluster/environment/repository" "github.com/devtron-labs/devtron/pkg/config/read" v1 "github.com/devtron-labs/devtron/pkg/infraConfig/bean/v1" k8s2 "github.com/devtron-labs/devtron/pkg/k8s" + "github.com/devtron-labs/devtron/pkg/pipeline" bean3 "github.com/devtron-labs/devtron/pkg/pipeline/bean" "github.com/devtron-labs/devtron/pkg/pipeline/executors" "github.com/devtron-labs/devtron/pkg/pipeline/infraProviders" "github.com/devtron-labs/devtron/pkg/pipeline/infraProviders/infraGetters" "github.com/devtron-labs/devtron/pkg/pipeline/types" + "github.com/devtron-labs/devtron/pkg/ucid" "go.uber.org/zap" v12 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -66,12 +68,13 @@ type WorkflowServiceImpl struct { ciCdConfig *types.CiCdConfig configMapService read.ConfigReadService envRepository repository2.EnvironmentRepository - globalCMCSService GlobalCMCSService + globalCMCSService pipeline.GlobalCMCSService argoWorkflowExecutor executors.ArgoWorkflowExecutor systemWorkflowExecutor executors.SystemWorkflowExecutor - k8sUtil *k8s.K8sServiceImpl k8sCommonService k8s2.K8sCommonService infraProvider infraProviders.InfraProvider + ucid ucid.Service + k8sUtil *k8s.K8sServiceImpl } // TODO: Move to bean @@ -80,12 +83,14 @@ func NewWorkflowServiceImpl(Logger *zap.SugaredLogger, envRepository repository2.EnvironmentRepository, ciCdConfig *types.CiCdConfig, configMapService read.ConfigReadService, - globalCMCSService GlobalCMCSService, + globalCMCSService pipeline.GlobalCMCSService, argoWorkflowExecutor executors.ArgoWorkflowExecutor, - k8sUtil *k8s.K8sServiceImpl, systemWorkflowExecutor executors.SystemWorkflowExecutor, k8sCommonService k8s2.K8sCommonService, - infraProvider infraProviders.InfraProvider) (*WorkflowServiceImpl, error) { + infraProvider infraProviders.InfraProvider, + ucid ucid.Service, + k8sUtil *k8s.K8sServiceImpl, +) (*WorkflowServiceImpl, error) { commonWorkflowService := &WorkflowServiceImpl{ Logger: Logger, ciCdConfig: ciCdConfig, @@ -97,6 +102,7 @@ func NewWorkflowServiceImpl(Logger *zap.SugaredLogger, systemWorkflowExecutor: systemWorkflowExecutor, k8sCommonService: k8sCommonService, infraProvider: infraProvider, + ucid: ucid, } restConfig, err := k8sUtil.GetK8sInClusterRestConfig() if err != nil { @@ -109,9 +115,6 @@ func NewWorkflowServiceImpl(Logger *zap.SugaredLogger, const ( CI_NODE_SELECTOR_APP_LABEL_KEY = "devtron.ai/node-selector" - - preCdStage = "preCD" - postCdStage = "postCD" ) func (impl *WorkflowServiceImpl) SubmitWorkflow(workflowRequest *types.WorkflowRequest) (*unstructured.UnstructuredList, string, error) { @@ -119,13 +122,16 @@ func (impl *WorkflowServiceImpl) SubmitWorkflow(workflowRequest *types.WorkflowR if err != nil { return nil, "", err } - workflowExecutor := impl.getWorkflowExecutor(workflowRequest.WorkflowExecutor) - if workflowExecutor == nil { - return nil, "", errors.New("workflow executor not found") + var createdWf *unstructured.UnstructuredList + canExecuteWorkflow, jobHelmChartPath, err := impl.checkIfCanExecuteWorkflowAndHandleVirtualExec(workflowRequest, workflowTemplate) + if canExecuteWorkflow { + workflowExecutor := impl.getWorkflowExecutor(workflowRequest.WorkflowExecutor) + if workflowExecutor == nil { + return nil, "", errors.New("workflow executor not found") + } + createdWf, err = workflowExecutor.ExecuteWorkflow(workflowTemplate) } - createdWf, err := workflowExecutor.ExecuteWorkflow(workflowTemplate) - jobHelmPackagePath := "" // due to ENT - return createdWf, jobHelmPackagePath, err + return createdWf, jobHelmChartPath, err } func (impl *WorkflowServiceImpl) createWorkflowTemplate(workflowRequest *types.WorkflowRequest) (bean3.WorkflowTemplate, error) { @@ -141,6 +147,11 @@ func (impl *WorkflowServiceImpl) createWorkflowTemplate(workflowRequest *types.W return bean3.WorkflowTemplate{}, err } + workflowTemplate, err = impl.updateWorkflowTemplateWithLabels(workflowRequest, workflowTemplate) + if err != nil { + impl.Logger.Errorw("error occurred while updating workflow template with labels", "err", err) + return bean3.WorkflowTemplate{}, err + } workflowRequest.AddNodeConstraintsFromConfig(&workflowTemplate, impl.ciCdConfig) infraConfiguration := &v1.InfraConfig{} shouldAddExistingCmCsInWorkflow := impl.shouldAddExistingCmCsInWorkflow(workflowRequest) @@ -213,12 +224,18 @@ func (impl *WorkflowServiceImpl) createWorkflowTemplate(workflowRequest *types.W clusterConfig, err := impl.getClusterConfig(workflowRequest) workflowTemplate.ClusterConfig = clusterConfig workflowTemplate.WorkflowType = workflowRequest.GetWorkflowTypeForWorkflowRequest() + devtronUCID, _, err := impl.ucid.GetUCIDWithOutCache() + if err != nil { + impl.Logger.Errorw("error in getting UCID", "err", err) + return bean3.WorkflowTemplate{}, err + } + workflowTemplate.DevtronInstanceUID = devtronUCID return workflowTemplate, nil } func (impl *WorkflowServiceImpl) shouldAddExistingCmCsInWorkflow(workflowRequest *types.WorkflowRequest) bool { // CmCs are not added for CI_JOB if IgnoreCmCsInCiJob is true - if workflowRequest.CiPipelineType == string(bean2.CI_JOB) && impl.ciCdConfig.IgnoreCmCsInCiJob { + if workflowRequest.CiPipelineType == string(common.CI_JOB) && impl.ciCdConfig.IgnoreCmCsInCiJob { return false } return true @@ -378,6 +395,7 @@ func (impl *WorkflowServiceImpl) getWorkflowExecutor(executorType cdWorkflow.Wor impl.Logger.Warnw("workflow executor not found", "type", executorType) return nil } + func (impl *WorkflowServiceImpl) GetWorkflow(executorType cdWorkflow.WorkflowExecutorType, name string, namespace string, restConfig *rest.Config) (*unstructured.UnstructuredList, error) { impl.Logger.Debug("getting wf", name) workflowExecutor := impl.getWorkflowExecutor(executorType) diff --git a/pkg/pipeline/WorkflowServiceIT_test.go b/pkg/executor/WorkflowServiceIT_test.go similarity index 99% rename from pkg/pipeline/WorkflowServiceIT_test.go rename to pkg/executor/WorkflowServiceIT_test.go index c025abe1fe..b1d569487f 100644 --- a/pkg/pipeline/WorkflowServiceIT_test.go +++ b/pkg/executor/WorkflowServiceIT_test.go @@ -14,7 +14,7 @@ * limitations under the License. */ -package pipeline +package executor import ( "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" @@ -35,6 +35,7 @@ import ( "github.com/devtron-labs/devtron/pkg/commonService" k8s2 "github.com/devtron-labs/devtron/pkg/k8s" "github.com/devtron-labs/devtron/pkg/k8s/informer" + "github.com/devtron-labs/devtron/pkg/pipeline" bean2 "github.com/devtron-labs/devtron/pkg/pipeline/bean" "github.com/devtron-labs/devtron/pkg/pipeline/executors" "github.com/devtron-labs/devtron/pkg/pipeline/types" @@ -59,10 +60,10 @@ var cmManifest3 = "{\"kind\":\"ConfigMap\",\"apiVersion\":\"v1\",\"metadata\":{\ var cmManifest4 = "{\"kind\":\"ConfigMap\",\"apiVersion\":\"v1\",\"metadata\":{\"name\":\"cm4-5-ci\",\"creationTimestamp\":null,\"ownerReferences\":[{\"apiVersion\":\"argoproj.io/v1alpha1\",\"kind\":\"Workflow\",\"name\":\"{{workflow.name}}\",\"uid\":\"{{workflow.uid}}\",\"blockOwnerDeletion\":true}]},\"data\":{\"key4\":\"value4\"}}" func getWorkflowServiceImpl(t *testing.T) *WorkflowServiceImpl { - logger, dbConnection := getDbConnAndLoggerService(t) + logger, dbConnection := pipeline.getDbConnAndLoggerService(t) ciCdConfig, _ := types.GetCiCdConfig() newGlobalCMCSRepositoryImpl := repository.NewGlobalCMCSRepositoryImpl(logger, dbConnection) - globalCMCSServiceImpl := NewGlobalCMCSServiceImpl(logger, newGlobalCMCSRepositoryImpl) + globalCMCSServiceImpl := pipeline.NewGlobalCMCSServiceImpl(logger, newGlobalCMCSRepositoryImpl) newEnvConfigOverrideRepository := chartConfig.NewEnvConfigOverrideRepository(dbConnection) newConfigMapRepositoryImpl := chartConfig.NewConfigMapRepositoryImpl(logger, dbConnection) newChartRepository := chartRepoRepository.NewChartRepository(dbConnection) diff --git a/pkg/executor/WorkflowService_ent.go b/pkg/executor/WorkflowService_ent.go new file mode 100644 index 0000000000..f01a801a90 --- /dev/null +++ b/pkg/executor/WorkflowService_ent.go @@ -0,0 +1,30 @@ +/* + * Copyright (c) 2020-2024. Devtron Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package executor + +import ( + bean3 "github.com/devtron-labs/devtron/pkg/pipeline/bean" + "github.com/devtron-labs/devtron/pkg/pipeline/types" +) + +func (impl *WorkflowServiceImpl) checkIfCanExecuteWorkflowAndHandleVirtualExec(workflowRequest *types.WorkflowRequest, workflowTemplate bean3.WorkflowTemplate) (canExecuteWorkflow bool, jobHelmChartPath string, err error) { + return true, "", nil +} + +func (impl *WorkflowServiceImpl) updateWorkflowTemplateWithLabels(workflowRequest *types.WorkflowRequest, workflowTemplate bean3.WorkflowTemplate) (bean3.WorkflowTemplate, error) { + return workflowTemplate, nil +} diff --git a/pkg/pipeline/WorkflowService_test.go b/pkg/executor/WorkflowService_test.go similarity index 99% rename from pkg/pipeline/WorkflowService_test.go rename to pkg/executor/WorkflowService_test.go index 4b4e937d68..1ebcb9639c 100644 --- a/pkg/pipeline/WorkflowService_test.go +++ b/pkg/executor/WorkflowService_test.go @@ -14,7 +14,7 @@ * limitations under the License. */ -package pipeline +package executor import ( "fmt" diff --git a/pkg/executor/wire_executor.go b/pkg/executor/wire_executor.go new file mode 100644 index 0000000000..a99bf31f61 --- /dev/null +++ b/pkg/executor/wire_executor.go @@ -0,0 +1,8 @@ +package executor + +import "github.com/google/wire" + +var ExecutorWireSet = wire.NewSet( + NewWorkflowServiceImpl, + wire.Bind(new(WorkflowService), new(*WorkflowServiceImpl)), +) diff --git a/pkg/gitops/GitOpsConfigService.go b/pkg/gitops/GitOpsConfigService.go index a0776f75b3..f095136674 100644 --- a/pkg/gitops/GitOpsConfigService.go +++ b/pkg/gitops/GitOpsConfigService.go @@ -251,7 +251,7 @@ func (impl *GitOpsConfigServiceImpl) registerGitOpsClientConfig(ctx context.Cont } } else { - clusterBean, err := impl.clusterReadService.FindOne(bean2.DEFAULT_CLUSTER) + clusterBean, err := impl.clusterReadService.FindOne(bean2.DefaultCluster) if err != nil { return nil, err } @@ -538,7 +538,7 @@ func (impl *GitOpsConfigServiceImpl) patchGitOpsClientConfig(model *repository.G } } else { - clusterBean, err := impl.clusterReadService.FindOne(bean2.DEFAULT_CLUSTER) + clusterBean, err := impl.clusterReadService.FindOne(bean2.DefaultCluster) if err != nil { return err } diff --git a/pkg/pipeline/BlobStorageConfigService.go b/pkg/pipeline/BlobStorageConfigService.go index fd6141c111..53ea2efde6 100644 --- a/pkg/pipeline/BlobStorageConfigService.go +++ b/pkg/pipeline/BlobStorageConfigService.go @@ -91,7 +91,7 @@ func (impl *BlobStorageConfigServiceImpl) FetchCmAndSecretBlobConfigFromExternal return cmConfig, secretConfig, nil } -func updateRequestWithExtClusterCmAndSecret(request *blob_storage.BlobStorageRequest, cmConfig *bean2.CmBlobStorageConfig, secretConfig *bean2.SecretBlobStorageConfig) *blob_storage.BlobStorageRequest { +func UpdateRequestWithExtClusterCmAndSecret(request *blob_storage.BlobStorageRequest, cmConfig *bean2.CmBlobStorageConfig, secretConfig *bean2.SecretBlobStorageConfig) *blob_storage.BlobStorageRequest { request.StorageType = cmConfig.CloudProvider request.AwsS3BaseConfig.AccessKey = cmConfig.S3AccessKey @@ -113,3 +113,8 @@ func updateRequestWithExtClusterCmAndSecret(request *blob_storage.BlobStorageReq return request } + +func IsExternalBlobStorageEnabled(isExternalRun bool, useBlobStorageConfigInCdWorkflow bool) bool { + // TODO impl.config.UseBlobStorageConfigInCdWorkflow fetches the live status, we need to check from db as well, we should put useExternalBlobStorage in db + return isExternalRun && !useBlobStorageConfigInCdWorkflow +} diff --git a/pkg/pipeline/BuildPipelineConfigService.go b/pkg/pipeline/BuildPipelineConfigService.go index c320898bdc..b003cbeedb 100644 --- a/pkg/pipeline/BuildPipelineConfigService.go +++ b/pkg/pipeline/BuildPipelineConfigService.go @@ -36,6 +36,7 @@ import ( repository3 "github.com/devtron-labs/devtron/pkg/build/git/gitMaterial/repository" "github.com/devtron-labs/devtron/pkg/build/pipeline" bean3 "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" + "github.com/devtron-labs/devtron/pkg/build/pipeline/bean/common" "github.com/devtron-labs/devtron/pkg/build/pipeline/read" pipelineConfigBean "github.com/devtron-labs/devtron/pkg/pipeline/bean" "github.com/devtron-labs/devtron/pkg/pipeline/history" @@ -307,11 +308,11 @@ func (impl *CiPipelineConfigServiceImpl) patchCiPipelineUpdateSource(baseCiConfi } // updating PipelineType from db if not present in request if modifiedCiPipeline.PipelineType == "" { - if bean3.PipelineType(pipeline.PipelineType) != "" { - modifiedCiPipeline.PipelineType = bean3.PipelineType(pipeline.PipelineType) + if common.PipelineType(pipeline.PipelineType) != "" { + modifiedCiPipeline.PipelineType = common.PipelineType(pipeline.PipelineType) } else { // updating default pipelineType if not present in request - modifiedCiPipeline.PipelineType = bean3.DefaultPipelineType + modifiedCiPipeline.PipelineType = common.DefaultPipelineType } } @@ -595,9 +596,9 @@ func (impl *CiPipelineConfigServiceImpl) GetCiPipeline(appId int) (ciConfig *bea Script: ciScript.Script, OutputLocation: ciScript.OutputLocation, } - if ciScript.Stage == BEFORE_DOCKER_BUILD { + if ciScript.Stage == common.BEFORE_DOCKER_BUILD { beforeDockerBuildScripts = append(beforeDockerBuildScripts, ciScriptResp) - } else if ciScript.Stage == AFTER_DOCKER_BUILD { + } else if ciScript.Stage == common.AFTER_DOCKER_BUILD { afterDockerBuildScripts = append(afterDockerBuildScripts, ciScriptResp) } } @@ -622,7 +623,7 @@ func (impl *CiPipelineConfigServiceImpl) GetCiPipeline(appId int) (ciConfig *bea AfterDockerBuildScripts: afterDockerBuildScripts, ScanEnabled: pipeline.ScanEnabled, IsDockerConfigOverridden: pipeline.IsDockerConfigOverridden, - PipelineType: bean3.PipelineType(pipeline.PipelineType), + PipelineType: common.PipelineType(pipeline.PipelineType), } ciEnvMapping, err := impl.ciPipelineRepository.FindCiEnvMappingByCiPipelineId(pipeline.Id) if err != nil && err != pg.ErrNoRows { @@ -750,9 +751,9 @@ func (impl *CiPipelineConfigServiceImpl) GetCiPipelineById(pipelineId int) (ciPi Script: ciScript.Script, OutputLocation: ciScript.OutputLocation, } - if ciScript.Stage == BEFORE_DOCKER_BUILD { + if ciScript.Stage == common.BEFORE_DOCKER_BUILD { beforeDockerBuildScripts = append(beforeDockerBuildScripts, ciScriptResp) - } else if ciScript.Stage == AFTER_DOCKER_BUILD { + } else if ciScript.Stage == common.AFTER_DOCKER_BUILD { afterDockerBuildScripts = append(afterDockerBuildScripts, ciScriptResp) } } @@ -778,7 +779,7 @@ func (impl *CiPipelineConfigServiceImpl) GetCiPipelineById(pipelineId int) (ciPi AfterDockerBuildScripts: afterDockerBuildScripts, ScanEnabled: pipeline.ScanEnabled, IsDockerConfigOverridden: pipeline.IsDockerConfigOverridden, - PipelineType: bean3.PipelineType(pipeline.PipelineType), + PipelineType: common.PipelineType(pipeline.PipelineType), } customTag, err := impl.customTagService.GetActiveCustomTagByEntityKeyAndValue(pipelineConfigBean.EntityTypeCiPipelineId, strconv.Itoa(pipeline.Id)) if err != nil && err != pg.ErrNoRows { @@ -902,7 +903,7 @@ func (impl *CiPipelineConfigServiceImpl) GetTriggerViewCiPipeline(appId int) (*b ParentCiPipeline: pipeline.ParentCiPipeline, ScanEnabled: pipeline.ScanEnabled, IsDockerConfigOverridden: pipeline.IsDockerConfigOverridden, - PipelineType: bean3.PipelineType(pipeline.PipelineType), + PipelineType: common.PipelineType(pipeline.PipelineType), } if ciTemplateBean, ok := ciOverrideTemplateMap[pipeline.Id]; ok { templateOverride := ciTemplateBean.CiTemplateOverride @@ -1268,7 +1269,7 @@ func (impl *CiPipelineConfigServiceImpl) UpdateCiTemplate(updateRequest *bean.Ci } for _, ciTemplateOverride := range ciTemplateOverrides { if _, ok := ciPipelineIdsMap[ciTemplateOverride.CiPipelineId]; ok { - if ciPipelineIdsMap[ciTemplateOverride.CiPipelineId].PipelineType == string(bean3.CI_JOB) { + if ciPipelineIdsMap[ciTemplateOverride.CiPipelineId].PipelineType == string(common.CI_JOB) { ciTemplateOverride.DockerRepository = updateRequest.DockerRepository ciTemplateOverride.DockerRegistryId = updateRequest.DockerRegistry _, err = impl.ciTemplateOverrideRepository.Update(ciTemplateOverride) @@ -1324,7 +1325,7 @@ func (impl *CiPipelineConfigServiceImpl) PatchCiPipeline(request *bean.CiPatchRe impl.logger.Errorw("err in fetching template for pipeline patch, ", "err", err, "appId", request.AppId) return nil, err } - if request.CiPipeline.PipelineType == bean3.CI_JOB { + if request.CiPipeline.PipelineType == common.CI_JOB { request.CiPipeline.IsDockerConfigOverridden = true request.CiPipeline.DockerConfigOverride = bean.DockerConfigOverride{ DockerRegistry: ciConfig.DockerRegistry, @@ -1517,15 +1518,15 @@ func (impl *CiPipelineConfigServiceImpl) GetCiPipelineMin(appId int, envIds []in var ciPipelineResp []*bean.CiPipelineMin for _, pipeline := range pipelines { parentCiPipeline := pipelineConfig.CiPipeline{} - pipelineType := bean3.CI_BUILD + pipelineType := common.CI_BUILD if pipelineParentCiMap[pipeline.Id] != nil { parentCiPipeline = *pipelineParentCiMap[pipeline.Id] - pipelineType = bean3.LINKED + pipelineType = common.LINKED } else if pipeline.IsExternal == true { - pipelineType = bean3.EXTERNAL - } else if pipeline.PipelineType == string(bean3.CI_JOB) { - pipelineType = bean3.CI_JOB + pipelineType = common.EXTERNAL + } else if pipeline.PipelineType == string(common.CI_JOB) { + pipelineType = common.CI_JOB } ciPipeline := &bean.CiPipelineMin{ @@ -1741,7 +1742,7 @@ func (impl *CiPipelineConfigServiceImpl) GetCiPipelineByEnvironment(request reso ExternalCiConfig: externalCiConfig, ScanEnabled: pipeline.ScanEnabled, IsDockerConfigOverridden: pipeline.IsDockerConfigOverridden, - PipelineType: bean3.PipelineType(pipeline.PipelineType), + PipelineType: common.PipelineType(pipeline.PipelineType), } parentPipelineAppId, ok := pipelineIdVsAppId[parentCiPipelineId] if ok { diff --git a/pkg/pipeline/BuildPipelineSwitchService.go b/pkg/pipeline/BuildPipelineSwitchService.go index 076d95a12e..858f03473c 100644 --- a/pkg/pipeline/BuildPipelineSwitchService.go +++ b/pkg/pipeline/BuildPipelineSwitchService.go @@ -21,7 +21,7 @@ import ( "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" "github.com/devtron-labs/devtron/internal/util" "github.com/devtron-labs/devtron/pkg/bean" - pipelineConfigBean "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" + "github.com/devtron-labs/devtron/pkg/build/pipeline/bean/common" "github.com/devtron-labs/devtron/pkg/build/pipeline/read" "github.com/devtron-labs/devtron/pkg/pipeline/adapter" "github.com/devtron-labs/devtron/pkg/pipeline/history" @@ -124,10 +124,10 @@ func (impl *BuildPipelineSwitchServiceImpl) SwitchToCiPipelineExceptExternal(req } //get the ciPipeline - var switchFromType pipelineConfigBean.PipelineType + var switchFromType common.PipelineType var switchFromPipelineId int if request.SwitchFromExternalCiPipelineId != 0 { - switchFromType = pipelineConfigBean.EXTERNAL + switchFromType = common.EXTERNAL switchFromPipelineId = request.SwitchFromExternalCiPipelineId } else { switchFromPipelineId = request.SwitchFromCiPipelineId @@ -145,9 +145,9 @@ func (impl *BuildPipelineSwitchServiceImpl) SwitchToCiPipelineExceptExternal(req return impl.createNewPipelineAndReplaceOldPipelineLinks(request.CiPipeline, ciConfig, switchFromPipelineId, switchFromType, request.UserId) } -func (impl *BuildPipelineSwitchServiceImpl) createNewPipelineAndReplaceOldPipelineLinks(ciPipelineReq *bean.CiPipeline, ciConfig *bean.CiConfigRequest, switchFromPipelineId int, switchFromType pipelineConfigBean.PipelineType, userId int32) (*bean.CiConfigRequest, error) { +func (impl *BuildPipelineSwitchServiceImpl) createNewPipelineAndReplaceOldPipelineLinks(ciPipelineReq *bean.CiPipeline, ciConfig *bean.CiConfigRequest, switchFromPipelineId int, switchFromType common.PipelineType, userId int32) (*bean.CiConfigRequest, error) { - isSelfLinkedCiPipeline := switchFromType != pipelineConfigBean.EXTERNAL && ciPipelineReq.IsLinkedCi() && ciPipelineReq.ParentCiPipeline == switchFromPipelineId + isSelfLinkedCiPipeline := switchFromType != common.EXTERNAL && ciPipelineReq.IsLinkedCi() && ciPipelineReq.ParentCiPipeline == switchFromPipelineId if isSelfLinkedCiPipeline { errMsg := "cannot create linked ci pipeline from the same source" return nil, util.DefaultApiError().WithInternalMessage(errMsg).WithUserMessage(errMsg).WithHttpStatusCode(http.StatusBadRequest) @@ -208,7 +208,7 @@ func (impl *BuildPipelineSwitchServiceImpl) createNewPipelineAndReplaceOldPipeli // add switchType and remove other id // make constants for error msgs -func (impl *BuildPipelineSwitchServiceImpl) validateCiPipelineSwitch(switchFromCiPipelineId int, switchToType, switchFromType pipelineConfigBean.PipelineType) error { +func (impl *BuildPipelineSwitchServiceImpl) validateCiPipelineSwitch(switchFromCiPipelineId int, switchToType, switchFromType common.PipelineType) error { // this will only allow below conversions // ext -> {ci_job,direct,linked} // direct -> {ci_job,linked} @@ -220,13 +220,13 @@ func (impl *BuildPipelineSwitchServiceImpl) validateCiPipelineSwitch(switchFromC } // refer SwitchToExternalCi - if switchToType == pipelineConfigBean.EXTERNAL { + if switchToType == common.EXTERNAL { return errors.New(string(cannotConvertToExternalCi)) } // we should not check the below logic for external_ci type as builds are not built in devtron and // linked pipelines won't be there as per current external-ci-pipeline architecture - if switchFromCiPipelineId > 0 && switchFromType != pipelineConfigBean.EXTERNAL { + if switchFromCiPipelineId > 0 && switchFromType != common.EXTERNAL { err := impl.validateSwitchPreConditions(switchFromCiPipelineId) if err != nil { return err @@ -252,13 +252,13 @@ func (impl *BuildPipelineSwitchServiceImpl) deleteCiAndItsWorkflowMappings(tx *p return err } -func (impl *BuildPipelineSwitchServiceImpl) deleteOldCiPipelineAndWorkflowMappingBeforeSwitch(tx *pg.Tx, switchFromPipelineId int, switchFromType pipelineConfigBean.PipelineType, userId int32) (*appWorkflow.AppWorkflowMapping, error) { +func (impl *BuildPipelineSwitchServiceImpl) deleteOldCiPipelineAndWorkflowMappingBeforeSwitch(tx *pg.Tx, switchFromPipelineId int, switchFromType common.PipelineType, userId int32) (*appWorkflow.AppWorkflowMapping, error) { // 1) delete build pipelines // 2) delete app workflowMappings var err error pipelineId := switchFromPipelineId pipelineType := "" - if switchFromType == pipelineConfigBean.EXTERNAL { + if switchFromType == common.EXTERNAL { err = impl.deleteExternalCi(tx, switchFromPipelineId, userId) pipelineType = appWorkflow.WEBHOOK } else { diff --git a/pkg/pipeline/CIHandler_ent.go b/pkg/pipeline/CIHandler_ent.go new file mode 100644 index 0000000000..4d41cf80a3 --- /dev/null +++ b/pkg/pipeline/CIHandler_ent.go @@ -0,0 +1,21 @@ +package pipeline + +import ( + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" +) + +type CiHandlerEnt interface { +} + +func (impl *CiHandlerImpl) updateResourceStatusInCache(ciWorkflowId int, podName string, namespace string, status string) { + //do nothing +} + +func (impl *CiHandlerImpl) getPipelineIdForTriggerView(pipeline *pipelineConfig.CiPipeline) (pipelineId int) { + if pipeline.ParentCiPipeline == 0 { + pipelineId = pipeline.Id + } else { + pipelineId = pipeline.ParentCiPipeline + } + return pipelineId +} diff --git a/pkg/pipeline/CdHandler.go b/pkg/pipeline/CdHandler.go index ba2618db49..7ceacc1c2e 100644 --- a/pkg/pipeline/CdHandler.go +++ b/pkg/pipeline/CdHandler.go @@ -17,49 +17,40 @@ package pipeline import ( - "bufio" "errors" "fmt" "github.com/devtron-labs/common-lib/utils" bean4 "github.com/devtron-labs/common-lib/utils/bean" - "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/adapter/cdWorkflow" - cdWorkflow2 "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow/cdWorkflow" - bean2 "github.com/devtron-labs/devtron/pkg/bean" + pipelineAdapter "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/adapter/cdWorkflow" + cdWorkflowBean "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow/cdWorkflow" "github.com/devtron-labs/devtron/pkg/build/artifacts/imageTagging" - "github.com/devtron-labs/devtron/pkg/cluster/adapter" - bean3 "github.com/devtron-labs/devtron/pkg/cluster/bean" + buildBean "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" repository3 "github.com/devtron-labs/devtron/pkg/cluster/environment/repository" common2 "github.com/devtron-labs/devtron/pkg/deployment/common" - "github.com/devtron-labs/devtron/pkg/pipeline/constants" - util2 "github.com/devtron-labs/devtron/pkg/pipeline/util" + eventProcessorBean "github.com/devtron-labs/devtron/pkg/eventProcessor/bean" "github.com/devtron-labs/devtron/pkg/pipeline/workflowStatus" bean5 "github.com/devtron-labs/devtron/pkg/pipeline/workflowStatus/bean" "github.com/devtron-labs/devtron/pkg/workflow/cd" - "net/http" - "os" - "path/filepath" + "slices" "strconv" "strings" "time" "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" - blob_storage "github.com/devtron-labs/common-lib/blob-storage" "github.com/devtron-labs/common-lib/utils/k8s" "github.com/devtron-labs/devtron/api/bean" "github.com/devtron-labs/devtron/internal/sql/repository" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" "github.com/devtron-labs/devtron/internal/util" "github.com/devtron-labs/devtron/pkg/auth/user" - "github.com/devtron-labs/devtron/pkg/cluster" pipelineBean "github.com/devtron-labs/devtron/pkg/pipeline/bean" "github.com/devtron-labs/devtron/pkg/pipeline/types" resourceGroup2 "github.com/devtron-labs/devtron/pkg/resourceGroup" - util3 "github.com/devtron-labs/devtron/util" + globalUtil "github.com/devtron-labs/devtron/util" "github.com/devtron-labs/devtron/util/rbac" "github.com/go-pg/pg" "go.opentelemetry.io/otel" "go.uber.org/zap" - "k8s.io/client-go/rest" ) const ( @@ -69,13 +60,10 @@ const ( ) type CdHandler interface { - UpdateWorkflow(workflowStatus v1alpha1.WorkflowStatus) (int, string, bool, error) + UpdateWorkflow(workflowStatus eventProcessorBean.CiCdStatus) (int, string, bool, error) GetCdBuildHistory(appId int, environmentId int, pipelineId int, offset int, size int) ([]pipelineBean.CdWorkflowWithArtifact, error) - GetRunningWorkflowLogs(environmentId int, pipelineId int, workflowId int) (*bufio.Reader, func() error, error) FetchCdWorkflowDetails(appId int, environmentId int, pipelineId int, buildId int) (types.WorkflowResponse, error) - DownloadCdWorkflowArtifacts(buildId int) (*os.File, error) FetchCdPrePostStageStatus(pipelineId int) ([]pipelineBean.CdWorkflowWithArtifact, error) - CancelStage(workflowRunnerId int, forceAbort bool, userId int32) (int, error) FetchAppWorkflowStatusForTriggerView(appId int) ([]*pipelineConfig.CdWorkflowStatus, error) FetchAppWorkflowStatusForTriggerViewForEnvironment(request resourceGroup2.ResourceGroupingRequest, token string) ([]*pipelineConfig.CdWorkflowStatus, error) FetchAppDeploymentStatusForEnvironments(request resourceGroup2.ResourceGroupingRequest, token string) ([]*pipelineConfig.AppDeploymentStatus, error) @@ -85,7 +73,6 @@ type CdHandler interface { type CdHandlerImpl struct { Logger *zap.SugaredLogger userService user.UserService - ciLogService CiLogService ciArtifactRepository repository.CiArtifactRepository ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository cdWorkflowRepository pipelineConfig.CdWorkflowRepository @@ -96,10 +83,7 @@ type CdHandlerImpl struct { resourceGroupService resourceGroup2.ResourceGroupService imageTaggingService imageTagging.ImageTaggingService k8sUtil *k8s.K8sServiceImpl - workflowService WorkflowService config *types.CdConfig - clusterService cluster.ClusterService - blobConfigStorageService BlobStorageConfigService customTagService CustomTagService deploymentConfigService common2.DeploymentConfigService workflowStageStatusService workflowStatus.WorkFlowStageStatusService @@ -107,15 +91,14 @@ type CdHandlerImpl struct { } func NewCdHandlerImpl(Logger *zap.SugaredLogger, userService user.UserService, - cdWorkflowRepository pipelineConfig.CdWorkflowRepository, ciLogService CiLogService, + cdWorkflowRepository pipelineConfig.CdWorkflowRepository, ciArtifactRepository repository.CiArtifactRepository, ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository, pipelineRepository pipelineConfig.PipelineRepository, envRepository repository3.EnvironmentRepository, ciWorkflowRepository pipelineConfig.CiWorkflowRepository, enforcerUtil rbac.EnforcerUtil, resourceGroupService resourceGroup2.ResourceGroupService, imageTaggingService imageTagging.ImageTaggingService, k8sUtil *k8s.K8sServiceImpl, - workflowService WorkflowService, clusterService cluster.ClusterService, - blobConfigStorageService BlobStorageConfigService, customTagService CustomTagService, + customTagService CustomTagService, deploymentConfigService common2.DeploymentConfigService, workflowStageStatusService workflowStatus.WorkFlowStageStatusService, cdWorkflowRunnerService cd.CdWorkflowRunnerService, @@ -123,7 +106,6 @@ func NewCdHandlerImpl(Logger *zap.SugaredLogger, userService user.UserService, cdh := &CdHandlerImpl{ Logger: Logger, userService: userService, - ciLogService: ciLogService, cdWorkflowRepository: cdWorkflowRepository, ciArtifactRepository: ciArtifactRepository, ciPipelineMaterialRepository: ciPipelineMaterialRepository, @@ -134,9 +116,6 @@ func NewCdHandlerImpl(Logger *zap.SugaredLogger, userService user.UserService, resourceGroupService: resourceGroupService, imageTaggingService: imageTaggingService, k8sUtil: k8sUtil, - workflowService: workflowService, - clusterService: clusterService, - blobConfigStorageService: blobConfigStorageService, customTagService: customTagService, deploymentConfigService: deploymentConfigService, workflowStageStatusService: workflowStageStatusService, @@ -150,177 +129,69 @@ func NewCdHandlerImpl(Logger *zap.SugaredLogger, userService user.UserService, return cdh } -func (impl *CdHandlerImpl) CancelStage(workflowRunnerId int, forceAbort bool, userId int32) (int, error) { - workflowRunner, err := impl.cdWorkflowRepository.FindWorkflowRunnerById(workflowRunnerId) - if err != nil { - impl.Logger.Errorw("err", "err", err) - return 0, err - } - pipeline, err := impl.pipelineRepository.FindById(workflowRunner.CdWorkflow.PipelineId) - if err != nil { - impl.Logger.Errorw("error while fetching cd pipeline", "err", err) - return 0, err - } - - env, err := impl.envRepository.FindById(pipeline.EnvironmentId) - if err != nil { - impl.Logger.Errorw("could not fetch stage env", "err", err) - return 0, err - } - - var clusterBean bean3.ClusterBean - if env != nil && env.Cluster != nil { - clusterBean = adapter.GetClusterBean(*env.Cluster) - } - clusterConfig := clusterBean.GetClusterConfig() - var isExtCluster bool - if workflowRunner.WorkflowType == types.PRE { - isExtCluster = pipeline.RunPreStageInEnv - } else if workflowRunner.WorkflowType == types.POST { - isExtCluster = pipeline.RunPostStageInEnv - } - var restConfig *rest.Config - if isExtCluster { - restConfig, err = impl.k8sUtil.GetRestConfigByCluster(clusterConfig) - if err != nil { - impl.Logger.Errorw("error in getting rest config by cluster id", "err", err) - return 0, err - } - } - // Terminate workflow - cancelWfDtoRequest := &types.CancelWfRequestDto{ - ExecutorType: workflowRunner.ExecutorType, - WorkflowName: workflowRunner.Name, - Namespace: workflowRunner.Namespace, - RestConfig: restConfig, - IsExt: isExtCluster, - Environment: nil, - } - err = impl.workflowService.TerminateWorkflow(cancelWfDtoRequest) - if err != nil && forceAbort { - impl.Logger.Errorw("error in terminating workflow, with force abort flag as true", "workflowName", workflowRunner.Name, "err", err) - cancelWfDtoRequest.WorkflowGenerateName = fmt.Sprintf("%d-%s", workflowRunnerId, workflowRunner.Name) - err1 := impl.workflowService.TerminateDanglingWorkflows(cancelWfDtoRequest) - if err1 != nil { - impl.Logger.Errorw("error in terminating dangling workflows", "cancelWfDtoRequest", cancelWfDtoRequest, "err", err) - // ignoring error here in case of force abort, confirmed from product - } - } else if err != nil && strings.Contains(err.Error(), "cannot find workflow") { - return 0, &util.ApiError{Code: "200", HttpStatusCode: http.StatusBadRequest, UserMessage: err.Error()} - } else if err != nil { - impl.Logger.Error("cannot terminate wf runner", "err", err) - return 0, err - } - if forceAbort { - err = impl.handleForceAbortCaseForCdStage(workflowRunner, forceAbort) - if err != nil { - impl.Logger.Errorw("error in handleForceAbortCaseForCdStage", "forceAbortFlag", forceAbort, "workflowRunner", workflowRunner, "err", err) - return 0, err - } - return workflowRunner.Id, nil - } - if len(workflowRunner.ImagePathReservationIds) > 0 { - err := impl.customTagService.DeactivateImagePathReservationByImageIds(workflowRunner.ImagePathReservationIds) - if err != nil { - impl.Logger.Errorw("error in deactivating image path reservation ids", "err", err) - return 0, err - } - } - workflowRunner.Status = cdWorkflow2.WorkflowCancel - workflowRunner.UpdatedOn = time.Now() - workflowRunner.UpdatedBy = userId - err = impl.cdWorkflowRunnerService.UpdateCdWorkflowRunnerWithStage(workflowRunner) - if err != nil { - impl.Logger.Error("cannot update deleted workflow runner status, but wf deleted", "err", err) - return 0, err - } - return workflowRunner.Id, nil -} - -func (impl *CdHandlerImpl) updateWorkflowRunnerForForceAbort(workflowRunner *pipelineConfig.CdWorkflowRunner) error { - workflowRunner.Status = cdWorkflow2.WorkflowCancel - workflowRunner.PodStatus = string(bean2.Failed) - workflowRunner.Message = constants.FORCE_ABORT_MESSAGE_AFTER_STARTING_STAGE - err := impl.cdWorkflowRunnerService.UpdateCdWorkflowRunnerWithStage(workflowRunner) - if err != nil { - impl.Logger.Errorw("error in updating workflow status in cd workflow runner in force abort case", "err", err) - return err - } - return nil -} - -func (impl *CdHandlerImpl) handleForceAbortCaseForCdStage(workflowRunner *pipelineConfig.CdWorkflowRunner, forceAbort bool) error { - isWorkflowInNonTerminalStage := workflowRunner.Status == string(v1alpha1.NodePending) || workflowRunner.Status == string(v1alpha1.NodeRunning) - if !isWorkflowInNonTerminalStage { - if forceAbort { - return impl.updateWorkflowRunnerForForceAbort(workflowRunner) - } else { - return &util.ApiError{Code: "200", HttpStatusCode: 400, UserMessage: "cannot cancel stage, stage not in progress"} - } - } - //this arises when someone deletes the workflow in resource browser and wants to force abort a cd stage(pre/post) - if workflowRunner.Status == string(v1alpha1.NodeRunning) && forceAbort { - return impl.updateWorkflowRunnerForForceAbort(workflowRunner) - } - return nil -} - -func (impl *CdHandlerImpl) UpdateWorkflow(workflowStatus v1alpha1.WorkflowStatus) (int, string, bool, error) { - wfStatusRs := impl.extractWorkfowStatus(workflowStatus) +func (impl *CdHandlerImpl) UpdateWorkflow(workflowStatus eventProcessorBean.CiCdStatus) (int, string, bool, error) { + wfStatusRs := impl.extractWorkflowStatus(workflowStatus) workflowName, status, podStatus, message, podName := wfStatusRs.WorkflowName, wfStatusRs.Status, wfStatusRs.PodStatus, wfStatusRs.Message, wfStatusRs.PodName - impl.Logger.Debugw("cd update for ", "wf ", workflowName, "status", status) + impl.Logger.Debugw("cd workflow status update event for", "wf ", workflowName, "status", status) if workflowName == "" { return 0, "", false, errors.New("invalid wf name") } workflowId, err := strconv.Atoi(workflowName[:strings.Index(workflowName, "-")]) if err != nil { - impl.Logger.Error("invalid wf status update req", "err", err) + impl.Logger.Errorw("invalid wf status update req", "workflowName", workflowName, "err", err) return 0, "", false, err } - savedWorkflow, err := impl.cdWorkflowRepository.FindWorkflowRunnerById(workflowId) + savedWorkflow, err := impl.cdWorkflowRepository.FindPreOrPostCdWorkflowRunnerById(workflowId) if err != nil { - impl.Logger.Error("cannot get saved wf", "err", err) + impl.Logger.Error("cannot get saved wf", "workflowId", workflowId, "err", err) return 0, "", false, err } cdArtifactLocationFormat := impl.config.GetArtifactLocationFormat() cdArtifactLocation := fmt.Sprintf(cdArtifactLocationFormat, savedWorkflow.CdWorkflowId, savedWorkflow.Id) if impl.stateChanged(status, podStatus, message, workflowStatus.FinishedAt.Time, savedWorkflow) { - if savedWorkflow.Status != cdWorkflow2.WorkflowCancel { + if !slices.Contains(cdWorkflowBean.WfrTerminalStatusList, savedWorkflow.PodStatus) { + savedWorkflow.Message = message + if !slices.Contains(cdWorkflowBean.WfrTerminalStatusList, savedWorkflow.Status) { + savedWorkflow.FinishedOn = workflowStatus.FinishedAt.Time + } + } else { + impl.Logger.Warnw("cd stage already in terminal state. skipped message and finishedOn from being updated", + "wfId", savedWorkflow.Id, "podStatus", savedWorkflow.PodStatus, "status", savedWorkflow.Status, "message", message, "finishedOn", workflowStatus.FinishedAt.Time) + } + if savedWorkflow.Status != cdWorkflowBean.WorkflowCancel { savedWorkflow.Status = status } savedWorkflow.CdArtifactLocation = cdArtifactLocation savedWorkflow.PodStatus = podStatus - savedWorkflow.Message = message - savedWorkflow.FinishedOn = workflowStatus.FinishedAt.Time savedWorkflow.Name = workflowName // removed log location from here since we are saving it at trigger savedWorkflow.PodName = podName savedWorkflow.UpdateAuditLog(1) - impl.Logger.Debugw("updating workflow ", "workflow", savedWorkflow) + impl.Logger.Debugw("updating cd workflow runner", "workflow", savedWorkflow) err = impl.cdWorkflowRunnerService.UpdateCdWorkflowRunnerWithStage(savedWorkflow) if err != nil { - impl.Logger.Error("update wf failed for id " + strconv.Itoa(savedWorkflow.Id)) - return 0, "", true, err + impl.Logger.Errorw("update wf failed for id", "wfId", savedWorkflow.Id, "err", err) + return savedWorkflow.Id, "", true, err } appId := savedWorkflow.CdWorkflow.Pipeline.AppId envId := savedWorkflow.CdWorkflow.Pipeline.EnvironmentId envDeploymentConfig, err := impl.deploymentConfigService.GetConfigForDevtronApps(appId, envId) if err != nil { impl.Logger.Errorw("error in fetching environment deployment config by appId and envId", "appId", appId, "envId", envId, "err", err) - return 0, "", true, err + return savedWorkflow.Id, savedWorkflow.Status, true, err } - util3.TriggerCDMetrics(cdWorkflow.GetTriggerMetricsFromRunnerObj(savedWorkflow, envDeploymentConfig), impl.config.ExposeCDMetrics) + globalUtil.TriggerCDMetrics(pipelineAdapter.GetTriggerMetricsFromRunnerObj(savedWorkflow, envDeploymentConfig), impl.config.ExposeCDMetrics) if string(v1alpha1.NodeError) == savedWorkflow.Status || string(v1alpha1.NodeFailed) == savedWorkflow.Status { - impl.Logger.Warnw("cd stage failed for workflow: ", "wfId", savedWorkflow.Id) + impl.Logger.Warnw("cd stage failed for workflow", "wfId", savedWorkflow.Id) } return savedWorkflow.Id, savedWorkflow.Status, true, nil } - return savedWorkflow.Id, savedWorkflow.Status, false, nil + return savedWorkflow.Id, status, false, nil } -func (impl *CdHandlerImpl) extractWorkfowStatus(workflowStatus v1alpha1.WorkflowStatus) *types.WorkflowStatus { +func (impl *CdHandlerImpl) extractWorkflowStatus(workflowStatus eventProcessorBean.CiCdStatus) *types.WorkflowStatus { workflowName := "" status := string(workflowStatus.Phase) podStatus := "Pending" @@ -435,9 +306,9 @@ func (impl *CdHandlerImpl) GetCdBuildHistory(appId int, environmentId int, pipel return cdWorkflowArtifact, err } - var ciMaterialsArr []pipelineConfig.CiPipelineMaterialResponse + var ciMaterialsArr []buildBean.CiPipelineMaterialResponse for _, ciMaterial := range ciMaterials { - res := pipelineConfig.CiPipelineMaterialResponse{ + res := buildBean.CiPipelineMaterialResponse{ Id: ciMaterial.Id, GitMaterialId: ciMaterial.GitMaterialId, GitMaterialName: ciMaterial.GitMaterial.Name[strings.Index(ciMaterial.GitMaterial.Name, "-")+1:], @@ -515,122 +386,6 @@ func (impl *CdHandlerImpl) GetCdBuildHistory(appId int, environmentId int, pipel return cdWorkflowArtifact, nil } -func (impl *CdHandlerImpl) GetRunningWorkflowLogs(environmentId int, pipelineId int, wfrId int) (*bufio.Reader, func() error, error) { - cdWorkflow, err := impl.cdWorkflowRepository.FindWorkflowRunnerById(wfrId) - if err != nil { - impl.Logger.Errorw("error on fetch wf runner", "err", err) - return nil, nil, err - } - - env, err := impl.envRepository.FindById(environmentId) - if err != nil { - impl.Logger.Errorw("could not fetch stage env", "err", err) - return nil, nil, err - } - - pipeline, err := impl.pipelineRepository.FindById(cdWorkflow.CdWorkflow.PipelineId) - if err != nil { - impl.Logger.Errorw("error while fetching cd pipeline", "err", err) - return nil, nil, err - } - var clusterBean bean3.ClusterBean - if env != nil && env.Cluster != nil { - clusterBean = adapter.GetClusterBean(*env.Cluster) - } - clusterConfig := clusterBean.GetClusterConfig() - var isExtCluster bool - if cdWorkflow.WorkflowType == types.PRE { - isExtCluster = pipeline.RunPreStageInEnv - } else if cdWorkflow.WorkflowType == types.POST { - isExtCluster = pipeline.RunPostStageInEnv - } - return impl.getWorkflowLogs(pipelineId, cdWorkflow, clusterConfig, isExtCluster) -} - -func (impl *CdHandlerImpl) getWorkflowLogs(pipelineId int, cdWorkflow *pipelineConfig.CdWorkflowRunner, clusterConfig *k8s.ClusterConfig, runStageInEnv bool) (*bufio.Reader, func() error, error) { - cdLogRequest := types.BuildLogRequest{ - PodName: cdWorkflow.PodName, - Namespace: cdWorkflow.Namespace, - } - - logStream, cleanUp, err := impl.ciLogService.FetchRunningWorkflowLogs(cdLogRequest, clusterConfig, runStageInEnv) - if logStream == nil || err != nil { - if !cdWorkflow.BlobStorageEnabled { - return nil, nil, errors.New("logs-not-stored-in-repository") - } else if string(v1alpha1.NodeSucceeded) == cdWorkflow.Status || string(v1alpha1.NodeError) == cdWorkflow.Status || string(v1alpha1.NodeFailed) == cdWorkflow.Status || cdWorkflow.Status == cdWorkflow2.WorkflowCancel { - impl.Logger.Debugw("pod is not live", "podName", cdWorkflow.PodName, "err", err) - return impl.getLogsFromRepository(pipelineId, cdWorkflow, clusterConfig, runStageInEnv) - } - if err != nil { - impl.Logger.Errorw("err on fetch workflow logs", "err", err) - return nil, nil, err - } else if logStream == nil { - return nil, cleanUp, fmt.Errorf("no logs found for pod %s", cdWorkflow.PodName) - } - } - logReader := bufio.NewReader(logStream) - return logReader, cleanUp, err -} - -func (impl *CdHandlerImpl) getLogsFromRepository(pipelineId int, cdWorkflow *pipelineConfig.CdWorkflowRunner, clusterConfig *k8s.ClusterConfig, isExt bool) (*bufio.Reader, func() error, error) { - impl.Logger.Debug("getting historic logs", "pipelineId", pipelineId) - - cdConfigLogsBucket := impl.config.GetDefaultBuildLogsBucket() // TODO -fixme - cdConfigCdCacheRegion := impl.config.GetDefaultCdLogsBucketRegion() - - cdLogRequest := types.BuildLogRequest{ - PipelineId: cdWorkflow.CdWorkflow.PipelineId, - WorkflowId: cdWorkflow.Id, - PodName: cdWorkflow.PodName, - LogsFilePath: cdWorkflow.LogLocation, // impl.ciCdConfig.CiDefaultBuildLogsKeyPrefix + "/" + cdWorkflow.Name + "/main.log", //TODO - fixme - CloudProvider: impl.config.CloudProvider, - AzureBlobConfig: &blob_storage.AzureBlobBaseConfig{ - Enabled: impl.config.CloudProvider == types.BLOB_STORAGE_AZURE, - AccountName: impl.config.AzureAccountName, - BlobContainerName: impl.config.AzureBlobContainerCiLog, - AccountKey: impl.config.AzureAccountKey, - }, - AwsS3BaseConfig: &blob_storage.AwsS3BaseConfig{ - AccessKey: impl.config.BlobStorageS3AccessKey, - Passkey: impl.config.BlobStorageS3SecretKey, - EndpointUrl: impl.config.BlobStorageS3Endpoint, - IsInSecure: impl.config.BlobStorageS3EndpointInsecure, - BucketName: cdConfigLogsBucket, - Region: cdConfigCdCacheRegion, - VersioningEnabled: impl.config.BlobStorageS3BucketVersioned, - }, - GcpBlobBaseConfig: &blob_storage.GcpBlobBaseConfig{ - BucketName: cdConfigLogsBucket, - CredentialFileJsonData: impl.config.BlobStorageGcpCredentialJson, - }, - } - useExternalBlobStorage := isExternalBlobStorageEnabled(isExt, impl.config.UseBlobStorageConfigInCdWorkflow) - if useExternalBlobStorage { - // fetch extClusterBlob cm and cs from k8s client, if they are present then read creds - // from them else return. - cmConfig, secretConfig, err := impl.blobConfigStorageService.FetchCmAndSecretBlobConfigFromExternalCluster(clusterConfig, cdWorkflow.Namespace) - if err != nil { - impl.Logger.Errorw("error in fetching config map and secret from external cluster", "err", err, "clusterConfig", clusterConfig) - return nil, nil, err - } - rq := &cdLogRequest - rq.SetBuildLogRequest(cmConfig, secretConfig) - } - - impl.Logger.Debugw("s3 log req ", "pipelineId", pipelineId, "runnerId", cdWorkflow.Id) - oldLogsStream, cleanUp, err := impl.ciLogService.FetchLogs(impl.config.BaseLogLocationPath, cdLogRequest) - if err != nil { - impl.Logger.Errorw("err", err) - return nil, nil, err - } - logReader := bufio.NewReader(oldLogsStream) - return logReader, cleanUp, err -} -func isExternalBlobStorageEnabled(isExternalRun bool, useBlobStorageConfigInCdWorkflow bool) bool { - // TODO impl.config.UseBlobStorageConfigInCdWorkflow fetches the live status, we need to check from db as well, we should put useExternalBlobStorage in db - return isExternalRun && !useBlobStorageConfigInCdWorkflow -} - func (impl *CdHandlerImpl) FetchCdWorkflowDetails(appId int, environmentId int, pipelineId int, buildId int) (types.WorkflowResponse, error) { workflowR, err := impl.cdWorkflowRepository.FindWorkflowRunnerById(buildId) if err != nil && err != pg.ErrNoRows { @@ -689,9 +444,9 @@ func (impl *CdHandlerImpl) FetchCdWorkflowDetails(appId int, environmentId int, return types.WorkflowResponse{}, err } - var ciMaterialsArr []pipelineConfig.CiPipelineMaterialResponse + var ciMaterialsArr []buildBean.CiPipelineMaterialResponse for _, m := range ciMaterials { - res := pipelineConfig.CiPipelineMaterialResponse{ + res := buildBean.CiPipelineMaterialResponse{ Id: m.Id, GitMaterialId: m.GitMaterialId, GitMaterialName: m.GitMaterial.Name[strings.Index(m.GitMaterial.Name, "-")+1:], @@ -735,89 +490,6 @@ func (impl *CdHandlerImpl) FetchCdWorkflowDetails(appId int, environmentId int, } -func (impl *CdHandlerImpl) DownloadCdWorkflowArtifacts(buildId int) (*os.File, error) { - wfr, err := impl.cdWorkflowRepository.FindWorkflowRunnerById(buildId) - if err != nil { - impl.Logger.Errorw("unable to fetch ciWorkflow", "err", err) - return nil, err - } - useExternalBlobStorage := isExternalBlobStorageEnabled(wfr.IsExternalRun(), impl.config.UseBlobStorageConfigInCdWorkflow) - if !wfr.BlobStorageEnabled { - return nil, errors.New("logs-not-stored-in-repository") - } - - cdConfigLogsBucket := impl.config.GetDefaultBuildLogsBucket() - cdConfigCdCacheRegion := impl.config.GetDefaultCdLogsBucketRegion() - - item := strconv.Itoa(wfr.Id) - awsS3BaseConfig := &blob_storage.AwsS3BaseConfig{ - AccessKey: impl.config.BlobStorageS3AccessKey, - Passkey: impl.config.BlobStorageS3SecretKey, - EndpointUrl: impl.config.BlobStorageS3Endpoint, - IsInSecure: impl.config.BlobStorageS3EndpointInsecure, - BucketName: cdConfigLogsBucket, - Region: cdConfigCdCacheRegion, - VersioningEnabled: impl.config.BlobStorageS3BucketVersioned, - } - azureBlobBaseConfig := &blob_storage.AzureBlobBaseConfig{ - Enabled: impl.config.CloudProvider == types.BLOB_STORAGE_AZURE, - AccountKey: impl.config.AzureAccountKey, - AccountName: impl.config.AzureAccountName, - BlobContainerName: impl.config.AzureBlobContainerCiLog, - } - gcpBlobBaseConfig := &blob_storage.GcpBlobBaseConfig{ - BucketName: cdConfigLogsBucket, - CredentialFileJsonData: impl.config.BlobStorageGcpCredentialJson, - } - cdArtifactLocationFormat := impl.config.GetArtifactLocationFormat() - key := fmt.Sprintf(cdArtifactLocationFormat, wfr.CdWorkflow.Id, wfr.Id) - if len(wfr.CdArtifactLocation) != 0 && util2.IsValidUrlSubPath(wfr.CdArtifactLocation) { - key = wfr.CdArtifactLocation - } else if util2.IsValidUrlSubPath(key) { - impl.cdWorkflowRepository.MigrateCdArtifactLocation(wfr.Id, key) - } - baseLogLocationPathConfig := impl.config.BaseLogLocationPath - blobStorageService := blob_storage.NewBlobStorageServiceImpl(nil) - destinationKey := filepath.Clean(filepath.Join(baseLogLocationPathConfig, item)) - request := &blob_storage.BlobStorageRequest{ - StorageType: impl.config.CloudProvider, - SourceKey: key, - DestinationKey: destinationKey, - AzureBlobBaseConfig: azureBlobBaseConfig, - AwsS3BaseConfig: awsS3BaseConfig, - GcpBlobBaseConfig: gcpBlobBaseConfig, - } - if useExternalBlobStorage { - clusterConfig, err := impl.clusterService.GetClusterConfigByClusterId(wfr.CdWorkflow.Pipeline.Environment.ClusterId) - if err != nil { - impl.Logger.Errorw("GetClusterConfigByClusterId, error in fetching clusterConfig", "err", err, "clusterId", wfr.CdWorkflow.Pipeline.Environment.ClusterId) - return nil, err - } - // fetch extClusterBlob cm and cs from k8s client, if they are present then read creds - // from them else return. - cmConfig, secretConfig, err := impl.blobConfigStorageService.FetchCmAndSecretBlobConfigFromExternalCluster(clusterConfig, wfr.Namespace) - if err != nil { - impl.Logger.Errorw("error in fetching config map and secret from external cluster", "err", err, "clusterConfig", clusterConfig) - return nil, err - } - request = updateRequestWithExtClusterCmAndSecret(request, cmConfig, secretConfig) - } - _, numBytes, err := blobStorageService.Get(request) - if err != nil { - impl.Logger.Errorw("error occurred while downloading file", "request", request, "error", err) - return nil, errors.New("failed to download resource") - } - - file, err := os.Open(destinationKey) - if err != nil { - impl.Logger.Errorw("unable to open file", "file", item, "err", err) - return nil, errors.New("unable to open file") - } - - impl.Logger.Infow("Downloaded ", "name", file.Name(), "bytes", numBytes) - return file, nil -} - func (impl *CdHandlerImpl) converterWFR(wfr pipelineConfig.CdWorkflowRunner) pipelineBean.CdWorkflowWithArtifact { workflow := pipelineBean.CdWorkflowWithArtifact{} if wfr.Id > 0 { diff --git a/pkg/pipeline/CiCdPipelineOrchestrator.go b/pkg/pipeline/CiCdPipelineOrchestrator.go index 3916ec29b2..a617e4635f 100644 --- a/pkg/pipeline/CiCdPipelineOrchestrator.go +++ b/pkg/pipeline/CiCdPipelineOrchestrator.go @@ -32,12 +32,12 @@ import ( repository6 "github.com/devtron-labs/devtron/pkg/build/git/gitMaterial/repository" "github.com/devtron-labs/devtron/pkg/build/pipeline" bean2 "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" + buildCommonBean "github.com/devtron-labs/devtron/pkg/build/pipeline/bean/common" read2 "github.com/devtron-labs/devtron/pkg/chart/read" repository2 "github.com/devtron-labs/devtron/pkg/cluster/environment/repository" "github.com/devtron-labs/devtron/pkg/deployment/common" "github.com/devtron-labs/devtron/pkg/deployment/common/read" "github.com/devtron-labs/devtron/pkg/deployment/gitOps/config" - constants3 "github.com/devtron-labs/devtron/pkg/pipeline/constants" util4 "github.com/devtron-labs/devtron/pkg/pipeline/util" "github.com/devtron-labs/devtron/pkg/plugin" "github.com/devtron-labs/devtron/util/beHelper" @@ -226,9 +226,6 @@ func NewCiCdPipelineOrchestrator( } } -const BEFORE_DOCKER_BUILD string = "BEFORE_DOCKER_BUILD" -const AFTER_DOCKER_BUILD string = "AFTER_DOCKER_BUILD" - func (impl CiCdPipelineOrchestratorImpl) PatchCiMaterialSource(patchRequest *bean.CiMaterialPatchRequest, userId int32) (*bean.CiMaterialPatchRequest, error) { pipeline, err := impl.findUniquePipelineForAppIdAndEnvironmentId(patchRequest.AppId, patchRequest.EnvironmentId) if err != nil { @@ -329,7 +326,7 @@ func (impl CiCdPipelineOrchestratorImpl) validateCiPipelineMaterial(ciPipelineMa func (impl CiCdPipelineOrchestratorImpl) getSkipMessage(ciPipeline *pipelineConfig.CiPipeline) string { switch ciPipeline.PipelineType { - case string(bean2.LINKED_CD): + case string(buildCommonBean.LINKED_CD): return "“Sync with Environment”" default: return "“Linked Build Pipeline”" @@ -1010,7 +1007,7 @@ func (impl CiCdPipelineOrchestratorImpl) CreateCiConf(createRequest *bean.CiConf var pipelineMaterials []*pipelineConfig.CiPipelineMaterial for _, r := range ciPipeline.CiMaterial { - if ciPipeline.PipelineType == bean2.LINKED_CD { + if ciPipeline.PipelineType == buildCommonBean.LINKED_CD { continue } material := &pipelineConfig.CiPipelineMaterial{ @@ -2505,7 +2502,7 @@ func (impl *CiCdPipelineOrchestratorImpl) GetWorkflowCacheConfig(appType helper. if appType == helper.Job { return util4.GetWorkflowCacheConfig(pipelineWorkflowCacheConfig, impl.workflowCacheConfig.IgnoreJob) } else { - if pipelineType == string(constants3.CI_JOB) { + if pipelineType == string(buildCommonBean.CI_JOB) { return util4.GetWorkflowCacheConfigWithBackwardCompatibility(pipelineWorkflowCacheConfig, impl.ciConfig.WorkflowCacheConfig, impl.workflowCacheConfig.IgnoreCIJob, impl.ciConfig.SkipCiJobBuildCachePushPull) } else { return util4.GetWorkflowCacheConfigWithBackwardCompatibility(pipelineWorkflowCacheConfig, impl.ciConfig.WorkflowCacheConfig, impl.workflowCacheConfig.IgnoreCI, impl.ciConfig.IgnoreDockerCacheForCI) diff --git a/pkg/pipeline/CiHandler.go b/pkg/pipeline/CiHandler.go index 5f05b28ef6..ef7f36a198 100644 --- a/pkg/pipeline/CiHandler.go +++ b/pkg/pipeline/CiHandler.go @@ -17,86 +17,62 @@ package pipeline import ( - "bufio" "context" "errors" "fmt" "github.com/devtron-labs/common-lib/utils" "github.com/devtron-labs/common-lib/utils/workFlow" - "github.com/devtron-labs/devtron/internal/sql/constants" - "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow/cdWorkflow" - bean6 "github.com/devtron-labs/devtron/pkg/auth/user/bean" - "github.com/devtron-labs/devtron/pkg/bean/common" + cdWorkflowBean "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow/cdWorkflow" "github.com/devtron-labs/devtron/pkg/build/artifacts/imageTagging" - bean4 "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" - "github.com/devtron-labs/devtron/pkg/cluster/adapter" - bean5 "github.com/devtron-labs/devtron/pkg/cluster/bean" - "github.com/devtron-labs/devtron/pkg/cluster/environment" + buildBean "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" repository2 "github.com/devtron-labs/devtron/pkg/cluster/environment/repository" - constants2 "github.com/devtron-labs/devtron/pkg/pipeline/constants" - util3 "github.com/devtron-labs/devtron/pkg/pipeline/util" + eventProcessorBean "github.com/devtron-labs/devtron/pkg/eventProcessor/bean" + "github.com/devtron-labs/devtron/pkg/pipeline/constants" "github.com/devtron-labs/devtron/pkg/pipeline/workflowStatus" - "io/ioutil" - "net/http" - "os" - "path/filepath" "regexp" + "slices" "strconv" "strings" "time" - blob_storage "github.com/devtron-labs/common-lib/blob-storage" - "github.com/devtron-labs/common-lib/utils/k8s" + "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" bean2 "github.com/devtron-labs/devtron/api/bean" + client "github.com/devtron-labs/devtron/client/events" "github.com/devtron-labs/devtron/client/gitSensor" + "github.com/devtron-labs/devtron/internal/sql/repository" "github.com/devtron-labs/devtron/internal/sql/repository/appWorkflow" + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" + "github.com/devtron-labs/devtron/internal/util" "github.com/devtron-labs/devtron/pkg/auth/user" - "github.com/devtron-labs/devtron/pkg/cluster" - k8s2 "github.com/devtron-labs/devtron/pkg/k8s" - bean3 "github.com/devtron-labs/devtron/pkg/pipeline/bean" + k8sPkg "github.com/devtron-labs/devtron/pkg/k8s" + pipelineConfigBean "github.com/devtron-labs/devtron/pkg/pipeline/bean" "github.com/devtron-labs/devtron/pkg/pipeline/executors" "github.com/devtron-labs/devtron/pkg/pipeline/types" - resourceGroup "github.com/devtron-labs/devtron/pkg/resourceGroup" + "github.com/devtron-labs/devtron/pkg/resourceGroup" "github.com/devtron-labs/devtron/util/rbac" - errors2 "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/client-go/rest" - - "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" - client "github.com/devtron-labs/devtron/client/events" - "github.com/devtron-labs/devtron/internal/sql/repository" - "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" - "github.com/devtron-labs/devtron/internal/util" - "github.com/devtron-labs/devtron/pkg/bean" - util2 "github.com/devtron-labs/devtron/util/event" "github.com/go-pg/pg" "go.uber.org/zap" ) type CiHandler interface { - HandleCIWebhook(gitCiTriggerRequest bean.GitCiTriggerRequest) (int, error) - HandleCIManual(ciTriggerRequest bean.CiTriggerRequest) (int, error) - CheckAndReTriggerCI(workflowStatus v1alpha1.WorkflowStatus) error - FetchMaterialsByPipelineId(pipelineId int, showAll bool) ([]pipelineConfig.CiPipelineMaterialResponse, error) - FetchMaterialsByPipelineIdAndGitMaterialId(pipelineId int, gitMaterialId int, showAll bool) ([]pipelineConfig.CiPipelineMaterialResponse, error) + // HandleCIWebhook(gitCiTriggerRequest bean.GitCiTriggerRequest) (int, error) + //HandleCIManual(ciTriggerRequest bean.CiTriggerRequest) (int, error) + //CheckAndReTriggerCI(workflowStatus eventProcessorBean.CiCdStatus) error + FetchMaterialsByPipelineId(pipelineId int, showAll bool) ([]buildBean.CiPipelineMaterialResponse, error) + FetchMaterialsByPipelineIdAndGitMaterialId(pipelineId int, gitMaterialId int, showAll bool) ([]buildBean.CiPipelineMaterialResponse, error) FetchWorkflowDetails(appId int, pipelineId int, buildId int) (types.WorkflowResponse, error) FetchArtifactsForCiJob(buildId int) (*types.ArtifactsForCiJob, error) - //FetchBuildById(appId int, pipelineId int) (WorkflowResponse, error) - CancelBuild(workflowId int, forceAbort bool) (int, error) - - GetRunningWorkflowLogs(workflowId int) (*bufio.Reader, func() error, error) - GetHistoricBuildLogs(workflowId int, ciWorkflow *pipelineConfig.CiWorkflow) (map[string]string, error) - //SyncWorkflows() error GetBuildHistory(pipelineId int, appId int, offset int, size int) ([]types.WorkflowResponse, error) - DownloadCiWorkflowArtifacts(pipelineId int, buildId int) (*os.File, error) - UpdateWorkflow(workflowStatus v1alpha1.WorkflowStatus) (int, error) + UpdateWorkflow(workflowStatus eventProcessorBean.CiCdStatus) (int, bool, error) FetchCiStatusForTriggerView(appId int) ([]*pipelineConfig.CiWorkflowStatus, error) FetchCiStatusForTriggerViewV1(appId int) ([]*pipelineConfig.CiWorkflowStatus, error) RefreshMaterialByCiPipelineMaterialId(gitMaterialId int) (refreshRes *gitSensor.RefreshGitMaterialResponse, err error) FetchMaterialInfoByArtifactId(ciArtifactId int, envId int) (*types.GitTriggerInfoResponse, error) - UpdateCiWorkflowStatusFailure(timeoutForFailureCiBuild int) error + //UpdateCiWorkflowStatusFailure(timeoutForFailureCiBuild int) error FetchCiStatusForTriggerViewForEnvironment(request resourceGroup.ResourceGroupingRequest, token string) ([]*pipelineConfig.CiWorkflowStatus, error) + CiHandlerEnt } type CiHandlerImpl struct { @@ -105,15 +81,12 @@ type CiHandlerImpl struct { ciService CiService gitSensorClient gitSensor.Client ciWorkflowRepository pipelineConfig.CiWorkflowRepository - workflowService WorkflowService - ciLogService CiLogService ciArtifactRepository repository.CiArtifactRepository userService user.UserService eventClient client.EventClient eventFactory client.EventFactory ciPipelineRepository pipelineConfig.CiPipelineRepository appListingRepository repository.AppListingRepository - K8sUtil *k8s.K8sServiceImpl cdPipelineRepository pipelineConfig.PipelineRepository enforcerUtil rbac.EnforcerUtil resourceGroupService resourceGroup.ResourceGroupService @@ -122,34 +95,28 @@ type CiHandlerImpl struct { customTagService CustomTagService appWorkflowRepository appWorkflow.AppWorkflowRepository config *types.CiConfig - k8sCommonService k8s2.K8sCommonService - clusterService cluster.ClusterService - blobConfigStorageService BlobStorageConfigService - envService environment.EnvironmentService + k8sCommonService k8sPkg.K8sCommonService workFlowStageStatusService workflowStatus.WorkFlowStageStatusService } -func NewCiHandlerImpl(Logger *zap.SugaredLogger, ciService CiService, ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository, gitSensorClient gitSensor.Client, ciWorkflowRepository pipelineConfig.CiWorkflowRepository, workflowService WorkflowService, - ciLogService CiLogService, ciArtifactRepository repository.CiArtifactRepository, userService user.UserService, eventClient client.EventClient, eventFactory client.EventFactory, ciPipelineRepository pipelineConfig.CiPipelineRepository, - appListingRepository repository.AppListingRepository, K8sUtil *k8s.K8sServiceImpl, cdPipelineRepository pipelineConfig.PipelineRepository, enforcerUtil rbac.EnforcerUtil, resourceGroupService resourceGroup.ResourceGroupService, envRepository repository2.EnvironmentRepository, - imageTaggingService imageTagging.ImageTaggingService, k8sCommonService k8s2.K8sCommonService, clusterService cluster.ClusterService, blobConfigStorageService BlobStorageConfigService, appWorkflowRepository appWorkflow.AppWorkflowRepository, customTagService CustomTagService, - envService environment.EnvironmentService, - workFlowStageStatusService workflowStatus.WorkFlowStageStatusService) *CiHandlerImpl { +func NewCiHandlerImpl(Logger *zap.SugaredLogger, ciService CiService, ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository, gitSensorClient gitSensor.Client, ciWorkflowRepository pipelineConfig.CiWorkflowRepository, + ciArtifactRepository repository.CiArtifactRepository, userService user.UserService, eventClient client.EventClient, eventFactory client.EventFactory, ciPipelineRepository pipelineConfig.CiPipelineRepository, + appListingRepository repository.AppListingRepository, cdPipelineRepository pipelineConfig.PipelineRepository, enforcerUtil rbac.EnforcerUtil, resourceGroupService resourceGroup.ResourceGroupService, envRepository repository2.EnvironmentRepository, + imageTaggingService imageTagging.ImageTaggingService, k8sCommonService k8sPkg.K8sCommonService, appWorkflowRepository appWorkflow.AppWorkflowRepository, customTagService CustomTagService, + workFlowStageStatusService workflowStatus.WorkFlowStageStatusService, +) *CiHandlerImpl { cih := &CiHandlerImpl{ Logger: Logger, ciService: ciService, ciPipelineMaterialRepository: ciPipelineMaterialRepository, gitSensorClient: gitSensorClient, ciWorkflowRepository: ciWorkflowRepository, - workflowService: workflowService, - ciLogService: ciLogService, ciArtifactRepository: ciArtifactRepository, userService: userService, eventClient: eventClient, eventFactory: eventFactory, ciPipelineRepository: ciPipelineRepository, appListingRepository: appListingRepository, - K8sUtil: K8sUtil, cdPipelineRepository: cdPipelineRepository, enforcerUtil: enforcerUtil, resourceGroupService: resourceGroupService, @@ -158,9 +125,6 @@ func NewCiHandlerImpl(Logger *zap.SugaredLogger, ciService CiService, ciPipeline customTagService: customTagService, appWorkflowRepository: appWorkflowRepository, k8sCommonService: k8sCommonService, - clusterService: clusterService, - blobConfigStorageService: blobConfigStorageService, - envService: envService, workFlowStageStatusService: workFlowStageStatusService, } config, err := types.GetCiConfig() @@ -172,173 +136,6 @@ func NewCiHandlerImpl(Logger *zap.SugaredLogger, ciService CiService, ciPipeline return cih } -func (impl *CiHandlerImpl) CheckAndReTriggerCI(workflowStatus v1alpha1.WorkflowStatus) error { - - //return if re-trigger feature is disabled - if !impl.config.WorkflowRetriesEnabled() { - impl.Logger.Debug("CI re-trigger is disabled") - return nil - } - - status, message, ciWorkFlow, err := impl.extractPodStatusAndWorkflow(workflowStatus) - if err != nil { - impl.Logger.Errorw("error in extractPodStatusAndWorkflow", "err", err) - return err - } - - if !executors.CheckIfReTriggerRequired(status, message, ciWorkFlow.Status) { - impl.Logger.Debugw("not re-triggering ci", "status", status, "message", message, "ciWorkflowStatus", ciWorkFlow.Status) - return nil - } - - impl.Logger.Debugw("re-triggering ci", "status", status, "message", message, "ciWorkflowStatus", ciWorkFlow.Status, "ciWorkFlowId", ciWorkFlow.Id) - - retryCount, refCiWorkflow, err := impl.getRefWorkflowAndCiRetryCount(ciWorkFlow) - if err != nil { - impl.Logger.Errorw("error while getting retry count value for a ciWorkflow", "ciWorkFlowId", ciWorkFlow.Id) - return err - } - - err = impl.reTriggerCi(retryCount, refCiWorkflow) - if err != nil { - impl.Logger.Errorw("error in reTriggerCi", "err", err, "status", status, "message", message, "retryCount", retryCount, "ciWorkFlowId", ciWorkFlow.Id) - } - return err -} - -func (impl *CiHandlerImpl) reTriggerCi(retryCount int, refCiWorkflow *pipelineConfig.CiWorkflow) error { - if retryCount >= impl.config.MaxCiWorkflowRetries { - impl.Logger.Infow("maximum retries exhausted for this ciWorkflow", "ciWorkflowId", refCiWorkflow.Id, "retries", retryCount, "configuredRetries", impl.config.MaxCiWorkflowRetries) - return nil - } - impl.Logger.Infow("re-triggering ci for a ci workflow", "ReferenceCiWorkflowId", refCiWorkflow.Id) - ciPipelineMaterialIds := make([]int, 0, len(refCiWorkflow.GitTriggers)) - for id, _ := range refCiWorkflow.GitTriggers { - ciPipelineMaterialIds = append(ciPipelineMaterialIds, id) - } - ciMaterials, err := impl.ciPipelineMaterialRepository.GetByIdsIncludeDeleted(ciPipelineMaterialIds) - if err != nil { - impl.Logger.Errorw("error in getting ci Pipeline Materials using ciPipeline Material Ids", "ciPipelineMaterialIds", ciPipelineMaterialIds, "err", err) - return err - } - - trigger := types.Trigger{} - trigger.BuildTriggerObject(refCiWorkflow, ciMaterials, bean6.SYSTEM_USER_ID, true, nil, "") - _, err = impl.ciService.TriggerCiPipeline(trigger) - - if err != nil { - impl.Logger.Errorw("error occurred in re-triggering ciWorkflow", "triggerDetails", trigger, "err", err) - return err - } - return nil -} - -func (impl *CiHandlerImpl) HandleCIManual(ciTriggerRequest bean.CiTriggerRequest) (int, error) { - impl.Logger.Debugw("HandleCIManual for pipeline ", "PipelineId", ciTriggerRequest.PipelineId) - commitHashes, runtimeParams, err := impl.buildManualTriggerCommitHashes(ciTriggerRequest) - if err != nil { - return 0, err - } - - ciArtifact, err := impl.ciArtifactRepository.GetLatestArtifactTimeByCiPipelineId(ciTriggerRequest.PipelineId) - if err != nil && err != pg.ErrNoRows { - impl.Logger.Errorw("Error in GetLatestArtifactTimeByCiPipelineId", "err", err, "pipelineId", ciTriggerRequest.PipelineId) - return 0, err - } - - createdOn := time.Time{} - if err != pg.ErrNoRows { - createdOn = ciArtifact.CreatedOn - } - - trigger := types.Trigger{ - PipelineId: ciTriggerRequest.PipelineId, - CommitHashes: commitHashes, - CiMaterials: nil, - TriggeredBy: ciTriggerRequest.TriggeredBy, - InvalidateCache: ciTriggerRequest.InvalidateCache, - RuntimeParameters: runtimeParams, - EnvironmentId: ciTriggerRequest.EnvironmentId, - PipelineType: ciTriggerRequest.PipelineType, - CiArtifactLastFetch: createdOn, - } - id, err := impl.ciService.TriggerCiPipeline(trigger) - - if err != nil { - return 0, err - } - return id, nil -} - -func (impl *CiHandlerImpl) HandleCIWebhook(gitCiTriggerRequest bean.GitCiTriggerRequest) (int, error) { - impl.Logger.Debugw("HandleCIWebhook for material ", "material", gitCiTriggerRequest.CiPipelineMaterial) - ciPipeline, err := impl.GetCiPipeline(gitCiTriggerRequest.CiPipelineMaterial.Id) - if err != nil { - impl.Logger.Errorw("err in getting ci_pipeline by ciPipelineMaterialId", "ciPipelineMaterialId", gitCiTriggerRequest.CiPipelineMaterial.Id, "err", err) - return 0, err - } - if ciPipeline.IsManual || ciPipeline.PipelineType == bean4.LINKED_CD.ToString() { - impl.Logger.Debugw("not handling for manual pipeline or in case of linked cd", "pipelineId", ciPipeline.Id) - return 0, err - } - - ciMaterials, err := impl.ciPipelineMaterialRepository.GetByPipelineId(ciPipeline.Id) - if err != nil { - impl.Logger.Errorw("err", "err", err) - return 0, err - } - isValidBuildSequence, err := impl.validateBuildSequence(gitCiTriggerRequest, ciPipeline.Id) - if !isValidBuildSequence { - return 0, errors.New("ignoring older build for ciMaterial " + strconv.Itoa(gitCiTriggerRequest.CiPipelineMaterial.Id) + - " commit " + gitCiTriggerRequest.CiPipelineMaterial.GitCommit.Commit) - } - // updating runtime params - runtimeParams := common.NewRuntimeParameters() - for k, v := range gitCiTriggerRequest.ExtraEnvironmentVariables { - runtimeParams = runtimeParams.AddSystemVariable(k, v) - } - commitHashes, err := impl.buildAutomaticTriggerCommitHashes(ciMaterials, gitCiTriggerRequest) - if err != nil { - return 0, err - } - - trigger := types.Trigger{ - PipelineId: ciPipeline.Id, - CommitHashes: commitHashes, - CiMaterials: ciMaterials, - TriggeredBy: gitCiTriggerRequest.TriggeredBy, - RuntimeParameters: runtimeParams, - } - id, err := impl.ciService.TriggerCiPipeline(trigger) - if err != nil { - return 0, err - } - return id, nil -} - -func (impl *CiHandlerImpl) validateBuildSequence(gitCiTriggerRequest bean.GitCiTriggerRequest, pipelineId int) (bool, error) { - isValid := true - lastTriggeredBuild, err := impl.ciWorkflowRepository.FindLastTriggeredWorkflow(pipelineId) - if !(lastTriggeredBuild.Status == string(v1alpha1.NodePending) || lastTriggeredBuild.Status == string(v1alpha1.NodeRunning)) { - return true, nil - } - if err != nil && !util.IsErrNoRows(err) { - impl.Logger.Errorw("cannot get last build for pipeline", "pipelineId", pipelineId) - return false, err - } - - ciPipelineMaterial := gitCiTriggerRequest.CiPipelineMaterial - - if ciPipelineMaterial.Type == string(constants.SOURCE_TYPE_BRANCH_FIXED) { - if ciPipelineMaterial.GitCommit.Date.Before(lastTriggeredBuild.GitTriggers[ciPipelineMaterial.Id].Date) { - impl.Logger.Warnw("older commit cannot be built for pipeline", "pipelineId", pipelineId, "ciMaterial", gitCiTriggerRequest.CiPipelineMaterial.Id) - isValid = false - } - } - - return isValid, nil -} - func (impl *CiHandlerImpl) RefreshMaterialByCiPipelineMaterialId(gitMaterialId int) (refreshRes *gitSensor.RefreshGitMaterialResponse, err error) { impl.Logger.Debugw("refreshing git material", "id", gitMaterialId) refreshRes, err = impl.gitSensorClient.RefreshGitMaterial(context.Background(), @@ -347,12 +144,12 @@ func (impl *CiHandlerImpl) RefreshMaterialByCiPipelineMaterialId(gitMaterialId i return refreshRes, err } -func (impl *CiHandlerImpl) FetchMaterialsByPipelineIdAndGitMaterialId(pipelineId int, gitMaterialId int, showAll bool) ([]pipelineConfig.CiPipelineMaterialResponse, error) { +func (impl *CiHandlerImpl) FetchMaterialsByPipelineIdAndGitMaterialId(pipelineId int, gitMaterialId int, showAll bool) ([]buildBean.CiPipelineMaterialResponse, error) { ciMaterials, err := impl.ciPipelineMaterialRepository.GetByPipelineIdAndGitMaterialId(pipelineId, gitMaterialId) if err != nil { impl.Logger.Errorw("ciMaterials fetch failed", "err", err) } - var ciPipelineMaterialResponses []pipelineConfig.CiPipelineMaterialResponse + var ciPipelineMaterialResponses []buildBean.CiPipelineMaterialResponse var responseMap = make(map[int]bool) ciMaterialHistoryMap := make(map[*pipelineConfig.CiPipelineMaterial]*gitSensor.MaterialChangeResp) @@ -369,13 +166,13 @@ func (impl *CiHandlerImpl) FetchMaterialsByPipelineIdAndGitMaterialId(pipelineId impl.Logger.Debugw("commits for material ", "m", m, "commits: ", changesResp) if apiErr != nil { impl.Logger.Warnw("git sensor FetchChanges failed for material", "id", m.Id) - return []pipelineConfig.CiPipelineMaterialResponse{}, apiErr + return []buildBean.CiPipelineMaterialResponse{}, apiErr } ciMaterialHistoryMap[m] = changesResp } for k, v := range ciMaterialHistoryMap { - r := pipelineConfig.CiPipelineMaterialResponse{ + r := buildBean.CiPipelineMaterialResponse{ Id: k.Id, GitMaterialId: k.GitMaterialId, GitMaterialName: k.GitMaterial.Name[strings.Index(k.GitMaterial.Name, "-")+1:], @@ -398,10 +195,10 @@ func (impl *CiHandlerImpl) FetchMaterialsByPipelineIdAndGitMaterialId(pipelineId regexMaterials, err := impl.ciPipelineMaterialRepository.GetRegexByPipelineId(pipelineId) if err != nil { impl.Logger.Errorw("regex ciMaterials fetch failed", "err", err) - return []pipelineConfig.CiPipelineMaterialResponse{}, err + return []buildBean.CiPipelineMaterialResponse{}, err } for _, k := range regexMaterials { - r := pipelineConfig.CiPipelineMaterialResponse{ + r := buildBean.CiPipelineMaterialResponse{ Id: k.Id, GitMaterialId: k.GitMaterialId, GitMaterialName: k.GitMaterial.Name[strings.Index(k.GitMaterial.Name, "-")+1:], @@ -424,12 +221,12 @@ func (impl *CiHandlerImpl) FetchMaterialsByPipelineIdAndGitMaterialId(pipelineId return ciPipelineMaterialResponses, nil } -func (impl *CiHandlerImpl) FetchMaterialsByPipelineId(pipelineId int, showAll bool) ([]pipelineConfig.CiPipelineMaterialResponse, error) { +func (impl *CiHandlerImpl) FetchMaterialsByPipelineId(pipelineId int, showAll bool) ([]buildBean.CiPipelineMaterialResponse, error) { ciMaterials, err := impl.ciPipelineMaterialRepository.GetByPipelineId(pipelineId) if err != nil { impl.Logger.Errorw("ciMaterials fetch failed", "err", err) } - var ciPipelineMaterialResponses []pipelineConfig.CiPipelineMaterialResponse + var ciPipelineMaterialResponses []buildBean.CiPipelineMaterialResponse var responseMap = make(map[int]bool) ciMaterialHistoryMap := make(map[*pipelineConfig.CiPipelineMaterial]*gitSensor.MaterialChangeResp) @@ -452,7 +249,7 @@ func (impl *CiHandlerImpl) FetchMaterialsByPipelineId(pipelineId int, showAll bo } for k, v := range ciMaterialHistoryMap { - r := pipelineConfig.CiPipelineMaterialResponse{ + r := buildBean.CiPipelineMaterialResponse{ Id: k.Id, GitMaterialId: k.GitMaterialId, GitMaterialName: k.GitMaterial.Name[strings.Index(k.GitMaterial.Name, "-")+1:], @@ -478,7 +275,7 @@ func (impl *CiHandlerImpl) FetchMaterialsByPipelineId(pipelineId int, showAll bo return nil, err } for _, k := range regexMaterials { - r := pipelineConfig.CiPipelineMaterialResponse{ + r := buildBean.CiPipelineMaterialResponse{ Id: k.Id, GitMaterialId: k.GitMaterialId, GitMaterialName: k.GitMaterial.Name[strings.Index(k.GitMaterial.Name, "-")+1:], @@ -507,9 +304,9 @@ func (impl *CiHandlerImpl) GetBuildHistory(pipelineId int, appId int, offset int if err != nil { impl.Logger.Errorw("ciMaterials fetch failed", "err", err) } - var ciPipelineMaterialResponses []pipelineConfig.CiPipelineMaterialResponse + var ciPipelineMaterialResponses []buildBean.CiPipelineMaterialResponse for _, m := range ciMaterials { - r := pipelineConfig.CiPipelineMaterialResponse{ + r := buildBean.CiPipelineMaterialResponse{ Id: m.Id, GitMaterialId: m.GitMaterialId, Type: string(m.Type), @@ -520,7 +317,7 @@ func (impl *CiHandlerImpl) GetBuildHistory(pipelineId int, appId int, offset int } ciPipelineMaterialResponses = append(ciPipelineMaterialResponses, r) } - //this map contains artifactId -> array of tags of that artifact + // this map contains artifactId -> array of tags of that artifact imageTagsDataMap, err := impl.imageTaggingService.GetTagsDataMapByAppId(appId) if err != nil { impl.Logger.Errorw("error in fetching image tags with appId", "err", err, "appId", appId) @@ -531,6 +328,7 @@ func (impl *CiHandlerImpl) GetBuildHistory(pipelineId int, appId int, offset int impl.Logger.Errorw("err", "err", err) return nil, err } + var workflowIds []int var artifactIds []int for _, w := range workFlows { @@ -544,7 +342,7 @@ func (impl *CiHandlerImpl) GetBuildHistory(pipelineId int, appId int, offset int return nil, err } - //this map contains artifactId -> imageComment of that artifact + // this map contains artifactId -> imageComment of that artifact imageCommetnsDataMap, err := impl.imageTaggingService.GetImageCommentsDataMapByArtifactIds(artifactIds) if err != nil { impl.Logger.Errorw("error in fetching imageCommetnsDataMap", "err", err, "appId", appId, "artifactIds", artifactIds) @@ -586,25 +384,25 @@ func (impl *CiHandlerImpl) GetBuildHistory(pipelineId int, appId int, offset int WorkflowExecutionStage: impl.workFlowStageStatusService.ConvertDBWorkflowStageToMap(allWfStagesDetail, w.Id, w.Status, w.PodStatus, w.Message, bean2.CI_WORKFLOW_TYPE.String(), w.StartedOn, w.FinishedOn), } - if w.Message == bean3.ImageTagUnavailableMessage { - customTag, err := impl.customTagService.GetCustomTagByEntityKeyAndValue(bean3.EntityTypeCiPipelineId, strconv.Itoa(w.CiPipelineId)) + if w.Message == pipelineConfigBean.ImageTagUnavailableMessage { + customTag, err := impl.customTagService.GetCustomTagByEntityKeyAndValue(pipelineConfigBean.EntityTypeCiPipelineId, strconv.Itoa(w.CiPipelineId)) if err != nil && err != pg.ErrNoRows { - //err == pg.ErrNoRows should never happen + // err == pg.ErrNoRows should never happen return nil, err } appWorkflows, err := impl.appWorkflowRepository.FindWFCIMappingByCIPipelineId(w.CiPipelineId) if err != nil && err != pg.ErrNoRows { return nil, err } - wfResponse.AppWorkflowId = appWorkflows[0].AppWorkflowId //it is guaranteed there will always be 1 entry (in case of ci_pipeline_id) + wfResponse.AppWorkflowId = appWorkflows[0].AppWorkflowId // it is guaranteed there will always be 1 entry (in case of ci_pipeline_id) wfResponse.CustomTag = &bean2.CustomTagErrorResponse{ TagPattern: customTag.TagPattern, AutoIncreasingNumber: customTag.AutoIncreasingNumber, - Message: bean3.ImageTagUnavailableMessage, + Message: pipelineConfigBean.ImageTagUnavailableMessage, } } if imageTagsDataMap[w.CiArtifactId] != nil { - wfResponse.ImageReleaseTags = imageTagsDataMap[w.CiArtifactId] //if artifact is not yet created,empty list will be sent + wfResponse.ImageReleaseTags = imageTagsDataMap[w.CiArtifactId] // if artifact is not yet created,empty list will be sent } if imageCommetnsDataMap[w.CiArtifactId] != nil { wfResponse.ImageComment = imageCommetnsDataMap[w.CiArtifactId] @@ -614,129 +412,6 @@ func (impl *CiHandlerImpl) GetBuildHistory(pipelineId int, appId int, offset int return ciWorkLowResponses, nil } -func (impl *CiHandlerImpl) CancelBuild(workflowId int, forceAbort bool) (int, error) { - workflow, err := impl.ciWorkflowRepository.FindById(workflowId) - if err != nil { - impl.Logger.Errorw("error in finding ci-workflow by workflow id", "ciWorkflowId", workflowId, "err", err) - return 0, err - } - isExt := workflow.Namespace != constants2.DefaultCiWorkflowNamespace - var env *repository2.Environment - var restConfig *rest.Config - if isExt { - restConfig, err = impl.getRestConfig(workflow) - if err != nil { - return 0, err - } - } - // Terminate workflow - cancelWfDtoRequest := &types.CancelWfRequestDto{ - ExecutorType: workflow.ExecutorType, - WorkflowName: workflow.Name, - Namespace: workflow.Namespace, - RestConfig: restConfig, - IsExt: isExt, - Environment: env, - } - // Terminate workflow - err = impl.workflowService.TerminateWorkflow(cancelWfDtoRequest) - if err != nil && forceAbort { - impl.Logger.Errorw("error in terminating workflow, with force abort flag flag as true", "workflowName", workflow.Name, "err", err) - - cancelWfDtoRequest.WorkflowGenerateName = fmt.Sprintf("%d-%s", workflowId, workflow.Name) - err1 := impl.workflowService.TerminateDanglingWorkflows(cancelWfDtoRequest) - if err1 != nil { - impl.Logger.Errorw("error in terminating dangling workflows", "cancelWfDtoRequest", cancelWfDtoRequest, "err", err) - // ignoring error here in case of force abort, confirmed from product - } - } else if err != nil && strings.Contains(err.Error(), "cannot find workflow") { - return 0, &util.ApiError{Code: "200", HttpStatusCode: http.StatusBadRequest, UserMessage: err.Error()} - } else if err != nil { - impl.Logger.Errorw("cannot terminate wf", "err", err) - return 0, err - } - if forceAbort { - err = impl.handleForceAbortCaseForCi(workflow, forceAbort) - if err != nil { - impl.Logger.Errorw("error in handleForceAbortCaseForCi", "forceAbortFlag", forceAbort, "workflow", workflow, "err", err) - return 0, err - } - return workflow.Id, nil - } - - workflow.Status = cdWorkflow.WorkflowCancel - if workflow.ExecutorType == cdWorkflow.WORKFLOW_EXECUTOR_TYPE_SYSTEM { - workflow.PodStatus = "Failed" - workflow.Message = constants2.TERMINATE_MESSAGE - } - err = impl.ciService.UpdateCiWorkflowWithStage(workflow) - if err != nil { - impl.Logger.Errorw("cannot update deleted workflow status, but wf deleted", "err", err) - return 0, err - } - imagePathReservationId := workflow.ImagePathReservationId - err = impl.customTagService.DeactivateImagePathReservation(imagePathReservationId) - if err != nil { - impl.Logger.Errorw("error in marking image tag unreserved", "err", err) - return 0, err - } - imagePathReservationIds := workflow.ImagePathReservationIds - if len(imagePathReservationIds) > 0 { - err = impl.customTagService.DeactivateImagePathReservationByImageIds(imagePathReservationIds) - if err != nil { - impl.Logger.Errorw("error in marking image tag unreserved", "err", err) - return 0, err - } - } - return workflow.Id, nil -} - -func (impl *CiHandlerImpl) handleForceAbortCaseForCi(workflow *pipelineConfig.CiWorkflow, forceAbort bool) error { - isWorkflowInNonTerminalStage := workflow.Status == string(v1alpha1.NodePending) || workflow.Status == string(v1alpha1.NodeRunning) - if !isWorkflowInNonTerminalStage { - if forceAbort { - return impl.updateWorkflowForForceAbort(workflow) - } else { - return &util.ApiError{Code: "200", HttpStatusCode: 400, UserMessage: "cannot cancel build, build not in progress"} - } - } - //this arises when someone deletes the workflow in resource browser and wants to force abort a ci - if workflow.Status == string(v1alpha1.NodeRunning) && forceAbort { - return impl.updateWorkflowForForceAbort(workflow) - } - return nil -} - -func (impl *CiHandlerImpl) updateWorkflowForForceAbort(workflow *pipelineConfig.CiWorkflow) error { - workflow.Status = cdWorkflow.WorkflowCancel - workflow.PodStatus = string(bean.Failed) - workflow.Message = constants2.FORCE_ABORT_MESSAGE_AFTER_STARTING_STAGE - err := impl.ciService.UpdateCiWorkflowWithStage(workflow) - if err != nil { - impl.Logger.Errorw("error in updating workflow status", "err", err) - return err - } - return nil -} - -func (impl *CiHandlerImpl) getRestConfig(workflow *pipelineConfig.CiWorkflow) (*rest.Config, error) { - env, err := impl.envRepository.FindById(workflow.EnvironmentId) - if err != nil { - impl.Logger.Errorw("could not fetch stage env", "err", err) - return nil, err - } - - clusterBean := adapter.GetClusterBean(*env.Cluster) - - clusterConfig := clusterBean.GetClusterConfig() - restConfig, err := impl.K8sUtil.GetRestConfigByCluster(clusterConfig) - if err != nil { - impl.Logger.Errorw("error in getting rest config by cluster id", "err", err) - return nil, err - } - return restConfig, nil -} - func (impl *CiHandlerImpl) FetchWorkflowDetails(appId int, pipelineId int, buildId int) (types.WorkflowResponse, error) { workflow, err := impl.ciWorkflowRepository.FindById(buildId) if err != nil { @@ -766,9 +441,9 @@ func (impl *CiHandlerImpl) FetchWorkflowDetails(appId int, pipelineId int, build return types.WorkflowResponse{}, err } - var ciMaterialsArr []pipelineConfig.CiPipelineMaterialResponse + var ciMaterialsArr []buildBean.CiPipelineMaterialResponse for _, m := range ciMaterials { - res := pipelineConfig.CiPipelineMaterialResponse{ + res := buildBean.CiPipelineMaterialResponse{ Id: m.Id, GitMaterialId: m.GitMaterialId, GitMaterialName: m.GitMaterial.Name[strings.Index(m.GitMaterial.Name, "-")+1:], @@ -794,6 +469,7 @@ func (impl *CiHandlerImpl) FetchWorkflowDetails(appId int, pipelineId int, build impl.ciWorkflowRepository.MigrateIsArtifactUploaded(workflow.Id, ciArtifact.IsArtifactUploaded) isArtifactUploaded = ciArtifact.IsArtifactUploaded } + wfStagesDetail, err := impl.workFlowStageStatusService.GetWorkflowStagesByWorkflowIdsAndWfType([]int{workflow.Id}, bean2.CI_WORKFLOW_TYPE.String()) if err != nil { impl.Logger.Errorw("error in fetching allWfStagesDetail", "err", err, "workflowId", workflow.Id) @@ -811,19 +487,19 @@ func (impl *CiHandlerImpl) FetchWorkflowDetails(appId int, pipelineId int, build CiPipelineId: workflow.CiPipelineId, Namespace: workflow.Namespace, LogLocation: workflow.LogLocation, - BlobStorageEnabled: workflow.BlobStorageEnabled, //TODO default value if value not found in db + BlobStorageEnabled: workflow.BlobStorageEnabled, // TODO default value if value not found in db GitTriggers: workflow.GitTriggers, CiMaterials: ciMaterialsArr, TriggeredBy: workflow.TriggeredBy, TriggeredByEmail: triggeredByUserEmailId, Artifact: ciArtifact.Image, - TargetPlatforms: utils.ConvertTargetPlatformStringToObject(ciArtifact.TargetPlatforms), ArtifactId: ciArtifact.Id, IsArtifactUploaded: isArtifactUploaded, EnvironmentId: workflow.EnvironmentId, EnvironmentName: environmentName, PipelineType: workflow.CiPipeline.PipelineType, PodName: workflow.PodName, + TargetPlatforms: utils.ConvertTargetPlatformStringToObject(ciArtifact.TargetPlatforms), WorkflowExecutionStage: impl.workFlowStageStatusService.ConvertDBWorkflowStageToMap(wfStagesDetail, workflow.Id, workflow.Status, workflow.PodStatus, workflow.Message, bean2.CI_WORKFLOW_TYPE.String(), workflow.StartedOn, workflow.FinishedOn), } return workflowResponse, nil @@ -840,278 +516,8 @@ func (impl *CiHandlerImpl) FetchArtifactsForCiJob(buildId int) (*types.Artifacts } return artifactsResponse, nil } -func (impl *CiHandlerImpl) GetRunningWorkflowLogs(workflowId int) (*bufio.Reader, func() error, error) { - ciWorkflow, err := impl.ciWorkflowRepository.FindById(workflowId) - if err != nil { - impl.Logger.Errorw("err", "err", err) - return nil, nil, err - } - return impl.getWorkflowLogs(ciWorkflow) -} - -func (impl *CiHandlerImpl) getWorkflowLogs(ciWorkflow *pipelineConfig.CiWorkflow) (*bufio.Reader, func() error, error) { - if string(v1alpha1.NodePending) == ciWorkflow.PodStatus { - return bufio.NewReader(strings.NewReader("")), func() error { return nil }, nil - } - ciLogRequest := types.BuildLogRequest{ - PodName: ciWorkflow.PodName, - Namespace: ciWorkflow.Namespace, - } - isExt := false - clusterConfig := &k8s.ClusterConfig{} - if ciWorkflow.EnvironmentId != 0 { - env, err := impl.envRepository.FindById(ciWorkflow.EnvironmentId) - if err != nil { - return nil, nil, err - } - var clusterBean bean5.ClusterBean - if env != nil && env.Cluster != nil { - clusterBean = adapter.GetClusterBean(*env.Cluster) - } - clusterConfig = clusterBean.GetClusterConfig() - isExt = true - } - - logStream, cleanUp, err := impl.ciLogService.FetchRunningWorkflowLogs(ciLogRequest, clusterConfig, isExt) - if logStream == nil || err != nil { - if !ciWorkflow.BlobStorageEnabled { - return nil, nil, &util.ApiError{Code: "200", HttpStatusCode: 400, UserMessage: "logs-not-stored-in-repository"} - } else if string(v1alpha1.NodeSucceeded) == ciWorkflow.Status || string(v1alpha1.NodeError) == ciWorkflow.Status || string(v1alpha1.NodeFailed) == ciWorkflow.Status || ciWorkflow.Status == cdWorkflow.WorkflowCancel { - impl.Logger.Debugw("pod is not live", "podName", ciWorkflow.PodName, "err", err) - return impl.getLogsFromRepository(ciWorkflow, clusterConfig, isExt) - } - if err != nil { - impl.Logger.Errorw("err on fetch workflow logs", "err", err) - return nil, nil, &util.ApiError{Code: "200", HttpStatusCode: 400, UserMessage: err.Error()} - } else if logStream == nil { - return nil, cleanUp, fmt.Errorf("no logs found for pod %s", ciWorkflow.PodName) - } - } - logReader := bufio.NewReader(logStream) - return logReader, cleanUp, err -} - -func (impl *CiHandlerImpl) getLogsFromRepository(ciWorkflow *pipelineConfig.CiWorkflow, clusterConfig *k8s.ClusterConfig, isExt bool) (*bufio.Reader, func() error, error) { - impl.Logger.Debug("getting historic logs", "ciWorkflowId", ciWorkflow.Id) - ciConfigLogsBucket := impl.config.GetDefaultBuildLogsBucket() - ciConfigCiCacheRegion := impl.config.DefaultCacheBucketRegion - logsFilePath := impl.config.GetDefaultBuildLogsKeyPrefix() + "/" + ciWorkflow.Name + "/main.log" // this is for backward compatibilty - if strings.Contains(ciWorkflow.LogLocation, "main.log") { - logsFilePath = ciWorkflow.LogLocation - } - ciLogRequest := types.BuildLogRequest{ - PipelineId: ciWorkflow.CiPipelineId, - WorkflowId: ciWorkflow.Id, - PodName: ciWorkflow.PodName, - LogsFilePath: logsFilePath, - CloudProvider: impl.config.CloudProvider, - AzureBlobConfig: &blob_storage.AzureBlobBaseConfig{ - Enabled: impl.config.CloudProvider == types.BLOB_STORAGE_AZURE, - AccountName: impl.config.AzureAccountName, - BlobContainerName: impl.config.AzureBlobContainerCiLog, - AccountKey: impl.config.AzureAccountKey, - }, - AwsS3BaseConfig: &blob_storage.AwsS3BaseConfig{ - AccessKey: impl.config.BlobStorageS3AccessKey, - Passkey: impl.config.BlobStorageS3SecretKey, - EndpointUrl: impl.config.BlobStorageS3Endpoint, - IsInSecure: impl.config.BlobStorageS3EndpointInsecure, - BucketName: ciConfigLogsBucket, - Region: ciConfigCiCacheRegion, - VersioningEnabled: impl.config.BlobStorageS3BucketVersioned, - }, - GcpBlobBaseConfig: &blob_storage.GcpBlobBaseConfig{ - BucketName: ciConfigLogsBucket, - CredentialFileJsonData: impl.config.BlobStorageGcpCredentialJson, - }, - } - useExternalBlobStorage := isExternalBlobStorageEnabled(isExt, impl.config.UseBlobStorageConfigInCiWorkflow) - if useExternalBlobStorage { - //fetch extClusterBlob cm and cs from k8s client, if they are present then read creds - //from them else return. - cmConfig, secretConfig, err := impl.blobConfigStorageService.FetchCmAndSecretBlobConfigFromExternalCluster(clusterConfig, ciWorkflow.Namespace) - if err != nil { - impl.Logger.Errorw("error in fetching config map and secret from external cluster", "err", err, "clusterConfig", clusterConfig) - return nil, nil, err - } - rq := &ciLogRequest - rq.SetBuildLogRequest(cmConfig, secretConfig) - } - oldLogsStream, cleanUp, err := impl.ciLogService.FetchLogs(impl.config.BaseLogLocationPath, ciLogRequest) - if err != nil { - impl.Logger.Errorw("err", "err", err) - return nil, nil, err - } - logReader := bufio.NewReader(oldLogsStream) - return logReader, cleanUp, err -} - -func (impl *CiHandlerImpl) DownloadCiWorkflowArtifacts(pipelineId int, buildId int) (*os.File, error) { - ciWorkflow, err := impl.ciWorkflowRepository.FindById(buildId) - if err != nil { - impl.Logger.Errorw("unable to fetch ciWorkflow", "err", err) - return nil, err - } - useExternalBlobStorage := isExternalBlobStorageEnabled(ciWorkflow.IsExternalRunInJobType(), impl.config.UseBlobStorageConfigInCiWorkflow) - if !ciWorkflow.BlobStorageEnabled { - return nil, errors.New("logs-not-stored-in-repository") - } - - if ciWorkflow.CiPipelineId != pipelineId { - impl.Logger.Error("invalid request, wf not in pipeline") - return nil, errors.New("invalid request, wf not in pipeline") - } - - ciConfigLogsBucket := impl.config.GetDefaultBuildLogsBucket() - item := strconv.Itoa(ciWorkflow.Id) - ciConfigCiCacheRegion := impl.config.DefaultCacheBucketRegion - azureBlobConfig := &blob_storage.AzureBlobBaseConfig{ - Enabled: impl.config.CloudProvider == types.BLOB_STORAGE_AZURE, - AccountName: impl.config.AzureAccountName, - BlobContainerName: impl.config.AzureBlobContainerCiLog, - AccountKey: impl.config.AzureAccountKey, - } - awsS3BaseConfig := &blob_storage.AwsS3BaseConfig{ - AccessKey: impl.config.BlobStorageS3AccessKey, - Passkey: impl.config.BlobStorageS3SecretKey, - EndpointUrl: impl.config.BlobStorageS3Endpoint, - IsInSecure: impl.config.BlobStorageS3EndpointInsecure, - BucketName: ciConfigLogsBucket, - Region: ciConfigCiCacheRegion, - VersioningEnabled: impl.config.BlobStorageS3BucketVersioned, - } - gcpBlobBaseConfig := &blob_storage.GcpBlobBaseConfig{ - BucketName: ciConfigLogsBucket, - CredentialFileJsonData: impl.config.BlobStorageGcpCredentialJson, - } - - ciArtifactLocationFormat := impl.config.GetArtifactLocationFormat() - key := fmt.Sprintf(ciArtifactLocationFormat, ciWorkflow.Id, ciWorkflow.Id) - if len(ciWorkflow.CiArtifactLocation) != 0 && util3.IsValidUrlSubPath(ciWorkflow.CiArtifactLocation) { - key = ciWorkflow.CiArtifactLocation - } else if util3.IsValidUrlSubPath(key) { - impl.ciWorkflowRepository.MigrateCiArtifactLocation(ciWorkflow.Id, key) - } - baseLogLocationPathConfig := impl.config.BaseLogLocationPath - blobStorageService := blob_storage.NewBlobStorageServiceImpl(nil) - destinationKey := filepath.Clean(filepath.Join(baseLogLocationPathConfig, item)) - request := &blob_storage.BlobStorageRequest{ - StorageType: impl.config.CloudProvider, - SourceKey: key, - DestinationKey: destinationKey, - AzureBlobBaseConfig: azureBlobConfig, - AwsS3BaseConfig: awsS3BaseConfig, - GcpBlobBaseConfig: gcpBlobBaseConfig, - } - if useExternalBlobStorage { - envBean, err := impl.envService.FindById(ciWorkflow.EnvironmentId) - if err != nil { - impl.Logger.Errorw("error in getting envBean by envId", "err", err, "envId", ciWorkflow.EnvironmentId) - return nil, err - } - clusterConfig, err := impl.clusterService.GetClusterConfigByClusterId(envBean.ClusterId) - if err != nil { - impl.Logger.Errorw("GetClusterConfigByClusterId, error in fetching clusterConfig by clusterId", "err", err, "clusterId", envBean.ClusterId) - return nil, err - } - //fetch extClusterBlob cm and cs from k8s client, if they are present then read creds - //from them else return. - cmConfig, secretConfig, err := impl.blobConfigStorageService.FetchCmAndSecretBlobConfigFromExternalCluster(clusterConfig, ciWorkflow.Namespace) - if err != nil { - impl.Logger.Errorw("error in fetching config map and secret from external cluster", "err", err, "clusterConfig", clusterConfig) - return nil, err - } - request = updateRequestWithExtClusterCmAndSecret(request, cmConfig, secretConfig) - } - _, numBytes, err := blobStorageService.Get(request) - if err != nil { - impl.Logger.Errorw("error occurred while downloading file", "request", request, "error", err) - return nil, errors.New("failed to download resource") - } - - file, err := os.Open(destinationKey) - if err != nil { - impl.Logger.Errorw("unable to open file", "file", item, "err", err) - return nil, errors.New("unable to open file") - } - - impl.Logger.Infow("Downloaded ", "filename", file.Name(), "bytes", numBytes) - return file, nil -} - -func (impl *CiHandlerImpl) GetHistoricBuildLogs(workflowId int, ciWorkflow *pipelineConfig.CiWorkflow) (map[string]string, error) { - var err error - if ciWorkflow == nil { - ciWorkflow, err = impl.ciWorkflowRepository.FindById(workflowId) - if err != nil { - impl.Logger.Errorw("err", "err", err) - return nil, err - } - } - ciConfigLogsBucket := impl.config.GetDefaultBuildLogsBucket() - ciConfigCiCacheRegion := impl.config.DefaultCacheBucketRegion - ciLogRequest := types.BuildLogRequest{ - PipelineId: ciWorkflow.CiPipelineId, - WorkflowId: ciWorkflow.Id, - PodName: ciWorkflow.PodName, - LogsFilePath: ciWorkflow.LogLocation, - CloudProvider: impl.config.CloudProvider, - AzureBlobConfig: &blob_storage.AzureBlobBaseConfig{ - Enabled: impl.config.CloudProvider == types.BLOB_STORAGE_AZURE, - AccountName: impl.config.AzureAccountName, - BlobContainerName: impl.config.AzureBlobContainerCiLog, - AccountKey: impl.config.AzureAccountKey, - }, - AwsS3BaseConfig: &blob_storage.AwsS3BaseConfig{ - AccessKey: impl.config.BlobStorageS3AccessKey, - Passkey: impl.config.BlobStorageS3SecretKey, - EndpointUrl: impl.config.BlobStorageS3Endpoint, - IsInSecure: impl.config.BlobStorageS3EndpointInsecure, - BucketName: ciConfigLogsBucket, - Region: ciConfigCiCacheRegion, - VersioningEnabled: impl.config.BlobStorageS3BucketVersioned, - }, - GcpBlobBaseConfig: &blob_storage.GcpBlobBaseConfig{ - BucketName: ciConfigLogsBucket, - CredentialFileJsonData: impl.config.BlobStorageGcpCredentialJson, - }, - } - useExternalBlobStorage := isExternalBlobStorageEnabled(ciWorkflow.IsExternalRunInJobType(), impl.config.UseBlobStorageConfigInCiWorkflow) - if useExternalBlobStorage { - envBean, err := impl.envService.FindById(ciWorkflow.EnvironmentId) - if err != nil { - impl.Logger.Errorw("error in getting envBean by envId", "err", err, "envId", ciWorkflow.EnvironmentId) - return nil, err - } - clusterConfig, err := impl.clusterService.GetClusterConfigByClusterId(envBean.ClusterId) - if err != nil { - impl.Logger.Errorw("GetClusterConfigByClusterId, error in fetching clusterConfig by clusterId", "err", err, "clusterId", envBean.ClusterId) - return nil, err - } - //fetch extClusterBlob cm and cs from k8s client, if they are present then read creds - //from them else return. - cmConfig, secretConfig, err := impl.blobConfigStorageService.FetchCmAndSecretBlobConfigFromExternalCluster(clusterConfig, ciWorkflow.Namespace) - if err != nil { - impl.Logger.Errorw("error in fetching config map and secret from external cluster", "err", err, "clusterConfig", clusterConfig) - return nil, err - } - rq := &ciLogRequest - rq.SetBuildLogRequest(cmConfig, secretConfig) - } - logsFile, cleanUp, err := impl.ciLogService.FetchLogs(impl.config.BaseLogLocationPath, ciLogRequest) - logs, err := ioutil.ReadFile(logsFile.Name()) - if err != nil { - impl.Logger.Errorw("err", "err", err) - return map[string]string{}, err - } - logStr := string(logs) - resp := make(map[string]string) - resp["logs"] = logStr - defer cleanUp() - return resp, err -} -func ExtractWorkflowStatus(workflowStatus v1alpha1.WorkflowStatus) (string, string, string, string, string, string) { +func ExtractWorkflowStatus(workflowStatus eventProcessorBean.CiCdStatus) (string, string, string, string, string, string) { workflowName := "" status := string(workflowStatus.Phase) podStatus := "" @@ -1119,7 +525,7 @@ func ExtractWorkflowStatus(workflowStatus v1alpha1.WorkflowStatus) (string, stri podName := "" logLocation := "" for k, v := range workflowStatus.Nodes { - if v.TemplateName == bean3.CI_WORKFLOW_NAME { + if v.TemplateName == pipelineConfigBean.CI_WORKFLOW_NAME { if v.BoundaryID == "" { workflowName = k } else { @@ -1141,92 +547,60 @@ func ExtractWorkflowStatus(workflowStatus v1alpha1.WorkflowStatus) (string, stri return workflowName, status, podStatus, message, logLocation, podName } -func (impl *CiHandlerImpl) extractPodStatusAndWorkflow(workflowStatus v1alpha1.WorkflowStatus) (string, string, *pipelineConfig.CiWorkflow, error) { - workflowName, status, _, message, _, _ := ExtractWorkflowStatus(workflowStatus) - if workflowName == "" { - impl.Logger.Errorw("extract workflow status, invalid wf name", "workflowName", workflowName, "status", status, "message", message) - return status, message, nil, errors.New("invalid wf name") - } - workflowId, err := strconv.Atoi(workflowName[:strings.Index(workflowName, "-")]) - if err != nil { - impl.Logger.Errorw("extract workflowId, invalid wf name", "workflowName", workflowName, "err", err) - return status, message, nil, err - } - - savedWorkflow, err := impl.ciWorkflowRepository.FindById(workflowId) - if err != nil { - impl.Logger.Errorw("cannot get saved wf", "workflowId", workflowId, "err", err) - return status, message, nil, err - } - - return status, message, savedWorkflow, err - -} - -func (impl *CiHandlerImpl) getRefWorkflowAndCiRetryCount(savedWorkflow *pipelineConfig.CiWorkflow) (int, *pipelineConfig.CiWorkflow, error) { - var err error - - if savedWorkflow.ReferenceCiWorkflowId != 0 { - savedWorkflow, err = impl.ciWorkflowRepository.FindById(savedWorkflow.ReferenceCiWorkflowId) - } - if err != nil { - impl.Logger.Errorw("cannot get saved wf", "err", err) - return 0, savedWorkflow, err - } - retryCount, err := impl.ciWorkflowRepository.FindRetriedWorkflowCountByReferenceId(savedWorkflow.Id) - return retryCount, savedWorkflow, err -} - -func (impl *CiHandlerImpl) UpdateWorkflow(workflowStatus v1alpha1.WorkflowStatus) (int, error) { +func (impl *CiHandlerImpl) UpdateWorkflow(workflowStatus eventProcessorBean.CiCdStatus) (int, bool, error) { workflowName, status, podStatus, message, _, podName := ExtractWorkflowStatus(workflowStatus) if workflowName == "" { impl.Logger.Errorw("extract workflow status, invalid wf name", "workflowName", workflowName, "status", status, "podStatus", podStatus, "message", message) - return 0, errors.New("invalid wf name") + return 0, false, errors.New("invalid wf name") } workflowId, err := strconv.Atoi(workflowName[:strings.Index(workflowName, "-")]) if err != nil { impl.Logger.Errorw("invalid wf status update req", "err", err) - return 0, err + return 0, false, err } savedWorkflow, err := impl.ciWorkflowRepository.FindById(workflowId) if err != nil { impl.Logger.Errorw("cannot get saved wf", "err", err) - return 0, err + return 0, false, err } - + impl.updateResourceStatusInCache(workflowId, podName, savedWorkflow.Namespace, status) ciArtifactLocationFormat := impl.config.GetArtifactLocationFormat() ciArtifactLocation := fmt.Sprintf(ciArtifactLocationFormat, savedWorkflow.Id, savedWorkflow.Id) if impl.stateChanged(status, podStatus, message, workflowStatus.FinishedAt.Time, savedWorkflow) { - if savedWorkflow.Status != cdWorkflow.WorkflowCancel { + if !slices.Contains(cdWorkflowBean.WfrTerminalStatusList, savedWorkflow.PodStatus) { + savedWorkflow.Message = message + if !slices.Contains(cdWorkflowBean.WfrTerminalStatusList, savedWorkflow.Status) { + savedWorkflow.FinishedOn = workflowStatus.FinishedAt.Time + } + } else { + impl.Logger.Warnw("cd stage already in terminal state. skipped message and finishedOn from being updated", + "wfId", savedWorkflow.Id, "podStatus", savedWorkflow.PodStatus, "status", savedWorkflow.Status, "message", message, "finishedOn", workflowStatus.FinishedAt.Time) + } + if savedWorkflow.Status != cdWorkflowBean.WorkflowCancel { savedWorkflow.Status = status } savedWorkflow.PodStatus = podStatus - savedWorkflow.Message = message - // NOTE: we are doing this for a quick fix where ci pending message become larger than 250 and in db we had set the charter limit to 250 - if len(message) > 250 { - savedWorkflow.Message = message[:250] - } - if savedWorkflow.ExecutorType == cdWorkflow.WORKFLOW_EXECUTOR_TYPE_SYSTEM && savedWorkflow.Status == cdWorkflow.WorkflowCancel { + if savedWorkflow.ExecutorType == cdWorkflowBean.WORKFLOW_EXECUTOR_TYPE_SYSTEM && savedWorkflow.Status == cdWorkflowBean.WorkflowCancel { savedWorkflow.PodStatus = "Failed" - savedWorkflow.Message = constants2.TERMINATE_MESSAGE + savedWorkflow.Message = constants.TERMINATE_MESSAGE } - savedWorkflow.FinishedOn = workflowStatus.FinishedAt.Time savedWorkflow.Name = workflowName - //savedWorkflow.LogLocation = "/ci-pipeline/" + strconv.Itoa(savedWorkflow.CiPipelineId) + "/workflow/" + strconv.Itoa(savedWorkflow.Id) + "/logs" //TODO need to fetch from workflow object - //savedWorkflow.LogLocation = logLocation // removed because we are saving log location at trigger + // savedWorkflow.LogLocation = "/ci-pipeline/" + strconv.Itoa(savedWorkflow.CiPipelineId) + "/workflow/" + strconv.Itoa(savedWorkflow.Id) + "/logs" //TODO need to fetch from workflow object + // savedWorkflow.LogLocation = logLocation // removed because we are saving log location at trigger savedWorkflow.CiArtifactLocation = ciArtifactLocation savedWorkflow.PodName = podName impl.Logger.Debugw("updating workflow ", "workflow", savedWorkflow) err = impl.ciService.UpdateCiWorkflowWithStage(savedWorkflow) if err != nil { impl.Logger.Error("update wf failed for id " + strconv.Itoa(savedWorkflow.Id)) - return 0, err + return savedWorkflow.Id, true, err } impl.sendCIFailEvent(savedWorkflow, status, message) + return savedWorkflow.Id, true, nil } - return savedWorkflow.Id, nil + return savedWorkflow.Id, false, nil } func (impl *CiHandlerImpl) sendCIFailEvent(savedWorkflow *pipelineConfig.CiWorkflow, status, message string) { @@ -1238,7 +612,7 @@ func (impl *CiHandlerImpl) sendCIFailEvent(savedWorkflow *pipelineConfig.CiWorkf impl.Logger.Warnw("ci failed for workflow: ", "wfId", savedWorkflow.Id) if extractErrorCode(savedWorkflow.Message) != workFlow.CiStageFailErrorCode { - go impl.WriteCIFailEvent(savedWorkflow) + impl.ciService.WriteCIFailEvent(savedWorkflow) } else { impl.Logger.Infof("Step failed notification received for wfID %d with message %s", savedWorkflow.Id, savedWorkflow.Message) } @@ -1257,27 +631,10 @@ func extractErrorCode(msg string) int { return -1 } -func (impl *CiHandlerImpl) WriteCIFailEvent(ciWorkflow *pipelineConfig.CiWorkflow) { - event, _ := impl.eventFactory.Build(util2.Fail, &ciWorkflow.CiPipelineId, ciWorkflow.CiPipeline.AppId, nil, util2.CI) - material := &client.MaterialTriggerInfo{} - material.GitTriggers = ciWorkflow.GitTriggers - event.CiWorkflowRunnerId = ciWorkflow.Id - event.UserId = int(ciWorkflow.TriggeredBy) - event = impl.eventFactory.BuildExtraCIData(event, material) - event.CiArtifactId = 0 - _, evtErr := impl.eventClient.WriteNotificationEvent(event) - if evtErr != nil { - impl.Logger.Errorw("error in writing event", "err", evtErr) - } -} - func (impl *CiHandlerImpl) BuildPayload(ciWorkflow *pipelineConfig.CiWorkflow) *client.Payload { payload := &client.Payload{} payload.AppName = ciWorkflow.CiPipeline.App.AppName payload.PipelineName = ciWorkflow.CiPipeline.Name - //payload["buildName"] = ciWorkflow.Name - //payload["podStatus"] = ciWorkflow.PodStatus - //payload["message"] = ciWorkflow.Message return payload } @@ -1286,191 +643,6 @@ func (impl *CiHandlerImpl) stateChanged(status string, podStatus string, msg str return savedWorkflow.Status != status || savedWorkflow.PodStatus != podStatus || savedWorkflow.Message != msg || savedWorkflow.FinishedOn != finishedAt } -func (impl *CiHandlerImpl) GetCiPipeline(ciMaterialId int) (*pipelineConfig.CiPipeline, error) { - ciMaterial, err := impl.ciPipelineMaterialRepository.GetById(ciMaterialId) - if err != nil { - return nil, err - } - ciPipeline := ciMaterial.CiPipeline - return ciPipeline, nil -} - -func (impl *CiHandlerImpl) buildAutomaticTriggerCommitHashes(ciMaterials []*pipelineConfig.CiPipelineMaterial, request bean.GitCiTriggerRequest) (map[int]pipelineConfig.GitCommit, error) { - commitHashes := map[int]pipelineConfig.GitCommit{} - for _, ciMaterial := range ciMaterials { - if ciMaterial.Id == request.CiPipelineMaterial.Id || len(ciMaterials) == 1 { - request.CiPipelineMaterial.GitCommit = SetGitCommitValuesForBuildingCommitHash(ciMaterial, request.CiPipelineMaterial.GitCommit) - commitHashes[ciMaterial.Id] = request.CiPipelineMaterial.GitCommit - } else { - // this is possible in case of non Webhook, as there would be only one pipeline material per git material in case of PR - lastCommit, err := impl.getLastSeenCommit(ciMaterial.Id) - if err != nil { - return map[int]pipelineConfig.GitCommit{}, err - } - lastCommit = SetGitCommitValuesForBuildingCommitHash(ciMaterial, lastCommit) - commitHashes[ciMaterial.Id] = lastCommit - } - } - return commitHashes, nil -} - -func SetGitCommitValuesForBuildingCommitHash(ciMaterial *pipelineConfig.CiPipelineMaterial, oldGitCommit pipelineConfig.GitCommit) pipelineConfig.GitCommit { - newGitCommit := oldGitCommit - newGitCommit.CiConfigureSourceType = ciMaterial.Type - newGitCommit.CiConfigureSourceValue = ciMaterial.Value - newGitCommit.GitRepoUrl = ciMaterial.GitMaterial.Url - newGitCommit.GitRepoName = ciMaterial.GitMaterial.Name[strings.Index(ciMaterial.GitMaterial.Name, "-")+1:] - return newGitCommit -} - -func (impl *CiHandlerImpl) buildManualTriggerCommitHashes(ciTriggerRequest bean.CiTriggerRequest) (map[int]pipelineConfig.GitCommit, *common.RuntimeParameters, error) { - commitHashes := map[int]pipelineConfig.GitCommit{} - runtimeParams := common.NewRuntimeParameters() - for _, ciPipelineMaterial := range ciTriggerRequest.CiPipelineMaterial { - - pipeLineMaterialFromDb, err := impl.ciPipelineMaterialRepository.GetById(ciPipelineMaterial.Id) - if err != nil { - impl.Logger.Errorw("err in fetching pipeline material by id", "err", err) - return map[int]pipelineConfig.GitCommit{}, nil, err - } - - pipelineType := pipeLineMaterialFromDb.Type - if pipelineType == constants.SOURCE_TYPE_BRANCH_FIXED { - gitCommit, err := impl.BuildManualTriggerCommitHashesForSourceTypeBranchFix(ciPipelineMaterial, pipeLineMaterialFromDb) - if err != nil { - impl.Logger.Errorw("err", "err", err) - return map[int]pipelineConfig.GitCommit{}, nil, err - } - commitHashes[ciPipelineMaterial.Id] = gitCommit - - } else if pipelineType == constants.SOURCE_TYPE_WEBHOOK { - gitCommit, extraEnvVariables, err := impl.BuildManualTriggerCommitHashesForSourceTypeWebhook(ciPipelineMaterial, pipeLineMaterialFromDb) - if err != nil { - impl.Logger.Errorw("err", "err", err) - return map[int]pipelineConfig.GitCommit{}, nil, err - } - commitHashes[ciPipelineMaterial.Id] = gitCommit - for key, value := range extraEnvVariables { - runtimeParams = runtimeParams.AddSystemVariable(key, value) - } - } - } - return commitHashes, runtimeParams, nil -} - -func (impl *CiHandlerImpl) BuildManualTriggerCommitHashesForSourceTypeBranchFix(ciPipelineMaterial bean.CiPipelineMaterial, pipeLineMaterialFromDb *pipelineConfig.CiPipelineMaterial) (pipelineConfig.GitCommit, error) { - commitMetadataRequest := &gitSensor.CommitMetadataRequest{ - PipelineMaterialId: ciPipelineMaterial.Id, - GitHash: ciPipelineMaterial.GitCommit.Commit, - GitTag: ciPipelineMaterial.GitTag, - } - gitCommitResponse, err := impl.gitSensorClient.GetCommitMetadataForPipelineMaterial(context.Background(), commitMetadataRequest) - if err != nil { - impl.Logger.Errorw("err in fetching commit metadata", "commitMetadataRequest", commitMetadataRequest, "err", err) - return pipelineConfig.GitCommit{}, err - } - if gitCommitResponse == nil { - return pipelineConfig.GitCommit{}, errors.New("commit not found") - } - - gitCommit := pipelineConfig.GitCommit{ - Commit: gitCommitResponse.Commit, - Author: gitCommitResponse.Author, - Date: gitCommitResponse.Date, - Message: gitCommitResponse.Message, - Changes: gitCommitResponse.Changes, - GitRepoName: pipeLineMaterialFromDb.GitMaterial.Name[strings.Index(pipeLineMaterialFromDb.GitMaterial.Name, "-")+1:], - GitRepoUrl: pipeLineMaterialFromDb.GitMaterial.Url, - CiConfigureSourceValue: pipeLineMaterialFromDb.Value, - CiConfigureSourceType: pipeLineMaterialFromDb.Type, - } - - return gitCommit, nil -} - -func (impl *CiHandlerImpl) BuildManualTriggerCommitHashesForSourceTypeWebhook(ciPipelineMaterial bean.CiPipelineMaterial, pipeLineMaterialFromDb *pipelineConfig.CiPipelineMaterial) (pipelineConfig.GitCommit, map[string]string, error) { - webhookDataInput := ciPipelineMaterial.GitCommit.WebhookData - - // fetch webhook data on the basis of Id - webhookDataRequest := &gitSensor.WebhookDataRequest{ - Id: webhookDataInput.Id, - CiPipelineMaterialId: ciPipelineMaterial.Id, - } - - webhookAndCiData, err := impl.gitSensorClient.GetWebhookData(context.Background(), webhookDataRequest) - if err != nil { - impl.Logger.Errorw("err", "err", err) - return pipelineConfig.GitCommit{}, nil, err - } - webhookData := webhookAndCiData.WebhookData - - // if webhook event is of merged type, then fetch latest commit for target branch - if webhookData.EventActionType == bean.WEBHOOK_EVENT_MERGED_ACTION_TYPE { - - // get target branch name from webhook - targetBranchName := webhookData.Data[bean.WEBHOOK_SELECTOR_TARGET_BRANCH_NAME_NAME] - if targetBranchName == "" { - impl.Logger.Error("target branch not found from webhook data") - return pipelineConfig.GitCommit{}, nil, err - } - - // get latest commit hash for target branch - latestCommitMetadataRequest := &gitSensor.CommitMetadataRequest{ - PipelineMaterialId: ciPipelineMaterial.Id, - BranchName: targetBranchName, - } - - latestCommit, err := impl.gitSensorClient.GetCommitMetadata(context.Background(), latestCommitMetadataRequest) - - if err != nil { - impl.Logger.Errorw("err", "err", err) - return pipelineConfig.GitCommit{}, nil, err - } - - // update webhookData (local) with target latest hash - webhookData.Data[bean.WEBHOOK_SELECTOR_TARGET_CHECKOUT_NAME] = latestCommit.Commit - - } - - // build git commit - gitCommit := pipelineConfig.GitCommit{ - GitRepoName: pipeLineMaterialFromDb.GitMaterial.Name[strings.Index(pipeLineMaterialFromDb.GitMaterial.Name, "-")+1:], - GitRepoUrl: pipeLineMaterialFromDb.GitMaterial.Url, - CiConfigureSourceValue: pipeLineMaterialFromDb.Value, - CiConfigureSourceType: pipeLineMaterialFromDb.Type, - WebhookData: pipelineConfig.WebhookData{ - Id: int(webhookData.Id), - EventActionType: webhookData.EventActionType, - Data: webhookData.Data, - }, - } - - return gitCommit, webhookAndCiData.ExtraEnvironmentVariables, nil -} - -func (impl *CiHandlerImpl) getLastSeenCommit(ciMaterialId int) (pipelineConfig.GitCommit, error) { - var materialIds []int - materialIds = append(materialIds, ciMaterialId) - headReq := &gitSensor.HeadRequest{ - MaterialIds: materialIds, - } - res, err := impl.gitSensorClient.GetHeadForPipelineMaterials(context.Background(), headReq) - if err != nil { - return pipelineConfig.GitCommit{}, err - } - if len(res) == 0 { - return pipelineConfig.GitCommit{}, errors.New("received empty response") - } - gitCommit := pipelineConfig.GitCommit{ - Commit: res[0].GitCommit.Commit, - Author: res[0].GitCommit.Author, - Date: res[0].GitCommit.Date, - Message: res[0].GitCommit.Message, - Changes: res[0].GitCommit.Changes, - } - return gitCommit, nil -} - func (impl *CiHandlerImpl) FetchCiStatusForTriggerViewV1(appId int) ([]*pipelineConfig.CiWorkflowStatus, error) { ciWorkflowStatuses, err := impl.ciWorkflowRepository.FIndCiWorkflowStatusesByAppId(appId) if err != nil && !util.IsErrNoRows(err) { @@ -1490,12 +662,7 @@ func (impl *CiHandlerImpl) FetchCiStatusForTriggerView(appId int) ([]*pipelineCo return ciWorkflowStatuses, err } for _, pipeline := range pipelines { - pipelineId := 0 - if pipeline.ParentCiPipeline == 0 { - pipelineId = pipeline.Id - } else { - pipelineId = pipeline.ParentCiPipeline - } + pipelineId := impl.getPipelineIdForTriggerView(pipeline) workflow, err := impl.ciWorkflowRepository.FindLastTriggeredWorkflow(pipelineId) if err != nil && !util.IsErrNoRows(err) { impl.Logger.Errorw("err", "pipelineId", pipelineId, "err", err) @@ -1540,7 +707,7 @@ func (impl *CiHandlerImpl) FetchMaterialInfoByArtifactId(ciArtifactId int, envId return &types.GitTriggerInfoResponse{}, err } - ciMaterialsArr := make([]pipelineConfig.CiPipelineMaterialResponse, 0) + ciMaterialsArr := make([]buildBean.CiPipelineMaterialResponse, 0) var triggeredByUserEmailId string //check workflow data only for non external builds if !ciPipeline.IsExternal { @@ -1596,7 +763,7 @@ func (impl *CiHandlerImpl) FetchMaterialInfoByArtifactId(ciArtifactId int, envId history = append(history, _gitCommit) - res := pipelineConfig.CiPipelineMaterialResponse{ + res := buildBean.CiPipelineMaterialResponse{ Id: m.Id, GitMaterialId: m.GitMaterialId, GitMaterialName: m.GitMaterial.Name[strings.Index(m.GitMaterial.Name, "-")+1:], @@ -1618,6 +785,7 @@ func (impl *CiHandlerImpl) FetchMaterialInfoByArtifactId(ciArtifactId int, envId //GitTriggers: workflow.GitTriggers, CiMaterials: ciMaterialsArr, TriggeredByEmail: triggeredByUserEmailId, + CiPipelineId: ciPipeline.Id, AppId: ciPipeline.AppId, AppName: deployDetail.AppName, EnvironmentId: deployDetail.EnvironmentId, @@ -1631,127 +799,6 @@ func (impl *CiHandlerImpl) FetchMaterialInfoByArtifactId(ciArtifactId int, envId return gitTriggerInfoResponse, nil } -func (impl *CiHandlerImpl) UpdateCiWorkflowStatusFailure(timeoutForFailureCiBuild int) error { - ciWorkflows, err := impl.ciWorkflowRepository.FindByStatusesIn([]string{constants2.Starting, constants2.Running}) - if err != nil { - impl.Logger.Errorw("error on fetching ci workflows", "err", err) - return err - } - client, err := impl.K8sUtil.GetClientForInCluster() - if err != nil { - impl.Logger.Errorw("error while fetching k8s client", "error", err) - return err - } - - for _, ciWorkflow := range ciWorkflows { - var isExt bool - var env *repository2.Environment - var restConfig *rest.Config - if ciWorkflow.Namespace != constants2.DefaultCiWorkflowNamespace { - isExt = true - env, err = impl.envRepository.FindById(ciWorkflow.EnvironmentId) - if err != nil { - impl.Logger.Errorw("could not fetch stage env", "err", err) - return err - } - restConfig, err = impl.getRestConfig(ciWorkflow) - if err != nil { - return err - } - } - - isEligibleToMarkFailed := false - isPodDeleted := false - if time.Since(ciWorkflow.StartedOn) > (time.Minute * time.Duration(timeoutForFailureCiBuild)) { - - //check weather pod is exists or not, if exits check its status - wf, err := impl.workflowService.GetWorkflowStatus(ciWorkflow.ExecutorType, ciWorkflow.Name, ciWorkflow.Namespace, restConfig) - if err != nil { - impl.Logger.Warnw("unable to fetch ci workflow", "err", err) - statusError, ok := err.(*errors2.StatusError) - if ok && statusError.Status().Code == http.StatusNotFound { - impl.Logger.Warnw("ci workflow not found", "err", err) - isEligibleToMarkFailed = true - } else { - continue - // skip this and process for next ci workflow - } - } - - //if ci workflow is exists, check its pod - if !isEligibleToMarkFailed { - ns := constants2.DefaultCiWorkflowNamespace - if isExt { - _, client, err = impl.k8sCommonService.GetCoreClientByClusterId(env.ClusterId) - if err != nil { - impl.Logger.Warnw("error in getting core v1 client using GetCoreClientByClusterId", "err", err, "clusterId", env.Cluster.Id) - continue - } - ns = env.Namespace - } - _, err := impl.K8sUtil.GetPodByName(ns, ciWorkflow.PodName, client) - if err != nil { - impl.Logger.Warnw("unable to fetch ci workflow - pod", "err", err) - statusError, ok := err.(*errors2.StatusError) - if ok && statusError.Status().Code == http.StatusNotFound { - impl.Logger.Warnw("pod not found", "err", err) - isEligibleToMarkFailed = true - } else { - continue - // skip this and process for next ci workflow - } - } - if ciWorkflow.ExecutorType == cdWorkflow.WORKFLOW_EXECUTOR_TYPE_SYSTEM { - if wf.Status == string(v1alpha1.WorkflowFailed) { - isPodDeleted = true - } - } else { - //check workflow status,get the status - if wf.Status == string(v1alpha1.WorkflowFailed) && wf.Message == constants2.POD_DELETED_MESSAGE { - isPodDeleted = true - } - } - } - } - if isEligibleToMarkFailed { - ciWorkflow.Status = "Failed" - ciWorkflow.PodStatus = "Failed" - if isPodDeleted { - ciWorkflow.Message = cdWorkflow.POD_DELETED_MESSAGE - //error logging handled inside handlePodDeleted - impl.handlePodDeleted(ciWorkflow) - } else { - ciWorkflow.Message = "marked failed by job" - } - err := impl.ciService.UpdateCiWorkflowWithStage(ciWorkflow) - if err != nil { - impl.Logger.Errorw("unable to update ci workflow, its eligible to mark failed", "err", err) - // skip this and process for next ci workflow - } - err = impl.customTagService.DeactivateImagePathReservation(ciWorkflow.ImagePathReservationId) - if err != nil { - impl.Logger.Errorw("unable to update ci workflow, its eligible to mark failed", "err", err) - } - } - } - return nil -} - -func (impl *CiHandlerImpl) handlePodDeleted(ciWorkflow *pipelineConfig.CiWorkflow) { - if !impl.config.WorkflowRetriesEnabled() { - impl.Logger.Debug("ci workflow retry feature disabled") - return - } - retryCount, refCiWorkflow, err := impl.getRefWorkflowAndCiRetryCount(ciWorkflow) - if err != nil { - impl.Logger.Errorw("error in getRefWorkflowAndCiRetryCount", "ciWorkflowId", ciWorkflow.Id, "err", err) - } - impl.Logger.Infow("re-triggering ci by UpdateCiWorkflowStatusFailedCron", "refCiWorkflowId", refCiWorkflow.Id, "ciWorkflow.Status", ciWorkflow.Status, "ciWorkflow.Message", ciWorkflow.Message, "retryCount", retryCount) - err = impl.reTriggerCi(retryCount, refCiWorkflow) - if err != nil { - impl.Logger.Errorw("error in reTriggerCi", "ciWorkflowId", refCiWorkflow.Id, "workflowStatus", ciWorkflow.Status, "ciWorkflowMessage", "ciWorkflow.Message", "retryCount", retryCount, "err", err) - } -} func (impl *CiHandlerImpl) FetchCiStatusForTriggerViewForEnvironment(request resourceGroup.ResourceGroupingRequest, token string) ([]*pipelineConfig.CiWorkflowStatus, error) { ciWorkflowStatuses := make([]*pipelineConfig.CiWorkflowStatus, 0) var cdPipelines []*pipelineConfig.Pipeline @@ -1761,7 +808,7 @@ func (impl *CiHandlerImpl) FetchCiStatusForTriggerViewForEnvironment(request res if err != nil { return nil, err } - //override appIds if already provided app group id in request. + // override appIds if already provided app group id in request. request.ResourceIds = appIds } if len(request.ResourceIds) > 0 { @@ -1794,7 +841,7 @@ func (impl *CiHandlerImpl) FetchCiStatusForTriggerViewForEnvironment(request res if len(ciPipelineIds) == 0 { return ciWorkflowStatuses, nil } - //authorization block starts here + // authorization block starts here var appObjectArr []string objects := impl.enforcerUtil.GetAppObjectByCiPipelineIds(ciPipelineIds) ciPipelineIds = []int{} @@ -1803,17 +850,12 @@ func (impl *CiHandlerImpl) FetchCiStatusForTriggerViewForEnvironment(request res } appResults, _ := request.CheckAuthBatch(token, appObjectArr, []string{}) for _, ciPipeline := range ciPipelines { - appObject := objects[ciPipeline.Id] //here only app permission have to check + appObject := objects[ciPipeline.Id] // here only app permission have to check if !appResults[appObject] { - //if user unauthorized, skip items + // if user unauthorized, skip items continue } - ciPipelineId := 0 - if ciPipeline.ParentCiPipeline == 0 { - ciPipelineId = ciPipeline.Id - } else { - ciPipelineId = ciPipeline.ParentCiPipeline - } + ciPipelineId := impl.getPipelineIdForTriggerView(ciPipeline) ciPipelineIds = append(ciPipelineIds, ciPipelineId) } if len(ciPipelineIds) == 0 { diff --git a/pkg/pipeline/CiLogService.go b/pkg/pipeline/CiLogService.go index 6ad0752f5c..d554917831 100644 --- a/pkg/pipeline/CiLogService.go +++ b/pkg/pipeline/CiLogService.go @@ -36,12 +36,11 @@ type CiLogService interface { type CiLogServiceImpl struct { logger *zap.SugaredLogger - ciService CiService kubeClient *kubernetes.Clientset k8sUtil *k8s.K8sServiceImpl } -func NewCiLogServiceImpl(logger *zap.SugaredLogger, ciService CiService, k8sUtil *k8s.K8sServiceImpl) (*CiLogServiceImpl, error) { +func NewCiLogServiceImpl(logger *zap.SugaredLogger, k8sUtil *k8s.K8sServiceImpl) (*CiLogServiceImpl, error) { _, _, clientSet, err := k8sUtil.GetK8sInClusterConfigAndClients() if err != nil { logger.Errorw("error in getting k8s in cluster client set", "err", err) @@ -49,7 +48,6 @@ func NewCiLogServiceImpl(logger *zap.SugaredLogger, ciService CiService, k8sUtil } return &CiLogServiceImpl{ logger: logger, - ciService: ciService, kubeClient: clientSet, k8sUtil: k8sUtil, }, nil diff --git a/pkg/pipeline/CiService.go b/pkg/pipeline/CiService.go index 3eb0faf6ee..c88ae5ca11 100644 --- a/pkg/pipeline/CiService.go +++ b/pkg/pipeline/CiService.go @@ -17,152 +17,50 @@ package pipeline import ( - "encoding/json" - "errors" - "fmt" "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" - "github.com/caarlos0/env" - "github.com/devtron-labs/common-lib/utils" - bean3 "github.com/devtron-labs/common-lib/utils/bean" - commonBean "github.com/devtron-labs/common-lib/workflow" bean5 "github.com/devtron-labs/devtron/api/bean" - "github.com/devtron-labs/devtron/internal/sql/constants" + client "github.com/devtron-labs/devtron/client/events" + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow/cdWorkflow" - "github.com/devtron-labs/devtron/pkg/attributes" - bean4 "github.com/devtron-labs/devtron/pkg/attributes/bean" - "github.com/devtron-labs/devtron/pkg/bean/common" - "github.com/devtron-labs/devtron/pkg/build/pipeline" - bean6 "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" - repository6 "github.com/devtron-labs/devtron/pkg/cluster/environment/repository" - "github.com/devtron-labs/devtron/pkg/pipeline/adapter" - pipelineConst "github.com/devtron-labs/devtron/pkg/pipeline/constants" - "github.com/devtron-labs/devtron/pkg/pipeline/infraProviders" + buildBean "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" + "github.com/devtron-labs/devtron/pkg/pipeline/types" "github.com/devtron-labs/devtron/pkg/pipeline/workflowStatus" - bean2 "github.com/devtron-labs/devtron/pkg/plugin/bean" "github.com/devtron-labs/devtron/pkg/sql" util3 "github.com/devtron-labs/devtron/util" - "github.com/devtron-labs/devtron/util/sliceUtil" - "path" - "path/filepath" - "slices" - "strconv" - "strings" - "time" - - repository5 "github.com/devtron-labs/devtron/internal/sql/repository" - appRepository "github.com/devtron-labs/devtron/internal/sql/repository/app" - repository3 "github.com/devtron-labs/devtron/internal/sql/repository/dockerRegistry" - "github.com/devtron-labs/devtron/internal/sql/repository/helper" - "github.com/devtron-labs/devtron/pkg/app" - "github.com/devtron-labs/devtron/pkg/auth/user" - pipelineConfigBean "github.com/devtron-labs/devtron/pkg/pipeline/bean" - "github.com/devtron-labs/devtron/pkg/pipeline/repository" - "github.com/devtron-labs/devtron/pkg/pipeline/types" - "github.com/devtron-labs/devtron/pkg/plugin" - repository2 "github.com/devtron-labs/devtron/pkg/plugin/repository" - "github.com/devtron-labs/devtron/pkg/resourceQualifiers" - "github.com/devtron-labs/devtron/pkg/variables" - repository4 "github.com/devtron-labs/devtron/pkg/variables/repository" - "github.com/go-pg/pg" - "net/http" - - "github.com/devtron-labs/common-lib/blob-storage" - client "github.com/devtron-labs/devtron/client/events" - "github.com/devtron-labs/devtron/internal/middleware" - "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" - "github.com/devtron-labs/devtron/internal/util" - "github.com/devtron-labs/devtron/pkg/bean" util2 "github.com/devtron-labs/devtron/util/event" "go.uber.org/zap" ) type CiService interface { - TriggerCiPipeline(trigger types.Trigger) (int, error) - GetCiMaterials(pipelineId int, ciMaterials []*pipelineConfig.CiPipelineMaterial) ([]*pipelineConfig.CiPipelineMaterial, error) + WriteCITriggerEvent(trigger types.Trigger, pipeline *pipelineConfig.CiPipeline, workflowRequest *types.WorkflowRequest) + WriteCIFailEvent(ciWorkflow *pipelineConfig.CiWorkflow) SaveCiWorkflowWithStage(wf *pipelineConfig.CiWorkflow) error UpdateCiWorkflowWithStage(wf *pipelineConfig.CiWorkflow) error } -type BuildxCacheFlags struct { - BuildxCacheModeMin bool `env:"BUILDX_CACHE_MODE_MIN" envDefault:"false" description:"To set build cache mode to minimum in buildx" ` - AsyncBuildxCacheExport bool `env:"ASYNC_BUILDX_CACHE_EXPORT" envDefault:"false" description:"To enable async container image cache export"` -} type CiServiceImpl struct { - Logger *zap.SugaredLogger - workflowService WorkflowService - ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository - workflowStageStatusService workflowStatus.WorkFlowStageStatusService - eventClient client.EventClient - eventFactory client.EventFactory - ciPipelineRepository pipelineConfig.CiPipelineRepository - ciArtifactRepository repository5.CiArtifactRepository - pipelineStageService PipelineStageService - userService user.UserService - ciTemplateService pipeline.CiTemplateReadService - appCrudOperationService app.AppCrudOperationService - envRepository repository6.EnvironmentRepository - appRepository appRepository.AppRepository - customTagService CustomTagService - config *types.CiConfig - scopedVariableManager variables.ScopedVariableManager - pluginInputVariableParser PluginInputVariableParser - globalPluginService plugin.GlobalPluginService - infraProvider infraProviders.InfraProvider - ciCdPipelineOrchestrator CiCdPipelineOrchestrator - buildxCacheFlags *BuildxCacheFlags - attributeService attributes.AttributesService - ciWorkflowRepository pipelineConfig.CiWorkflowRepository - transactionManager sql.TransactionWrapper + Logger *zap.SugaredLogger + workflowStageStatusService workflowStatus.WorkFlowStageStatusService + eventClient client.EventClient + eventFactory client.EventFactory + config *types.CiConfig + ciWorkflowRepository pipelineConfig.CiWorkflowRepository + transactionManager sql.TransactionWrapper } -func NewCiServiceImpl(Logger *zap.SugaredLogger, workflowService WorkflowService, - ciPipelineMaterialRepository pipelineConfig.CiPipelineMaterialRepository, +func NewCiServiceImpl(Logger *zap.SugaredLogger, workflowStageStatusService workflowStatus.WorkFlowStageStatusService, eventClient client.EventClient, eventFactory client.EventFactory, - ciPipelineRepository pipelineConfig.CiPipelineRepository, - ciArtifactRepository repository5.CiArtifactRepository, - pipelineStageService PipelineStageService, - userService user.UserService, - ciTemplateService pipeline.CiTemplateReadService, appCrudOperationService app.AppCrudOperationService, envRepository repository6.EnvironmentRepository, appRepository appRepository.AppRepository, - scopedVariableManager variables.ScopedVariableManager, - customTagService CustomTagService, - pluginInputVariableParser PluginInputVariableParser, - globalPluginService plugin.GlobalPluginService, - infraProvider infraProviders.InfraProvider, - ciCdPipelineOrchestrator CiCdPipelineOrchestrator, attributeService attributes.AttributesService, ciWorkflowRepository pipelineConfig.CiWorkflowRepository, transactionManager sql.TransactionWrapper, ) *CiServiceImpl { - buildxCacheFlags := &BuildxCacheFlags{} - err := env.Parse(buildxCacheFlags) - if err != nil { - Logger.Infow("error occurred while parsing BuildxCacheFlags env,so setting BuildxCacheModeMin and AsyncBuildxCacheExport to default value", "err", err) - } cis := &CiServiceImpl{ - Logger: Logger, - workflowService: workflowService, - ciPipelineMaterialRepository: ciPipelineMaterialRepository, - workflowStageStatusService: workflowStageStatusService, - eventClient: eventClient, - eventFactory: eventFactory, - ciPipelineRepository: ciPipelineRepository, - ciArtifactRepository: ciArtifactRepository, - pipelineStageService: pipelineStageService, - userService: userService, - ciTemplateService: ciTemplateService, - appCrudOperationService: appCrudOperationService, - envRepository: envRepository, - appRepository: appRepository, - scopedVariableManager: scopedVariableManager, - customTagService: customTagService, - pluginInputVariableParser: pluginInputVariableParser, - globalPluginService: globalPluginService, - infraProvider: infraProvider, - ciCdPipelineOrchestrator: ciCdPipelineOrchestrator, - buildxCacheFlags: buildxCacheFlags, - attributeService: attributeService, - ciWorkflowRepository: ciWorkflowRepository, - transactionManager: transactionManager, + Logger: Logger, + workflowStageStatusService: workflowStageStatusService, + eventClient: eventClient, + eventFactory: eventFactory, + ciWorkflowRepository: ciWorkflowRepository, + transactionManager: transactionManager, } config, err := types.GetCiConfig() if err != nil { @@ -172,325 +70,9 @@ func NewCiServiceImpl(Logger *zap.SugaredLogger, workflowService WorkflowService return cis } -func (impl *CiServiceImpl) GetCiMaterials(pipelineId int, ciMaterials []*pipelineConfig.CiPipelineMaterial) ([]*pipelineConfig.CiPipelineMaterial, error) { - if !(len(ciMaterials) == 0) { - return ciMaterials, nil - } else { - ciMaterials, err := impl.ciPipelineMaterialRepository.GetByPipelineId(pipelineId) - if err != nil { - impl.Logger.Errorw("err", "err", err) - return nil, err - } - impl.Logger.Debug("ciMaterials for pipeline trigger ", ciMaterials) - return ciMaterials, nil - } -} - -func (impl *CiServiceImpl) handleRuntimeParamsValidations(trigger types.Trigger, ciMaterials []*pipelineConfig.CiPipelineMaterial, workflowRequest *types.WorkflowRequest) error { - // externalCi artifact is meant only for CI_JOB - if trigger.PipelineType != string(bean6.CI_JOB) { - return nil - } - - // checking if user has given run time parameters for externalCiArtifact, if given then sending git material to Ci-Runner - externalCiArtifact, exists := trigger.RuntimeParameters.GetGlobalRuntimeVariables()[pipelineConst.ExtraEnvVarExternalCiArtifactKey] - // validate externalCiArtifact as docker image - if exists { - externalCiArtifact = strings.TrimSpace(externalCiArtifact) - if !strings.Contains(externalCiArtifact, ":") { - if utils.IsValidDockerTagName(externalCiArtifact) { - fullImageUrl, err := utils.BuildDockerImagePath(bean3.DockerRegistryInfo{ - DockerImageTag: externalCiArtifact, - DockerRegistryId: workflowRequest.DockerRegistryId, - DockerRegistryType: workflowRequest.DockerRegistryType, - DockerRegistryURL: workflowRequest.DockerRegistryURL, - DockerRepository: workflowRequest.DockerRepository, - }) - if err != nil { - impl.Logger.Errorw("Error in building docker image", "err", err) - return err - } - externalCiArtifact = fullImageUrl - } else { - impl.Logger.Errorw("validation error", "externalCiArtifact", externalCiArtifact) - return fmt.Errorf("invalid image name or url given in externalCiArtifact") - } - - } - // This will overwrite the existing runtime parameters value for constants.externalCiArtifact - trigger.RuntimeParameters = trigger.RuntimeParameters.AddRuntimeGlobalVariable(pipelineConst.ExtraEnvVarExternalCiArtifactKey, externalCiArtifact) - var artifactExists bool - var err error - - imageDigest, ok := trigger.RuntimeParameters.GetGlobalRuntimeVariables()[pipelineConst.ExtraEnvVarImageDigestKey] - if !ok || len(imageDigest) == 0 { - artifactExists, err = impl.ciArtifactRepository.IfArtifactExistByImage(externalCiArtifact, trigger.PipelineId) - if err != nil { - impl.Logger.Errorw("error in fetching ci artifact", "err", err) - return err - } - if artifactExists { - impl.Logger.Errorw("ci artifact already exists with same image name", "artifact", externalCiArtifact) - return fmt.Errorf("ci artifact already exists with same image name") - } - } else { - artifactExists, err = impl.ciArtifactRepository.IfArtifactExistByImageDigest(imageDigest, externalCiArtifact, trigger.PipelineId) - if err != nil { - impl.Logger.Errorw("error in fetching ci artifact", "err", err, "imageDigest", imageDigest) - return err - } - if artifactExists { - impl.Logger.Errorw("ci artifact already exists with same digest", "artifact", externalCiArtifact) - return fmt.Errorf("ci artifact already exists with same digest") - } - - } - - } - if trigger.PipelineType == string(bean6.CI_JOB) && len(ciMaterials) != 0 && !exists && externalCiArtifact == "" { - ciMaterials[0].GitMaterial = nil - ciMaterials[0].GitMaterialId = 0 - } - return nil -} - -func (impl *CiServiceImpl) markCurrentCiWorkflowFailed(savedCiWf *pipelineConfig.CiWorkflow, validationErr error) error { - // currently such requirement is not there - if savedCiWf == nil { - return nil - } - if savedCiWf.Id != 0 && slices.Contains(cdWorkflow.WfrTerminalStatusList, savedCiWf.Status) { - impl.Logger.Debug("workflow is already in terminal state", "status", savedCiWf.Status, "workflowId", savedCiWf.Id, "message", savedCiWf.Message) - return nil - } - - savedCiWf.Status = cdWorkflow.WorkflowFailed - savedCiWf.Message = validationErr.Error() - savedCiWf.FinishedOn = time.Now() - - var dbErr error - if savedCiWf.Id == 0 { - dbErr = impl.SaveCiWorkflowWithStage(savedCiWf) - } else { - dbErr = impl.UpdateCiWorkflowWithStage(savedCiWf) - } - - if dbErr != nil { - impl.Logger.Errorw("save/update workflow error", "err", dbErr) - return dbErr - } - return nil -} - -func (impl *CiServiceImpl) TriggerCiPipeline(trigger types.Trigger) (int, error) { - impl.Logger.Debug("ci pipeline manual trigger") - ciMaterials, err := impl.GetCiMaterials(trigger.PipelineId, trigger.CiMaterials) - if err != nil { - return 0, err - } - - ciPipelineScripts, err := impl.ciPipelineRepository.FindCiScriptsByCiPipelineId(trigger.PipelineId) - if err != nil && !util.IsErrNoRows(err) { - return 0, err - } - - var pipeline *pipelineConfig.CiPipeline - for _, m := range ciMaterials { - pipeline = m.CiPipeline - break - } - - scope := resourceQualifiers.Scope{ - AppId: pipeline.App.Id, - } - ciWorkflowConfigNamespace := impl.config.GetDefaultNamespace() - envModal, isJob, err := impl.getEnvironmentForJob(pipeline, trigger) - if err != nil { - return 0, err - } - if isJob && envModal != nil { - ciWorkflowConfigNamespace = envModal.Namespace - - // This will be populated for jobs running in selected environment - scope.EnvId = envModal.Id - scope.ClusterId = envModal.ClusterId - - scope.SystemMetadata = &resourceQualifiers.SystemMetadata{ - EnvironmentName: envModal.Name, - ClusterName: envModal.Cluster.ClusterName, - Namespace: envModal.Namespace, - } - } - if scope.SystemMetadata == nil { - scope.SystemMetadata = &resourceQualifiers.SystemMetadata{ - Namespace: ciWorkflowConfigNamespace, - AppName: pipeline.App.AppName, - } - } - - savedCiWf, err := impl.saveNewWorkflow(pipeline, ciWorkflowConfigNamespace, trigger.CommitHashes, trigger.TriggeredBy, trigger.EnvironmentId, isJob, trigger.ReferenceCiWorkflowId) - if err != nil { - impl.Logger.Errorw("could not save new workflow", "err", err) - return 0, err - } - - // preCiSteps, postCiSteps, refPluginsData, err := impl.pipelineStageService.BuildPrePostAndRefPluginStepsDataForWfRequest(pipeline.Id, ciEvent) - request := pipelineConfigBean.NewBuildPrePostStepDataReq(pipeline.Id, pipelineConfigBean.CiStage, scope) - prePostAndRefPluginResponse, err := impl.pipelineStageService.BuildPrePostAndRefPluginStepsDataForWfRequest(request) - if err != nil { - impl.Logger.Errorw("error in getting pre steps data for wf request", "err", err, "ciPipelineId", pipeline.Id) - dbErr := impl.markCurrentCiWorkflowFailed(savedCiWf, err) - if dbErr != nil { - impl.Logger.Errorw("saving workflow error", "err", dbErr) - } - return 0, err - } - preCiSteps := prePostAndRefPluginResponse.PreStageSteps - postCiSteps := prePostAndRefPluginResponse.PostStageSteps - refPluginsData := prePostAndRefPluginResponse.RefPluginData - variableSnapshot := prePostAndRefPluginResponse.VariableSnapshot - - if len(preCiSteps) == 0 && isJob { - errMsg := fmt.Sprintf("No tasks are configured in this job pipeline") - validationErr := util.NewApiError(http.StatusNotFound, errMsg, errMsg) - - return 0, validationErr - } - - // get env variables of git trigger data and add it in the extraEnvVariables - gitTriggerEnvVariables, _, err := impl.ciCdPipelineOrchestrator.GetGitCommitEnvVarDataForCICDStage(savedCiWf.GitTriggers) - if err != nil { - impl.Logger.Errorw("error in getting gitTrigger env data for stage", "gitTriggers", savedCiWf.GitTriggers, "err", err) - return 0, err - } - - for k, v := range gitTriggerEnvVariables { - trigger.RuntimeParameters = trigger.RuntimeParameters.AddSystemVariable(k, v) - } - - workflowRequest, err := impl.buildWfRequestForCiPipeline(pipeline, trigger, ciMaterials, savedCiWf, ciWorkflowConfigNamespace, ciPipelineScripts, preCiSteps, postCiSteps, refPluginsData, isJob) - if err != nil { - impl.Logger.Errorw("make workflow req", "err", err) - return 0, err - } - err = impl.handleRuntimeParamsValidations(trigger, ciMaterials, workflowRequest) - if err != nil { - savedCiWf.Status = cdWorkflow.WorkflowAborted - savedCiWf.Message = err.Error() - err1 := impl.UpdateCiWorkflowWithStage(savedCiWf) - if err1 != nil { - impl.Logger.Errorw("could not save workflow, after failing due to conflicting image tag") - } - return 0, err - } - - workflowRequest.Scope = scope - workflowRequest.AppId = pipeline.AppId - workflowRequest.BuildxCacheModeMin = impl.buildxCacheFlags.BuildxCacheModeMin - workflowRequest.AsyncBuildxCacheExport = impl.buildxCacheFlags.AsyncBuildxCacheExport - if impl.config != nil && impl.config.BuildxK8sDriverOptions != "" { - err = impl.setBuildxK8sDriverData(workflowRequest) - if err != nil { - impl.Logger.Errorw("error in setBuildxK8sDriverData", "BUILDX_K8S_DRIVER_OPTIONS", impl.config.BuildxK8sDriverOptions, "err", err) - return 0, err - } - } - - // savedCiWf.LogLocation = impl.ciCdConfig.CiDefaultBuildLogsKeyPrefix + "/" + workflowRequest.WorkflowNamePrefix + "/main.log" - savedCiWf.LogLocation = fmt.Sprintf("%s/%s/main.log", impl.config.GetDefaultBuildLogsKeyPrefix(), workflowRequest.WorkflowNamePrefix) - err = impl.updateCiWorkflow(workflowRequest, savedCiWf) - - appLabels, err := impl.appCrudOperationService.GetLabelsByAppId(pipeline.AppId) - if err != nil { - return 0, err - } - workflowRequest.AppLabels = appLabels - workflowRequest.Env = envModal - if isJob { - workflowRequest.Type = pipelineConfigBean.JOB_WORKFLOW_PIPELINE_TYPE - } else { - workflowRequest.Type = pipelineConfigBean.CI_WORKFLOW_PIPELINE_TYPE - } - - workflowRequest.CiPipelineType = trigger.PipelineType - err = impl.executeCiPipeline(workflowRequest) - if err != nil { - impl.Logger.Errorw("error in executing ci pipeline", "err", err) - dbErr := impl.markCurrentCiWorkflowFailed(savedCiWf, err) - if dbErr != nil { - impl.Logger.Errorw("update ci workflow error", "err", dbErr) - } - return 0, err - } - impl.Logger.Debugw("ci triggered", " pipeline ", trigger.PipelineId) - - var variableSnapshotHistories = sliceUtil.GetBeansPtr( - repository4.GetSnapshotBean(savedCiWf.Id, repository4.HistoryReferenceTypeCIWORKFLOW, variableSnapshot)) - if len(variableSnapshotHistories) > 0 { - err = impl.scopedVariableManager.SaveVariableHistoriesForTrigger(variableSnapshotHistories, trigger.TriggeredBy) - if err != nil { - impl.Logger.Errorf("Not able to save variable snapshot for CI trigger %s", err) - } - } - - middleware.CiTriggerCounter.WithLabelValues(pipeline.App.AppName, pipeline.Name).Inc() - go impl.WriteCITriggerEvent(trigger, pipeline, workflowRequest) - return savedCiWf.Id, err -} - -func (impl *CiServiceImpl) setBuildxK8sDriverData(workflowRequest *types.WorkflowRequest) error { - ciBuildConfig := workflowRequest.CiBuildConfig - if ciBuildConfig != nil { - if dockerBuildConfig := ciBuildConfig.DockerBuildConfig; dockerBuildConfig != nil { - k8sDriverOptions, err := impl.getK8sDriverOptions() - if err != nil { - errMsg := "error in parsing BUILDX_K8S_DRIVER_OPTIONS from the devtron-cm, " - err = errors.New(errMsg + "error : " + err.Error()) - impl.Logger.Errorw(errMsg, "err", err) - } - dockerBuildConfig.BuildxK8sDriverOptions = k8sDriverOptions - - } - } - return nil -} - -func (impl *CiServiceImpl) getK8sDriverOptions() ([]map[string]string, error) { - buildxK8sDriverOptions := make([]map[string]string, 0) - err := json.Unmarshal([]byte(impl.config.BuildxK8sDriverOptions), &buildxK8sDriverOptions) - if err != nil { - return nil, err - } else { - return buildxK8sDriverOptions, nil - } -} - -func (impl *CiServiceImpl) getEnvironmentForJob(pipeline *pipelineConfig.CiPipeline, trigger types.Trigger) (*repository6.Environment, bool, error) { - app, err := impl.appRepository.FindById(pipeline.AppId) - if err != nil { - impl.Logger.Errorw("could not find app", "err", err) - return nil, false, err - } - - var env *repository6.Environment - isJob := false - if app.AppType == helper.Job { - isJob = true - if trigger.EnvironmentId != 0 { - env, err = impl.envRepository.FindById(trigger.EnvironmentId) - if err != nil { - impl.Logger.Errorw("could not find environment", "err", err) - return nil, isJob, err - } - return env, isJob, nil - } - } - return nil, isJob, nil -} - func (impl *CiServiceImpl) WriteCITriggerEvent(trigger types.Trigger, pipeline *pipelineConfig.CiPipeline, workflowRequest *types.WorkflowRequest) { event, _ := impl.eventFactory.Build(util2.Trigger, &pipeline.Id, pipeline.AppId, nil, util2.CI) - material := &client.MaterialTriggerInfo{} + material := &buildBean.MaterialTriggerInfo{} material.GitTriggers = trigger.CommitHashes @@ -503,635 +85,17 @@ func (impl *CiServiceImpl) WriteCITriggerEvent(trigger types.Trigger, pipeline * } } -// TODO: Send all trigger data -func (impl *CiServiceImpl) BuildPayload(trigger types.Trigger, pipeline *pipelineConfig.CiPipeline) *client.Payload { - payload := &client.Payload{} - payload.AppName = pipeline.App.AppName - payload.PipelineName = pipeline.Name - return payload -} - -func (impl *CiServiceImpl) saveNewWorkflow(pipeline *pipelineConfig.CiPipeline, ciWorkflowConfigNamespace string, - commitHashes map[int]pipelineConfig.GitCommit, userId int32, EnvironmentId int, isJob bool, refCiWorkflowId int) (wf *pipelineConfig.CiWorkflow, error error) { - - ciWorkflow := &pipelineConfig.CiWorkflow{ - Name: pipeline.Name + "-" + strconv.Itoa(pipeline.Id), - Status: cdWorkflow.WorkflowStarting, // starting CIStage - Message: "", - StartedOn: time.Now(), - CiPipelineId: pipeline.Id, - Namespace: impl.config.GetDefaultNamespace(), - BlobStorageEnabled: impl.config.BlobStorageEnabled, - GitTriggers: commitHashes, - LogLocation: "", - TriggeredBy: userId, - ReferenceCiWorkflowId: refCiWorkflowId, - ExecutorType: impl.config.GetWorkflowExecutorType(), - } - if isJob { - ciWorkflow.Namespace = ciWorkflowConfigNamespace - ciWorkflow.EnvironmentId = EnvironmentId - } - err := impl.SaveCiWorkflowWithStage(ciWorkflow) - if err != nil { - impl.Logger.Errorw("saving workflow error", "err", err) - return &pipelineConfig.CiWorkflow{}, err - } - impl.Logger.Debugw("workflow saved ", "id", ciWorkflow.Id) - return ciWorkflow, nil -} - -func (impl *CiServiceImpl) executeCiPipeline(workflowRequest *types.WorkflowRequest) error { - _, _, err := impl.workflowService.SubmitWorkflow(workflowRequest) - if err != nil { - impl.Logger.Errorw("workflow error", "err", err) - return err - } - return nil -} - -func (impl *CiServiceImpl) buildS3ArtifactLocation(ciWorkflowConfigLogsBucket string, savedWf *pipelineConfig.CiWorkflow) (string, string, string) { - ciArtifactLocationFormat := impl.config.GetArtifactLocationFormat() - ArtifactLocation := fmt.Sprintf("s3://"+path.Join(ciWorkflowConfigLogsBucket, ciArtifactLocationFormat), savedWf.Id, savedWf.Id) - artifactFileName := fmt.Sprintf(ciArtifactLocationFormat, savedWf.Id, savedWf.Id) - return ArtifactLocation, ciWorkflowConfigLogsBucket, artifactFileName -} - -func (impl *CiServiceImpl) buildDefaultArtifactLocation(savedWf *pipelineConfig.CiWorkflow) string { - ciArtifactLocationFormat := impl.config.GetArtifactLocationFormat() - ArtifactLocation := fmt.Sprintf(ciArtifactLocationFormat, savedWf.Id, savedWf.Id) - return ArtifactLocation -} - -func (impl *CiServiceImpl) buildWfRequestForCiPipeline(pipeline *pipelineConfig.CiPipeline, trigger types.Trigger, ciMaterials []*pipelineConfig.CiPipelineMaterial, savedWf *pipelineConfig.CiWorkflow, ciWorkflowConfigNamespace string, ciPipelineScripts []*pipelineConfig.CiPipelineScript, preCiSteps []*pipelineConfigBean.StepObject, postCiSteps []*pipelineConfigBean.StepObject, refPluginsData []*pipelineConfigBean.RefPluginObject, isJob bool) (*types.WorkflowRequest, error) { - var ciProjectDetails []pipelineConfigBean.CiProjectDetails - commitHashes := trigger.CommitHashes - for _, ciMaterial := range ciMaterials { - // ignore those materials which have inactive git material - if ciMaterial == nil || ciMaterial.GitMaterial == nil || !ciMaterial.GitMaterial.Active { - continue - } - commitHashForPipelineId := commitHashes[ciMaterial.Id] - ciProjectDetail := pipelineConfigBean.CiProjectDetails{ - GitRepository: ciMaterial.GitMaterial.Url, - MaterialName: ciMaterial.GitMaterial.Name, - CheckoutPath: ciMaterial.GitMaterial.CheckoutPath, - FetchSubmodules: ciMaterial.GitMaterial.FetchSubmodules, - CommitHash: commitHashForPipelineId.Commit, - Author: commitHashForPipelineId.Author, - SourceType: ciMaterial.Type, - SourceValue: ciMaterial.Value, - GitTag: ciMaterial.GitTag, - Message: commitHashForPipelineId.Message, - Type: string(ciMaterial.Type), - CommitTime: commitHashForPipelineId.Date.Format(bean.LayoutRFC3339), - GitOptions: pipelineConfigBean.GitOptions{ - UserName: ciMaterial.GitMaterial.GitProvider.UserName, - Password: ciMaterial.GitMaterial.GitProvider.Password, - SshPrivateKey: ciMaterial.GitMaterial.GitProvider.SshPrivateKey, - AccessToken: ciMaterial.GitMaterial.GitProvider.AccessToken, - AuthMode: ciMaterial.GitMaterial.GitProvider.AuthMode, - EnableTLSVerification: ciMaterial.GitMaterial.GitProvider.EnableTLSVerification, - TlsKey: ciMaterial.GitMaterial.GitProvider.TlsKey, - TlsCert: ciMaterial.GitMaterial.GitProvider.TlsCert, - CaCert: ciMaterial.GitMaterial.GitProvider.CaCert, - }, - } - - if ciMaterial.Type == constants.SOURCE_TYPE_WEBHOOK { - webhookData := commitHashForPipelineId.WebhookData - ciProjectDetail.WebhookData = pipelineConfig.WebhookData{ - Id: webhookData.Id, - EventActionType: webhookData.EventActionType, - Data: webhookData.Data, - } - } - - ciProjectDetails = append(ciProjectDetails, ciProjectDetail) - } - - var beforeDockerBuildScripts []*bean.CiScript - var afterDockerBuildScripts []*bean.CiScript - for _, ciPipelineScript := range ciPipelineScripts { - ciTask := &bean.CiScript{ - Id: ciPipelineScript.Id, - Index: ciPipelineScript.Index, - Name: ciPipelineScript.Name, - Script: ciPipelineScript.Script, - OutputLocation: ciPipelineScript.OutputLocation, - } - - if ciPipelineScript.Stage == BEFORE_DOCKER_BUILD { - beforeDockerBuildScripts = append(beforeDockerBuildScripts, ciTask) - } else if ciPipelineScript.Stage == AFTER_DOCKER_BUILD { - afterDockerBuildScripts = append(afterDockerBuildScripts, ciTask) - } - } - - if !(len(beforeDockerBuildScripts) == 0 && len(afterDockerBuildScripts) == 0) { - // found beforeDockerBuildScripts/afterDockerBuildScripts - // building preCiSteps & postCiSteps from them, refPluginsData not needed - preCiSteps = buildCiStepsDataFromDockerBuildScripts(beforeDockerBuildScripts) - postCiSteps = buildCiStepsDataFromDockerBuildScripts(afterDockerBuildScripts) - refPluginsData = []*pipelineConfigBean.RefPluginObject{} - } - host, err := impl.attributeService.GetByKey(bean4.HostUrlKey) - if err != nil { - impl.Logger.Errorw("error in getting host url", "err", err, "hostUrl", host.Value) - return nil, err - } - ciWorkflowConfigCiCacheBucket := impl.config.DefaultCacheBucket - - ciWorkflowConfigCiCacheRegion := impl.config.DefaultCacheBucketRegion - - ciWorkflowConfigCiImage := impl.config.GetDefaultImage() - ciTemplate := pipeline.CiTemplate - ciLevelArgs := pipeline.DockerArgs - - if ciLevelArgs == "" { - ciLevelArgs = "{}" - } - - if pipeline.CiTemplate.DockerBuildOptions == "" { - pipeline.CiTemplate.DockerBuildOptions = "{}" - } - userEmailId, err := impl.userService.GetActiveEmailById(trigger.TriggeredBy) - if err != nil { - impl.Logger.Errorw("unable to find user email by id", "err", err, "id", trigger.TriggeredBy) - return nil, err - } - var dockerfilePath string - var dockerRepository string - var checkoutPath string - var ciBuildConfigBean *bean6.CiBuildConfigBean - dockerRegistry := &repository3.DockerArtifactStore{} - ciBaseBuildConfigEntity := ciTemplate.CiBuildConfig - ciBaseBuildConfigBean, err := adapter.ConvertDbBuildConfigToBean(ciBaseBuildConfigEntity) - if err != nil { - impl.Logger.Errorw("error occurred while converting buildconfig dbEntity to configBean", "ciBuildConfigEntity", ciBaseBuildConfigEntity, "err", err) - return nil, errors.New("error while parsing ci build config") - } - if !pipeline.IsExternal && pipeline.IsDockerConfigOverridden { - templateOverrideBean, err := impl.ciTemplateService.FindTemplateOverrideByCiPipelineId(pipeline.Id) - if err != nil { - return nil, err - } - ciBuildConfigBean = templateOverrideBean.CiBuildConfig - // updating args coming from ciBaseBuildConfigEntity because it is not part of Ci override - if ciBuildConfigBean != nil && ciBuildConfigBean.DockerBuildConfig != nil && ciBaseBuildConfigBean != nil && ciBaseBuildConfigBean.DockerBuildConfig != nil { - ciBuildConfigBean.DockerBuildConfig.Args = ciBaseBuildConfigBean.DockerBuildConfig.Args - } - templateOverride := templateOverrideBean.CiTemplateOverride - checkoutPath = templateOverride.GitMaterial.CheckoutPath - dockerfilePath = templateOverride.DockerfilePath - dockerRepository = templateOverride.DockerRepository - dockerRegistry = templateOverride.DockerRegistry - } else { - checkoutPath = ciTemplate.GitMaterial.CheckoutPath - dockerfilePath = ciTemplate.DockerfilePath - dockerRegistry = ciTemplate.DockerRegistry - dockerRepository = ciTemplate.DockerRepository - ciBuildConfigBean = ciBaseBuildConfigBean - if ciBuildConfigBean != nil { - ciBuildConfigBean.BuildContextGitMaterialId = ciTemplate.BuildContextGitMaterialId - } - - } - if checkoutPath == "" { - checkoutPath = "./" - } - var dockerImageTag string - customTag, err := impl.customTagService.GetActiveCustomTagByEntityKeyAndValue(pipelineConfigBean.EntityTypeCiPipelineId, strconv.Itoa(pipeline.Id)) - if err != nil && err != pg.ErrNoRows { - return nil, err - } - if customTag.Id != 0 && customTag.Enabled == true { - imagePathReservation, err := impl.customTagService.GenerateImagePath(pipelineConfigBean.EntityTypeCiPipelineId, strconv.Itoa(pipeline.Id), dockerRegistry.RegistryURL, dockerRepository) - if err != nil { - if errors.Is(err, pipelineConfigBean.ErrImagePathInUse) { - errMsg := pipelineConfigBean.ImageTagUnavailableMessage - validationErr := util.NewApiError(http.StatusConflict, errMsg, errMsg) - dbErr := impl.markCurrentCiWorkflowFailed(savedWf, validationErr) - if dbErr != nil { - impl.Logger.Errorw("could not save workflow, after failing due to conflicting image tag", "err", dbErr, "savedWf", savedWf.Id) - } - return nil, err - } - return nil, err - } - savedWf.ImagePathReservationIds = []int{imagePathReservation.Id} - // imagePath = docker.io/avd0/dashboard:fd23414b - imagePathSplit := strings.Split(imagePathReservation.ImagePath, ":") - if len(imagePathSplit) >= 1 { - dockerImageTag = imagePathSplit[len(imagePathSplit)-1] - } - } else { - dockerImageTag = impl.buildImageTag(commitHashes, pipeline.Id, savedWf.Id) - } - - // copyContainerImage plugin specific logic - var registryCredentialMap map[string]bean2.RegistryCredentials - var pluginArtifactStage string - var imageReservationIds []int - var registryDestinationImageMap map[string][]string - if !isJob { - registryDestinationImageMap, registryCredentialMap, pluginArtifactStage, imageReservationIds, err = impl.GetWorkflowRequestVariablesForCopyContainerImagePlugin(preCiSteps, postCiSteps, dockerImageTag, customTag.Id, - fmt.Sprintf(pipelineConfigBean.ImagePathPattern, - dockerRegistry.RegistryURL, - dockerRepository, - dockerImageTag), - dockerRegistry.Id) - if err != nil { - impl.Logger.Errorw("error in getting env variables for copyContainerImage plugin") - dbErr := impl.markCurrentCiWorkflowFailed(savedWf, err) - if dbErr != nil { - impl.Logger.Errorw("could not save workflow, after failing due to conflicting image tag", "err", dbErr, "savedWf", savedWf.Id) - } - return nil, err - } - - savedWf.ImagePathReservationIds = append(savedWf.ImagePathReservationIds, imageReservationIds...) - } - // mergedArgs := string(merged) - oldArgs := ciTemplate.Args - ciBuildConfigBean, err = adapter.OverrideCiBuildConfig(dockerfilePath, oldArgs, ciLevelArgs, ciTemplate.DockerBuildOptions, ciTemplate.TargetPlatform, ciBuildConfigBean) - if err != nil { - impl.Logger.Errorw("error occurred while overriding ci build config", "oldArgs", oldArgs, "ciLevelArgs", ciLevelArgs, "error", err) - return nil, errors.New("error while parsing ci build config") - } - buildContextCheckoutPath, err := impl.ciPipelineMaterialRepository.GetCheckoutPath(ciBuildConfigBean.BuildContextGitMaterialId) - if err != nil && err != pg.ErrNoRows { - impl.Logger.Errorw("error occurred while getting checkout path from git material", "gitMaterialId", ciBuildConfigBean.BuildContextGitMaterialId, "error", err) - return nil, err - } - if buildContextCheckoutPath == "" { - buildContextCheckoutPath = checkoutPath - } - if ciBuildConfigBean.UseRootBuildContext { - // use root build context i.e '.' - buildContextCheckoutPath = "." - } - - ciBuildConfigBean.PipelineType = trigger.PipelineType - - if ciBuildConfigBean.CiBuildType == bean6.SELF_DOCKERFILE_BUILD_TYPE || ciBuildConfigBean.CiBuildType == bean6.MANAGED_DOCKERFILE_BUILD_TYPE { - ciBuildConfigBean.DockerBuildConfig.BuildContext = filepath.Join(buildContextCheckoutPath, ciBuildConfigBean.DockerBuildConfig.BuildContext) - dockerBuildConfig := ciBuildConfigBean.DockerBuildConfig - dockerfilePath = filepath.Join(checkoutPath, dockerBuildConfig.DockerfilePath) - dockerBuildConfig.DockerfilePath = dockerfilePath - checkoutPath = dockerfilePath[:strings.LastIndex(dockerfilePath, "/")+1] - } else if ciBuildConfigBean.CiBuildType == bean6.BUILDPACK_BUILD_TYPE { - buildPackConfig := ciBuildConfigBean.BuildPackConfig - checkoutPath = filepath.Join(checkoutPath, buildPackConfig.ProjectPath) - } - - defaultTargetPlatform := impl.config.DefaultTargetPlatform - useBuildx := impl.config.UseBuildx - - if ciBuildConfigBean.DockerBuildConfig != nil { - if ciBuildConfigBean.DockerBuildConfig.TargetPlatform == "" && useBuildx { - ciBuildConfigBean.DockerBuildConfig.TargetPlatform = defaultTargetPlatform - ciBuildConfigBean.DockerBuildConfig.UseBuildx = useBuildx - } - ciBuildConfigBean.DockerBuildConfig.BuildxProvenanceMode = impl.config.BuildxProvenanceMode - } - - workflowRequest := &types.WorkflowRequest{ - WorkflowNamePrefix: strconv.Itoa(savedWf.Id) + "-" + savedWf.Name, - PipelineName: pipeline.Name, - PipelineId: pipeline.Id, - CiCacheFileName: pipeline.Name + "-" + strconv.Itoa(pipeline.Id) + ".tar.gz", - CiProjectDetails: ciProjectDetails, - Namespace: ciWorkflowConfigNamespace, - BlobStorageConfigured: savedWf.BlobStorageEnabled, - CiImage: ciWorkflowConfigCiImage, - WorkflowId: savedWf.Id, - TriggeredBy: savedWf.TriggeredBy, - CacheLimit: impl.config.CacheLimit, - ScanEnabled: pipeline.ScanEnabled, - CloudProvider: impl.config.CloudProvider, - DefaultAddressPoolBaseCidr: impl.config.GetDefaultAddressPoolBaseCidr(), - DefaultAddressPoolSize: impl.config.GetDefaultAddressPoolSize(), - PreCiSteps: preCiSteps, - PostCiSteps: postCiSteps, - RefPlugins: refPluginsData, - AppName: pipeline.App.AppName, - TriggerByAuthor: userEmailId, - CiBuildConfig: ciBuildConfigBean, - CiBuildDockerMtuValue: impl.config.CiRunnerDockerMTUValue, - CacheInvalidate: trigger.InvalidateCache, - SystemEnvironmentVariables: trigger.RuntimeParameters.GetSystemVariables(), - EnableBuildContext: impl.config.EnableBuildContext, - OrchestratorHost: impl.config.OrchestratorHost, - HostUrl: host.Value, - OrchestratorToken: impl.config.OrchestratorToken, - ImageRetryCount: impl.config.ImageRetryCount, - ImageRetryInterval: impl.config.ImageRetryInterval, - WorkflowExecutor: impl.config.GetWorkflowExecutorType(), - Type: pipelineConfigBean.CI_WORKFLOW_PIPELINE_TYPE, - CiArtifactLastFetch: trigger.CiArtifactLastFetch, - RegistryDestinationImageMap: registryDestinationImageMap, - RegistryCredentialMap: registryCredentialMap, - PluginArtifactStage: pluginArtifactStage, - ImageScanMaxRetries: impl.config.ImageScanMaxRetries, - ImageScanRetryDelay: impl.config.ImageScanRetryDelay, - UseDockerApiToGetDigest: impl.config.UseDockerApiToGetDigest, - } - workflowRequest.SetAwsInspectorConfig("") - //in oss, there is no pipeline level workflow cache config, so we pass inherit to get the app level config - workflowCacheConfig := impl.ciCdPipelineOrchestrator.GetWorkflowCacheConfig(pipeline.App.AppType, trigger.PipelineType, common.WorkflowCacheConfigInherit) - workflowRequest.IgnoreDockerCachePush = !workflowCacheConfig.Value - workflowRequest.IgnoreDockerCachePull = !workflowCacheConfig.Value - impl.Logger.Debugw("Ignore Cache values", "IgnoreDockerCachePush", workflowRequest.IgnoreDockerCachePush, "IgnoreDockerCachePull", workflowRequest.IgnoreDockerCachePull) - if pipeline.App.AppType == helper.Job { - workflowRequest.AppName = pipeline.App.DisplayName - } - if pipeline.ScanEnabled { - scanToolMetadata, scanVia, err := impl.fetchImageScanExecutionMedium() - if err != nil { - impl.Logger.Errorw("error occurred getting scanned via", "err", err) - return nil, err - } - workflowRequest.SetExecuteImageScanningVia(scanVia) - if scanVia.IsScanMediumExternal() { - imageScanExecutionSteps, refPlugins, err := impl.fetchImageScanExecutionStepsForWfRequest(scanToolMetadata) - if err != nil { - impl.Logger.Errorw("error occurred, fetchImageScanExecutionStepsForWfRequest", "scanToolMetadata", scanToolMetadata, "err", err) - return nil, err - } - workflowRequest.SetImageScanningSteps(imageScanExecutionSteps) - workflowRequest.RefPlugins = append(workflowRequest.RefPlugins, refPlugins...) - } - } - - if dockerRegistry != nil { - workflowRequest.DockerRegistryId = dockerRegistry.Id - workflowRequest.DockerRegistryType = string(dockerRegistry.RegistryType) - workflowRequest.DockerImageTag = dockerImageTag - workflowRequest.DockerRegistryURL = dockerRegistry.RegistryURL - workflowRequest.DockerRepository = dockerRepository - workflowRequest.CheckoutPath = checkoutPath - workflowRequest.DockerUsername = dockerRegistry.Username - workflowRequest.DockerPassword = dockerRegistry.Password - workflowRequest.AwsRegion = dockerRegistry.AWSRegion - workflowRequest.AccessKey = dockerRegistry.AWSAccessKeyId - workflowRequest.SecretKey = dockerRegistry.AWSSecretAccessKey - workflowRequest.DockerConnection = dockerRegistry.Connection - workflowRequest.DockerCert = dockerRegistry.Cert - - } - ciWorkflowConfigLogsBucket := impl.config.GetDefaultBuildLogsBucket() - - switch workflowRequest.CloudProvider { - case types.BLOB_STORAGE_S3: - // No AccessKey is used for uploading artifacts, instead IAM based auth is used - workflowRequest.CiCacheRegion = ciWorkflowConfigCiCacheRegion - workflowRequest.CiCacheLocation = ciWorkflowConfigCiCacheBucket - workflowRequest.CiArtifactLocation, workflowRequest.CiArtifactBucket, workflowRequest.CiArtifactFileName = impl.buildS3ArtifactLocation(ciWorkflowConfigLogsBucket, savedWf) - workflowRequest.BlobStorageS3Config = &blob_storage.BlobStorageS3Config{ - AccessKey: impl.config.BlobStorageS3AccessKey, - Passkey: impl.config.BlobStorageS3SecretKey, - EndpointUrl: impl.config.BlobStorageS3Endpoint, - IsInSecure: impl.config.BlobStorageS3EndpointInsecure, - CiCacheBucketName: ciWorkflowConfigCiCacheBucket, - CiCacheRegion: ciWorkflowConfigCiCacheRegion, - CiCacheBucketVersioning: impl.config.BlobStorageS3BucketVersioned, - CiArtifactBucketName: workflowRequest.CiArtifactBucket, - CiArtifactRegion: impl.config.GetDefaultCdLogsBucketRegion(), - CiArtifactBucketVersioning: impl.config.BlobStorageS3BucketVersioned, - CiLogBucketName: impl.config.GetDefaultBuildLogsBucket(), - CiLogRegion: impl.config.GetDefaultCdLogsBucketRegion(), - CiLogBucketVersioning: impl.config.BlobStorageS3BucketVersioned, - } - case types.BLOB_STORAGE_GCP: - workflowRequest.GcpBlobConfig = &blob_storage.GcpBlobConfig{ - CredentialFileJsonData: impl.config.BlobStorageGcpCredentialJson, - CacheBucketName: ciWorkflowConfigCiCacheBucket, - LogBucketName: ciWorkflowConfigLogsBucket, - ArtifactBucketName: ciWorkflowConfigLogsBucket, - } - workflowRequest.CiArtifactLocation = impl.buildDefaultArtifactLocation(savedWf) - workflowRequest.CiArtifactFileName = workflowRequest.CiArtifactLocation - case types.BLOB_STORAGE_AZURE: - workflowRequest.AzureBlobConfig = &blob_storage.AzureBlobConfig{ - Enabled: impl.config.CloudProvider == types.BLOB_STORAGE_AZURE, - AccountName: impl.config.AzureAccountName, - BlobContainerCiCache: impl.config.AzureBlobContainerCiCache, - AccountKey: impl.config.AzureAccountKey, - BlobContainerCiLog: impl.config.AzureBlobContainerCiLog, - BlobContainerArtifact: impl.config.AzureBlobContainerCiLog, - } - workflowRequest.BlobStorageS3Config = &blob_storage.BlobStorageS3Config{ - EndpointUrl: impl.config.AzureGatewayUrl, - IsInSecure: impl.config.AzureGatewayConnectionInsecure, - CiLogBucketName: impl.config.AzureBlobContainerCiLog, - CiLogRegion: impl.config.DefaultCacheBucketRegion, - CiLogBucketVersioning: impl.config.BlobStorageS3BucketVersioned, - AccessKey: impl.config.AzureAccountName, - } - workflowRequest.CiArtifactLocation = impl.buildDefaultArtifactLocation(savedWf) - workflowRequest.CiArtifactFileName = workflowRequest.CiArtifactLocation - default: - if impl.config.BlobStorageEnabled { - return nil, fmt.Errorf("blob storage %s not supported", workflowRequest.CloudProvider) - } - } - return workflowRequest, nil -} - -func (impl *CiServiceImpl) GetWorkflowRequestVariablesForCopyContainerImagePlugin(preCiSteps []*pipelineConfigBean.StepObject, postCiSteps []*pipelineConfigBean.StepObject, customTag string, customTagId int, buildImagePath string, buildImagedockerRegistryId string) (map[string][]string, map[string]bean2.RegistryCredentials, string, []int, error) { - - copyContainerImagePluginDetail, err := impl.globalPluginService.GetRefPluginIdByRefPluginName(COPY_CONTAINER_IMAGE) - if err != nil && err != pg.ErrNoRows { - impl.Logger.Errorw("error in getting copyContainerImage plugin id", "err", err) - return nil, nil, "", nil, err - } - - pluginIdToVersionMap := make(map[int]string) - for _, p := range copyContainerImagePluginDetail { - pluginIdToVersionMap[p.Id] = p.Version - } - - for _, step := range preCiSteps { - if _, ok := pluginIdToVersionMap[step.RefPluginId]; ok { - // for copyContainerImage plugin parse destination images and save its data in image path reservation table - return nil, nil, "", nil, errors.New("copyContainerImage plugin not allowed in pre-ci step, please remove it and try again") - } - } - - registryCredentialMap := make(map[string]bean2.RegistryCredentials) - registryDestinationImageMap := make(map[string][]string) - var allDestinationImages []string //saving all images to be reserved in this array - - for _, step := range postCiSteps { - if version, ok := pluginIdToVersionMap[step.RefPluginId]; ok { - destinationImageMap, credentialMap, err := impl.pluginInputVariableParser.HandleCopyContainerImagePluginInputVariables(step.InputVars, customTag, buildImagePath, buildImagedockerRegistryId) - if err != nil { - impl.Logger.Errorw("error in parsing copyContainerImage input variable", "err", err) - return nil, nil, "", nil, err - } - if version == COPY_CONTAINER_IMAGE_VERSION_V1 { - // this is needed in ci runner only for v1 - registryDestinationImageMap = destinationImageMap - } - for _, images := range destinationImageMap { - allDestinationImages = append(allDestinationImages, images...) - } - for k, v := range credentialMap { - registryCredentialMap[k] = v - } - } - } - - pluginArtifactStage := repository5.POST_CI - for _, image := range allDestinationImages { - if image == buildImagePath { - return nil, registryCredentialMap, pluginArtifactStage, nil, - pipelineConfigBean.ErrImagePathInUse - } - } - savedCIArtifacts, err := impl.ciArtifactRepository.FindCiArtifactByImagePaths(allDestinationImages) - if err != nil { - impl.Logger.Errorw("error in fetching artifacts by image path", "err", err) - return nil, nil, pluginArtifactStage, nil, err - } - if len(savedCIArtifacts) > 0 { - // if already present in ci artifact, return "image path already in use error" - return nil, nil, pluginArtifactStage, nil, pipelineConfigBean.ErrImagePathInUse - } - imagePathReservationIds, err := impl.ReserveImagesGeneratedAtPlugin(customTagId, allDestinationImages) - if err != nil { - return nil, nil, pluginArtifactStage, imagePathReservationIds, err - } - return registryDestinationImageMap, registryCredentialMap, pluginArtifactStage, imagePathReservationIds, nil -} - -func (impl *CiServiceImpl) ReserveImagesGeneratedAtPlugin(customTagId int, destinationImages []string) ([]int, error) { - var imagePathReservationIds []int - for _, image := range destinationImages { - imagePathReservationData, err := impl.customTagService.ReserveImagePath(image, customTagId) - if err != nil { - impl.Logger.Errorw("Error in marking custom tag reserved", "err", err) - return imagePathReservationIds, err - } - imagePathReservationIds = append(imagePathReservationIds, imagePathReservationData.Id) - } - return imagePathReservationIds, nil -} - -func buildCiStepsDataFromDockerBuildScripts(dockerBuildScripts []*bean.CiScript) []*pipelineConfigBean.StepObject { - // before plugin support, few variables were set as env vars in ci-runner - // these variables are now moved to global vars in plugin steps, but to avoid error in old scripts adding those variables in payload - inputVars := []*commonBean.VariableObject{ - { - Name: "DOCKER_IMAGE_TAG", - Format: "STRING", - VariableType: commonBean.VariableTypeRefGlobal, - ReferenceVariableName: "DOCKER_IMAGE_TAG", - }, - { - Name: "DOCKER_REPOSITORY", - Format: "STRING", - VariableType: commonBean.VariableTypeRefGlobal, - ReferenceVariableName: "DOCKER_REPOSITORY", - }, - { - Name: "DOCKER_REGISTRY_URL", - Format: "STRING", - VariableType: commonBean.VariableTypeRefGlobal, - ReferenceVariableName: "DOCKER_REGISTRY_URL", - }, - { - Name: "DOCKER_IMAGE", - Format: "STRING", - VariableType: commonBean.VariableTypeRefGlobal, - ReferenceVariableName: "DOCKER_IMAGE", - }, - } - var ciSteps []*pipelineConfigBean.StepObject - for _, dockerBuildScript := range dockerBuildScripts { - ciStep := &pipelineConfigBean.StepObject{ - Name: dockerBuildScript.Name, - Index: dockerBuildScript.Index, - Script: dockerBuildScript.Script, - ArtifactPaths: []string{dockerBuildScript.OutputLocation}, - StepType: string(repository.PIPELINE_STEP_TYPE_INLINE), - ExecutorType: string(repository2.SCRIPT_TYPE_SHELL), - InputVars: inputVars, - } - ciSteps = append(ciSteps, ciStep) - } - return ciSteps -} - -func (impl *CiServiceImpl) buildImageTag(commitHashes map[int]pipelineConfig.GitCommit, id int, wfId int) string { - dockerImageTag := "" - toAppendDevtronParamInTag := true - for _, v := range commitHashes { - if v.WebhookData.Id == 0 { - if v.Commit == "" { - continue - } - dockerImageTag = getUpdatedDockerImageTagWithCommitOrCheckOutData(dockerImageTag, _getTruncatedImageTag(v.Commit)) - } else { - _targetCheckout := v.WebhookData.Data[bean.WEBHOOK_SELECTOR_TARGET_CHECKOUT_NAME] - if _targetCheckout == "" { - continue - } - // if not PR based then meaning tag based - isPRBasedEvent := v.WebhookData.EventActionType == bean.WEBHOOK_EVENT_MERGED_ACTION_TYPE - if !isPRBasedEvent && impl.config.CiCdConfig.UseImageTagFromGitProviderForTagBasedBuild { - dockerImageTag = getUpdatedDockerImageTagWithCommitOrCheckOutData(dockerImageTag, _targetCheckout) - } else { - dockerImageTag = getUpdatedDockerImageTagWithCommitOrCheckOutData(dockerImageTag, _getTruncatedImageTag(_targetCheckout)) - } - if isPRBasedEvent { - _sourceCheckout := v.WebhookData.Data[bean.WEBHOOK_SELECTOR_SOURCE_CHECKOUT_NAME] - dockerImageTag = getUpdatedDockerImageTagWithCommitOrCheckOutData(dockerImageTag, _getTruncatedImageTag(_sourceCheckout)) - } else { - toAppendDevtronParamInTag = !impl.config.CiCdConfig.UseImageTagFromGitProviderForTagBasedBuild - } - } - } - toAppendDevtronParamInTag = toAppendDevtronParamInTag && dockerImageTag != "" - if toAppendDevtronParamInTag { - dockerImageTag = fmt.Sprintf("%s-%d-%d", dockerImageTag, id, wfId) - } - // replace / with underscore, as docker image tag doesn't support slash. it gives error - dockerImageTag = strings.ReplaceAll(dockerImageTag, "/", "_") - return dockerImageTag -} - -func getUpdatedDockerImageTagWithCommitOrCheckOutData(dockerImageTag, commitOrCheckOutData string) string { - if dockerImageTag == "" { - dockerImageTag = commitOrCheckOutData - } else { - if commitOrCheckOutData != "" { - dockerImageTag = fmt.Sprintf("%s-%s", dockerImageTag, commitOrCheckOutData) - } - } - return dockerImageTag -} - -func (impl *CiServiceImpl) updateCiWorkflow(request *types.WorkflowRequest, savedWf *pipelineConfig.CiWorkflow) error { - ciBuildConfig := request.CiBuildConfig - ciBuildType := string(ciBuildConfig.CiBuildType) - savedWf.CiBuildType = ciBuildType - return impl.UpdateCiWorkflowWithStage(savedWf) -} - -func _getTruncatedImageTag(imageTag string) string { - _length := len(imageTag) - if _length == 0 { - return imageTag - } - - _truncatedLength := 8 - - if _length < _truncatedLength { - return imageTag - } else { - return imageTag[:_truncatedLength] +func (impl *CiServiceImpl) WriteCIFailEvent(ciWorkflow *pipelineConfig.CiWorkflow) { + event, _ := impl.eventFactory.Build(util2.Fail, &ciWorkflow.CiPipelineId, ciWorkflow.CiPipeline.AppId, nil, util2.CI) + material := &buildBean.MaterialTriggerInfo{} + material.GitTriggers = ciWorkflow.GitTriggers + event.CiWorkflowRunnerId = ciWorkflow.Id + event.UserId = int(ciWorkflow.TriggeredBy) + event = impl.eventFactory.BuildExtraCIData(event, material) + event.CiArtifactId = 0 + _, evtErr := impl.eventClient.WriteNotificationEvent(event) + if evtErr != nil { + impl.Logger.Errorw("error in writing event", "err", evtErr) } } diff --git a/pkg/pipeline/CiService_ent.go b/pkg/pipeline/CiService_ent.go deleted file mode 100644 index 342429d052..0000000000 --- a/pkg/pipeline/CiService_ent.go +++ /dev/null @@ -1,16 +0,0 @@ -package pipeline - -import ( - "github.com/devtron-labs/common-lib/imageScan/bean" - bean2 "github.com/devtron-labs/devtron/pkg/pipeline/bean" - "github.com/devtron-labs/devtron/pkg/pipeline/types" - "github.com/devtron-labs/devtron/pkg/policyGovernance/security/scanTool/repository" -) - -func (impl *CiServiceImpl) fetchImageScanExecutionMedium() (*repository.ScanToolMetadata, bean.ScanExecutionMedium, error) { - return &repository.ScanToolMetadata{}, "", nil -} - -func (impl *CiServiceImpl) fetchImageScanExecutionStepsForWfRequest(scanToolMetadata *repository.ScanToolMetadata) ([]*types.ImageScanningSteps, []*bean2.RefPluginObject, error) { - return nil, nil, nil -} diff --git a/pkg/pipeline/PipelineStageService.go b/pkg/pipeline/PipelineStageService.go index 2ff4123f0f..2a538b7282 100644 --- a/pkg/pipeline/PipelineStageService.go +++ b/pkg/pipeline/PipelineStageService.go @@ -74,6 +74,11 @@ func NewPipelineStageService(logger *zap.SugaredLogger, } } +const ( + preCdStage = "preCD" + postCdStage = "postCD" +) + type PipelineStageServiceImpl struct { logger *zap.SugaredLogger pipelineStageRepository repository.PipelineStageRepository diff --git a/pkg/pipeline/adapter/adapter.go b/pkg/pipeline/adapter/adapter.go index badb5c0f3e..d480731a2a 100644 --- a/pkg/pipeline/adapter/adapter.go +++ b/pkg/pipeline/adapter/adapter.go @@ -23,6 +23,7 @@ import ( "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/ciPipeline" "github.com/devtron-labs/devtron/pkg/bean" bean2 "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" + "github.com/devtron-labs/devtron/pkg/build/pipeline/bean/common" bean3 "github.com/devtron-labs/devtron/pkg/cluster/environment/bean" repository2 "github.com/devtron-labs/devtron/pkg/cluster/environment/repository" pipelineConfigBean "github.com/devtron-labs/devtron/pkg/pipeline/bean" @@ -190,7 +191,7 @@ func mergeMap(oldDockerArgs map[string]string, ciLevelDockerArgs map[string]stri // IsLinkedCD will return if the pipelineConfig.CiPipeline is a Linked CD func IsLinkedCD(ci pipelineConfig.CiPipeline) bool { - return ci.ParentCiPipeline != 0 && ci.PipelineType == string(bean2.LINKED_CD) + return ci.ParentCiPipeline != 0 && ci.PipelineType == string(common.LINKED_CD) } // IsLinkedCI will return if the pipelineConfig.CiPipeline is a Linked CI @@ -199,7 +200,7 @@ func IsLinkedCI(ci *pipelineConfig.CiPipeline) bool { return false } return ci.ParentCiPipeline != 0 && - ci.PipelineType == string(bean2.LINKED) + ci.PipelineType == string(common.LINKED) } // IsCIJob will return if the pipelineConfig.CiPipeline is a CI JOB @@ -207,7 +208,7 @@ func IsCIJob(ci *pipelineConfig.CiPipeline) bool { if ci == nil { return false } - return ci.PipelineType == string(bean2.CI_JOB) + return ci.PipelineType == string(common.CI_JOB) } // GetSourceCiDownStreamResponse will take the models []bean.LinkedCIDetails and []pipelineConfig.CdWorkflowRunner (for last deployment status) and generate the []CiPipeline.SourceCiDownStreamResponse diff --git a/pkg/pipeline/bean/CdHandlerBean.go b/pkg/pipeline/bean/CdHandlerBean.go index 4b6480e365..5dc408c226 100644 --- a/pkg/pipeline/bean/CdHandlerBean.go +++ b/pkg/pipeline/bean/CdHandlerBean.go @@ -4,37 +4,38 @@ import ( "github.com/devtron-labs/common-lib/utils/bean" "github.com/devtron-labs/devtron/internal/sql/repository/imageTagging" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" + buildBean "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" bean2 "github.com/devtron-labs/devtron/pkg/pipeline/workflowStatus/bean" "time" ) type CdWorkflowWithArtifact struct { - Id int `json:"id"` - CdWorkflowId int `json:"cd_workflow_id"` - Name string `json:"name"` - Status string `json:"status"` - PodStatus string `json:"pod_status"` - Message string `json:"message"` - StartedOn time.Time `json:"started_on"` - FinishedOn time.Time `json:"finished_on"` - PipelineId int `json:"pipeline_id"` - Namespace string `json:"namespace"` - LogFilePath string `json:"log_file_path"` - TriggeredBy int32 `json:"triggered_by"` - EmailId string `json:"email_id"` - Image string `json:"image"` - TargetPlatforms []*bean.TargetPlatform `json:"targetPlatforms"` - MaterialInfo string `json:"material_info,omitempty"` - DataSource string `json:"data_source,omitempty"` - CiArtifactId int `json:"ci_artifact_id,omitempty"` - IsArtifactUploaded bool `json:"isArtifactUploaded"` - WorkflowType string `json:"workflow_type,omitempty"` - ExecutorType string `json:"executor_type,omitempty"` - BlobStorageEnabled bool `json:"blobStorageEnabled"` - GitTriggers map[int]pipelineConfig.GitCommit `json:"gitTriggers"` - CiMaterials []pipelineConfig.CiPipelineMaterialResponse `json:"ciMaterials"` - ImageReleaseTags []*repository.ImageTag `json:"imageReleaseTags"` - ImageComment *repository.ImageComment `json:"imageComment"` - RefCdWorkflowRunnerId int `json:"referenceCdWorkflowRunnerId"` - WorkflowExecutionStage map[string][]*bean2.WorkflowStageDto `json:"workflowExecutionStages"` + Id int `json:"id"` + CdWorkflowId int `json:"cd_workflow_id"` + Name string `json:"name"` + Status string `json:"status"` + PodStatus string `json:"pod_status"` + Message string `json:"message"` + StartedOn time.Time `json:"started_on"` + FinishedOn time.Time `json:"finished_on"` + PipelineId int `json:"pipeline_id"` + Namespace string `json:"namespace"` + LogFilePath string `json:"log_file_path"` + TriggeredBy int32 `json:"triggered_by"` + EmailId string `json:"email_id"` + Image string `json:"image"` + TargetPlatforms []*bean.TargetPlatform `json:"targetPlatforms"` + MaterialInfo string `json:"material_info,omitempty"` + DataSource string `json:"data_source,omitempty"` + CiArtifactId int `json:"ci_artifact_id,omitempty"` + IsArtifactUploaded bool `json:"isArtifactUploaded"` + WorkflowType string `json:"workflow_type,omitempty"` + ExecutorType string `json:"executor_type,omitempty"` + BlobStorageEnabled bool `json:"blobStorageEnabled"` + GitTriggers map[int]pipelineConfig.GitCommit `json:"gitTriggers"` + CiMaterials []buildBean.CiPipelineMaterialResponse `json:"ciMaterials"` + ImageReleaseTags []*repository.ImageTag `json:"imageReleaseTags"` + ImageComment *repository.ImageComment `json:"imageComment"` + RefCdWorkflowRunnerId int `json:"referenceCdWorkflowRunnerId"` + WorkflowExecutionStage map[string][]*bean2.WorkflowStageDto `json:"workflowExecutionStages"` } diff --git a/pkg/pipeline/bean/WorkflowTemplate.go b/pkg/pipeline/bean/WorkflowTemplate.go index 55e91e565c..dcab419b5e 100644 --- a/pkg/pipeline/bean/WorkflowTemplate.go +++ b/pkg/pipeline/bean/WorkflowTemplate.go @@ -18,6 +18,7 @@ package bean import ( blob_storage "github.com/devtron-labs/common-lib/blob-storage" + informerBean "github.com/devtron-labs/common-lib/informer" "github.com/devtron-labs/devtron/api/bean" v1 "k8s.io/api/core/v1" v12 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -47,17 +48,20 @@ type WorkflowTemplate struct { RefPlugins []*RefPluginObject TerminationGracePeriod int WorkflowType string + DevtronInstanceUID string } const ( - CI_WORKFLOW_NAME = "ci" - CI_WORKFLOW_WITH_STAGES = "ci-stages-with-env" - CiStage = "CI" - JobStage = "JOB" - CdStage = "CD" - CD_WORKFLOW_NAME = "cd" - CD_WORKFLOW_WITH_STAGES = "cd-stages-with-env" - WorkflowGenerateNamePrefix = "devtron.ai/generate-name-prefix" + CI_WORKFLOW_NAME = "ci" + CI_WORKFLOW_WITH_STAGES = "ci-stages-with-env" + CiStage = "CI" + JobStage = "JOB" + CdStage = "CD" + CD_WORKFLOW_NAME = "cd" + CD_WORKFLOW_WITH_STAGES = "cd-stages-with-env" + WorkflowGenerateNamePrefix = "devtron.ai/generate-name-prefix" + DevtronLabelPurposeKey = "devtron.ai/purpose" + DevtronLabelPurposeWorkflow = "workflow" ) func (workflowTemplate *WorkflowTemplate) GetEntrypoint() string { @@ -76,8 +80,11 @@ func (workflowTemplate *WorkflowTemplate) SetActiveDeadlineSeconds(timeout int64 } func (workflowTemplate *WorkflowTemplate) CreateObjectMetadata() *v12.ObjectMeta { - - workflowLabels := map[string]string{WorkflowGenerateNamePrefix: workflowTemplate.WorkflowNamePrefix} + workflowLabels := map[string]string{ + WorkflowGenerateNamePrefix: workflowTemplate.WorkflowNamePrefix, + informerBean.WorkflowTypeLabelKey: workflowTemplate.WorkflowType, + informerBean.DevtronOwnerInstanceLabelKey: workflowTemplate.DevtronInstanceUID, + } switch workflowTemplate.WorkflowType { case CI_WORKFLOW_NAME: workflowLabels["devtron.ai/workflow-purpose"] = "ci" diff --git a/pkg/pipeline/constants/constants.go b/pkg/pipeline/constants/constants.go index 0d5cd5bcab..0028c37d45 100644 --- a/pkg/pipeline/constants/constants.go +++ b/pkg/pipeline/constants/constants.go @@ -19,19 +19,6 @@ package constants const CDPipelineNotFoundErr = "cd pipeline not found" const CiPipelineNotFoundErr = "ci pipeline not found" -type PipelineType string - -// default PipelineType -const DefaultPipelineType = CI_BUILD - -const ( - CI_BUILD PipelineType = "CI_BUILD" - LINKED PipelineType = "LINKED" - EXTERNAL PipelineType = "EXTERNAL" - CI_JOB PipelineType = "CI_JOB" - LINKED_CD PipelineType = "LINKED_CD" -) - type PatchPipelineActionResponse string const ( @@ -43,32 +30,13 @@ const ( PATCH_PIPELINE_ACTION_NOT_APPLICABLE PatchPipelineActionResponse = "N/A" ) -func (r PipelineType) ToString() string { - return string(r) -} - -func (pType PipelineType) IsValidPipelineType() bool { - switch pType { - case CI_BUILD, LINKED, EXTERNAL, CI_JOB, LINKED_CD: - return true - default: - return false - } -} - const ( UNIQUE_DEPLOYMENT_APP_NAME = "unique_deployment_app_name" ) -const ( - ExtraEnvVarExternalCiArtifactKey = "externalCiArtifact" - ExtraEnvVarImageDigestKey = "imageDigest" -) - const DefaultCiWorkflowNamespace = "devtron-ci" const Running = "Running" const Starting = "Starting" -const POD_DELETED_MESSAGE = "pod deleted" const TERMINATE_MESSAGE = "workflow shutdown with strategy: Terminate" const FORCE_ABORT_MESSAGE_AFTER_STARTING_STAGE = "workflow shutdown with strategy: Force Abort" const POD_TIMEOUT_MESSAGE = "Pod was active on the node longer than the specified deadline" diff --git a/pkg/pipeline/executors/WorkflowUtils.go b/pkg/pipeline/executors/WorkflowUtils.go index 0c9f18e911..bbf7439660 100644 --- a/pkg/pipeline/executors/WorkflowUtils.go +++ b/pkg/pipeline/executors/WorkflowUtils.go @@ -21,6 +21,7 @@ import ( "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" "github.com/argoproj/argo-workflows/v3/pkg/client/clientset/versioned" v1alpha12 "github.com/argoproj/argo-workflows/v3/pkg/client/clientset/versioned/typed/workflow/v1alpha1" + informerBean "github.com/devtron-labs/common-lib/informer" "github.com/devtron-labs/common-lib/utils" bean2 "github.com/devtron-labs/devtron/api/bean" "github.com/devtron-labs/devtron/internal/sql/repository" @@ -287,16 +288,20 @@ func GetClientInstance(config *rest.Config, namespace string) (v1alpha12.Workflo } func CheckIfReTriggerRequired(status, message, workflowRunnerStatus string) bool { - return ((status == string(v1alpha1.NodeError) || status == string(v1alpha1.NodeFailed)) && - message == cdWorkflow.POD_DELETED_MESSAGE) && (workflowRunnerStatus != cdWorkflow.WorkflowCancel && workflowRunnerStatus != cdWorkflow.WorkflowAborted) + return (status == string(v1alpha1.NodeError) || status == string(v1alpha1.NodeFailed)) && + message == cdWorkflow.POD_DELETED_MESSAGE && + workflowRunnerStatus != cdWorkflow.WorkflowCancel && + workflowRunnerStatus != cdWorkflow.WorkflowAborted && + workflowRunnerStatus != cdWorkflow.WorkflowSucceeded } func GetWorkflowLabelsForSystemExecutor(workflowTemplate bean.WorkflowTemplate) map[string]string { return map[string]string{ - DEVTRON_WORKFLOW_LABEL_KEY: DEVTRON_WORKFLOW_LABEL_VALUE, - "devtron.ai/purpose": "workflow", - "workflowType": workflowTemplate.WorkflowType, - bean.WorkflowGenerateNamePrefix: workflowTemplate.WorkflowNamePrefix, + DEVTRON_WORKFLOW_LABEL_KEY: DEVTRON_WORKFLOW_LABEL_VALUE, + bean.DevtronLabelPurposeKey: bean.DevtronLabelPurposeWorkflow, + informerBean.WorkflowTypeLabelKey: workflowTemplate.WorkflowType, + bean.WorkflowGenerateNamePrefix: workflowTemplate.WorkflowNamePrefix, + informerBean.DevtronOwnerInstanceLabelKey: workflowTemplate.DevtronInstanceUID, } } diff --git a/pkg/pipeline/pipelineStageVariableParser.go b/pkg/pipeline/pipelineStageVariableParser.go index b97517d30b..bd0d3bce77 100644 --- a/pkg/pipeline/pipelineStageVariableParser.go +++ b/pkg/pipeline/pipelineStageVariableParser.go @@ -30,13 +30,9 @@ import ( ) type copyContainerImagePluginInputVariable = string -type RefPluginName = string const ( - COPY_CONTAINER_IMAGE RefPluginName = "Copy container image" - COPY_CONTAINER_IMAGE_VERSION_V1 = "1.0.0" - COPY_CONTAINER_IMAGE_VERSION_V2 = "2.0.0" - EMPTY_STRING = " " + EMPTY_STRING = " " ) const ( diff --git a/pkg/pipeline/types/CiCdConfig.go b/pkg/pipeline/types/CiCdConfig.go index 5d50162141..246a0ec801 100644 --- a/pkg/pipeline/types/CiCdConfig.go +++ b/pkg/pipeline/types/CiCdConfig.go @@ -27,6 +27,7 @@ import ( "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow/cdWorkflow" "github.com/devtron-labs/devtron/pkg/bean/common" + buildBean "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" "github.com/devtron-labs/devtron/pkg/cluster/environment/repository" "github.com/devtron-labs/devtron/pkg/pipeline/bean" v12 "k8s.io/api/core/v1" @@ -550,17 +551,18 @@ type ArtifactsForCiJob struct { } type GitTriggerInfoResponse struct { - CiMaterials []pipelineConfig.CiPipelineMaterialResponse `json:"ciMaterials"` - TriggeredByEmail string `json:"triggeredByEmail"` - LastDeployedTime string `json:"lastDeployedTime,omitempty"` - AppId int `json:"appId"` - AppName string `json:"appName"` - EnvironmentId int `json:"environmentId"` - EnvironmentName string `json:"environmentName"` - Default bool `json:"default,omitempty"` - ImageTaggingData ImageTaggingResponseDTO `json:"imageTaggingData"` - Image string `json:"image"` - TargetPlatforms []*bean2.TargetPlatform `json:"targetPlatforms"` + CiMaterials []buildBean.CiPipelineMaterialResponse `json:"ciMaterials"` + TriggeredByEmail string `json:"triggeredByEmail"` + LastDeployedTime string `json:"lastDeployedTime,omitempty"` + CiPipelineId int `json:"ciPipelineId"` + AppId int `json:"appId"` + AppName string `json:"appName"` + EnvironmentId int `json:"environmentId"` + EnvironmentName string `json:"environmentName"` + Default bool `json:"default,omitempty"` + ImageTaggingData ImageTaggingResponseDTO `json:"imageTaggingData"` + Image string `json:"image"` + TargetPlatforms []*bean2.TargetPlatform `json:"targetPlatforms"` } type Trigger struct { diff --git a/pkg/pipeline/types/Workflow.go b/pkg/pipeline/types/Workflow.go index dca45ab8fc..dceb9a03b0 100644 --- a/pkg/pipeline/types/Workflow.go +++ b/pkg/pipeline/types/Workflow.go @@ -31,6 +31,7 @@ import ( "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/workflow/cdWorkflow" bean2 "github.com/devtron-labs/devtron/pkg/bean" bean5 "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" + buildBean "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" repository4 "github.com/devtron-labs/devtron/pkg/cluster/environment/repository" infraBean "github.com/devtron-labs/devtron/pkg/infraConfig/bean/v1" "github.com/devtron-labs/devtron/pkg/pipeline/bean" @@ -649,37 +650,37 @@ type ChildCdMetadata struct { } type WorkflowResponse struct { - Id int `json:"id"` - Name string `json:"name"` - Status string `json:"status"` - PodStatus string `json:"podStatus"` - Message string `json:"message"` - StartedOn time.Time `json:"startedOn"` - FinishedOn time.Time `json:"finishedOn"` - CiPipelineId int `json:"ciPipelineId"` - Namespace string `json:"namespace"` - LogLocation string `json:"logLocation"` - BlobStorageEnabled bool `json:"blobStorageEnabled"` - GitTriggers map[int]pipelineConfig.GitCommit `json:"gitTriggers"` - CiMaterials []pipelineConfig.CiPipelineMaterialResponse `json:"ciMaterials"` - TriggeredBy int32 `json:"triggeredBy"` - Artifact string `json:"artifact"` - TriggeredByEmail string `json:"triggeredByEmail"` - Stage string `json:"stage"` - ArtifactId int `json:"artifactId"` - IsArtifactUploaded bool `json:"isArtifactUploaded"` - IsVirtualEnvironment bool `json:"isVirtualEnvironment"` - PodName string `json:"podName"` - EnvironmentId int `json:"environmentId"` - EnvironmentName string `json:"environmentName"` - ImageReleaseTags []*repository3.ImageTag `json:"imageReleaseTags"` - ImageComment *repository3.ImageComment `json:"imageComment"` - AppWorkflowId int `json:"appWorkflowId"` - CustomTag *bean3.CustomTagErrorResponse `json:"customTag,omitempty"` - PipelineType string `json:"pipelineType"` - ReferenceWorkflowId int `json:"referenceWorkflowId"` - TargetPlatforms []*bean7.TargetPlatform `json:"targetPlatforms"` - WorkflowExecutionStage map[string][]*bean6.WorkflowStageDto `json:"workflowExecutionStages"` + Id int `json:"id"` + Name string `json:"name"` + Status string `json:"status"` + PodStatus string `json:"podStatus"` + Message string `json:"message"` + StartedOn time.Time `json:"startedOn"` + FinishedOn time.Time `json:"finishedOn"` + CiPipelineId int `json:"ciPipelineId"` + Namespace string `json:"namespace"` + LogLocation string `json:"logLocation"` + BlobStorageEnabled bool `json:"blobStorageEnabled"` + GitTriggers map[int]pipelineConfig.GitCommit `json:"gitTriggers"` + CiMaterials []buildBean.CiPipelineMaterialResponse `json:"ciMaterials"` + TriggeredBy int32 `json:"triggeredBy"` + Artifact string `json:"artifact"` + TriggeredByEmail string `json:"triggeredByEmail"` + Stage string `json:"stage"` + ArtifactId int `json:"artifactId"` + IsArtifactUploaded bool `json:"isArtifactUploaded"` + IsVirtualEnvironment bool `json:"isVirtualEnvironment"` + PodName string `json:"podName"` + EnvironmentId int `json:"environmentId"` + EnvironmentName string `json:"environmentName"` + ImageReleaseTags []*repository3.ImageTag `json:"imageReleaseTags"` + ImageComment *repository3.ImageComment `json:"imageComment"` + AppWorkflowId int `json:"appWorkflowId"` + CustomTag *bean3.CustomTagErrorResponse `json:"customTag,omitempty"` + PipelineType string `json:"pipelineType"` + ReferenceWorkflowId int `json:"referenceWorkflowId"` + TargetPlatforms []*bean7.TargetPlatform `json:"targetPlatforms"` + WorkflowExecutionStage map[string][]*bean6.WorkflowStageDto `json:"workflowExecutionStages"` } type ConfigMapSecretDto struct { diff --git a/pkg/pipeline/types/Workflow_ent.go b/pkg/pipeline/types/Workflow_ent.go index 2a5c03e78f..ecf78b9e1a 100644 --- a/pkg/pipeline/types/Workflow_ent.go +++ b/pkg/pipeline/types/Workflow_ent.go @@ -30,6 +30,6 @@ func (workflowRequest *WorkflowRequest) SetImageScanningSteps(imageScanningSteps return } -func (workflowRequest *WorkflowRequest) SetAwsInspectorConfig(awsInspectorConfig string) { +func (workflowRequest *WorkflowRequest) SetEntOnlyFields(trigger Trigger, ciConfig *CiConfig) { return } diff --git a/pkg/ucid/bean.go b/pkg/ucid/bean.go new file mode 100644 index 0000000000..acd595ad57 --- /dev/null +++ b/pkg/ucid/bean.go @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2024. Devtron Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ucid + +const ( + DevtronUniqueClientIdConfigMap = "devtron-ucid" + DevtronUniqueClientIdConfigMapKey = "UCID" + UIEventKey = "uiEventKey" + InstallEventKey = "installEvent" +) diff --git a/pkg/ucid/ucid.go b/pkg/ucid/ucid.go new file mode 100644 index 0000000000..65c03e8887 --- /dev/null +++ b/pkg/ucid/ucid.go @@ -0,0 +1,96 @@ +/* + * Copyright (c) 2024. Devtron Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ucid + +import ( + "fmt" + "github.com/devtron-labs/common-lib/utils/k8s" + "github.com/devtron-labs/devtron/pkg/util" + globalUtil "github.com/devtron-labs/devtron/util" + goCache "github.com/patrickmn/go-cache" + "go.uber.org/zap" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +type Service interface { + GetUCIDWithCache(cache *goCache.Cache) (string, bool, error) + GetUCIDWithOutCache() (string, bool, error) +} + +type ServiceImpl struct { + logger *zap.SugaredLogger + K8sUtil *k8s.K8sServiceImpl + aCDAuthConfig *util.ACDAuthConfig +} + +func NewServiceImpl( + logger *zap.SugaredLogger, + K8sUtil *k8s.K8sServiceImpl, + aCDAuthConfig *util.ACDAuthConfig, +) *ServiceImpl { + return &ServiceImpl{ + logger: logger, + K8sUtil: K8sUtil, + aCDAuthConfig: aCDAuthConfig, + } +} + +func (impl *ServiceImpl) GetUCIDWithOutCache() (string, bool, error) { + return impl.GetUCIDWithCache(nil) +} + +func (impl *ServiceImpl) GetUCIDWithCache(cache *goCache.Cache) (string, bool, error) { + if cache != nil { + ucid, found := cache.Get(DevtronUniqueClientIdConfigMapKey) + if found { + return ucid.(string), true, nil + } + } + client, err := impl.K8sUtil.GetClientForInCluster() + if err != nil { + impl.logger.Errorw("exception while getting unique client id", "error", err) + return "", false, err + } + cm, err := impl.K8sUtil.GetConfigMap(impl.aCDAuthConfig.ACDConfigMapNamespace, DevtronUniqueClientIdConfigMap, client) + if errStatus, ok := status.FromError(err); !ok || errStatus.Code() == codes.NotFound || errStatus.Code() == codes.Unknown { + // if not found, create new cm + cm = &v1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: DevtronUniqueClientIdConfigMap}} + data := map[string]string{} + data[DevtronUniqueClientIdConfigMapKey] = globalUtil.Generate(16) // generate unique random number + data[InstallEventKey] = "1" // used in operator to detect event is install or upgrade + data[UIEventKey] = "1" + cm.Data = data + _, err = impl.K8sUtil.CreateConfigMap(impl.aCDAuthConfig.ACDConfigMapNamespace, cm, client) + if err != nil { + impl.logger.Errorw("exception while getting unique client id", "error", err) + return "", false, err + } + } + if cm == nil { + impl.logger.Errorw("configmap not found while getting unique client id") + return "", false, fmt.Errorf("configmap %q not found while getting unique client id", DevtronUniqueClientIdConfigMap) + } + dataMap := cm.Data + ucid := dataMap[DevtronUniqueClientIdConfigMapKey] + if cache != nil { + cache.Set(DevtronUniqueClientIdConfigMapKey, ucid, goCache.DefaultExpiration) + } + return ucid, false, nil +} diff --git a/pkg/ucid/wire_ucid.go b/pkg/ucid/wire_ucid.go new file mode 100644 index 0000000000..f8f4402af2 --- /dev/null +++ b/pkg/ucid/wire_ucid.go @@ -0,0 +1,8 @@ +package ucid + +import "github.com/google/wire" + +var WireSet = wire.NewSet( + NewServiceImpl, + wire.Bind(new(Service), new(*ServiceImpl)), +) diff --git a/pkg/workflow/dag/WorkflowDagExecutor.go b/pkg/workflow/dag/WorkflowDagExecutor.go index d87f6bfd97..a53e90dbd6 100644 --- a/pkg/workflow/dag/WorkflowDagExecutor.go +++ b/pkg/workflow/dag/WorkflowDagExecutor.go @@ -24,6 +24,7 @@ import ( "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" "github.com/devtron-labs/common-lib/async" "github.com/devtron-labs/common-lib/utils" + "github.com/devtron-labs/common-lib/utils/k8s" "github.com/devtron-labs/common-lib/utils/workFlow" bean6 "github.com/devtron-labs/devtron/api/helm-app/bean" client2 "github.com/devtron-labs/devtron/api/helm-app/service" @@ -38,6 +39,10 @@ import ( bean7 "github.com/devtron-labs/devtron/pkg/auth/user/bean" "github.com/devtron-labs/devtron/pkg/build/artifacts" bean5 "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" + buildCommonBean "github.com/devtron-labs/devtron/pkg/build/pipeline/bean/common" + "github.com/devtron-labs/devtron/pkg/build/trigger" + "github.com/devtron-labs/devtron/pkg/cluster/adapter" + repository5 "github.com/devtron-labs/devtron/pkg/cluster/environment/repository" common2 "github.com/devtron-labs/devtron/pkg/deployment/common" "github.com/devtron-labs/devtron/pkg/deployment/manifest" "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps" @@ -45,6 +50,8 @@ import ( triggerBean "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/bean" "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/userDeploymentRequest/service" eventProcessorBean "github.com/devtron-labs/devtron/pkg/eventProcessor/bean" + "github.com/devtron-labs/devtron/pkg/executor" + k8sPkg "github.com/devtron-labs/devtron/pkg/k8s" "github.com/devtron-labs/devtron/pkg/pipeline" constants2 "github.com/devtron-labs/devtron/pkg/pipeline/constants" repository2 "github.com/devtron-labs/devtron/pkg/plugin/repository" @@ -58,6 +65,9 @@ import ( "github.com/devtron-labs/devtron/pkg/workflow/dag/helper" error2 "github.com/devtron-labs/devtron/util/error" util2 "github.com/devtron-labs/devtron/util/event" + errors2 "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/client-go/rest" + "net/http" "strings" "sync" "time" @@ -83,6 +93,7 @@ import ( ) type WorkflowDagExecutor interface { + UpdateCiWorkflowStatusFailure(timeoutForFailureCiBuild int) error UpdateCiWorkflowForCiSuccess(request *bean2.CiArtifactWebhookRequest) (err error) HandleCiSuccessEvent(triggerContext triggerBean.TriggerContext, ciPipelineId int, request *bean2.CiArtifactWebhookRequest, imagePushedAt time.Time) (id int, err error) HandlePreStageSuccessEvent(triggerContext triggerBean.TriggerContext, cdStageCompleteEvent eventProcessorBean.CdStageCompleteEvent) error @@ -125,7 +136,7 @@ type WorkflowDagExecutorImpl struct { helmAppService client2.HelmAppService cdWorkflowCommonService cd.CdWorkflowCommonService - cdTriggerService devtronApps.TriggerService + cdHandlerService devtronApps.HandlerService userDeploymentRequestService service.UserDeploymentRequestService manifestCreationService manifest.ManifestCreationService @@ -134,6 +145,12 @@ type WorkflowDagExecutorImpl struct { asyncRunnable *async.Runnable scanHistoryRepository repository3.ImageScanHistoryRepository imageScanService imageScanning.ImageScanService + + K8sUtil *k8s.K8sServiceImpl + envRepository repository5.EnvironmentRepository + k8sCommonService k8sPkg.K8sCommonService + workflowService executor.WorkflowService + ciHandlerService trigger.HandlerService } func NewWorkflowDagExecutorImpl(Logger *zap.SugaredLogger, pipelineRepository pipelineConfig.PipelineRepository, @@ -154,7 +171,7 @@ func NewWorkflowDagExecutorImpl(Logger *zap.SugaredLogger, pipelineRepository pi ciService pipeline.CiService, helmAppService client2.HelmAppService, cdWorkflowCommonService cd.CdWorkflowCommonService, - cdTriggerService devtronApps.TriggerService, + cdHandlerService devtronApps.HandlerService, userDeploymentRequestService service.UserDeploymentRequestService, manifestCreationService manifest.ManifestCreationService, commonArtifactService artifacts.CommonArtifactService, @@ -162,6 +179,11 @@ func NewWorkflowDagExecutorImpl(Logger *zap.SugaredLogger, pipelineRepository pi asyncRunnable *async.Runnable, scanHistoryRepository repository3.ImageScanHistoryRepository, imageScanService imageScanning.ImageScanService, + K8sUtil *k8s.K8sServiceImpl, + envRepository repository5.EnvironmentRepository, + k8sCommonService k8sPkg.K8sCommonService, + workflowService executor.WorkflowService, + ciHandlerService trigger.HandlerService, ) *WorkflowDagExecutorImpl { wde := &WorkflowDagExecutorImpl{logger: Logger, pipelineRepository: pipelineRepository, @@ -180,7 +202,7 @@ func NewWorkflowDagExecutorImpl(Logger *zap.SugaredLogger, pipelineRepository pi pipelineStatusTimelineService: pipelineStatusTimelineService, helmAppService: helmAppService, cdWorkflowCommonService: cdWorkflowCommonService, - cdTriggerService: cdTriggerService, + cdHandlerService: cdHandlerService, userDeploymentRequestService: userDeploymentRequestService, manifestCreationService: manifestCreationService, commonArtifactService: commonArtifactService, @@ -190,6 +212,11 @@ func NewWorkflowDagExecutorImpl(Logger *zap.SugaredLogger, pipelineRepository pi imageScanService: imageScanService, cdWorkflowRunnerService: cdWorkflowRunnerService, ciService: ciService, + K8sUtil: K8sUtil, + envRepository: envRepository, + k8sCommonService: k8sCommonService, + workflowService: workflowService, + ciHandlerService: ciHandlerService, } config, err := types.GetCdConfig() if err != nil { @@ -209,6 +236,130 @@ func NewWorkflowDagExecutorImpl(Logger *zap.SugaredLogger, pipelineRepository pi return wde } +func (impl *WorkflowDagExecutorImpl) UpdateCiWorkflowStatusFailure(timeoutForFailureCiBuild int) error { + ciWorkflows, err := impl.ciWorkflowRepository.FindByStatusesIn([]string{constants2.Starting, constants2.Running}) + if err != nil { + impl.logger.Errorw("error on fetching ci workflows", "err", err) + return err + } + client, err := impl.K8sUtil.GetClientForInCluster() + if err != nil { + impl.logger.Errorw("error while fetching k8s client", "error", err) + return err + } + + for _, ciWorkflow := range ciWorkflows { + var isExt bool + var env *repository5.Environment + var restConfig *rest.Config + if ciWorkflow.Namespace != constants2.DefaultCiWorkflowNamespace { + isExt = true + env, err = impl.envRepository.FindById(ciWorkflow.EnvironmentId) + if err != nil { + impl.logger.Errorw("could not fetch stage env", "err", err) + return err + } + restConfig, err = impl.getRestConfig(ciWorkflow) + if err != nil { + return err + } + } + + isEligibleToMarkFailed := false + isPodDeleted := false + if time.Since(ciWorkflow.StartedOn) > (time.Minute * time.Duration(timeoutForFailureCiBuild)) { + + //check weather pod is exists or not, if exits check its status + wf, err := impl.workflowService.GetWorkflowStatus(ciWorkflow.ExecutorType, ciWorkflow.Name, ciWorkflow.Namespace, restConfig) + if err != nil { + impl.logger.Warnw("unable to fetch ci workflow", "err", err) + statusError, ok := err.(*errors2.StatusError) + if ok && statusError.Status().Code == http.StatusNotFound { + impl.logger.Warnw("ci workflow not found", "err", err) + isEligibleToMarkFailed = true + } else { + continue + // skip this and process for next ci workflow + } + } + + //if ci workflow is exists, check its pod + if !isEligibleToMarkFailed { + ns := constants2.DefaultCiWorkflowNamespace + if isExt { + _, client, err = impl.k8sCommonService.GetCoreClientByClusterId(env.ClusterId) + if err != nil { + impl.logger.Warnw("error in getting core v1 client using GetCoreClientByClusterId", "err", err, "clusterId", env.Cluster.Id) + continue + } + ns = env.Namespace + } + _, err := impl.K8sUtil.GetPodByName(ns, ciWorkflow.PodName, client) + if err != nil { + impl.logger.Warnw("unable to fetch ci workflow - pod", "err", err) + statusError, ok := err.(*errors2.StatusError) + if ok && statusError.Status().Code == http.StatusNotFound { + impl.logger.Warnw("pod not found", "err", err) + isEligibleToMarkFailed = true + } else { + continue + // skip this and process for next ci workflow + } + } + if ciWorkflow.ExecutorType == cdWorkflow2.WORKFLOW_EXECUTOR_TYPE_SYSTEM { + if wf.Status == string(v1alpha1.WorkflowFailed) { + isPodDeleted = true + } + } else { + // check workflow status,get the status + if wf.Status == string(v1alpha1.WorkflowFailed) && wf.Message == cdWorkflow2.POD_DELETED_MESSAGE { + isPodDeleted = true + } + } + } + } + if isEligibleToMarkFailed { + ciWorkflow.Status = "Failed" + ciWorkflow.PodStatus = "Failed" + if isPodDeleted { + ciWorkflow.Message = cdWorkflow2.POD_DELETED_MESSAGE + // error logging handled inside handlePodDeleted + impl.ciHandlerService.HandlePodDeleted(ciWorkflow) + } else { + ciWorkflow.Message = "marked failed by job" + } + err := impl.ciService.UpdateCiWorkflowWithStage(ciWorkflow) + if err != nil { + impl.logger.Errorw("unable to update ci workflow, its eligible to mark failed", "err", err) + // skip this and process for next ci workflow + } + err = impl.customTagService.DeactivateImagePathReservation(ciWorkflow.ImagePathReservationId) + if err != nil { + impl.logger.Errorw("unable to update ci workflow, its eligible to mark failed", "err", err) + } + } + } + return nil +} + +func (impl *WorkflowDagExecutorImpl) getRestConfig(workflow *pipelineConfig.CiWorkflow) (*rest.Config, error) { + env, err := impl.envRepository.FindById(workflow.EnvironmentId) + if err != nil { + impl.logger.Errorw("could not fetch stage env", "err", err) + return nil, err + } + + clusterBean := adapter.GetClusterBean(*env.Cluster) + + clusterConfig := clusterBean.GetClusterConfig() + restConfig, err := impl.K8sUtil.GetRestConfigByCluster(clusterConfig) + if err != nil { + impl.logger.Errorw("error in getting rest config by cluster id", "err", err) + return nil, err + } + return restConfig, nil +} + func (impl *WorkflowDagExecutorImpl) HandleCdStageReTrigger(runner *pipelineConfig.CdWorkflowRunner) error { // do not re-trigger if retries = 0 if !impl.config.WorkflowRetriesEnabled() { @@ -250,13 +401,13 @@ func (impl *WorkflowDagExecutorImpl) HandleCdStageReTrigger(runner *pipelineConf } if runner.WorkflowType == bean.CD_WORKFLOW_TYPE_PRE { - _, err = impl.cdTriggerService.TriggerPreStage(triggerRequest) + _, err = impl.cdHandlerService.TriggerPreStage(triggerRequest) if err != nil { impl.logger.Errorw("error in TriggerPreStage ", "err", err, "cdWorkflowRunnerId", runner.Id) return err } } else if runner.WorkflowType == bean.CD_WORKFLOW_TYPE_POST { - _, err = impl.cdTriggerService.TriggerPostStage(triggerRequest) + _, err = impl.cdHandlerService.TriggerPostStage(triggerRequest) if err != nil { impl.logger.Errorw("error in TriggerPostStage ", "err", err, "cdWorkflowRunnerId", runner.Id) return err @@ -409,7 +560,7 @@ func (impl *WorkflowDagExecutorImpl) ProcessDevtronAsyncInstallRequest(cdAsyncIn impl.logger.Errorw("error in getting deployment config by appId and envId", "appId", overrideRequest.AppId, "envId", overrideRequest.EnvId, "err", err) return err } - releaseId, _, releaseErr := impl.cdTriggerService.TriggerRelease(newCtx, overrideRequest, envDeploymentConfig, cdAsyncInstallReq.TriggeredAt, cdAsyncInstallReq.TriggeredBy) + releaseId, _, releaseErr := impl.cdHandlerService.TriggerRelease(newCtx, overrideRequest, envDeploymentConfig, cdAsyncInstallReq.TriggeredAt, cdAsyncInstallReq.TriggeredBy) if releaseErr != nil { impl.logger.Errorw("error encountered in ProcessDevtronAsyncInstallRequest", "err", releaseErr, "cdWfrId", cdWfr.Id) impl.handleAsyncTriggerReleaseError(newCtx, releaseErr, cdWfr, overrideRequest) @@ -539,13 +690,13 @@ func (impl *WorkflowDagExecutorImpl) triggerIfAutoStageCdPipeline(request trigge // pre stage exists if request.Pipeline.PreTriggerType == pipelineConfig.TRIGGER_TYPE_AUTOMATIC { impl.logger.Debugw("trigger pre stage for pipeline", "artifactId", request.Artifact.Id, "pipelineId", request.Pipeline.Id) - _, err = impl.cdTriggerService.TriggerPreStage(request) // TODO handle error here + _, err = impl.cdHandlerService.TriggerPreStage(request) // TODO handle error here return err } } else if request.Pipeline.TriggerType == pipelineConfig.TRIGGER_TYPE_AUTOMATIC { // trigger deployment impl.logger.Debugw("trigger cd for pipeline", "artifactId", request.Artifact.Id, "pipelineId", request.Pipeline.Id) - err = impl.cdTriggerService.TriggerAutomaticDeployment(request) + err = impl.cdHandlerService.TriggerAutomaticDeployment(request) return err } return nil @@ -615,7 +766,7 @@ func (impl *WorkflowDagExecutorImpl) HandlePreStageSuccessEvent(triggerContext t } else { ciArtifactId = cdStageCompleteEvent.CiArtifactDTO.Id } - err = impl.cdTriggerService.TriggerAutoCDOnPreStageSuccess(triggerContext, cdStageCompleteEvent.CdPipelineId, ciArtifactId, cdStageCompleteEvent.WorkflowId) + err = impl.cdHandlerService.TriggerAutoCDOnPreStageSuccess(triggerContext, cdStageCompleteEvent.CdPipelineId, ciArtifactId, cdStageCompleteEvent.WorkflowId) if err != nil { impl.logger.Errorw("error in triggering cd on pre cd succcess", "err", err) return err @@ -659,7 +810,7 @@ func (impl *WorkflowDagExecutorImpl) HandleDeploymentSuccessEvent(triggerContext RefCdWorkflowRunnerId: 0, } triggerRequest.TriggerContext.Context = context.Background() - _, err = impl.cdTriggerService.TriggerPostStage(triggerRequest) + _, err = impl.cdHandlerService.TriggerPostStage(triggerRequest) if err != nil { impl.logger.Errorw("error in triggering post stage after successful deployment event", "err", err, "cdWorkflow", cdWorkflow) return err @@ -806,7 +957,7 @@ func (impl *WorkflowDagExecutorImpl) HandleCiSuccessEvent(triggerContext trigger // image scanning plugin can only be applied in Post-ci, scanning in pre-ci doesn't make sense pipelineStage := repository4.PIPELINE_STAGE_TYPE_POST_CI - if pipelineModal.PipelineType == constants2.CI_JOB.ToString() { + if pipelineModal.PipelineType == buildCommonBean.CI_JOB.ToString() { pipelineStage = repository4.PIPELINE_STAGE_TYPE_PRE_CI } // this flag comes from ci-runner when scanning is enabled from ciPipeline modal @@ -830,7 +981,7 @@ func (impl *WorkflowDagExecutorImpl) HandleCiSuccessEvent(triggerContext trigger var pluginArtifacts []*repository.CiArtifact for registry, artifacts := range request.PluginRegistryArtifactDetails { for _, image := range artifacts { - if pipelineModal.PipelineType == string(bean5.CI_JOB) && image == "" { + if pipelineModal.PipelineType == string(buildCommonBean.CI_JOB) && image == "" { continue } pluginArtifact := &repository.CiArtifact{ @@ -994,7 +1145,7 @@ func (impl *WorkflowDagExecutorImpl) HandleCiStepFailedEvent(ciPipelineId int, r func (impl *WorkflowDagExecutorImpl) WriteCiStepFailedEvent(pipeline *pipelineConfig.CiPipeline, request *bean2.CiArtifactWebhookRequest, ciWorkflow *pipelineConfig.CiWorkflow) { event, _ := impl.eventFactory.Build(util2.Fail, &pipeline.Id, pipeline.AppId, nil, util2.CI) - material := &client.MaterialTriggerInfo{} + material := &bean5.MaterialTriggerInfo{} material.GitTriggers = ciWorkflow.GitTriggers event.CiWorkflowRunnerId = ciWorkflow.Id event.UserId = int(ciWorkflow.TriggeredBy) diff --git a/scripts/sql/32803400_workflow_message.down.sql b/scripts/sql/32803400_workflow_message.down.sql new file mode 100644 index 0000000000..7b794d76d6 --- /dev/null +++ b/scripts/sql/32803400_workflow_message.down.sql @@ -0,0 +1 @@ +-- Migration down is not applicable here as cd_workflow_runner.message has moved to higher DataType \ No newline at end of file diff --git a/scripts/sql/32803400_workflow_message.up.sql b/scripts/sql/32803400_workflow_message.up.sql new file mode 100644 index 0000000000..bc3083ee77 --- /dev/null +++ b/scripts/sql/32803400_workflow_message.up.sql @@ -0,0 +1,11 @@ +BEGIN; + +-- cd_workflow_runner.message has a limit of 1000 characters. migrating to +ALTER TABLE "public"."cd_workflow_runner" + ALTER COLUMN "message" TYPE TEXT; + +-- ci_workflow.message has a limit of 250 characters. migrating to +ALTER TABLE "public"."ci_workflow" + ALTER COLUMN "message" TYPE TEXT; + +COMMIT; \ No newline at end of file diff --git a/util/helper.go b/util/helper.go index c8d3e00c27..50fbbf91ab 100644 --- a/util/helper.go +++ b/util/helper.go @@ -25,8 +25,8 @@ import ( "fmt" "github.com/devtron-labs/devtron/internal/middleware" "github.com/juju/errors" + "go.opentelemetry.io/otel" "io" - "io/ioutil" "math/rand" "net/http" "os" @@ -139,8 +139,10 @@ func GenerateNewWorkflowName(name string) string { return fmt.Sprintf("%s-clone-%s", name, Generate(4)) } -func HttpRequest(url string) (map[string]interface{}, error) { - req, err := http.NewRequest(http.MethodGet, url, nil) +func HttpRequest(ctx context.Context, url string) (map[string]interface{}, error) { + newCtx, span := otel.Tracer("orchestrator").Start(ctx, "util.HttpRequest") + defer span.End() + req, err := http.NewRequestWithContext(newCtx, http.MethodGet, url, nil) if err != nil { return nil, err } @@ -151,7 +153,7 @@ func HttpRequest(url string) (map[string]interface{}, error) { return nil, err } if res.StatusCode >= 200 && res.StatusCode <= 299 { - resBody, err := ioutil.ReadAll(res.Body) + resBody, err := io.ReadAll(res.Body) if err != nil { return nil, err } diff --git a/vendor/github.com/devtron-labs/common-lib/informer/bean.go b/vendor/github.com/devtron-labs/common-lib/informer/bean.go new file mode 100644 index 0000000000..38aed85afc --- /dev/null +++ b/vendor/github.com/devtron-labs/common-lib/informer/bean.go @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2024. Devtron Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package informer + +const ( + ClusterModifyEventSecretType = "cluster.request/modify" + ClusterActionAdd = "add" + ClusterActionUpdate = "update" + ClusterActionDelete = "delete" + SecretFieldAction = "action" + SecretFieldClusterId = "cluster_id" +) + +const ( + WorkflowTypeLabelKey = "workflowType" + CiWorkflowName = "ci" + CdWorkflowName = "cd" +) + +const ( + DevtronOwnerInstanceLabelKey = "devtron.ai/owner-instance" +) + +const ( + PodDeletedMessage = "pod deleted" +) diff --git a/client/telemetry/PosthogClient.go b/vendor/github.com/devtron-labs/common-lib/telemetry/PosthogClient.go similarity index 70% rename from client/telemetry/PosthogClient.go rename to vendor/github.com/devtron-labs/common-lib/telemetry/PosthogClient.go index 565470dc5c..79a8c72f88 100644 --- a/client/telemetry/PosthogClient.go +++ b/vendor/github.com/devtron-labs/common-lib/telemetry/PosthogClient.go @@ -17,11 +17,13 @@ package telemetry import ( + "context" "encoding/base64" "encoding/json" + "go.opentelemetry.io/otel" "time" - "github.com/devtron-labs/devtron/util" + "github.com/devtron-labs/common-lib/utils/http" "github.com/patrickmn/go-cache" "github.com/posthog/posthog-go" "go.uber.org/zap" @@ -32,26 +34,13 @@ type PosthogClient struct { cache *cache.Cache } -var ( - PosthogApiKey string = "" - PosthogEndpoint string = "https://app.posthog.com" - SummaryCronExpr string = "0 0 * * *" // Run once a day, midnight - HeartbeatCronExpr string = "0 0/6 * * *" - CacheExpiry int = 1440 - PosthogEncodedApiKey string = "" - IsOptOut bool = false -) - -const ( - TelemetryApiKeyEndpoint string = "aHR0cHM6Ly90ZWxlbWV0cnkuZGV2dHJvbi5haS9kZXZ0cm9uL3RlbGVtZXRyeS9wb3N0aG9nSW5mbw==" - TelemetryOptOutApiBaseUrl string = "aHR0cHM6Ly90ZWxlbWV0cnkuZGV2dHJvbi5haS9kZXZ0cm9uL3RlbGVtZXRyeS9vcHQtb3V0" - ResponseApiKey string = "PosthogApiKey" - ResponseUrlKey string = "PosthogEndpoint" -) +func (p *PosthogClient) GetCache() *cache.Cache { + return p.cache +} func NewPosthogClient(logger *zap.SugaredLogger) (*PosthogClient, error) { if PosthogApiKey == "" { - encodedApiKey, apiKey, posthogUrl, err := getPosthogApiKey(TelemetryApiKeyEndpoint, logger) + encodedApiKey, apiKey, posthogUrl, err := getPosthogApiKey(context.Background(), TelemetryApiKeyEndpoint, logger) if err != nil { logger.Errorw("exception caught while getting api key", "err", err) } else { @@ -75,14 +64,16 @@ func NewPosthogClient(logger *zap.SugaredLogger) (*PosthogClient, error) { return pgClient, nil } -func getPosthogApiKey(encodedPosthogApiKeyUrl string, logger *zap.SugaredLogger) (string, string, string, error) { +func getPosthogApiKey(ctx context.Context, encodedPosthogApiKeyUrl string, logger *zap.SugaredLogger) (string, string, string, error) { + newCtx, span := otel.Tracer("common").Start(ctx, "telemetry.getPosthogApiKey") + defer span.End() decodedPosthogApiKeyUrl, err := base64.StdEncoding.DecodeString(encodedPosthogApiKeyUrl) if err != nil { logger.Errorw("error fetching posthog api key, decode error", "err", err) return "", "", "", err } apiKeyUrl := string(decodedPosthogApiKeyUrl) - response, err := util.HttpRequest(apiKeyUrl) + response, err := http.HttpRequest(newCtx, apiKeyUrl) if err != nil { logger.Errorw("error fetching posthog api key, http call", "err", err) return "", "", "", err @@ -94,7 +85,7 @@ func getPosthogApiKey(encodedPosthogApiKeyUrl string, logger *zap.SugaredLogger) return "", "", "", err } var datamap map[string]string - if err := json.Unmarshal(posthogInfoByte, &datamap); err != nil { + if err = json.Unmarshal(posthogInfoByte, &datamap); err != nil { logger.Errorw("error while unmarshal data", "err", err) return "", "", "", err } diff --git a/vendor/github.com/devtron-labs/common-lib/telemetry/bean.go b/vendor/github.com/devtron-labs/common-lib/telemetry/bean.go new file mode 100644 index 0000000000..b85621d0f3 --- /dev/null +++ b/vendor/github.com/devtron-labs/common-lib/telemetry/bean.go @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2024. Devtron Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package telemetry + +var ( + PosthogApiKey string = "" + PosthogEndpoint string = "https://app.posthog.com" + SummaryCronExpr string = "0 0 * * *" // Run once a day, midnight + HeartbeatCronExpr string = "0 0/6 * * *" + CacheExpiry int = 1440 + PosthogEncodedApiKey string = "" + IsOptOut bool = false +) + +const ( + TelemetryApiKeyEndpoint string = "aHR0cHM6Ly90ZWxlbWV0cnkuZGV2dHJvbi5haS9kZXZ0cm9uL3RlbGVtZXRyeS9wb3N0aG9nSW5mbw==" + TelemetryOptOutApiBaseUrl string = "aHR0cHM6Ly90ZWxlbWV0cnkuZGV2dHJvbi5haS9kZXZ0cm9uL3RlbGVtZXRyeS9vcHQtb3V0" + ResponseApiKey string = "PosthogApiKey" + ResponseUrlKey string = "PosthogEndpoint" +) diff --git a/vendor/github.com/devtron-labs/common-lib/utils/http/HttpUtil.go b/vendor/github.com/devtron-labs/common-lib/utils/http/HttpUtil.go index 933b97f4f9..9f8524e489 100644 --- a/vendor/github.com/devtron-labs/common-lib/utils/http/HttpUtil.go +++ b/vendor/github.com/devtron-labs/common-lib/utils/http/HttpUtil.go @@ -17,11 +17,13 @@ package http import ( + "context" "crypto/tls" "crypto/x509" "encoding/json" "github.com/pkg/errors" - "io/ioutil" + "go.opentelemetry.io/otel" + "io" "net/http" "os" ) @@ -83,8 +85,10 @@ func CertPoolFromFile(filename string) (*x509.CertPool, error) { return cp, nil } -func HttpRequest(url string) (map[string]interface{}, error) { - req, err := http.NewRequest(http.MethodGet, url, nil) +func HttpRequest(ctx context.Context, url string) (map[string]interface{}, error) { + newCtx, span := otel.Tracer("common").Start(ctx, "http.HttpRequest") + defer span.End() + req, err := http.NewRequestWithContext(newCtx, http.MethodGet, url, nil) if err != nil { return nil, err } @@ -95,7 +99,7 @@ func HttpRequest(url string) (map[string]interface{}, error) { return nil, err } if res.StatusCode >= 200 && res.StatusCode <= 299 { - resBody, err := ioutil.ReadAll(res.Body) + resBody, err := io.ReadAll(res.Body) if err != nil { return nil, err } diff --git a/vendor/github.com/golang-jwt/jwt/v4/parser.go b/vendor/github.com/golang-jwt/jwt/v4/parser.go index 9dd36e5a5a..0fc510a0aa 100644 --- a/vendor/github.com/golang-jwt/jwt/v4/parser.go +++ b/vendor/github.com/golang-jwt/jwt/v4/parser.go @@ -7,6 +7,8 @@ import ( "strings" ) +const tokenDelimiter = "." + type Parser struct { // If populated, only these methods will be considered valid. // @@ -122,9 +124,10 @@ func (p *Parser) ParseWithClaims(tokenString string, claims Claims, keyFunc Keyf // It's only ever useful in cases where you know the signature is valid (because it has // been checked previously in the stack) and you want to extract values from it. func (p *Parser) ParseUnverified(tokenString string, claims Claims) (token *Token, parts []string, err error) { - parts = strings.Split(tokenString, ".") - if len(parts) != 3 { - return nil, parts, NewValidationError("token contains an invalid number of segments", ValidationErrorMalformed) + var ok bool + parts, ok = splitToken(tokenString) + if !ok { + return nil, nil, NewValidationError("token contains an invalid number of segments", ValidationErrorMalformed) } token = &Token{Raw: tokenString} @@ -174,3 +177,30 @@ func (p *Parser) ParseUnverified(tokenString string, claims Claims) (token *Toke return token, parts, nil } + +// splitToken splits a token string into three parts: header, claims, and signature. It will only +// return true if the token contains exactly two delimiters and three parts. In all other cases, it +// will return nil parts and false. +func splitToken(token string) ([]string, bool) { + parts := make([]string, 3) + header, remain, ok := strings.Cut(token, tokenDelimiter) + if !ok { + return nil, false + } + parts[0] = header + claims, remain, ok := strings.Cut(remain, tokenDelimiter) + if !ok { + return nil, false + } + parts[1] = claims + // One more cut to ensure the signature is the last part of the token and there are no more + // delimiters. This avoids an issue where malicious input could contain additional delimiters + // causing unecessary overhead parsing tokens. + signature, _, unexpected := strings.Cut(remain, tokenDelimiter) + if unexpected { + return nil, false + } + parts[2] = signature + + return parts, true +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 4a1cd3503d..436ab6735d 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -472,7 +472,7 @@ github.com/davecgh/go-spew/spew # github.com/deckarep/golang-set v1.8.0 ## explicit; go 1.17 github.com/deckarep/golang-set -# github.com/devtron-labs/authenticator v0.4.35-0.20240809073103-6e11da8083f8 => github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250421131910-ad3aa9bb920e +# github.com/devtron-labs/authenticator v0.4.35-0.20240809073103-6e11da8083f8 => github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250422075757-fe1e5dd1b92b ## explicit; go 1.21 github.com/devtron-labs/authenticator/apiToken github.com/devtron-labs/authenticator/client @@ -480,7 +480,7 @@ github.com/devtron-labs/authenticator/jwt github.com/devtron-labs/authenticator/middleware github.com/devtron-labs/authenticator/oidc github.com/devtron-labs/authenticator/password -# github.com/devtron-labs/common-lib v0.18.1-0.20241001061923-eda545dc839e => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250421131910-ad3aa9bb920e +# github.com/devtron-labs/common-lib v0.18.1-0.20241001061923-eda545dc839e => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250422075757-fe1e5dd1b92b ## explicit; go 1.21 github.com/devtron-labs/common-lib/async github.com/devtron-labs/common-lib/blob-storage @@ -492,10 +492,12 @@ github.com/devtron-labs/common-lib/fetchAllEnv github.com/devtron-labs/common-lib/git-manager github.com/devtron-labs/common-lib/git-manager/util github.com/devtron-labs/common-lib/imageScan/bean +github.com/devtron-labs/common-lib/informer github.com/devtron-labs/common-lib/middlewares github.com/devtron-labs/common-lib/pubsub-lib github.com/devtron-labs/common-lib/pubsub-lib/metrics github.com/devtron-labs/common-lib/pubsub-lib/model +github.com/devtron-labs/common-lib/telemetry github.com/devtron-labs/common-lib/utils github.com/devtron-labs/common-lib/utils/bean github.com/devtron-labs/common-lib/utils/grpc @@ -699,7 +701,7 @@ github.com/gogo/protobuf/jsonpb github.com/gogo/protobuf/proto github.com/gogo/protobuf/sortkeys github.com/gogo/protobuf/types -# github.com/golang-jwt/jwt/v4 v4.5.1 +# github.com/golang-jwt/jwt/v4 v4.5.2 ## explicit; go 1.16 github.com/golang-jwt/jwt/v4 # github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da @@ -2350,8 +2352,8 @@ xorm.io/xorm/log xorm.io/xorm/names xorm.io/xorm/schemas xorm.io/xorm/tags -# github.com/devtron-labs/authenticator => github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250421131910-ad3aa9bb920e -# github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250421131910-ad3aa9bb920e +# github.com/devtron-labs/authenticator => github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250422075757-fe1e5dd1b92b +# github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250422075757-fe1e5dd1b92b # github.com/go-check/check => github.com/go-check/check v0.0.0-20180628173108-788fd7840127 # github.com/googleapis/gnostic => github.com/googleapis/gnostic v0.5.5 # k8s.io/api => k8s.io/api v0.29.7 diff --git a/wire_gen.go b/wire_gen.go index 78a257534d..60f4d5a649 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -1,6 +1,6 @@ // Code generated by Wire. DO NOT EDIT. -//go:generate go run -mod=mod github.com/google/wire/cmd/wire +//go:generate go run github.com/google/wire/cmd/wire //go:build !wireinject // +build !wireinject @@ -12,6 +12,7 @@ import ( "github.com/devtron-labs/authenticator/middleware" "github.com/devtron-labs/common-lib/cloud-provider-identifier" "github.com/devtron-labs/common-lib/pubsub-lib" + "github.com/devtron-labs/common-lib/telemetry" "github.com/devtron-labs/common-lib/utils/grpc" "github.com/devtron-labs/common-lib/utils/k8s" apiToken2 "github.com/devtron-labs/devtron/api/apiToken" @@ -49,7 +50,7 @@ import ( "github.com/devtron-labs/devtron/api/restHandler/app/pipeline/configure" history2 "github.com/devtron-labs/devtron/api/restHandler/app/pipeline/history" status3 "github.com/devtron-labs/devtron/api/restHandler/app/pipeline/status" - "github.com/devtron-labs/devtron/api/restHandler/app/pipeline/trigger" + trigger2 "github.com/devtron-labs/devtron/api/restHandler/app/pipeline/trigger" "github.com/devtron-labs/devtron/api/restHandler/app/pipeline/webhook" "github.com/devtron-labs/devtron/api/restHandler/app/workflow" "github.com/devtron-labs/devtron/api/restHandler/scopedVariable" @@ -62,7 +63,7 @@ import ( configure2 "github.com/devtron-labs/devtron/api/router/app/pipeline/configure" history3 "github.com/devtron-labs/devtron/api/router/app/pipeline/history" status4 "github.com/devtron-labs/devtron/api/router/app/pipeline/status" - trigger2 "github.com/devtron-labs/devtron/api/router/app/pipeline/trigger" + trigger3 "github.com/devtron-labs/devtron/api/router/app/pipeline/trigger" workflow2 "github.com/devtron-labs/devtron/api/router/app/workflow" server2 "github.com/devtron-labs/devtron/api/server" "github.com/devtron-labs/devtron/api/sse" @@ -90,7 +91,7 @@ import ( "github.com/devtron-labs/devtron/client/grafana" "github.com/devtron-labs/devtron/client/lens" "github.com/devtron-labs/devtron/client/proxy" - "github.com/devtron-labs/devtron/client/telemetry" + telemetry2 "github.com/devtron-labs/devtron/client/telemetry" repository2 "github.com/devtron-labs/devtron/internal/sql/repository" "github.com/devtron-labs/devtron/internal/sql/repository/app" "github.com/devtron-labs/devtron/internal/sql/repository/appStatus" @@ -100,7 +101,7 @@ import ( "github.com/devtron-labs/devtron/internal/sql/repository/deploymentConfig" repository9 "github.com/devtron-labs/devtron/internal/sql/repository/dockerRegistry" "github.com/devtron-labs/devtron/internal/sql/repository/helper" - repository22 "github.com/devtron-labs/devtron/internal/sql/repository/imageTagging" + repository23 "github.com/devtron-labs/devtron/internal/sql/repository/imageTagging" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" "github.com/devtron-labs/devtron/internal/sql/repository/resourceGroup" "github.com/devtron-labs/devtron/internal/util" @@ -147,14 +148,15 @@ import ( read21 "github.com/devtron-labs/devtron/pkg/build/git/gitHost/read" repository26 "github.com/devtron-labs/devtron/pkg/build/git/gitHost/repository" read15 "github.com/devtron-labs/devtron/pkg/build/git/gitMaterial/read" - repository20 "github.com/devtron-labs/devtron/pkg/build/git/gitMaterial/repository" + repository21 "github.com/devtron-labs/devtron/pkg/build/git/gitMaterial/repository" "github.com/devtron-labs/devtron/pkg/build/git/gitProvider" read9 "github.com/devtron-labs/devtron/pkg/build/git/gitProvider/read" - repository11 "github.com/devtron-labs/devtron/pkg/build/git/gitProvider/repository" + repository12 "github.com/devtron-labs/devtron/pkg/build/git/gitProvider/repository" "github.com/devtron-labs/devtron/pkg/build/git/gitWebhook" - repository23 "github.com/devtron-labs/devtron/pkg/build/git/gitWebhook/repository" + repository11 "github.com/devtron-labs/devtron/pkg/build/git/gitWebhook/repository" pipeline2 "github.com/devtron-labs/devtron/pkg/build/pipeline" read14 "github.com/devtron-labs/devtron/pkg/build/pipeline/read" + "github.com/devtron-labs/devtron/pkg/build/trigger" service7 "github.com/devtron-labs/devtron/pkg/bulkAction/service" "github.com/devtron-labs/devtron/pkg/chart" "github.com/devtron-labs/devtron/pkg/chart/gitOpsConfig" @@ -184,7 +186,7 @@ import ( "github.com/devtron-labs/devtron/pkg/deployment/manifest/configMapAndSecret" read19 "github.com/devtron-labs/devtron/pkg/deployment/manifest/configMapAndSecret/read" "github.com/devtron-labs/devtron/pkg/deployment/manifest/deployedAppMetrics" - repository16 "github.com/devtron-labs/devtron/pkg/deployment/manifest/deployedAppMetrics/repository" + repository17 "github.com/devtron-labs/devtron/pkg/deployment/manifest/deployedAppMetrics/repository" "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate" "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/chartRef" read12 "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/chartRef/read" @@ -199,12 +201,13 @@ import ( "github.com/devtron-labs/devtron/pkg/devtronResource" "github.com/devtron-labs/devtron/pkg/devtronResource/history/deployment/cdPipeline" read10 "github.com/devtron-labs/devtron/pkg/devtronResource/read" - repository13 "github.com/devtron-labs/devtron/pkg/devtronResource/repository" + repository14 "github.com/devtron-labs/devtron/pkg/devtronResource/repository" "github.com/devtron-labs/devtron/pkg/dockerRegistry" "github.com/devtron-labs/devtron/pkg/eventProcessor" "github.com/devtron-labs/devtron/pkg/eventProcessor/celEvaluator" "github.com/devtron-labs/devtron/pkg/eventProcessor/in" "github.com/devtron-labs/devtron/pkg/eventProcessor/out" + "github.com/devtron-labs/devtron/pkg/executor" "github.com/devtron-labs/devtron/pkg/externalLink" "github.com/devtron-labs/devtron/pkg/fluxApplication" "github.com/devtron-labs/devtron/pkg/generateManifest" @@ -213,7 +216,7 @@ import ( "github.com/devtron-labs/devtron/pkg/gitops" "github.com/devtron-labs/devtron/pkg/imageDigestPolicy" config4 "github.com/devtron-labs/devtron/pkg/infraConfig/config" - repository14 "github.com/devtron-labs/devtron/pkg/infraConfig/repository" + repository15 "github.com/devtron-labs/devtron/pkg/infraConfig/repository" "github.com/devtron-labs/devtron/pkg/infraConfig/repository/audit" service2 "github.com/devtron-labs/devtron/pkg/infraConfig/service" audit2 "github.com/devtron-labs/devtron/pkg/infraConfig/service/audit" @@ -233,21 +236,21 @@ import ( "github.com/devtron-labs/devtron/pkg/pipeline/draftAwareConfigService" "github.com/devtron-labs/devtron/pkg/pipeline/executors" "github.com/devtron-labs/devtron/pkg/pipeline/history" - repository21 "github.com/devtron-labs/devtron/pkg/pipeline/history/repository" + repository22 "github.com/devtron-labs/devtron/pkg/pipeline/history/repository" "github.com/devtron-labs/devtron/pkg/pipeline/infraProviders" "github.com/devtron-labs/devtron/pkg/pipeline/infraProviders/infraGetters/ci" "github.com/devtron-labs/devtron/pkg/pipeline/infraProviders/infraGetters/job" - repository18 "github.com/devtron-labs/devtron/pkg/pipeline/repository" + repository19 "github.com/devtron-labs/devtron/pkg/pipeline/repository" "github.com/devtron-labs/devtron/pkg/pipeline/types" "github.com/devtron-labs/devtron/pkg/pipeline/workflowStatus" - repository17 "github.com/devtron-labs/devtron/pkg/pipeline/workflowStatus/repository" + repository18 "github.com/devtron-labs/devtron/pkg/pipeline/workflowStatus/repository" "github.com/devtron-labs/devtron/pkg/plugin" - repository19 "github.com/devtron-labs/devtron/pkg/plugin/repository" + repository20 "github.com/devtron-labs/devtron/pkg/plugin/repository" "github.com/devtron-labs/devtron/pkg/policyGovernance/security/imageScanning" read18 "github.com/devtron-labs/devtron/pkg/policyGovernance/security/imageScanning/read" repository24 "github.com/devtron-labs/devtron/pkg/policyGovernance/security/imageScanning/repository" "github.com/devtron-labs/devtron/pkg/policyGovernance/security/scanTool" - repository15 "github.com/devtron-labs/devtron/pkg/policyGovernance/security/scanTool/repository" + repository16 "github.com/devtron-labs/devtron/pkg/policyGovernance/security/scanTool/repository" resourceGroup2 "github.com/devtron-labs/devtron/pkg/resourceGroup" "github.com/devtron-labs/devtron/pkg/resourceQualifiers" "github.com/devtron-labs/devtron/pkg/server" @@ -258,11 +261,12 @@ import ( read4 "github.com/devtron-labs/devtron/pkg/team/read" repository8 "github.com/devtron-labs/devtron/pkg/team/repository" "github.com/devtron-labs/devtron/pkg/terminal" + "github.com/devtron-labs/devtron/pkg/ucid" "github.com/devtron-labs/devtron/pkg/userResource" util3 "github.com/devtron-labs/devtron/pkg/util" "github.com/devtron-labs/devtron/pkg/variables" "github.com/devtron-labs/devtron/pkg/variables/parsers" - repository12 "github.com/devtron-labs/devtron/pkg/variables/repository" + repository13 "github.com/devtron-labs/devtron/pkg/variables/repository" "github.com/devtron-labs/devtron/pkg/webhook/helm" "github.com/devtron-labs/devtron/pkg/workflow/cd" read20 "github.com/devtron-labs/devtron/pkg/workflow/cd/read" @@ -487,11 +491,12 @@ func InitializeApp() (*App, error) { clusterRbacServiceImpl := rbac2.NewClusterRbacServiceImpl(environmentServiceImpl, enforcerImpl, enforcerUtilImpl, clusterServiceImplExtended, sugaredLogger, userServiceImpl, clusterReadServiceImpl) clusterRestHandlerImpl := cluster3.NewClusterRestHandlerImpl(clusterServiceImplExtended, genericNoteServiceImpl, clusterDescriptionServiceImpl, sugaredLogger, userServiceImpl, validate, enforcerImpl, deleteServiceExtendedImpl, environmentServiceImpl, clusterRbacServiceImpl) clusterRouterImpl := cluster3.NewClusterRouterImpl(clusterRestHandlerImpl) + gitWebhookRepositoryImpl := repository11.NewGitWebhookRepositoryImpl(db) ciCdConfig, err := types.GetCiCdConfig() if err != nil { return nil, err } - gitProviderRepositoryImpl := repository11.NewGitProviderRepositoryImpl(db) + gitProviderRepositoryImpl := repository12.NewGitProviderRepositoryImpl(db) gitProviderReadServiceImpl := read9.NewGitProviderReadService(sugaredLogger, gitProviderRepositoryImpl) commonBaseServiceImpl := commonService.NewCommonBaseServiceImpl(sugaredLogger, environmentVariables, moduleReadServiceImpl) commonServiceImpl := commonService.NewCommonServiceImpl(sugaredLogger, chartRepositoryImpl, envConfigOverrideRepositoryImpl, dockerArtifactStoreRepositoryImpl, attributesRepositoryImpl, environmentRepositoryImpl, appRepositoryImpl, gitOpsConfigReadServiceImpl, gitProviderReadServiceImpl, envConfigOverrideReadServiceImpl, commonBaseServiceImpl, teamReadServiceImpl) @@ -499,8 +504,8 @@ func InitializeApp() (*App, error) { mergeUtil := util.MergeUtil{ Logger: sugaredLogger, } - scopedVariableRepositoryImpl := repository12.NewScopedVariableRepository(db, sugaredLogger, transactionUtilImpl) - devtronResourceSearchableKeyRepositoryImpl := repository13.NewDevtronResourceSearchableKeyRepositoryImpl(sugaredLogger, db) + scopedVariableRepositoryImpl := repository13.NewScopedVariableRepository(db, sugaredLogger, transactionUtilImpl) + devtronResourceSearchableKeyRepositoryImpl := repository14.NewDevtronResourceSearchableKeyRepositoryImpl(sugaredLogger, db) devtronResourceSearchableKeyServiceImpl, err := read10.NewDevtronResourceSearchableKeyServiceImpl(sugaredLogger, devtronResourceSearchableKeyRepositoryImpl) if err != nil { return nil, err @@ -517,9 +522,9 @@ func InitializeApp() (*App, error) { if err != nil { return nil, err } - variableEntityMappingRepositoryImpl := repository12.NewVariableEntityMappingRepository(sugaredLogger, db, transactionUtilImpl) + variableEntityMappingRepositoryImpl := repository13.NewVariableEntityMappingRepository(sugaredLogger, db, transactionUtilImpl) variableEntityMappingServiceImpl := variables.NewVariableEntityMappingServiceImpl(variableEntityMappingRepositoryImpl, sugaredLogger) - variableSnapshotHistoryRepositoryImpl := repository12.NewVariableSnapshotHistoryRepository(sugaredLogger, db) + variableSnapshotHistoryRepositoryImpl := repository13.NewVariableSnapshotHistoryRepository(sugaredLogger, db) variableSnapshotHistoryServiceImpl := variables.NewVariableSnapshotHistoryServiceImpl(variableSnapshotHistoryRepositoryImpl, sugaredLogger) variableTemplateParserImpl, err := parsers.NewVariableTemplateParserImpl(sugaredLogger) if err != nil { @@ -540,7 +545,7 @@ func InitializeApp() (*App, error) { if err != nil { return nil, err } - infraConfigRepositoryImpl := repository14.NewInfraProfileRepositoryImpl(db, transactionUtilImpl) + infraConfigRepositoryImpl := repository15.NewInfraProfileRepositoryImpl(db, transactionUtilImpl) pipelineOverrideRepositoryImpl := chartConfig.NewPipelineOverrideRepository(db) utilMergeUtil := &util.MergeUtil{ Logger: sugaredLogger, @@ -573,7 +578,7 @@ func InitializeApp() (*App, error) { if err != nil { return nil, err } - scanToolMetadataRepositoryImpl := repository15.NewScanToolMetadataRepositoryImpl(db, sugaredLogger) + scanToolMetadataRepositoryImpl := repository16.NewScanToolMetadataRepositoryImpl(db, sugaredLogger) scanToolMetadataServiceImpl := scanTool.NewScanToolMetadataServiceImpl(sugaredLogger, scanToolMetadataRepositoryImpl) moduleServiceImpl := module.NewModuleServiceImpl(sugaredLogger, serverEnvConfigServerEnvConfig, moduleRepositoryImpl, moduleActionAuditLogRepositoryImpl, helmAppServiceImpl, serverDataStoreServerDataStore, serverCacheServiceImpl, moduleCacheServiceImpl, moduleCronServiceImpl, moduleServiceHelperImpl, moduleResourceStatusRepositoryImpl, scanToolMetadataServiceImpl, environmentVariables, moduleEnvConfig) eventRESTClientImpl := client2.NewEventRESTClientImpl(sugaredLogger, httpClient, eventClientConfig, pubSubClientServiceImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, attributesRepositoryImpl, moduleServiceImpl) @@ -607,11 +612,11 @@ func InitializeApp() (*App, error) { ciTemplateOverrideRepositoryImpl := pipelineConfig.NewCiTemplateOverrideRepositoryImpl(db, sugaredLogger) ciPipelineConfigReadServiceImpl := read14.NewCiPipelineConfigReadServiceImpl(sugaredLogger, ciPipelineRepositoryImpl, ciTemplateOverrideRepositoryImpl) dockerRegistryIpsConfigServiceImpl := dockerRegistry.NewDockerRegistryIpsConfigServiceImpl(sugaredLogger, dockerRegistryIpsConfigRepositoryImpl, k8sServiceImpl, dockerArtifactStoreRepositoryImpl, clusterReadServiceImpl, ciPipelineConfigReadServiceImpl) - appLevelMetricsRepositoryImpl := repository16.NewAppLevelMetricsRepositoryImpl(db, sugaredLogger) - envLevelAppMetricsRepositoryImpl := repository16.NewEnvLevelAppMetricsRepositoryImpl(db, sugaredLogger) + appLevelMetricsRepositoryImpl := repository17.NewAppLevelMetricsRepositoryImpl(db, sugaredLogger) + envLevelAppMetricsRepositoryImpl := repository17.NewEnvLevelAppMetricsRepositoryImpl(db, sugaredLogger) deployedAppMetricsServiceImpl := deployedAppMetrics.NewDeployedAppMetricsServiceImpl(sugaredLogger, appLevelMetricsRepositoryImpl, envLevelAppMetricsRepositoryImpl, chartRefServiceImpl) appListingServiceImpl := app2.NewAppListingServiceImpl(sugaredLogger, appListingRepositoryImpl, appDetailsReadServiceImpl, appRepositoryImpl, appListingViewBuilderImpl, pipelineRepositoryImpl, linkoutsRepositoryImpl, cdWorkflowRepositoryImpl, pipelineOverrideRepositoryImpl, environmentRepositoryImpl, chartRepositoryImpl, ciPipelineRepositoryImpl, dockerRegistryIpsConfigServiceImpl, userRepositoryImpl, deployedAppMetricsServiceImpl, ciArtifactRepositoryImpl, envConfigOverrideReadServiceImpl, ciPipelineConfigReadServiceImpl) - workflowStageRepositoryImpl := repository17.NewWorkflowStageRepositoryImpl(sugaredLogger, db) + workflowStageRepositoryImpl := repository18.NewWorkflowStageRepositoryImpl(sugaredLogger, db) workFlowStageStatusServiceImpl := workflowStatus.NewWorkflowStageFlowStatusServiceImpl(sugaredLogger, workflowStageRepositoryImpl, ciWorkflowRepositoryImpl, cdWorkflowRepositoryImpl, transactionUtilImpl) cdWorkflowRunnerServiceImpl := cd.NewCdWorkflowRunnerServiceImpl(sugaredLogger, cdWorkflowRepositoryImpl, workFlowStageStatusServiceImpl, transactionUtilImpl) appServiceImpl := app2.NewAppService(pipelineOverrideRepositoryImpl, utilMergeUtil, sugaredLogger, pipelineRepositoryImpl, eventRESTClientImpl, eventSimpleFactoryImpl, appRepositoryImpl, configMapRepositoryImpl, chartRepositoryImpl, cdWorkflowRepositoryImpl, commonServiceImpl, chartTemplateServiceImpl, pipelineStatusTimelineRepositoryImpl, pipelineStatusTimelineResourcesServiceImpl, pipelineStatusSyncDetailServiceImpl, pipelineStatusTimelineServiceImpl, appServiceConfig, appStatusServiceImpl, installedAppReadServiceImpl, installedAppVersionHistoryRepositoryImpl, scopedVariableCMCSManagerImpl, acdConfig, gitOpsConfigReadServiceImpl, gitOperationServiceImpl, deploymentTemplateServiceImpl, appListingServiceImpl, deploymentConfigServiceImpl, envConfigOverrideReadServiceImpl, cdWorkflowRunnerServiceImpl) @@ -626,12 +631,13 @@ func InitializeApp() (*App, error) { } ciInfraGetter := ci.NewCiInfraGetter(sugaredLogger, infraConfigServiceImpl, infraConfigAuditServiceImpl) infraProviderImpl := infraProviders.NewInfraProviderImpl(sugaredLogger, infraGetter, ciInfraGetter) - workflowServiceImpl, err := pipeline.NewWorkflowServiceImpl(sugaredLogger, environmentRepositoryImpl, ciCdConfig, configReadServiceImpl, globalCMCSServiceImpl, argoWorkflowExecutorImpl, k8sServiceImpl, systemWorkflowExecutorImpl, k8sCommonServiceImpl, infraProviderImpl) + serviceImpl := ucid.NewServiceImpl(sugaredLogger, k8sServiceImpl, acdAuthConfig) + workflowServiceImpl, err := executor.NewWorkflowServiceImpl(sugaredLogger, environmentRepositoryImpl, ciCdConfig, configReadServiceImpl, globalCMCSServiceImpl, argoWorkflowExecutorImpl, systemWorkflowExecutorImpl, k8sCommonServiceImpl, infraProviderImpl, serviceImpl, k8sServiceImpl) if err != nil { return nil, err } - pipelineStageRepositoryImpl := repository18.NewPipelineStageRepository(sugaredLogger, db) - globalPluginRepositoryImpl := repository19.NewGlobalPluginRepository(sugaredLogger, db) + pipelineStageRepositoryImpl := repository19.NewPipelineStageRepository(sugaredLogger, db) + globalPluginRepositoryImpl := repository20.NewGlobalPluginRepository(sugaredLogger, db) globalPluginServiceImpl := plugin.NewGlobalPluginService(sugaredLogger, globalPluginRepositoryImpl, pipelineStageRepositoryImpl, userServiceImpl) pipelineStageServiceImpl := pipeline.NewPipelineStageService(sugaredLogger, pipelineStageRepositoryImpl, globalPluginRepositoryImpl, pipelineRepositoryImpl, scopedVariableManagerImpl, globalPluginServiceImpl) ciTemplateRepositoryImpl := pipelineConfig.NewCiTemplateRepositoryImpl(db, sugaredLogger) @@ -641,12 +647,11 @@ func InitializeApp() (*App, error) { if err != nil { return nil, err } - materialRepositoryImpl := repository20.NewMaterialRepositoryImpl(db) + materialRepositoryImpl := repository21.NewMaterialRepositoryImpl(db) gitMaterialReadServiceImpl := read15.NewGitMaterialReadServiceImpl(sugaredLogger, materialRepositoryImpl) appCrudOperationServiceImpl := app2.NewAppCrudOperationServiceImpl(appLabelRepositoryImpl, sugaredLogger, appRepositoryImpl, userRepositoryImpl, installedAppRepositoryImpl, genericNoteServiceImpl, installedAppDBServiceImpl, crudOperationServiceConfig, dbMigrationServiceImpl, gitMaterialReadServiceImpl) imageTagRepositoryImpl := repository2.NewImageTagRepository(db, sugaredLogger) customTagServiceImpl := pipeline.NewCustomTagService(sugaredLogger, imageTagRepositoryImpl) - pluginInputVariableParserImpl := pipeline.NewPluginInputVariableParserImpl(sugaredLogger, dockerRegistryConfigImpl, customTagServiceImpl) clientConfig, err := gitSensor.GetConfig() if err != nil { return nil, err @@ -655,54 +660,54 @@ func InitializeApp() (*App, error) { if err != nil { return nil, err } - prePostCdScriptHistoryRepositoryImpl := repository21.NewPrePostCdScriptHistoryRepositoryImpl(sugaredLogger, db) - configMapHistoryRepositoryImpl := repository21.NewConfigMapHistoryRepositoryImpl(sugaredLogger, db, transactionUtilImpl) + prePostCdScriptHistoryRepositoryImpl := repository22.NewPrePostCdScriptHistoryRepositoryImpl(sugaredLogger, db) + configMapHistoryRepositoryImpl := repository22.NewConfigMapHistoryRepositoryImpl(sugaredLogger, db, transactionUtilImpl) configMapHistoryServiceImpl := configMapAndSecret.NewConfigMapHistoryServiceImpl(sugaredLogger, configMapHistoryRepositoryImpl, pipelineRepositoryImpl, configMapRepositoryImpl, userServiceImpl, scopedVariableCMCSManagerImpl) prePostCdScriptHistoryServiceImpl := history.NewPrePostCdScriptHistoryServiceImpl(sugaredLogger, prePostCdScriptHistoryRepositoryImpl, configMapRepositoryImpl, configMapHistoryServiceImpl) - gitMaterialHistoryRepositoryImpl := repository21.NewGitMaterialHistoryRepositoyImpl(db) + gitMaterialHistoryRepositoryImpl := repository22.NewGitMaterialHistoryRepositoyImpl(db) gitMaterialHistoryServiceImpl := history.NewGitMaterialHistoryServiceImpl(gitMaterialHistoryRepositoryImpl, sugaredLogger) - ciPipelineHistoryRepositoryImpl := repository21.NewCiPipelineHistoryRepositoryImpl(db, sugaredLogger) + ciPipelineHistoryRepositoryImpl := repository22.NewCiPipelineHistoryRepositoryImpl(db, sugaredLogger) ciPipelineHistoryServiceImpl := history.NewCiPipelineHistoryServiceImpl(ciPipelineHistoryRepositoryImpl, sugaredLogger, ciPipelineRepositoryImpl) ciBuildConfigRepositoryImpl := pipelineConfig.NewCiBuildConfigRepositoryImpl(db, sugaredLogger) ciBuildConfigServiceImpl := pipeline.NewCiBuildConfigServiceImpl(sugaredLogger, ciBuildConfigRepositoryImpl) ciTemplateServiceImpl := pipeline.NewCiTemplateServiceImpl(sugaredLogger, ciBuildConfigServiceImpl, ciTemplateRepositoryImpl, ciTemplateOverrideRepositoryImpl) pipelineConfigRepositoryImpl := chartConfig.NewPipelineConfigRepository(db) configMapServiceImpl := pipeline.NewConfigMapServiceImpl(chartRepositoryImpl, sugaredLogger, chartRepoRepositoryImpl, mergeUtil, pipelineConfigRepositoryImpl, configMapRepositoryImpl, commonServiceImpl, appRepositoryImpl, configMapHistoryServiceImpl, environmentRepositoryImpl, scopedVariableCMCSManagerImpl) - deploymentTemplateHistoryRepositoryImpl := repository21.NewDeploymentTemplateHistoryRepositoryImpl(sugaredLogger, db) + deploymentTemplateHistoryRepositoryImpl := repository22.NewDeploymentTemplateHistoryRepositoryImpl(sugaredLogger, db) deploymentTemplateHistoryServiceImpl := deploymentTemplate.NewDeploymentTemplateHistoryServiceImpl(sugaredLogger, deploymentTemplateHistoryRepositoryImpl, pipelineRepositoryImpl, chartRepositoryImpl, userServiceImpl, cdWorkflowRepositoryImpl, scopedVariableManagerImpl, deployedAppMetricsServiceImpl, chartRefServiceImpl) chartReadServiceImpl := read16.NewChartReadServiceImpl(sugaredLogger, chartRepositoryImpl, deploymentConfigServiceImpl, deployedAppMetricsServiceImpl, gitOpsConfigReadServiceImpl, chartRefReadServiceImpl) chartServiceImpl := chart.NewChartServiceImpl(chartRepositoryImpl, sugaredLogger, chartTemplateServiceImpl, chartRepoRepositoryImpl, appRepositoryImpl, mergeUtil, envConfigOverrideRepositoryImpl, pipelineConfigRepositoryImpl, environmentRepositoryImpl, deploymentTemplateHistoryServiceImpl, scopedVariableManagerImpl, deployedAppMetricsServiceImpl, chartRefServiceImpl, gitOpsConfigReadServiceImpl, deploymentConfigServiceImpl, envConfigOverrideReadServiceImpl, chartReadServiceImpl) ciCdPipelineOrchestratorImpl := pipeline.NewCiCdPipelineOrchestrator(appRepositoryImpl, sugaredLogger, materialRepositoryImpl, pipelineRepositoryImpl, ciPipelineRepositoryImpl, ciPipelineMaterialRepositoryImpl, cdWorkflowRepositoryImpl, clientImpl, ciCdConfig, appWorkflowRepositoryImpl, environmentRepositoryImpl, attributesServiceImpl, appCrudOperationServiceImpl, userAuthServiceImpl, prePostCdScriptHistoryServiceImpl, pipelineStageServiceImpl, gitMaterialHistoryServiceImpl, ciPipelineHistoryServiceImpl, ciTemplateReadServiceImpl, ciTemplateServiceImpl, dockerArtifactStoreRepositoryImpl, ciArtifactRepositoryImpl, configMapServiceImpl, customTagServiceImpl, genericNoteServiceImpl, chartServiceImpl, transactionUtilImpl, gitOpsConfigReadServiceImpl, deploymentConfigServiceImpl, deploymentConfigReadServiceImpl, chartReadServiceImpl) - ciServiceImpl := pipeline.NewCiServiceImpl(sugaredLogger, workflowServiceImpl, ciPipelineMaterialRepositoryImpl, workFlowStageStatusServiceImpl, eventRESTClientImpl, eventSimpleFactoryImpl, ciPipelineRepositoryImpl, ciArtifactRepositoryImpl, pipelineStageServiceImpl, userServiceImpl, ciTemplateReadServiceImpl, appCrudOperationServiceImpl, environmentRepositoryImpl, appRepositoryImpl, scopedVariableManagerImpl, customTagServiceImpl, pluginInputVariableParserImpl, globalPluginServiceImpl, infraProviderImpl, ciCdPipelineOrchestratorImpl, attributesServiceImpl, ciWorkflowRepositoryImpl, transactionUtilImpl) - ciLogServiceImpl, err := pipeline.NewCiLogServiceImpl(sugaredLogger, ciServiceImpl, k8sServiceImpl) - if err != nil { - return nil, err - } - resourceGroupRepositoryImpl := resourceGroup.NewResourceGroupRepositoryImpl(db) - resourceGroupMappingRepositoryImpl := resourceGroup.NewResourceGroupMappingRepositoryImpl(db) - resourceGroupServiceImpl := resourceGroup2.NewResourceGroupServiceImpl(sugaredLogger, resourceGroupRepositoryImpl, resourceGroupMappingRepositoryImpl, enforcerUtilImpl, devtronResourceSearchableKeyServiceImpl, appStatusRepositoryImpl) - imageTaggingRepositoryImpl := repository22.NewImageTaggingRepositoryImpl(db, transactionUtilImpl) - imageTaggingReadServiceImpl, err := read17.NewImageTaggingReadServiceImpl(imageTaggingRepositoryImpl, sugaredLogger) + pluginInputVariableParserImpl := pipeline.NewPluginInputVariableParserImpl(sugaredLogger, dockerRegistryConfigImpl, customTagServiceImpl) + ciServiceImpl := pipeline.NewCiServiceImpl(sugaredLogger, workFlowStageStatusServiceImpl, eventRESTClientImpl, eventSimpleFactoryImpl, ciWorkflowRepositoryImpl, transactionUtilImpl) + ciLogServiceImpl, err := pipeline.NewCiLogServiceImpl(sugaredLogger, k8sServiceImpl) if err != nil { return nil, err } - imageTaggingServiceImpl := imageTagging.NewImageTaggingServiceImpl(imageTaggingRepositoryImpl, imageTaggingReadServiceImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, environmentRepositoryImpl, sugaredLogger) blobStorageConfigServiceImpl := pipeline.NewBlobStorageConfigServiceImpl(sugaredLogger, k8sServiceImpl, ciCdConfig) - ciHandlerImpl := pipeline.NewCiHandlerImpl(sugaredLogger, ciServiceImpl, ciPipelineMaterialRepositoryImpl, clientImpl, ciWorkflowRepositoryImpl, workflowServiceImpl, ciLogServiceImpl, ciArtifactRepositoryImpl, userServiceImpl, eventRESTClientImpl, eventSimpleFactoryImpl, ciPipelineRepositoryImpl, appListingRepositoryImpl, k8sServiceImpl, pipelineRepositoryImpl, enforcerUtilImpl, resourceGroupServiceImpl, environmentRepositoryImpl, imageTaggingServiceImpl, k8sCommonServiceImpl, clusterServiceImplExtended, blobStorageConfigServiceImpl, appWorkflowRepositoryImpl, customTagServiceImpl, environmentServiceImpl, workFlowStageStatusServiceImpl) - gitWebhookRepositoryImpl := repository23.NewGitWebhookRepositoryImpl(db) - gitWebhookServiceImpl := gitWebhook.NewGitWebhookServiceImpl(sugaredLogger, ciHandlerImpl, gitWebhookRepositoryImpl) + handlerServiceImpl := trigger.NewHandlerServiceImpl(sugaredLogger, workflowServiceImpl, ciPipelineMaterialRepositoryImpl, ciPipelineRepositoryImpl, ciArtifactRepositoryImpl, pipelineStageServiceImpl, userServiceImpl, ciTemplateReadServiceImpl, appCrudOperationServiceImpl, environmentRepositoryImpl, appRepositoryImpl, scopedVariableManagerImpl, customTagServiceImpl, ciCdPipelineOrchestratorImpl, attributesServiceImpl, pluginInputVariableParserImpl, globalPluginServiceImpl, ciServiceImpl, ciWorkflowRepositoryImpl, clientImpl, ciLogServiceImpl, blobStorageConfigServiceImpl, clusterServiceImplExtended, environmentServiceImpl, k8sServiceImpl) + gitWebhookServiceImpl := gitWebhook.NewGitWebhookServiceImpl(sugaredLogger, gitWebhookRepositoryImpl, handlerServiceImpl) gitWebhookRestHandlerImpl := restHandler.NewGitWebhookRestHandlerImpl(sugaredLogger, gitWebhookServiceImpl) ecrConfig, err := pipeline.GetEcrConfig() if err != nil { return nil, err } - ciTemplateHistoryRepositoryImpl := repository21.NewCiTemplateHistoryRepositoryImpl(db, sugaredLogger) + ciTemplateHistoryRepositoryImpl := repository22.NewCiTemplateHistoryRepositoryImpl(db, sugaredLogger) ciTemplateHistoryServiceImpl := history.NewCiTemplateHistoryServiceImpl(ciTemplateHistoryRepositoryImpl, sugaredLogger) + resourceGroupRepositoryImpl := resourceGroup.NewResourceGroupRepositoryImpl(db) + resourceGroupMappingRepositoryImpl := resourceGroup.NewResourceGroupMappingRepositoryImpl(db) + resourceGroupServiceImpl := resourceGroup2.NewResourceGroupServiceImpl(sugaredLogger, resourceGroupRepositoryImpl, resourceGroupMappingRepositoryImpl, enforcerUtilImpl, devtronResourceSearchableKeyServiceImpl, appStatusRepositoryImpl) buildPipelineSwitchServiceImpl := pipeline.NewBuildPipelineSwitchServiceImpl(sugaredLogger, ciPipelineConfigReadServiceImpl, ciPipelineRepositoryImpl, ciCdPipelineOrchestratorImpl, pipelineRepositoryImpl, ciWorkflowRepositoryImpl, appWorkflowRepositoryImpl, ciPipelineHistoryServiceImpl, ciTemplateOverrideRepositoryImpl, ciPipelineMaterialRepositoryImpl) ciPipelineConfigServiceImpl := pipeline.NewCiPipelineConfigServiceImpl(sugaredLogger, ciCdPipelineOrchestratorImpl, dockerArtifactStoreRepositoryImpl, gitMaterialReadServiceImpl, appRepositoryImpl, pipelineRepositoryImpl, ciPipelineConfigReadServiceImpl, ciPipelineRepositoryImpl, ecrConfig, appWorkflowRepositoryImpl, ciCdConfig, attributesServiceImpl, pipelineStageServiceImpl, ciPipelineMaterialRepositoryImpl, ciTemplateServiceImpl, ciTemplateReadServiceImpl, ciTemplateOverrideRepositoryImpl, ciTemplateHistoryServiceImpl, enforcerUtilImpl, ciWorkflowRepositoryImpl, resourceGroupServiceImpl, customTagServiceImpl, cdWorkflowRepositoryImpl, buildPipelineSwitchServiceImpl, pipelineStageRepositoryImpl, globalPluginRepositoryImpl, appListingServiceImpl) ciMaterialConfigServiceImpl := pipeline.NewCiMaterialConfigServiceImpl(sugaredLogger, materialRepositoryImpl, ciTemplateReadServiceImpl, ciCdPipelineOrchestratorImpl, ciPipelineRepositoryImpl, gitMaterialHistoryServiceImpl, pipelineRepositoryImpl, ciPipelineMaterialRepositoryImpl, transactionUtilImpl, gitMaterialReadServiceImpl) + imageTaggingRepositoryImpl := repository23.NewImageTaggingRepositoryImpl(db, transactionUtilImpl) + imageTaggingReadServiceImpl, err := read17.NewImageTaggingReadServiceImpl(imageTaggingRepositoryImpl, sugaredLogger) + if err != nil { + return nil, err + } + imageTaggingServiceImpl := imageTagging.NewImageTaggingServiceImpl(imageTaggingRepositoryImpl, imageTaggingReadServiceImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, environmentRepositoryImpl, sugaredLogger) deploymentGroupRepositoryImpl := repository2.NewDeploymentGroupRepositoryImpl(sugaredLogger, db) - pipelineStrategyHistoryRepositoryImpl := repository21.NewPipelineStrategyHistoryRepositoryImpl(sugaredLogger, db) + pipelineStrategyHistoryRepositoryImpl := repository22.NewPipelineStrategyHistoryRepositoryImpl(sugaredLogger, db) pipelineStrategyHistoryServiceImpl := history.NewPipelineStrategyHistoryServiceImpl(sugaredLogger, pipelineStrategyHistoryRepositoryImpl, userServiceImpl) propertiesConfigServiceImpl := pipeline.NewPropertiesConfigServiceImpl(sugaredLogger, envConfigOverrideRepositoryImpl, chartRepositoryImpl, environmentRepositoryImpl, deploymentTemplateHistoryServiceImpl, scopedVariableManagerImpl, deployedAppMetricsServiceImpl, envConfigOverrideReadServiceImpl, deploymentConfigServiceImpl, chartServiceImpl) installedAppDBExtendedServiceImpl := FullMode.NewInstalledAppDBExtendedServiceImpl(installedAppDBServiceImpl, appStatusServiceImpl, gitOpsConfigReadServiceImpl) @@ -733,7 +738,8 @@ func InitializeApp() (*App, error) { deploymentTemplateValidationServiceEntImpl := validator.NewDeploymentTemplateValidationServiceEntImpl() deploymentTemplateValidationServiceImpl := validator.NewDeploymentTemplateValidationServiceImpl(sugaredLogger, chartRefServiceImpl, scopedVariableManagerImpl, deployedAppMetricsServiceImpl, deploymentTemplateValidationServiceEntImpl) devtronAppGitOpConfigServiceImpl := gitOpsConfig.NewDevtronAppGitOpConfigServiceImpl(sugaredLogger, chartRepositoryImpl, chartServiceImpl, gitOpsConfigReadServiceImpl, gitOpsValidationServiceImpl, argoClientWrapperServiceImpl, deploymentConfigServiceImpl, chartReadServiceImpl) - cdHandlerImpl := pipeline.NewCdHandlerImpl(sugaredLogger, userServiceImpl, cdWorkflowRepositoryImpl, ciLogServiceImpl, ciArtifactRepositoryImpl, ciPipelineMaterialRepositoryImpl, pipelineRepositoryImpl, environmentRepositoryImpl, ciWorkflowRepositoryImpl, enforcerUtilImpl, resourceGroupServiceImpl, imageTaggingServiceImpl, k8sServiceImpl, workflowServiceImpl, clusterServiceImplExtended, blobStorageConfigServiceImpl, customTagServiceImpl, deploymentConfigServiceImpl, workFlowStageStatusServiceImpl, cdWorkflowRunnerServiceImpl) + ciHandlerImpl := pipeline.NewCiHandlerImpl(sugaredLogger, ciServiceImpl, ciPipelineMaterialRepositoryImpl, clientImpl, ciWorkflowRepositoryImpl, ciArtifactRepositoryImpl, userServiceImpl, eventRESTClientImpl, eventSimpleFactoryImpl, ciPipelineRepositoryImpl, appListingRepositoryImpl, pipelineRepositoryImpl, enforcerUtilImpl, resourceGroupServiceImpl, environmentRepositoryImpl, imageTaggingServiceImpl, k8sCommonServiceImpl, appWorkflowRepositoryImpl, customTagServiceImpl, workFlowStageStatusServiceImpl) + cdHandlerImpl := pipeline.NewCdHandlerImpl(sugaredLogger, userServiceImpl, cdWorkflowRepositoryImpl, ciArtifactRepositoryImpl, ciPipelineMaterialRepositoryImpl, pipelineRepositoryImpl, environmentRepositoryImpl, ciWorkflowRepositoryImpl, enforcerUtilImpl, resourceGroupServiceImpl, imageTaggingServiceImpl, k8sServiceImpl, customTagServiceImpl, deploymentConfigServiceImpl, workFlowStageStatusServiceImpl, cdWorkflowRunnerServiceImpl) appWorkflowServiceImpl := appWorkflow2.NewAppWorkflowServiceImpl(sugaredLogger, appWorkflowRepositoryImpl, ciCdPipelineOrchestratorImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, enforcerUtilImpl, resourceGroupServiceImpl, appRepositoryImpl, userAuthServiceImpl, chartServiceImpl, deploymentConfigServiceImpl, pipelineBuilderImpl) appCloneServiceImpl := appClone.NewAppCloneServiceImpl(sugaredLogger, pipelineBuilderImpl, attributesServiceImpl, chartServiceImpl, configMapServiceImpl, appWorkflowServiceImpl, appListingServiceImpl, propertiesConfigServiceImpl, pipelineStageServiceImpl, ciTemplateReadServiceImpl, appRepositoryImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, ciPipelineConfigServiceImpl, gitOpsConfigReadServiceImpl, chartReadServiceImpl) deploymentTemplateRepositoryImpl := repository2.NewDeploymentTemplateRepositoryImpl(db, sugaredLogger) @@ -752,7 +758,6 @@ func InitializeApp() (*App, error) { policyServiceImpl := imageScanning.NewPolicyServiceImpl(environmentServiceImpl, sugaredLogger, appRepositoryImpl, pipelineOverrideRepositoryImpl, cvePolicyRepositoryImpl, clusterServiceImplExtended, pipelineRepositoryImpl, imageScanResultRepositoryImpl, imageScanDeployInfoRepositoryImpl, imageScanObjectMetaRepositoryImpl, httpClient, ciArtifactRepositoryImpl, ciCdConfig, imageScanHistoryReadServiceImpl, cveStoreRepositoryImpl, ciTemplateRepositoryImpl, clusterReadServiceImpl, transactionUtilImpl) imageScanResultReadServiceImpl := read18.NewImageScanResultReadServiceImpl(sugaredLogger, imageScanResultRepositoryImpl) draftAwareConfigServiceImpl := draftAwareConfigService.NewDraftAwareResourceServiceImpl(sugaredLogger, configMapServiceImpl, chartServiceImpl, propertiesConfigServiceImpl) - pipelineConfigRestHandlerImpl := configure.NewPipelineRestHandlerImpl(pipelineBuilderImpl, sugaredLogger, deploymentTemplateValidationServiceImpl, chartServiceImpl, devtronAppGitOpConfigServiceImpl, propertiesConfigServiceImpl, userServiceImpl, teamServiceImpl, enforcerImpl, ciHandlerImpl, validate, clientImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, enforcerUtilImpl, dockerRegistryConfigImpl, cdHandlerImpl, appCloneServiceImpl, generateManifestDeploymentTemplateServiceImpl, appWorkflowServiceImpl, gitMaterialReadServiceImpl, policyServiceImpl, imageScanResultReadServiceImpl, ciPipelineMaterialRepositoryImpl, imageTaggingReadServiceImpl, imageTaggingServiceImpl, ciArtifactRepositoryImpl, deployedAppMetricsServiceImpl, chartRefServiceImpl, ciCdPipelineOrchestratorImpl, gitProviderReadServiceImpl, teamReadServiceImpl, environmentRepositoryImpl, chartReadServiceImpl, draftAwareConfigServiceImpl) gitOpsManifestPushServiceImpl := publish.NewGitOpsManifestPushServiceImpl(sugaredLogger, pipelineStatusTimelineServiceImpl, pipelineOverrideRepositoryImpl, acdConfig, chartRefServiceImpl, gitOpsConfigReadServiceImpl, chartServiceImpl, gitOperationServiceImpl, argoClientWrapperServiceImpl, transactionUtilImpl, deploymentConfigServiceImpl, chartTemplateServiceImpl) manifestCreationServiceImpl := manifest.NewManifestCreationServiceImpl(sugaredLogger, dockerRegistryIpsConfigServiceImpl, chartRefServiceImpl, scopedVariableCMCSManagerImpl, k8sCommonServiceImpl, deployedAppMetricsServiceImpl, imageDigestPolicyServiceImpl, utilMergeUtil, appCrudOperationServiceImpl, deploymentTemplateServiceImpl, argoClientWrapperServiceImpl, configMapHistoryRepositoryImpl, configMapRepositoryImpl, chartRepositoryImpl, envConfigOverrideRepositoryImpl, environmentRepositoryImpl, pipelineRepositoryImpl, ciArtifactRepositoryImpl, pipelineOverrideRepositoryImpl, pipelineStrategyHistoryRepositoryImpl, pipelineConfigRepositoryImpl, deploymentTemplateHistoryRepositoryImpl, deploymentConfigServiceImpl, envConfigOverrideReadServiceImpl) configMapHistoryReadServiceImpl := read19.NewConfigMapHistoryReadService(sugaredLogger, configMapHistoryRepositoryImpl, scopedVariableCMCSManagerImpl) @@ -761,16 +766,17 @@ func InitializeApp() (*App, error) { userDeploymentRequestServiceImpl := service3.NewUserDeploymentRequestServiceImpl(sugaredLogger, userDeploymentRequestRepositoryImpl) imageScanDeployInfoReadServiceImpl := read18.NewImageScanDeployInfoReadService(sugaredLogger, imageScanDeployInfoRepositoryImpl) imageScanDeployInfoServiceImpl := imageScanning.NewImageScanDeployInfoService(sugaredLogger, imageScanDeployInfoRepositoryImpl) - manifestPushConfigRepositoryImpl := repository18.NewManifestPushConfigRepository(sugaredLogger, db) + manifestPushConfigRepositoryImpl := repository19.NewManifestPushConfigRepository(sugaredLogger, db) scanToolExecutionHistoryMappingRepositoryImpl := repository24.NewScanToolExecutionHistoryMappingRepositoryImpl(db, sugaredLogger) cdWorkflowReadServiceImpl := read20.NewCdWorkflowReadServiceImpl(sugaredLogger, cdWorkflowRepositoryImpl) imageScanServiceImpl := imageScanning.NewImageScanServiceImpl(sugaredLogger, imageScanHistoryRepositoryImpl, imageScanResultRepositoryImpl, imageScanObjectMetaRepositoryImpl, cveStoreRepositoryImpl, imageScanDeployInfoRepositoryImpl, userServiceImpl, appRepositoryImpl, environmentServiceImpl, ciArtifactRepositoryImpl, policyServiceImpl, pipelineRepositoryImpl, ciPipelineRepositoryImpl, scanToolMetadataRepositoryImpl, scanToolExecutionHistoryMappingRepositoryImpl, cvePolicyRepositoryImpl, cdWorkflowReadServiceImpl) - triggerServiceImpl, err := devtronApps.NewTriggerServiceImpl(sugaredLogger, cdWorkflowCommonServiceImpl, gitOpsManifestPushServiceImpl, gitOpsConfigReadServiceImpl, argoK8sClientImpl, acdConfig, argoClientWrapperServiceImpl, pipelineStatusTimelineServiceImpl, chartTemplateServiceImpl, workflowEventPublishServiceImpl, manifestCreationServiceImpl, deployedConfigurationHistoryServiceImpl, pipelineStageServiceImpl, globalPluginServiceImpl, customTagServiceImpl, pluginInputVariableParserImpl, prePostCdScriptHistoryServiceImpl, scopedVariableCMCSManagerImpl, workflowServiceImpl, imageDigestPolicyServiceImpl, userServiceImpl, clientImpl, helmAppServiceImpl, enforcerUtilImpl, userDeploymentRequestServiceImpl, helmAppClientImpl, eventSimpleFactoryImpl, eventRESTClientImpl, environmentVariables, appRepositoryImpl, ciPipelineMaterialRepositoryImpl, imageScanHistoryReadServiceImpl, imageScanDeployInfoReadServiceImpl, imageScanDeployInfoServiceImpl, pipelineRepositoryImpl, pipelineOverrideRepositoryImpl, manifestPushConfigRepositoryImpl, chartRepositoryImpl, environmentRepositoryImpl, cdWorkflowRepositoryImpl, ciWorkflowRepositoryImpl, ciArtifactRepositoryImpl, ciTemplateReadServiceImpl, gitMaterialReadServiceImpl, appLabelRepositoryImpl, ciPipelineRepositoryImpl, appWorkflowRepositoryImpl, dockerArtifactStoreRepositoryImpl, imageScanServiceImpl, k8sServiceImpl, transactionUtilImpl, deploymentConfigServiceImpl, ciCdPipelineOrchestratorImpl, gitOperationServiceImpl, attributesServiceImpl, clusterRepositoryImpl, cdWorkflowRunnerServiceImpl) + devtronAppsHandlerServiceImpl, err := devtronApps.NewHandlerServiceImpl(sugaredLogger, cdWorkflowCommonServiceImpl, gitOpsManifestPushServiceImpl, gitOpsConfigReadServiceImpl, argoK8sClientImpl, acdConfig, argoClientWrapperServiceImpl, pipelineStatusTimelineServiceImpl, chartTemplateServiceImpl, workflowEventPublishServiceImpl, manifestCreationServiceImpl, deployedConfigurationHistoryServiceImpl, pipelineStageServiceImpl, globalPluginServiceImpl, customTagServiceImpl, pluginInputVariableParserImpl, prePostCdScriptHistoryServiceImpl, scopedVariableCMCSManagerImpl, imageDigestPolicyServiceImpl, userServiceImpl, helmAppServiceImpl, enforcerUtilImpl, userDeploymentRequestServiceImpl, helmAppClientImpl, eventSimpleFactoryImpl, eventRESTClientImpl, environmentVariables, appRepositoryImpl, ciPipelineMaterialRepositoryImpl, imageScanHistoryReadServiceImpl, imageScanDeployInfoReadServiceImpl, imageScanDeployInfoServiceImpl, pipelineRepositoryImpl, pipelineOverrideRepositoryImpl, manifestPushConfigRepositoryImpl, chartRepositoryImpl, environmentRepositoryImpl, cdWorkflowRepositoryImpl, ciWorkflowRepositoryImpl, ciArtifactRepositoryImpl, ciTemplateReadServiceImpl, gitMaterialReadServiceImpl, appLabelRepositoryImpl, ciPipelineRepositoryImpl, appWorkflowRepositoryImpl, dockerArtifactStoreRepositoryImpl, imageScanServiceImpl, k8sServiceImpl, transactionUtilImpl, deploymentConfigServiceImpl, ciCdPipelineOrchestratorImpl, gitOperationServiceImpl, attributesServiceImpl, clusterRepositoryImpl, cdWorkflowRunnerServiceImpl, clusterServiceImplExtended, ciLogServiceImpl, workflowServiceImpl, blobStorageConfigServiceImpl) if err != nil { return nil, err } + pipelineConfigRestHandlerImpl := configure.NewPipelineRestHandlerImpl(pipelineBuilderImpl, sugaredLogger, deploymentTemplateValidationServiceImpl, chartServiceImpl, devtronAppGitOpConfigServiceImpl, propertiesConfigServiceImpl, userServiceImpl, teamServiceImpl, enforcerImpl, ciHandlerImpl, validate, clientImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, enforcerUtilImpl, dockerRegistryConfigImpl, cdHandlerImpl, appCloneServiceImpl, generateManifestDeploymentTemplateServiceImpl, appWorkflowServiceImpl, gitMaterialReadServiceImpl, policyServiceImpl, imageScanResultReadServiceImpl, ciPipelineMaterialRepositoryImpl, imageTaggingReadServiceImpl, imageTaggingServiceImpl, ciArtifactRepositoryImpl, deployedAppMetricsServiceImpl, chartRefServiceImpl, ciCdPipelineOrchestratorImpl, gitProviderReadServiceImpl, teamReadServiceImpl, environmentRepositoryImpl, chartReadServiceImpl, draftAwareConfigServiceImpl, handlerServiceImpl, devtronAppsHandlerServiceImpl) commonArtifactServiceImpl := artifacts.NewCommonArtifactServiceImpl(sugaredLogger, ciArtifactRepositoryImpl) - workflowDagExecutorImpl := dag.NewWorkflowDagExecutorImpl(sugaredLogger, pipelineRepositoryImpl, cdWorkflowRepositoryImpl, ciArtifactRepositoryImpl, enforcerUtilImpl, appWorkflowRepositoryImpl, pipelineStageServiceImpl, ciWorkflowRepositoryImpl, ciPipelineRepositoryImpl, pipelineStageRepositoryImpl, globalPluginRepositoryImpl, eventRESTClientImpl, eventSimpleFactoryImpl, customTagServiceImpl, pipelineStatusTimelineServiceImpl, cdWorkflowRunnerServiceImpl, ciServiceImpl, helmAppServiceImpl, cdWorkflowCommonServiceImpl, triggerServiceImpl, userDeploymentRequestServiceImpl, manifestCreationServiceImpl, commonArtifactServiceImpl, deploymentConfigServiceImpl, runnable, imageScanHistoryRepositoryImpl, imageScanServiceImpl) + workflowDagExecutorImpl := dag.NewWorkflowDagExecutorImpl(sugaredLogger, pipelineRepositoryImpl, cdWorkflowRepositoryImpl, ciArtifactRepositoryImpl, enforcerUtilImpl, appWorkflowRepositoryImpl, pipelineStageServiceImpl, ciWorkflowRepositoryImpl, ciPipelineRepositoryImpl, pipelineStageRepositoryImpl, globalPluginRepositoryImpl, eventRESTClientImpl, eventSimpleFactoryImpl, customTagServiceImpl, pipelineStatusTimelineServiceImpl, cdWorkflowRunnerServiceImpl, ciServiceImpl, helmAppServiceImpl, cdWorkflowCommonServiceImpl, devtronAppsHandlerServiceImpl, userDeploymentRequestServiceImpl, manifestCreationServiceImpl, commonArtifactServiceImpl, deploymentConfigServiceImpl, runnable, imageScanHistoryRepositoryImpl, imageScanServiceImpl, k8sServiceImpl, environmentRepositoryImpl, k8sCommonServiceImpl, workflowServiceImpl, handlerServiceImpl) externalCiRestHandlerImpl := restHandler.NewExternalCiRestHandlerImpl(sugaredLogger, validate, userServiceImpl, enforcerImpl, workflowDagExecutorImpl) pubSubClientRestHandlerImpl := restHandler.NewPubSubClientRestHandlerImpl(pubSubClientServiceImpl, sugaredLogger, ciCdConfig) webhookRouterImpl := router.NewWebhookRouterImpl(gitWebhookRestHandlerImpl, pipelineConfigRestHandlerImpl, externalCiRestHandlerImpl, pubSubClientRestHandlerImpl) @@ -933,15 +939,15 @@ func InitializeApp() (*App, error) { return nil, err } providerIdentifierServiceImpl := providerIdentifier.NewProviderIdentifierServiceImpl(sugaredLogger) - telemetryEventClientImplExtended, err := telemetry.NewTelemetryEventClientImplExtended(sugaredLogger, httpClient, clusterServiceImplExtended, k8sServiceImpl, acdAuthConfig, environmentServiceImpl, userServiceImpl, appListingRepositoryImpl, posthogClient, ciPipelineConfigReadServiceImpl, pipelineRepositoryImpl, gitProviderRepositoryImpl, attributesRepositoryImpl, ssoLoginServiceImpl, appRepositoryImpl, ciWorkflowRepositoryImpl, cdWorkflowRepositoryImpl, dockerArtifactStoreRepositoryImpl, gitMaterialReadServiceImpl, ciTemplateRepositoryImpl, chartRepositoryImpl, userAuditServiceImpl, ciBuildConfigServiceImpl, moduleRepositoryImpl, serverDataStoreServerDataStore, helmAppClientImpl, installedAppReadServiceImpl, userAttributesRepositoryImpl, providerIdentifierServiceImpl, cronLoggerImpl, gitOpsConfigReadServiceImpl, environmentVariables) + telemetryEventClientImplExtended, err := telemetry2.NewTelemetryEventClientImplExtended(sugaredLogger, httpClient, clusterServiceImplExtended, k8sServiceImpl, acdAuthConfig, environmentServiceImpl, userServiceImpl, appListingRepositoryImpl, posthogClient, serviceImpl, ciPipelineConfigReadServiceImpl, pipelineRepositoryImpl, gitProviderRepositoryImpl, attributesRepositoryImpl, ssoLoginServiceImpl, appRepositoryImpl, ciWorkflowRepositoryImpl, cdWorkflowRepositoryImpl, dockerArtifactStoreRepositoryImpl, gitMaterialReadServiceImpl, ciTemplateRepositoryImpl, chartRepositoryImpl, userAuditServiceImpl, ciBuildConfigServiceImpl, moduleRepositoryImpl, serverDataStoreServerDataStore, helmAppClientImpl, installedAppReadServiceImpl, userAttributesRepositoryImpl, providerIdentifierServiceImpl, cronLoggerImpl, gitOpsConfigReadServiceImpl, environmentVariables) if err != nil { return nil, err } telemetryRestHandlerImpl := restHandler.NewTelemetryRestHandlerImpl(sugaredLogger, telemetryEventClientImplExtended, enforcerImpl, userServiceImpl) telemetryRouterImpl := router.NewTelemetryRouterImpl(sugaredLogger, telemetryRestHandlerImpl) bulkUpdateRepositoryImpl := bulkUpdate.NewBulkUpdateRepository(db, sugaredLogger) - deployedAppServiceImpl := deployedApp.NewDeployedAppServiceImpl(sugaredLogger, k8sCommonServiceImpl, triggerServiceImpl, environmentRepositoryImpl, pipelineRepositoryImpl, cdWorkflowRepositoryImpl) - bulkUpdateServiceImpl := service7.NewBulkUpdateServiceImpl(bulkUpdateRepositoryImpl, sugaredLogger, environmentRepositoryImpl, pipelineRepositoryImpl, appRepositoryImpl, deploymentTemplateHistoryServiceImpl, configMapHistoryServiceImpl, pipelineBuilderImpl, enforcerUtilImpl, ciHandlerImpl, ciPipelineRepositoryImpl, appWorkflowRepositoryImpl, appWorkflowServiceImpl, scopedVariableManagerImpl, deployedAppMetricsServiceImpl, chartRefServiceImpl, deployedAppServiceImpl, cdPipelineEventPublishServiceImpl) + deployedAppServiceImpl := deployedApp.NewDeployedAppServiceImpl(sugaredLogger, k8sCommonServiceImpl, devtronAppsHandlerServiceImpl, environmentRepositoryImpl, pipelineRepositoryImpl, cdWorkflowRepositoryImpl) + bulkUpdateServiceImpl := service7.NewBulkUpdateServiceImpl(bulkUpdateRepositoryImpl, sugaredLogger, environmentRepositoryImpl, pipelineRepositoryImpl, appRepositoryImpl, deploymentTemplateHistoryServiceImpl, configMapHistoryServiceImpl, pipelineBuilderImpl, enforcerUtilImpl, ciHandlerImpl, ciPipelineRepositoryImpl, appWorkflowRepositoryImpl, appWorkflowServiceImpl, scopedVariableManagerImpl, deployedAppMetricsServiceImpl, chartRefServiceImpl, deployedAppServiceImpl, cdPipelineEventPublishServiceImpl, handlerServiceImpl) bulkUpdateRestHandlerImpl := restHandler.NewBulkUpdateRestHandlerImpl(pipelineBuilderImpl, sugaredLogger, bulkUpdateServiceImpl, chartServiceImpl, propertiesConfigServiceImpl, userServiceImpl, teamServiceImpl, enforcerImpl, ciHandlerImpl, validate, clientImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, enforcerUtilImpl, environmentServiceImpl, gitRegistryConfigImpl, dockerRegistryConfigImpl, cdHandlerImpl, appCloneServiceImpl, appWorkflowServiceImpl, materialRepositoryImpl) bulkUpdateRouterImpl := router.NewBulkUpdateRouterImpl(bulkUpdateRestHandlerImpl) webhookSecretValidatorImpl := gitWebhook.NewWebhookSecretValidatorImpl(sugaredLogger) @@ -952,18 +958,18 @@ func InitializeApp() (*App, error) { webhookListenerRouterImpl := router.NewWebhookListenerRouterImpl(webhookEventHandlerImpl) appFilteringRestHandlerImpl := appList.NewAppFilteringRestHandlerImpl(sugaredLogger, teamServiceImpl, enforcerImpl, userServiceImpl, clusterServiceImplExtended, environmentServiceImpl, teamReadServiceImpl) appFilteringRouterImpl := appList2.NewAppFilteringRouterImpl(appFilteringRestHandlerImpl) - serviceImpl := resourceTree.NewServiceImpl(sugaredLogger, appListingServiceImpl, appStatusServiceImpl, argoApplicationServiceExtendedImpl, cdApplicationStatusUpdateHandlerImpl, helmAppReadServiceImpl, helmAppServiceImpl, k8sApplicationServiceImpl, k8sCommonServiceImpl, environmentReadServiceImpl) - appListingRestHandlerImpl := appList.NewAppListingRestHandlerImpl(appListingServiceImpl, enforcerImpl, pipelineBuilderImpl, sugaredLogger, enforcerUtilImpl, deploymentGroupServiceImpl, userServiceImpl, k8sCommonServiceImpl, installedAppDBExtendedServiceImpl, installedAppResourceServiceImpl, pipelineRepositoryImpl, k8sApplicationServiceImpl, deploymentConfigServiceImpl, serviceImpl) + resourceTreeServiceImpl := resourceTree.NewServiceImpl(sugaredLogger, appListingServiceImpl, appStatusServiceImpl, argoApplicationServiceExtendedImpl, cdApplicationStatusUpdateHandlerImpl, helmAppReadServiceImpl, helmAppServiceImpl, k8sApplicationServiceImpl, k8sCommonServiceImpl, environmentReadServiceImpl) + appListingRestHandlerImpl := appList.NewAppListingRestHandlerImpl(appListingServiceImpl, enforcerImpl, pipelineBuilderImpl, sugaredLogger, enforcerUtilImpl, deploymentGroupServiceImpl, userServiceImpl, k8sCommonServiceImpl, installedAppDBExtendedServiceImpl, installedAppResourceServiceImpl, pipelineRepositoryImpl, k8sApplicationServiceImpl, deploymentConfigServiceImpl, resourceTreeServiceImpl) appListingRouterImpl := appList2.NewAppListingRouterImpl(appListingRestHandlerImpl) appInfoRestHandlerImpl := appInfo.NewAppInfoRestHandlerImpl(sugaredLogger, appCrudOperationServiceImpl, userServiceImpl, validate, enforcerUtilImpl, enforcerImpl, helmAppServiceImpl, enforcerUtilHelmImpl, genericNoteServiceImpl, commonEnforcementUtilImpl) appInfoRouterImpl := appInfo2.NewAppInfoRouterImpl(sugaredLogger, appInfoRestHandlerImpl) pipelineDeploymentConfigServiceImpl := pipeline.NewPipelineDeploymentConfigServiceImpl(sugaredLogger, chartRepositoryImpl, pipelineRepositoryImpl, pipelineConfigRepositoryImpl, configMapRepositoryImpl, scopedVariableCMCSManagerImpl, deployedAppMetricsServiceImpl, chartRefServiceImpl, configMapHistoryReadServiceImpl, envConfigOverrideReadServiceImpl) - pipelineTriggerRestHandlerImpl := trigger.NewPipelineRestHandler(appServiceImpl, userServiceImpl, validate, enforcerImpl, teamServiceImpl, sugaredLogger, enforcerUtilImpl, deploymentGroupServiceImpl, pipelineDeploymentConfigServiceImpl, deployedAppServiceImpl, triggerServiceImpl, workflowEventPublishServiceImpl) + pipelineTriggerRestHandlerImpl := trigger2.NewPipelineRestHandler(appServiceImpl, userServiceImpl, validate, enforcerImpl, teamServiceImpl, sugaredLogger, enforcerUtilImpl, deploymentGroupServiceImpl, pipelineDeploymentConfigServiceImpl, deployedAppServiceImpl, devtronAppsHandlerServiceImpl, workflowEventPublishServiceImpl) sseSSE := sse.NewSSE() - pipelineTriggerRouterImpl := trigger2.NewPipelineTriggerRouter(pipelineTriggerRestHandlerImpl, sseSSE) + pipelineTriggerRouterImpl := trigger3.NewPipelineTriggerRouter(pipelineTriggerRestHandlerImpl, sseSSE) webhookDataRestHandlerImpl := webhook.NewWebhookDataRestHandlerImpl(sugaredLogger, userServiceImpl, ciPipelineMaterialRepositoryImpl, enforcerUtilImpl, enforcerImpl, clientImpl, webhookEventDataConfigImpl) pipelineConfigRouterImpl := configure2.NewPipelineRouterImpl(pipelineConfigRestHandlerImpl, webhookDataRestHandlerImpl) - prePostCiScriptHistoryRepositoryImpl := repository21.NewPrePostCiScriptHistoryRepositoryImpl(sugaredLogger, db) + prePostCiScriptHistoryRepositoryImpl := repository22.NewPrePostCiScriptHistoryRepositoryImpl(sugaredLogger, db) prePostCiScriptHistoryServiceImpl := history.NewPrePostCiScriptHistoryServiceImpl(sugaredLogger, prePostCiScriptHistoryRepositoryImpl) pipelineHistoryRestHandlerImpl := history2.NewPipelineHistoryRestHandlerImpl(sugaredLogger, userServiceImpl, enforcerImpl, pipelineStrategyHistoryServiceImpl, deploymentTemplateHistoryServiceImpl, configMapHistoryServiceImpl, prePostCiScriptHistoryServiceImpl, prePostCdScriptHistoryServiceImpl, enforcerUtilImpl, deployedConfigurationHistoryServiceImpl) pipelineHistoryRouterImpl := history3.NewPipelineHistoryRouterImpl(pipelineHistoryRestHandlerImpl) @@ -1034,7 +1040,7 @@ func InitializeApp() (*App, error) { if err != nil { return nil, err } - ciStatusUpdateCronImpl := cron2.NewCiStatusUpdateCronImpl(sugaredLogger, appServiceImpl, ciWorkflowStatusUpdateConfig, ciPipelineRepositoryImpl, ciHandlerImpl, cronLoggerImpl) + ciStatusUpdateCronImpl := cron2.NewCiStatusUpdateCronImpl(sugaredLogger, appServiceImpl, ciWorkflowStatusUpdateConfig, ciPipelineRepositoryImpl, cronLoggerImpl, workflowDagExecutorImpl) resourceGroupRestHandlerImpl := restHandler.NewResourceGroupRestHandlerImpl(sugaredLogger, enforcerImpl, userServiceImpl, resourceGroupServiceImpl, validate) resourceGroupingRouterImpl := router.NewResourceGroupingRouterImpl(pipelineConfigRestHandlerImpl, appWorkflowRestHandlerImpl, resourceGroupRestHandlerImpl) rbacRoleServiceImpl := user.NewRbacRoleServiceImpl(sugaredLogger, rbacRoleDataRepositoryImpl) @@ -1046,7 +1052,7 @@ func InitializeApp() (*App, error) { if err != nil { return nil, err } - ciTriggerCronImpl := cron2.NewCiTriggerCronImpl(sugaredLogger, ciTriggerCronConfig, pipelineStageRepositoryImpl, ciHandlerImpl, ciArtifactRepositoryImpl, globalPluginRepositoryImpl, cronLoggerImpl) + ciTriggerCronImpl := cron2.NewCiTriggerCronImpl(sugaredLogger, ciTriggerCronConfig, pipelineStageRepositoryImpl, ciArtifactRepositoryImpl, globalPluginRepositoryImpl, cronLoggerImpl, handlerServiceImpl) proxyConfig, err := proxy.GetProxyConfig() if err != nil { return nil, err @@ -1082,12 +1088,12 @@ func InitializeApp() (*App, error) { cdWorkflowServiceImpl := cd.NewCdWorkflowServiceImpl(sugaredLogger, cdWorkflowRepositoryImpl) cdWorkflowRunnerReadServiceImpl := read20.NewCdWorkflowRunnerReadServiceImpl(sugaredLogger, cdWorkflowRepositoryImpl) webhookServiceImpl := pipeline.NewWebhookServiceImpl(ciArtifactRepositoryImpl, sugaredLogger, ciPipelineRepositoryImpl, ciWorkflowRepositoryImpl, cdWorkflowCommonServiceImpl, workFlowStageStatusServiceImpl, ciServiceImpl) - workflowEventProcessorImpl, err := in.NewWorkflowEventProcessorImpl(sugaredLogger, pubSubClientServiceImpl, cdWorkflowServiceImpl, cdWorkflowReadServiceImpl, cdWorkflowRunnerServiceImpl, cdWorkflowRunnerReadServiceImpl, workflowDagExecutorImpl, ciHandlerImpl, cdHandlerImpl, eventSimpleFactoryImpl, eventRESTClientImpl, triggerServiceImpl, deployedAppServiceImpl, webhookServiceImpl, validate, environmentVariables, cdWorkflowCommonServiceImpl, cdPipelineConfigServiceImpl, userDeploymentRequestServiceImpl, pipelineRepositoryImpl, ciArtifactRepositoryImpl, cdWorkflowRepositoryImpl, deploymentConfigServiceImpl) + workflowEventProcessorImpl, err := in.NewWorkflowEventProcessorImpl(sugaredLogger, pubSubClientServiceImpl, cdWorkflowServiceImpl, cdWorkflowReadServiceImpl, cdWorkflowRunnerServiceImpl, cdWorkflowRunnerReadServiceImpl, workflowDagExecutorImpl, ciHandlerImpl, cdHandlerImpl, eventSimpleFactoryImpl, eventRESTClientImpl, devtronAppsHandlerServiceImpl, deployedAppServiceImpl, webhookServiceImpl, validate, environmentVariables, cdWorkflowCommonServiceImpl, cdPipelineConfigServiceImpl, userDeploymentRequestServiceImpl, serviceImpl, pipelineRepositoryImpl, ciArtifactRepositoryImpl, cdWorkflowRepositoryImpl, deploymentConfigServiceImpl, handlerServiceImpl) if err != nil { return nil, err } ciPipelineEventProcessorImpl := in.NewCIPipelineEventProcessorImpl(sugaredLogger, pubSubClientServiceImpl, gitWebhookServiceImpl) - cdPipelineEventProcessorImpl := in.NewCDPipelineEventProcessorImpl(sugaredLogger, pubSubClientServiceImpl, cdWorkflowCommonServiceImpl, workflowStatusServiceImpl, triggerServiceImpl, pipelineRepositoryImpl, installedAppReadServiceImpl) + cdPipelineEventProcessorImpl := in.NewCDPipelineEventProcessorImpl(sugaredLogger, pubSubClientServiceImpl, cdWorkflowCommonServiceImpl, workflowStatusServiceImpl, devtronAppsHandlerServiceImpl, pipelineRepositoryImpl, installedAppReadServiceImpl) deployedApplicationEventProcessorImpl := in.NewDeployedApplicationEventProcessorImpl(sugaredLogger, pubSubClientServiceImpl, appServiceImpl, gitOpsConfigReadServiceImpl, installedAppDBExtendedServiceImpl, workflowDagExecutorImpl, cdWorkflowCommonServiceImpl, pipelineBuilderImpl, appStoreDeploymentServiceImpl, pipelineRepositoryImpl, installedAppReadServiceImpl, deploymentConfigServiceImpl) appStoreAppsEventProcessorImpl := in.NewAppStoreAppsEventProcessorImpl(sugaredLogger, pubSubClientServiceImpl, chartGroupServiceImpl, installedAppVersionHistoryRepositoryImpl) centralEventProcessor, err := eventProcessor.NewCentralEventProcessor(sugaredLogger, workflowEventProcessorImpl, ciPipelineEventProcessorImpl, cdPipelineEventProcessorImpl, deployedApplicationEventProcessorImpl, appStoreAppsEventProcessorImpl) From 44e36db51c15fc1d35d855774f7fc572d78d99c5 Mon Sep 17 00:00:00 2001 From: Shivam Nagar <124123645+Shivam-nagar23@users.noreply.github.com> Date: Thu, 24 Apr 2025 18:31:36 +0530 Subject: [PATCH 32/34] retrigger bug (#6534) --- pkg/eventProcessor/in/WorkflowEventProcessorService.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/eventProcessor/in/WorkflowEventProcessorService.go b/pkg/eventProcessor/in/WorkflowEventProcessorService.go index cb34134dd1..03c8e8ad63 100644 --- a/pkg/eventProcessor/in/WorkflowEventProcessorService.go +++ b/pkg/eventProcessor/in/WorkflowEventProcessorService.go @@ -501,7 +501,7 @@ func (impl *WorkflowEventProcessorImpl) SubscribeCDWorkflowStatusUpdate() error wfStatusInEvent := string(wfStatus.Phase) if wfStatusInEvent == string(v1alpha1.NodeSucceeded) || wfStatusInEvent == string(v1alpha1.NodeFailed) || wfStatusInEvent == string(v1alpha1.NodeError) { // the re-trigger should only happen when we get a pod deleted event. - if executors.CheckIfReTriggerRequired(status, wfStatus.Message, wfr.Status) { + if executors.CheckIfReTriggerRequired(status, wfr.Message, wfr.Status) { err = impl.workflowDagExecutor.HandleCdStageReTrigger(wfr) if err != nil { // check if this log required or not From b99bc5cb757b98667d6dfa233399e6dc6d11ef3d Mon Sep 17 00:00:00 2001 From: Shivam Nagar <124123645+Shivam-nagar23@users.noreply.github.com> Date: Fri, 25 Apr 2025 08:44:12 +0530 Subject: [PATCH 33/34] workflow status message fix (#6536) --- .../in/WorkflowEventProcessorService.go | 4 ++-- pkg/pipeline/CdHandler.go | 18 +++++++++--------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/pkg/eventProcessor/in/WorkflowEventProcessorService.go b/pkg/eventProcessor/in/WorkflowEventProcessorService.go index 03c8e8ad63..113ce26ff6 100644 --- a/pkg/eventProcessor/in/WorkflowEventProcessorService.go +++ b/pkg/eventProcessor/in/WorkflowEventProcessorService.go @@ -474,7 +474,7 @@ func (impl *WorkflowEventProcessorImpl) SubscribeCDWorkflowStatusUpdate() error return } } - wfrId, status, stateChanged, err := impl.cdHandler.UpdateWorkflow(wfStatus) + wfrId, status, stateChanged, wfStatusMessage, err := impl.cdHandler.UpdateWorkflow(wfStatus) impl.logger.Debugw("cd UpdateWorkflow for wfStatus", "wfrId", wfrId, "status", status, "wfStatus", wfStatus) if err != nil { impl.logger.Errorw("error in cd workflow status update", "wfrId", wfrId, "status", status, "wfStatus", wfStatus, "err", err) @@ -501,7 +501,7 @@ func (impl *WorkflowEventProcessorImpl) SubscribeCDWorkflowStatusUpdate() error wfStatusInEvent := string(wfStatus.Phase) if wfStatusInEvent == string(v1alpha1.NodeSucceeded) || wfStatusInEvent == string(v1alpha1.NodeFailed) || wfStatusInEvent == string(v1alpha1.NodeError) { // the re-trigger should only happen when we get a pod deleted event. - if executors.CheckIfReTriggerRequired(status, wfr.Message, wfr.Status) { + if executors.CheckIfReTriggerRequired(status, wfStatusMessage, wfr.Status) { err = impl.workflowDagExecutor.HandleCdStageReTrigger(wfr) if err != nil { // check if this log required or not diff --git a/pkg/pipeline/CdHandler.go b/pkg/pipeline/CdHandler.go index 7ceacc1c2e..4fbdba573d 100644 --- a/pkg/pipeline/CdHandler.go +++ b/pkg/pipeline/CdHandler.go @@ -60,7 +60,7 @@ const ( ) type CdHandler interface { - UpdateWorkflow(workflowStatus eventProcessorBean.CiCdStatus) (int, string, bool, error) + UpdateWorkflow(workflowStatus eventProcessorBean.CiCdStatus) (int, string, bool, string, error) GetCdBuildHistory(appId int, environmentId int, pipelineId int, offset int, size int) ([]pipelineBean.CdWorkflowWithArtifact, error) FetchCdWorkflowDetails(appId int, environmentId int, pipelineId int, buildId int) (types.WorkflowResponse, error) FetchCdPrePostStageStatus(pipelineId int) ([]pipelineBean.CdWorkflowWithArtifact, error) @@ -129,23 +129,23 @@ func NewCdHandlerImpl(Logger *zap.SugaredLogger, userService user.UserService, return cdh } -func (impl *CdHandlerImpl) UpdateWorkflow(workflowStatus eventProcessorBean.CiCdStatus) (int, string, bool, error) { +func (impl *CdHandlerImpl) UpdateWorkflow(workflowStatus eventProcessorBean.CiCdStatus) (int, string, bool, string, error) { wfStatusRs := impl.extractWorkflowStatus(workflowStatus) workflowName, status, podStatus, message, podName := wfStatusRs.WorkflowName, wfStatusRs.Status, wfStatusRs.PodStatus, wfStatusRs.Message, wfStatusRs.PodName impl.Logger.Debugw("cd workflow status update event for", "wf ", workflowName, "status", status) if workflowName == "" { - return 0, "", false, errors.New("invalid wf name") + return 0, "", false, "", errors.New("invalid wf name") } workflowId, err := strconv.Atoi(workflowName[:strings.Index(workflowName, "-")]) if err != nil { impl.Logger.Errorw("invalid wf status update req", "workflowName", workflowName, "err", err) - return 0, "", false, err + return 0, "", false, "", err } savedWorkflow, err := impl.cdWorkflowRepository.FindPreOrPostCdWorkflowRunnerById(workflowId) if err != nil { impl.Logger.Error("cannot get saved wf", "workflowId", workflowId, "err", err) - return 0, "", false, err + return 0, "", false, "", err } cdArtifactLocationFormat := impl.config.GetArtifactLocationFormat() @@ -173,22 +173,22 @@ func (impl *CdHandlerImpl) UpdateWorkflow(workflowStatus eventProcessorBean.CiCd err = impl.cdWorkflowRunnerService.UpdateCdWorkflowRunnerWithStage(savedWorkflow) if err != nil { impl.Logger.Errorw("update wf failed for id", "wfId", savedWorkflow.Id, "err", err) - return savedWorkflow.Id, "", true, err + return savedWorkflow.Id, "", true, "", err } appId := savedWorkflow.CdWorkflow.Pipeline.AppId envId := savedWorkflow.CdWorkflow.Pipeline.EnvironmentId envDeploymentConfig, err := impl.deploymentConfigService.GetConfigForDevtronApps(appId, envId) if err != nil { impl.Logger.Errorw("error in fetching environment deployment config by appId and envId", "appId", appId, "envId", envId, "err", err) - return savedWorkflow.Id, savedWorkflow.Status, true, err + return savedWorkflow.Id, savedWorkflow.Status, true, "", err } globalUtil.TriggerCDMetrics(pipelineAdapter.GetTriggerMetricsFromRunnerObj(savedWorkflow, envDeploymentConfig), impl.config.ExposeCDMetrics) if string(v1alpha1.NodeError) == savedWorkflow.Status || string(v1alpha1.NodeFailed) == savedWorkflow.Status { impl.Logger.Warnw("cd stage failed for workflow", "wfId", savedWorkflow.Id) } - return savedWorkflow.Id, savedWorkflow.Status, true, nil + return savedWorkflow.Id, savedWorkflow.Status, true, message, nil } - return savedWorkflow.Id, status, false, nil + return savedWorkflow.Id, status, false, message, nil } func (impl *CdHandlerImpl) extractWorkflowStatus(workflowStatus eventProcessorBean.CiCdStatus) *types.WorkflowStatus { From e7da702670a2870bd199be51f80747246db5b7b6 Mon Sep 17 00:00:00 2001 From: Vikram Singh Date: Fri, 25 Apr 2025 13:23:44 +0530 Subject: [PATCH 34/34] vendor update --- go.mod | 4 ++-- go.sum | 8 ++++---- vendor/modules.txt | 8 ++++---- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/go.mod b/go.mod index 0e23cf5e03..8948173d8e 100644 --- a/go.mod +++ b/go.mod @@ -307,8 +307,8 @@ require ( replace ( github.com/argoproj/argo-workflows/v3 v3.5.13 => github.com/devtron-labs/argo-workflows/v3 v3.5.13 - github.com/devtron-labs/authenticator => github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250422075757-fe1e5dd1b92b - github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250422075757-fe1e5dd1b92b + github.com/devtron-labs/authenticator => github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250425074923-08ed2e1491ae + github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250425074923-08ed2e1491ae github.com/go-check/check => github.com/go-check/check v0.0.0-20180628173108-788fd7840127 github.com/googleapis/gnostic => github.com/googleapis/gnostic v0.5.5 k8s.io/api => k8s.io/api v0.29.7 diff --git a/go.sum b/go.sum index 2824784a2a..48c61b8e1b 100644 --- a/go.sum +++ b/go.sum @@ -829,10 +829,10 @@ github.com/denisenkom/go-mssqldb v0.0.0-20200428022330-06a60b6afbbc h1:VRRKCwnzq github.com/denisenkom/go-mssqldb v0.0.0-20200428022330-06a60b6afbbc/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= github.com/devtron-labs/argo-workflows/v3 v3.5.13 h1:3pINq0gXOSeTw2z/vYe+j80lRpSN5Rp/8mfQORh8SmU= github.com/devtron-labs/argo-workflows/v3 v3.5.13/go.mod h1:/vqxcovDPT4zqr4DjR5v7CF8ggpY1l3TSa2CIG3jmjA= -github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250422075757-fe1e5dd1b92b h1:b6AkVK1t2NNJcCusTF3w5UN9OwFMS6XEn7MUsSkraTY= -github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250422075757-fe1e5dd1b92b/go.mod h1:FfaLDXN1ZXxyRpnskBqVIYkpkWDCzBmDgIO9xqLnxdQ= -github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250422075757-fe1e5dd1b92b h1:PyI1pxuq7KiSbM2ObApJBcMSRkjXkMfNBZGzy9EJGQI= -github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250422075757-fe1e5dd1b92b/go.mod h1:zkNShlkcHxsmnL0gKNbs0uyRL8lZonGKr5Km63uTLI0= +github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250425074923-08ed2e1491ae h1:K8qIyCD3K3I6nL7h4+jFIDzomMh6Ggro74FZ3RExtl0= +github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250425074923-08ed2e1491ae/go.mod h1:FfaLDXN1ZXxyRpnskBqVIYkpkWDCzBmDgIO9xqLnxdQ= +github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250425074923-08ed2e1491ae h1:BC6tr0ocl072m7dx+ZzuIWC/GdKRb13vAuE4EPC/vzk= +github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250425074923-08ed2e1491ae/go.mod h1:zkNShlkcHxsmnL0gKNbs0uyRL8lZonGKr5Km63uTLI0= github.com/devtron-labs/go-bitbucket v0.9.60-beta h1:VEx1jvDgdtDPS6A1uUFoaEi0l1/oLhbr+90xOwr6sDU= github.com/devtron-labs/go-bitbucket v0.9.60-beta/go.mod h1:GnuiCesvh8xyHeMCb+twm8lBR/kQzJYSKL28ZfObp1Y= github.com/devtron-labs/protos v0.0.3-0.20250323220609-ecf8a0f7305e h1:U6UdYbW8a7xn5IzFPd8cywjVVPfutGJCudjePAfL/Hs= diff --git a/vendor/modules.txt b/vendor/modules.txt index 436ab6735d..9dcbbdbec1 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -472,7 +472,7 @@ github.com/davecgh/go-spew/spew # github.com/deckarep/golang-set v1.8.0 ## explicit; go 1.17 github.com/deckarep/golang-set -# github.com/devtron-labs/authenticator v0.4.35-0.20240809073103-6e11da8083f8 => github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250422075757-fe1e5dd1b92b +# github.com/devtron-labs/authenticator v0.4.35-0.20240809073103-6e11da8083f8 => github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250425074923-08ed2e1491ae ## explicit; go 1.21 github.com/devtron-labs/authenticator/apiToken github.com/devtron-labs/authenticator/client @@ -480,7 +480,7 @@ github.com/devtron-labs/authenticator/jwt github.com/devtron-labs/authenticator/middleware github.com/devtron-labs/authenticator/oidc github.com/devtron-labs/authenticator/password -# github.com/devtron-labs/common-lib v0.18.1-0.20241001061923-eda545dc839e => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250422075757-fe1e5dd1b92b +# github.com/devtron-labs/common-lib v0.18.1-0.20241001061923-eda545dc839e => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250425074923-08ed2e1491ae ## explicit; go 1.21 github.com/devtron-labs/common-lib/async github.com/devtron-labs/common-lib/blob-storage @@ -2352,8 +2352,8 @@ xorm.io/xorm/log xorm.io/xorm/names xorm.io/xorm/schemas xorm.io/xorm/tags -# github.com/devtron-labs/authenticator => github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250422075757-fe1e5dd1b92b -# github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250422075757-fe1e5dd1b92b +# github.com/devtron-labs/authenticator => github.com/devtron-labs/devtron-services/authenticator v0.0.0-20250425074923-08ed2e1491ae +# github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20250425074923-08ed2e1491ae # github.com/go-check/check => github.com/go-check/check v0.0.0-20180628173108-788fd7840127 # github.com/googleapis/gnostic => github.com/googleapis/gnostic v0.5.5 # k8s.io/api => k8s.io/api v0.29.7