diff --git a/api/restHandler/app/workflow/AppWorkflowRestHandler.go b/api/restHandler/app/workflow/AppWorkflowRestHandler.go index e72c34e39c..3e38234153 100644 --- a/api/restHandler/app/workflow/AppWorkflowRestHandler.go +++ b/api/restHandler/app/workflow/AppWorkflowRestHandler.go @@ -280,9 +280,9 @@ func (impl AppWorkflowRestHandlerImpl) FindAppWorkflow(w http.ResponseWriter, r } else { workflows[bean3.Workflows] = []appWorkflow.AppWorkflowDto{} } - isAppLevelGitOpsConfigured, err := impl.chartService.IsGitOpsRepoConfiguredForDevtronApps(appId) + isAppLevelGitOpsConfigured, err := impl.chartService.IsGitOpsRepoConfiguredForDevtronApp(appId) if err != nil && !util.IsErrNoRows(err) { - impl.Logger.Errorw("service err, IsGitOpsRepoConfiguredForDevtronApps", "appId", appId, "envIds", envIds, "err", err) + impl.Logger.Errorw("service err, IsGitOpsRepoConfiguredForDevtronApp", "appId", appId, "envIds", envIds, "err", err) common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) return } diff --git a/cmd/external-app/wire_gen.go b/cmd/external-app/wire_gen.go index 7cada24e76..917252738d 100644 --- a/cmd/external-app/wire_gen.go +++ b/cmd/external-app/wire_gen.go @@ -1,6 +1,6 @@ // Code generated by Wire. DO NOT EDIT. -//go:generate go run github.com/google/wire/cmd/wire +//go:generate go run -mod=mod github.com/google/wire/cmd/wire //go:build !wireinject // +build !wireinject diff --git a/internal/sql/repository/deploymentConfig/repository.go b/internal/sql/repository/deploymentConfig/repository.go index 28508c47fa..ecaf1f59bc 100644 --- a/internal/sql/repository/deploymentConfig/repository.go +++ b/internal/sql/repository/deploymentConfig/repository.go @@ -134,17 +134,15 @@ func (impl *RepositoryImpl) GetAppLevelConfigByAppIds(appIds []int) ([]*Deployme func (impl *RepositoryImpl) GetAppAndEnvLevelConfigsInBulk(appIdToEnvIdsMap map[int][]int) ([]*DeploymentConfig, error) { var result []*DeploymentConfig err := impl.dbConnection.Model(&result). - WhereGroup(func(query *orm.Query) (*orm.Query, error) { + WhereOrGroup(func(query *orm.Query) (*orm.Query, error) { for appId, envIds := range appIdToEnvIdsMap { if len(envIds) == 0 { continue } - query = query.WhereOr(" app_id = ? and environment_id in (?) and active=true ", appId, pg.In(envIds)) + query = query.Where("app_id = ?", appId).Where("environment_id in (?)", pg.In((envIds))).Where("active = ?", true) } return query, nil - }). - Where("active = ?", true). - Select() + }).Select() return result, err } diff --git a/internal/sql/repository/pipelineConfig/PipelineRepository.go b/internal/sql/repository/pipelineConfig/PipelineRepository.go index a442aba552..7ca6c44567 100644 --- a/internal/sql/repository/pipelineConfig/PipelineRepository.go +++ b/internal/sql/repository/pipelineConfig/PipelineRepository.go @@ -100,9 +100,11 @@ type PipelineRepository interface { FindAllPipelineCreatedCountInLast24Hour() (pipelineCount int, err error) FindAllDeletedPipelineCountInLast24Hour() (pipelineCount int, err error) FindActiveByEnvId(envId int) (pipelines []*Pipeline, err error) + FindActivePipelineAppIdsByEnvId(envId int) ([]int, error) FindActivePipelineByEnvId(envId int) (pipelines []*Pipeline, err error) FindActiveByEnvIds(envId []int) (pipelines []*Pipeline, err error) FindActiveByInFilter(envId int, appIdIncludes []int) (pipelines []*Pipeline, err error) + FindActivePipelineAppIdsByInFilter(envId int, appIdIncludes []int) ([]int, error) FindActiveByNotFilter(envId int, appIdExcludes []int) (pipelines []*Pipeline, err error) FindAllPipelinesByChartsOverrideAndAppIdAndChartId(chartOverridden bool, appId int, chartId int) (pipelines []*Pipeline, err error) FindActiveByAppIdAndPipelineId(appId int, pipelineId int) ([]*Pipeline, error) @@ -128,6 +130,7 @@ type PipelineRepository interface { FindWithEnvironmentByCiIds(ctx context.Context, cIPipelineIds []int) ([]*Pipeline, error) FindDeploymentAppTypeByAppIdAndEnvId(appId, envId int) (string, error) FindByAppIdToEnvIdsMapping(appIdToEnvIds map[int][]int) ([]*Pipeline, error) + FindDeploymentAppTypeByIds(ids []int) (pipelines []*Pipeline, err error) } type CiArtifactDTO struct { @@ -483,6 +486,15 @@ func (impl PipelineRepositoryImpl) FindActiveByEnvId(envId int) (pipelines []*Pi return pipelines, err } +func (impl PipelineRepositoryImpl) FindActivePipelineAppIdsByEnvId(envId int) ([]int, error) { + var appIds []int + err := impl.dbConnection.Model((*Pipeline)(nil)).Column("app_id"). + Where("environment_id = ?", envId). + Where("deleted = ?", false). + Select(&appIds) + return appIds, err +} + func (impl PipelineRepositoryImpl) FindActivePipelineByEnvId(envId int) (pipelines []*Pipeline, err error) { err = impl.dbConnection.Model(&pipelines).Column("pipeline.*", "App", "Environment"). Where("environment_id = ?", envId). @@ -509,6 +521,15 @@ func (impl PipelineRepositoryImpl) FindActiveByInFilter(envId int, appIdIncludes return pipelines, err } +func (impl PipelineRepositoryImpl) FindActivePipelineAppIdsByInFilter(envId int, appIdIncludes []int) ([]int, error) { + var appIds []int + err := impl.dbConnection.Model((*Pipeline)(nil)).Column("app_id"). + Where("environment_id = ?", envId). + Where("app_id in (?)", pg.In(appIdIncludes)). + Where("deleted = ?", false).Select(&appIds) + return appIds, err +} + func (impl PipelineRepositoryImpl) FindActiveByNotFilter(envId int, appIdExcludes []int) (pipelines []*Pipeline, err error) { err = impl.dbConnection.Model(&pipelines).Column("pipeline.*", "App", "Environment"). Where("environment_id = ?", envId). @@ -812,3 +833,9 @@ func (impl PipelineRepositoryImpl) FindByAppIdToEnvIdsMapping(appIdToEnvIds map[ Select() return pipelines, err } + +func (impl PipelineRepositoryImpl) FindDeploymentAppTypeByIds(ids []int) (pipelines []*Pipeline, err error) { + err = impl.dbConnection.Model(&pipelines).Column("id", "app_id", "env_id", "deployment_app_type"). + Where("id in (?)", pg.In(ids)).Where("deleted = ?", false).Select() + return pipelines, err +} diff --git a/pkg/appWorkflow/AppWorkflowService.go b/pkg/appWorkflow/AppWorkflowService.go index 6c6bbb7160..7905f729d3 100644 --- a/pkg/appWorkflow/AppWorkflowService.go +++ b/pkg/appWorkflow/AppWorkflowService.go @@ -851,7 +851,7 @@ func (impl AppWorkflowServiceImpl) FindCdPipelinesByAppId(appId int) (*bean.CdPi AppId: appId, } - isAppLevelGitOpsConfigured, err := impl.chartService.IsGitOpsRepoConfiguredForDevtronApps(appId) + isAppLevelGitOpsConfigured, err := impl.chartService.IsGitOpsRepoConfiguredForDevtronApp(appId) if err != nil { impl.Logger.Errorw("error in fetching latest chart details for app by appId") return nil, err diff --git a/pkg/chart/ChartService.go b/pkg/chart/ChartService.go index 2061977d99..edce7ca958 100644 --- a/pkg/chart/ChartService.go +++ b/pkg/chart/ChartService.go @@ -67,7 +67,8 @@ type ChartService interface { ConfigureGitOpsRepoUrlForApp(appId int, repoUrl, chartLocation string, isCustomRepo bool, userId int32) (*bean2.DeploymentConfig, error) - IsGitOpsRepoConfiguredForDevtronApps(appId int) (bool, error) + IsGitOpsRepoConfiguredForDevtronApp(appId int) (bool, error) + IsGitOpsRepoConfiguredForDevtronApps(appIds []int) (map[int]bool, error) IsGitOpsRepoAlreadyRegistered(gitOpsRepoUrl string) (bool, error) } @@ -542,7 +543,7 @@ func (impl *ChartServiceImpl) getNewVersion(chartRepo, chartName, refChartLocati return placeholders[0] + "." + placeholders[1] + "." + strconv.FormatInt(count, 10), nil } -func (impl *ChartServiceImpl) IsGitOpsRepoConfiguredForDevtronApps(appId int) (bool, error) { +func (impl *ChartServiceImpl) IsGitOpsRepoConfiguredForDevtronApp(appId int) (bool, error) { gitOpsConfigStatus, err := impl.gitOpsConfigReadService.IsGitOpsConfigured() if err != nil { impl.logger.Errorw("error in fetching latest chart for app by appId") @@ -560,6 +561,30 @@ func (impl *ChartServiceImpl) IsGitOpsRepoConfiguredForDevtronApps(appId int) (b return !apiGitOpsBean.IsGitOpsRepoNotConfigured(latestChartConfiguredInApp.GitRepoUrl), nil } +func (impl *ChartServiceImpl) IsGitOpsRepoConfiguredForDevtronApps(appIds []int) (map[int]bool, error) { + gitOpsConfigStatus, err := impl.gitOpsConfigReadService.IsGitOpsConfigured() + if err != nil { + impl.logger.Errorw("error in fetching latest chart for app by appId") + return nil, err + } + appIdRepoConfiguredMap := make(map[int]bool, len(appIds)) + for _, appId := range appIds { + if !gitOpsConfigStatus.IsGitOpsConfigured { + appIdRepoConfiguredMap[appId] = false + } else if !gitOpsConfigStatus.AllowCustomRepository { + appIdRepoConfiguredMap[appId] = true + } else { + latestChartConfiguredInApp, err := impl.FindLatestChartForAppByAppId(appId) + if err != nil { + impl.logger.Errorw("error in fetching latest chart for app by appId") + return nil, err + } + appIdRepoConfiguredMap[appId] = !apiGitOpsBean.IsGitOpsRepoNotConfigured(latestChartConfiguredInApp.GitRepoUrl) + } + } + return appIdRepoConfiguredMap, nil +} + func (impl *ChartServiceImpl) FindLatestChartForAppByAppId(appId int) (chartTemplate *TemplateRequest, err error) { chart, err := impl.chartRepository.FindLatestChartForAppByAppId(appId) if err != nil { diff --git a/pkg/deployment/common/bean/bean.go b/pkg/deployment/common/bean/bean.go index 4e29a85533..8817d3cb45 100644 --- a/pkg/deployment/common/bean/bean.go +++ b/pkg/deployment/common/bean/bean.go @@ -22,6 +22,7 @@ type UniqueDeploymentConfigIdentifier string type DeploymentConfigSelector struct { AppId int EnvironmentId int + CDPipelineId int } func (u UniqueDeploymentConfigIdentifier) String() string { diff --git a/pkg/deployment/common/deploymentConfigService.go b/pkg/deployment/common/deploymentConfigService.go index 928e200937..003ad389ea 100644 --- a/pkg/deployment/common/deploymentConfigService.go +++ b/pkg/deployment/common/deploymentConfigService.go @@ -1,6 +1,7 @@ package common import ( + "fmt" "github.com/devtron-labs/devtron/api/bean/gitOps" appRepository "github.com/devtron-labs/devtron/internal/sql/repository/app" "github.com/devtron-labs/devtron/internal/sql/repository/deploymentConfig" @@ -24,6 +25,7 @@ type DeploymentConfigService interface { GetAndMigrateConfigIfAbsentForHelmApp(appId, envId int) (*bean.DeploymentConfig, error) GetAppLevelConfigForDevtronApp(appId int) (*bean.DeploymentConfig, error) UpdateRepoUrlForAppAndEnvId(repoURL string, appId, envId int) error + GetDeploymentAppTypeForCDInBulk(pipelines []*pipelineConfig.Pipeline) (map[int]string, error) } type DeploymentConfigServiceImpl struct { @@ -430,3 +432,30 @@ func (impl *DeploymentConfigServiceImpl) UpdateRepoUrlForAppAndEnvId(repoURL str } return nil } + +func (impl *DeploymentConfigServiceImpl) GetDeploymentAppTypeForCDInBulk(pipelines []*pipelineConfig.Pipeline) (map[int]string, error) { + resp := make(map[int]string, len(pipelines)) //map of pipelineId and deploymentAppType + if impl.deploymentServiceTypeConfig.UseDeploymentConfigData { + appIdEnvIdMapping := make(map[int][]int, len(pipelines)) + appIdEnvIdKeyPipelineIdMap := make(map[string]int, len(pipelines)) + for _, pipeline := range pipelines { + appIdEnvIdMapping[pipeline.AppId] = append(appIdEnvIdMapping[pipeline.AppId], pipeline.EnvironmentId) + appIdEnvIdKeyPipelineIdMap[fmt.Sprintf("%d-%d", pipeline.AppId, pipeline.EnvironmentId)] = pipeline.Id + } + configs, err := impl.deploymentConfigRepository.GetAppAndEnvLevelConfigsInBulk(appIdEnvIdMapping) + if err != nil { + impl.logger.Errorw("error, GetAppAndEnvLevelConfigsInBulk", "appIdEnvIdMapping", appIdEnvIdMapping, "err", err) + return nil, err + } + for _, config := range configs { + pipelineId := appIdEnvIdKeyPipelineIdMap[fmt.Sprintf("%d-%d", config.AppId, config.EnvironmentId)] + resp[pipelineId] = config.DeploymentAppType + } + } + for _, pipeline := range pipelines { + if _, ok := resp[pipeline.Id]; !ok { //not found in map, either flag is disabled or config not migrated yet. Getting from old data + resp[pipeline.Id] = pipeline.DeploymentAppType + } + } + return resp, nil +} diff --git a/pkg/pipeline/AppDeploymentTypeChangeManager.go b/pkg/pipeline/AppDeploymentTypeChangeManager.go index c29c7572ba..d78c59e60a 100644 --- a/pkg/pipeline/AppDeploymentTypeChangeManager.go +++ b/pkg/pipeline/AppDeploymentTypeChangeManager.go @@ -418,14 +418,6 @@ func (impl *AppDeploymentTypeChangeManagerImpl) TriggerDeploymentAfterTypeChange "err", err) } - deploymentConfigSelector := make([]*bean4.DeploymentConfigSelector, len(pipelineIds)) - for _, pipeline := range pipelines { - deploymentConfigSelector = append(deploymentConfigSelector, &bean4.DeploymentConfigSelector{ - AppId: pipeline.AppId, - EnvironmentId: pipeline.EnvironmentId, - }) - } - for _, p := range pipelines { envDeploymentConfig, err := impl.deploymentConfigService.GetAndMigrateConfigIfAbsentForDevtronApps(p.AppId, p.EnvironmentId) if err != nil { diff --git a/pkg/pipeline/BuildPipelineConfigService.go b/pkg/pipeline/BuildPipelineConfigService.go index c9446d9b7c..dd4e570d53 100644 --- a/pkg/pipeline/BuildPipelineConfigService.go +++ b/pkg/pipeline/BuildPipelineConfigService.go @@ -33,7 +33,9 @@ import ( pipelineConfigBean "github.com/devtron-labs/devtron/pkg/pipeline/bean" "github.com/devtron-labs/devtron/pkg/pipeline/bean/CiPipeline" "github.com/devtron-labs/devtron/pkg/pipeline/history" + "github.com/devtron-labs/devtron/pkg/pipeline/repository" "github.com/devtron-labs/devtron/pkg/pipeline/types" + repository2 "github.com/devtron-labs/devtron/pkg/plugin/repository" resourceGroup2 "github.com/devtron-labs/devtron/pkg/resourceGroup" "github.com/devtron-labs/devtron/pkg/sql" "github.com/devtron-labs/devtron/util/rbac" @@ -133,6 +135,8 @@ type CiPipelineConfigServiceImpl struct { customTagService CustomTagService cdWorkflowRepository pipelineConfig.CdWorkflowRepository buildPipelineSwitchService BuildPipelineSwitchService + pipelineStageRepository repository.PipelineStageRepository + globalPluginRepository repository2.GlobalPluginRepository } func NewCiPipelineConfigServiceImpl(logger *zap.SugaredLogger, @@ -157,8 +161,8 @@ func NewCiPipelineConfigServiceImpl(logger *zap.SugaredLogger, customTagService CustomTagService, cdWorkflowRepository pipelineConfig.CdWorkflowRepository, buildPipelineSwitchService BuildPipelineSwitchService, -) *CiPipelineConfigServiceImpl { - + pipelineStageRepository repository.PipelineStageRepository, + globalPluginRepository repository2.GlobalPluginRepository) *CiPipelineConfigServiceImpl { securityConfig := &SecurityConfig{} err := env.Parse(securityConfig) if err != nil { @@ -188,6 +192,8 @@ func NewCiPipelineConfigServiceImpl(logger *zap.SugaredLogger, customTagService: customTagService, cdWorkflowRepository: cdWorkflowRepository, buildPipelineSwitchService: buildPipelineSwitchService, + pipelineStageRepository: pipelineStageRepository, + globalPluginRepository: globalPluginRepository, } } diff --git a/pkg/pipeline/CiCdPipelineOrchestrator.go b/pkg/pipeline/CiCdPipelineOrchestrator.go index 17f3ce9a69..d5834cbf30 100644 --- a/pkg/pipeline/CiCdPipelineOrchestrator.go +++ b/pkg/pipeline/CiCdPipelineOrchestrator.go @@ -1824,16 +1824,18 @@ func (impl CiCdPipelineOrchestratorImpl) DeleteCdPipeline(pipelineId int, userId } func (impl CiCdPipelineOrchestratorImpl) getPipelineIdAndPrePostStageMapping(dbPipelines []*pipelineConfig.Pipeline) (map[int][]*pipelineConfigBean.PipelineStageDto, error) { var err error - pipelineIdAndPrePostStageMapping := make(map[int][]*pipelineConfigBean.PipelineStageDto) - var dbPipelineIds []int + pipelineIdAndPrePostStageMapping := make(map[int][]*pipelineConfigBean.PipelineStageDto, len(dbPipelines)) + dbPipelineIds := make([]int, 0, len(dbPipelines)) + pipelineMap := make(map[int]*pipelineConfig.Pipeline, len(dbPipelines)) for _, pipeline := range dbPipelines { + pipelineMap[pipeline.Id] = pipeline dbPipelineIds = append(dbPipelineIds, pipeline.Id) } if len(dbPipelineIds) > 0 { - pipelineIdAndPrePostStageMapping, err = impl.pipelineStageService.GetCdPipelineStageDataDeepCopyForPipelineIds(dbPipelineIds) + pipelineIdAndPrePostStageMapping, err = impl.pipelineStageService.GetCdPipelineStageDataDeepCopyForPipelineIds(dbPipelineIds, pipelineMap) if err != nil { impl.logger.Errorw("error in fetching pipelinePrePostStageMapping", "err", err, "cdPipelineIds", dbPipelineIds) - return pipelineIdAndPrePostStageMapping, err + return nil, err } } return pipelineIdAndPrePostStageMapping, nil @@ -1856,7 +1858,7 @@ func (impl CiCdPipelineOrchestratorImpl) GetCdPipelinesForApp(appId int) (cdPipe isAppLevelGitOpsConfigured := false if len(dbPipelines) != 0 { - isAppLevelGitOpsConfigured, err = impl.chartService.IsGitOpsRepoConfiguredForDevtronApps(appId) + isAppLevelGitOpsConfigured, err = impl.chartService.IsGitOpsRepoConfiguredForDevtronApp(appId) if err != nil { impl.logger.Errorw("error in fetching latest chart for app by appId") return nil, err @@ -1943,22 +1945,19 @@ func (impl CiCdPipelineOrchestratorImpl) GetCdPipelinesForApp(appId int) (cdPipe func (impl CiCdPipelineOrchestratorImpl) GetCdPipelinesForEnv(envId int, requestedAppIds []int) (cdPipelines *bean.CdPipelines, err error) { var dbPipelines []*pipelineConfig.Pipeline + appIds := make([]int, 0, len(requestedAppIds)*envId) if len(requestedAppIds) > 0 { - dbPipelines, err = impl.pipelineRepository.FindActiveByInFilter(envId, requestedAppIds) + appIds, err = impl.pipelineRepository.FindActivePipelineAppIdsByInFilter(envId, requestedAppIds) } else { - dbPipelines, err = impl.pipelineRepository.FindActiveByEnvId(envId) + appIds, err = impl.pipelineRepository.FindActivePipelineAppIdsByEnvId(envId) } - if err != nil { - impl.logger.Errorw("error in fetching pipelines", "envId", envId, "err", err) + if err != nil && !util.IsErrNoRows(err) { + impl.logger.Errorw("error in fetching pipeline's appIds", "envId", envId, "requestedAppIds", requestedAppIds, "err", err) return nil, err } - var appIds []int - for _, pipeline := range dbPipelines { - appIds = append(appIds, pipeline.AppId) - } if len(appIds) == 0 { - err = &util.ApiError{Code: "404", HttpStatusCode: 200, UserMessage: "no cd pipeline found"} + err = &util.ApiError{Code: "404", HttpStatusCode: http.StatusNotFound, UserMessage: "no cd pipeline found"} return cdPipelines, err } @@ -1973,36 +1972,18 @@ func (impl CiCdPipelineOrchestratorImpl) GetCdPipelinesForEnv(envId int, request impl.logger.Errorw("error in fetching pipelineIdAndPrePostStageMapping", "err", err) return nil, err } - var pipelines []*bean.CDPipelineConfigObject - isAppLevelGitOpsConfigured := false - gitOpsConfigStatus, err := impl.gitOpsConfigReadService.IsGitOpsConfigured() + appIdAppLevelGitOpsConfiguredMap, err := impl.chartService.IsGitOpsRepoConfiguredForDevtronApps(appIds) if err != nil { - impl.logger.Errorw("error in fetching global GitOps configuration") + impl.logger.Errorw("error in fetching latest chart details for app by appId") return nil, err } - if gitOpsConfigStatus.IsGitOpsConfigured && !gitOpsConfigStatus.AllowCustomRepository { - isAppLevelGitOpsConfigured = true + pipelines := make([]*bean.CDPipelineConfigObject, 0, len(dbPipelines)) + pipelineIdDeploymentTypeMap, err := impl.deploymentConfigService.GetDeploymentAppTypeForCDInBulk(dbPipelines) + if err != nil { + impl.logger.Errorw("error, GetDeploymentAppTypeForCDInBulk", "pipelines", dbPipelines, "err", err) + return nil, err } - isDeploymentConfigUsed := impl.deploymentConfigService.IsDeploymentConfigUsed() for _, dbPipeline := range dbPipelines { - if !isAppLevelGitOpsConfigured { - // TODO: optimize this to get for the all appIds in one go - isAppLevelGitOpsConfigured, err = impl.chartService.IsGitOpsRepoConfiguredForDevtronApps(dbPipeline.AppId) - if err != nil { - impl.logger.Errorw("error in fetching latest chart details for app by appId") - return nil, err - } - } - deploymentAppType := dbPipeline.DeploymentAppType - if isDeploymentConfigUsed { - // TODO: optimize this to get for the all pipelineIds in one go - envDeploymentConfig, err := impl.deploymentConfigService.GetConfigForDevtronApps(dbPipeline.AppId, dbPipeline.EnvironmentId) - if err != nil { - impl.logger.Errorw("error in fetching environment deployment config by appId and envId", "appId", dbPipeline.AppId, "envId", dbPipeline.EnvironmentId, "err", err) - return nil, err - } - deploymentAppType = envDeploymentConfig.DeploymentAppType - } pipeline := &bean.CDPipelineConfigObject{ Id: dbPipeline.Id, Name: dbPipeline.Name, @@ -2012,13 +1993,13 @@ func (impl CiCdPipelineOrchestratorImpl) GetCdPipelinesForEnv(envId int, request TriggerType: dbPipeline.TriggerType, RunPreStageInEnv: dbPipeline.RunPreStageInEnv, RunPostStageInEnv: dbPipeline.RunPostStageInEnv, - DeploymentAppType: deploymentAppType, + DeploymentAppType: pipelineIdDeploymentTypeMap[dbPipeline.Id], AppName: dbPipeline.App.AppName, AppId: dbPipeline.AppId, TeamId: dbPipeline.App.TeamId, EnvironmentIdentifier: dbPipeline.Environment.EnvironmentIdentifier, IsVirtualEnvironment: dbPipeline.Environment.IsVirtualEnvironment, - IsGitOpsRepoNotConfigured: !isAppLevelGitOpsConfigured, + IsGitOpsRepoNotConfigured: !appIdAppLevelGitOpsConfiguredMap[dbPipeline.AppId], } if len(dbPipeline.PreStageConfig) > 0 { preStage := bean.CdStage{} @@ -2118,7 +2099,7 @@ func (impl CiCdPipelineOrchestratorImpl) GetCdPipelinesForAppAndEnv(appId int, e impl.logger.Error(err) return nil, err } - isAppLevelGitOpsConfigured, err := impl.chartService.IsGitOpsRepoConfiguredForDevtronApps(appId) + isAppLevelGitOpsConfigured, err := impl.chartService.IsGitOpsRepoConfiguredForDevtronApp(appId) if err != nil { impl.logger.Errorw("error in fetching latest chart details for app by appId") return nil, err diff --git a/pkg/pipeline/DeploymentPipelineConfigService.go b/pkg/pipeline/DeploymentPipelineConfigService.go index 423a76e775..c622597c15 100644 --- a/pkg/pipeline/DeploymentPipelineConfigService.go +++ b/pkg/pipeline/DeploymentPipelineConfigService.go @@ -362,7 +362,7 @@ func (impl *CdPipelineConfigServiceImpl) GetCdPipelineById(pipelineId int) (cdPi } var preDeployStage *pipelineConfigBean.PipelineStageDto var postDeployStage *pipelineConfigBean.PipelineStageDto - preDeployStage, postDeployStage, err = impl.pipelineStageService.GetCdPipelineStageDataDeepCopy(dbPipeline.Id) + preDeployStage, postDeployStage, err = impl.pipelineStageService.GetCdPipelineStageDataDeepCopy(dbPipeline) if err != nil { impl.logger.Errorw("error in getting pre/post-CD stage data", "err", err, "cdPipelineId", dbPipeline.Id) return nil, err @@ -1276,7 +1276,7 @@ func (impl *CdPipelineConfigServiceImpl) GetCdPipelinesByEnvironment(request res customTagStage = repository5.PIPELINE_STAGE_TYPE_POST_CD } if !isAppLevelGitOpsConfigured { - isAppLevelGitOpsConfigured, err = impl.chartService.IsGitOpsRepoConfiguredForDevtronApps(dbPipeline.AppId) + isAppLevelGitOpsConfigured, err = impl.chartService.IsGitOpsRepoConfiguredForDevtronApp(dbPipeline.AppId) if err != nil { impl.logger.Errorw("error in fetching latest chart details for app by appId") return nil, err diff --git a/pkg/pipeline/PipelineStageService.go b/pkg/pipeline/PipelineStageService.go index 0d32cfeac3..d4e7e7cfea 100644 --- a/pkg/pipeline/PipelineStageService.go +++ b/pkg/pipeline/PipelineStageService.go @@ -19,7 +19,9 @@ package pipeline import ( "encoding/json" "errors" + "fmt" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" + "github.com/devtron-labs/devtron/internal/util" "github.com/devtron-labs/devtron/pkg/pipeline/bean" "github.com/devtron-labs/devtron/pkg/pipeline/repository" "github.com/devtron-labs/devtron/pkg/plugin" @@ -30,6 +32,7 @@ import ( repository3 "github.com/devtron-labs/devtron/pkg/variables/repository" "github.com/go-pg/pg" "go.uber.org/zap" + "net/http" "time" ) @@ -40,8 +43,8 @@ type PipelineStageService interface { DeletePipelineStage(stageReq *bean.PipelineStageDto, userId int32, tx *pg.Tx) error BuildPrePostAndRefPluginStepsDataForWfRequest(pipelineId int, stageType string, scope resourceQualifiers.Scope) (*bean.PrePostAndRefPluginStepsResponse, error) GetCiPipelineStageDataDeepCopy(ciPipelineId int) (preCiStage *bean.PipelineStageDto, postCiStage *bean.PipelineStageDto, err error) - GetCdPipelineStageDataDeepCopy(cdPipelineId int) (*bean.PipelineStageDto, *bean.PipelineStageDto, error) - GetCdPipelineStageDataDeepCopyForPipelineIds(cdPipelineIds []int) (map[int][]*bean.PipelineStageDto, error) + GetCdPipelineStageDataDeepCopy(cdPipeline *pipelineConfig.Pipeline) (*bean.PipelineStageDto, *bean.PipelineStageDto, error) + GetCdPipelineStageDataDeepCopyForPipelineIds(cdPipelineIds []int, pipelineMap map[int]*pipelineConfig.Pipeline) (map[int][]*bean.PipelineStageDto, error) GetCdStageByCdPipelineIdAndStageType(cdPipelineId int, stageType repository.PipelineStageType) (*repository.PipelineStage, error) // DeletePipelineStageIfReq function is used to delete corrupted pipelineStage data // , there was a bug(https://github.com/devtron-labs/devtron/issues/3826) where we were not deleting pipeline stage entry even after deleting all the pipelineStageSteps @@ -88,13 +91,13 @@ func (impl *PipelineStageServiceImpl) GetCiPipelineStageDataDeepCopy(ciPipelineI var postCiStage *bean.PipelineStageDto for _, ciStage := range ciStages { if ciStage.Type == repository.PIPELINE_STAGE_TYPE_PRE_CI { - preCiStage, err = impl.BuildPipelineStageDataDeepCopy(ciStage) + preCiStage, err = impl.BuildPipelineStageDataDeepCopy(ciStage, nil) if err != nil { impl.logger.Errorw("error in getting ci stage data", "err", err, "ciStage", ciStage) return nil, nil, err } } else if ciStage.Type == repository.PIPELINE_STAGE_TYPE_POST_CI { - postCiStage, err = impl.BuildPipelineStageDataDeepCopy(ciStage) + postCiStage, err = impl.BuildPipelineStageDataDeepCopy(ciStage, nil) if err != nil { impl.logger.Errorw("error in getting ci stage data", "err", err, "ciStage", ciStage) return nil, nil, err @@ -110,7 +113,7 @@ func (impl *PipelineStageServiceImpl) GetCdStageByCdPipelineIdAndStageType(cdPip return impl.pipelineStageRepository.GetCdStageByCdPipelineIdAndStageType(cdPipelineId, stageType) } -func (impl *PipelineStageServiceImpl) GetCdPipelineStageDataDeepCopy(cdPipelineId int) (*bean.PipelineStageDto, *bean.PipelineStageDto, error) { +func (impl *PipelineStageServiceImpl) GetCdPipelineStageDataDeepCopy(cdPipeline *pipelineConfig.Pipeline) (*bean.PipelineStageDto, *bean.PipelineStageDto, error) { //migrate plugin_metadata to plugin_parent_metadata, also pluginVersionsMetadata will have updated entries after migrating, this is a one time operation err := impl.globalPluginService.MigratePluginData() if err != nil { @@ -119,7 +122,7 @@ func (impl *PipelineStageServiceImpl) GetCdPipelineStageDataDeepCopy(cdPipelineI } //getting all stages by cd pipeline id - cdStages, err := impl.pipelineStageRepository.GetAllCdStagesByCdPipelineId(cdPipelineId) + cdStages, err := impl.pipelineStageRepository.GetAllCdStagesByCdPipelineId(cdPipeline.Id) if err != nil { impl.logger.Errorw("error in getting all cdStages by cdPipelineId", "err", err, "cdPipelineStages", cdStages) return nil, nil, err @@ -132,19 +135,19 @@ func (impl *PipelineStageServiceImpl) GetCdPipelineStageDataDeepCopy(cdPipelineI var postDeployStage *bean.PipelineStageDto for _, cdStage := range cdStages { if cdStage.Type == repository.PIPELINE_STAGE_TYPE_PRE_CD { - preDeployStage, err = impl.BuildPipelineStageDataDeepCopy(cdStage) + preDeployStage, err = impl.BuildPipelineStageDataDeepCopy(cdStage, cdPipeline) if err != nil { impl.logger.Errorw("error in getting cd stage data", "err", err, "cdStage", cdStage) return nil, nil, err } } else if cdStage.Type == repository.PIPELINE_STAGE_TYPE_POST_CD { - postDeployStage, err = impl.BuildPipelineStageDataDeepCopy(cdStage) + postDeployStage, err = impl.BuildPipelineStageDataDeepCopy(cdStage, cdPipeline) if err != nil { impl.logger.Errorw("error in getting cd stage data", "err", err, "cdStage", cdStage) return nil, nil, err } } else { - impl.logger.Errorw("found improper stage mapped with cdPipeline", "cdPipelineId", cdPipelineId, "stage", cdStage) + impl.logger.Errorw("found improper stage mapped with cdPipeline", "cdPipelineId", cdPipeline.Id, "stage", cdStage) } } if preDeployStage != nil { @@ -156,51 +159,51 @@ func (impl *PipelineStageServiceImpl) GetCdPipelineStageDataDeepCopy(cdPipelineI return preDeployStage, postDeployStage, nil } -func (impl *PipelineStageServiceImpl) GetCdPipelineStageDataDeepCopyForPipelineIds(cdPipelineIds []int) (map[int][]*bean.PipelineStageDto, error) { - pipelinePrePostStageMappingResp := make(map[int][]*bean.PipelineStageDto) - pipelineStageMapping := make(map[int][]*repository.PipelineStage) +func (impl *PipelineStageServiceImpl) GetCdPipelineStageDataDeepCopyForPipelineIds(cdPipelineIds []int, pipelineMap map[int]*pipelineConfig.Pipeline) (map[int][]*bean.PipelineStageDto, error) { + pipelinePrePostStageMappingResp := make(map[int][]*bean.PipelineStageDto, len(cdPipelineIds)) pipelineStages, err := impl.pipelineStageRepository.GetAllCdStagesByCdPipelineIds(cdPipelineIds) if err != nil { impl.logger.Errorw("error in getting pipelineStages from cdPipelineIds", "err", err, "cdPipelineIds", cdPipelineIds) - return pipelinePrePostStageMappingResp, err + return nil, err } if len(pipelineStages) == 0 { return pipelinePrePostStageMappingResp, nil } for _, pipelineStage := range pipelineStages { - pipelineStageMapping[pipelineStage.CdPipelineId] = append(pipelineStageMapping[pipelineStage.CdPipelineId], pipelineStage) - } - - for pipelineId, pipelineStages := range pipelineStageMapping { - var preDeployStage *bean.PipelineStageDto - var postDeployStage *bean.PipelineStageDto - for _, pipelineStage := range pipelineStages { - if pipelineStage.Type == repository.PIPELINE_STAGE_TYPE_PRE_CD { - preDeployStage, err = impl.BuildPipelineStageDataDeepCopy(pipelineStage) - if err != nil { - impl.logger.Errorw("error in getting cd stage data", "err", err, "cdStage", bean.CdStage) - return pipelinePrePostStageMappingResp, err - } - preDeployStage.Name = "Pre-Deployment" - } else if pipelineStage.Type == repository.PIPELINE_STAGE_TYPE_POST_CD { - postDeployStage, err = impl.BuildPipelineStageDataDeepCopy(pipelineStage) - if err != nil { - impl.logger.Errorw("error in getting cd stage data", "err", err, "cdStage", bean.CdStage) - return pipelinePrePostStageMappingResp, err - } - postDeployStage.Name = "Post-Deployment" - } else { - impl.logger.Errorw("found improper stage mapped with cdPipeline", "cdPipelineId", pipelineId, "stage", bean.CdStage) + pipelineId := pipelineStage.CdPipelineId + var preDeployStage, postDeployStage *bean.PipelineStageDto + pipeline := pipelineMap[pipelineId] + if pipeline == nil { + impl.logger.Errorw("error in finding pipeline by id", "pipelineId", pipelineId) + return nil, &util.ApiError{Code: "404", HttpStatusCode: http.StatusNotFound, InternalMessage: fmt.Sprintf("pipeline not found, id : %d", pipelineId), UserMessage: "pipeline not found"} + } + if _, ok := pipelinePrePostStageMappingResp[pipelineId]; !ok { + pipelinePrePostStageMappingResp[pipelineId] = make([]*bean.PipelineStageDto, 2) //for old logic compatibility (assumptions that pre is 0 element and post is 1 element) + } + if pipelineStage.Type == repository.PIPELINE_STAGE_TYPE_PRE_CD { + preDeployStage, err = impl.BuildPipelineStageDataDeepCopy(pipelineStage, pipeline) + if err != nil { + impl.logger.Errorw("error in getting cd stage data", "err", err, "cdStage", bean.CdStage) + return nil, err + } + preDeployStage.Name = "Pre-Deployment" + pipelinePrePostStageMappingResp[pipelineId][0] = preDeployStage + } else if pipelineStage.Type == repository.PIPELINE_STAGE_TYPE_POST_CD { + postDeployStage, err = impl.BuildPipelineStageDataDeepCopy(pipelineStage, pipeline) + if err != nil { + impl.logger.Errorw("error in getting cd stage data", "err", err, "cdStage", bean.CdStage) + return nil, err } + postDeployStage.Name = "Post-Deployment" + pipelinePrePostStageMappingResp[pipelineId][1] = postDeployStage + } else { + impl.logger.Errorw("found improper stage mapped with cdPipeline", "cdPipelineId", pipelineId, "stage", bean.CdStage) } - pipelinePrePostStageMappingResp[pipelineId] = append(pipelinePrePostStageMappingResp[pipelineId], preDeployStage) - pipelinePrePostStageMappingResp[pipelineId] = append(pipelinePrePostStageMappingResp[pipelineId], postDeployStage) - } return pipelinePrePostStageMappingResp, nil } -func (impl *PipelineStageServiceImpl) BuildPipelineStageDataDeepCopy(pipelineStage *repository.PipelineStage) (*bean.PipelineStageDto, error) { +func (impl *PipelineStageServiceImpl) BuildPipelineStageDataDeepCopy(pipelineStage *repository.PipelineStage, pipeline *pipelineConfig.Pipeline) (*bean.PipelineStageDto, error) { stageData := &bean.PipelineStageDto{ Id: pipelineStage.Id, Name: pipelineStage.Name, @@ -208,11 +211,6 @@ func (impl *PipelineStageServiceImpl) BuildPipelineStageDataDeepCopy(pipelineSta Type: pipelineStage.Type, } if pipelineStage.Type == repository.PIPELINE_STAGE_TYPE_PRE_CD || pipelineStage.Type == repository.PIPELINE_STAGE_TYPE_POST_CD { - pipeline, err := impl.pipelineRepository.FindById(pipelineStage.CdPipelineId) - if err != nil { - impl.logger.Errorw("error in getting pipeline from cdPipelineId", "err", err, "cdPipelineId", pipelineStage.CdPipelineId) - return nil, err - } if pipelineStage.Type == repository.PIPELINE_STAGE_TYPE_PRE_CD { stageData.TriggerType = pipeline.PreTriggerType } diff --git a/wire_gen.go b/wire_gen.go index 56b7d5a193..2931b38ebe 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -1,6 +1,6 @@ // Code generated by Wire. DO NOT EDIT. -//go:generate go run github.com/google/wire/cmd/wire +//go:generate go run -mod=mod github.com/google/wire/cmd/wire //go:build !wireinject // +build !wireinject @@ -584,7 +584,7 @@ func InitializeApp() (*App, error) { ciTemplateHistoryRepositoryImpl := repository12.NewCiTemplateHistoryRepositoryImpl(db, sugaredLogger) ciTemplateHistoryServiceImpl := history.NewCiTemplateHistoryServiceImpl(ciTemplateHistoryRepositoryImpl, sugaredLogger) buildPipelineSwitchServiceImpl := pipeline.NewBuildPipelineSwitchServiceImpl(sugaredLogger, ciPipelineRepositoryImpl, ciCdPipelineOrchestratorImpl, pipelineRepositoryImpl, ciWorkflowRepositoryImpl, appWorkflowRepositoryImpl, ciPipelineHistoryServiceImpl, ciTemplateOverrideRepositoryImpl, ciPipelineMaterialRepositoryImpl) - ciPipelineConfigServiceImpl := pipeline.NewCiPipelineConfigServiceImpl(sugaredLogger, ciCdPipelineOrchestratorImpl, dockerArtifactStoreRepositoryImpl, materialRepositoryImpl, appRepositoryImpl, pipelineRepositoryImpl, ciPipelineRepositoryImpl, ecrConfig, appWorkflowRepositoryImpl, ciCdConfig, attributesServiceImpl, pipelineStageServiceImpl, ciPipelineMaterialRepositoryImpl, ciTemplateServiceImpl, ciTemplateOverrideRepositoryImpl, ciTemplateHistoryServiceImpl, enforcerUtilImpl, ciWorkflowRepositoryImpl, resourceGroupServiceImpl, customTagServiceImpl, cdWorkflowRepositoryImpl, buildPipelineSwitchServiceImpl) + ciPipelineConfigServiceImpl := pipeline.NewCiPipelineConfigServiceImpl(sugaredLogger, ciCdPipelineOrchestratorImpl, dockerArtifactStoreRepositoryImpl, materialRepositoryImpl, appRepositoryImpl, pipelineRepositoryImpl, ciPipelineRepositoryImpl, ecrConfig, appWorkflowRepositoryImpl, ciCdConfig, attributesServiceImpl, pipelineStageServiceImpl, ciPipelineMaterialRepositoryImpl, ciTemplateServiceImpl, ciTemplateOverrideRepositoryImpl, ciTemplateHistoryServiceImpl, enforcerUtilImpl, ciWorkflowRepositoryImpl, resourceGroupServiceImpl, customTagServiceImpl, cdWorkflowRepositoryImpl, buildPipelineSwitchServiceImpl, pipelineStageRepositoryImpl, globalPluginRepositoryImpl) ciMaterialConfigServiceImpl := pipeline.NewCiMaterialConfigServiceImpl(sugaredLogger, materialRepositoryImpl, ciTemplateServiceImpl, ciCdPipelineOrchestratorImpl, ciPipelineRepositoryImpl, gitMaterialHistoryServiceImpl, pipelineRepositoryImpl, ciPipelineMaterialRepositoryImpl, transactionUtilImpl) deploymentGroupRepositoryImpl := repository2.NewDeploymentGroupRepositoryImpl(sugaredLogger, db) pipelineStrategyHistoryRepositoryImpl := repository12.NewPipelineStrategyHistoryRepositoryImpl(sugaredLogger, db)