From 8083da354dd9c6c8cc5d37d7ee3b250c9ce0ec0a Mon Sep 17 00:00:00 2001 From: ayushmaheshwari Date: Wed, 26 Jun 2024 13:22:48 +0530 Subject: [PATCH 01/45] inital commit - deployment config service --- Wire.go | 12 +- .../trigger/PipelineTriggerRestHandler.go | 4 +- .../repository/deploymentConfig/repository.go | 73 +++++++++ .../pipelineConfig/PipelineRepository.go | 10 ++ pkg/chart/ChartService.go | 24 +-- pkg/deployment/common/adapter.go | 38 +++++ pkg/deployment/common/bean/bean.go | 29 ++++ .../common/deploymentConfigService.go | 154 ++++++++++++++++++ pkg/deployment/common/helper.go | 10 ++ .../read/deploymentConfigReadService.go | 16 ++ .../gitOps/config/GitOpsConfigReadService.go | 43 +++++ pkg/pipeline/DeploymentConfigService.go | 20 +-- util/HttpUtil.go | 13 ++ wire_gen.go | 10 +- 14 files changed, 429 insertions(+), 27 deletions(-) create mode 100644 internal/sql/repository/deploymentConfig/repository.go create mode 100644 pkg/deployment/common/adapter.go create mode 100644 pkg/deployment/common/bean/bean.go create mode 100644 pkg/deployment/common/deploymentConfigService.go create mode 100644 pkg/deployment/common/helper.go create mode 100644 pkg/deployment/common/read/deploymentConfigReadService.go diff --git a/Wire.go b/Wire.go index 08f44eeab9..97f8acfd7b 100644 --- a/Wire.go +++ b/Wire.go @@ -93,6 +93,7 @@ import ( appWorkflow2 "github.com/devtron-labs/devtron/internal/sql/repository/appWorkflow" "github.com/devtron-labs/devtron/internal/sql/repository/bulkUpdate" "github.com/devtron-labs/devtron/internal/sql/repository/chartConfig" + "github.com/devtron-labs/devtron/internal/sql/repository/deploymentConfig" dockerRegistryRepository "github.com/devtron-labs/devtron/internal/sql/repository/dockerRegistry" "github.com/devtron-labs/devtron/internal/sql/repository/helper" repository8 "github.com/devtron-labs/devtron/internal/sql/repository/imageTagging" @@ -121,6 +122,7 @@ import ( "github.com/devtron-labs/devtron/pkg/commonService" delete2 "github.com/devtron-labs/devtron/pkg/delete" deployment2 "github.com/devtron-labs/devtron/pkg/deployment" + "github.com/devtron-labs/devtron/pkg/deployment/common" git2 "github.com/devtron-labs/devtron/pkg/deployment/gitOps/git" "github.com/devtron-labs/devtron/pkg/deployment/manifest/publish" "github.com/devtron-labs/devtron/pkg/deploymentGroup" @@ -872,8 +874,8 @@ func InitializeApp() (*App, error) { wire.Bind(new(repository.UserAttributesRepository), new(*repository.UserAttributesRepositoryImpl)), pipelineConfig.NewPipelineStatusTimelineRepositoryImpl, wire.Bind(new(pipelineConfig.PipelineStatusTimelineRepository), new(*pipelineConfig.PipelineStatusTimelineRepositoryImpl)), - wire.Bind(new(pipeline.DeploymentConfigService), new(*pipeline.DeploymentConfigServiceImpl)), - pipeline.NewDeploymentConfigServiceImpl, + wire.Bind(new(pipeline.PipelineDeploymentConfigService), new(*pipeline.PipelineDeploymentConfigServiceImpl)), + pipeline.NewPipelineDeploymentConfigServiceImpl, pipelineConfig.NewCiTemplateOverrideRepositoryImpl, wire.Bind(new(pipelineConfig.CiTemplateOverrideRepository), new(*pipelineConfig.CiTemplateOverrideRepositoryImpl)), pipelineConfig.NewCiBuildConfigRepositoryImpl, @@ -970,6 +972,12 @@ func InitializeApp() (*App, error) { cel.NewCELServiceImpl, wire.Bind(new(cel.EvaluatorService), new(*cel.EvaluatorServiceImpl)), + + deploymentConfig.NewRepositoryImpl, + wire.Bind(new(deploymentConfig.Repository), new(*deploymentConfig.RepositoryImpl)), + + common.NewDeploymentConfigServiceImpl, + wire.Bind(new(common.DeploymentConfigService), new(*common.DeploymentConfigServiceImpl)), ) return &App{}, nil } diff --git a/api/restHandler/app/pipeline/trigger/PipelineTriggerRestHandler.go b/api/restHandler/app/pipeline/trigger/PipelineTriggerRestHandler.go index 6b6f7751f1..6415e556a0 100644 --- a/api/restHandler/app/pipeline/trigger/PipelineTriggerRestHandler.go +++ b/api/restHandler/app/pipeline/trigger/PipelineTriggerRestHandler.go @@ -66,7 +66,7 @@ type PipelineTriggerRestHandlerImpl struct { enforcerUtil rbac.EnforcerUtil deploymentGroupService deploymentGroup.DeploymentGroupService argoUserService argo.ArgoUserService - deploymentConfigService pipeline.DeploymentConfigService + deploymentConfigService pipeline.PipelineDeploymentConfigService deployedAppService deployedApp.DeployedAppService cdTriggerService devtronApps.TriggerService workflowEventPublishService out.WorkflowEventPublishService @@ -75,7 +75,7 @@ type PipelineTriggerRestHandlerImpl struct { func NewPipelineRestHandler(appService app.AppService, userAuthService user.UserService, validator *validator.Validate, enforcer casbin.Enforcer, teamService team.TeamService, logger *zap.SugaredLogger, enforcerUtil rbac.EnforcerUtil, deploymentGroupService deploymentGroup.DeploymentGroupService, - argoUserService argo.ArgoUserService, deploymentConfigService pipeline.DeploymentConfigService, + argoUserService argo.ArgoUserService, deploymentConfigService pipeline.PipelineDeploymentConfigService, deployedAppService deployedApp.DeployedAppService, cdTriggerService devtronApps.TriggerService, workflowEventPublishService out.WorkflowEventPublishService) *PipelineTriggerRestHandlerImpl { diff --git a/internal/sql/repository/deploymentConfig/repository.go b/internal/sql/repository/deploymentConfig/repository.go new file mode 100644 index 0000000000..6017683c89 --- /dev/null +++ b/internal/sql/repository/deploymentConfig/repository.go @@ -0,0 +1,73 @@ +package deploymentConfig + +import ( + "github.com/devtron-labs/devtron/pkg/sql" + "github.com/go-pg/pg" +) + +type DeploymentConfig struct { + Id int `sql:"id,pk"` + AppId int `sql:"app_id"` + EnvironmentId int `sql:"environment_id"` + DeploymentAppType string `sql:"deployment_app_type"` + ConfigType string `sql:"config_type"` + RepoUrl string `sql:"repo_url"` + ChartLocation string `sql:"chart_location"` + CredentialType string `sql:"credential_type"` + CredentialIdInt int `sql:"credential_id_int"` + CredentialIdString string `sql:"credential_id_string"` + Active bool `sql:"active"` + sql.AuditLog +} + +type Repository interface { + Save(config *DeploymentConfig) (*DeploymentConfig, error) + Update(config *DeploymentConfig) (*DeploymentConfig, error) + GetById(id int) (*DeploymentConfig, error) + GetByAppIdAndEnvId(appId, envId int) (*DeploymentConfig, error) + GetAppLevelConfig(appId int) (*DeploymentConfig, error) +} + +type RepositoryImpl struct { + dbConnection *pg.DB +} + +func NewRepositoryImpl(dbConnection *pg.DB) *RepositoryImpl { + return &RepositoryImpl{dbConnection: dbConnection} +} + +func (impl RepositoryImpl) Save(config *DeploymentConfig) (*DeploymentConfig, error) { + err := impl.dbConnection.Insert(config) + return config, err +} + +func (impl RepositoryImpl) Update(config *DeploymentConfig) (*DeploymentConfig, error) { + err := impl.dbConnection.Update(config) + return config, err +} + +func (impl RepositoryImpl) GetById(id int) (*DeploymentConfig, error) { + result := &DeploymentConfig{} + err := impl.dbConnection.Model(result).Where("id = ?", id).Where("active = ?", true).Select() + return result, err +} + +func (impl RepositoryImpl) GetByAppIdAndEnvId(appId, envId int) (*DeploymentConfig, error) { + result := &DeploymentConfig{} + err := impl.dbConnection.Model(result). + Where("app_id = ?", appId). + Where("environment_id = ? ", envId). + Where("active = ?", true). + Select() + return result, err +} + +func (impl RepositoryImpl) GetAppLevelConfig(appId int) (*DeploymentConfig, error) { + result := &DeploymentConfig{} + err := impl.dbConnection.Model(result). + Where("app_id = ? ", appId). + Where("environment_id is NULL"). + Where("active = ?", true). + Select() + return result, err +} diff --git a/internal/sql/repository/pipelineConfig/PipelineRepository.go b/internal/sql/repository/pipelineConfig/PipelineRepository.go index 35645bb35d..1d21c54a9d 100644 --- a/internal/sql/repository/pipelineConfig/PipelineRepository.go +++ b/internal/sql/repository/pipelineConfig/PipelineRepository.go @@ -125,6 +125,7 @@ type PipelineRepository interface { UpdateOldCiPipelineIdToNewCiPipelineId(tx *pg.Tx, oldCiPipelineId, newCiPipelineId int) error // FindWithEnvironmentByCiIds Possibility of duplicate environment names when filtered by unique pipeline ids FindWithEnvironmentByCiIds(ctx context.Context, cIPipelineIds []int) ([]*Pipeline, error) + FindDeploymentAppTypeByAppIdAndEnvId(appId, envId int) (string, error) } type CiArtifactDTO struct { @@ -783,3 +784,12 @@ func (impl PipelineRepositoryImpl) FindWithEnvironmentByCiIds(ctx context.Contex } return cDPipelines, nil } + +func (impl PipelineRepositoryImpl) FindDeploymentAppTypeByAppIdAndEnvId(appId, envId int) (string, error) { + var deploymentAppType string + err := impl.dbConnection.Model((*Pipeline)(nil)). + Column("deployment_app_type"). + Where("app_id = ? and environment_id=? and active=true", appId, envId). + Select(&deploymentAppType) + return deploymentAppType, err +} diff --git a/pkg/chart/ChartService.go b/pkg/chart/ChartService.go index ba3af40d85..fba5362a5c 100644 --- a/pkg/chart/ChartService.go +++ b/pkg/chart/ChartService.go @@ -28,6 +28,7 @@ import ( "github.com/devtron-labs/devtron/internal/util" chartRepoRepository "github.com/devtron-labs/devtron/pkg/chartRepo/repository" repository4 "github.com/devtron-labs/devtron/pkg/cluster/repository" + "github.com/devtron-labs/devtron/pkg/deployment/common" "github.com/devtron-labs/devtron/pkg/deployment/gitOps/config" "github.com/devtron-labs/devtron/pkg/deployment/manifest/deployedAppMetrics" "github.com/devtron-labs/devtron/pkg/deployment/manifest/deployedAppMetrics/bean" @@ -85,6 +86,7 @@ type ChartServiceImpl struct { deployedAppMetricsService deployedAppMetrics.DeployedAppMetricsService chartRefService chartRef.ChartRefService gitOpsConfigReadService config.GitOpsConfigReadService + deploymentConfigService common.DeploymentConfigService } func NewChartServiceImpl(chartRepository chartRepoRepository.ChartRepository, @@ -100,7 +102,8 @@ func NewChartServiceImpl(chartRepository chartRepoRepository.ChartRepository, scopedVariableManager variables.ScopedVariableManager, deployedAppMetricsService deployedAppMetrics.DeployedAppMetricsService, chartRefService chartRef.ChartRefService, - gitOpsConfigReadService config.GitOpsConfigReadService) *ChartServiceImpl { + gitOpsConfigReadService config.GitOpsConfigReadService, + deploymentConfigService common.DeploymentConfigService) *ChartServiceImpl { return &ChartServiceImpl{ chartRepository: chartRepository, logger: logger, @@ -116,6 +119,7 @@ func NewChartServiceImpl(chartRepository chartRepoRepository.ChartRepository, deployedAppMetricsService: deployedAppMetricsService, chartRefService: chartRefService, gitOpsConfigReadService: gitOpsConfigReadService, + deploymentConfigService: deploymentConfigService, } } @@ -258,15 +262,15 @@ func (impl *ChartServiceImpl) Create(templateRequest TemplateRequest, ctx contex ChartVersion: chartMeta.Version, Status: models.CHARTSTATUS_NEW, Active: true, - ChartLocation: chartLocation, - GitRepoUrl: gitRepoUrl, - ReferenceTemplate: templateName, - ChartRefId: templateRequest.ChartRefId, - Latest: true, - Previous: false, - IsBasicViewLocked: templateRequest.IsBasicViewLocked, - CurrentViewEditor: templateRequest.CurrentViewEditor, - AuditLog: sql.AuditLog{CreatedBy: templateRequest.UserId, CreatedOn: time.Now(), UpdatedOn: time.Now(), UpdatedBy: templateRequest.UserId}, + //ChartLocation: chartLocation, + //GitRepoUrl: gitRepoUrl, + ReferenceTemplate: templateName, + ChartRefId: templateRequest.ChartRefId, + Latest: true, + Previous: false, + IsBasicViewLocked: templateRequest.IsBasicViewLocked, + CurrentViewEditor: templateRequest.CurrentViewEditor, + AuditLog: sql.AuditLog{CreatedBy: templateRequest.UserId, CreatedOn: time.Now(), UpdatedOn: time.Now(), UpdatedBy: templateRequest.UserId}, } err = impl.chartRepository.Save(chart) diff --git a/pkg/deployment/common/adapter.go b/pkg/deployment/common/adapter.go new file mode 100644 index 0000000000..6b8df190c8 --- /dev/null +++ b/pkg/deployment/common/adapter.go @@ -0,0 +1,38 @@ +package common + +import ( + "github.com/devtron-labs/devtron/internal/sql/repository/deploymentConfig" + "github.com/devtron-labs/devtron/pkg/deployment/common/bean" +) + +func ConvertDeploymentConfigDTOToDbObj(config *bean.DeploymentConfig) *deploymentConfig.DeploymentConfig { + return &deploymentConfig.DeploymentConfig{ + Id: config.Id, + AppId: config.AppId, + EnvironmentId: config.EnvironmentId, + DeploymentAppType: config.DeploymentAppType, + ConfigType: config.ConfigType, + RepoUrl: config.RepoURL, + ChartLocation: config.ChartLocation, + CredentialType: config.CredentialType, + CredentialIdInt: config.CredentialIdInt, + CredentialIdString: config.CredentialIdString, + Active: config.Active, + } +} + +func ConvertDeploymentConfigDbObjToDTO(dbObj *deploymentConfig.DeploymentConfig) *bean.DeploymentConfig { + return &bean.DeploymentConfig{ + Id: dbObj.Id, + AppId: dbObj.AppId, + EnvironmentId: dbObj.EnvironmentId, + DeploymentAppType: dbObj.DeploymentAppType, + ConfigType: dbObj.ConfigType, + RepoURL: dbObj.RepoUrl, + ChartLocation: dbObj.ChartLocation, + CredentialType: dbObj.CredentialType, + CredentialIdInt: dbObj.CredentialIdInt, + CredentialIdString: dbObj.CredentialIdString, + Active: dbObj.Active, + } +} diff --git a/pkg/deployment/common/bean/bean.go b/pkg/deployment/common/bean/bean.go new file mode 100644 index 0000000000..677a2efb7a --- /dev/null +++ b/pkg/deployment/common/bean/bean.go @@ -0,0 +1,29 @@ +package bean + +type DeploymentConfig struct { + Id int + AppId int + EnvironmentId int + ConfigType string + DeploymentAppType string + RepoURL string + RepoName string + ChartLocation string + CredentialType string + CredentialIdInt int + CredentialIdString string + Active bool +} + +type DeploymentConfigType string + +const ( + CUSTOM DeploymentConfigType = "custom" + SYSTEM_GENERATED DeploymentConfigType = "system_generated" +) + +type DeploymentConfigCredentialType string + +const ( + GitOps DeploymentConfigCredentialType = "gitOps" +) diff --git a/pkg/deployment/common/deploymentConfigService.go b/pkg/deployment/common/deploymentConfigService.go new file mode 100644 index 0000000000..e848f3ed64 --- /dev/null +++ b/pkg/deployment/common/deploymentConfigService.go @@ -0,0 +1,154 @@ +package common + +import ( + "github.com/devtron-labs/devtron/internal/sql/repository/deploymentConfig" + "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" + bean3 "github.com/devtron-labs/devtron/pkg/auth/user/bean" + chartRepoRepository "github.com/devtron-labs/devtron/pkg/chartRepo/repository" + "github.com/devtron-labs/devtron/pkg/deployment/common/bean" + "github.com/devtron-labs/devtron/pkg/deployment/gitOps/config" + bean2 "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/bean" + "github.com/go-pg/pg" + "go.uber.org/zap" +) + +type DeploymentConfigService interface { + CreateOrUpdateConfig(config *bean.DeploymentConfig, userId int32) (*bean.DeploymentConfig, error) + GetDeploymentConfig(appId, envId int) (*bean.DeploymentConfig, error) +} + +type DeploymentConfigServiceImpl struct { + deploymentConfigRepository deploymentConfig.Repository + logger *zap.SugaredLogger + chartRepository chartRepoRepository.ChartRepository + pipelineRepository pipelineConfig.PipelineRepository + gitOpsConfigReadService config.GitOpsConfigReadService +} + +func NewDeploymentConfigServiceImpl( + deploymentConfigRepository deploymentConfig.Repository, + logger *zap.SugaredLogger, + chartRepository chartRepoRepository.ChartRepository, + pipelineRepository pipelineConfig.PipelineRepository, + gitOpsConfigReadService config.GitOpsConfigReadService, +) *DeploymentConfigServiceImpl { + return &DeploymentConfigServiceImpl{ + deploymentConfigRepository: deploymentConfigRepository, + logger: logger, + chartRepository: chartRepository, + pipelineRepository: pipelineRepository, + gitOpsConfigReadService: gitOpsConfigReadService, + } +} + +func (impl *DeploymentConfigServiceImpl) CreateOrUpdateConfig(config *bean.DeploymentConfig, userId int32) (*bean.DeploymentConfig, error) { + + configDbObj, err := impl.GetConfigDBObj(config.AppId, config.EnvironmentId) + if err != nil && err != pg.ErrNoRows { + impl.logger.Errorw("error in fetching deployment config from DB by appId and envId", + "appId", config.AppId, "envId", config.EnvironmentId, "err", err) + } + + newDBObj := ConvertDeploymentConfigDTOToDbObj(config) + + if configDbObj == nil || (configDbObj != nil && configDbObj.Id == 0) { + newDBObj.AuditLog.CreateAuditLog(userId) + newDBObj, err = impl.deploymentConfigRepository.Save(newDBObj) + if err != nil { + impl.logger.Errorw("error in saving deploymentConfig", "appId", config.AppId, "envId", config.EnvironmentId, "err", err) + return nil, err + } + } else { + newDBObj.AuditLog.UpdateAuditLog(userId) + newDBObj, err = impl.deploymentConfigRepository.Update(newDBObj) + if err != nil { + impl.logger.Errorw("error in updating deploymentConfig", "appId", config.AppId, "envId", config.EnvironmentId, "err", err) + return nil, err + } + } + + return ConvertDeploymentConfigDbObjToDTO(newDBObj), nil +} + +func (impl *DeploymentConfigServiceImpl) GetDeploymentConfig(appId, envId int) (*bean.DeploymentConfig, error) { + configDbObj, err := impl.GetConfigDBObj(appId, envId) + if err != nil && err != pg.ErrNoRows { + impl.logger.Errorw("error in fetching deployment config from DB by appId and envId", + "appId", appId, "envId", envId, "err", err) + } + if err == pg.ErrNoRows || (configDbObj == nil || (configDbObj != nil && configDbObj.Id == 0)) { + configDbObj, err = impl.migrateOldDataToDeploymentConfig(appId, envId) + if err != nil { + impl.logger.Errorw("error in migrating old data to deployment config", "appId", appId, "envId", envId, "err", err) + return nil, err + } + } + return ConvertDeploymentConfigDbObjToDTO(configDbObj), nil +} + +func (impl *DeploymentConfigServiceImpl) migrateOldDataToDeploymentConfig(appId int, envId int) (*deploymentConfig.DeploymentConfig, error) { + + chart, err := impl.chartRepository.FindLatestChartForAppByAppId(appId) + if err != nil { + impl.logger.Errorw("error in fetch chart for git repo migration by appId", "appId", appId, "err", err) + return nil, err + } + var deploymentAppType string + if envId > 0 { + deploymentAppType, err = impl.pipelineRepository.FindDeploymentAppTypeByAppIdAndEnvId(appId, envId) + if err != nil { + impl.logger.Errorw("error in getting deployment app type by appId and envId", "appId", appId, "envId", envId, "err", err) + return nil, err + } + } + + ConfigDbObj := &deploymentConfig.DeploymentConfig{ + AppId: appId, + EnvironmentId: envId, + DeploymentAppType: deploymentAppType, + Active: true, + } + + switch deploymentAppType { + //TODO: handling for other deployment app type in future + case bean2.ArgoCd: + ConfigDbObj.ConfigType = GetDeploymentConfigType(chart.IsCustomGitRepository) + ConfigDbObj.RepoUrl = chart.GitRepoUrl + ConfigDbObj.ChartLocation = chart.ChartLocation + ConfigDbObj.CredentialType = string(bean.GitOps) + gitOpsConfig, err := impl.gitOpsConfigReadService.GetGitOpsProviderByRepoURL(ConfigDbObj.RepoUrl) + if err != nil { + impl.logger.Infow("error in fetching gitOps config by repoUrl, skipping migration to deployment config", "repoURL", ConfigDbObj.RepoUrl) + return ConfigDbObj, nil + } + ConfigDbObj.CredentialIdInt = gitOpsConfig.Id + } + + ConfigDbObj.AuditLog.CreateAuditLog(bean3.SYSTEM_USER_ID) + ConfigDbObj, err = impl.deploymentConfigRepository.Save(ConfigDbObj) + if err != nil { + impl.logger.Errorw("error in saving deployment config in DB", "appId", appId, "envId", envId, "err", err) + return nil, err + } + + return ConfigDbObj, nil +} + +func (impl *DeploymentConfigServiceImpl) GetConfigDBObj(appId, envId int) (*deploymentConfig.DeploymentConfig, error) { + var configDbObj *deploymentConfig.DeploymentConfig + var err error + if envId == 0 { + configDbObj, err = impl.deploymentConfigRepository.GetAppLevelConfig(appId) + if err != nil { + impl.logger.Errorw("error in getiting deployment config db object by appId", "appId", configDbObj.AppId, "err", err) + return nil, err + } + } else { + configDbObj, err = impl.deploymentConfigRepository.GetByAppIdAndEnvId(appId, envId) + if err != nil { + impl.logger.Errorw("error in getiting deployment config db object by appId and envId", "appId", configDbObj.AppId, "envId", configDbObj.EnvironmentId, "err", err) + return nil, err + } + } + return configDbObj, nil +} diff --git a/pkg/deployment/common/helper.go b/pkg/deployment/common/helper.go new file mode 100644 index 0000000000..1ad3b12ff1 --- /dev/null +++ b/pkg/deployment/common/helper.go @@ -0,0 +1,10 @@ +package common + +import "github.com/devtron-labs/devtron/pkg/deployment/common/bean" + +func GetDeploymentConfigType(isCustomGitOpsRepo bool) string { + if isCustomGitOpsRepo { + return string(bean.CUSTOM) + } + return string(bean.SYSTEM_GENERATED) +} diff --git a/pkg/deployment/common/read/deploymentConfigReadService.go b/pkg/deployment/common/read/deploymentConfigReadService.go new file mode 100644 index 0000000000..e7a97a29c9 --- /dev/null +++ b/pkg/deployment/common/read/deploymentConfigReadService.go @@ -0,0 +1,16 @@ +package read + +import ( + "github.com/devtron-labs/devtron/internal/sql/repository/deploymentConfig" + "github.com/devtron-labs/devtron/pkg/deployment/common/bean" + "go.uber.org/zap" +) + +type DeploymentConfigReadService interface { + GetByAppIdAndEnvId(appId, envId int) (*bean.DeploymentConfig, error) +} + +type DeploymentConfigReadServiceImpl struct { + deploymentConfigRepository deploymentConfig.Repository + logger *zap.SugaredLogger +} diff --git a/pkg/deployment/gitOps/config/GitOpsConfigReadService.go b/pkg/deployment/gitOps/config/GitOpsConfigReadService.go index 073f791d6e..bb0f9f4bcc 100644 --- a/pkg/deployment/gitOps/config/GitOpsConfigReadService.go +++ b/pkg/deployment/gitOps/config/GitOpsConfigReadService.go @@ -38,6 +38,7 @@ type GitOpsConfigReadService interface { GetBitbucketMetadata() (*bean.BitbucketProviderMetadata, error) GetGitOpsConfigActive() (*bean2.GitOpsConfigDto, error) GetConfiguredGitOpsCount() (int, error) + GetGitOpsProviderByRepoURL(gitRepoUrl string) (*bean2.GitOpsConfigDto, error) } type GitOpsConfigReadServiceImpl struct { @@ -158,3 +159,45 @@ func (impl *GitOpsConfigReadServiceImpl) GetConfiguredGitOpsCount() (int, error) count, err := impl.gitOpsRepository.GetAllGitOpsConfigCount() return count, err } + +func (impl *GitOpsConfigReadServiceImpl) GetGitOpsProviderByRepoURL(gitRepoUrl string) (*bean2.GitOpsConfigDto, error) { + models, err := impl.gitOpsRepository.GetAllGitOpsConfig() + if err != nil { + impl.logger.Errorw("error, GetGitOpsConfigActive", "err", err) + return nil, err + } + + var gitOpsConfig *bean2.GitOpsConfigDto + + requestHost, err := util.GetHost(gitRepoUrl) + if err != nil { + return nil, fmt.Errorf("unable to parse host from repo URL: %s", gitRepoUrl) + } + + for _, model := range models { + if host, err := util.GetHost(model.Host); err != nil && host == requestHost { + gitOpsConfig = &bean2.GitOpsConfigDto{ + Id: model.Id, + Provider: model.Provider, + GitHubOrgId: model.GitHubOrgId, + GitLabGroupId: model.GitLabGroupId, + Active: model.Active, + Token: model.Token, + Host: model.Host, + Username: model.Username, + UserId: model.CreatedBy, + AzureProjectName: model.AzureProject, + BitBucketWorkspaceId: model.BitBucketWorkspaceId, + BitBucketProjectKey: model.BitBucketProjectKey, + AllowCustomRepository: model.AllowCustomRepository, + } + // written with assumption that only one GitOpsConfig is present in DB for each provider(github, gitlab, etc) + break + } + } + if gitOpsConfig == nil { + return nil, fmt.Errorf("no gitops config found in DB for given repoURL: %s", gitRepoUrl) + } + + return gitOpsConfig, nil +} diff --git a/pkg/pipeline/DeploymentConfigService.go b/pkg/pipeline/DeploymentConfigService.go index aee803793b..798f7da2c4 100644 --- a/pkg/pipeline/DeploymentConfigService.go +++ b/pkg/pipeline/DeploymentConfigService.go @@ -36,11 +36,11 @@ import ( "go.uber.org/zap" ) -type DeploymentConfigService interface { +type PipelineDeploymentConfigService interface { GetLatestDeploymentConfigurationByPipelineId(ctx context.Context, pipelineId int, userHasAdminAccess bool) (*history.AllDeploymentConfigurationDetail, error) } -type DeploymentConfigServiceImpl struct { +type PipelineDeploymentConfigServiceImpl struct { logger *zap.SugaredLogger envConfigOverrideRepository chartConfig.EnvConfigOverrideRepository chartRepository chartRepoRepository.ChartRepository @@ -53,7 +53,7 @@ type DeploymentConfigServiceImpl struct { chartRefService chartRef.ChartRefService } -func NewDeploymentConfigServiceImpl(logger *zap.SugaredLogger, +func NewPipelineDeploymentConfigServiceImpl(logger *zap.SugaredLogger, envConfigOverrideRepository chartConfig.EnvConfigOverrideRepository, chartRepository chartRepoRepository.ChartRepository, pipelineRepository pipelineConfig.PipelineRepository, @@ -62,8 +62,8 @@ func NewDeploymentConfigServiceImpl(logger *zap.SugaredLogger, configMapHistoryService history.ConfigMapHistoryService, scopedVariableManager variables.ScopedVariableCMCSManager, deployedAppMetricsService deployedAppMetrics.DeployedAppMetricsService, - chartRefService chartRef.ChartRefService) *DeploymentConfigServiceImpl { - return &DeploymentConfigServiceImpl{ + chartRefService chartRef.ChartRefService) *PipelineDeploymentConfigServiceImpl { + return &PipelineDeploymentConfigServiceImpl{ logger: logger, envConfigOverrideRepository: envConfigOverrideRepository, chartRepository: chartRepository, @@ -77,7 +77,7 @@ func NewDeploymentConfigServiceImpl(logger *zap.SugaredLogger, } } -func (impl *DeploymentConfigServiceImpl) GetLatestDeploymentConfigurationByPipelineId(ctx context.Context, pipelineId int, userHasAdminAccess bool) (*history.AllDeploymentConfigurationDetail, error) { +func (impl *PipelineDeploymentConfigServiceImpl) GetLatestDeploymentConfigurationByPipelineId(ctx context.Context, pipelineId int, userHasAdminAccess bool) (*history.AllDeploymentConfigurationDetail, error) { configResp := &history.AllDeploymentConfigurationDetail{} pipeline, err := impl.pipelineRepository.FindById(pipelineId) if err != nil { @@ -109,7 +109,7 @@ func (impl *DeploymentConfigServiceImpl) GetLatestDeploymentConfigurationByPipel return configResp, nil } -func (impl *DeploymentConfigServiceImpl) GetLatestDeploymentTemplateConfig(ctx context.Context, pipeline *pipelineConfig.Pipeline) (*history.HistoryDetailDto, error) { +func (impl *PipelineDeploymentConfigServiceImpl) GetLatestDeploymentTemplateConfig(ctx context.Context, pipeline *pipelineConfig.Pipeline) (*history.HistoryDetailDto, error) { isAppMetricsEnabled, err := impl.deployedAppMetricsService.GetMetricsFlagForAPipelineByAppIdAndEnvId(pipeline.AppId, pipeline.EnvironmentId) if err != nil { impl.logger.Errorw("error, GetMetricsFlagForAPipelineByAppIdAndEnvId", "err", err, "appId", pipeline.AppId, "envId", pipeline.EnvironmentId) @@ -203,7 +203,7 @@ func (impl *DeploymentConfigServiceImpl) GetLatestDeploymentTemplateConfig(ctx c return deploymentTemplateConfig, nil } -func (impl *DeploymentConfigServiceImpl) GetLatestPipelineStrategyConfig(pipeline *pipelineConfig.Pipeline) (*history.HistoryDetailDto, error) { +func (impl *PipelineDeploymentConfigServiceImpl) GetLatestPipelineStrategyConfig(pipeline *pipelineConfig.Pipeline) (*history.HistoryDetailDto, error) { pipelineStrategy, err := impl.pipelineConfigRepository.GetDefaultStrategyByPipelineId(pipeline.Id) if err != nil { @@ -221,7 +221,7 @@ func (impl *DeploymentConfigServiceImpl) GetLatestPipelineStrategyConfig(pipelin return pipelineStrategyConfig, nil } -func (impl *DeploymentConfigServiceImpl) GetLatestCMCSConfig(ctx context.Context, pipeline *pipelineConfig.Pipeline, userHasAdminAccess bool) ([]*history.ComponentLevelHistoryDetailDto, []*history.ComponentLevelHistoryDetailDto, error) { +func (impl *PipelineDeploymentConfigServiceImpl) GetLatestCMCSConfig(ctx context.Context, pipeline *pipelineConfig.Pipeline, userHasAdminAccess bool) ([]*history.ComponentLevelHistoryDetailDto, []*history.ComponentLevelHistoryDetailDto, error) { configAppLevel, err := impl.configMapRepository.GetByAppIdAppLevel(pipeline.AppId) if err != nil && pg.ErrNoRows != err { @@ -293,7 +293,7 @@ func (impl *DeploymentConfigServiceImpl) GetLatestCMCSConfig(ctx context.Context return cmConfigsDto, secretConfigsDto, nil } -func (impl *DeploymentConfigServiceImpl) GetMergedCMCSConfigMap(appLevelConfig, envLevelConfig string, configType repository2.ConfigType) (map[string]*bean.ConfigData, error) { +func (impl *PipelineDeploymentConfigServiceImpl) GetMergedCMCSConfigMap(appLevelConfig, envLevelConfig string, configType repository2.ConfigType) (map[string]*bean.ConfigData, error) { envLevelMap := make(map[string]*bean.ConfigData, 0) finalMap := make(map[string]*bean.ConfigData, 0) if configType == repository2.CONFIGMAP_TYPE { diff --git a/util/HttpUtil.go b/util/HttpUtil.go index d240688b13..3e6a022bfd 100644 --- a/util/HttpUtil.go +++ b/util/HttpUtil.go @@ -21,6 +21,7 @@ import ( "fmt" "io/ioutil" "net/http" + "net/url" "strconv" "time" ) @@ -55,3 +56,15 @@ func ReadFromUrlWithRetry(url string) ([]byte, error) { } return nil, err } + +func GetHost(urlStr string) (string, error) { + u, err := url.Parse(urlStr) + if err == nil { + return u.Host, nil + } + u, err = url.Parse("//" + urlStr) + if err != nil { + return "", fmt.Errorf("invalid url: %w", err) + } + return u.Host, nil +} diff --git a/wire_gen.go b/wire_gen.go index 2d0e72a2b2..28e8aeadce 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -84,6 +84,7 @@ import ( "github.com/devtron-labs/devtron/internal/sql/repository/appWorkflow" "github.com/devtron-labs/devtron/internal/sql/repository/bulkUpdate" "github.com/devtron-labs/devtron/internal/sql/repository/chartConfig" + "github.com/devtron-labs/devtron/internal/sql/repository/deploymentConfig" repository5 "github.com/devtron-labs/devtron/internal/sql/repository/dockerRegistry" "github.com/devtron-labs/devtron/internal/sql/repository/helper" repository12 "github.com/devtron-labs/devtron/internal/sql/repository/imageTagging" @@ -131,6 +132,7 @@ import ( "github.com/devtron-labs/devtron/pkg/clusterTerminalAccess" "github.com/devtron-labs/devtron/pkg/commonService" delete2 "github.com/devtron-labs/devtron/pkg/delete" + "github.com/devtron-labs/devtron/pkg/deployment/common" "github.com/devtron-labs/devtron/pkg/deployment/deployedApp" "github.com/devtron-labs/devtron/pkg/deployment/gitOps/config" "github.com/devtron-labs/devtron/pkg/deployment/gitOps/git" @@ -559,7 +561,9 @@ func InitializeApp() (*App, error) { configMapServiceImpl := pipeline.NewConfigMapServiceImpl(chartRepositoryImpl, sugaredLogger, chartRepoRepositoryImpl, utilMergeUtil, pipelineConfigRepositoryImpl, configMapRepositoryImpl, envConfigOverrideRepositoryImpl, commonServiceImpl, appRepositoryImpl, configMapHistoryServiceImpl, environmentRepositoryImpl, scopedVariableCMCSManagerImpl) deploymentTemplateHistoryRepositoryImpl := repository13.NewDeploymentTemplateHistoryRepositoryImpl(sugaredLogger, db) deploymentTemplateHistoryServiceImpl := history.NewDeploymentTemplateHistoryServiceImpl(sugaredLogger, deploymentTemplateHistoryRepositoryImpl, pipelineRepositoryImpl, chartRepositoryImpl, userServiceImpl, cdWorkflowRepositoryImpl, scopedVariableManagerImpl, deployedAppMetricsServiceImpl, chartRefServiceImpl) - chartServiceImpl := chart.NewChartServiceImpl(chartRepositoryImpl, sugaredLogger, chartTemplateServiceImpl, chartRepoRepositoryImpl, appRepositoryImpl, utilMergeUtil, envConfigOverrideRepositoryImpl, pipelineConfigRepositoryImpl, environmentRepositoryImpl, deploymentTemplateHistoryServiceImpl, scopedVariableManagerImpl, deployedAppMetricsServiceImpl, chartRefServiceImpl, gitOpsConfigReadServiceImpl) + repositoryImpl := deploymentConfig.NewRepositoryImpl(db) + deploymentConfigServiceImpl := common.NewDeploymentConfigServiceImpl(repositoryImpl, sugaredLogger, chartRepositoryImpl, pipelineRepositoryImpl, gitOpsConfigReadServiceImpl) + chartServiceImpl := chart.NewChartServiceImpl(chartRepositoryImpl, sugaredLogger, chartTemplateServiceImpl, chartRepoRepositoryImpl, appRepositoryImpl, utilMergeUtil, envConfigOverrideRepositoryImpl, pipelineConfigRepositoryImpl, environmentRepositoryImpl, deploymentTemplateHistoryServiceImpl, scopedVariableManagerImpl, deployedAppMetricsServiceImpl, chartRefServiceImpl, gitOpsConfigReadServiceImpl, deploymentConfigServiceImpl) ciCdPipelineOrchestratorImpl := pipeline.NewCiCdPipelineOrchestrator(appRepositoryImpl, sugaredLogger, materialRepositoryImpl, pipelineRepositoryImpl, ciPipelineRepositoryImpl, ciPipelineMaterialRepositoryImpl, cdWorkflowRepositoryImpl, clientImpl, ciCdConfig, appWorkflowRepositoryImpl, environmentRepositoryImpl, attributesServiceImpl, appCrudOperationServiceImpl, userAuthServiceImpl, prePostCdScriptHistoryServiceImpl, pipelineStageServiceImpl, gitMaterialHistoryServiceImpl, ciPipelineHistoryServiceImpl, ciTemplateServiceImpl, dockerArtifactStoreRepositoryImpl, ciArtifactRepositoryImpl, configMapServiceImpl, customTagServiceImpl, genericNoteServiceImpl, chartServiceImpl, transactionUtilImpl) ecrConfig, err := pipeline.GetEcrConfig() if err != nil { @@ -816,8 +820,8 @@ func InitializeApp() (*App, error) { appListingRouterImpl := appList2.NewAppListingRouterImpl(appListingRestHandlerImpl) appInfoRestHandlerImpl := appInfo.NewAppInfoRestHandlerImpl(sugaredLogger, appCrudOperationServiceImpl, userServiceImpl, validate, enforcerUtilImpl, enforcerImpl, helmAppServiceImpl, enforcerUtilHelmImpl, genericNoteServiceImpl) appInfoRouterImpl := appInfo2.NewAppInfoRouterImpl(sugaredLogger, appInfoRestHandlerImpl) - deploymentConfigServiceImpl := pipeline.NewDeploymentConfigServiceImpl(sugaredLogger, envConfigOverrideRepositoryImpl, chartRepositoryImpl, pipelineRepositoryImpl, pipelineConfigRepositoryImpl, configMapRepositoryImpl, configMapHistoryServiceImpl, scopedVariableCMCSManagerImpl, deployedAppMetricsServiceImpl, chartRefServiceImpl) - pipelineTriggerRestHandlerImpl := trigger.NewPipelineRestHandler(appServiceImpl, userServiceImpl, validate, enforcerImpl, teamServiceImpl, sugaredLogger, enforcerUtilImpl, deploymentGroupServiceImpl, argoUserServiceImpl, deploymentConfigServiceImpl, deployedAppServiceImpl, triggerServiceImpl, workflowEventPublishServiceImpl) + pipelineDeploymentConfigServiceImpl := pipeline.NewPipelineDeploymentConfigServiceImpl(sugaredLogger, envConfigOverrideRepositoryImpl, chartRepositoryImpl, pipelineRepositoryImpl, pipelineConfigRepositoryImpl, configMapRepositoryImpl, configMapHistoryServiceImpl, scopedVariableCMCSManagerImpl, deployedAppMetricsServiceImpl, chartRefServiceImpl) + pipelineTriggerRestHandlerImpl := trigger.NewPipelineRestHandler(appServiceImpl, userServiceImpl, validate, enforcerImpl, teamServiceImpl, sugaredLogger, enforcerUtilImpl, deploymentGroupServiceImpl, argoUserServiceImpl, pipelineDeploymentConfigServiceImpl, deployedAppServiceImpl, triggerServiceImpl, workflowEventPublishServiceImpl) sseSSE := sse.NewSSE() pipelineTriggerRouterImpl := trigger2.NewPipelineTriggerRouter(pipelineTriggerRestHandlerImpl, sseSSE) webhookDataRestHandlerImpl := webhook.NewWebhookDataRestHandlerImpl(sugaredLogger, userServiceImpl, ciPipelineMaterialRepositoryImpl, enforcerUtilImpl, enforcerImpl, clientImpl, webhookEventDataConfigImpl) From 08152bf3e94befcf6265c3d8d3b14aabdbfc92da Mon Sep 17 00:00:00 2001 From: ayushmaheshwari Date: Thu, 27 Jun 2024 16:16:21 +0530 Subject: [PATCH 02/45] wip: migration changes for devtron apps --- .../app/appList/AppListingRestHandler.go | 45 +++- .../pipelineConfig/PipelineRepository.go | 2 +- .../adapter/cdWorkflow/adapter.go | 5 +- pkg/app/AppService.go | 17 +- .../status/PipelineStatusTimelineService.go | 14 +- pkg/appWorkflow/AppWorkflowService.go | 16 +- pkg/chart/ChartService.go | 223 +++++++++++------- .../DevtronAppGitOpsConfigService.go | 14 +- pkg/deployment/common/bean/bean.go | 8 + .../common/deploymentConfigService.go | 101 +++++--- pkg/deployment/common/helper.go | 4 + .../manifest/ManifestCreationService.go | 14 +- .../manifest/publish/ManifestPushService.go | 2 +- .../trigger/devtronApps/TriggerService.go | 54 +++-- .../trigger/devtronApps/adapter/adapter.go | 5 +- .../in/WorkflowEventProcessorService.go | 26 +- .../AppDeploymentTypeChangeManager.go | 61 +++-- pkg/pipeline/CdHandler.go | 16 +- pkg/pipeline/CiCdPipelineOrchestrator.go | 32 ++- .../DeploymentPipelineConfigService.go | 85 +++++-- pkg/pipeline/PipelineBuilder.go | 10 - pkg/workflow/cd/CdWorkflowCommonService.go | 25 +- pkg/workflow/dag/WorkflowDagExecutor.go | 14 +- pkg/workflow/status/WorkflowStatusService.go | 15 +- wire_gen.go | 34 +-- 25 files changed, 603 insertions(+), 239 deletions(-) diff --git a/api/restHandler/app/appList/AppListingRestHandler.go b/api/restHandler/app/appList/AppListingRestHandler.go index a995a4f12b..fe9be69008 100644 --- a/api/restHandler/app/appList/AppListingRestHandler.go +++ b/api/restHandler/app/appList/AppListingRestHandler.go @@ -29,6 +29,8 @@ import ( "github.com/devtron-labs/devtron/pkg/appStore/installedApp/service/FullMode/resource" util4 "github.com/devtron-labs/devtron/pkg/appStore/util" bean2 "github.com/devtron-labs/devtron/pkg/cluster/repository/bean" + common2 "github.com/devtron-labs/devtron/pkg/deployment/common" + bean3 "github.com/devtron-labs/devtron/pkg/deployment/common/bean" "net/http" "strconv" "time" @@ -110,6 +112,7 @@ type AppListingRestHandlerImpl struct { genericNoteService genericNotes.GenericNoteService k8sApplicationService application3.K8sApplicationService deploymentTemplateService generateManifest.DeploymentTemplateService + deploymentConfigService common2.DeploymentConfigService } type AppStatus struct { @@ -140,7 +143,9 @@ func NewAppListingRestHandlerImpl(application application.ServiceClient, pipelineRepository pipelineConfig.PipelineRepository, appStatusService appStatus.AppStatusService, installedAppRepository repository.InstalledAppRepository, genericNoteService genericNotes.GenericNoteService, - k8sApplicationService application3.K8sApplicationService, deploymentTemplateService generateManifest.DeploymentTemplateService, + k8sApplicationService application3.K8sApplicationService, + deploymentTemplateService generateManifest.DeploymentTemplateService, + deploymentConfigService common2.DeploymentConfigService, ) *AppListingRestHandlerImpl { appListingHandler := &AppListingRestHandlerImpl{ application: application, @@ -164,6 +169,7 @@ func NewAppListingRestHandlerImpl(application application.ServiceClient, genericNoteService: genericNoteService, k8sApplicationService: k8sApplicationService, deploymentTemplateService: deploymentTemplateService, + deploymentConfigService: deploymentConfigService, } return appListingHandler } @@ -542,7 +548,13 @@ func (handler AppListingRestHandlerImpl) FetchAppDetails(w http.ResponseWriter, common.WriteJsonResp(w, fmt.Errorf("error in getting acd token"), nil, http.StatusInternalServerError) return } - resourceTree, err := handler.fetchResourceTree(w, r, appId, envId, acdToken, cdPipeline) + envDeploymentConfig, err := handler.deploymentConfigService.GetDeploymentConfig(appId, envId) + if err != nil { + handler.logger.Errorw("error in fetching deployment config", "appId", appId, "envId", envId, "err", err) + common.WriteJsonResp(w, fmt.Errorf("error in getting deployment config for env"), nil, http.StatusInternalServerError) + return + } + resourceTree, err := handler.fetchResourceTree(w, r, appId, envId, acdToken, cdPipeline, envDeploymentConfig) if appDetail.DeploymentAppType == util.PIPELINE_DEPLOYMENT_TYPE_ACD { apiError, ok := err.(*util.ApiError) if ok && apiError != nil { @@ -635,20 +647,26 @@ func (handler AppListingRestHandlerImpl) FetchResourceTree(w http.ResponseWriter common.WriteJsonResp(w, fmt.Errorf("error in getting acd token"), nil, http.StatusInternalServerError) return } - resourceTree, err := handler.fetchResourceTree(w, r, appId, envId, acdToken, cdPipeline) + envDeploymentConfig, err := handler.deploymentConfigService.GetDeploymentConfig(appId, envId) + if err != nil { + handler.logger.Errorw("error in fetching deployment config", "appId", appId, "envId", envId, "err", err) + common.WriteJsonResp(w, fmt.Errorf("error in getting deployment config for env"), nil, http.StatusInternalServerError) + return + } + resourceTree, err := handler.fetchResourceTree(w, r, appId, envId, acdToken, cdPipeline, envDeploymentConfig) if err != nil { handler.logger.Errorw("error in fetching resource tree", "err", err, "appId", appId, "envId", envId) - handler.handleResourceTreeErrAndDeletePipelineIfNeeded(w, err, acdToken, cdPipeline) + handler.handleResourceTreeErrAndDeletePipelineIfNeeded(w, err, acdToken, cdPipeline, envDeploymentConfig) return } common.WriteJsonResp(w, err, resourceTree, http.StatusOK) } func (handler AppListingRestHandlerImpl) handleResourceTreeErrAndDeletePipelineIfNeeded(w http.ResponseWriter, err error, - acdToken string, cdPipeline *pipelineConfig.Pipeline) { + acdToken string, cdPipeline *pipelineConfig.Pipeline, deploymentConfig *bean3.DeploymentConfig) { var apiError *util.ApiError ok := errors.As(err, &apiError) - if cdPipeline.DeploymentAppType == util.PIPELINE_DEPLOYMENT_TYPE_ACD { + if deploymentConfig.DeploymentAppType == util.PIPELINE_DEPLOYMENT_TYPE_ACD { if ok && apiError != nil { if apiError.Code == constants.AppDetailResourceTreeNotFound && cdPipeline.DeploymentAppDeleteRequest == true && cdPipeline.DeploymentAppCreated == true { acdAppFound, appDeleteErr := handler.pipeline.MarkGitOpsDevtronAppsDeletedWhereArgoAppIsDeleted(acdToken, cdPipeline) @@ -943,7 +961,13 @@ func (handler AppListingRestHandlerImpl) GetHostUrlsByBatch(w http.ResponseWrite } cdPipeline := pipelines[0] - resourceTree, err = handler.fetchResourceTree(w, r, appId, envId, acdToken, cdPipeline) + envDeploymentConfig, err := handler.deploymentConfigService.GetDeploymentConfig(appId, envId) + if err != nil { + handler.logger.Errorw("error in fetching deployment config", "appId", appId, "envId", envId, "err", err) + common.WriteJsonResp(w, fmt.Errorf("error in getting deployment config for env"), nil, http.StatusInternalServerError) + return + } + resourceTree, err = handler.fetchResourceTree(w, r, appId, envId, acdToken, cdPipeline, envDeploymentConfig) } _, ok := resourceTree["nodes"] if !ok { @@ -991,13 +1015,14 @@ func (handler AppListingRestHandlerImpl) getAppDetails(ctx context.Context, appI } // TODO: move this to service -func (handler AppListingRestHandlerImpl) fetchResourceTree(w http.ResponseWriter, r *http.Request, appId int, envId int, acdToken string, cdPipeline *pipelineConfig.Pipeline) (map[string]interface{}, error) { +func (handler AppListingRestHandlerImpl) fetchResourceTree(w http.ResponseWriter, r *http.Request, appId int, envId int, acdToken string, cdPipeline *pipelineConfig.Pipeline, deploymentConfig *bean3.DeploymentConfig) (map[string]interface{}, error) { var resourceTree map[string]interface{} if !cdPipeline.DeploymentAppCreated { handler.logger.Infow("deployment for this pipeline does not exist", "pipelineId", cdPipeline.Id) return resourceTree, nil } - if len(cdPipeline.DeploymentAppName) > 0 && cdPipeline.EnvironmentId > 0 && util.IsAcdApp(cdPipeline.DeploymentAppType) { + + if len(cdPipeline.DeploymentAppName) > 0 && cdPipeline.EnvironmentId > 0 && util.IsAcdApp(deploymentConfig.DeploymentAppType) { // RBAC enforcer Ends query := &application2.ResourcesQuery{ ApplicationName: &cdPipeline.DeploymentAppName, @@ -1075,7 +1100,7 @@ func (handler AppListingRestHandlerImpl) fetchResourceTree(w http.ResponseWriter } }() - } else if len(cdPipeline.DeploymentAppName) > 0 && cdPipeline.EnvironmentId > 0 && util.IsHelmApp(cdPipeline.DeploymentAppType) { + } else if len(cdPipeline.DeploymentAppName) > 0 && cdPipeline.EnvironmentId > 0 && util.IsHelmApp(deploymentConfig.DeploymentAppType) { config, err := handler.helmAppService.GetClusterConf(cdPipeline.Environment.ClusterId) if err != nil { handler.logger.Errorw("error in fetching cluster detail", "err", err) diff --git a/internal/sql/repository/pipelineConfig/PipelineRepository.go b/internal/sql/repository/pipelineConfig/PipelineRepository.go index 1d21c54a9d..c51753cc06 100644 --- a/internal/sql/repository/pipelineConfig/PipelineRepository.go +++ b/internal/sql/repository/pipelineConfig/PipelineRepository.go @@ -789,7 +789,7 @@ func (impl PipelineRepositoryImpl) FindDeploymentAppTypeByAppIdAndEnvId(appId, e var deploymentAppType string err := impl.dbConnection.Model((*Pipeline)(nil)). Column("deployment_app_type"). - Where("app_id = ? and environment_id=? and active=true", appId, envId). + Where("app_id = ? and environment_id=?", appId, envId). Select(&deploymentAppType) return deploymentAppType, err } diff --git a/internal/sql/repository/pipelineConfig/adapter/cdWorkflow/adapter.go b/internal/sql/repository/pipelineConfig/adapter/cdWorkflow/adapter.go index 306811b15d..9e73c3006c 100644 --- a/internal/sql/repository/pipelineConfig/adapter/cdWorkflow/adapter.go +++ b/internal/sql/repository/pipelineConfig/adapter/cdWorkflow/adapter.go @@ -2,17 +2,18 @@ package cdWorkflow import ( "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" + "github.com/devtron-labs/devtron/pkg/deployment/common/bean" "github.com/devtron-labs/devtron/util" "time" ) type UpdateOptions = func(cdWfr *pipelineConfig.CdWorkflowRunner) -func GetTriggerMetricsFromRunnerObj(runner *pipelineConfig.CdWorkflowRunner) util.CDMetrics { +func GetTriggerMetricsFromRunnerObj(runner *pipelineConfig.CdWorkflowRunner, deploymentConfig *bean.DeploymentConfig) util.CDMetrics { return util.CDMetrics{ AppName: runner.CdWorkflow.Pipeline.DeploymentAppName, Status: runner.Status, - DeploymentType: runner.CdWorkflow.Pipeline.DeploymentAppType, + DeploymentType: deploymentConfig.DeploymentAppType, EnvironmentName: runner.CdWorkflow.Pipeline.Environment.Name, Time: time.Since(runner.StartedOn).Seconds() - time.Since(runner.FinishedOn).Seconds(), } diff --git a/pkg/app/AppService.go b/pkg/app/AppService.go index ff744a4160..a43e7a0b20 100644 --- a/pkg/app/AppService.go +++ b/pkg/app/AppService.go @@ -26,6 +26,8 @@ import ( "github.com/devtron-labs/devtron/internal/sql/models" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/adapter/cdWorkflow" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/timelineStatus" + common2 "github.com/devtron-labs/devtron/pkg/deployment/common" + bean2 "github.com/devtron-labs/devtron/pkg/deployment/common/bean" commonBean "github.com/devtron-labs/devtron/pkg/deployment/gitOps/common/bean" "github.com/devtron-labs/devtron/pkg/deployment/gitOps/config" "github.com/devtron-labs/devtron/pkg/deployment/gitOps/git" @@ -122,6 +124,7 @@ type AppServiceImpl struct { gitOperationService git.GitOperationService deploymentTemplateService deploymentTemplate.DeploymentTemplateService appListingService AppListingService + deploymentConfigService common2.DeploymentConfigService } type AppService interface { @@ -162,7 +165,8 @@ func NewAppService( scopedVariableManager variables.ScopedVariableCMCSManager, acdConfig *argocdServer.ACDConfig, gitOpsConfigReadService config.GitOpsConfigReadService, gitOperationService git.GitOperationService, deploymentTemplateService deploymentTemplate.DeploymentTemplateService, - appListingService AppListingService) *AppServiceImpl { + appListingService AppListingService, + deploymentConfigService common2.DeploymentConfigService) *AppServiceImpl { appServiceImpl := &AppServiceImpl{ environmentConfigRepository: environmentConfigRepository, mergeUtil: mergeUtil, @@ -193,6 +197,7 @@ func NewAppService( gitOperationService: gitOperationService, deploymentTemplateService: deploymentTemplateService, appListingService: appListingService, + deploymentConfigService: deploymentConfigService, } return appServiceImpl } @@ -830,6 +835,7 @@ type ValuesOverrideResponse struct { PipelineOverride *chartConfig.PipelineOverride Artifact *repository.CiArtifact Pipeline *pipelineConfig.Pipeline + DeploymentConfig *bean2.DeploymentConfig } func (impl *AppServiceImpl) buildACDContext() (acdContext context.Context, err error) { @@ -1147,7 +1153,14 @@ func (impl *AppServiceImpl) UpdateCdWorkflowRunnerByACDObject(app *v1alpha1.Appl impl.logger.Errorw("error on update cd workflow runner", "wfr", wfr, "app", app, "err", err) return err } - util2.TriggerCDMetrics(cdWorkflow.GetTriggerMetricsFromRunnerObj(wfr), impl.appStatusConfig.ExposeCDMetrics) + appId := wfr.CdWorkflow.Pipeline.AppId + envId := wfr.CdWorkflow.Pipeline.EnvironmentId + envDeploymentConfig, err := impl.deploymentConfigService.GetDeploymentConfig(appId, envId) + if err != nil { + impl.logger.Errorw("error in fetching environment deployment config by appId and envId", "appId", appId, "envId", envId, "err", err) + return err + } + util2.TriggerCDMetrics(cdWorkflow.GetTriggerMetricsFromRunnerObj(wfr, envDeploymentConfig), impl.appStatusConfig.ExposeCDMetrics) return nil } diff --git a/pkg/app/status/PipelineStatusTimelineService.go b/pkg/app/status/PipelineStatusTimelineService.go index ba94d5a912..b4b24cc724 100644 --- a/pkg/app/status/PipelineStatusTimelineService.go +++ b/pkg/app/status/PipelineStatusTimelineService.go @@ -21,6 +21,7 @@ import ( "fmt" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/timelineStatus" "github.com/devtron-labs/devtron/pkg/app/status/bean" + "github.com/devtron-labs/devtron/pkg/deployment/common" "time" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" @@ -56,6 +57,7 @@ type PipelineStatusTimelineServiceImpl struct { pipelineStatusSyncDetailService PipelineStatusSyncDetailService installedAppRepository repository.InstalledAppRepository installedAppVersionHistory repository.InstalledAppVersionHistoryRepository + deploymentConfigService common.DeploymentConfigService } func NewPipelineStatusTimelineServiceImpl(logger *zap.SugaredLogger, @@ -66,6 +68,7 @@ func NewPipelineStatusTimelineServiceImpl(logger *zap.SugaredLogger, pipelineStatusSyncDetailService PipelineStatusSyncDetailService, installedAppRepository repository.InstalledAppRepository, installedAppVersionHistory repository.InstalledAppVersionHistoryRepository, + deploymentConfigService common.DeploymentConfigService, ) *PipelineStatusTimelineServiceImpl { return &PipelineStatusTimelineServiceImpl{ logger: logger, @@ -76,6 +79,7 @@ func NewPipelineStatusTimelineServiceImpl(logger *zap.SugaredLogger, pipelineStatusSyncDetailService: pipelineStatusSyncDetailService, installedAppRepository: installedAppRepository, installedAppVersionHistory: installedAppVersionHistory, + deploymentConfigService: deploymentConfigService, } } @@ -215,11 +219,19 @@ func (impl *PipelineStatusTimelineServiceImpl) FetchTimelines(appId, envId, wfrI return nil, err } } + deploymentStartedOn = wfr.StartedOn deploymentFinishedOn = wfr.FinishedOn triggeredBy = wfr.TriggeredBy wfrStatus = wfr.Status - deploymentAppType = wfr.CdWorkflow.Pipeline.DeploymentAppType + + envDeploymentConfig, err := impl.deploymentConfigService.GetDeploymentConfig(appId, envId) + if err != nil { + impl.logger.Errorw("error in fetching environment deployment config by appId and envId", "appId", appId, "envId", envId, "err", err) + return nil, err + } + + deploymentAppType = envDeploymentConfig.DeploymentAppType triggeredByUserEmailId, err := impl.userService.GetEmailById(triggeredBy) if err != nil { impl.logger.Errorw("error in getting user email by id", "err", err, "userId", triggeredBy) diff --git a/pkg/appWorkflow/AppWorkflowService.go b/pkg/appWorkflow/AppWorkflowService.go index ba819dd19f..184473c26e 100644 --- a/pkg/appWorkflow/AppWorkflowService.go +++ b/pkg/appWorkflow/AppWorkflowService.go @@ -19,6 +19,7 @@ package appWorkflow import ( "errors" "fmt" + "github.com/devtron-labs/devtron/pkg/deployment/common" util2 "github.com/devtron-labs/devtron/util" "time" @@ -79,6 +80,7 @@ type AppWorkflowServiceImpl struct { enforcerUtil rbac.EnforcerUtil userAuthService user.UserAuthService chartService chart.ChartService + deploymentConfigService common.DeploymentConfigService } type AppWorkflowDto struct { @@ -159,7 +161,9 @@ type PipelineIdentifier struct { func NewAppWorkflowServiceImpl(logger *zap.SugaredLogger, appWorkflowRepository appWorkflow.AppWorkflowRepository, ciCdPipelineOrchestrator pipeline.CiCdPipelineOrchestrator, ciPipelineRepository pipelineConfig.CiPipelineRepository, pipelineRepository pipelineConfig.PipelineRepository, enforcerUtil rbac.EnforcerUtil, resourceGroupService resourceGroup2.ResourceGroupService, - appRepository appRepository.AppRepository, userAuthService user.UserAuthService, chartService chart.ChartService) *AppWorkflowServiceImpl { + appRepository appRepository.AppRepository, userAuthService user.UserAuthService, chartService chart.ChartService, + deploymentConfigService common.DeploymentConfigService, +) *AppWorkflowServiceImpl { return &AppWorkflowServiceImpl{ Logger: logger, appWorkflowRepository: appWorkflowRepository, @@ -171,6 +175,7 @@ func NewAppWorkflowServiceImpl(logger *zap.SugaredLogger, appWorkflowRepository appRepository: appRepository, userAuthService: userAuthService, chartService: chartService, + deploymentConfigService: deploymentConfigService, } } @@ -851,6 +856,13 @@ func (impl AppWorkflowServiceImpl) FindCdPipelinesByAppId(appId int) (*bean.CdPi } for _, pipeline := range dbPipelines { + + envDeploymentConfig, err := impl.deploymentConfigService.GetDeploymentConfig(appId, pipeline.EnvironmentId) + if err != nil { + impl.Logger.Errorw("error in fetching environment deployment config by appId and envId", "appId", appId, "envId", pipeline.EnvironmentId, "err", err) + return nil, err + } + cdPipelineConfigObj := &bean.CDPipelineConfigObject{ Id: pipeline.Id, EnvironmentId: pipeline.EnvironmentId, @@ -858,7 +870,7 @@ func (impl AppWorkflowServiceImpl) FindCdPipelinesByAppId(appId int) (*bean.CdPi CiPipelineId: pipeline.CiPipelineId, TriggerType: pipeline.TriggerType, Name: pipeline.Name, - DeploymentAppType: pipeline.DeploymentAppType, + DeploymentAppType: envDeploymentConfig.DeploymentAppType, AppName: pipeline.DeploymentAppName, AppId: pipeline.AppId, IsGitOpsRepoNotConfigured: !isAppLevelGitOpsConfigured, diff --git a/pkg/chart/ChartService.go b/pkg/chart/ChartService.go index fba5362a5c..cfc9372d1d 100644 --- a/pkg/chart/ChartService.go +++ b/pkg/chart/ChartService.go @@ -29,6 +29,7 @@ import ( chartRepoRepository "github.com/devtron-labs/devtron/pkg/chartRepo/repository" repository4 "github.com/devtron-labs/devtron/pkg/cluster/repository" "github.com/devtron-labs/devtron/pkg/deployment/common" + bean2 "github.com/devtron-labs/devtron/pkg/deployment/common/bean" "github.com/devtron-labs/devtron/pkg/deployment/gitOps/config" "github.com/devtron-labs/devtron/pkg/deployment/manifest/deployedAppMetrics" "github.com/devtron-labs/devtron/pkg/deployment/manifest/deployedAppMetrics/bean" @@ -64,8 +65,7 @@ type ChartService interface { ChartRefAutocompleteForAppOrEnv(appId int, envId int) (*chartRefBean.ChartRefAutocompleteResponse, error) - ConfigureGitOpsRepoUrl(appId int, repoUrl, chartLocation string, isCustomRepo bool, userId int32) error - OverrideGitOpsRepoUrl(appId int, repoUrl string, userId int32) error + ConfigureGitOpsRepoUrlForApp(appId int, repoUrl, chartLocation string, isCustomRepo bool, userId int32) error IsGitOpsRepoConfiguredForDevtronApps(appId int) (bool, error) IsGitOpsRepoAlreadyRegistered(gitOpsRepoUrl string) (bool, error) @@ -211,9 +211,10 @@ func (impl *ChartServiceImpl) Create(templateRequest TemplateRequest, ctx contex if err != nil { return nil, err } - if currentLatestChart.GitRepoUrl != "" { - gitRepoUrl = currentLatestChart.GitRepoUrl - } + + //if currentLatestChart.GitRepoUrl != "" { + // gitRepoUrl = currentLatestChart.GitRepoUrl + //} } // ENDS @@ -280,6 +281,38 @@ func (impl *ChartServiceImpl) Create(templateRequest TemplateRequest, ctx contex return nil, err } + // create deployment config + activeGitOpsConfig, err := impl.gitOpsConfigReadService.GetGitOpsConfigActive() + if err != nil { + impl.logger.Errorw("error in getting active gitOps config", "err", err) + return nil, err + } + + currentDeploymentConfig, err := impl.deploymentConfigService.GetDeploymentConfig(templateRequest.AppId, 0) + if err != nil && err != pg.ErrNoRows { + impl.logger.Errorw("error in getting current deployment config for appId", "appId", templateRequest.AppId, "err", err) + return nil, err + } + + if currentDeploymentConfig != nil && currentDeploymentConfig.Id > 0 { + gitRepoUrl = currentDeploymentConfig.RepoURL + } + + deploymentConfig := &bean2.DeploymentConfig{ + AppId: templateRequest.AppId, + ConfigType: common.GetDeploymentConfigType(templateRequest.IsCustomGitRepository), + RepoURL: gitRepoUrl, + ChartLocation: chartLocation, + CredentialType: bean2.GitOps.String(), + CredentialIdInt: activeGitOpsConfig.Id, + Active: true, + } + deploymentConfig, err = impl.deploymentConfigService.CreateOrUpdateConfig(deploymentConfig, templateRequest.UserId) + if err != nil { + impl.logger.Errorw("error in saving deployment config", "appId", templateRequest.AppId, "err", err) + return nil, err + } + //creating history entry for deployment template err = impl.deploymentTemplateHistoryService.CreateDeploymentTemplateHistoryFromGlobalTemplate(chart, nil, templateRequest.IsAppMetricsEnabled) if err != nil { @@ -305,7 +338,7 @@ func (impl *ChartServiceImpl) Create(templateRequest TemplateRequest, ctx contex return nil, err } - chartVal, err := impl.chartAdaptor(chart, appLevelMetricsUpdateReq.EnableMetrics) + chartVal, err := impl.chartAdaptor(chart, appLevelMetricsUpdateReq.EnableMetrics, deploymentConfig) return chartVal, err } @@ -351,15 +384,39 @@ func (impl *ChartServiceImpl) CreateChartFromEnvOverride(templateRequest Templat if err != nil { return nil, err } - currentLatestChart, err := impl.chartRepository.FindLatestChartForAppByAppId(templateRequest.AppId) - if err != nil && pg.ErrNoRows != err { + + currentDeploymentConfig, err := impl.deploymentConfigService.GetDeploymentConfig(templateRequest.AppId, 0) + if err != nil && err != pg.ErrNoRows { return nil, err } + chartLocation := filepath.Join(templateName, version) gitRepoUrl := apiGitOpsBean.GIT_REPO_NOT_CONFIGURED - if currentLatestChart.Id > 0 && currentLatestChart.GitRepoUrl != "" { - gitRepoUrl = currentLatestChart.GitRepoUrl + if currentDeploymentConfig != nil && currentDeploymentConfig.Id > 0 && len(currentDeploymentConfig.RepoURL) > 0 { + gitRepoUrl = currentDeploymentConfig.RepoURL + } + + // create deployment config + activeGitOpsConfig, err := impl.gitOpsConfigReadService.GetGitOpsConfigActive() + if err != nil { + impl.logger.Errorw("error in getting active gitOps config", "err", err) + return nil, err + } + deploymentConfig := &bean2.DeploymentConfig{ + AppId: templateRequest.AppId, + ConfigType: common.GetDeploymentConfigType(templateRequest.IsCustomGitRepository), + RepoURL: gitRepoUrl, + ChartLocation: chartLocation, + CredentialType: bean2.GitOps.String(), + CredentialIdInt: activeGitOpsConfig.Id, + Active: true, + } + deploymentConfig, err = impl.deploymentConfigService.CreateOrUpdateConfig(deploymentConfig, templateRequest.UserId) + if err != nil { + impl.logger.Errorw("error in saving deployment config", "appId", templateRequest.AppId, "err", err) + return nil, err } + override, err := templateRequest.ValuesOverride.MarshalJSON() if err != nil { return nil, err @@ -393,15 +450,15 @@ func (impl *ChartServiceImpl) CreateChartFromEnvOverride(templateRequest Templat ChartVersion: chartMeta.Version, Status: models.CHARTSTATUS_NEW, Active: true, - ChartLocation: chartLocation, - GitRepoUrl: gitRepoUrl, - ReferenceTemplate: templateName, - ChartRefId: templateRequest.ChartRefId, - Latest: false, - Previous: false, - IsBasicViewLocked: templateRequest.IsBasicViewLocked, - CurrentViewEditor: templateRequest.CurrentViewEditor, - AuditLog: sql.AuditLog{CreatedBy: templateRequest.UserId, CreatedOn: time.Now(), UpdatedOn: time.Now(), UpdatedBy: templateRequest.UserId}, + //ChartLocation: chartLocation, + //GitRepoUrl: gitRepoUrl, + ReferenceTemplate: templateName, + ChartRefId: templateRequest.ChartRefId, + Latest: false, + Previous: false, + IsBasicViewLocked: templateRequest.IsBasicViewLocked, + CurrentViewEditor: templateRequest.CurrentViewEditor, + AuditLog: sql.AuditLog{CreatedBy: templateRequest.UserId, CreatedOn: time.Now(), UpdatedOn: time.Now(), UpdatedBy: templateRequest.UserId}, } err = impl.chartRepository.Save(chart) @@ -409,6 +466,7 @@ func (impl *ChartServiceImpl) CreateChartFromEnvOverride(templateRequest Templat impl.logger.Errorw("error in saving chart ", "chart", chart, "error", err) return nil, err } + //creating history entry for deployment template err = impl.deploymentTemplateHistoryService.CreateDeploymentTemplateHistoryFromGlobalTemplate(chart, nil, appMetrics) if err != nil { @@ -421,18 +479,18 @@ func (impl *ChartServiceImpl) CreateChartFromEnvOverride(templateRequest Templat return nil, err } - chartVal, err := impl.chartAdaptor(chart, false) + chartVal, err := impl.chartAdaptor(chart, false, deploymentConfig) return chartVal, err } // converts db object to bean -func (impl *ChartServiceImpl) chartAdaptor(chart *chartRepoRepository.Chart, isAppMetricsEnabled bool) (*TemplateRequest, error) { +func (impl *ChartServiceImpl) chartAdaptor(chart *chartRepoRepository.Chart, isAppMetricsEnabled bool, deploymentConfig *bean2.DeploymentConfig) (*TemplateRequest, error) { if chart == nil || chart.Id == 0 { return &TemplateRequest{}, &util.ApiError{UserMessage: "no chart found"} } gitRepoUrl := "" - if !apiGitOpsBean.IsGitOpsRepoNotConfigured(chart.GitRepoUrl) { - gitRepoUrl = chart.GitRepoUrl + if !apiGitOpsBean.IsGitOpsRepoNotConfigured(deploymentConfig.RepoURL) { + gitRepoUrl = deploymentConfig.RepoURL } templateRequest := &TemplateRequest{ RefChartTemplate: chart.ReferenceTemplate, @@ -539,12 +597,19 @@ func (impl *ChartServiceImpl) FindLatestChartForAppByAppId(appId int) (chartTemp impl.logger.Errorw("error in fetching chart ", "appId", appId, "err", err) return nil, err } + + deploymentConfig, err := impl.deploymentConfigService.GetDeploymentConfig(appId, 0) + if err != nil { + impl.logger.Errorw("error in fetching deployment config by appId", "appId", appId, "err", err) + return nil, err + } + isAppMetricsEnabled, err := impl.deployedAppMetricsService.GetMetricsFlagByAppId(appId) if err != nil { impl.logger.Errorw("error in fetching app-metrics", "appId", appId, "err", err) return nil, err } - chartTemplate, err = impl.chartAdaptor(chart, isAppMetricsEnabled) + chartTemplate, err = impl.chartAdaptor(chart, isAppMetricsEnabled, deploymentConfig) return chartTemplate, err } @@ -559,7 +624,12 @@ func (impl *ChartServiceImpl) GetByAppIdAndChartRefId(appId int, chartRefId int) impl.logger.Errorw("error in fetching app-metrics", "appId", appId, "err", err) return nil, err } - chartTemplate, err = impl.chartAdaptor(chart, isAppMetricsEnabled) + deploymentConfig, err := impl.deploymentConfigService.GetDeploymentConfig(appId, 0) + if err != nil { + impl.logger.Errorw("error in fetching deployment config by appId", "appId", appId, "err", err) + return nil, err + } + chartTemplate, err = impl.chartAdaptor(chart, isAppMetricsEnabled, deploymentConfig) return chartTemplate, err } @@ -790,7 +860,12 @@ func (impl *ChartServiceImpl) FindPreviousChartByAppId(appId int) (chartTemplate impl.logger.Errorw("error in fetching chart ", "appId", appId, "err", err) return nil, err } - chartTemplate, err = impl.chartAdaptor(chart, false) + deploymentConfig, err := impl.deploymentConfigService.GetDeploymentConfig(appId, 0) + if err != nil { + impl.logger.Errorw("error in fetching deployment config by appId", "appId", appId, "err", err) + return nil, err + } + chartTemplate, err = impl.chartAdaptor(chart, false, deploymentConfig) return chartTemplate, err } @@ -894,70 +969,54 @@ func (impl *ChartServiceImpl) CheckIfChartRefUserUploadedByAppId(id int) (bool, return chartData.UserUploaded, err } -func (impl *ChartServiceImpl) ConfigureGitOpsRepoUrl(appId int, repoUrl, chartLocation string, isCustomRepo bool, userId int32) error { - charts, err := impl.chartRepository.FindActiveChartsByAppId(appId) - if err != nil && !util.IsErrNoRows(err) { - return err - } - tx, err := impl.chartRepository.StartTx() - if err != nil { - impl.logger.Errorw("error in starting transaction to update charts", "error", err) - return err - } - defer impl.chartRepository.RollbackTx(tx) - var updatedCharts []*chartRepoRepository.Chart - for _, ch := range charts { - if apiGitOpsBean.IsGitOpsRepoNotConfigured(ch.GitRepoUrl) { - ch.GitRepoUrl = repoUrl - ch.IsCustomGitRepository = isCustomRepo - ch.ChartLocation = chartLocation - ch.UpdateAuditLog(userId) - updatedCharts = append(updatedCharts, ch) - } - } - err = impl.chartRepository.UpdateAllInTx(tx, updatedCharts) - if err != nil { - return err - } - err = impl.chartRepository.CommitTx(tx) - if err != nil { - impl.logger.Errorw("error in committing transaction to update charts", "error", err) - return err - } - return nil -} +func (impl *ChartServiceImpl) ConfigureGitOpsRepoUrlForApp(appId int, repoUrl, chartLocation string, isCustomRepo bool, userId int32) error { -func (impl *ChartServiceImpl) OverrideGitOpsRepoUrl(appId int, repoUrl string, userId int32) error { - charts, err := impl.chartRepository.FindActiveChartsByAppId(appId) - if err != nil { - return err + deploymentConfig := &bean2.DeploymentConfig{ + AppId: appId, + ConfigType: common.GetDeploymentConfigType(isCustomRepo), + RepoURL: repoUrl, + ChartLocation: chartLocation, + Active: true, } - tx, err := impl.chartRepository.StartTx() + deploymentConfig, err := impl.deploymentConfigService.CreateOrUpdateConfig(deploymentConfig, userId) if err != nil { - impl.logger.Errorw("error in starting transaction to update charts", "error", err) - return err - } - defer impl.chartRepository.RollbackTx(tx) - var updatedCharts []*chartRepoRepository.Chart - for _, ch := range charts { - if !ch.IsCustomGitRepository { - ch.GitRepoUrl = repoUrl - ch.UpdateAuditLog(userId) - updatedCharts = append(updatedCharts, ch) - } - } - err = impl.chartRepository.UpdateAllInTx(tx, updatedCharts) - if err != nil { - return err - } - err = impl.chartRepository.CommitTx(tx) - if err != nil { - impl.logger.Errorw("error in committing transaction to update charts", "error", err) + impl.logger.Errorw("error in saving deployment config for app", "appId", appId, "err", err) return err } return nil } +//func (impl *ChartServiceImpl) OverrideGitOpsRepoUrl(appId int, repoUrl string, userId int32) error { +// charts, err := impl.chartRepository.FindActiveChartsByAppId(appId) +// if err != nil { +// return err +// } +// tx, err := impl.chartRepository.StartTx() +// if err != nil { +// impl.logger.Errorw("error in starting transaction to update charts", "error", err) +// return err +// } +// defer impl.chartRepository.RollbackTx(tx) +// var updatedCharts []*chartRepoRepository.Chart +// for _, ch := range charts { +// if !ch.IsCustomGitRepository { +// ch.GitRepoUrl = repoUrl +// ch.UpdateAuditLog(userId) +// updatedCharts = append(updatedCharts, ch) +// } +// } +// err = impl.chartRepository.UpdateAllInTx(tx, updatedCharts) +// if err != nil { +// return err +// } +// err = impl.chartRepository.CommitTx(tx) +// if err != nil { +// impl.logger.Errorw("error in committing transaction to update charts", "error", err) +// return err +// } +// return nil +//} + func (impl *ChartServiceImpl) IsGitOpsRepoAlreadyRegistered(gitOpsRepoUrl string) (bool, error) { chartModel, err := impl.chartRepository.FindChartByGitRepoUrl(gitOpsRepoUrl) if err != nil && !util.IsErrNoRows(err) { diff --git a/pkg/chart/gitOpsConfig/DevtronAppGitOpsConfigService.go b/pkg/chart/gitOpsConfig/DevtronAppGitOpsConfigService.go index 21f8eb42dc..c5d568d99f 100644 --- a/pkg/chart/gitOpsConfig/DevtronAppGitOpsConfigService.go +++ b/pkg/chart/gitOpsConfig/DevtronAppGitOpsConfigService.go @@ -22,6 +22,7 @@ import ( "github.com/devtron-labs/devtron/client/argocdServer" chartService "github.com/devtron-labs/devtron/pkg/chart" "github.com/devtron-labs/devtron/pkg/chart/gitOpsConfig/bean" + "github.com/devtron-labs/devtron/pkg/deployment/common" commonBean "github.com/devtron-labs/devtron/pkg/deployment/gitOps/common/bean" "github.com/devtron-labs/devtron/pkg/deployment/gitOps/config" "github.com/devtron-labs/devtron/pkg/deployment/gitOps/validation" @@ -48,6 +49,7 @@ type DevtronAppGitOpConfigServiceImpl struct { gitOpsConfigReadService config.GitOpsConfigReadService gitOpsValidationService validation.GitOpsValidationService argoClientWrapperService argocdServer.ArgoClientWrapperService + deploymentConfigService common.DeploymentConfigService } func NewDevtronAppGitOpConfigServiceImpl(logger *zap.SugaredLogger, @@ -55,7 +57,8 @@ func NewDevtronAppGitOpConfigServiceImpl(logger *zap.SugaredLogger, chartService chartService.ChartService, gitOpsConfigReadService config.GitOpsConfigReadService, gitOpsValidationService validation.GitOpsValidationService, - argoClientWrapperService argocdServer.ArgoClientWrapperService) *DevtronAppGitOpConfigServiceImpl { + argoClientWrapperService argocdServer.ArgoClientWrapperService, + deploymentConfigService common.DeploymentConfigService) *DevtronAppGitOpConfigServiceImpl { return &DevtronAppGitOpConfigServiceImpl{ logger: logger, chartRepository: chartRepository, @@ -63,6 +66,7 @@ func NewDevtronAppGitOpConfigServiceImpl(logger *zap.SugaredLogger, gitOpsConfigReadService: gitOpsConfigReadService, gitOpsValidationService: gitOpsValidationService, argoClientWrapperService: argoClientWrapperService, + deploymentConfigService: deploymentConfigService, } } @@ -138,7 +142,7 @@ func (impl *DevtronAppGitOpConfigServiceImpl) SaveAppLevelGitOpsConfiguration(ap return err } isCustomGitOpsRepo := gitOpsConfigurationStatus.AllowCustomRepository && appGitOpsRequest.GitOpsRepoURL != apiGitOpsBean.GIT_REPO_DEFAULT - err = impl.chartService.ConfigureGitOpsRepoUrl(appGitOpsRequest.AppId, chartGitAttr.RepoUrl, chartGitAttr.ChartLocation, isCustomGitOpsRepo, appGitOpsRequest.UserId) + err = impl.chartService.ConfigureGitOpsRepoUrlForApp(appGitOpsRequest.AppId, chartGitAttr.RepoUrl, chartGitAttr.ChartLocation, isCustomGitOpsRepo, appGitOpsRequest.UserId) if err != nil { impl.logger.Errorw("error in updating git repo url in charts", "err", err) return err @@ -192,10 +196,10 @@ func (impl *DevtronAppGitOpConfigServiceImpl) GetAppLevelGitOpsConfiguration(app } func (impl *DevtronAppGitOpConfigServiceImpl) isGitRepoUrlPresent(appId int) bool { - fetchedChart, err := impl.chartRepository.FindLatestChartForAppByAppId(appId) + deploymentConfig, err := impl.deploymentConfigService.GetDeploymentConfig(appId, 0) if err != nil { - impl.logger.Errorw("error fetching git repo url from the latest chart") + impl.logger.Errorw("error fetching git repo url from deploymentConfig for latest chart") return false } - return !apiGitOpsBean.IsGitOpsRepoNotConfigured(fetchedChart.GitRepoUrl) + return !apiGitOpsBean.IsGitOpsRepoNotConfigured(deploymentConfig.RepoURL) } diff --git a/pkg/deployment/common/bean/bean.go b/pkg/deployment/common/bean/bean.go index 677a2efb7a..af6ad8404d 100644 --- a/pkg/deployment/common/bean/bean.go +++ b/pkg/deployment/common/bean/bean.go @@ -22,8 +22,16 @@ const ( SYSTEM_GENERATED DeploymentConfigType = "system_generated" ) +func (d DeploymentConfigType) String() string { + return string(d) +} + type DeploymentConfigCredentialType string const ( GitOps DeploymentConfigCredentialType = "gitOps" ) + +func (d DeploymentConfigCredentialType) String() string { + return string(d) +} diff --git a/pkg/deployment/common/deploymentConfigService.go b/pkg/deployment/common/deploymentConfigService.go index e848f3ed64..87ff407f24 100644 --- a/pkg/deployment/common/deploymentConfigService.go +++ b/pkg/deployment/common/deploymentConfigService.go @@ -71,67 +71,100 @@ func (impl *DeploymentConfigServiceImpl) CreateOrUpdateConfig(config *bean.Deplo } func (impl *DeploymentConfigServiceImpl) GetDeploymentConfig(appId, envId int) (*bean.DeploymentConfig, error) { - configDbObj, err := impl.GetConfigDBObj(appId, envId) - if err != nil && err != pg.ErrNoRows { - impl.logger.Errorw("error in fetching deployment config from DB by appId and envId", - "appId", appId, "envId", envId, "err", err) + + appLevelConfigDbObj, err := impl.deploymentConfigRepository.GetAppLevelConfig(appId) + if err != nil { + impl.logger.Errorw("error in getiting deployment config db object by appId", "appId", appId, "err", err) + return nil, err } - if err == pg.ErrNoRows || (configDbObj == nil || (configDbObj != nil && configDbObj.Id == 0)) { - configDbObj, err = impl.migrateOldDataToDeploymentConfig(appId, envId) + if err == pg.ErrNoRows { + appLevelConfigDbObj, err = impl.migrateAppLevelDataTODeploymentConfig(appId) if err != nil { - impl.logger.Errorw("error in migrating old data to deployment config", "appId", appId, "envId", envId, "err", err) + impl.logger.Errorw("error in migrating app level config to deployment config", "appId", appId, "err", err) return nil, err } } - return ConvertDeploymentConfigDbObjToDTO(configDbObj), nil -} -func (impl *DeploymentConfigServiceImpl) migrateOldDataToDeploymentConfig(appId int, envId int) (*deploymentConfig.DeploymentConfig, error) { + if envId > 0 { + appAndEnvLevelConfig, err := impl.deploymentConfigRepository.GetByAppIdAndEnvId(appId, envId) + if err != nil { + impl.logger.Errorw("error in getiting deployment config db object by appId and envId", "appId", appId, "envId", envId, "err", err) + return nil, err + } + if err == pg.ErrNoRows { + appAndEnvLevelConfig, err = impl.migrateAppAndEnvLevelDataToDeploymentConfig(appLevelConfigDbObj, appId, envId) + if err != nil { + impl.logger.Errorw("error in migrating app level config to deployment config", "appId", appId, "err", err) + return nil, err + } + } + return ConvertDeploymentConfigDbObjToDTO(appAndEnvLevelConfig), nil + } + + return ConvertDeploymentConfigDbObjToDTO(appLevelConfigDbObj), nil +} +func (impl *DeploymentConfigServiceImpl) migrateAppLevelDataTODeploymentConfig(appId int) (*deploymentConfig.DeploymentConfig, error) { chart, err := impl.chartRepository.FindLatestChartForAppByAppId(appId) if err != nil { impl.logger.Errorw("error in fetch chart for git repo migration by appId", "appId", appId, "err", err) return nil, err } - var deploymentAppType string - if envId > 0 { - deploymentAppType, err = impl.pipelineRepository.FindDeploymentAppTypeByAppIdAndEnvId(appId, envId) - if err != nil { - impl.logger.Errorw("error in getting deployment app type by appId and envId", "appId", appId, "envId", envId, "err", err) - return nil, err - } + ConfigDbObj := &deploymentConfig.DeploymentConfig{ + ConfigType: GetDeploymentConfigType(chart.IsCustomGitRepository), + AppId: appId, + Active: true, + RepoUrl: chart.GitRepoUrl, + ChartLocation: chart.ChartLocation, + } + ConfigDbObj.AuditLog.CreateAuditLog(1) + ConfigDbObj, err = impl.deploymentConfigRepository.Save(ConfigDbObj) + if err != nil { + impl.logger.Errorw("error in saving deployment config in DB", "appId", appId, "err", err) + return nil, err } + return ConfigDbObj, nil +} - ConfigDbObj := &deploymentConfig.DeploymentConfig{ - AppId: appId, - EnvironmentId: envId, - DeploymentAppType: deploymentAppType, - Active: true, +func (impl *DeploymentConfigServiceImpl) migrateAppAndEnvLevelDataToDeploymentConfig(appLevelConfig *deploymentConfig.DeploymentConfig, appId int, envId int) (*deploymentConfig.DeploymentConfig, error) { + + configDbObj := &deploymentConfig.DeploymentConfig{ + AppId: appId, + EnvironmentId: envId, + ConfigType: appLevelConfig.ConfigType, + RepoUrl: appLevelConfig.RepoUrl, + ChartLocation: appLevelConfig.ChartLocation, + Active: true, } - switch deploymentAppType { + deploymentAppType, err := impl.pipelineRepository.FindDeploymentAppTypeByAppIdAndEnvId(appId, envId) + if err != nil { + impl.logger.Errorw("error in getting deployment app type by appId and envId", "appId", appId, "envId", envId, "err", err) + return nil, err + } + + configDbObj.DeploymentAppType = deploymentAppType + + switch configDbObj.DeploymentAppType { //TODO: handling for other deployment app type in future case bean2.ArgoCd: - ConfigDbObj.ConfigType = GetDeploymentConfigType(chart.IsCustomGitRepository) - ConfigDbObj.RepoUrl = chart.GitRepoUrl - ConfigDbObj.ChartLocation = chart.ChartLocation - ConfigDbObj.CredentialType = string(bean.GitOps) - gitOpsConfig, err := impl.gitOpsConfigReadService.GetGitOpsProviderByRepoURL(ConfigDbObj.RepoUrl) + configDbObj.CredentialType = bean.GitOps.String() + gitOpsConfig, err := impl.gitOpsConfigReadService.GetGitOpsProviderByRepoURL(configDbObj.RepoUrl) if err != nil { - impl.logger.Infow("error in fetching gitOps config by repoUrl, skipping migration to deployment config", "repoURL", ConfigDbObj.RepoUrl) - return ConfigDbObj, nil + impl.logger.Infow("error in fetching gitOps config by repoUrl, skipping migration to deployment config", "repoURL", configDbObj.RepoUrl) + return configDbObj, nil } - ConfigDbObj.CredentialIdInt = gitOpsConfig.Id + configDbObj.CredentialIdInt = gitOpsConfig.Id } - ConfigDbObj.AuditLog.CreateAuditLog(bean3.SYSTEM_USER_ID) - ConfigDbObj, err = impl.deploymentConfigRepository.Save(ConfigDbObj) + configDbObj.AuditLog.CreateAuditLog(bean3.SYSTEM_USER_ID) + configDbObj, err = impl.deploymentConfigRepository.Save(configDbObj) if err != nil { impl.logger.Errorw("error in saving deployment config in DB", "appId", appId, "envId", envId, "err", err) return nil, err } - return ConfigDbObj, nil + return configDbObj, nil } func (impl *DeploymentConfigServiceImpl) GetConfigDBObj(appId, envId int) (*deploymentConfig.DeploymentConfig, error) { diff --git a/pkg/deployment/common/helper.go b/pkg/deployment/common/helper.go index 1ad3b12ff1..71667ef6c6 100644 --- a/pkg/deployment/common/helper.go +++ b/pkg/deployment/common/helper.go @@ -8,3 +8,7 @@ func GetDeploymentConfigType(isCustomGitOpsRepo bool) string { } return string(bean.SYSTEM_GENERATED) } + +func IsCustomGitOpsRepo(deploymentConfigType string) bool { + return deploymentConfigType == bean.CUSTOM.String() +} diff --git a/pkg/deployment/manifest/ManifestCreationService.go b/pkg/deployment/manifest/ManifestCreationService.go index 763447b26f..92c2952706 100644 --- a/pkg/deployment/manifest/ManifestCreationService.go +++ b/pkg/deployment/manifest/ManifestCreationService.go @@ -34,6 +34,7 @@ import ( bean2 "github.com/devtron-labs/devtron/pkg/bean" chartRepoRepository "github.com/devtron-labs/devtron/pkg/chartRepo/repository" repository2 "github.com/devtron-labs/devtron/pkg/cluster/repository" + "github.com/devtron-labs/devtron/pkg/deployment/common" bean3 "github.com/devtron-labs/devtron/pkg/deployment/manifest/bean" "github.com/devtron-labs/devtron/pkg/deployment/manifest/deployedAppMetrics" "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate" @@ -93,6 +94,7 @@ type ManifestCreationServiceImpl struct { strategyHistoryRepository repository3.PipelineStrategyHistoryRepository pipelineConfigRepository chartConfig.PipelineConfigRepository deploymentTemplateHistoryRepository repository3.DeploymentTemplateHistoryRepository + deploymentConfigService common.DeploymentConfigService } func NewManifestCreationServiceImpl(logger *zap.SugaredLogger, @@ -116,7 +118,8 @@ func NewManifestCreationServiceImpl(logger *zap.SugaredLogger, pipelineOverrideRepository chartConfig.PipelineOverrideRepository, strategyHistoryRepository repository3.PipelineStrategyHistoryRepository, pipelineConfigRepository chartConfig.PipelineConfigRepository, - deploymentTemplateHistoryRepository repository3.DeploymentTemplateHistoryRepository) *ManifestCreationServiceImpl { + deploymentTemplateHistoryRepository repository3.DeploymentTemplateHistoryRepository, + deploymentConfigService common.DeploymentConfigService) *ManifestCreationServiceImpl { return &ManifestCreationServiceImpl{ logger: logger, dockerRegistryIpsConfigService: dockerRegistryIpsConfigService, @@ -140,6 +143,7 @@ func NewManifestCreationServiceImpl(logger *zap.SugaredLogger, strategyHistoryRepository: strategyHistoryRepository, pipelineConfigRepository: pipelineConfigRepository, deploymentTemplateHistoryRepository: deploymentTemplateHistoryRepository, + deploymentConfigService: deploymentConfigService, } } @@ -218,6 +222,14 @@ func (impl *ManifestCreationServiceImpl) GetValuesOverrideForTrigger(overrideReq } valuesOverrideResponse.EnvOverride = envOverride + envDeploymentConfig, err := impl.deploymentConfigService.GetDeploymentConfig(pipeline.AppId, pipeline.EnvironmentId) + if err != nil { + impl.logger.Errorw("error in getting deployment config by appId and envId", "appId", pipeline.AppId, "envId", pipeline.EnvironmentId, "err", err) + return valuesOverrideResponse, err + } + + valuesOverrideResponse.DeploymentConfig = envDeploymentConfig + // Conditional Block based on PipelineOverrideCreated --> start if !isPipelineOverrideCreated { pipelineOverride, err = impl.savePipelineOverride(newCtx, overrideRequest, envOverride.Id, triggeredAt) diff --git a/pkg/deployment/manifest/publish/ManifestPushService.go b/pkg/deployment/manifest/publish/ManifestPushService.go index b5c10610a0..62d8fe8caa 100644 --- a/pkg/deployment/manifest/publish/ManifestPushService.go +++ b/pkg/deployment/manifest/publish/ManifestPushService.go @@ -144,7 +144,7 @@ func (impl *GitOpsManifestPushServiceImpl) PushChart(ctx context.Context, manife manifestPushTemplate.RepoUrl = newGitRepoUrl manifestPushResponse.NewGitRepoUrl = newGitRepoUrl // below function will override gitRepoUrl for charts even if user has already configured gitOps repoURL - err = impl.chartService.OverrideGitOpsRepoUrl(manifestPushTemplate.AppId, newGitRepoUrl, manifestPushTemplate.UserId) + err = impl.chartService.ConfigureGitOpsRepoUrlForApp(manifestPushTemplate.AppId, newGitRepoUrl, manifestPushTemplate.ChartLocation, manifestPushTemplate.IsCustomGitRepository, manifestPushTemplate.UserId) if err != nil { impl.logger.Errorw("error in updating git repo url in charts", "err", err) manifestPushResponse.Error = fmt.Errorf("No repository configured for Gitops! Error while migrating gitops repository: '%s'", newGitRepoUrl) diff --git a/pkg/deployment/trigger/devtronApps/TriggerService.go b/pkg/deployment/trigger/devtronApps/TriggerService.go index 1eb922c8d6..07e3ee0420 100644 --- a/pkg/deployment/trigger/devtronApps/TriggerService.go +++ b/pkg/deployment/trigger/devtronApps/TriggerService.go @@ -50,6 +50,8 @@ import ( bean2 "github.com/devtron-labs/devtron/pkg/bean" chartRepoRepository "github.com/devtron-labs/devtron/pkg/chartRepo/repository" repository2 "github.com/devtron-labs/devtron/pkg/cluster/repository" + "github.com/devtron-labs/devtron/pkg/deployment/common" + bean9 "github.com/devtron-labs/devtron/pkg/deployment/common/bean" "github.com/devtron-labs/devtron/pkg/deployment/gitOps/config" "github.com/devtron-labs/devtron/pkg/deployment/manifest" bean5 "github.com/devtron-labs/devtron/pkg/deployment/manifest/deploymentTemplate/chartRef/bean" @@ -157,6 +159,7 @@ type TriggerServiceImpl struct { dockerArtifactStoreRepository repository4.DockerArtifactStoreRepository K8sUtil *util5.K8sServiceImpl transactionUtilImpl *sql.TransactionUtilImpl + deploymentConfigService common.DeploymentConfigService } func NewTriggerServiceImpl(logger *zap.SugaredLogger, @@ -209,7 +212,8 @@ func NewTriggerServiceImpl(logger *zap.SugaredLogger, dockerArtifactStoreRepository repository4.DockerArtifactStoreRepository, imageScanService security2.ImageScanService, K8sUtil *util5.K8sServiceImpl, - transactionUtilImpl *sql.TransactionUtilImpl) (*TriggerServiceImpl, error) { + transactionUtilImpl *sql.TransactionUtilImpl, + deploymentConfigService common.DeploymentConfigService) (*TriggerServiceImpl, error) { impl := &TriggerServiceImpl{ logger: logger, cdWorkflowCommonService: cdWorkflowCommonService, @@ -262,6 +266,7 @@ func NewTriggerServiceImpl(logger *zap.SugaredLogger, imageScanService: imageScanService, K8sUtil: K8sUtil, transactionUtilImpl: transactionUtilImpl, + deploymentConfigService: deploymentConfigService, } config, err := types.GetCdConfig() if err != nil { @@ -363,7 +368,12 @@ func (impl *TriggerServiceImpl) ManualCdTrigger(triggerContext bean.TriggerConte if err != nil { return 0, err } - adapter.SetPipelineFieldsInOverrideRequest(overrideRequest, cdPipeline) + envDeploymentConfig, err := impl.deploymentConfigService.GetDeploymentConfig(cdPipeline.AppId, cdPipeline.EnvironmentId) + if err != nil { + impl.logger.Errorw("error in fetching environment deployment config by appId and envId", "appId", cdPipeline.AppId, "envId", cdPipeline.EnvironmentId, "err", err) + return 0, err + } + adapter.SetPipelineFieldsInOverrideRequest(overrideRequest, cdPipeline, envDeploymentConfig) switch overrideRequest.CdWorkflowType { case bean3.CD_WORKFLOW_TYPE_PRE: @@ -664,7 +674,12 @@ func (impl *TriggerServiceImpl) releasePipeline(ctx context.Context, pipeline *p DeploymentWithConfig: bean3.DEPLOYMENT_CONFIG_TYPE_LAST_SAVED, WfrId: wfrId, } - adapter.SetPipelineFieldsInOverrideRequest(request, pipeline) + envDeploymentConfig, err := impl.deploymentConfigService.GetDeploymentConfig(pipeline.AppId, pipeline.EnvironmentId) + if err != nil { + impl.logger.Errorw("error in fetching environment deployment config by appId and envId", "appId", pipeline.AppId, "envId", pipeline.EnvironmentId, "err", err) + return err + } + adapter.SetPipelineFieldsInOverrideRequest(request, pipeline, envDeploymentConfig) releaseCtx, err := impl.argoUserService.GetACDContext(ctx) if err != nil { @@ -811,7 +826,7 @@ func (impl *TriggerServiceImpl) performGitOps(ctx context.Context, } if manifestPushResponse.IsNewGitRepoConfigured() { // Update GitOps repo url after repo new repo created - valuesOverrideResponse.EnvOverride.Chart.GitRepoUrl = manifestPushResponse.NewGitRepoUrl + valuesOverrideResponse.DeploymentConfig.RepoURL = manifestPushResponse.NewGitRepoUrl } return nil } @@ -916,9 +931,9 @@ func (impl *TriggerServiceImpl) buildManifestPushTemplate(overrideRequest *bean3 manifestPushTemplate.ChartReferenceTemplate = valuesOverrideResponse.EnvOverride.Chart.ReferenceTemplate manifestPushTemplate.ChartName = valuesOverrideResponse.EnvOverride.Chart.ChartName manifestPushTemplate.ChartVersion = valuesOverrideResponse.EnvOverride.Chart.ChartVersion - manifestPushTemplate.ChartLocation = valuesOverrideResponse.EnvOverride.Chart.ChartLocation - manifestPushTemplate.RepoUrl = valuesOverrideResponse.EnvOverride.Chart.GitRepoUrl - manifestPushTemplate.IsCustomGitRepository = valuesOverrideResponse.EnvOverride.Chart.IsCustomGitRepository + manifestPushTemplate.ChartLocation = valuesOverrideResponse.DeploymentConfig.ChartLocation + manifestPushTemplate.RepoUrl = valuesOverrideResponse.DeploymentConfig.RepoURL + manifestPushTemplate.IsCustomGitRepository = common.IsCustomGitOpsRepo(valuesOverrideResponse.DeploymentConfig.ConfigType) } return manifestPushTemplate, err } @@ -978,7 +993,7 @@ func (impl *TriggerServiceImpl) createHelmAppForCdPipeline(ctx context.Context, referenceTemplate := envOverride.Chart.ReferenceTemplate referenceTemplatePath := path.Join(bean5.RefChartDirPath, referenceTemplate) - if util.IsHelmApp(pipelineModel.DeploymentAppType) { + if util.IsHelmApp(valuesOverrideResponse.DeploymentConfig.DeploymentAppType) { var sanitizedK8sVersion string //handle specific case for all cronjob charts from cronjob-chart_1-2-0 to cronjob-chart_1-5-0 where semverCompare //comparison func has wrong api version mentioned, so for already installed charts via these charts that comparison @@ -1126,7 +1141,7 @@ func (impl *TriggerServiceImpl) deployArgoCdApp(ctx context.Context, overrideReq } impl.logger.Debugw("ArgoCd application created", "name", name) - updateAppInArgoCd, err := impl.updateArgoPipeline(newCtx, valuesOverrideResponse.Pipeline, valuesOverrideResponse.EnvOverride) + updateAppInArgoCd, err := impl.updateArgoPipeline(newCtx, valuesOverrideResponse.Pipeline, valuesOverrideResponse.EnvOverride, valuesOverrideResponse.DeploymentConfig) if err != nil { impl.logger.Errorw("error in updating argocd app ", "err", err) return err @@ -1159,7 +1174,7 @@ func (impl *TriggerServiceImpl) deployArgoCdApp(ctx context.Context, overrideReq } // update repoUrl, revision and argo app sync mode (auto/manual) if needed -func (impl *TriggerServiceImpl) updateArgoPipeline(ctx context.Context, pipeline *pipelineConfig.Pipeline, envOverride *chartConfig.EnvConfigOverride) (bool, error) { +func (impl *TriggerServiceImpl) updateArgoPipeline(ctx context.Context, pipeline *pipelineConfig.Pipeline, envOverride *chartConfig.EnvConfigOverride, deploymentConfig *bean9.DeploymentConfig) (bool, error) { if ctx == nil { impl.logger.Errorw("err in syncing ACD, ctx is NULL", "pipelineName", pipeline.Name) return false, nil @@ -1177,11 +1192,11 @@ func (impl *TriggerServiceImpl) updateArgoPipeline(ctx context.Context, pipeline appStatus, _ := status2.FromError(err) if appStatus.Code() == codes.OK { impl.logger.Debugw("argo app exists", "app", argoAppName, "pipeline", pipeline.Name) - if impl.argoClientWrapperService.IsArgoAppPatchRequired(argoApplication.Spec.Source, envOverride.Chart.GitRepoUrl, envOverride.Chart.ChartLocation) { + if impl.argoClientWrapperService.IsArgoAppPatchRequired(argoApplication.Spec.Source, deploymentConfig.RepoURL, envOverride.Chart.ChartLocation) { patchRequestDto := &bean7.ArgoCdAppPatchReqDto{ ArgoAppName: argoAppName, - ChartLocation: envOverride.Chart.ChartLocation, - GitRepoUrl: envOverride.Chart.GitRepoUrl, + ChartLocation: deploymentConfig.ChartLocation, + GitRepoUrl: deploymentConfig.RepoURL, TargetRevision: bean7.TargetRevisionMaster, PatchType: bean7.PatchTypeMerge, } @@ -1190,7 +1205,7 @@ func (impl *TriggerServiceImpl) updateArgoPipeline(ctx context.Context, pipeline impl.logger.Errorw("error in patching argo pipeline", "err", err, "req", patchRequestDto) return false, err } - if envOverride.Chart.GitRepoUrl != argoApplication.Spec.Source.RepoURL { + if deploymentConfig.RepoURL != argoApplication.Spec.Source.RepoURL { impl.logger.Infow("patching argo application's repo url", "argoAppName", argoAppName) } impl.logger.Debugw("pipeline update req", "res", patchRequestDto) @@ -1443,12 +1458,17 @@ func (impl *TriggerServiceImpl) handleCustomGitOpsRepoValidation(runner *pipelin isGitOpsConfigured = true } if isGitOpsConfigured && gitOpsConfig.AllowCustomRepository { - chart, err := impl.chartRepository.FindLatestChartForAppByAppId(pipeline.AppId) + //chart, err := impl.chartRepository.FindLatestChartForAppByAppId(pipeline.AppId) + //if err != nil { + // impl.logger.Errorw("error in fetching latest chart for app by appId", "err", err, "appId", pipeline.AppId) + // return err + //} + deploymentConfig, err := impl.deploymentConfigService.GetDeploymentConfig(pipeline.AppId, pipeline.EnvironmentId) if err != nil { - impl.logger.Errorw("error in fetching latest chart for app by appId", "err", err, "appId", pipeline.AppId) + impl.logger.Errorw("error in getting deployment config by appId and envId", "appId", pipeline.AppId, "envId", pipeline.EnvironmentId, "err", err) return err } - if gitOps.IsGitOpsRepoNotConfigured(chart.GitRepoUrl) { + if gitOps.IsGitOpsRepoNotConfigured(deploymentConfig.RepoURL) { if err = impl.cdWorkflowCommonService.MarkCurrentDeploymentFailed(runner, errors.New(pipelineConfig.GITOPS_REPO_NOT_CONFIGURED), triggeredBy); err != nil { impl.logger.Errorw("error while updating current runner status to failed, TriggerDeployment", "wfrId", runner.Id, "err", err) } diff --git a/pkg/deployment/trigger/devtronApps/adapter/adapter.go b/pkg/deployment/trigger/devtronApps/adapter/adapter.go index f67eaa40e2..f2e25302e3 100644 --- a/pkg/deployment/trigger/devtronApps/adapter/adapter.go +++ b/pkg/deployment/trigger/devtronApps/adapter/adapter.go @@ -20,12 +20,13 @@ import ( apiBean "github.com/devtron-labs/devtron/api/bean" helmBean "github.com/devtron-labs/devtron/api/helm-app/service/bean" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" + bean2 "github.com/devtron-labs/devtron/pkg/deployment/common/bean" "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/bean" eventProcessorBean "github.com/devtron-labs/devtron/pkg/eventProcessor/bean" "time" ) -func SetPipelineFieldsInOverrideRequest(overrideRequest *apiBean.ValuesOverrideRequest, pipeline *pipelineConfig.Pipeline) { +func SetPipelineFieldsInOverrideRequest(overrideRequest *apiBean.ValuesOverrideRequest, pipeline *pipelineConfig.Pipeline, deploymentConfig *bean2.DeploymentConfig) { overrideRequest.PipelineId = pipeline.Id overrideRequest.PipelineName = pipeline.Name overrideRequest.EnvId = pipeline.EnvironmentId @@ -33,7 +34,7 @@ func SetPipelineFieldsInOverrideRequest(overrideRequest *apiBean.ValuesOverrideR overrideRequest.ClusterId = pipeline.Environment.ClusterId overrideRequest.AppId = pipeline.AppId overrideRequest.AppName = pipeline.App.AppName - overrideRequest.DeploymentAppType = pipeline.DeploymentAppType + overrideRequest.DeploymentAppType = deploymentConfig.DeploymentAppType overrideRequest.Namespace = pipeline.Environment.Namespace overrideRequest.ReleaseName = pipeline.DeploymentAppName } diff --git a/pkg/eventProcessor/in/WorkflowEventProcessorService.go b/pkg/eventProcessor/in/WorkflowEventProcessorService.go index 673c24fee5..7ea34cb9d4 100644 --- a/pkg/eventProcessor/in/WorkflowEventProcessorService.go +++ b/pkg/eventProcessor/in/WorkflowEventProcessorService.go @@ -33,6 +33,7 @@ import ( util3 "github.com/devtron-labs/devtron/internal/util" "github.com/devtron-labs/devtron/pkg/app" userBean "github.com/devtron-labs/devtron/pkg/auth/user/bean" + "github.com/devtron-labs/devtron/pkg/deployment/common" "github.com/devtron-labs/devtron/pkg/deployment/deployedApp" deploymentBean "github.com/devtron-labs/devtron/pkg/deployment/deployedApp/bean" "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps" @@ -88,9 +89,10 @@ type WorkflowEventProcessorImpl struct { appServiceConfig *app.AppServiceConfig // repositories import to be removed - pipelineRepository pipelineConfig.PipelineRepository - ciArtifactRepository repository.CiArtifactRepository - cdWorkflowRepository pipelineConfig.CdWorkflowRepository + pipelineRepository pipelineConfig.PipelineRepository + ciArtifactRepository repository.CiArtifactRepository + cdWorkflowRepository pipelineConfig.CdWorkflowRepository + deploymentConfigService common.DeploymentConfigService } func NewWorkflowEventProcessorImpl(logger *zap.SugaredLogger, @@ -111,7 +113,8 @@ func NewWorkflowEventProcessorImpl(logger *zap.SugaredLogger, userDeploymentRequestService service.UserDeploymentRequestService, pipelineRepository pipelineConfig.PipelineRepository, ciArtifactRepository repository.CiArtifactRepository, - cdWorkflowRepository pipelineConfig.CdWorkflowRepository) (*WorkflowEventProcessorImpl, error) { + cdWorkflowRepository pipelineConfig.CdWorkflowRepository, + deploymentConfigService common.DeploymentConfigService) (*WorkflowEventProcessorImpl, error) { impl := &WorkflowEventProcessorImpl{ logger: logger, pubSubClient: pubSubClient, @@ -136,6 +139,7 @@ func NewWorkflowEventProcessorImpl(logger *zap.SugaredLogger, pipelineRepository: pipelineRepository, ciArtifactRepository: ciArtifactRepository, cdWorkflowRepository: cdWorkflowRepository, + deploymentConfigService: deploymentConfigService, } appServiceConfig, err := app.GetAppServiceConfig() if err != nil { @@ -753,7 +757,12 @@ func (impl *WorkflowEventProcessorImpl) SubscribeCDPipelineDeleteEvent() error { impl.logger.Errorw("error in fetching pipeline by pipelineId", "err", err, "pipelineId", cdPipelineDeleteEvent.PipelineId) return } - if util3.IsHelmApp(pipeline.DeploymentAppType) || util3.IsAcdApp(pipeline.DeploymentAppType) { + envDeploymentConfig, err := impl.deploymentConfigService.GetDeploymentConfig(pipeline.AppId, pipeline.EnvironmentId) + if err != nil { + impl.logger.Errorw("error in fetching environment deployment config by appId and envId", "appId", pipeline.AppId, "envId", pipeline.EnvironmentId, "err", err) + return + } + if util3.IsHelmApp(envDeploymentConfig.DeploymentAppType) || util3.IsAcdApp(envDeploymentConfig.DeploymentAppType) { impl.RemoveReleaseContextForPipeline(cdPipelineDeleteEvent.PipelineId, cdPipelineDeleteEvent.TriggeredBy) // there is a possibility that when the pipeline was deleted, async request nats message was not consumed completely and could have led to dangling deployment app // trying to delete deployment app once @@ -1026,7 +1035,12 @@ func (impl *WorkflowEventProcessorImpl) setAdditionalDataInAsyncInstallReq(ctx c if err != nil { return err } - triggerAdapter.SetPipelineFieldsInOverrideRequest(cdAsyncInstallReq.ValuesOverrideRequest, pipelineModel) + envDeploymentConfig, err := impl.deploymentConfigService.GetDeploymentConfig(pipelineModel.AppId, pipelineModel.EnvironmentId) + if err != nil { + impl.logger.Errorw("error in fetching environment deployment config by appId and envId", "appId", pipelineModel.AppId, "envId", pipelineModel.EnvironmentId, "err", err) + return err + } + triggerAdapter.SetPipelineFieldsInOverrideRequest(cdAsyncInstallReq.ValuesOverrideRequest, pipelineModel, envDeploymentConfig) if cdAsyncInstallReq.ValuesOverrideRequest.DeploymentType == models.DEPLOYMENTTYPE_UNKNOWN { cdAsyncInstallReq.ValuesOverrideRequest.DeploymentType = models.DEPLOYMENTTYPE_DEPLOY } diff --git a/pkg/pipeline/AppDeploymentTypeChangeManager.go b/pkg/pipeline/AppDeploymentTypeChangeManager.go index 6dd9f55a5b..daebb3d358 100644 --- a/pkg/pipeline/AppDeploymentTypeChangeManager.go +++ b/pkg/pipeline/AppDeploymentTypeChangeManager.go @@ -31,6 +31,8 @@ import ( app2 "github.com/devtron-labs/devtron/pkg/app" "github.com/devtron-labs/devtron/pkg/bean" chartService "github.com/devtron-labs/devtron/pkg/chart" + "github.com/devtron-labs/devtron/pkg/deployment/common" + bean4 "github.com/devtron-labs/devtron/pkg/deployment/common/bean" commonBean "github.com/devtron-labs/devtron/pkg/deployment/gitOps/common/bean" "github.com/devtron-labs/devtron/pkg/deployment/gitOps/config" bean3 "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/bean" @@ -75,6 +77,7 @@ type AppDeploymentTypeChangeManagerImpl struct { gitOpsConfigReadService config.GitOpsConfigReadService chartService chartService.ChartService workflowEventPublishService out.WorkflowEventPublishService + deploymentConfigService common.DeploymentConfigService } func NewAppDeploymentTypeChangeManagerImpl( @@ -88,7 +91,8 @@ func NewAppDeploymentTypeChangeManagerImpl( cdPipelineConfigService CdPipelineConfigService, gitOpsConfigReadService config.GitOpsConfigReadService, chartService chartService.ChartService, - workflowEventPublishService out.WorkflowEventPublishService) *AppDeploymentTypeChangeManagerImpl { + workflowEventPublishService out.WorkflowEventPublishService, + deploymentConfigService common.DeploymentConfigService) *AppDeploymentTypeChangeManagerImpl { return &AppDeploymentTypeChangeManagerImpl{ logger: logger, pipelineRepository: pipelineRepository, @@ -101,6 +105,7 @@ func NewAppDeploymentTypeChangeManagerImpl( gitOpsConfigReadService: gitOpsConfigReadService, chartService: chartService, workflowEventPublishService: workflowEventPublishService, + deploymentConfigService: deploymentConfigService, } } @@ -413,6 +418,12 @@ func (impl *AppDeploymentTypeChangeManagerImpl) DeleteDeploymentApps(ctx context // Iterate over all the pipelines in the environment for given deployment app type for _, pipeline := range pipelines { + envDeploymentConfig, err := impl.deploymentConfigService.GetDeploymentConfig(pipeline.AppId, pipeline.EnvironmentId) + if err != nil { + impl.logger.Errorw("error in fetching environment deployment config by appId and envId", "appId", pipeline.AppId, "envId", pipeline.EnvironmentId, "err", err) + continue + } + var isValid bool // check if pipeline info like app name and environment is empty or not if failedPipelines, isValid = impl.isPipelineInfoValid(pipeline, failedPipelines); !isValid { @@ -421,17 +432,15 @@ func (impl *AppDeploymentTypeChangeManagerImpl) DeleteDeploymentApps(ctx context var healthChkErr error // check health of the app if it is argocd deployment type - if _, healthChkErr = impl.handleNotDeployedAppsIfArgoDeploymentType(pipeline, failedPipelines); healthChkErr != nil { + if _, healthChkErr = impl.handleNotDeployedAppsIfArgoDeploymentType(pipeline, failedPipelines, envDeploymentConfig); healthChkErr != nil { // cannot delete unhealthy app continue } deploymentAppName := fmt.Sprintf("%s-%s", pipeline.App.AppName, pipeline.Environment.Name) - var err error - // delete request - if pipeline.DeploymentAppType == bean3.ArgoCd { + if envDeploymentConfig.DeploymentAppType == bean3.ArgoCd { err = impl.deleteArgoCdApp(ctx, pipeline, deploymentAppName, true) } else { @@ -463,10 +472,26 @@ func (impl *AppDeploymentTypeChangeManagerImpl) DeleteDeploymentApps(ctx context impl.logger.Errorw("error in registering acd app", "err", AcdRegisterErr) } if AcdRegisterErr == nil { - RepoURLUpdateErr = impl.chartService.ConfigureGitOpsRepoUrl(pipeline.AppId, chartGitAttr.RepoUrl, chartGitAttr.ChartLocation, false, userId) + RepoURLUpdateErr = impl.chartService.ConfigureGitOpsRepoUrlForApp(pipeline.AppId, chartGitAttr.RepoUrl, chartGitAttr.ChartLocation, false, userId) if RepoURLUpdateErr != nil { impl.logger.Errorw("error in updating git repo url in charts", "err", RepoURLUpdateErr) } + //configure gitOps repoUrl for ENV + activeGitOpsConfig, err := impl.gitOpsConfigReadService.GetGitOpsConfigActive() + if err != nil { + impl.logger.Errorw("error in getting active gitOps config", "err", err) + } + envDeploymentConfig.DeploymentAppType = bean3.ArgoCd + envDeploymentConfig.ConfigType = common.GetDeploymentConfigType(chart.IsCustomGitRepository) + envDeploymentConfig.RepoURL = chartGitAttr.RepoUrl + envDeploymentConfig.ChartLocation = chartGitAttr.ChartLocation + envDeploymentConfig.CredentialIdInt = activeGitOpsConfig.Id + envDeploymentConfig.CredentialType = bean4.GitOps.String() + + envDeploymentConfig, RepoURLUpdateErr = impl.deploymentConfigService.CreateOrUpdateConfig(envDeploymentConfig, userId) + if RepoURLUpdateErr != nil { + impl.logger.Errorw("error in saving deployment config for app", "appId", pipeline.AppId, "envId", pipeline.EnvironmentId, "err", err) + } } } } @@ -509,11 +534,11 @@ func (impl *AppDeploymentTypeChangeManagerImpl) DeleteDeploymentApps(ctx context continue } - err = impl.deleteHelmApp(ctx, pipeline) + err = impl.deleteHelmApp(ctx, pipeline, envDeploymentConfig.DeploymentAppType) } if err != nil { - impl.logger.Errorw("error deleting app on "+pipeline.DeploymentAppType, + impl.logger.Errorw("error deleting app on "+envDeploymentConfig.DeploymentAppType, "deployment app name", deploymentAppName, "err", err) @@ -577,10 +602,10 @@ func (impl *AppDeploymentTypeChangeManagerImpl) isPipelineInfoValid(pipeline *pi return failedPipelines, true } -func (impl *AppDeploymentTypeChangeManagerImpl) handleNotHealthyAppsIfArgoDeploymentType(pipeline *pipelineConfig.Pipeline, +func (impl *AppDeploymentTypeChangeManagerImpl) handleNotHealthyAppsIfArgoDeploymentType(pipeline *pipelineConfig.Pipeline, deploymentConfig *bean4.DeploymentConfig, failedPipelines []*bean.DeploymentChangeStatus) ([]*bean.DeploymentChangeStatus, error) { - if pipeline.DeploymentAppType == bean3.ArgoCd { + if deploymentConfig.DeploymentAppType == bean3.ArgoCd { // check if app status is Healthy status, err := impl.appStatusRepository.Get(pipeline.AppId, pipeline.EnvironmentId) @@ -609,9 +634,9 @@ func (impl *AppDeploymentTypeChangeManagerImpl) handleNotHealthyAppsIfArgoDeploy } func (impl *AppDeploymentTypeChangeManagerImpl) handleNotDeployedAppsIfArgoDeploymentType(pipeline *pipelineConfig.Pipeline, - failedPipelines []*bean.DeploymentChangeStatus) ([]*bean.DeploymentChangeStatus, error) { + failedPipelines []*bean.DeploymentChangeStatus, deploymentConfig *bean4.DeploymentConfig) ([]*bean.DeploymentChangeStatus, error) { - if pipeline.DeploymentAppType == string(bean3.ArgoCd) { + if deploymentConfig.DeploymentAppType == string(bean3.ArgoCd) { // check if app status is Healthy status, err := impl.appStatusRepository.Get(pipeline.AppId, pipeline.EnvironmentId) @@ -656,10 +681,12 @@ func (impl *AppDeploymentTypeChangeManagerImpl) fetchDeletedApp(ctx context.Cont failedPipelines := make([]*bean.DeploymentChangeStatus, 0) // Iterate over all the pipelines in the environment for given deployment app type for _, pipeline := range pipelines { - + envDeploymentConfig, err := impl.deploymentConfigService.GetDeploymentConfig(pipeline.AppId, pipeline.EnvironmentId) + if err != nil { + impl.logger.Errorw("error in fetching environment deployment config by appId and envId", "appId", pipeline.AppId, "envId", pipeline.EnvironmentId, "err", err) + } deploymentAppName := fmt.Sprintf("%s-%s", pipeline.App.AppName, pipeline.Environment.Name) - var err error - if pipeline.DeploymentAppType == bean3.ArgoCd { + if envDeploymentConfig.DeploymentAppType == bean3.ArgoCd { appIdentifier := &helmBean.AppIdentifier{ ClusterId: pipeline.Environment.ClusterId, ReleaseName: pipeline.DeploymentAppName, @@ -740,14 +767,14 @@ func (impl *AppDeploymentTypeChangeManagerImpl) appendToDeploymentChangeStatusLi } // deleteHelmApp takes in context and pipeline object and deletes the release in helm -func (impl *AppDeploymentTypeChangeManagerImpl) deleteHelmApp(ctx context.Context, pipeline *pipelineConfig.Pipeline) error { +func (impl *AppDeploymentTypeChangeManagerImpl) deleteHelmApp(ctx context.Context, pipeline *pipelineConfig.Pipeline, deploymentAppType string) error { if !pipeline.DeploymentAppCreated { return nil } // validation - if !util.IsHelmApp(pipeline.DeploymentAppType) { + if !util.IsHelmApp(deploymentAppType) { return errors.New("unable to delete pipeline with id: " + strconv.Itoa(pipeline.Id) + ", not a helm app") } diff --git a/pkg/pipeline/CdHandler.go b/pkg/pipeline/CdHandler.go index 876fabffeb..15dcec283f 100644 --- a/pkg/pipeline/CdHandler.go +++ b/pkg/pipeline/CdHandler.go @@ -21,6 +21,7 @@ import ( "errors" "fmt" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/adapter/cdWorkflow" + common2 "github.com/devtron-labs/devtron/pkg/deployment/common" "os" "path/filepath" "strconv" @@ -88,6 +89,7 @@ type CdHandlerImpl struct { clusterService cluster.ClusterService blobConfigStorageService BlobStorageConfigService customTagService CustomTagService + deploymentConfigService common2.DeploymentConfigService } func NewCdHandlerImpl(Logger *zap.SugaredLogger, userService user.UserService, @@ -99,7 +101,9 @@ func NewCdHandlerImpl(Logger *zap.SugaredLogger, userService user.UserService, resourceGroupService resourceGroup2.ResourceGroupService, imageTaggingService ImageTaggingService, k8sUtil *k8s.K8sServiceImpl, workflowService WorkflowService, clusterService cluster.ClusterService, - blobConfigStorageService BlobStorageConfigService, customTagService CustomTagService) *CdHandlerImpl { + blobConfigStorageService BlobStorageConfigService, customTagService CustomTagService, + deploymentConfigService common2.DeploymentConfigService, +) *CdHandlerImpl { cdh := &CdHandlerImpl{ Logger: Logger, userService: userService, @@ -118,6 +122,7 @@ func NewCdHandlerImpl(Logger *zap.SugaredLogger, userService user.UserService, clusterService: clusterService, blobConfigStorageService: blobConfigStorageService, customTagService: customTagService, + deploymentConfigService: deploymentConfigService, } config, err := types.GetCdConfig() if err != nil { @@ -244,7 +249,14 @@ func (impl *CdHandlerImpl) UpdateWorkflow(workflowStatus v1alpha1.WorkflowStatus impl.Logger.Error("update wf failed for id " + strconv.Itoa(savedWorkflow.Id)) return 0, "", err } - util3.TriggerCDMetrics(cdWorkflow.GetTriggerMetricsFromRunnerObj(savedWorkflow), impl.config.ExposeCDMetrics) + appId := savedWorkflow.CdWorkflow.Pipeline.AppId + envId := savedWorkflow.CdWorkflow.Pipeline.EnvironmentId + envDeploymentConfig, err := impl.deploymentConfigService.GetDeploymentConfig(appId, envId) + if err != nil { + impl.Logger.Errorw("error in fetching environment deployment config by appId and envId", "appId", appId, "envId", envId, "err", err) + return 0, "", err + } + util3.TriggerCDMetrics(cdWorkflow.GetTriggerMetricsFromRunnerObj(savedWorkflow, envDeploymentConfig), impl.config.ExposeCDMetrics) if string(v1alpha1.NodeError) == savedWorkflow.Status || string(v1alpha1.NodeFailed) == savedWorkflow.Status { impl.Logger.Warnw("cd stage failed for workflow: ", "wfId", savedWorkflow.Id) } diff --git a/pkg/pipeline/CiCdPipelineOrchestrator.go b/pkg/pipeline/CiCdPipelineOrchestrator.go index 081cc12d6e..2ca2ae7cac 100644 --- a/pkg/pipeline/CiCdPipelineOrchestrator.go +++ b/pkg/pipeline/CiCdPipelineOrchestrator.go @@ -26,6 +26,8 @@ import ( "errors" "fmt" attributesBean "github.com/devtron-labs/devtron/pkg/attributes/bean" + "github.com/devtron-labs/devtron/pkg/deployment/common" + "github.com/devtron-labs/devtron/pkg/deployment/gitOps/config" "golang.org/x/exp/slices" "net/http" "path" @@ -131,6 +133,8 @@ type CiCdPipelineOrchestratorImpl struct { customTagService CustomTagService chartService chart.ChartService transactionManager sql.TransactionWrapper + gitOpsConfigReadService config.GitOpsConfigReadService + deploymentConfigService common.DeploymentConfigService } func NewCiCdPipelineOrchestrator( @@ -157,7 +161,9 @@ func NewCiCdPipelineOrchestrator( configMapService ConfigMapService, customTagService CustomTagService, genericNoteService genericNotes.GenericNoteService, - chartService chart.ChartService, transactionManager sql.TransactionWrapper) *CiCdPipelineOrchestratorImpl { + chartService chart.ChartService, transactionManager sql.TransactionWrapper, + gitOpsConfigReadService config.GitOpsConfigReadService, + deploymentConfigService common.DeploymentConfigService) *CiCdPipelineOrchestratorImpl { return &CiCdPipelineOrchestratorImpl{ appRepository: pipelineGroupRepository, logger: logger, @@ -185,6 +191,8 @@ func NewCiCdPipelineOrchestrator( customTagService: customTagService, chartService: chartService, transactionManager: transactionManager, + gitOpsConfigReadService: gitOpsConfigReadService, + deploymentConfigService: deploymentConfigService, } } @@ -1609,9 +1617,9 @@ func (impl CiCdPipelineOrchestratorImpl) CreateCDPipelines(pipelineRequest *bean RunPreStageInEnv: pipelineRequest.RunPreStageInEnv, RunPostStageInEnv: pipelineRequest.RunPostStageInEnv, DeploymentAppCreated: false, - DeploymentAppType: pipelineRequest.DeploymentAppType, - DeploymentAppName: fmt.Sprintf("%s-%s", appName, env.Name), - AuditLog: sql.AuditLog{UpdatedBy: userId, CreatedBy: userId, UpdatedOn: time.Now(), CreatedOn: time.Now()}, + //DeploymentAppType: pipelineRequest.DeploymentAppType, + DeploymentAppName: fmt.Sprintf("%s-%s", appName, env.Name), + AuditLog: sql.AuditLog{UpdatedBy: userId, CreatedBy: userId, UpdatedOn: time.Now(), CreatedOn: time.Now()}, } err = impl.pipelineRepository.Save([]*pipelineConfig.Pipeline{pipeline}, tx) if err != nil { @@ -1772,6 +1780,13 @@ func (impl CiCdPipelineOrchestratorImpl) GetCdPipelinesForApp(appId int) (cdPipe var pipelines []*bean.CDPipelineConfigObject for _, dbPipeline := range dbPipelines { + + envDeploymentConfig, err := impl.deploymentConfigService.GetDeploymentConfig(dbPipeline.AppId, dbPipeline.EnvironmentId) + if err != nil { + impl.logger.Errorw("error in fetching environment deployment config by appId and envId", "appId", appId, "envId", dbPipeline.EnvironmentId, "err", err) + return nil, err + } + preStage := bean.CdStage{} if len(dbPipeline.PreStageConfig) > 0 { preStage.Name = "Pre-Deployment" @@ -1817,7 +1832,7 @@ func (impl CiCdPipelineOrchestratorImpl) GetCdPipelinesForApp(appId int) (cdPipe RunPostStageInEnv: dbPipeline.RunPostStageInEnv, PreStageConfigMapSecretNames: preStageConfigmapSecrets, PostStageConfigMapSecretNames: postStageConfigmapSecrets, - DeploymentAppType: dbPipeline.DeploymentAppType, + DeploymentAppType: envDeploymentConfig.DeploymentAppType, DeploymentAppCreated: dbPipeline.DeploymentAppCreated, DeploymentAppDeleteRequest: dbPipeline.DeploymentAppDeleteRequest, IsVirtualEnvironment: dbPipeline.Environment.IsVirtualEnvironment, @@ -1880,6 +1895,11 @@ func (impl CiCdPipelineOrchestratorImpl) GetCdPipelinesForEnv(envId int, request impl.logger.Errorw("error in fetching latest chart details for app by appId") return nil, err } + envDeploymentConfig, err := impl.deploymentConfigService.GetDeploymentConfig(dbPipeline.AppId, dbPipeline.EnvironmentId) + if err != nil { + impl.logger.Errorw("error in fetching environment deployment config by appId and envId", "appId", dbPipeline.AppId, "envId", dbPipeline.EnvironmentId, "err", err) + return nil, err + } pipeline := &bean.CDPipelineConfigObject{ Id: dbPipeline.Id, Name: dbPipeline.Name, @@ -1889,7 +1909,7 @@ func (impl CiCdPipelineOrchestratorImpl) GetCdPipelinesForEnv(envId int, request TriggerType: dbPipeline.TriggerType, RunPreStageInEnv: dbPipeline.RunPreStageInEnv, RunPostStageInEnv: dbPipeline.RunPostStageInEnv, - DeploymentAppType: dbPipeline.DeploymentAppType, + DeploymentAppType: envDeploymentConfig.DeploymentAppType, AppName: dbPipeline.App.AppName, AppId: dbPipeline.AppId, TeamId: dbPipeline.App.TeamId, diff --git a/pkg/pipeline/DeploymentPipelineConfigService.go b/pkg/pipeline/DeploymentPipelineConfigService.go index 40efabf736..16548747ed 100644 --- a/pkg/pipeline/DeploymentPipelineConfigService.go +++ b/pkg/pipeline/DeploymentPipelineConfigService.go @@ -43,6 +43,8 @@ import ( chartRepoRepository "github.com/devtron-labs/devtron/pkg/chartRepo/repository" repository2 "github.com/devtron-labs/devtron/pkg/cluster/repository" clutserBean "github.com/devtron-labs/devtron/pkg/cluster/repository/bean" + "github.com/devtron-labs/devtron/pkg/deployment/common" + bean4 "github.com/devtron-labs/devtron/pkg/deployment/common/bean" commonBean "github.com/devtron-labs/devtron/pkg/deployment/gitOps/common/bean" "github.com/devtron-labs/devtron/pkg/deployment/gitOps/config" "github.com/devtron-labs/devtron/pkg/deployment/gitOps/git" @@ -157,6 +159,7 @@ type CdPipelineConfigServiceImpl struct { imageDigestPolicyService imageDigestPolicy.ImageDigestPolicyService pipelineConfigEventPublishService out.PipelineConfigEventPublishService deploymentTypeOverrideService config2.DeploymentTypeOverrideService + deploymentConfigService common.DeploymentConfigService } func NewCdPipelineConfigServiceImpl(logger *zap.SugaredLogger, pipelineRepository pipelineConfig.PipelineRepository, @@ -180,7 +183,8 @@ func NewCdPipelineConfigServiceImpl(logger *zap.SugaredLogger, pipelineRepositor chartService chart.ChartService, imageDigestPolicyService imageDigestPolicy.ImageDigestPolicyService, pipelineConfigEventPublishService out.PipelineConfigEventPublishService, - deploymentTypeOverrideService config2.DeploymentTypeOverrideService) *CdPipelineConfigServiceImpl { + deploymentTypeOverrideService config2.DeploymentTypeOverrideService, + deploymentConfigService common.DeploymentConfigService) *CdPipelineConfigServiceImpl { return &CdPipelineConfigServiceImpl{ logger: logger, pipelineRepository: pipelineRepository, @@ -217,6 +221,7 @@ func NewCdPipelineConfigServiceImpl(logger *zap.SugaredLogger, pipelineRepositor imageDigestPolicyService: imageDigestPolicyService, pipelineConfigEventPublishService: pipelineConfigEventPublishService, deploymentTypeOverrideService: deploymentTypeOverrideService, + deploymentConfigService: deploymentConfigService, } } @@ -322,6 +327,12 @@ func (impl *CdPipelineConfigServiceImpl) GetCdPipelineById(pipelineId int) (cdPi return nil, err } + envDeploymentConfig, err := impl.deploymentConfigService.GetDeploymentConfig(dbPipeline.AppId, dbPipeline.EnvironmentId) + if err != nil { + impl.logger.Errorw("error in fetching environment deployment config by appId and envId", "appId", dbPipeline.AppId, "envId", dbPipeline.EnvironmentId, "err", err) + return nil, err + } + cdPipeline = &bean.CDPipelineConfigObject{ Id: dbPipeline.Id, Name: dbPipeline.Name, @@ -340,7 +351,7 @@ func (impl *CdPipelineConfigServiceImpl) GetCdPipelineById(pipelineId int) (cdPi CdArgoSetup: environment.Cluster.CdArgoSetup, ParentPipelineId: appWorkflowMapping.ParentId, ParentPipelineType: appWorkflowMapping.ParentType, - DeploymentAppType: dbPipeline.DeploymentAppType, + DeploymentAppType: envDeploymentConfig.DeploymentAppType, DeploymentAppCreated: dbPipeline.DeploymentAppCreated, IsVirtualEnvironment: dbPipeline.Environment.IsVirtualEnvironment, CustomTagObject: customTag, @@ -402,13 +413,15 @@ func (impl *CdPipelineConfigServiceImpl) CreateCdPipelines(pipelineCreateRequest // TODO: creating git repo for all apps irrespective of acd or helm if gitOpsConfigurationStatus.IsGitOpsConfigured && isGitOpsRequiredForCD && !pipelineCreateRequest.IsCloneAppReq { - chart, err := impl.chartService.FindLatestChartForAppByAppId(app.Id) + + AppDeploymentConfig, err := impl.deploymentConfigService.GetDeploymentConfig(app.Id, 0) if err != nil { - impl.logger.Errorw("Error in fetching latest chart for pipeline", "err", err, "appId", app.Id) + impl.logger.Errorw("error in fetching deployment config by appId", "appId", app.Id, "err", err) return nil, err } - if gitOps.IsGitOpsRepoNotConfigured(chart.GitRepoUrl) { - if gitOpsConfigurationStatus.AllowCustomRepository || chart.IsCustomGitRepository { + + if gitOps.IsGitOpsRepoNotConfigured(AppDeploymentConfig.RepoURL) { + if gitOpsConfigurationStatus.AllowCustomRepository || AppDeploymentConfig.ConfigType == bean4.CUSTOM.String() { apiErr := &util.ApiError{ HttpStatusCode: http.StatusConflict, UserMessage: pipelineConfig.GITOPS_REPO_NOT_CONFIGURED, @@ -426,13 +439,35 @@ func (impl *CdPipelineConfigServiceImpl) CreateCdPipelines(pipelineCreateRequest impl.logger.Errorw("error in registering app in acd", "err", err) return nil, err } - // below function will update gitRepoUrl for charts if user has not already provided gitOps repoURL - err = impl.chartService.ConfigureGitOpsRepoUrl(pipelineCreateRequest.AppId, chartGitAttr.RepoUrl, chartGitAttr.ChartLocation, false, pipelineCreateRequest.UserId) + AppDeploymentConfig.RepoURL = chartGitAttr.RepoUrl + AppDeploymentConfig, err = impl.deploymentConfigService.CreateOrUpdateConfig(AppDeploymentConfig, pipelineCreateRequest.UserId) + if err != nil { + impl.logger.Errorw("error in fetching creating env config", "appId", app.Id, "err", err) + return nil, err + } + } + for _, pipeline := range pipelineCreateRequest.Pipelines { + envDeploymentConfig := &bean4.DeploymentConfig{ + AppId: app.Id, + EnvironmentId: pipeline.EnvironmentId, + ConfigType: AppDeploymentConfig.ConfigType, + DeploymentAppType: AppDeploymentConfig.DeploymentAppType, + RepoURL: AppDeploymentConfig.RepoURL, + RepoName: AppDeploymentConfig.RepoName, + ChartLocation: AppDeploymentConfig.ChartLocation, + CredentialType: AppDeploymentConfig.CredentialType, + CredentialIdInt: AppDeploymentConfig.CredentialIdInt, + CredentialIdString: AppDeploymentConfig.CredentialIdString, + Active: true, + } + envDeploymentConfig, err := impl.deploymentConfigService.CreateOrUpdateConfig(envDeploymentConfig, pipelineCreateRequest.UserId) if err != nil { - impl.logger.Errorw("error in updating git repo url in charts", "err", err) + impl.logger.Errorw("error in fetching creating env config", "appId", app.Id, "envId", pipeline.EnvironmentId, "err", err) return nil, err } + } + } for _, pipeline := range pipelineCreateRequest.Pipelines { @@ -820,10 +855,15 @@ func (impl *CdPipelineConfigServiceImpl) DeleteCdPipeline(pipeline *pipelineConf impl.logger.Errorw("error in deleting imageDigestPolicy for pipeline", "err", err, "pipelineId", pipeline.Id) return nil, err } + envDeploymentConfig, err := impl.deploymentConfigService.GetDeploymentConfig(pipeline.AppId, pipeline.EnvironmentId) + if err != nil { + impl.logger.Errorw("error in fetching environment deployment config by appId and envId", "appId", pipeline.AppId, "envId", pipeline.EnvironmentId, "err", err) + return nil, err + } //delete app from argo cd, if created if pipeline.DeploymentAppCreated == true { deploymentAppName := fmt.Sprintf("%s-%s", pipeline.App.AppName, pipeline.Environment.Name) - if util.IsAcdApp(pipeline.DeploymentAppType) { + if util.IsAcdApp(envDeploymentConfig.DeploymentAppType) { if !forceDelete && !deleteResponse.ClusterReachable { impl.logger.Errorw("cluster connection error", "err", clusterBean.ErrorInConnecting) if cascadeDelete { @@ -859,7 +899,7 @@ func (impl *CdPipelineConfigServiceImpl) DeleteCdPipeline(pipeline *pipelineConf } impl.logger.Infow("app deleted from argocd", "id", pipeline.Id, "pipelineName", pipeline.Name, "app", deploymentAppName) } - } else if util.IsHelmApp(pipeline.DeploymentAppType) { + } else if util.IsHelmApp(envDeploymentConfig.DeploymentAppType) { err = impl.DeleteHelmTypePipelineDeploymentApp(ctx, forceDelete, pipeline) if err != nil { impl.logger.Errorw("error, DeleteHelmTypePipelineDeploymentApp", "err", err, "pipelineId", pipeline.Id) @@ -909,8 +949,13 @@ func (impl *CdPipelineConfigServiceImpl) DeleteACDAppCdPipelineWithNonCascade(pi _, err := impl.DeleteCdPipeline(pipeline, ctx, bean.FORCE_DELETE, false, userId) return err } + envDeploymentConfig, err := impl.deploymentConfigService.GetDeploymentConfig(pipeline.AppId, pipeline.EnvironmentId) + if err != nil { + impl.logger.Errorw("error in fetching environment deployment config by appId and envId", "appId", pipeline.AppId, "envId", pipeline.EnvironmentId, "err", err) + return err + } //delete app from argo cd with non-cascade, if created - if pipeline.DeploymentAppCreated && util.IsAcdApp(pipeline.DeploymentAppType) { + if pipeline.DeploymentAppCreated && util.IsAcdApp(envDeploymentConfig.DeploymentAppType) { appDetails, err := impl.appRepo.FindById(pipeline.AppId) deploymentAppName := fmt.Sprintf("%s-%s", appDetails.AppName, pipeline.Environment.Name) impl.logger.Debugw("acd app is already deleted for this pipeline", "pipeline", pipeline) @@ -1288,12 +1333,19 @@ func (impl *CdPipelineConfigServiceImpl) GetCdPipelinesByEnvironmentMin(request //if user unauthorized, skip items continue } + + envDeploymentConfig, err := impl.deploymentConfigService.GetDeploymentConfig(dbPipeline.AppId, dbPipeline.EnvironmentId) + if err != nil { + impl.logger.Errorw("error in fetching environment deployment config by appId and envId", "appId", dbPipeline.AppId, "envId", dbPipeline.EnvironmentId, "err", err) + return nil, err + } + pcObject := &bean.CDPipelineConfigObject{ AppId: dbPipeline.AppId, AppName: dbPipeline.App.AppName, EnvironmentId: dbPipeline.EnvironmentId, Id: dbPipeline.Id, - DeploymentAppType: dbPipeline.DeploymentAppType, + DeploymentAppType: envDeploymentConfig.DeploymentAppType, IsVirtualEnvironment: dbPipeline.Environment.IsVirtualEnvironment, } cdPipelines = append(cdPipelines, pcObject) @@ -1983,8 +2035,13 @@ func (impl *CdPipelineConfigServiceImpl) DeleteCdPipelinePartial(pipeline *pipel //delete app from argo cd, if created if pipeline.DeploymentAppCreated && !pipeline.DeploymentAppDeleteRequest { + envDeploymentConfig, err := impl.deploymentConfigService.GetDeploymentConfig(pipeline.AppId, pipeline.EnvironmentId) + if err != nil { + impl.logger.Errorw("error in fetching environment deployment config by appId and envId", "appId", pipeline.AppId, "envId", pipeline.EnvironmentId, "err", err) + return deleteResponse, err + } deploymentAppName := fmt.Sprintf("%s-%s", pipeline.App.AppName, pipeline.Environment.Name) - if util.IsAcdApp(pipeline.DeploymentAppType) { + if util.IsAcdApp(envDeploymentConfig.DeploymentAppType) { if !forceDelete && !deleteResponse.ClusterReachable { impl.logger.Errorw("cluster connection error", "err", clusterBean.ErrorInConnecting) if cascadeDelete { diff --git a/pkg/pipeline/PipelineBuilder.go b/pkg/pipeline/PipelineBuilder.go index 1dcabe8ce2..7fdc24e702 100644 --- a/pkg/pipeline/PipelineBuilder.go +++ b/pkg/pipeline/PipelineBuilder.go @@ -201,16 +201,6 @@ func getPatchMessage(err error) string { return "" } -func (impl *PipelineBuilderImpl) isGitRepoUrlPresent(appId int) bool { - fetchedChart, err := impl.chartRepository.FindLatestByAppId(appId) - - if err != nil || len(fetchedChart.GitRepoUrl) == 0 { - impl.logger.Errorw("error fetching git repo url or it is not present") - return false - } - return true -} - type DeploymentType struct { Deployment Deployment `json:"deployment"` } diff --git a/pkg/workflow/cd/CdWorkflowCommonService.go b/pkg/workflow/cd/CdWorkflowCommonService.go index 42cd13e811..6b9e217257 100644 --- a/pkg/workflow/cd/CdWorkflowCommonService.go +++ b/pkg/workflow/cd/CdWorkflowCommonService.go @@ -28,6 +28,7 @@ import ( "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/timelineStatus" "github.com/devtron-labs/devtron/internal/util" "github.com/devtron-labs/devtron/pkg/app/status" + common2 "github.com/devtron-labs/devtron/pkg/deployment/common" "github.com/devtron-labs/devtron/pkg/pipeline/types" globalUtil "github.com/devtron-labs/devtron/util" "go.opentelemetry.io/otel" @@ -55,13 +56,15 @@ type CdWorkflowCommonServiceImpl struct { config *types.CdConfig pipelineRepository pipelineConfig.PipelineRepository pipelineStatusTimelineRepository pipelineConfig.PipelineStatusTimelineRepository + deploymentConfigService common2.DeploymentConfigService } func NewCdWorkflowCommonServiceImpl(logger *zap.SugaredLogger, cdWorkflowRepository pipelineConfig.CdWorkflowRepository, pipelineStatusTimelineService status.PipelineStatusTimelineService, pipelineRepository pipelineConfig.PipelineRepository, - pipelineStatusTimelineRepository pipelineConfig.PipelineStatusTimelineRepository) (*CdWorkflowCommonServiceImpl, error) { + pipelineStatusTimelineRepository pipelineConfig.PipelineStatusTimelineRepository, + deploymentConfigService common2.DeploymentConfigService) (*CdWorkflowCommonServiceImpl, error) { config, err := types.GetCdConfig() if err != nil { return nil, err @@ -73,6 +76,7 @@ func NewCdWorkflowCommonServiceImpl(logger *zap.SugaredLogger, config: config, pipelineRepository: pipelineRepository, pipelineStatusTimelineRepository: pipelineStatusTimelineRepository, + deploymentConfigService: deploymentConfigService, }, nil } @@ -185,7 +189,14 @@ func (impl *CdWorkflowCommonServiceImpl) MarkCurrentDeploymentFailed(runner *pip return err } } - globalUtil.TriggerCDMetrics(cdWorkflow.GetTriggerMetricsFromRunnerObj(runner), impl.config.ExposeCDMetrics) + appId := runner.CdWorkflow.Pipeline.AppId + envId := runner.CdWorkflow.Pipeline.EnvironmentId + envDeploymentConfig, err := impl.deploymentConfigService.GetDeploymentConfig(appId, envId) + if err != nil { + impl.logger.Errorw("error in fetching environment deployment config by appId and envId", "appId", appId, "envId", envId, "err", err) + return err + } + globalUtil.TriggerCDMetrics(cdWorkflow.GetTriggerMetricsFromRunnerObj(runner, envDeploymentConfig), impl.config.ExposeCDMetrics) } return nil } @@ -242,7 +253,15 @@ func (impl *CdWorkflowCommonServiceImpl) UpdatePreviousQueuedRunnerStatus(cdWfrI Pipeline: pipeline, } } - globalUtil.TriggerCDMetrics(cdWorkflow.GetTriggerMetricsFromRunnerObj(queuedRunner), impl.config.ExposeCDMetrics) + + appId := queuedRunner.CdWorkflow.Pipeline.AppId + envId := queuedRunner.CdWorkflow.Pipeline.EnvironmentId + envDeploymentConfig, err := impl.deploymentConfigService.GetDeploymentConfig(appId, envId) + if err != nil { + impl.logger.Errorw("error in fetching environment deployment config by appId and envId", "appId", appId, "envId", envId, "err", err) + return err + } + globalUtil.TriggerCDMetrics(cdWorkflow.GetTriggerMetricsFromRunnerObj(queuedRunner, envDeploymentConfig), impl.config.ExposeCDMetrics) queuedRunnerIds = append(queuedRunnerIds, queuedRunner.Id) } err = impl.cdWorkflowRepository.UpdateRunnerStatusToFailedForIds(pipelineConfig.ErrorDeploymentSuperseded.Error(), triggeredBy, queuedRunnerIds...) diff --git a/pkg/workflow/dag/WorkflowDagExecutor.go b/pkg/workflow/dag/WorkflowDagExecutor.go index 787e8bdf3c..73eb007a7a 100644 --- a/pkg/workflow/dag/WorkflowDagExecutor.go +++ b/pkg/workflow/dag/WorkflowDagExecutor.go @@ -30,6 +30,7 @@ import ( "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/adapter/cdWorkflow" "github.com/devtron-labs/devtron/pkg/app/status" "github.com/devtron-labs/devtron/pkg/build/artifacts" + common2 "github.com/devtron-labs/devtron/pkg/deployment/common" "github.com/devtron-labs/devtron/pkg/deployment/manifest" "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps" triggerAdapter "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/adapter" @@ -113,6 +114,7 @@ type WorkflowDagExecutorImpl struct { manifestCreationService manifest.ManifestCreationService commonArtifactService artifacts.CommonArtifactService + deploymentConfigService common2.DeploymentConfigService } func NewWorkflowDagExecutorImpl(Logger *zap.SugaredLogger, pipelineRepository pipelineConfig.PipelineRepository, @@ -134,7 +136,8 @@ func NewWorkflowDagExecutorImpl(Logger *zap.SugaredLogger, pipelineRepository pi cdTriggerService devtronApps.TriggerService, userDeploymentRequestService service.UserDeploymentRequestService, manifestCreationService manifest.ManifestCreationService, - commonArtifactService artifacts.CommonArtifactService) *WorkflowDagExecutorImpl { + commonArtifactService artifacts.CommonArtifactService, + deploymentConfigService common2.DeploymentConfigService) *WorkflowDagExecutorImpl { wde := &WorkflowDagExecutorImpl{logger: Logger, pipelineRepository: pipelineRepository, cdWorkflowRepository: cdWorkflowRepository, @@ -156,6 +159,7 @@ func NewWorkflowDagExecutorImpl(Logger *zap.SugaredLogger, pipelineRepository pi userDeploymentRequestService: userDeploymentRequestService, manifestCreationService: manifestCreationService, commonArtifactService: commonArtifactService, + deploymentConfigService: deploymentConfigService, } config, err := types.GetCdConfig() if err != nil { @@ -323,7 +327,13 @@ func (impl *WorkflowDagExecutorImpl) handleAsyncTriggerReleaseError(ctx context. } } if util4.IsTerminalRunnerStatus(cdWfr.Status) { - util4.TriggerCDMetrics(cdWorkflow.GetTriggerMetricsFromRunnerObj(cdWfr), impl.config.ExposeCDMetrics) + appId := cdWfr.CdWorkflow.Pipeline.AppId + envId := cdWfr.CdWorkflow.Pipeline.EnvironmentId + envDeploymentConfig, err := impl.deploymentConfigService.GetDeploymentConfig(appId, envId) + if err != nil { + impl.logger.Errorw("error in fetching environment deployment config by appId and envId", "appId", appId, "envId", envId, "err", err) + } + util4.TriggerCDMetrics(cdWorkflow.GetTriggerMetricsFromRunnerObj(cdWfr, envDeploymentConfig), impl.config.ExposeCDMetrics) } impl.logger.Infow("updated workflow runner status for helm app", "wfr", cdWfr) } else { diff --git a/pkg/workflow/status/WorkflowStatusService.go b/pkg/workflow/status/WorkflowStatusService.go index 7ca4b5aae2..3f7d3ff9cf 100644 --- a/pkg/workflow/status/WorkflowStatusService.go +++ b/pkg/workflow/status/WorkflowStatusService.go @@ -34,6 +34,7 @@ import ( app_status "github.com/devtron-labs/devtron/pkg/appStatus" repository3 "github.com/devtron-labs/devtron/pkg/appStore/installedApp/repository" repository2 "github.com/devtron-labs/devtron/pkg/cluster/repository" + common2 "github.com/devtron-labs/devtron/pkg/deployment/common" bean3 "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/bean" "github.com/devtron-labs/devtron/pkg/eventProcessor/out" "github.com/devtron-labs/devtron/pkg/pipeline/types" @@ -86,7 +87,8 @@ type WorkflowStatusServiceImpl struct { pipelineRepository pipelineConfig.PipelineRepository appListingService app.AppListingService - application application.ServiceClient + application application.ServiceClient + deploymentConfigService common2.DeploymentConfigService } func NewWorkflowStatusServiceImpl(logger *zap.SugaredLogger, @@ -108,6 +110,7 @@ func NewWorkflowStatusServiceImpl(logger *zap.SugaredLogger, pipelineRepository pipelineConfig.PipelineRepository, application application.ServiceClient, appListingService app.AppListingService, + deploymentConfigService common2.DeploymentConfigService, ) (*WorkflowStatusServiceImpl, error) { impl := &WorkflowStatusServiceImpl{ logger: logger, @@ -131,6 +134,7 @@ func NewWorkflowStatusServiceImpl(logger *zap.SugaredLogger, pipelineRepository: pipelineRepository, application: application, appListingService: appListingService, + deploymentConfigService: deploymentConfigService, } config, err := types.GetCdConfig() if err != nil { @@ -175,8 +179,15 @@ func (impl *WorkflowStatusServiceImpl) CheckHelmAppStatusPeriodicallyAndUpdateIn impl.logger.Errorw("error on update cd workflow runner", "wfr", wfr, "err", err) return err } + appId := wfr.CdWorkflow.Pipeline.AppId + envId := wfr.CdWorkflow.Pipeline.EnvironmentId + envDeploymentConfig, err := impl.deploymentConfigService.GetDeploymentConfig(appId, envId) + if err != nil { + impl.logger.Errorw("error in fetching environment deployment config by appId and envId", "appId", appId, "envId", envId, "err", err) + return err + } if slices.Contains(pipelineConfig.WfrTerminalStatusList, wfr.Status) { - util3.TriggerCDMetrics(cdWorkflow.GetTriggerMetricsFromRunnerObj(wfr), impl.config.ExposeCDMetrics) + util3.TriggerCDMetrics(cdWorkflow.GetTriggerMetricsFromRunnerObj(wfr, envDeploymentConfig), impl.config.ExposeCDMetrics) } impl.logger.Infow("updated workflow runner status for helm app", "wfr", wfr) diff --git a/wire_gen.go b/wire_gen.go index 28e8aeadce..3e20b4395e 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -424,7 +424,9 @@ func InitializeApp() (*App, error) { pipelineStatusSyncDetailRepositoryImpl := pipelineConfig.NewPipelineStatusSyncDetailRepositoryImpl(db, sugaredLogger) pipelineStatusSyncDetailServiceImpl := status.NewPipelineStatusSyncDetailServiceImpl(sugaredLogger, pipelineStatusSyncDetailRepositoryImpl) installedAppVersionHistoryRepositoryImpl := repository3.NewInstalledAppVersionHistoryRepositoryImpl(sugaredLogger, db) - pipelineStatusTimelineServiceImpl := status.NewPipelineStatusTimelineServiceImpl(sugaredLogger, pipelineStatusTimelineRepositoryImpl, cdWorkflowRepositoryImpl, userServiceImpl, pipelineStatusTimelineResourcesServiceImpl, pipelineStatusSyncDetailServiceImpl, installedAppRepositoryImpl, installedAppVersionHistoryRepositoryImpl) + repositoryImpl := deploymentConfig.NewRepositoryImpl(db) + deploymentConfigServiceImpl := common.NewDeploymentConfigServiceImpl(repositoryImpl, sugaredLogger, chartRepositoryImpl, pipelineRepositoryImpl, gitOpsConfigReadServiceImpl) + pipelineStatusTimelineServiceImpl := status.NewPipelineStatusTimelineServiceImpl(sugaredLogger, pipelineStatusTimelineRepositoryImpl, cdWorkflowRepositoryImpl, userServiceImpl, pipelineStatusTimelineResourcesServiceImpl, pipelineStatusSyncDetailServiceImpl, installedAppRepositoryImpl, installedAppVersionHistoryRepositoryImpl, deploymentConfigServiceImpl) appServiceConfig, err := app2.GetAppServiceConfig() if err != nil { return nil, err @@ -487,7 +489,7 @@ func InitializeApp() (*App, error) { envLevelAppMetricsRepositoryImpl := repository9.NewEnvLevelAppMetricsRepositoryImpl(db, sugaredLogger) deployedAppMetricsServiceImpl := deployedAppMetrics.NewDeployedAppMetricsServiceImpl(sugaredLogger, appLevelMetricsRepositoryImpl, envLevelAppMetricsRepositoryImpl, chartRefServiceImpl) appListingServiceImpl := app2.NewAppListingServiceImpl(sugaredLogger, appListingRepositoryImpl, applicationServiceClientImpl, appRepositoryImpl, appListingViewBuilderImpl, pipelineRepositoryImpl, linkoutsRepositoryImpl, cdWorkflowRepositoryImpl, pipelineOverrideRepositoryImpl, environmentRepositoryImpl, argoUserServiceImpl, envConfigOverrideRepositoryImpl, chartRepositoryImpl, ciPipelineRepositoryImpl, dockerRegistryIpsConfigServiceImpl, userRepositoryImpl, deployedAppMetricsServiceImpl, ciArtifactRepositoryImpl) - appServiceImpl := app2.NewAppService(envConfigOverrideRepositoryImpl, pipelineOverrideRepositoryImpl, mergeUtil, sugaredLogger, pipelineRepositoryImpl, eventRESTClientImpl, eventSimpleFactoryImpl, applicationServiceClientImpl, appRepositoryImpl, configMapRepositoryImpl, chartRepositoryImpl, cdWorkflowRepositoryImpl, commonServiceImpl, chartTemplateServiceImpl, argoUserServiceImpl, pipelineStatusTimelineRepositoryImpl, pipelineStatusTimelineResourcesServiceImpl, pipelineStatusSyncDetailServiceImpl, pipelineStatusTimelineServiceImpl, appServiceConfig, appStatusServiceImpl, installedAppRepositoryImpl, installedAppVersionHistoryRepositoryImpl, scopedVariableCMCSManagerImpl, acdConfig, gitOpsConfigReadServiceImpl, gitOperationServiceImpl, deploymentTemplateServiceImpl, appListingServiceImpl) + appServiceImpl := app2.NewAppService(envConfigOverrideRepositoryImpl, pipelineOverrideRepositoryImpl, mergeUtil, sugaredLogger, pipelineRepositoryImpl, eventRESTClientImpl, eventSimpleFactoryImpl, applicationServiceClientImpl, appRepositoryImpl, configMapRepositoryImpl, chartRepositoryImpl, cdWorkflowRepositoryImpl, commonServiceImpl, chartTemplateServiceImpl, argoUserServiceImpl, pipelineStatusTimelineRepositoryImpl, pipelineStatusTimelineResourcesServiceImpl, pipelineStatusSyncDetailServiceImpl, pipelineStatusTimelineServiceImpl, appServiceConfig, appStatusServiceImpl, installedAppRepositoryImpl, installedAppVersionHistoryRepositoryImpl, scopedVariableCMCSManagerImpl, acdConfig, gitOpsConfigReadServiceImpl, gitOperationServiceImpl, deploymentTemplateServiceImpl, appListingServiceImpl, deploymentConfigServiceImpl) globalCMCSRepositoryImpl := repository2.NewGlobalCMCSRepositoryImpl(sugaredLogger, db) globalCMCSServiceImpl := pipeline.NewGlobalCMCSServiceImpl(sugaredLogger, globalCMCSRepositoryImpl) argoWorkflowExecutorImpl := executors.NewArgoWorkflowExecutorImpl(sugaredLogger) @@ -561,10 +563,8 @@ func InitializeApp() (*App, error) { configMapServiceImpl := pipeline.NewConfigMapServiceImpl(chartRepositoryImpl, sugaredLogger, chartRepoRepositoryImpl, utilMergeUtil, pipelineConfigRepositoryImpl, configMapRepositoryImpl, envConfigOverrideRepositoryImpl, commonServiceImpl, appRepositoryImpl, configMapHistoryServiceImpl, environmentRepositoryImpl, scopedVariableCMCSManagerImpl) deploymentTemplateHistoryRepositoryImpl := repository13.NewDeploymentTemplateHistoryRepositoryImpl(sugaredLogger, db) deploymentTemplateHistoryServiceImpl := history.NewDeploymentTemplateHistoryServiceImpl(sugaredLogger, deploymentTemplateHistoryRepositoryImpl, pipelineRepositoryImpl, chartRepositoryImpl, userServiceImpl, cdWorkflowRepositoryImpl, scopedVariableManagerImpl, deployedAppMetricsServiceImpl, chartRefServiceImpl) - repositoryImpl := deploymentConfig.NewRepositoryImpl(db) - deploymentConfigServiceImpl := common.NewDeploymentConfigServiceImpl(repositoryImpl, sugaredLogger, chartRepositoryImpl, pipelineRepositoryImpl, gitOpsConfigReadServiceImpl) chartServiceImpl := chart.NewChartServiceImpl(chartRepositoryImpl, sugaredLogger, chartTemplateServiceImpl, chartRepoRepositoryImpl, appRepositoryImpl, utilMergeUtil, envConfigOverrideRepositoryImpl, pipelineConfigRepositoryImpl, environmentRepositoryImpl, deploymentTemplateHistoryServiceImpl, scopedVariableManagerImpl, deployedAppMetricsServiceImpl, chartRefServiceImpl, gitOpsConfigReadServiceImpl, deploymentConfigServiceImpl) - ciCdPipelineOrchestratorImpl := pipeline.NewCiCdPipelineOrchestrator(appRepositoryImpl, sugaredLogger, materialRepositoryImpl, pipelineRepositoryImpl, ciPipelineRepositoryImpl, ciPipelineMaterialRepositoryImpl, cdWorkflowRepositoryImpl, clientImpl, ciCdConfig, appWorkflowRepositoryImpl, environmentRepositoryImpl, attributesServiceImpl, appCrudOperationServiceImpl, userAuthServiceImpl, prePostCdScriptHistoryServiceImpl, pipelineStageServiceImpl, gitMaterialHistoryServiceImpl, ciPipelineHistoryServiceImpl, ciTemplateServiceImpl, dockerArtifactStoreRepositoryImpl, ciArtifactRepositoryImpl, configMapServiceImpl, customTagServiceImpl, genericNoteServiceImpl, chartServiceImpl, transactionUtilImpl) + ciCdPipelineOrchestratorImpl := pipeline.NewCiCdPipelineOrchestrator(appRepositoryImpl, sugaredLogger, materialRepositoryImpl, pipelineRepositoryImpl, ciPipelineRepositoryImpl, ciPipelineMaterialRepositoryImpl, cdWorkflowRepositoryImpl, clientImpl, ciCdConfig, appWorkflowRepositoryImpl, environmentRepositoryImpl, attributesServiceImpl, appCrudOperationServiceImpl, userAuthServiceImpl, prePostCdScriptHistoryServiceImpl, pipelineStageServiceImpl, gitMaterialHistoryServiceImpl, ciPipelineHistoryServiceImpl, ciTemplateServiceImpl, dockerArtifactStoreRepositoryImpl, ciArtifactRepositoryImpl, configMapServiceImpl, customTagServiceImpl, genericNoteServiceImpl, chartServiceImpl, transactionUtilImpl, gitOpsConfigReadServiceImpl, deploymentConfigServiceImpl) ecrConfig, err := pipeline.GetEcrConfig() if err != nil { return nil, err @@ -583,12 +583,12 @@ func InitializeApp() (*App, error) { imageDigestPolicyServiceImpl := imageDigestPolicy.NewImageDigestPolicyServiceImpl(sugaredLogger, qualifierMappingServiceImpl, devtronResourceSearchableKeyServiceImpl) pipelineConfigEventPublishServiceImpl := out.NewPipelineConfigEventPublishServiceImpl(sugaredLogger, pubSubClientServiceImpl) deploymentTypeOverrideServiceImpl := providerConfig.NewDeploymentTypeOverrideServiceImpl(sugaredLogger, environmentVariables, attributesServiceImpl) - cdPipelineConfigServiceImpl := pipeline.NewCdPipelineConfigServiceImpl(sugaredLogger, pipelineRepositoryImpl, environmentRepositoryImpl, pipelineConfigRepositoryImpl, appWorkflowRepositoryImpl, pipelineStageServiceImpl, appRepositoryImpl, appServiceImpl, deploymentGroupRepositoryImpl, ciCdPipelineOrchestratorImpl, appStatusRepositoryImpl, ciPipelineRepositoryImpl, prePostCdScriptHistoryServiceImpl, clusterRepositoryImpl, helmAppServiceImpl, enforcerUtilImpl, pipelineStrategyHistoryServiceImpl, chartRepositoryImpl, resourceGroupServiceImpl, propertiesConfigServiceImpl, deploymentTemplateHistoryServiceImpl, scopedVariableManagerImpl, environmentVariables, applicationServiceClientImpl, customTagServiceImpl, ciPipelineConfigServiceImpl, buildPipelineSwitchServiceImpl, argoClientWrapperServiceImpl, deployedAppMetricsServiceImpl, gitOpsConfigReadServiceImpl, gitOperationServiceImpl, chartServiceImpl, imageDigestPolicyServiceImpl, pipelineConfigEventPublishServiceImpl, deploymentTypeOverrideServiceImpl) + cdPipelineConfigServiceImpl := pipeline.NewCdPipelineConfigServiceImpl(sugaredLogger, pipelineRepositoryImpl, environmentRepositoryImpl, pipelineConfigRepositoryImpl, appWorkflowRepositoryImpl, pipelineStageServiceImpl, appRepositoryImpl, appServiceImpl, deploymentGroupRepositoryImpl, ciCdPipelineOrchestratorImpl, appStatusRepositoryImpl, ciPipelineRepositoryImpl, prePostCdScriptHistoryServiceImpl, clusterRepositoryImpl, helmAppServiceImpl, enforcerUtilImpl, pipelineStrategyHistoryServiceImpl, chartRepositoryImpl, resourceGroupServiceImpl, propertiesConfigServiceImpl, deploymentTemplateHistoryServiceImpl, scopedVariableManagerImpl, environmentVariables, applicationServiceClientImpl, customTagServiceImpl, ciPipelineConfigServiceImpl, buildPipelineSwitchServiceImpl, argoClientWrapperServiceImpl, deployedAppMetricsServiceImpl, gitOpsConfigReadServiceImpl, gitOperationServiceImpl, chartServiceImpl, imageDigestPolicyServiceImpl, pipelineConfigEventPublishServiceImpl, deploymentTypeOverrideServiceImpl, deploymentConfigServiceImpl) appArtifactManagerImpl := pipeline.NewAppArtifactManagerImpl(sugaredLogger, cdWorkflowRepositoryImpl, userServiceImpl, imageTaggingServiceImpl, ciArtifactRepositoryImpl, ciWorkflowRepositoryImpl, pipelineStageServiceImpl, cdPipelineConfigServiceImpl, dockerArtifactStoreRepositoryImpl, ciPipelineRepositoryImpl, ciTemplateServiceImpl) devtronAppCMCSServiceImpl := pipeline.NewDevtronAppCMCSServiceImpl(sugaredLogger, appServiceImpl, attributesRepositoryImpl) globalStrategyMetadataChartRefMappingRepositoryImpl := chartRepoRepository.NewGlobalStrategyMetadataChartRefMappingRepositoryImpl(db, sugaredLogger) devtronAppStrategyServiceImpl := pipeline.NewDevtronAppStrategyServiceImpl(sugaredLogger, chartRepositoryImpl, globalStrategyMetadataChartRefMappingRepositoryImpl, ciCdPipelineOrchestratorImpl, cdPipelineConfigServiceImpl) - cdWorkflowCommonServiceImpl, err := cd.NewCdWorkflowCommonServiceImpl(sugaredLogger, cdWorkflowRepositoryImpl, pipelineStatusTimelineServiceImpl, pipelineRepositoryImpl, pipelineStatusTimelineRepositoryImpl) + cdWorkflowCommonServiceImpl, err := cd.NewCdWorkflowCommonServiceImpl(sugaredLogger, cdWorkflowRepositoryImpl, pipelineStatusTimelineServiceImpl, pipelineRepositoryImpl, pipelineStatusTimelineRepositoryImpl, deploymentConfigServiceImpl) if err != nil { return nil, err } @@ -601,15 +601,15 @@ func InitializeApp() (*App, error) { if err != nil { return nil, err } - appDeploymentTypeChangeManagerImpl := pipeline.NewAppDeploymentTypeChangeManagerImpl(sugaredLogger, pipelineRepositoryImpl, appServiceImpl, appStatusRepositoryImpl, helmAppServiceImpl, applicationServiceClientImpl, appArtifactManagerImpl, cdPipelineConfigServiceImpl, gitOpsConfigReadServiceImpl, chartServiceImpl, workflowEventPublishServiceImpl) + appDeploymentTypeChangeManagerImpl := pipeline.NewAppDeploymentTypeChangeManagerImpl(sugaredLogger, pipelineRepositoryImpl, appServiceImpl, appStatusRepositoryImpl, helmAppServiceImpl, applicationServiceClientImpl, appArtifactManagerImpl, cdPipelineConfigServiceImpl, gitOpsConfigReadServiceImpl, chartServiceImpl, workflowEventPublishServiceImpl, deploymentConfigServiceImpl) devtronAppConfigServiceImpl := pipeline.NewDevtronAppConfigServiceImpl(sugaredLogger, ciCdPipelineOrchestratorImpl, appRepositoryImpl, pipelineRepositoryImpl, resourceGroupServiceImpl, enforcerUtilImpl, ciMaterialConfigServiceImpl) pipelineBuilderImpl := pipeline.NewPipelineBuilderImpl(sugaredLogger, materialRepositoryImpl, chartRepositoryImpl, ciPipelineConfigServiceImpl, ciMaterialConfigServiceImpl, appArtifactManagerImpl, devtronAppCMCSServiceImpl, devtronAppStrategyServiceImpl, appDeploymentTypeChangeManagerImpl, cdPipelineConfigServiceImpl, devtronAppConfigServiceImpl) deploymentTemplateValidationServiceImpl := deploymentTemplate.NewDeploymentTemplateValidationServiceImpl(sugaredLogger, chartRefServiceImpl, scopedVariableManagerImpl) installedAppDBExtendedServiceImpl := FullMode.NewInstalledAppDBExtendedServiceImpl(installedAppDBServiceImpl, appStatusServiceImpl, gitOpsConfigReadServiceImpl) gitOpsValidationServiceImpl := validation.NewGitOpsValidationServiceImpl(sugaredLogger, gitFactory, gitOperationServiceImpl, gitOpsConfigReadServiceImpl, chartTemplateServiceImpl, chartServiceImpl, installedAppDBExtendedServiceImpl) - devtronAppGitOpConfigServiceImpl := gitOpsConfig.NewDevtronAppGitOpConfigServiceImpl(sugaredLogger, chartRepositoryImpl, chartServiceImpl, gitOpsConfigReadServiceImpl, gitOpsValidationServiceImpl, argoClientWrapperServiceImpl) - cdHandlerImpl := pipeline.NewCdHandlerImpl(sugaredLogger, userServiceImpl, cdWorkflowRepositoryImpl, ciLogServiceImpl, ciArtifactRepositoryImpl, ciPipelineMaterialRepositoryImpl, pipelineRepositoryImpl, environmentRepositoryImpl, ciWorkflowRepositoryImpl, enforcerUtilImpl, resourceGroupServiceImpl, imageTaggingServiceImpl, k8sServiceImpl, workflowServiceImpl, clusterServiceImplExtended, blobStorageConfigServiceImpl, customTagServiceImpl) - appWorkflowServiceImpl := appWorkflow2.NewAppWorkflowServiceImpl(sugaredLogger, appWorkflowRepositoryImpl, ciCdPipelineOrchestratorImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, enforcerUtilImpl, resourceGroupServiceImpl, appRepositoryImpl, userAuthServiceImpl, chartServiceImpl) + devtronAppGitOpConfigServiceImpl := gitOpsConfig.NewDevtronAppGitOpConfigServiceImpl(sugaredLogger, chartRepositoryImpl, chartServiceImpl, gitOpsConfigReadServiceImpl, gitOpsValidationServiceImpl, argoClientWrapperServiceImpl, deploymentConfigServiceImpl) + cdHandlerImpl := pipeline.NewCdHandlerImpl(sugaredLogger, userServiceImpl, cdWorkflowRepositoryImpl, ciLogServiceImpl, ciArtifactRepositoryImpl, ciPipelineMaterialRepositoryImpl, pipelineRepositoryImpl, environmentRepositoryImpl, ciWorkflowRepositoryImpl, enforcerUtilImpl, resourceGroupServiceImpl, imageTaggingServiceImpl, k8sServiceImpl, workflowServiceImpl, clusterServiceImplExtended, blobStorageConfigServiceImpl, customTagServiceImpl, deploymentConfigServiceImpl) + appWorkflowServiceImpl := appWorkflow2.NewAppWorkflowServiceImpl(sugaredLogger, appWorkflowRepositoryImpl, ciCdPipelineOrchestratorImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, enforcerUtilImpl, resourceGroupServiceImpl, appRepositoryImpl, userAuthServiceImpl, chartServiceImpl, deploymentConfigServiceImpl) appCloneServiceImpl := appClone.NewAppCloneServiceImpl(sugaredLogger, pipelineBuilderImpl, attributesServiceImpl, chartServiceImpl, configMapServiceImpl, appWorkflowServiceImpl, appListingServiceImpl, propertiesConfigServiceImpl, pipelineStageServiceImpl, ciTemplateServiceImpl, appRepositoryImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, ciPipelineConfigServiceImpl, gitOpsConfigReadServiceImpl) deploymentTemplateRepositoryImpl := repository2.NewDeploymentTemplateRepositoryImpl(db, sugaredLogger) generateManifestDeploymentTemplateServiceImpl, err := generateManifest.NewDeploymentTemplateServiceImpl(sugaredLogger, chartServiceImpl, appListingServiceImpl, deploymentTemplateRepositoryImpl, helmAppServiceImpl, chartTemplateServiceImpl, helmAppClientImpl, k8sServiceImpl, propertiesConfigServiceImpl, deploymentTemplateHistoryServiceImpl, environmentRepositoryImpl, appRepositoryImpl, scopedVariableManagerImpl, chartRefServiceImpl, pipelineOverrideRepositoryImpl, chartRepositoryImpl) @@ -626,19 +626,19 @@ func InitializeApp() (*App, error) { pipelineConfigRestHandlerImpl := configure.NewPipelineRestHandlerImpl(pipelineBuilderImpl, sugaredLogger, deploymentTemplateValidationServiceImpl, chartServiceImpl, devtronAppGitOpConfigServiceImpl, propertiesConfigServiceImpl, userServiceImpl, teamServiceImpl, enforcerImpl, ciHandlerImpl, validate, clientImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, enforcerUtilImpl, dockerRegistryConfigImpl, cdHandlerImpl, appCloneServiceImpl, generateManifestDeploymentTemplateServiceImpl, appWorkflowServiceImpl, materialRepositoryImpl, policyServiceImpl, imageScanResultRepositoryImpl, gitProviderRepositoryImpl, argoUserServiceImpl, ciPipelineMaterialRepositoryImpl, imageTaggingServiceImpl, ciArtifactRepositoryImpl, deployedAppMetricsServiceImpl, chartRefServiceImpl, ciCdPipelineOrchestratorImpl) gitOpsManifestPushServiceImpl := publish.NewGitOpsManifestPushServiceImpl(sugaredLogger, pipelineStatusTimelineServiceImpl, pipelineOverrideRepositoryImpl, acdConfig, chartRefServiceImpl, gitOpsConfigReadServiceImpl, chartServiceImpl, gitOperationServiceImpl, argoClientWrapperServiceImpl, transactionUtilImpl) argoK8sClientImpl := argocdServer.NewArgoK8sClientImpl(sugaredLogger, k8sServiceImpl) - manifestCreationServiceImpl := manifest.NewManifestCreationServiceImpl(sugaredLogger, dockerRegistryIpsConfigServiceImpl, chartRefServiceImpl, scopedVariableCMCSManagerImpl, k8sCommonServiceImpl, deployedAppMetricsServiceImpl, imageDigestPolicyServiceImpl, mergeUtil, appCrudOperationServiceImpl, deploymentTemplateServiceImpl, applicationServiceClientImpl, configMapHistoryRepositoryImpl, configMapRepositoryImpl, chartRepositoryImpl, envConfigOverrideRepositoryImpl, environmentRepositoryImpl, pipelineRepositoryImpl, ciArtifactRepositoryImpl, pipelineOverrideRepositoryImpl, pipelineStrategyHistoryRepositoryImpl, pipelineConfigRepositoryImpl, deploymentTemplateHistoryRepositoryImpl) + manifestCreationServiceImpl := manifest.NewManifestCreationServiceImpl(sugaredLogger, dockerRegistryIpsConfigServiceImpl, chartRefServiceImpl, scopedVariableCMCSManagerImpl, k8sCommonServiceImpl, deployedAppMetricsServiceImpl, imageDigestPolicyServiceImpl, mergeUtil, appCrudOperationServiceImpl, deploymentTemplateServiceImpl, applicationServiceClientImpl, configMapHistoryRepositoryImpl, configMapRepositoryImpl, chartRepositoryImpl, envConfigOverrideRepositoryImpl, environmentRepositoryImpl, pipelineRepositoryImpl, ciArtifactRepositoryImpl, pipelineOverrideRepositoryImpl, pipelineStrategyHistoryRepositoryImpl, pipelineConfigRepositoryImpl, deploymentTemplateHistoryRepositoryImpl, deploymentConfigServiceImpl) deployedConfigurationHistoryServiceImpl := history.NewDeployedConfigurationHistoryServiceImpl(sugaredLogger, userServiceImpl, deploymentTemplateHistoryServiceImpl, pipelineStrategyHistoryServiceImpl, configMapHistoryServiceImpl, cdWorkflowRepositoryImpl, scopedVariableCMCSManagerImpl) userDeploymentRequestRepositoryImpl := repository15.NewUserDeploymentRequestRepositoryImpl(db, transactionUtilImpl) userDeploymentRequestServiceImpl := service2.NewUserDeploymentRequestServiceImpl(sugaredLogger, userDeploymentRequestRepositoryImpl) manifestPushConfigRepositoryImpl := repository10.NewManifestPushConfigRepository(sugaredLogger, db) scanToolExecutionHistoryMappingRepositoryImpl := security.NewScanToolExecutionHistoryMappingRepositoryImpl(db, sugaredLogger) imageScanServiceImpl := security2.NewImageScanServiceImpl(sugaredLogger, imageScanHistoryRepositoryImpl, imageScanResultRepositoryImpl, imageScanObjectMetaRepositoryImpl, cveStoreRepositoryImpl, imageScanDeployInfoRepositoryImpl, userServiceImpl, teamRepositoryImpl, appRepositoryImpl, environmentServiceImpl, ciArtifactRepositoryImpl, policyServiceImpl, pipelineRepositoryImpl, ciPipelineRepositoryImpl, scanToolMetadataRepositoryImpl, scanToolExecutionHistoryMappingRepositoryImpl, cvePolicyRepositoryImpl) - triggerServiceImpl, err := devtronApps.NewTriggerServiceImpl(sugaredLogger, cdWorkflowCommonServiceImpl, gitOpsManifestPushServiceImpl, gitOpsConfigReadServiceImpl, argoK8sClientImpl, acdConfig, argoClientWrapperServiceImpl, pipelineStatusTimelineServiceImpl, chartTemplateServiceImpl, workflowEventPublishServiceImpl, manifestCreationServiceImpl, deployedConfigurationHistoryServiceImpl, argoUserServiceImpl, pipelineStageServiceImpl, globalPluginServiceImpl, customTagServiceImpl, pluginInputVariableParserImpl, prePostCdScriptHistoryServiceImpl, scopedVariableCMCSManagerImpl, workflowServiceImpl, imageDigestPolicyServiceImpl, userServiceImpl, clientImpl, helmAppServiceImpl, enforcerUtilImpl, userDeploymentRequestServiceImpl, helmAppClientImpl, eventSimpleFactoryImpl, eventRESTClientImpl, environmentVariables, appRepositoryImpl, ciPipelineMaterialRepositoryImpl, imageScanHistoryRepositoryImpl, imageScanDeployInfoRepositoryImpl, pipelineRepositoryImpl, pipelineOverrideRepositoryImpl, manifestPushConfigRepositoryImpl, chartRepositoryImpl, environmentRepositoryImpl, cdWorkflowRepositoryImpl, ciWorkflowRepositoryImpl, ciArtifactRepositoryImpl, ciTemplateServiceImpl, materialRepositoryImpl, appLabelRepositoryImpl, ciPipelineRepositoryImpl, appWorkflowRepositoryImpl, dockerArtifactStoreRepositoryImpl, imageScanServiceImpl, k8sServiceImpl, transactionUtilImpl) + triggerServiceImpl, err := devtronApps.NewTriggerServiceImpl(sugaredLogger, cdWorkflowCommonServiceImpl, gitOpsManifestPushServiceImpl, gitOpsConfigReadServiceImpl, argoK8sClientImpl, acdConfig, argoClientWrapperServiceImpl, pipelineStatusTimelineServiceImpl, chartTemplateServiceImpl, workflowEventPublishServiceImpl, manifestCreationServiceImpl, deployedConfigurationHistoryServiceImpl, argoUserServiceImpl, pipelineStageServiceImpl, globalPluginServiceImpl, customTagServiceImpl, pluginInputVariableParserImpl, prePostCdScriptHistoryServiceImpl, scopedVariableCMCSManagerImpl, workflowServiceImpl, imageDigestPolicyServiceImpl, userServiceImpl, clientImpl, helmAppServiceImpl, enforcerUtilImpl, userDeploymentRequestServiceImpl, helmAppClientImpl, eventSimpleFactoryImpl, eventRESTClientImpl, environmentVariables, appRepositoryImpl, ciPipelineMaterialRepositoryImpl, imageScanHistoryRepositoryImpl, imageScanDeployInfoRepositoryImpl, pipelineRepositoryImpl, pipelineOverrideRepositoryImpl, manifestPushConfigRepositoryImpl, chartRepositoryImpl, environmentRepositoryImpl, cdWorkflowRepositoryImpl, ciWorkflowRepositoryImpl, ciArtifactRepositoryImpl, ciTemplateServiceImpl, materialRepositoryImpl, appLabelRepositoryImpl, ciPipelineRepositoryImpl, appWorkflowRepositoryImpl, dockerArtifactStoreRepositoryImpl, imageScanServiceImpl, k8sServiceImpl, transactionUtilImpl, deploymentConfigServiceImpl) if err != nil { return nil, err } commonArtifactServiceImpl := artifacts.NewCommonArtifactServiceImpl(sugaredLogger, ciArtifactRepositoryImpl) - workflowDagExecutorImpl := dag.NewWorkflowDagExecutorImpl(sugaredLogger, pipelineRepositoryImpl, cdWorkflowRepositoryImpl, ciArtifactRepositoryImpl, enforcerUtilImpl, appWorkflowRepositoryImpl, pipelineStageServiceImpl, ciWorkflowRepositoryImpl, ciPipelineRepositoryImpl, pipelineStageRepositoryImpl, globalPluginRepositoryImpl, eventRESTClientImpl, eventSimpleFactoryImpl, customTagServiceImpl, pipelineStatusTimelineServiceImpl, helmAppServiceImpl, cdWorkflowCommonServiceImpl, triggerServiceImpl, userDeploymentRequestServiceImpl, manifestCreationServiceImpl, commonArtifactServiceImpl) + workflowDagExecutorImpl := dag.NewWorkflowDagExecutorImpl(sugaredLogger, pipelineRepositoryImpl, cdWorkflowRepositoryImpl, ciArtifactRepositoryImpl, enforcerUtilImpl, appWorkflowRepositoryImpl, pipelineStageServiceImpl, ciWorkflowRepositoryImpl, ciPipelineRepositoryImpl, pipelineStageRepositoryImpl, globalPluginRepositoryImpl, eventRESTClientImpl, eventSimpleFactoryImpl, customTagServiceImpl, pipelineStatusTimelineServiceImpl, helmAppServiceImpl, cdWorkflowCommonServiceImpl, triggerServiceImpl, userDeploymentRequestServiceImpl, manifestCreationServiceImpl, commonArtifactServiceImpl, deploymentConfigServiceImpl) externalCiRestHandlerImpl := restHandler.NewExternalCiRestHandlerImpl(sugaredLogger, validate, userServiceImpl, enforcerImpl, workflowDagExecutorImpl) pubSubClientRestHandlerImpl := restHandler.NewPubSubClientRestHandlerImpl(pubSubClientServiceImpl, sugaredLogger, ciCdConfig) webhookRouterImpl := router.NewWebhookRouterImpl(gitWebhookRestHandlerImpl, pipelineConfigRestHandlerImpl, externalCiRestHandlerImpl, pubSubClientRestHandlerImpl) @@ -713,7 +713,7 @@ func InitializeApp() (*App, error) { return nil, err } cdPipelineEventPublishServiceImpl := out.NewCDPipelineEventPublishServiceImpl(sugaredLogger, pubSubClientServiceImpl) - workflowStatusServiceImpl, err := status2.NewWorkflowStatusServiceImpl(sugaredLogger, workflowDagExecutorImpl, pipelineStatusTimelineServiceImpl, appServiceImpl, appStatusServiceImpl, acdConfig, appServiceConfig, argoUserServiceImpl, pipelineStatusSyncDetailServiceImpl, argoClientWrapperServiceImpl, cdPipelineEventPublishServiceImpl, cdWorkflowRepositoryImpl, pipelineOverrideRepositoryImpl, installedAppVersionHistoryRepositoryImpl, appRepositoryImpl, environmentRepositoryImpl, installedAppRepositoryImpl, pipelineStatusTimelineRepositoryImpl, pipelineRepositoryImpl, applicationServiceClientImpl, appListingServiceImpl) + workflowStatusServiceImpl, err := status2.NewWorkflowStatusServiceImpl(sugaredLogger, workflowDagExecutorImpl, pipelineStatusTimelineServiceImpl, appServiceImpl, appStatusServiceImpl, acdConfig, appServiceConfig, argoUserServiceImpl, pipelineStatusSyncDetailServiceImpl, argoClientWrapperServiceImpl, cdPipelineEventPublishServiceImpl, cdWorkflowRepositoryImpl, pipelineOverrideRepositoryImpl, installedAppVersionHistoryRepositoryImpl, appRepositoryImpl, environmentRepositoryImpl, installedAppRepositoryImpl, pipelineStatusTimelineRepositoryImpl, pipelineRepositoryImpl, applicationServiceClientImpl, appListingServiceImpl, deploymentConfigServiceImpl) if err != nil { return nil, err } @@ -816,7 +816,7 @@ func InitializeApp() (*App, error) { webhookListenerRouterImpl := router.NewWebhookListenerRouterImpl(webhookEventHandlerImpl) appFilteringRestHandlerImpl := appList.NewAppFilteringRestHandlerImpl(sugaredLogger, teamServiceImpl, enforcerImpl, userServiceImpl, clusterServiceImplExtended, environmentServiceImpl) appFilteringRouterImpl := appList2.NewAppFilteringRouterImpl(appFilteringRestHandlerImpl) - appListingRestHandlerImpl := appList.NewAppListingRestHandlerImpl(applicationServiceClientImpl, appListingServiceImpl, enforcerImpl, pipelineBuilderImpl, sugaredLogger, enforcerUtilImpl, deploymentGroupServiceImpl, userServiceImpl, helmAppClientImpl, helmAppServiceImpl, argoUserServiceImpl, k8sCommonServiceImpl, installedAppDBExtendedServiceImpl, installedAppResourceServiceImpl, cdApplicationStatusUpdateHandlerImpl, pipelineRepositoryImpl, appStatusServiceImpl, installedAppRepositoryImpl, genericNoteServiceImpl, k8sApplicationServiceImpl, generateManifestDeploymentTemplateServiceImpl) + appListingRestHandlerImpl := appList.NewAppListingRestHandlerImpl(applicationServiceClientImpl, appListingServiceImpl, enforcerImpl, pipelineBuilderImpl, sugaredLogger, enforcerUtilImpl, deploymentGroupServiceImpl, userServiceImpl, helmAppClientImpl, helmAppServiceImpl, argoUserServiceImpl, k8sCommonServiceImpl, installedAppDBExtendedServiceImpl, installedAppResourceServiceImpl, cdApplicationStatusUpdateHandlerImpl, pipelineRepositoryImpl, appStatusServiceImpl, installedAppRepositoryImpl, genericNoteServiceImpl, k8sApplicationServiceImpl, generateManifestDeploymentTemplateServiceImpl, deploymentConfigServiceImpl) appListingRouterImpl := appList2.NewAppListingRouterImpl(appListingRestHandlerImpl) appInfoRestHandlerImpl := appInfo.NewAppInfoRestHandlerImpl(sugaredLogger, appCrudOperationServiceImpl, userServiceImpl, validate, enforcerUtilImpl, enforcerImpl, helmAppServiceImpl, enforcerUtilHelmImpl, genericNoteServiceImpl) appInfoRouterImpl := appInfo2.NewAppInfoRouterImpl(sugaredLogger, appInfoRestHandlerImpl) @@ -936,7 +936,7 @@ func InitializeApp() (*App, error) { cdWorkflowServiceImpl := cd.NewCdWorkflowServiceImpl(sugaredLogger, cdWorkflowRepositoryImpl) cdWorkflowRunnerServiceImpl := cd.NewCdWorkflowRunnerServiceImpl(sugaredLogger, cdWorkflowRepositoryImpl) webhookServiceImpl := pipeline.NewWebhookServiceImpl(ciArtifactRepositoryImpl, sugaredLogger, ciPipelineRepositoryImpl, ciWorkflowRepositoryImpl, cdWorkflowCommonServiceImpl) - workflowEventProcessorImpl, err := in.NewWorkflowEventProcessorImpl(sugaredLogger, pubSubClientServiceImpl, cdWorkflowServiceImpl, cdWorkflowRunnerServiceImpl, workflowDagExecutorImpl, argoUserServiceImpl, ciHandlerImpl, cdHandlerImpl, eventSimpleFactoryImpl, eventRESTClientImpl, triggerServiceImpl, deployedAppServiceImpl, webhookServiceImpl, validate, environmentVariables, cdWorkflowCommonServiceImpl, cdPipelineConfigServiceImpl, userDeploymentRequestServiceImpl, pipelineRepositoryImpl, ciArtifactRepositoryImpl, cdWorkflowRepositoryImpl) + workflowEventProcessorImpl, err := in.NewWorkflowEventProcessorImpl(sugaredLogger, pubSubClientServiceImpl, cdWorkflowServiceImpl, cdWorkflowRunnerServiceImpl, workflowDagExecutorImpl, argoUserServiceImpl, ciHandlerImpl, cdHandlerImpl, eventSimpleFactoryImpl, eventRESTClientImpl, triggerServiceImpl, deployedAppServiceImpl, webhookServiceImpl, validate, environmentVariables, cdWorkflowCommonServiceImpl, cdPipelineConfigServiceImpl, userDeploymentRequestServiceImpl, pipelineRepositoryImpl, ciArtifactRepositoryImpl, cdWorkflowRepositoryImpl, deploymentConfigServiceImpl) if err != nil { return nil, err } From 689c0e87baa3c66c98cc910fba1c4bd9db8a8da5 Mon Sep 17 00:00:00 2001 From: ayushmaheshwari Date: Sun, 30 Jun 2024 22:46:51 +0530 Subject: [PATCH 03/45] changes for helm apps --- api/appStore/InstalledAppRestHandler.go | 15 ++-- api/bean/AppView.go | 82 ++++++++--------- .../app/appList/AppListingRestHandler.go | 34 +++---- .../repository/deploymentConfig/repository.go | 46 ++++++---- .../status/PipelineStatusTimelineService.go | 7 +- pkg/appStore/adapter/Adapter.go | 17 ++-- pkg/appStore/bean/bean.go | 22 +++++ .../repository/InstalledAppRepository.go | 11 ++- .../service/AppStoreDeploymentDBService.go | 28 +++++- .../service/AppStoreDeploymentService.go | 88 ++++++++++++++----- .../service/EAMode/InstalledAppDBService.go | 36 ++++++-- .../deployment/FullModeDeploymentService.go | 14 ++- .../deployment/InstalledAppGitOpsService.go | 12 ++- ...InstalledAppDeploymentTypeChangeService.go | 25 ++++-- .../service/FullMode/resource/NotesService.go | 8 +- .../FullMode/resource/ResourceTreeService.go | 15 ++-- pkg/chart/ChartService.go | 6 +- .../common/deploymentConfigService.go | 82 +++++++++++++++-- .../gitOps/config/GitOpsConfigReadService.go | 10 +++ .../DeploymentPipelineConfigService.go | 10 ++- scripts/sql/260_deployment_config.up.sql | 22 +++++ wire_gen.go | 14 +-- 22 files changed, 445 insertions(+), 159 deletions(-) create mode 100644 scripts/sql/260_deployment_config.up.sql diff --git a/api/appStore/InstalledAppRestHandler.go b/api/appStore/InstalledAppRestHandler.go index 9077817a59..a765340119 100644 --- a/api/appStore/InstalledAppRestHandler.go +++ b/api/appStore/InstalledAppRestHandler.go @@ -27,6 +27,7 @@ import ( "github.com/devtron-labs/devtron/pkg/appStore/installedApp/service/FullMode/resource" util3 "github.com/devtron-labs/devtron/pkg/appStore/util" "github.com/devtron-labs/devtron/pkg/bean" + "gopkg.in/go-playground/validator.v9" "net/http" "reflect" "strconv" @@ -48,6 +49,7 @@ import ( "github.com/devtron-labs/devtron/pkg/auth/authorisation/casbin" "github.com/devtron-labs/devtron/pkg/auth/user" "github.com/devtron-labs/devtron/pkg/cluster" + bean3 "github.com/devtron-labs/devtron/pkg/deployment/common/bean" "github.com/devtron-labs/devtron/util" "github.com/devtron-labs/devtron/util/argo" "github.com/devtron-labs/devtron/util/rbac" @@ -55,7 +57,6 @@ import ( "github.com/go-pg/pg" "github.com/gorilla/mux" "go.uber.org/zap" - "gopkg.in/go-playground/validator.v9" ) type InstalledAppRestHandler interface { @@ -744,8 +745,8 @@ func (handler *InstalledAppRestHandlerImpl) FetchAppDetailsForInstalledApp(w htt resourceTreeAndNotesContainer.ResourceTree = map[string]interface{}{} if len(installedApp.App.AppName) > 0 && len(installedApp.Environment.Name) > 0 { - err = handler.fetchResourceTree(w, r, &resourceTreeAndNotesContainer, *installedApp, "", "") - if installedApp.DeploymentAppType == util2.PIPELINE_DEPLOYMENT_TYPE_ACD { + err = handler.fetchResourceTree(w, r, &resourceTreeAndNotesContainer, *installedApp, appDetail.DeploymentConfig, "", "") + if appDetail.DeploymentAppType == util2.PIPELINE_DEPLOYMENT_TYPE_ACD { apiError, ok := err.(*util2.ApiError) if ok && apiError != nil { if apiError.Code == constants.AppDetailResourceTreeNotFound && installedApp.DeploymentAppDeleteRequest == true { @@ -868,8 +869,8 @@ func (handler *InstalledAppRestHandlerImpl) FetchResourceTree(w http.ResponseWri resourceTreeAndNotesContainer.ResourceTree = map[string]interface{}{} if len(installedApp.App.AppName) > 0 && len(installedApp.Environment.Name) > 0 { - err = handler.fetchResourceTree(w, r, &resourceTreeAndNotesContainer, *installedApp, appDetail.HelmReleaseInstallStatus, appDetail.Status) - if installedApp.DeploymentAppType == util2.PIPELINE_DEPLOYMENT_TYPE_ACD { + err = handler.fetchResourceTree(w, r, &resourceTreeAndNotesContainer, *installedApp, appDetail.DeploymentConfig, appDetail.HelmReleaseInstallStatus, appDetail.Status) + if appDetail.DeploymentAppType == util2.PIPELINE_DEPLOYMENT_TYPE_ACD { //resource tree has been fetched now prepare to sync application deployment status with this resource tree call handler.syncDeploymentStatusWithResourceTreeCall(appDetail) apiError, ok := err.(*util2.ApiError) @@ -960,10 +961,10 @@ func (handler *InstalledAppRestHandlerImpl) FetchResourceTreeForACDApp(w http.Re common.WriteJsonResp(w, err, appDetail, http.StatusOK) } -func (handler *InstalledAppRestHandlerImpl) fetchResourceTree(w http.ResponseWriter, r *http.Request, resourceTreeAndNotesContainer *bean2.AppDetailsContainer, installedApp repository.InstalledApps, helmReleaseInstallStatus string, status string) error { +func (handler *InstalledAppRestHandlerImpl) fetchResourceTree(w http.ResponseWriter, r *http.Request, resourceTreeAndNotesContainer *bean2.AppDetailsContainer, installedApp repository.InstalledApps, deploymentConfig *bean3.DeploymentConfig, helmReleaseInstallStatus string, status string) error { ctx := r.Context() cn, _ := w.(http.CloseNotifier) - err := handler.installedAppResourceService.FetchResourceTree(ctx, cn, resourceTreeAndNotesContainer, installedApp, helmReleaseInstallStatus, status) + err := handler.installedAppResourceService.FetchResourceTree(ctx, cn, resourceTreeAndNotesContainer, installedApp, deploymentConfig, helmReleaseInstallStatus, status) return err } diff --git a/api/bean/AppView.go b/api/bean/AppView.go index 03314b52c7..03ca0809cb 100644 --- a/api/bean/AppView.go +++ b/api/bean/AppView.go @@ -19,6 +19,7 @@ package bean import ( "encoding/json" "github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1" + "github.com/devtron-labs/devtron/pkg/deployment/common/bean" "time" ) @@ -141,46 +142,47 @@ type AppEnvironmentContainer struct { } type DeploymentDetailContainer struct { - InstalledAppId int `json:"installedAppId,omitempty"` - AppId int `json:"appId,omitempty"` - CdPipelineId int `json:"cdPipelineId,omitempty"` - TriggerType string `json:"triggerType,omitempty"` - ParentEnvironmentName string `json:"parentEnvironmentName"` - AppStoreInstalledAppVersionId int `json:"appStoreInstalledAppVersionId,omitempty"` - AppStoreChartName string `json:"appStoreChartName,omitempty"` - AppStoreChartId int `json:"appStoreChartId,omitempty"` - AppStoreAppName string `json:"appStoreAppName,omitempty"` - AppStoreAppVersion string `json:"appStoreAppVersion,omitempty"` - AppName string `json:"appName"` - EnvironmentId int `json:"environmentId"` - EnvironmentName string `json:"environmentName"` - Namespace string `json:"namespace,omitempty"` - Status string `json:"status,omitempty"` - StatusMessage string `json:"statusMessage,omitempty"` - LastDeployedTime string `json:"lastDeployedTime,omitempty"` - LastDeployedBy string `json:"lastDeployedBy,omitempty"` - MaterialInfo json.RawMessage `json:"materialInfo,omitempty"` - MaterialInfoJsonString string `json:"-"` - ReleaseVersion string `json:"releaseVersion,omitempty"` - Default bool `json:"default,omitempty"` - DataSource string `json:"dataSource,omitempty"` - LastDeployedPipeline string `json:"lastDeployedPipeline,omitempty"` - Deprecated bool `json:"deprecated"` - K8sVersion string `json:"k8sVersion"` - CiArtifactId int `json:"ciArtifactId"` - ParentArtifactId int `json:"parentArtifactId"` - ClusterId int `json:"clusterId"` - DeploymentAppType string `json:"deploymentAppType"` - CiPipelineId int `json:"ciPipelineId,omitempty"` - IsExternalCi bool `json:"externalCi"` - ClusterName string `json:"clusterName,omitempty"` - DockerRegistryId string `json:"dockerRegistryId,omitempty"` - IpsAccessProvided bool `json:"ipsAccessProvided"` - DeploymentAppDeleteRequest bool `json:"deploymentAppDeleteRequest"` - Description string `json:"description" validate:"max=40"` - IsVirtualEnvironment bool `json:"isVirtualEnvironment"` - HelmPackageName string `json:"helmPackageName"` - HelmReleaseInstallStatus string `json:"-"` + InstalledAppId int `json:"installedAppId,omitempty"` + AppId int `json:"appId,omitempty"` + CdPipelineId int `json:"cdPipelineId,omitempty"` + TriggerType string `json:"triggerType,omitempty"` + ParentEnvironmentName string `json:"parentEnvironmentName"` + AppStoreInstalledAppVersionId int `json:"appStoreInstalledAppVersionId,omitempty"` + AppStoreChartName string `json:"appStoreChartName,omitempty"` + AppStoreChartId int `json:"appStoreChartId,omitempty"` + AppStoreAppName string `json:"appStoreAppName,omitempty"` + AppStoreAppVersion string `json:"appStoreAppVersion,omitempty"` + AppName string `json:"appName"` + EnvironmentId int `json:"environmentId"` + EnvironmentName string `json:"environmentName"` + Namespace string `json:"namespace,omitempty"` + Status string `json:"status,omitempty"` + StatusMessage string `json:"statusMessage,omitempty"` + LastDeployedTime string `json:"lastDeployedTime,omitempty"` + LastDeployedBy string `json:"lastDeployedBy,omitempty"` + MaterialInfo json.RawMessage `json:"materialInfo,omitempty"` + MaterialInfoJsonString string `json:"-"` + ReleaseVersion string `json:"releaseVersion,omitempty"` + Default bool `json:"default,omitempty"` + DataSource string `json:"dataSource,omitempty"` + LastDeployedPipeline string `json:"lastDeployedPipeline,omitempty"` + Deprecated bool `json:"deprecated"` + K8sVersion string `json:"k8sVersion"` + CiArtifactId int `json:"ciArtifactId"` + ParentArtifactId int `json:"parentArtifactId"` + ClusterId int `json:"clusterId"` + DeploymentAppType string `json:"deploymentAppType"` + CiPipelineId int `json:"ciPipelineId,omitempty"` + IsExternalCi bool `json:"externalCi"` + ClusterName string `json:"clusterName,omitempty"` + DockerRegistryId string `json:"dockerRegistryId,omitempty"` + IpsAccessProvided bool `json:"ipsAccessProvided"` + DeploymentAppDeleteRequest bool `json:"deploymentAppDeleteRequest"` + Description string `json:"description" validate:"max=40"` + IsVirtualEnvironment bool `json:"isVirtualEnvironment"` + HelmPackageName string `json:"helmPackageName"` + HelmReleaseInstallStatus string `json:"-"` + DeploymentConfig *bean.DeploymentConfig `json:"-"` } type AppDetailContainer struct { diff --git a/api/restHandler/app/appList/AppListingRestHandler.go b/api/restHandler/app/appList/AppListingRestHandler.go index fe9be69008..7a9133e14c 100644 --- a/api/restHandler/app/appList/AppListingRestHandler.go +++ b/api/restHandler/app/appList/AppListingRestHandler.go @@ -21,28 +21,18 @@ import ( "encoding/json" "errors" "fmt" - "github.com/devtron-labs/devtron/api/helm-app/gRPC" - client "github.com/devtron-labs/devtron/api/helm-app/service" - util3 "github.com/devtron-labs/devtron/api/util" - argoApplication "github.com/devtron-labs/devtron/client/argocdServer/bean" - "github.com/devtron-labs/devtron/pkg/appStore/installedApp/service/FullMode" - "github.com/devtron-labs/devtron/pkg/appStore/installedApp/service/FullMode/resource" - util4 "github.com/devtron-labs/devtron/pkg/appStore/util" - bean2 "github.com/devtron-labs/devtron/pkg/cluster/repository/bean" - common2 "github.com/devtron-labs/devtron/pkg/deployment/common" - bean3 "github.com/devtron-labs/devtron/pkg/deployment/common/bean" - "net/http" - "strconv" - "time" - application2 "github.com/argoproj/argo-cd/v2/pkg/apiclient/application" "github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1" k8sCommonBean "github.com/devtron-labs/common-lib/utils/k8s/commonBean" "github.com/devtron-labs/common-lib/utils/k8s/health" k8sObjectUtils "github.com/devtron-labs/common-lib/utils/k8sObjectsUtil" "github.com/devtron-labs/devtron/api/bean" + "github.com/devtron-labs/devtron/api/helm-app/gRPC" + client "github.com/devtron-labs/devtron/api/helm-app/service" "github.com/devtron-labs/devtron/api/restHandler/common" + util3 "github.com/devtron-labs/devtron/api/util" "github.com/devtron-labs/devtron/client/argocdServer/application" + argoApplication "github.com/devtron-labs/devtron/client/argocdServer/bean" "github.com/devtron-labs/devtron/client/cron" "github.com/devtron-labs/devtron/internal/constants" "github.com/devtron-labs/devtron/internal/middleware" @@ -52,9 +42,16 @@ import ( "github.com/devtron-labs/devtron/pkg/app" "github.com/devtron-labs/devtron/pkg/appStatus" "github.com/devtron-labs/devtron/pkg/appStore/installedApp/repository" + "github.com/devtron-labs/devtron/pkg/appStore/installedApp/service/FullMode" + "github.com/devtron-labs/devtron/pkg/appStore/installedApp/service/FullMode/resource" + util4 "github.com/devtron-labs/devtron/pkg/appStore/util" "github.com/devtron-labs/devtron/pkg/auth/authorisation/casbin" "github.com/devtron-labs/devtron/pkg/auth/user" "github.com/devtron-labs/devtron/pkg/cluster" + bean2 "github.com/devtron-labs/devtron/pkg/cluster/repository/bean" + common2 "github.com/devtron-labs/devtron/pkg/deployment/common" + bean3 "github.com/devtron-labs/devtron/pkg/deployment/common/bean" + bean4 "github.com/devtron-labs/devtron/pkg/deployment/common/bean" "github.com/devtron-labs/devtron/pkg/deploymentGroup" "github.com/devtron-labs/devtron/pkg/generateManifest" "github.com/devtron-labs/devtron/pkg/genericNotes" @@ -69,6 +66,9 @@ import ( "github.com/gorilla/mux" "go.opentelemetry.io/otel" "go.uber.org/zap" + "net/http" + "strconv" + "time" ) type AppListingRestHandler interface { @@ -852,10 +852,10 @@ func (handler AppListingRestHandlerImpl) RedirectToLinkouts(w http.ResponseWrite } http.Redirect(w, r, link, http.StatusOK) } -func (handler AppListingRestHandlerImpl) fetchResourceTreeFromInstallAppService(w http.ResponseWriter, r *http.Request, resourceTreeAndNotesContainer bean.AppDetailsContainer, installedApps repository.InstalledApps) (bean.AppDetailsContainer, error) { +func (handler AppListingRestHandlerImpl) fetchResourceTreeFromInstallAppService(w http.ResponseWriter, r *http.Request, resourceTreeAndNotesContainer bean.AppDetailsContainer, installedApps repository.InstalledApps, deploymentConfig *bean4.DeploymentConfig) (bean.AppDetailsContainer, error) { rctx := r.Context() cn, _ := w.(http.CloseNotifier) - err := handler.installedAppResourceService.FetchResourceTree(rctx, cn, &resourceTreeAndNotesContainer, installedApps, "", "") + err := handler.installedAppResourceService.FetchResourceTree(rctx, cn, &resourceTreeAndNotesContainer, installedApps, deploymentConfig, "", "") return resourceTreeAndNotesContainer, err } func (handler AppListingRestHandlerImpl) GetHostUrlsByBatch(w http.ResponseWriter, r *http.Request) { @@ -932,7 +932,7 @@ func (handler AppListingRestHandlerImpl) GetHostUrlsByBatch(w http.ResponseWrite handler.installedAppService.ChangeAppNameToDisplayNameForInstalledApp(installedApp) } resourceTreeAndNotesContainer := bean.AppDetailsContainer{} - resourceTreeAndNotesContainer, err = handler.fetchResourceTreeFromInstallAppService(w, r, resourceTreeAndNotesContainer, *installedApp) + resourceTreeAndNotesContainer, err = handler.fetchResourceTreeFromInstallAppService(w, r, resourceTreeAndNotesContainer, *installedApp, appDetail.DeploymentConfig) if err != nil { common.WriteJsonResp(w, fmt.Errorf("error in fetching resource tree"), nil, http.StatusInternalServerError) return diff --git a/internal/sql/repository/deploymentConfig/repository.go b/internal/sql/repository/deploymentConfig/repository.go index 6017683c89..c994f2ffa9 100644 --- a/internal/sql/repository/deploymentConfig/repository.go +++ b/internal/sql/repository/deploymentConfig/repository.go @@ -6,23 +6,25 @@ import ( ) type DeploymentConfig struct { - Id int `sql:"id,pk"` - AppId int `sql:"app_id"` - EnvironmentId int `sql:"environment_id"` - DeploymentAppType string `sql:"deployment_app_type"` - ConfigType string `sql:"config_type"` - RepoUrl string `sql:"repo_url"` - ChartLocation string `sql:"chart_location"` - CredentialType string `sql:"credential_type"` - CredentialIdInt int `sql:"credential_id_int"` - CredentialIdString string `sql:"credential_id_string"` - Active bool `sql:"active"` + tableName struct{} `sql:"deployment_config" pg:",discard_unknown_columns"` + Id int `sql:"id,pk"` + AppId int `sql:"app_id"` + EnvironmentId int `sql:"environment_id"` + DeploymentAppType string `sql:"deployment_app_type"` + ConfigType string `sql:"config_type"` + RepoUrl string `sql:"repo_url"` + RepoName string `sql:"repo_name"` + ChartLocation string `sql:"chart_location"` + CredentialType string `sql:"credential_type"` + CredentialIdInt int `sql:"credential_id_int"` + CredentialIdString string `sql:"credential_id_string"` + Active bool `sql:"active"` sql.AuditLog } type Repository interface { - Save(config *DeploymentConfig) (*DeploymentConfig, error) - Update(config *DeploymentConfig) (*DeploymentConfig, error) + Save(tx *pg.Tx, config *DeploymentConfig) (*DeploymentConfig, error) + Update(tx *pg.Tx, config *DeploymentConfig) (*DeploymentConfig, error) GetById(id int) (*DeploymentConfig, error) GetByAppIdAndEnvId(appId, envId int) (*DeploymentConfig, error) GetAppLevelConfig(appId int) (*DeploymentConfig, error) @@ -36,13 +38,23 @@ func NewRepositoryImpl(dbConnection *pg.DB) *RepositoryImpl { return &RepositoryImpl{dbConnection: dbConnection} } -func (impl RepositoryImpl) Save(config *DeploymentConfig) (*DeploymentConfig, error) { - err := impl.dbConnection.Insert(config) +func (impl RepositoryImpl) Save(tx *pg.Tx, config *DeploymentConfig) (*DeploymentConfig, error) { + var err error + if tx != nil { + err = tx.Insert(config) + } else { + err = impl.dbConnection.Insert(config) + } return config, err } -func (impl RepositoryImpl) Update(config *DeploymentConfig) (*DeploymentConfig, error) { - err := impl.dbConnection.Update(config) +func (impl RepositoryImpl) Update(tx *pg.Tx, config *DeploymentConfig) (*DeploymentConfig, error) { + var err error + if tx != nil { + err = tx.Update(config) + } else { + err = impl.dbConnection.Update(config) + } return config, err } diff --git a/pkg/app/status/PipelineStatusTimelineService.go b/pkg/app/status/PipelineStatusTimelineService.go index b4b24cc724..2706e577df 100644 --- a/pkg/app/status/PipelineStatusTimelineService.go +++ b/pkg/app/status/PipelineStatusTimelineService.go @@ -302,6 +302,11 @@ func (impl *PipelineStatusTimelineServiceImpl) FetchTimelinesForAppStore(install impl.logger.Errorw("error in getting installed_app_version by appId and envId", "err", err, "appId", installedAppId, "envId", envId) return nil, err } + deploymentConfig, err := impl.deploymentConfigService.GetDeploymentConfigForHelmApp(installedAppVersion.InstalledApp.AppId, installedAppVersion.InstalledApp.EnvironmentId) + if err != nil { + impl.logger.Errorw("error in getiting deployment config db object by appId and envId", "appId", installedAppVersion.InstalledApp.AppId, "envId", installedAppVersion.InstalledApp.EnvironmentId, "err", err) + return nil, err + } installedAppVersionHistory := &repository.InstalledAppVersionHistory{} if installedAppVersionHistoryId == 0 { //fetching latest installed_app_version_history from installed_app_version_id @@ -327,7 +332,7 @@ func (impl *PipelineStatusTimelineServiceImpl) FetchTimelinesForAppStore(install deploymentFinishedOn = installedAppVersionHistory.FinishedOn } installedAppVersionHistoryStatus = installedAppVersionHistory.Status - deploymentAppType = installedAppVersion.InstalledApp.DeploymentAppType + deploymentAppType = deploymentConfig.DeploymentAppType triggeredByUserEmailId, err := impl.userService.GetEmailById(installedAppVersionHistory.CreatedBy) if err != nil { impl.logger.Errorw("error in getting user email by id", "err", err, "userId", installedAppVersionHistory.CreatedBy) diff --git a/pkg/appStore/adapter/Adapter.go b/pkg/appStore/adapter/Adapter.go index dc575eb90d..440ceaf0bf 100644 --- a/pkg/appStore/adapter/Adapter.go +++ b/pkg/appStore/adapter/Adapter.go @@ -28,6 +28,7 @@ import ( "github.com/devtron-labs/devtron/pkg/bean" "github.com/devtron-labs/devtron/pkg/cluster/adapter" clutserBean "github.com/devtron-labs/devtron/pkg/cluster/repository/bean" + bean2 "github.com/devtron-labs/devtron/pkg/deployment/common/bean" "time" ) @@ -114,7 +115,7 @@ func NewInstalledAppDeploymentAction(deploymentAppType string) *appStoreBean.Ins } // GenerateInstallAppVersionDTO converts repository.InstalledApps and repository.InstalledAppVersions db object to appStoreBean.InstallAppVersionDTO bean -func GenerateInstallAppVersionDTO(installedApp *repository.InstalledApps, installedAppVersion *repository.InstalledAppVersions) *appStoreBean.InstallAppVersionDTO { +func GenerateInstallAppVersionDTO(installedApp *repository.InstalledApps, installedAppVersion *repository.InstalledAppVersions, config *bean2.DeploymentConfig) *appStoreBean.InstallAppVersionDTO { chartVersionApp := installedAppVersion.AppStoreApplicationVersion var chartRepoName, chartRepoUrl, Username, Password string @@ -149,7 +150,7 @@ func GenerateInstallAppVersionDTO(installedApp *repository.InstalledApps, instal }, }, } - UpdateInstallAppDetails(installAppDto, installedApp) + UpdateInstallAppDetails(installAppDto, installedApp, config) UpdateInstalledAppVersionsMetaData(installAppDto, installedAppVersion) UpdateAppDetails(installAppDto, &installedApp.App) UpdateAdditionalEnvDetails(installAppDto, envBean) @@ -159,7 +160,7 @@ func GenerateInstallAppVersionDTO(installedApp *repository.InstalledApps, instal // GenerateInstallAppVersionMinDTO converts repository.InstalledApps db object to appStoreBean.InstallAppVersionDTO bean; // Note: It only generates a minimal DTO and doesn't include repository.InstalledAppVersions data, also it's safe not to // use this bean for creating db model again -func GenerateInstallAppVersionMinDTO(installedApp *repository.InstalledApps) *appStoreBean.InstallAppVersionDTO { +func GenerateInstallAppVersionMinDTO(installedApp *repository.InstalledApps, config *bean2.DeploymentConfig) *appStoreBean.InstallAppVersionDTO { installAppVersionDto := &appStoreBean.InstallAppVersionDTO{ EnvironmentId: installedApp.EnvironmentId, InstalledAppId: installedApp.Id, @@ -171,7 +172,7 @@ func GenerateInstallAppVersionMinDTO(installedApp *repository.InstalledApps) *ap EnvironmentName: installedApp.Environment.Name, TeamId: installedApp.App.TeamId, TeamName: installedApp.App.Team.Name, - DeploymentAppType: installedApp.DeploymentAppType, + DeploymentAppType: config.DeploymentAppType, IsVirtualEnvironment: installedApp.Environment.IsVirtualEnvironment, } if util4.IsExternalChartStoreApp(installedApp.App.DisplayName) { @@ -245,7 +246,7 @@ func UpdateAppDetails(request *appStoreBean.InstallAppVersionDTO, app *app.App) } // UpdateInstallAppDetails update repository.InstalledApps data into the same InstallAppVersionDTO -func UpdateInstallAppDetails(request *appStoreBean.InstallAppVersionDTO, installedApp *repository.InstalledApps) { +func UpdateInstallAppDetails(request *appStoreBean.InstallAppVersionDTO, installedApp *repository.InstalledApps, config *bean2.DeploymentConfig) { if request == nil || installedApp == nil { return } @@ -253,9 +254,9 @@ func UpdateInstallAppDetails(request *appStoreBean.InstallAppVersionDTO, install request.AppId = installedApp.AppId request.EnvironmentId = installedApp.EnvironmentId request.Status = installedApp.Status - request.DeploymentAppType = installedApp.DeploymentAppType - if util.IsAcdApp(installedApp.DeploymentAppType) { - request.GitOpsRepoURL = installedApp.GitOpsRepoUrl + request.DeploymentAppType = config.DeploymentAppType + if util.IsAcdApp(config.DeploymentAppType) { + request.GitOpsRepoURL = config.RepoURL } } diff --git a/pkg/appStore/bean/bean.go b/pkg/appStore/bean/bean.go index a8b6c5c582..c02c8af36a 100644 --- a/pkg/appStore/bean/bean.go +++ b/pkg/appStore/bean/bean.go @@ -22,6 +22,8 @@ import ( apiBean "github.com/devtron-labs/devtron/api/bean/gitOps" openapi "github.com/devtron-labs/devtron/api/helm-app/openapiClient" "github.com/devtron-labs/devtron/pkg/cluster/repository/bean" + bean2 "github.com/devtron-labs/devtron/pkg/deployment/common/bean" + "github.com/devtron-labs/devtron/util/gitUtil" "time" ) @@ -170,6 +172,26 @@ type InstallAppVersionChartRepoDTO struct { Password string `json:"-"` } +func (chart *InstallAppVersionDTO) GetDeploymentConfig(gitOpsConfigId int) *bean2.DeploymentConfig { + var configType string + if chart.IsCustomRepository { + configType = bean2.CUSTOM.String() + } else { + configType = bean2.SYSTEM_GENERATED.String() + } + return &bean2.DeploymentConfig{ + AppId: chart.AppId, + EnvironmentId: chart.EnvironmentId, + ConfigType: configType, + DeploymentAppType: chart.DeploymentAppType, + RepoURL: chart.GitOpsRepoURL, + RepoName: gitUtil.GetGitRepoNameFromGitRepoUrl(chart.GitOpsRepoURL), + CredentialType: bean2.GitOps.String(), + CredentialIdInt: gitOpsConfigId, + Active: true, + } +} + // / type RefChartProxyDir string diff --git a/pkg/appStore/installedApp/repository/InstalledAppRepository.go b/pkg/appStore/installedApp/repository/InstalledAppRepository.go index 3fb6631d64..287b2b4f4e 100644 --- a/pkg/appStore/installedApp/repository/InstalledAppRepository.go +++ b/pkg/appStore/installedApp/repository/InstalledAppRepository.go @@ -20,6 +20,7 @@ import ( "fmt" "github.com/devtron-labs/common-lib/utils/k8s/health" "github.com/devtron-labs/devtron/internal/sql/repository/app" + "github.com/devtron-labs/devtron/internal/sql/repository/deploymentConfig" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/timelineStatus" util2 "github.com/devtron-labs/devtron/internal/util" @@ -53,6 +54,7 @@ type InstalledApps struct { Notes string `json:"notes"` App app.App Environment repository.Environment + DeploymentConfig deploymentConfig.DeploymentConfig sql.AuditLog } @@ -711,14 +713,16 @@ func (impl InstalledAppRepositoryImpl) GetDeploymentSuccessfulStatusCountForTele } func (impl InstalledAppRepositoryImpl) GetGitOpsInstalledAppsWhereArgoAppDeletedIsTrue(installedAppId int, envId int) (InstalledApps, error) { + //TODO: test var installedApps InstalledApps err := impl.dbConnection.Model(&installedApps). - Column("installed_apps.*", "App.app_name", "Environment.namespace", "Environment.cluster_id", "Environment.environment_name"). + Column("installed_apps.*", "App.app_name", "Environment.namespace", "Environment.cluster_id", "Environment.environment_name", "DeploymentConfig"). + Join(" LEFT JOIN deployment_configuration dc on (installed_apps.app_id=dc.app_id and installed_apps.environment_id=dc.environment_id)"). Where("deployment_app_delete_request = ?", true). Where("installed_apps.active = ?", true). Where("installed_apps.id = ?", installedAppId). Where("installed_apps.environment_id = ?", envId). - Where("deployment_app_type = ?", util2.PIPELINE_DEPLOYMENT_TYPE_ACD). + Where("(installed_apps.deployment_app_type = ? or dc.deployment_app_type = ? )", util2.PIPELINE_DEPLOYMENT_TYPE_ACD, util2.PIPELINE_DEPLOYMENT_TYPE_ACD). Select() if err != nil && err != pg.ErrNoRows { impl.Logger.Errorw("error in fetching pipeline while udating delete status", "err", err) @@ -861,8 +865,9 @@ func (impl InstalledAppRepositoryImpl) GetActiveInstalledAppByEnvIdAndDeployment query := impl.dbConnection. Model(&installedApps). - Column("installed_apps.*", "App", "Environment"). + Column("installed_apps.*", "App", "Environment", "DeploymentConfig"). Join("inner join app a on installed_apps.app_id = a.id"). + Join("inner join "). Where("installed_apps.environment_id = ?", envId). Where("installed_apps.deployment_app_type = ?", deploymentType). Where("installed_apps.active = ?", true) diff --git a/pkg/appStore/installedApp/service/AppStoreDeploymentDBService.go b/pkg/appStore/installedApp/service/AppStoreDeploymentDBService.go index 71b376bb5f..cfa9531125 100644 --- a/pkg/appStore/installedApp/service/AppStoreDeploymentDBService.go +++ b/pkg/appStore/installedApp/service/AppStoreDeploymentDBService.go @@ -35,6 +35,7 @@ import ( "github.com/devtron-labs/devtron/pkg/bean" clusterService "github.com/devtron-labs/devtron/pkg/cluster" clutserBean "github.com/devtron-labs/devtron/pkg/cluster/repository/bean" + "github.com/devtron-labs/devtron/pkg/deployment/common" "github.com/devtron-labs/devtron/pkg/deployment/gitOps/config" gitOpsBean "github.com/devtron-labs/devtron/pkg/deployment/gitOps/config/bean" validationBean "github.com/devtron-labs/devtron/pkg/deployment/gitOps/validation/bean" @@ -89,6 +90,7 @@ type AppStoreDeploymentDBServiceImpl struct { fullModeDeploymentService deployment.FullModeDeploymentService appStoreValidator AppStoreValidator installedAppDbService EAMode.InstalledAppDBService + deploymentConfigService common.DeploymentConfigService } func NewAppStoreDeploymentDBServiceImpl(logger *zap.SugaredLogger, @@ -102,7 +104,8 @@ func NewAppStoreDeploymentDBServiceImpl(logger *zap.SugaredLogger, gitOpsConfigReadService config.GitOpsConfigReadService, deploymentTypeOverrideService providerConfig.DeploymentTypeOverrideService, fullModeDeploymentService deployment.FullModeDeploymentService, appStoreValidator AppStoreValidator, - installedAppDbService EAMode.InstalledAppDBService) *AppStoreDeploymentDBServiceImpl { + installedAppDbService EAMode.InstalledAppDBService, + deploymentConfigService common.DeploymentConfigService) *AppStoreDeploymentDBServiceImpl { return &AppStoreDeploymentDBServiceImpl{ logger: logger, installedAppRepository: installedAppRepository, @@ -117,6 +120,7 @@ func NewAppStoreDeploymentDBServiceImpl(logger *zap.SugaredLogger, fullModeDeploymentService: fullModeDeploymentService, appStoreValidator: appStoreValidator, installedAppDbService: installedAppDbService, + deploymentConfigService: deploymentConfigService, } } @@ -203,9 +207,24 @@ func (impl *AppStoreDeploymentDBServiceImpl) AppStoreDeployOperationDB(installRe impl.logger.Errorw("error while creating install app", "error", err) return nil, err } + installRequest.InstalledAppId = installedApp.Id // Stage 3: ends + //save deployment config + gitOpsConfig, err := impl.gitOpsConfigReadService.GetGitOpsConfigActive() + if err != nil { + impl.logger.Errorw("error in getting active gitops config", "err", err) + return nil, err + } + + deploymentConfig := installRequest.GetDeploymentConfig(gitOpsConfig.Id) + deploymentConfig, err = impl.deploymentConfigService.CreateOrUpdateConfig(tx, deploymentConfig, installRequest.UserId) + if err != nil { + impl.logger.Errorw("error in creating deployment config for installed app", "appId", installedApp.AppId, "envId", installedApp.EnvironmentId, "err", err) + return nil, err + } + // Stage 4: save installed_app_versions model installedAppVersions := adapter.NewInstallAppVersionsModel(installRequest) _, err = impl.installedAppRepository.CreateInstalledAppVersion(installedAppVersions, tx) @@ -291,7 +310,12 @@ func (impl *AppStoreDeploymentDBServiceImpl) GetInstalledApp(id int) (*appStoreB impl.logger.Errorw("error while fetching from db", "error", err) return nil, err } - chartTemplate := adapter.GenerateInstallAppVersionMinDTO(app) + deploymentConfig, err := impl.deploymentConfigService.GetDeploymentConfigForHelmApp(app.AppId, app.EnvironmentId) + if err != nil { + impl.logger.Errorw("error in getiting deployment config db object by appId and envId", "appId", app.AppId, "envId", app.EnvironmentId, "err", err) + return nil, err + } + chartTemplate := adapter.GenerateInstallAppVersionMinDTO(app, deploymentConfig) return chartTemplate, nil } diff --git a/pkg/appStore/installedApp/service/AppStoreDeploymentService.go b/pkg/appStore/installedApp/service/AppStoreDeploymentService.go index b522192e17..5636b902cb 100644 --- a/pkg/appStore/installedApp/service/AppStoreDeploymentService.go +++ b/pkg/appStore/installedApp/service/AppStoreDeploymentService.go @@ -41,6 +41,8 @@ import ( "github.com/devtron-labs/devtron/pkg/appStore/installedApp/service/FullMode/deployment" bean2 "github.com/devtron-labs/devtron/pkg/appStore/installedApp/service/bean" "github.com/devtron-labs/devtron/pkg/cluster" + "github.com/devtron-labs/devtron/pkg/deployment/common" + bean5 "github.com/devtron-labs/devtron/pkg/deployment/common/bean" "github.com/devtron-labs/devtron/pkg/deployment/gitOps/config" util2 "github.com/devtron-labs/devtron/util" "github.com/go-pg/pg" @@ -84,6 +86,7 @@ type AppStoreDeploymentServiceImpl struct { gitOpsConfigReadService config.GitOpsConfigReadService deletePostProcessor DeletePostProcessor appStoreValidator AppStoreValidator + deploymentConfigService common.DeploymentConfigService } func NewAppStoreDeploymentServiceImpl(logger *zap.SugaredLogger, @@ -101,7 +104,8 @@ func NewAppStoreDeploymentServiceImpl(logger *zap.SugaredLogger, envVariables *util2.EnvironmentVariables, aCDConfig *argocdServer.ACDConfig, gitOpsConfigReadService config.GitOpsConfigReadService, deletePostProcessor DeletePostProcessor, - appStoreValidator AppStoreValidator) *AppStoreDeploymentServiceImpl { + appStoreValidator AppStoreValidator, + deploymentConfigService common.DeploymentConfigService) *AppStoreDeploymentServiceImpl { return &AppStoreDeploymentServiceImpl{ logger: logger, installedAppRepository: installedAppRepository, @@ -120,6 +124,7 @@ func NewAppStoreDeploymentServiceImpl(logger *zap.SugaredLogger, gitOpsConfigReadService: gitOpsConfigReadService, deletePostProcessor: deletePostProcessor, appStoreValidator: appStoreValidator, + deploymentConfigService: deploymentConfigService, } } @@ -245,8 +250,13 @@ func (impl *AppStoreDeploymentServiceImpl) DeleteInstalledApp(ctx context.Contex return nil, err } + deploymentConfig, err := impl.deploymentConfigService.GetDeploymentConfigForHelmApp(model.AppId, model.EnvironmentId) + if err != nil { + impl.logger.Errorw("error in getiting deployment config db object by appId and envId", "appId", model.AppId, "envId", model.EnvironmentId, "err", err) + return nil, err + } if installAppVersionRequest.AcdPartialDelete == true { - if !util2.IsBaseStack() && !util2.IsHelmApp(app.AppOfferingMode) && !util.IsHelmApp(model.DeploymentAppType) { + if !util2.IsBaseStack() && !util2.IsHelmApp(app.AppOfferingMode) && !util.IsHelmApp(deploymentConfig.DeploymentAppType) { if !installAppVersionRequest.InstalledAppDeleteResponse.ClusterReachable { impl.logger.Errorw("cluster connection error", "err", environment.ErrorInConnecting) if !installAppVersionRequest.NonCascadeDelete { @@ -322,7 +332,7 @@ func (impl *AppStoreDeploymentServiceImpl) DeleteInstalledApp(ctx context.Contex } } - if util2.IsBaseStack() || util2.IsHelmApp(app.AppOfferingMode) || util.IsHelmApp(model.DeploymentAppType) { + if util2.IsBaseStack() || util2.IsHelmApp(app.AppOfferingMode) || util.IsHelmApp(deploymentConfig.DeploymentAppType) { // there might be a case if helm release gets uninstalled from helm cli. //in this case on deleting the app from API, it should not give error as it should get deleted from db, otherwise due to delete error, db does not get clean // so in helm, we need to check first if the release exists or not, if exists then only delete @@ -448,18 +458,23 @@ func (impl *AppStoreDeploymentServiceImpl) RollbackApplication(ctx context.Conte impl.logger.Errorw("error while fetching installed app", "error", err) return false, err } - installedApp.GitOpsRepoURL = installedAppModel.GitOpsRepoUrl + deploymentConfig, err := impl.deploymentConfigService.GetDeploymentConfigForHelmApp(installedApp.AppId, installedApp.EnvironmentId) + if err != nil { + impl.logger.Errorw("error in getiting deployment config db object by appId and envId", "appId", installedApp.AppId, "envId", installedApp.EnvironmentId, "err", err) + return false, err + } + installedApp.GitOpsRepoURL = deploymentConfig.RepoURL // migrate installedApp.GitOpsRepoName to installedApp.GitOpsRepoUrl - if util.IsAcdApp(installedAppModel.DeploymentAppType) && - len(installedAppModel.GitOpsRepoName) != 0 && - len(installedAppModel.GitOpsRepoUrl) == 0 { + if util.IsAcdApp(deploymentConfig.DeploymentAppType) && + len(deploymentConfig.RepoName) != 0 && + len(deploymentConfig.RepoURL) == 0 { //as the installedApp.GitOpsRepoName is not an empty string; migrate installedApp.GitOpsRepoName to installedApp.GitOpsRepoUrl - migrationErr := impl.handleGitOpsRepoUrlMigration(tx, installedAppModel, userId) + migrationErr := impl.handleGitOpsRepoUrlMigration(tx, installedAppModel, deploymentConfig, userId) if migrationErr != nil { impl.logger.Errorw("error in GitOps repository url migration", "err", migrationErr) return false, err } - installedApp.GitOpsRepoURL = installedAppModel.GitOpsRepoUrl + installedApp.GitOpsRepoURL = deploymentConfig.RepoURL } // migration ends } @@ -598,23 +613,30 @@ func (impl *AppStoreDeploymentServiceImpl) UpdateInstalledApp(ctx context.Contex installedApp.Environment.ClusterId: installedApp.Environment.Namespace, } + deploymentConfig, err := impl.deploymentConfigService.GetDeploymentConfigForHelmApp(installedApp.AppId, installedApp.EnvironmentId) + if err != nil { + impl.logger.Errorw("error in getting deploymentConfig by appId and envId", "appId", installedApp.AppId, "envId", installedApp.EnvironmentId, "err", err) + return nil, err + } + err = impl.helmAppService.CheckIfNsExistsForClusterIds(clusterIdToNsMap) if err != nil { return nil, err } - upgradeAppRequest.UpdateDeploymentAppType(installedApp.DeploymentAppType) + upgradeAppRequest.UpdateDeploymentAppType(deploymentConfig.DeploymentAppType) - installedAppDeploymentAction := adapter.NewInstalledAppDeploymentAction(installedApp.DeploymentAppType) + installedAppDeploymentAction := adapter.NewInstalledAppDeploymentAction(deploymentConfig.DeploymentAppType) // migrate installedApp.GitOpsRepoName to installedApp.GitOpsRepoUrl - if util.IsAcdApp(installedApp.DeploymentAppType) && - len(installedApp.GitOpsRepoName) != 0 && - len(installedApp.GitOpsRepoUrl) == 0 { + if util.IsAcdApp(deploymentConfig.DeploymentAppType) && + len(deploymentConfig.RepoName) != 0 && + len(deploymentConfig.RepoURL) == 0 { //as the installedApp.GitOpsRepoName is not an empty string; migrate installedApp.GitOpsRepoName to installedApp.GitOpsRepoUrl gitRepoUrl, err := impl.fullModeDeploymentService.GetGitRepoUrl(installedApp.GitOpsRepoName) if err != nil { impl.logger.Errorw("error in GitOps repository url migration", "err", err) return nil, err } + deploymentConfig.RepoURL = gitRepoUrl installedApp.GitOpsRepoUrl = gitRepoUrl } // migration ends @@ -673,7 +695,7 @@ func (impl *AppStoreDeploymentServiceImpl) UpdateInstalledApp(ctx context.Contex // populate appStoreBean.InstallAppVersionDTO from the DB models upgradeAppRequest.Id = installedAppVersion.Id upgradeAppRequest.InstalledAppVersionId = installedAppVersion.Id - adapter.UpdateInstallAppDetails(upgradeAppRequest, installedApp) + adapter.UpdateInstallAppDetails(upgradeAppRequest, installedApp, deploymentConfig) adapter.UpdateAppDetails(upgradeAppRequest, &installedApp.App) environment, err := impl.environmentService.GetExtendedEnvBeanById(installedApp.EnvironmentId) if err != nil { @@ -715,7 +737,7 @@ func (impl *AppStoreDeploymentServiceImpl) UpdateInstalledApp(ctx context.Contex return nil, err } // required if gitOps repo name is changed, gitOps repo name will change if env variable which we use as suffix changes - monoRepoMigrationRequired = impl.checkIfMonoRepoMigrationRequired(installedApp) + monoRepoMigrationRequired = impl.checkIfMonoRepoMigrationRequired(installedApp, deploymentConfig) argocdAppName := installedApp.App.AppName + "-" + installedApp.Environment.Name upgradeAppRequest.ACDAppName = argocdAppName @@ -761,12 +783,20 @@ func (impl *AppStoreDeploymentServiceImpl) UpdateInstalledApp(ctx context.Contex if monoRepoMigrationRequired { //if monorepo case is true then repoUrl is changed then also update repo url in database installedApp.UpdateGitOpsRepository(gitOpsResponse.ChartGitAttribute.RepoUrl, installedApp.IsCustomRepository) + deploymentConfig.RepoURL = gitOpsResponse.ChartGitAttribute.RepoUrl } installedApp, err = impl.installedAppRepository.UpdateInstalledApp(installedApp, tx) if err != nil { impl.logger.Errorw("error in updating installed app", "err", err) return nil, err } + + deploymentConfig, err = impl.deploymentConfigService.CreateOrUpdateConfig(tx, deploymentConfig, upgradeAppRequest.UserId) + if err != nil { + impl.logger.Errorw("error in updating deployment config for helm apps", "appId", deploymentConfig.AppId, "envId", deploymentConfig.EnvironmentId, "err", err) + return nil, err + } + //STEP 8: finish with return response err = tx.Commit() if err != nil { @@ -833,7 +863,7 @@ func (impl *AppStoreDeploymentServiceImpl) MarkGitOpsInstalledAppsDeletedIfArgoA apiError.InternalMessage = "error in fetching partially deleted argoCd apps from installed app repo" return apiError } - if !util.IsAcdApp(installedApp.DeploymentAppType) || !installedApp.DeploymentAppDeleteRequest { + if (!util.IsAcdApp(installedApp.DeploymentAppType) && !util.IsAcdApp(installedApp.DeploymentConfig.DeploymentAppType)) || !installedApp.DeploymentAppDeleteRequest { return nil } // Operates for ArgoCd apps only @@ -956,13 +986,13 @@ func (impl *AppStoreDeploymentServiceImpl) linkHelmApplicationToChartStore(insta } // checkIfMonoRepoMigrationRequired checks if gitOps repo name is changed -func (impl *AppStoreDeploymentServiceImpl) checkIfMonoRepoMigrationRequired(installedApp *repository.InstalledApps) bool { +func (impl *AppStoreDeploymentServiceImpl) checkIfMonoRepoMigrationRequired(installedApp *repository.InstalledApps, deploymentConfig *bean5.DeploymentConfig) bool { monoRepoMigrationRequired := false - if !util.IsAcdApp(installedApp.DeploymentAppType) || gitOps.IsGitOpsRepoNotConfigured(installedApp.GitOpsRepoUrl) || installedApp.IsCustomRepository { + if !util.IsAcdApp(deploymentConfig.DeploymentAppType) || gitOps.IsGitOpsRepoNotConfigured(deploymentConfig.RepoURL) || deploymentConfig.ConfigType == bean5.CUSTOM.String() { return false } var err error - gitOpsRepoName := impl.gitOpsConfigReadService.GetGitOpsRepoNameFromUrl(installedApp.GitOpsRepoUrl) + gitOpsRepoName := impl.gitOpsConfigReadService.GetGitOpsRepoNameFromUrl(deploymentConfig.RepoURL) if len(gitOpsRepoName) == 0 { gitOpsRepoName, err = impl.fullModeDeploymentService.GetAcdAppGitOpsRepoName(installedApp.App.AppName, installedApp.Environment.Name) if err != nil || gitOpsRepoName == "" { @@ -980,7 +1010,7 @@ func (impl *AppStoreDeploymentServiceImpl) checkIfMonoRepoMigrationRequired(inst } // handleGitOpsRepoUrlMigration will migrate git_ops_repo_name to git_ops_repo_url -func (impl *AppStoreDeploymentServiceImpl) handleGitOpsRepoUrlMigration(tx *pg.Tx, installedApp *repository.InstalledApps, userId int32) error { +func (impl *AppStoreDeploymentServiceImpl) handleGitOpsRepoUrlMigration(tx *pg.Tx, installedApp *repository.InstalledApps, deploymentConfig *bean5.DeploymentConfig, userId int32) error { var ( localTx *pg.Tx err error @@ -1004,15 +1034,27 @@ func (impl *AppStoreDeploymentServiceImpl) handleGitOpsRepoUrlMigration(tx *pg.T installedApp.GitOpsRepoUrl = gitRepoUrl installedApp.UpdatedOn = time.Now() installedApp.UpdatedBy = userId + + var dbTx *pg.Tx if localTx != nil { - _, err = impl.installedAppRepository.UpdateInstalledApp(installedApp, localTx) + dbTx = localTx } else { - _, err = impl.installedAppRepository.UpdateInstalledApp(installedApp, tx) + dbTx = tx } + + _, err = impl.installedAppRepository.UpdateInstalledApp(installedApp, dbTx) if err != nil { impl.logger.Errorw("error in updating installed app model", "err", err) return err } + + deploymentConfig.RepoURL = gitRepoUrl + deploymentConfig, err = impl.deploymentConfigService.CreateOrUpdateConfig(dbTx, deploymentConfig, userId) + if err != nil { + impl.logger.Errorw("error in updating deployment config", "err", err) + return err + } + if localTx != nil { err = localTx.Commit() if err != nil { diff --git a/pkg/appStore/installedApp/service/EAMode/InstalledAppDBService.go b/pkg/appStore/installedApp/service/EAMode/InstalledAppDBService.go index 0ebaa31646..2e83752ab5 100644 --- a/pkg/appStore/installedApp/service/EAMode/InstalledAppDBService.go +++ b/pkg/appStore/installedApp/service/EAMode/InstalledAppDBService.go @@ -22,6 +22,7 @@ import ( util4 "github.com/devtron-labs/devtron/pkg/appStore/util" bean3 "github.com/devtron-labs/devtron/pkg/auth/user/bean" "github.com/devtron-labs/devtron/pkg/cluster" + "github.com/devtron-labs/devtron/pkg/deployment/common" "net/http" "strconv" "strings" @@ -66,6 +67,7 @@ type InstalledAppDBServiceImpl struct { UserService user.UserService EnvironmentService cluster.EnvironmentService InstalledAppRepositoryHistory appStoreRepo.InstalledAppVersionHistoryRepository + deploymentConfigService common.DeploymentConfigService } func NewInstalledAppDBServiceImpl(logger *zap.SugaredLogger, @@ -73,7 +75,8 @@ func NewInstalledAppDBServiceImpl(logger *zap.SugaredLogger, appRepository app.AppRepository, userService user.UserService, environmentService cluster.EnvironmentService, - installedAppRepositoryHistory appStoreRepo.InstalledAppVersionHistoryRepository) *InstalledAppDBServiceImpl { + installedAppRepositoryHistory appStoreRepo.InstalledAppVersionHistoryRepository, + deploymentConfigService common.DeploymentConfigService) *InstalledAppDBServiceImpl { return &InstalledAppDBServiceImpl{ Logger: logger, InstalledAppRepository: installedAppRepository, @@ -177,6 +180,13 @@ func (impl *InstalledAppDBServiceImpl) FindAppDetailsForAppstoreApplication(inst impl.Logger.Errorw("error in getting helm release status from db", "err", err) return bean2.AppDetailContainer{}, err } + + deploymentConfig, err := impl.deploymentConfigService.GetDeploymentConfigForHelmApp(installedAppVerison.InstalledApp.AppId, installedAppVerison.InstalledApp.EnvironmentId) + if err != nil { + impl.Logger.Errorw("error in getiting deployment config db object by appId and envId", "appId", installedAppVerison.InstalledApp.AppId, "envId", installedAppVerison.InstalledApp.EnvironmentId, "err", err) + return bean2.AppDetailContainer{}, err + } + var chartName string if installedAppVerison.AppStoreApplicationVersion.AppStore.ChartRepoId != 0 { chartName = installedAppVerison.AppStoreApplicationVersion.AppStore.ChartRepo.Name @@ -200,11 +210,12 @@ func (impl *InstalledAppDBServiceImpl) FindAppDetailsForAppstoreApplication(inst Deprecated: installedAppVerison.AppStoreApplicationVersion.Deprecated, ClusterId: installedAppVerison.InstalledApp.Environment.ClusterId, ClusterName: installedAppVerison.InstalledApp.Environment.Cluster.ClusterName, - DeploymentAppType: installedAppVerison.InstalledApp.DeploymentAppType, + DeploymentAppType: deploymentConfig.DeploymentAppType, DeploymentAppDeleteRequest: installedAppVerison.InstalledApp.DeploymentAppDeleteRequest, IsVirtualEnvironment: installedAppVerison.InstalledApp.Environment.IsVirtualEnvironment, HelmReleaseInstallStatus: helmReleaseInstallStatus, Status: status, + DeploymentConfig: deploymentConfig, } if util4.IsExternalChartStoreApp(installedAppVerison.InstalledApp.App.DisplayName) { deploymentContainer.AppName = installedAppVerison.InstalledApp.App.DisplayName @@ -247,7 +258,12 @@ func (impl *InstalledAppDBServiceImpl) GetInstalledAppByClusterNamespaceAndName( if err != nil { return nil, err } - return adapter.GenerateInstallAppVersionDTO(installedApp, installedAppVersion), nil + deploymentConfig, err := impl.deploymentConfigService.GetDeploymentConfigForHelmApp(installedApp.AppId, installedApp.EnvironmentId) + if err != nil { + impl.Logger.Errorw("error in getiting deployment config db object by appId and envId", "appId", installedApp.AppId, "envId", installedApp.EnvironmentId, "err", err) + return nil, err + } + return adapter.GenerateInstallAppVersionDTO(installedApp, installedAppVersion, deploymentConfig), nil } return nil, nil @@ -259,7 +275,12 @@ func (impl *InstalledAppDBServiceImpl) GetInstalledAppByInstalledAppId(installed return nil, err } installedApp := &installedAppVersion.InstalledApp - return adapter.GenerateInstallAppVersionDTO(installedApp, installedAppVersion), nil + deploymentConfig, err := impl.deploymentConfigService.GetDeploymentConfigForHelmApp(installedApp.AppId, installedApp.EnvironmentId) + if err != nil { + impl.Logger.Errorw("error in getiting deployment config db object by appId and envId", "appId", installedApp.AppId, "envId", installedApp.EnvironmentId, "err", err) + return nil, err + } + return adapter.GenerateInstallAppVersionDTO(installedApp, installedAppVersion, deploymentConfig), nil } func (impl *InstalledAppDBServiceImpl) GetInstalledAppVersion(id int, userId int32) (*appStoreBean.InstallAppVersionDTO, error) { @@ -271,6 +292,11 @@ func (impl *InstalledAppDBServiceImpl) GetInstalledAppVersion(id int, userId int impl.Logger.Errorw("error while fetching from db", "error", err) return nil, err } + deploymentConfig, err := impl.deploymentConfigService.GetDeploymentConfigForHelmApp(model.InstalledApp.AppId, model.InstalledApp.EnvironmentId) + if err != nil { + impl.Logger.Errorw("error in getiting deployment config db object by appId and envId", "appId", model.InstalledApp.AppId, "envId", model.InstalledApp.EnvironmentId, "err", err) + return nil, err + } // update InstallAppVersion configurations installAppVersion := &appStoreBean.InstallAppVersionDTO{ Id: model.Id, @@ -287,7 +313,7 @@ func (impl *InstalledAppDBServiceImpl) GetInstalledAppVersion(id int, userId int adapter.UpdateAppDetails(installAppVersion, &model.InstalledApp.App) // update InstallApp configurations - adapter.UpdateInstallAppDetails(installAppVersion, &model.InstalledApp) + adapter.UpdateInstallAppDetails(installAppVersion, &model.InstalledApp, deploymentConfig) // update AppStoreApplication configurations adapter.UpdateAppStoreApplicationDetails(installAppVersion, &model.AppStoreApplicationVersion) diff --git a/pkg/appStore/installedApp/service/FullMode/deployment/FullModeDeploymentService.go b/pkg/appStore/installedApp/service/FullMode/deployment/FullModeDeploymentService.go index 533460a5b2..0a3741196b 100644 --- a/pkg/appStore/installedApp/service/FullMode/deployment/FullModeDeploymentService.go +++ b/pkg/appStore/installedApp/service/FullMode/deployment/FullModeDeploymentService.go @@ -26,6 +26,7 @@ import ( "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/timelineStatus" "github.com/devtron-labs/devtron/pkg/appStore/installedApp/service/common" repository5 "github.com/devtron-labs/devtron/pkg/cluster/repository" + "github.com/devtron-labs/devtron/pkg/deployment/common" commonBean "github.com/devtron-labs/devtron/pkg/deployment/gitOps/common/bean" "github.com/devtron-labs/devtron/pkg/deployment/gitOps/config" "github.com/devtron-labs/devtron/pkg/deployment/gitOps/git" @@ -99,6 +100,7 @@ type FullModeDeploymentServiceImpl struct { gitOpsConfigReadService config.GitOpsConfigReadService gitOpsValidationService validation.GitOpsValidationService environmentRepository repository5.EnvironmentRepository + deploymentConfigService common.DeploymentConfigService } func NewFullModeDeploymentServiceImpl( @@ -122,7 +124,8 @@ func NewFullModeDeploymentServiceImpl( gitOperationService git.GitOperationService, gitOpsConfigReadService config.GitOpsConfigReadService, gitOpsValidationService validation.GitOpsValidationService, - environmentRepository repository5.EnvironmentRepository) *FullModeDeploymentServiceImpl { + environmentRepository repository5.EnvironmentRepository, + deploymentConfigService common.DeploymentConfigService) *FullModeDeploymentServiceImpl { return &FullModeDeploymentServiceImpl{ Logger: logger, acdClient: acdClient, @@ -145,6 +148,7 @@ func NewFullModeDeploymentServiceImpl( gitOpsConfigReadService: gitOpsConfigReadService, gitOpsValidationService: gitOpsValidationService, environmentRepository: environmentRepository, + deploymentConfigService: deploymentConfigService, } } @@ -244,6 +248,12 @@ func (impl *FullModeDeploymentServiceImpl) RollbackRelease(ctx context.Context, return installedApp, false, err } + deploymentConfig, err := impl.deploymentConfigService.GetDeploymentConfigForHelmApp(installedApp.AppId, installedApp.EnvironmentId) + if err != nil { + impl.Logger.Errorw("error in getiting deployment config db object by appId and envId", "appId", installedApp.AppId, "envId", installedApp.EnvironmentId, "err", err) + return installedApp, false, err + } + //validate relations if installedApp.InstalledAppId != installedAppVersion.InstalledAppId { err = &util.ApiError{Code: "400", HttpStatusCode: 400, UserMessage: "bad request, requested version are not belongs to each other", InternalMessage: ""} @@ -255,7 +265,7 @@ func (impl *FullModeDeploymentServiceImpl) RollbackRelease(ctx context.Context, installedApp.ValuesOverrideYaml = versionHistory.ValuesYamlRaw installedApp.AppStoreId = installedAppVersion.AppStoreApplicationVersion.AppStoreId installedApp.AppStoreName = installedAppVersion.AppStoreApplicationVersion.AppStore.Name - installedApp.GitOpsRepoURL = installedAppVersion.InstalledApp.GitOpsRepoUrl + installedApp.GitOpsRepoURL = deploymentConfig.RepoURL installedApp.ACDAppName = fmt.Sprintf("%s-%s", installedApp.AppName, installedApp.EnvironmentName) //create an entry in version history table diff --git a/pkg/appStore/installedApp/service/FullMode/deployment/InstalledAppGitOpsService.go b/pkg/appStore/installedApp/service/FullMode/deployment/InstalledAppGitOpsService.go index 7b884e40d2..2b04fdf907 100644 --- a/pkg/appStore/installedApp/service/FullMode/deployment/InstalledAppGitOpsService.go +++ b/pkg/appStore/installedApp/service/FullMode/deployment/InstalledAppGitOpsService.go @@ -320,7 +320,17 @@ func (impl *FullModeDeploymentServiceImpl) getGitCommitConfig(installAppVersionR apiErr := &util.ApiError{HttpStatusCode: http.StatusNotFound, Code: strconv.Itoa(http.StatusNotFound), InternalMessage: "Invalid request! No InstalledApp found.", UserMessage: "Invalid request! No InstalledApp found."} return nil, apiErr } - installAppVersionRequest.GitOpsRepoURL = InstalledApp.GitOpsRepoUrl + deploymentConfig, err := impl.deploymentConfigService.GetDeploymentConfigForHelmApp(InstalledApp.AppId, InstalledApp.EnvironmentId) + if err != nil { + impl.Logger.Errorw("error in getiting deployment config db object by appId and envId", "appId", InstalledApp.AppId, "envId", InstalledApp.EnvironmentId, "err", err) + return nil, err + } + if util.IsErrNoRows(err) { + apiErr := &util.ApiError{HttpStatusCode: http.StatusNotFound, Code: strconv.Itoa(http.StatusNotFound), InternalMessage: "Invalid request! No InstalledApp found.", UserMessage: "Invalid request! No InstalledApp found."} + return nil, apiErr + } + //installAppVersionRequest.GitOpsRepoURL = InstalledApp.GitOpsRepoUrl + installAppVersionRequest.GitOpsRepoURL = deploymentConfig.RepoURL } gitOpsRepoName := impl.gitOpsConfigReadService.GetGitOpsRepoNameFromUrl(installAppVersionRequest.GitOpsRepoURL) userEmailId, userName := impl.gitOpsConfigReadService.GetUserEmailIdAndNameForGitOpsCommit(installAppVersionRequest.UserId) diff --git a/pkg/appStore/installedApp/service/FullMode/deploymentTypeChange/InstalledAppDeploymentTypeChangeService.go b/pkg/appStore/installedApp/service/FullMode/deploymentTypeChange/InstalledAppDeploymentTypeChangeService.go index 2c3c231f38..4554df574f 100644 --- a/pkg/appStore/installedApp/service/FullMode/deploymentTypeChange/InstalledAppDeploymentTypeChangeService.go +++ b/pkg/appStore/installedApp/service/FullMode/deploymentTypeChange/InstalledAppDeploymentTypeChangeService.go @@ -41,6 +41,8 @@ import ( "github.com/devtron-labs/devtron/pkg/bean" "github.com/devtron-labs/devtron/pkg/cluster" repository5 "github.com/devtron-labs/devtron/pkg/cluster/repository" + "github.com/devtron-labs/devtron/pkg/deployment/common" + bean3 "github.com/devtron-labs/devtron/pkg/deployment/common/bean" "github.com/devtron-labs/devtron/pkg/deployment/gitOps/config" bean2 "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/bean" "github.com/devtron-labs/devtron/pkg/k8s" @@ -79,6 +81,7 @@ type InstalledAppDeploymentTypeChangeServiceImpl struct { helmAppService client.HelmAppService argoUserService argo.ArgoUserService clusterService cluster.ClusterService + deploymentConfigService common.DeploymentConfigService } func NewInstalledAppDeploymentTypeChangeServiceImpl(logger *zap.SugaredLogger, @@ -93,7 +96,8 @@ func NewInstalledAppDeploymentTypeChangeServiceImpl(logger *zap.SugaredLogger, argoClientWrapperService argocdServer.ArgoClientWrapperService, chartGroupService chartGroup.ChartGroupService, helmAppService client.HelmAppService, argoUserService argo.ArgoUserService, clusterService cluster.ClusterService, - appRepository appRepository.AppRepository) *InstalledAppDeploymentTypeChangeServiceImpl { + appRepository appRepository.AppRepository, + deploymentConfigService common.DeploymentConfigService) *InstalledAppDeploymentTypeChangeServiceImpl { return &InstalledAppDeploymentTypeChangeServiceImpl{ logger: logger, installedAppRepository: installedAppRepository, @@ -112,6 +116,7 @@ func NewInstalledAppDeploymentTypeChangeServiceImpl(logger *zap.SugaredLogger, argoUserService: argoUserService, clusterService: clusterService, appRepository: appRepository, + deploymentConfigService: deploymentConfigService, } } @@ -293,6 +298,12 @@ func (impl *InstalledAppDeploymentTypeChangeServiceImpl) deleteInstalledApps(ctx for _, installedApp := range installedApps { installedApp.Environment.Cluster = cluster + deploymentConfig, err := impl.deploymentConfigService.GetDeploymentConfigForHelmApp(installedApp.AppId, installedApp.EnvironmentId) + if err != nil { + impl.logger.Errorw("error in getiting deployment config db object by appId and envId", "appId", installedApp.AppId, "envId", installedApp.EnvironmentId, "err", err) + return nil, err + } + var isValid bool // check if installed app info like app name and environment is empty or not if failedToDeleteApps, isValid = impl.isInstalledAppInfoValid(installedApp, failedToDeleteApps); !isValid { @@ -301,17 +312,16 @@ func (impl *InstalledAppDeploymentTypeChangeServiceImpl) deleteInstalledApps(ctx var healthChkErr error // check health of the app if it is argo-cd deployment type - if _, healthChkErr = impl.handleNotDeployedAppsIfArgoDeploymentType(installedApp, failedToDeleteApps); healthChkErr != nil { + if _, healthChkErr = impl.handleNotDeployedAppsIfArgoDeploymentType(installedApp, deploymentConfig, failedToDeleteApps); healthChkErr != nil { // cannot delete unhealthy app continue } deploymentAppName := fmt.Sprintf("%s-%s", installedApp.App.AppName, installedApp.Environment.Name) - var err error // delete request - if installedApp.DeploymentAppType == bean2.ArgoCd { + if deploymentConfig.DeploymentAppType == bean2.ArgoCd { err = impl.fullModeDeploymentService.DeleteACD(deploymentAppName, ctx, false) - } else if installedApp.DeploymentAppType == bean2.Helm { + } else if deploymentConfig.DeploymentAppType == bean2.Helm { // For converting from Helm to ArgoCD, GitOps should be configured if gitOpsConfigErr != nil || !gitOpsConfigStatus.IsGitOpsConfigured { err = &util.ApiError{HttpStatusCode: http.StatusBadRequest, Code: "200", UserMessage: errors.New("GitOps not configured or unable to fetch GitOps configuration")} @@ -337,7 +347,7 @@ func (impl *InstalledAppDeploymentTypeChangeServiceImpl) deleteInstalledApps(ctx } if err != nil { - impl.logger.Errorw("error deleting app on "+installedApp.DeploymentAppType, + impl.logger.Errorw("error deleting app on "+deploymentConfig.DeploymentAppType, "deployment app name", deploymentAppName, "err", err) @@ -369,9 +379,10 @@ func (impl *InstalledAppDeploymentTypeChangeServiceImpl) isInstalledAppInfoValid } func (impl *InstalledAppDeploymentTypeChangeServiceImpl) handleNotDeployedAppsIfArgoDeploymentType(installedApp *repository2.InstalledApps, + deploymentConfig *bean3.DeploymentConfig, failedToDeleteApps []*bean.DeploymentChangeStatus) ([]*bean.DeploymentChangeStatus, error) { - if installedApp.DeploymentAppType == string(bean2.ArgoCd) { + if deploymentConfig.DeploymentAppType == bean2.ArgoCd { // check if app status is Healthy status, err := impl.appStatusRepository.Get(installedApp.AppId, installedApp.EnvironmentId) diff --git a/pkg/appStore/installedApp/service/FullMode/resource/NotesService.go b/pkg/appStore/installedApp/service/FullMode/resource/NotesService.go index cba5302ec9..0bdcdc91ab 100644 --- a/pkg/appStore/installedApp/service/FullMode/resource/NotesService.go +++ b/pkg/appStore/installedApp/service/FullMode/resource/NotesService.go @@ -83,7 +83,13 @@ func (impl *InstalledAppResourceServiceImpl) findNotesForArgoApplication(install } var notes string - if util.IsAcdApp(installedAppVerison.InstalledApp.DeploymentAppType) { + deploymentConfig, err := impl.deploymentConfigurationService.GetDeploymentConfigForHelmApp(installedAppVerison.InstalledApp.AppId, installedAppVerison.InstalledApp.EnvironmentId) + if err != nil { + impl.logger.Errorw("error in getiting deployment config db object by appId and envId", "appId", installedAppVerison.InstalledApp.AppId, "envId", installedAppVerison.InstalledApp.EnvironmentId, "err", err) + return "", err + } + + if util.IsAcdApp(deploymentConfig.DeploymentAppType) { appStoreAppVersion, err := impl.appStoreApplicationVersionRepository.FindById(installedAppVerison.AppStoreApplicationVersion.Id) if err != nil { impl.logger.Errorw("error fetching app store app version in installed app service", "err", err) diff --git a/pkg/appStore/installedApp/service/FullMode/resource/ResourceTreeService.go b/pkg/appStore/installedApp/service/FullMode/resource/ResourceTreeService.go index d81fdc0945..6c91f3f446 100644 --- a/pkg/appStore/installedApp/service/FullMode/resource/ResourceTreeService.go +++ b/pkg/appStore/installedApp/service/FullMode/resource/ResourceTreeService.go @@ -34,6 +34,8 @@ import ( "github.com/devtron-labs/devtron/pkg/appStore/bean" appStoreDiscoverRepository "github.com/devtron-labs/devtron/pkg/appStore/discover/repository" "github.com/devtron-labs/devtron/pkg/appStore/installedApp/repository" + "github.com/devtron-labs/devtron/pkg/deployment/common" + bean2 "github.com/devtron-labs/devtron/pkg/deployment/common/bean" "github.com/devtron-labs/devtron/pkg/k8s" application3 "github.com/devtron-labs/devtron/pkg/k8s/application" util3 "github.com/devtron-labs/devtron/pkg/util" @@ -51,7 +53,7 @@ import ( type InstalledAppResourceService interface { FetchResourceTreeWithHibernateForACD(rctx context.Context, cn http.CloseNotifier, appDetail *bean.AppDetailContainer) bean.AppDetailContainer - FetchResourceTree(rctx context.Context, cn http.CloseNotifier, appDetailsContainer *bean.AppDetailsContainer, installedApp repository.InstalledApps, helmReleaseInstallStatus string, status string) error + FetchResourceTree(rctx context.Context, cn http.CloseNotifier, appDetailsContainer *bean.AppDetailsContainer, installedApp repository.InstalledApps, deploymentConfig *bean2.DeploymentConfig, helmReleaseInstallStatus string, status string) error FetchChartNotes(installedAppId int, envId int, token string, checkNotesAuth func(token string, appName string, envId int) bool) (string, error) } @@ -69,6 +71,7 @@ type InstalledAppResourceServiceImpl struct { k8sCommonService k8s.K8sCommonService k8sApplicationService application3.K8sApplicationService K8sUtil k8s2.K8sService + deploymentConfigurationService common.DeploymentConfigService } func NewInstalledAppResourceServiceImpl(logger *zap.SugaredLogger, @@ -79,7 +82,8 @@ func NewInstalledAppResourceServiceImpl(logger *zap.SugaredLogger, installedAppRepositoryHistory repository.InstalledAppVersionHistoryRepository, argoUserService argo.ArgoUserService, helmAppClient gRPC.HelmAppClient, helmAppService client.HelmAppService, appStatusService appStatus.AppStatusService, - k8sCommonService k8s.K8sCommonService, k8sApplicationService application3.K8sApplicationService, K8sUtil k8s2.K8sService) *InstalledAppResourceServiceImpl { + k8sCommonService k8s.K8sCommonService, k8sApplicationService application3.K8sApplicationService, K8sUtil k8s2.K8sService, + deploymentConfigurationService common.DeploymentConfigService) *InstalledAppResourceServiceImpl { return &InstalledAppResourceServiceImpl{ logger: logger, installedAppRepository: installedAppRepository, @@ -94,16 +98,17 @@ func NewInstalledAppResourceServiceImpl(logger *zap.SugaredLogger, k8sCommonService: k8sCommonService, k8sApplicationService: k8sApplicationService, K8sUtil: K8sUtil, + deploymentConfigurationService: deploymentConfigurationService, } } -func (impl *InstalledAppResourceServiceImpl) FetchResourceTree(rctx context.Context, cn http.CloseNotifier, appDetailsContainer *bean.AppDetailsContainer, installedApp repository.InstalledApps, helmReleaseInstallStatus string, status string) error { +func (impl *InstalledAppResourceServiceImpl) FetchResourceTree(rctx context.Context, cn http.CloseNotifier, appDetailsContainer *bean.AppDetailsContainer, installedApp repository.InstalledApps, deploymentConfig *bean2.DeploymentConfig, helmReleaseInstallStatus string, status string) error { var err error var resourceTree map[string]interface{} deploymentAppName := fmt.Sprintf("%s-%s", installedApp.App.AppName, installedApp.Environment.Name) - if util.IsAcdApp(installedApp.DeploymentAppType) { + if util.IsAcdApp(deploymentConfig.DeploymentAppType) { resourceTree, err = impl.fetchResourceTreeForACD(rctx, cn, installedApp.App.Id, installedApp.EnvironmentId, installedApp.Environment.ClusterId, deploymentAppName, installedApp.Environment.Namespace) - } else if util.IsHelmApp(installedApp.DeploymentAppType) { + } else if util.IsHelmApp(deploymentConfig.DeploymentAppType) { config, err := impl.helmAppService.GetClusterConf(installedApp.Environment.ClusterId) if err != nil { impl.logger.Errorw("error in fetching cluster detail", "err", err) diff --git a/pkg/chart/ChartService.go b/pkg/chart/ChartService.go index cfc9372d1d..bf2caedd4b 100644 --- a/pkg/chart/ChartService.go +++ b/pkg/chart/ChartService.go @@ -307,7 +307,7 @@ func (impl *ChartServiceImpl) Create(templateRequest TemplateRequest, ctx contex CredentialIdInt: activeGitOpsConfig.Id, Active: true, } - deploymentConfig, err = impl.deploymentConfigService.CreateOrUpdateConfig(deploymentConfig, templateRequest.UserId) + deploymentConfig, err = impl.deploymentConfigService.CreateOrUpdateConfig(nil, deploymentConfig, templateRequest.UserId) if err != nil { impl.logger.Errorw("error in saving deployment config", "appId", templateRequest.AppId, "err", err) return nil, err @@ -411,7 +411,7 @@ func (impl *ChartServiceImpl) CreateChartFromEnvOverride(templateRequest Templat CredentialIdInt: activeGitOpsConfig.Id, Active: true, } - deploymentConfig, err = impl.deploymentConfigService.CreateOrUpdateConfig(deploymentConfig, templateRequest.UserId) + deploymentConfig, err = impl.deploymentConfigService.CreateOrUpdateConfig(nil, deploymentConfig, templateRequest.UserId) if err != nil { impl.logger.Errorw("error in saving deployment config", "appId", templateRequest.AppId, "err", err) return nil, err @@ -978,7 +978,7 @@ func (impl *ChartServiceImpl) ConfigureGitOpsRepoUrlForApp(appId int, repoUrl, c ChartLocation: chartLocation, Active: true, } - deploymentConfig, err := impl.deploymentConfigService.CreateOrUpdateConfig(deploymentConfig, userId) + deploymentConfig, err := impl.deploymentConfigService.CreateOrUpdateConfig(nil, deploymentConfig, userId) if err != nil { impl.logger.Errorw("error in saving deployment config for app", "appId", appId, "err", err) return err diff --git a/pkg/deployment/common/deploymentConfigService.go b/pkg/deployment/common/deploymentConfigService.go index 87ff407f24..389e374453 100644 --- a/pkg/deployment/common/deploymentConfigService.go +++ b/pkg/deployment/common/deploymentConfigService.go @@ -1,20 +1,24 @@ package common import ( + appRepository "github.com/devtron-labs/devtron/internal/sql/repository/app" "github.com/devtron-labs/devtron/internal/sql/repository/deploymentConfig" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" + "github.com/devtron-labs/devtron/pkg/appStore/installedApp/repository" bean3 "github.com/devtron-labs/devtron/pkg/auth/user/bean" chartRepoRepository "github.com/devtron-labs/devtron/pkg/chartRepo/repository" "github.com/devtron-labs/devtron/pkg/deployment/common/bean" "github.com/devtron-labs/devtron/pkg/deployment/gitOps/config" + "github.com/devtron-labs/devtron/pkg/deployment/gitOps/git" bean2 "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/bean" "github.com/go-pg/pg" "go.uber.org/zap" ) type DeploymentConfigService interface { - CreateOrUpdateConfig(config *bean.DeploymentConfig, userId int32) (*bean.DeploymentConfig, error) + CreateOrUpdateConfig(tx *pg.Tx, config *bean.DeploymentConfig, userId int32) (*bean.DeploymentConfig, error) GetDeploymentConfig(appId, envId int) (*bean.DeploymentConfig, error) + GetDeploymentConfigForHelmApp(appId, envId int) (*bean.DeploymentConfig, error) } type DeploymentConfigServiceImpl struct { @@ -23,6 +27,9 @@ type DeploymentConfigServiceImpl struct { chartRepository chartRepoRepository.ChartRepository pipelineRepository pipelineConfig.PipelineRepository gitOpsConfigReadService config.GitOpsConfigReadService + appRepository appRepository.AppRepository + installedAppRepository repository.InstalledAppRepository + git.GitOperationService } func NewDeploymentConfigServiceImpl( @@ -31,6 +38,8 @@ func NewDeploymentConfigServiceImpl( chartRepository chartRepoRepository.ChartRepository, pipelineRepository pipelineConfig.PipelineRepository, gitOpsConfigReadService config.GitOpsConfigReadService, + appRepository appRepository.AppRepository, + installedAppRepository repository.InstalledAppRepository, ) *DeploymentConfigServiceImpl { return &DeploymentConfigServiceImpl{ deploymentConfigRepository: deploymentConfigRepository, @@ -38,10 +47,12 @@ func NewDeploymentConfigServiceImpl( chartRepository: chartRepository, pipelineRepository: pipelineRepository, gitOpsConfigReadService: gitOpsConfigReadService, + appRepository: appRepository, + installedAppRepository: installedAppRepository, } } -func (impl *DeploymentConfigServiceImpl) CreateOrUpdateConfig(config *bean.DeploymentConfig, userId int32) (*bean.DeploymentConfig, error) { +func (impl *DeploymentConfigServiceImpl) CreateOrUpdateConfig(tx *pg.Tx, config *bean.DeploymentConfig, userId int32) (*bean.DeploymentConfig, error) { configDbObj, err := impl.GetConfigDBObj(config.AppId, config.EnvironmentId) if err != nil && err != pg.ErrNoRows { @@ -53,14 +64,14 @@ func (impl *DeploymentConfigServiceImpl) CreateOrUpdateConfig(config *bean.Deplo if configDbObj == nil || (configDbObj != nil && configDbObj.Id == 0) { newDBObj.AuditLog.CreateAuditLog(userId) - newDBObj, err = impl.deploymentConfigRepository.Save(newDBObj) + newDBObj, err = impl.deploymentConfigRepository.Save(tx, newDBObj) if err != nil { impl.logger.Errorw("error in saving deploymentConfig", "appId", config.AppId, "envId", config.EnvironmentId, "err", err) return nil, err } } else { newDBObj.AuditLog.UpdateAuditLog(userId) - newDBObj, err = impl.deploymentConfigRepository.Update(newDBObj) + newDBObj, err = impl.deploymentConfigRepository.Update(tx, newDBObj) if err != nil { impl.logger.Errorw("error in updating deploymentConfig", "appId", config.AppId, "envId", config.EnvironmentId, "err", err) return nil, err @@ -73,7 +84,7 @@ func (impl *DeploymentConfigServiceImpl) CreateOrUpdateConfig(config *bean.Deplo func (impl *DeploymentConfigServiceImpl) GetDeploymentConfig(appId, envId int) (*bean.DeploymentConfig, error) { appLevelConfigDbObj, err := impl.deploymentConfigRepository.GetAppLevelConfig(appId) - if err != nil { + if err != nil && err != pg.ErrNoRows { impl.logger.Errorw("error in getiting deployment config db object by appId", "appId", appId, "err", err) return nil, err } @@ -87,7 +98,7 @@ func (impl *DeploymentConfigServiceImpl) GetDeploymentConfig(appId, envId int) ( if envId > 0 { appAndEnvLevelConfig, err := impl.deploymentConfigRepository.GetByAppIdAndEnvId(appId, envId) - if err != nil { + if err != nil && err != pg.ErrNoRows { impl.logger.Errorw("error in getiting deployment config db object by appId and envId", "appId", appId, "envId", envId, "err", err) return nil, err } @@ -118,7 +129,7 @@ func (impl *DeploymentConfigServiceImpl) migrateAppLevelDataTODeploymentConfig(a ChartLocation: chart.ChartLocation, } ConfigDbObj.AuditLog.CreateAuditLog(1) - ConfigDbObj, err = impl.deploymentConfigRepository.Save(ConfigDbObj) + ConfigDbObj, err = impl.deploymentConfigRepository.Save(nil, ConfigDbObj) if err != nil { impl.logger.Errorw("error in saving deployment config in DB", "appId", appId, "err", err) return nil, err @@ -158,7 +169,7 @@ func (impl *DeploymentConfigServiceImpl) migrateAppAndEnvLevelDataToDeploymentCo } configDbObj.AuditLog.CreateAuditLog(bean3.SYSTEM_USER_ID) - configDbObj, err = impl.deploymentConfigRepository.Save(configDbObj) + configDbObj, err = impl.deploymentConfigRepository.Save(nil, configDbObj) if err != nil { impl.logger.Errorw("error in saving deployment config in DB", "appId", appId, "envId", envId, "err", err) return nil, err @@ -185,3 +196,58 @@ func (impl *DeploymentConfigServiceImpl) GetConfigDBObj(appId, envId int) (*depl } return configDbObj, nil } + +func (impl *DeploymentConfigServiceImpl) GetDeploymentConfigForHelmApp(appId, envId int) (*bean.DeploymentConfig, error) { + + helmDeploymentConfig, err := impl.deploymentConfigRepository.GetByAppIdAndEnvId(appId, envId) + if err != nil && err != pg.ErrNoRows { + impl.logger.Errorw("error in fetching deployment config by by appId and envId", "appId", appId, "envId", envId, "err", err) + return nil, err + } + + if err == pg.ErrNoRows { + helmDeploymentConfig, err = impl.migrateHelmAppDataToDeploymentConfig(appId, envId) + if err != nil { + impl.logger.Errorw("error in migrating helm deployment config", "appId", appId, "envId", envId, "err", err) + return nil, err + } + } + return ConvertDeploymentConfigDbObjToDTO(helmDeploymentConfig), nil +} + +func (impl *DeploymentConfigServiceImpl) migrateHelmAppDataToDeploymentConfig(appId, envId int) (*deploymentConfig.DeploymentConfig, error) { + + installedApp, err := impl.installedAppRepository.GetInstalledAppsByAppId(appId) + if err != nil { + impl.logger.Errorw("error in getting installed app by appId", "appId", appId, "err", err) + return nil, err + } + + helmDeploymentConfig := &deploymentConfig.DeploymentConfig{ + AppId: appId, + EnvironmentId: envId, + DeploymentAppType: installedApp.DeploymentAppType, + ConfigType: GetDeploymentConfigType(installedApp.IsCustomRepository), + RepoUrl: installedApp.GitOpsRepoUrl, + RepoName: installedApp.GitOpsRepoName, + Active: true, + } + + switch helmDeploymentConfig.DeploymentAppType { + case bean2.ArgoCd: + gitOpsConfig, err := impl.gitOpsConfigReadService.GetGitOpsProviderByRepoURL(installedApp.GitOpsRepoUrl) + if err != nil { + impl.logger.Infow("error in fetching gitOps config by repoUrl, skipping migration to deployment config", "repoURL", installedApp.GitOpsRepoUrl) + return nil, err + } + helmDeploymentConfig.ConfigType = bean.GitOps.String() + helmDeploymentConfig.CredentialIdInt = gitOpsConfig.Id + } + helmDeploymentConfig.CreateAuditLog(bean3.SYSTEM_USER_ID) + helmDeploymentConfig, err = impl.deploymentConfigRepository.Save(nil, helmDeploymentConfig) + if err != nil { + impl.logger.Errorw("error in saving deployment config for helm app", "appId", appId, "envId", envId, "err", err) + return nil, err + } + return helmDeploymentConfig, nil +} diff --git a/pkg/deployment/gitOps/config/GitOpsConfigReadService.go b/pkg/deployment/gitOps/config/GitOpsConfigReadService.go index bb0f9f4bcc..e0a9328cb5 100644 --- a/pkg/deployment/gitOps/config/GitOpsConfigReadService.go +++ b/pkg/deployment/gitOps/config/GitOpsConfigReadService.go @@ -161,6 +161,16 @@ func (impl *GitOpsConfigReadServiceImpl) GetConfiguredGitOpsCount() (int, error) } func (impl *GitOpsConfigReadServiceImpl) GetGitOpsProviderByRepoURL(gitRepoUrl string) (*bean2.GitOpsConfigDto, error) { + + if gitRepoUrl == bean2.GIT_REPO_NOT_CONFIGURED { + model, err := impl.GetGitOpsConfigActive() + if err != nil { + impl.logger.Errorw("error in getting default gitOps provider", "err", err) + return nil, err + } + return model, nil + } + models, err := impl.gitOpsRepository.GetAllGitOpsConfig() if err != nil { impl.logger.Errorw("error, GetGitOpsConfigActive", "err", err) diff --git a/pkg/pipeline/DeploymentPipelineConfigService.go b/pkg/pipeline/DeploymentPipelineConfigService.go index 16548747ed..1a5eae7254 100644 --- a/pkg/pipeline/DeploymentPipelineConfigService.go +++ b/pkg/pipeline/DeploymentPipelineConfigService.go @@ -440,7 +440,7 @@ func (impl *CdPipelineConfigServiceImpl) CreateCdPipelines(pipelineCreateRequest return nil, err } AppDeploymentConfig.RepoURL = chartGitAttr.RepoUrl - AppDeploymentConfig, err = impl.deploymentConfigService.CreateOrUpdateConfig(AppDeploymentConfig, pipelineCreateRequest.UserId) + AppDeploymentConfig, err = impl.deploymentConfigService.CreateOrUpdateConfig(nil, AppDeploymentConfig, pipelineCreateRequest.UserId) if err != nil { impl.logger.Errorw("error in fetching creating env config", "appId", app.Id, "err", err) return nil, err @@ -460,7 +460,7 @@ func (impl *CdPipelineConfigServiceImpl) CreateCdPipelines(pipelineCreateRequest CredentialIdString: AppDeploymentConfig.CredentialIdString, Active: true, } - envDeploymentConfig, err := impl.deploymentConfigService.CreateOrUpdateConfig(envDeploymentConfig, pipelineCreateRequest.UserId) + envDeploymentConfig, err := impl.deploymentConfigService.CreateOrUpdateConfig(nil, envDeploymentConfig, pipelineCreateRequest.UserId) if err != nil { impl.logger.Errorw("error in fetching creating env config", "appId", app.Id, "envId", pipeline.EnvironmentId, "err", err) return nil, err @@ -860,6 +860,12 @@ func (impl *CdPipelineConfigServiceImpl) DeleteCdPipeline(pipeline *pipelineConf impl.logger.Errorw("error in fetching environment deployment config by appId and envId", "appId", pipeline.AppId, "envId", pipeline.EnvironmentId, "err", err) return nil, err } + envDeploymentConfig.Active = false + envDeploymentConfig, err = impl.deploymentConfigService.CreateOrUpdateConfig(tx, envDeploymentConfig, userId) + if err != nil { + impl.logger.Errorw("error in deleting deployment config for pipeline", "appId", pipeline.AppId, "envId", pipeline.EnvironmentId, "err", err) + return nil, err + } //delete app from argo cd, if created if pipeline.DeploymentAppCreated == true { deploymentAppName := fmt.Sprintf("%s-%s", pipeline.App.AppName, pipeline.Environment.Name) diff --git a/scripts/sql/260_deployment_config.up.sql b/scripts/sql/260_deployment_config.up.sql new file mode 100644 index 0000000000..d436cea276 --- /dev/null +++ b/scripts/sql/260_deployment_config.up.sql @@ -0,0 +1,22 @@ +CREATE SEQUENCE IF NOT EXISTS id_seq_deployment_config; + +CREATE TABLE IF NOT EXISTS "public"."deployment_config" +( + "id" int NOT NULL DEFAULT nextval('id_seq_deployment_config'::regclass), + "app_id" int, + "environment_id" int, + "deployment_app_type" VARCHAR(50), + "config_type" VARCHAR(50), + "repo_url" VARCHAR(50), + "repo_name" VARCHAR(50), + "chart_location" VARCHAR(50), + "credential_type" VARCHAR(50), + "credential_id_int" int, + "credential_id_string" VARCHAR(50), + "active" bool, + "created_on" timestamptz, + "created_by" integer, + "updated_on" timestamptz, + "updated_by" integer, + PRIMARY KEY ("id") +); \ No newline at end of file diff --git a/wire_gen.go b/wire_gen.go index 3e20b4395e..65ce614493 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -425,7 +425,7 @@ func InitializeApp() (*App, error) { pipelineStatusSyncDetailServiceImpl := status.NewPipelineStatusSyncDetailServiceImpl(sugaredLogger, pipelineStatusSyncDetailRepositoryImpl) installedAppVersionHistoryRepositoryImpl := repository3.NewInstalledAppVersionHistoryRepositoryImpl(sugaredLogger, db) repositoryImpl := deploymentConfig.NewRepositoryImpl(db) - deploymentConfigServiceImpl := common.NewDeploymentConfigServiceImpl(repositoryImpl, sugaredLogger, chartRepositoryImpl, pipelineRepositoryImpl, gitOpsConfigReadServiceImpl) + deploymentConfigServiceImpl := common.NewDeploymentConfigServiceImpl(repositoryImpl, sugaredLogger, chartRepositoryImpl, pipelineRepositoryImpl, gitOpsConfigReadServiceImpl, appRepositoryImpl, installedAppRepositoryImpl) pipelineStatusTimelineServiceImpl := status.NewPipelineStatusTimelineServiceImpl(sugaredLogger, pipelineStatusTimelineRepositoryImpl, cdWorkflowRepositoryImpl, userServiceImpl, pipelineStatusTimelineResourcesServiceImpl, pipelineStatusSyncDetailServiceImpl, installedAppRepositoryImpl, installedAppVersionHistoryRepositoryImpl, deploymentConfigServiceImpl) appServiceConfig, err := app2.GetAppServiceConfig() if err != nil { @@ -518,7 +518,7 @@ func InitializeApp() (*App, error) { ciTemplateServiceImpl := pipeline.NewCiTemplateServiceImpl(sugaredLogger, ciBuildConfigServiceImpl, ciTemplateRepositoryImpl, ciTemplateOverrideRepositoryImpl) appLabelRepositoryImpl := pipelineConfig.NewAppLabelRepositoryImpl(db) materialRepositoryImpl := pipelineConfig.NewMaterialRepositoryImpl(db) - installedAppDBServiceImpl := EAMode.NewInstalledAppDBServiceImpl(sugaredLogger, installedAppRepositoryImpl, appRepositoryImpl, userServiceImpl, environmentServiceImpl, installedAppVersionHistoryRepositoryImpl) + installedAppDBServiceImpl := EAMode.NewInstalledAppDBServiceImpl(sugaredLogger, installedAppRepositoryImpl, appRepositoryImpl, userServiceImpl, environmentServiceImpl, installedAppVersionHistoryRepositoryImpl, deploymentConfigServiceImpl) crudOperationServiceConfig, err := app2.GetCrudOperationServiceConfig() if err != nil { return nil, err @@ -692,7 +692,7 @@ func InitializeApp() (*App, error) { if err != nil { return nil, err } - installedAppResourceServiceImpl := resource.NewInstalledAppResourceServiceImpl(sugaredLogger, installedAppRepositoryImpl, appStoreApplicationVersionRepositoryImpl, applicationServiceClientImpl, acdAuthConfig, installedAppVersionHistoryRepositoryImpl, argoUserServiceImpl, helmAppClientImpl, helmAppServiceImpl, appStatusServiceImpl, k8sCommonServiceImpl, k8sApplicationServiceImpl, k8sServiceImpl) + installedAppResourceServiceImpl := resource.NewInstalledAppResourceServiceImpl(sugaredLogger, installedAppRepositoryImpl, appStoreApplicationVersionRepositoryImpl, applicationServiceClientImpl, acdAuthConfig, installedAppVersionHistoryRepositoryImpl, argoUserServiceImpl, helmAppClientImpl, helmAppServiceImpl, appStatusServiceImpl, k8sCommonServiceImpl, k8sApplicationServiceImpl, k8sServiceImpl, deploymentConfigServiceImpl) chartGroupEntriesRepositoryImpl := repository17.NewChartGroupEntriesRepositoryImpl(db, sugaredLogger) chartGroupReposotoryImpl := repository17.NewChartGroupReposotoryImpl(db, sugaredLogger) chartGroupDeploymentRepositoryImpl := repository17.NewChartGroupDeploymentRepositoryImpl(db, sugaredLogger) @@ -701,12 +701,12 @@ func InitializeApp() (*App, error) { clusterInstalledAppsRepositoryImpl := repository3.NewClusterInstalledAppsRepositoryImpl(db, sugaredLogger) appStoreValuesServiceImpl := service3.NewAppStoreValuesServiceImpl(sugaredLogger, appStoreApplicationVersionRepositoryImpl, installedAppRepositoryImpl, appStoreVersionValuesRepositoryImpl, userServiceImpl) appStoreDeploymentCommonServiceImpl := appStoreDeploymentCommon.NewAppStoreDeploymentCommonServiceImpl(sugaredLogger, appStoreApplicationVersionRepositoryImpl, chartTemplateServiceImpl) - fullModeDeploymentServiceImpl := deployment.NewFullModeDeploymentServiceImpl(sugaredLogger, applicationServiceClientImpl, argoK8sClientImpl, acdAuthConfig, chartGroupDeploymentRepositoryImpl, installedAppRepositoryImpl, installedAppVersionHistoryRepositoryImpl, argoUserServiceImpl, appStoreDeploymentCommonServiceImpl, helmAppServiceImpl, appStatusServiceImpl, pipelineStatusTimelineServiceImpl, userServiceImpl, pipelineStatusTimelineRepositoryImpl, appStoreApplicationVersionRepositoryImpl, argoClientWrapperServiceImpl, acdConfig, gitOperationServiceImpl, gitOpsConfigReadServiceImpl, gitOpsValidationServiceImpl, environmentRepositoryImpl) + fullModeDeploymentServiceImpl := deployment.NewFullModeDeploymentServiceImpl(sugaredLogger, applicationServiceClientImpl, argoK8sClientImpl, acdAuthConfig, chartGroupDeploymentRepositoryImpl, installedAppRepositoryImpl, installedAppVersionHistoryRepositoryImpl, argoUserServiceImpl, appStoreDeploymentCommonServiceImpl, helmAppServiceImpl, appStatusServiceImpl, pipelineStatusTimelineServiceImpl, userServiceImpl, pipelineStatusTimelineRepositoryImpl, appStoreApplicationVersionRepositoryImpl, argoClientWrapperServiceImpl, acdConfig, gitOperationServiceImpl, gitOpsConfigReadServiceImpl, gitOpsValidationServiceImpl, environmentRepositoryImpl, deploymentConfigServiceImpl) appStoreValidatorImpl := service4.NewAppAppStoreValidatorImpl(sugaredLogger) - appStoreDeploymentDBServiceImpl := service4.NewAppStoreDeploymentDBServiceImpl(sugaredLogger, installedAppRepositoryImpl, appStoreApplicationVersionRepositoryImpl, appRepositoryImpl, environmentServiceImpl, clusterServiceImplExtended, installedAppVersionHistoryRepositoryImpl, environmentVariables, gitOpsConfigReadServiceImpl, deploymentTypeOverrideServiceImpl, fullModeDeploymentServiceImpl, appStoreValidatorImpl, installedAppDBServiceImpl) + appStoreDeploymentDBServiceImpl := service4.NewAppStoreDeploymentDBServiceImpl(sugaredLogger, installedAppRepositoryImpl, appStoreApplicationVersionRepositoryImpl, appRepositoryImpl, environmentServiceImpl, clusterServiceImplExtended, installedAppVersionHistoryRepositoryImpl, environmentVariables, gitOpsConfigReadServiceImpl, deploymentTypeOverrideServiceImpl, fullModeDeploymentServiceImpl, appStoreValidatorImpl, installedAppDBServiceImpl, deploymentConfigServiceImpl) eaModeDeploymentServiceImpl := EAMode.NewEAModeDeploymentServiceImpl(sugaredLogger, helmAppServiceImpl, appStoreApplicationVersionRepositoryImpl, helmAppClientImpl, installedAppRepositoryImpl, ociRegistryConfigRepositoryImpl) deletePostProcessorImpl := service4.NewDeletePostProcessorImpl(sugaredLogger) - appStoreDeploymentServiceImpl := service4.NewAppStoreDeploymentServiceImpl(sugaredLogger, installedAppRepositoryImpl, installedAppDBServiceImpl, appStoreDeploymentDBServiceImpl, chartGroupDeploymentRepositoryImpl, appStoreApplicationVersionRepositoryImpl, appRepositoryImpl, eaModeDeploymentServiceImpl, fullModeDeploymentServiceImpl, environmentServiceImpl, helmAppServiceImpl, installedAppVersionHistoryRepositoryImpl, environmentVariables, acdConfig, gitOpsConfigReadServiceImpl, deletePostProcessorImpl, appStoreValidatorImpl) + appStoreDeploymentServiceImpl := service4.NewAppStoreDeploymentServiceImpl(sugaredLogger, installedAppRepositoryImpl, installedAppDBServiceImpl, appStoreDeploymentDBServiceImpl, chartGroupDeploymentRepositoryImpl, appStoreApplicationVersionRepositoryImpl, appRepositoryImpl, eaModeDeploymentServiceImpl, fullModeDeploymentServiceImpl, environmentServiceImpl, helmAppServiceImpl, installedAppVersionHistoryRepositoryImpl, environmentVariables, acdConfig, gitOpsConfigReadServiceImpl, deletePostProcessorImpl, appStoreValidatorImpl, deploymentConfigServiceImpl) appStoreAppsEventPublishServiceImpl := out.NewAppStoreAppsEventPublishServiceImpl(sugaredLogger, pubSubClientServiceImpl) chartGroupServiceImpl, err := chartGroup.NewChartGroupServiceImpl(sugaredLogger, chartGroupEntriesRepositoryImpl, chartGroupReposotoryImpl, chartGroupDeploymentRepositoryImpl, installedAppRepositoryImpl, appStoreVersionValuesRepositoryImpl, appStoreRepositoryImpl, userAuthServiceImpl, appStoreApplicationVersionRepositoryImpl, environmentServiceImpl, teamRepositoryImpl, clusterInstalledAppsRepositoryImpl, appStoreValuesServiceImpl, appStoreDeploymentServiceImpl, appStoreDeploymentDBServiceImpl, argoUserServiceImpl, pipelineStatusTimelineServiceImpl, acdConfig, fullModeDeploymentServiceImpl, gitOperationServiceImpl, installedAppDBExtendedServiceImpl, appStoreAppsEventPublishServiceImpl) if err != nil { @@ -718,7 +718,7 @@ func InitializeApp() (*App, error) { return nil, err } cdApplicationStatusUpdateHandlerImpl := cron2.NewCdApplicationStatusUpdateHandlerImpl(sugaredLogger, appServiceImpl, workflowDagExecutorImpl, installedAppDBServiceImpl, appServiceConfig, pipelineStatusTimelineRepositoryImpl, eventRESTClientImpl, appListingRepositoryImpl, cdWorkflowRepositoryImpl, pipelineRepositoryImpl, installedAppVersionHistoryRepositoryImpl, installedAppRepositoryImpl, cronLoggerImpl, cdWorkflowCommonServiceImpl, workflowStatusServiceImpl) - installedAppDeploymentTypeChangeServiceImpl := deploymentTypeChange.NewInstalledAppDeploymentTypeChangeServiceImpl(sugaredLogger, installedAppRepositoryImpl, installedAppVersionHistoryRepositoryImpl, appStatusRepositoryImpl, gitOpsConfigReadServiceImpl, environmentRepositoryImpl, applicationServiceClientImpl, k8sCommonServiceImpl, k8sServiceImpl, fullModeDeploymentServiceImpl, eaModeDeploymentServiceImpl, argoClientWrapperServiceImpl, chartGroupServiceImpl, helmAppServiceImpl, argoUserServiceImpl, clusterServiceImplExtended, appRepositoryImpl) + installedAppDeploymentTypeChangeServiceImpl := deploymentTypeChange.NewInstalledAppDeploymentTypeChangeServiceImpl(sugaredLogger, installedAppRepositoryImpl, installedAppVersionHistoryRepositoryImpl, appStatusRepositoryImpl, gitOpsConfigReadServiceImpl, environmentRepositoryImpl, applicationServiceClientImpl, k8sCommonServiceImpl, k8sServiceImpl, fullModeDeploymentServiceImpl, eaModeDeploymentServiceImpl, argoClientWrapperServiceImpl, chartGroupServiceImpl, helmAppServiceImpl, argoUserServiceImpl, clusterServiceImplExtended, appRepositoryImpl, deploymentConfigServiceImpl) installedAppRestHandlerImpl := appStore.NewInstalledAppRestHandlerImpl(sugaredLogger, userServiceImpl, enforcerImpl, enforcerUtilImpl, enforcerUtilHelmImpl, installedAppDBExtendedServiceImpl, installedAppResourceServiceImpl, chartGroupServiceImpl, validate, clusterServiceImplExtended, applicationServiceClientImpl, appStoreDeploymentServiceImpl, appStoreDeploymentDBServiceImpl, helmAppClientImpl, argoUserServiceImpl, cdApplicationStatusUpdateHandlerImpl, installedAppRepositoryImpl, appCrudOperationServiceImpl, installedAppDeploymentTypeChangeServiceImpl) appStoreValuesRestHandlerImpl := appStoreValues.NewAppStoreValuesRestHandlerImpl(sugaredLogger, userServiceImpl, appStoreValuesServiceImpl) appStoreValuesRouterImpl := appStoreValues.NewAppStoreValuesRouterImpl(appStoreValuesRestHandlerImpl) From 852de8f412bcb5f44b645ba8ce0e17bf91233e3c Mon Sep 17 00:00:00 2001 From: ayushmaheshwari Date: Mon, 1 Jul 2024 13:26:18 +0530 Subject: [PATCH 04/45] dev testing fixes --- .../pipelineConfig/PipelineRepository.go | 2 +- .../repository/InstalledAppRepository.go | 10 ++-------- .../service/AppStoreDeploymentService.go | 9 ++++++++- pkg/chart/ChartService.go | 10 ---------- pkg/deployment/common/deploymentConfigService.go | 5 +++-- .../gitOps/config/GitOpsConfigReadService.go | 6 +++++- pkg/pipeline/AppDeploymentTypeChangeManager.go | 2 +- pkg/pipeline/DeploymentPipelineConfigService.go | 2 +- scripts/sql/260_deployment_config.down.sql | 1 + scripts/sql/260_deployment_config.up.sql | 14 +++++++------- util/version.go | 3 ++- 11 files changed, 31 insertions(+), 33 deletions(-) create mode 100644 scripts/sql/260_deployment_config.down.sql diff --git a/internal/sql/repository/pipelineConfig/PipelineRepository.go b/internal/sql/repository/pipelineConfig/PipelineRepository.go index c51753cc06..9642b9f426 100644 --- a/internal/sql/repository/pipelineConfig/PipelineRepository.go +++ b/internal/sql/repository/pipelineConfig/PipelineRepository.go @@ -789,7 +789,7 @@ func (impl PipelineRepositoryImpl) FindDeploymentAppTypeByAppIdAndEnvId(appId, e var deploymentAppType string err := impl.dbConnection.Model((*Pipeline)(nil)). Column("deployment_app_type"). - Where("app_id = ? and environment_id=?", appId, envId). + Where("app_id = ? and environment_id=? and deleted=false", appId, envId). Select(&deploymentAppType) return deploymentAppType, err } diff --git a/pkg/appStore/installedApp/repository/InstalledAppRepository.go b/pkg/appStore/installedApp/repository/InstalledAppRepository.go index 287b2b4f4e..5e641abae8 100644 --- a/pkg/appStore/installedApp/repository/InstalledAppRepository.go +++ b/pkg/appStore/installedApp/repository/InstalledAppRepository.go @@ -20,7 +20,6 @@ import ( "fmt" "github.com/devtron-labs/common-lib/utils/k8s/health" "github.com/devtron-labs/devtron/internal/sql/repository/app" - "github.com/devtron-labs/devtron/internal/sql/repository/deploymentConfig" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig/bean/timelineStatus" util2 "github.com/devtron-labs/devtron/internal/util" @@ -54,7 +53,6 @@ type InstalledApps struct { Notes string `json:"notes"` App app.App Environment repository.Environment - DeploymentConfig deploymentConfig.DeploymentConfig sql.AuditLog } @@ -713,16 +711,13 @@ func (impl InstalledAppRepositoryImpl) GetDeploymentSuccessfulStatusCountForTele } func (impl InstalledAppRepositoryImpl) GetGitOpsInstalledAppsWhereArgoAppDeletedIsTrue(installedAppId int, envId int) (InstalledApps, error) { - //TODO: test var installedApps InstalledApps err := impl.dbConnection.Model(&installedApps). - Column("installed_apps.*", "App.app_name", "Environment.namespace", "Environment.cluster_id", "Environment.environment_name", "DeploymentConfig"). - Join(" LEFT JOIN deployment_configuration dc on (installed_apps.app_id=dc.app_id and installed_apps.environment_id=dc.environment_id)"). + Column("installed_apps.*", "App.app_name", "Environment.namespace", "Environment.cluster_id", "Environment.environment_name"). Where("deployment_app_delete_request = ?", true). Where("installed_apps.active = ?", true). Where("installed_apps.id = ?", installedAppId). Where("installed_apps.environment_id = ?", envId). - Where("(installed_apps.deployment_app_type = ? or dc.deployment_app_type = ? )", util2.PIPELINE_DEPLOYMENT_TYPE_ACD, util2.PIPELINE_DEPLOYMENT_TYPE_ACD). Select() if err != nil && err != pg.ErrNoRows { impl.Logger.Errorw("error in fetching pipeline while udating delete status", "err", err) @@ -865,9 +860,8 @@ func (impl InstalledAppRepositoryImpl) GetActiveInstalledAppByEnvIdAndDeployment query := impl.dbConnection. Model(&installedApps). - Column("installed_apps.*", "App", "Environment", "DeploymentConfig"). + Column("installed_apps.*", "App", "Environment"). Join("inner join app a on installed_apps.app_id = a.id"). - Join("inner join "). Where("installed_apps.environment_id = ?", envId). Where("installed_apps.deployment_app_type = ?", deploymentType). Where("installed_apps.active = ?", true) diff --git a/pkg/appStore/installedApp/service/AppStoreDeploymentService.go b/pkg/appStore/installedApp/service/AppStoreDeploymentService.go index 5636b902cb..2db93e7759 100644 --- a/pkg/appStore/installedApp/service/AppStoreDeploymentService.go +++ b/pkg/appStore/installedApp/service/AppStoreDeploymentService.go @@ -863,7 +863,14 @@ func (impl *AppStoreDeploymentServiceImpl) MarkGitOpsInstalledAppsDeletedIfArgoA apiError.InternalMessage = "error in fetching partially deleted argoCd apps from installed app repo" return apiError } - if (!util.IsAcdApp(installedApp.DeploymentAppType) && !util.IsAcdApp(installedApp.DeploymentConfig.DeploymentAppType)) || !installedApp.DeploymentAppDeleteRequest { + deploymentConfig, err := impl.deploymentConfigService.GetDeploymentConfig(installedAppId, envId) + if err != nil { + impl.logger.Errorw("error in getting deployment config by appId and envId", "appId", installedAppId, "envId", envId, "err", err) + apiError.HttpStatusCode = http.StatusInternalServerError + apiError.InternalMessage = "error in fetching partially deleted argoCd apps from installed app repo" + return apiError + } + if (!util.IsAcdApp(installedApp.DeploymentAppType) && !util.IsAcdApp(deploymentConfig.DeploymentAppType)) || !installedApp.DeploymentAppDeleteRequest { return nil } // Operates for ArgoCd apps only diff --git a/pkg/chart/ChartService.go b/pkg/chart/ChartService.go index bf2caedd4b..fcd31aa757 100644 --- a/pkg/chart/ChartService.go +++ b/pkg/chart/ChartService.go @@ -288,16 +288,6 @@ func (impl *ChartServiceImpl) Create(templateRequest TemplateRequest, ctx contex return nil, err } - currentDeploymentConfig, err := impl.deploymentConfigService.GetDeploymentConfig(templateRequest.AppId, 0) - if err != nil && err != pg.ErrNoRows { - impl.logger.Errorw("error in getting current deployment config for appId", "appId", templateRequest.AppId, "err", err) - return nil, err - } - - if currentDeploymentConfig != nil && currentDeploymentConfig.Id > 0 { - gitRepoUrl = currentDeploymentConfig.RepoURL - } - deploymentConfig := &bean2.DeploymentConfig{ AppId: templateRequest.AppId, ConfigType: common.GetDeploymentConfigType(templateRequest.IsCustomGitRepository), diff --git a/pkg/deployment/common/deploymentConfigService.go b/pkg/deployment/common/deploymentConfigService.go index 389e374453..a5bd9e0a3b 100644 --- a/pkg/deployment/common/deploymentConfigService.go +++ b/pkg/deployment/common/deploymentConfigService.go @@ -70,6 +70,7 @@ func (impl *DeploymentConfigServiceImpl) CreateOrUpdateConfig(tx *pg.Tx, config return nil, err } } else { + newDBObj.Id = configDbObj.Id newDBObj.AuditLog.UpdateAuditLog(userId) newDBObj, err = impl.deploymentConfigRepository.Update(tx, newDBObj) if err != nil { @@ -163,7 +164,7 @@ func (impl *DeploymentConfigServiceImpl) migrateAppAndEnvLevelDataToDeploymentCo gitOpsConfig, err := impl.gitOpsConfigReadService.GetGitOpsProviderByRepoURL(configDbObj.RepoUrl) if err != nil { impl.logger.Infow("error in fetching gitOps config by repoUrl, skipping migration to deployment config", "repoURL", configDbObj.RepoUrl) - return configDbObj, nil + return configDbObj, err } configDbObj.CredentialIdInt = gitOpsConfig.Id } @@ -240,7 +241,7 @@ func (impl *DeploymentConfigServiceImpl) migrateHelmAppDataToDeploymentConfig(ap impl.logger.Infow("error in fetching gitOps config by repoUrl, skipping migration to deployment config", "repoURL", installedApp.GitOpsRepoUrl) return nil, err } - helmDeploymentConfig.ConfigType = bean.GitOps.String() + helmDeploymentConfig.ConfigType = bean.SYSTEM_GENERATED.String() helmDeploymentConfig.CredentialIdInt = gitOpsConfig.Id } helmDeploymentConfig.CreateAuditLog(bean3.SYSTEM_USER_ID) diff --git a/pkg/deployment/gitOps/config/GitOpsConfigReadService.go b/pkg/deployment/gitOps/config/GitOpsConfigReadService.go index e0a9328cb5..6bf8fcc2dc 100644 --- a/pkg/deployment/gitOps/config/GitOpsConfigReadService.go +++ b/pkg/deployment/gitOps/config/GitOpsConfigReadService.go @@ -185,7 +185,11 @@ func (impl *GitOpsConfigReadServiceImpl) GetGitOpsProviderByRepoURL(gitRepoUrl s } for _, model := range models { - if host, err := util.GetHost(model.Host); err != nil && host == requestHost { + host, err := util.GetHost(model.Host) + if err != nil { + return nil, fmt.Errorf("unable to parse host from repo URL: %s", gitRepoUrl) + } + if host == requestHost { gitOpsConfig = &bean2.GitOpsConfigDto{ Id: model.Id, Provider: model.Provider, diff --git a/pkg/pipeline/AppDeploymentTypeChangeManager.go b/pkg/pipeline/AppDeploymentTypeChangeManager.go index daebb3d358..fdea822217 100644 --- a/pkg/pipeline/AppDeploymentTypeChangeManager.go +++ b/pkg/pipeline/AppDeploymentTypeChangeManager.go @@ -488,7 +488,7 @@ func (impl *AppDeploymentTypeChangeManagerImpl) DeleteDeploymentApps(ctx context envDeploymentConfig.CredentialIdInt = activeGitOpsConfig.Id envDeploymentConfig.CredentialType = bean4.GitOps.String() - envDeploymentConfig, RepoURLUpdateErr = impl.deploymentConfigService.CreateOrUpdateConfig(envDeploymentConfig, userId) + envDeploymentConfig, RepoURLUpdateErr = impl.deploymentConfigService.CreateOrUpdateConfig(nil, envDeploymentConfig, userId) if RepoURLUpdateErr != nil { impl.logger.Errorw("error in saving deployment config for app", "appId", pipeline.AppId, "envId", pipeline.EnvironmentId, "err", err) } diff --git a/pkg/pipeline/DeploymentPipelineConfigService.go b/pkg/pipeline/DeploymentPipelineConfigService.go index 1a5eae7254..4529b6e4ea 100644 --- a/pkg/pipeline/DeploymentPipelineConfigService.go +++ b/pkg/pipeline/DeploymentPipelineConfigService.go @@ -451,7 +451,7 @@ func (impl *CdPipelineConfigServiceImpl) CreateCdPipelines(pipelineCreateRequest AppId: app.Id, EnvironmentId: pipeline.EnvironmentId, ConfigType: AppDeploymentConfig.ConfigType, - DeploymentAppType: AppDeploymentConfig.DeploymentAppType, + DeploymentAppType: pipeline.DeploymentAppType, RepoURL: AppDeploymentConfig.RepoURL, RepoName: AppDeploymentConfig.RepoName, ChartLocation: AppDeploymentConfig.ChartLocation, diff --git a/scripts/sql/260_deployment_config.down.sql b/scripts/sql/260_deployment_config.down.sql new file mode 100644 index 0000000000..a4f8551daa --- /dev/null +++ b/scripts/sql/260_deployment_config.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS deployment_config; \ No newline at end of file diff --git a/scripts/sql/260_deployment_config.up.sql b/scripts/sql/260_deployment_config.up.sql index d436cea276..8175c22678 100644 --- a/scripts/sql/260_deployment_config.up.sql +++ b/scripts/sql/260_deployment_config.up.sql @@ -5,14 +5,14 @@ CREATE TABLE IF NOT EXISTS "public"."deployment_config" "id" int NOT NULL DEFAULT nextval('id_seq_deployment_config'::regclass), "app_id" int, "environment_id" int, - "deployment_app_type" VARCHAR(50), - "config_type" VARCHAR(50), - "repo_url" VARCHAR(50), - "repo_name" VARCHAR(50), - "chart_location" VARCHAR(50), - "credential_type" VARCHAR(50), + "deployment_app_type" VARCHAR(100), + "config_type" VARCHAR(100), + "repo_url" VARCHAR(250), + "repo_name" VARCHAR(200), + "chart_location" VARCHAR(250), + "credential_type" VARCHAR(100), "credential_id_int" int, - "credential_id_string" VARCHAR(50), + "credential_id_string" VARCHAR(100), "active" bool, "created_on" timestamptz, "created_by" integer, diff --git a/util/version.go b/util/version.go index 4d467aff49..4e2c029459 100644 --- a/util/version.go +++ b/util/version.go @@ -39,7 +39,8 @@ func IsBaseStack() bool { } func IsFullStack() bool { - return GetDevtronVersion().ServerMode == SERVER_MODE_FULL + isFullStack := GetDevtronVersion().ServerMode == SERVER_MODE_FULL + return isFullStack } func IsHelmApp(appOfferingMode string) bool { From 2e39d037d5da6b584cbcedfe8136e9b02177d550 Mon Sep 17 00:00:00 2001 From: ayushmaheshwari Date: Mon, 1 Jul 2024 16:04:10 +0530 Subject: [PATCH 05/45] wire fix --- cmd/external-app/wire.go | 8 ++++++++ cmd/external-app/wire_gen.go | 11 ++++++++--- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/cmd/external-app/wire.go b/cmd/external-app/wire.go index 15a9287f4d..7903ba0c40 100644 --- a/cmd/external-app/wire.go +++ b/cmd/external-app/wire.go @@ -57,6 +57,7 @@ import ( "github.com/devtron-labs/devtron/internal/sql/repository" app2 "github.com/devtron-labs/devtron/internal/sql/repository/app" "github.com/devtron-labs/devtron/internal/sql/repository/appStatus" + "github.com/devtron-labs/devtron/internal/sql/repository/deploymentConfig" dockerRegistryRepository "github.com/devtron-labs/devtron/internal/sql/repository/dockerRegistry" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" security2 "github.com/devtron-labs/devtron/internal/sql/repository/security" @@ -67,6 +68,7 @@ import ( "github.com/devtron-labs/devtron/pkg/appStore/installedApp/service/FullMode/deployment" "github.com/devtron-labs/devtron/pkg/attributes" delete2 "github.com/devtron-labs/devtron/pkg/delete" + "github.com/devtron-labs/devtron/pkg/deployment/common" "github.com/devtron-labs/devtron/pkg/deployment/gitOps" "github.com/devtron-labs/devtron/pkg/deployment/providerConfig" "github.com/devtron-labs/devtron/pkg/kubernetesResourceAuditLogs" @@ -236,6 +238,12 @@ func InitializeApp() (*App, error) { // end: docker registry wire set injection cron.NewCronLoggerImpl, appStore.AppStoreWireSet, + + deploymentConfig.NewRepositoryImpl, + wire.Bind(new(deploymentConfig.Repository), new(*deploymentConfig.RepositoryImpl)), + + common.NewDeploymentConfigServiceImpl, + wire.Bind(new(common.DeploymentConfigService), new(*common.DeploymentConfigServiceImpl)), ) return &App{}, nil } diff --git a/cmd/external-app/wire_gen.go b/cmd/external-app/wire_gen.go index 3e303d78ad..f8c868b19f 100644 --- a/cmd/external-app/wire_gen.go +++ b/cmd/external-app/wire_gen.go @@ -47,6 +47,7 @@ import ( repository3 "github.com/devtron-labs/devtron/internal/sql/repository" "github.com/devtron-labs/devtron/internal/sql/repository/app" "github.com/devtron-labs/devtron/internal/sql/repository/appStatus" + "github.com/devtron-labs/devtron/internal/sql/repository/deploymentConfig" repository5 "github.com/devtron-labs/devtron/internal/sql/repository/dockerRegistry" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" "github.com/devtron-labs/devtron/internal/sql/repository/security" @@ -75,6 +76,7 @@ import ( repository2 "github.com/devtron-labs/devtron/pkg/cluster/repository" "github.com/devtron-labs/devtron/pkg/clusterTerminalAccess" delete2 "github.com/devtron-labs/devtron/pkg/delete" + "github.com/devtron-labs/devtron/pkg/deployment/common" "github.com/devtron-labs/devtron/pkg/deployment/gitOps/config" "github.com/devtron-labs/devtron/pkg/deployment/providerConfig" "github.com/devtron-labs/devtron/pkg/externalLink" @@ -254,21 +256,24 @@ func InitializeApp() (*App, error) { return nil, err } installedAppVersionHistoryRepositoryImpl := repository4.NewInstalledAppVersionHistoryRepositoryImpl(sugaredLogger, db) - installedAppDBServiceImpl := EAMode.NewInstalledAppDBServiceImpl(sugaredLogger, installedAppRepositoryImpl, appRepositoryImpl, userServiceImpl, environmentServiceImpl, installedAppVersionHistoryRepositoryImpl) + repositoryImpl := deploymentConfig.NewRepositoryImpl(db) + chartRepositoryImpl := chartRepoRepository.NewChartRepository(db, transactionUtilImpl) gitOpsConfigRepositoryImpl := repository3.NewGitOpsConfigRepositoryImpl(sugaredLogger, db) gitOpsConfigReadServiceImpl := config.NewGitOpsConfigReadServiceImpl(sugaredLogger, gitOpsConfigRepositoryImpl, userServiceImpl, environmentVariables) + deploymentConfigServiceImpl := common.NewDeploymentConfigServiceImpl(repositoryImpl, sugaredLogger, chartRepositoryImpl, pipelineRepositoryImpl, gitOpsConfigReadServiceImpl, appRepositoryImpl, installedAppRepositoryImpl) + installedAppDBServiceImpl := EAMode.NewInstalledAppDBServiceImpl(sugaredLogger, installedAppRepositoryImpl, appRepositoryImpl, userServiceImpl, environmentServiceImpl, installedAppVersionHistoryRepositoryImpl, deploymentConfigServiceImpl) attributesServiceImpl := attributes.NewAttributesServiceImpl(sugaredLogger, attributesRepositoryImpl) deploymentTypeOverrideServiceImpl := providerConfig.NewDeploymentTypeOverrideServiceImpl(sugaredLogger, environmentVariables, attributesServiceImpl) eaModeDeploymentServiceImpl := EAMode.NewEAModeDeploymentServiceImpl(sugaredLogger, helmAppServiceImpl, appStoreApplicationVersionRepositoryImpl, helmAppClientImpl, installedAppRepositoryImpl, ociRegistryConfigRepositoryImpl) appStoreValidatorImpl := service2.NewAppAppStoreValidatorImpl(sugaredLogger) - appStoreDeploymentDBServiceImpl := service2.NewAppStoreDeploymentDBServiceImpl(sugaredLogger, installedAppRepositoryImpl, appStoreApplicationVersionRepositoryImpl, appRepositoryImpl, environmentServiceImpl, clusterServiceImpl, installedAppVersionHistoryRepositoryImpl, environmentVariables, gitOpsConfigReadServiceImpl, deploymentTypeOverrideServiceImpl, eaModeDeploymentServiceImpl, appStoreValidatorImpl, installedAppDBServiceImpl) + appStoreDeploymentDBServiceImpl := service2.NewAppStoreDeploymentDBServiceImpl(sugaredLogger, installedAppRepositoryImpl, appStoreApplicationVersionRepositoryImpl, appRepositoryImpl, environmentServiceImpl, clusterServiceImpl, installedAppVersionHistoryRepositoryImpl, environmentVariables, gitOpsConfigReadServiceImpl, deploymentTypeOverrideServiceImpl, eaModeDeploymentServiceImpl, appStoreValidatorImpl, installedAppDBServiceImpl, deploymentConfigServiceImpl) chartGroupDeploymentRepositoryImpl := repository7.NewChartGroupDeploymentRepositoryImpl(db, sugaredLogger) acdConfig, err := argocdServer.GetACDDeploymentConfig() if err != nil { return nil, err } deletePostProcessorImpl := service2.NewDeletePostProcessorImpl(sugaredLogger) - appStoreDeploymentServiceImpl := service2.NewAppStoreDeploymentServiceImpl(sugaredLogger, installedAppRepositoryImpl, installedAppDBServiceImpl, appStoreDeploymentDBServiceImpl, chartGroupDeploymentRepositoryImpl, appStoreApplicationVersionRepositoryImpl, appRepositoryImpl, eaModeDeploymentServiceImpl, eaModeDeploymentServiceImpl, environmentServiceImpl, helmAppServiceImpl, installedAppVersionHistoryRepositoryImpl, environmentVariables, acdConfig, gitOpsConfigReadServiceImpl, deletePostProcessorImpl, appStoreValidatorImpl) + appStoreDeploymentServiceImpl := service2.NewAppStoreDeploymentServiceImpl(sugaredLogger, installedAppRepositoryImpl, installedAppDBServiceImpl, appStoreDeploymentDBServiceImpl, chartGroupDeploymentRepositoryImpl, appStoreApplicationVersionRepositoryImpl, appRepositoryImpl, eaModeDeploymentServiceImpl, eaModeDeploymentServiceImpl, environmentServiceImpl, helmAppServiceImpl, installedAppVersionHistoryRepositoryImpl, environmentVariables, acdConfig, gitOpsConfigReadServiceImpl, deletePostProcessorImpl, appStoreValidatorImpl, deploymentConfigServiceImpl) helmAppRestHandlerImpl := client2.NewHelmAppRestHandlerImpl(sugaredLogger, helmAppServiceImpl, enforcerImpl, clusterServiceImpl, enforcerUtilHelmImpl, appStoreDeploymentServiceImpl, installedAppDBServiceImpl, userServiceImpl, attributesServiceImpl, serverEnvConfigServerEnvConfig) helmAppRouterImpl := client2.NewHelmAppRouterImpl(helmAppRestHandlerImpl) argoApplicationServiceImpl := argoApplication.NewArgoApplicationServiceImpl(sugaredLogger, clusterRepositoryImpl, k8sServiceImpl, helmUserServiceImpl, helmAppServiceImpl) From 096ba0449fdbd2f23d5a2ef0b50b79655c49134e Mon Sep 17 00:00:00 2001 From: ayushmaheshwari Date: Tue, 2 Jul 2024 01:01:59 +0530 Subject: [PATCH 06/45] bulk get function --- .../repository/deploymentConfig/repository.go | 37 +++ .../pipelineConfig/PipelineRepository.go | 15 + .../repository/InstalledAppRepository.go | 13 + pkg/deployment/common/bean/bean.go | 5 + .../common/deploymentConfigService.go | 285 ++++++++++++++++++ .../gitOps/config/GitOpsConfigReadService.go | 62 ++++ 6 files changed, 417 insertions(+) diff --git a/internal/sql/repository/deploymentConfig/repository.go b/internal/sql/repository/deploymentConfig/repository.go index c994f2ffa9..cd7e373ed0 100644 --- a/internal/sql/repository/deploymentConfig/repository.go +++ b/internal/sql/repository/deploymentConfig/repository.go @@ -3,6 +3,7 @@ package deploymentConfig import ( "github.com/devtron-labs/devtron/pkg/sql" "github.com/go-pg/pg" + "github.com/go-pg/pg/orm" ) type DeploymentConfig struct { @@ -24,10 +25,13 @@ type DeploymentConfig struct { type Repository interface { Save(tx *pg.Tx, config *DeploymentConfig) (*DeploymentConfig, error) + SaveAll(tx *pg.Tx, configs []*DeploymentConfig) ([]*DeploymentConfig, error) Update(tx *pg.Tx, config *DeploymentConfig) (*DeploymentConfig, error) GetById(id int) (*DeploymentConfig, error) GetByAppIdAndEnvId(appId, envId int) (*DeploymentConfig, error) GetAppLevelConfig(appId int) (*DeploymentConfig, error) + GetAppLevelConfigByAppIds(appIds []int) ([]*DeploymentConfig, error) + GetAppAndEnvLevelConfigsInBulk(envIdToAppIdMap map[int]int) ([]*DeploymentConfig, error) } type RepositoryImpl struct { @@ -48,6 +52,16 @@ func (impl RepositoryImpl) Save(tx *pg.Tx, config *DeploymentConfig) (*Deploymen return config, err } +func (impl RepositoryImpl) SaveAll(tx *pg.Tx, configs []*DeploymentConfig) ([]*DeploymentConfig, error) { + var err error + if tx != nil { + err = tx.Insert(configs) + } else { + err = impl.dbConnection.Insert(configs) + } + return configs, err +} + func (impl RepositoryImpl) Update(tx *pg.Tx, config *DeploymentConfig) (*DeploymentConfig, error) { var err error if tx != nil { @@ -83,3 +97,26 @@ func (impl RepositoryImpl) GetAppLevelConfig(appId int) (*DeploymentConfig, erro Select() return result, err } + +func (impl RepositoryImpl) GetAppLevelConfigByAppIds(appIds []int) ([]*DeploymentConfig, error) { + var result []*DeploymentConfig + err := impl.dbConnection.Model(&result). + Where("app_id in (?) ", appIds). + Where("active = ?", true). + Select() + return result, err +} + +func (impl RepositoryImpl) GetAppAndEnvLevelConfigsInBulk(envIdToAppIdMap map[int]int) ([]*DeploymentConfig, error) { + var result []*DeploymentConfig + err := impl.dbConnection.Model(&result). + WhereGroup(func(query *orm.Query) (*orm.Query, error) { + for envId, appId := range envIdToAppIdMap { + query = query.Where("environment_id = ? and app_id = ? ", envId, appId) + } + return query, nil + }). + Where("active = ?", true). + Select() + return result, err +} diff --git a/internal/sql/repository/pipelineConfig/PipelineRepository.go b/internal/sql/repository/pipelineConfig/PipelineRepository.go index 9642b9f426..6aaf65a0b9 100644 --- a/internal/sql/repository/pipelineConfig/PipelineRepository.go +++ b/internal/sql/repository/pipelineConfig/PipelineRepository.go @@ -28,6 +28,7 @@ import ( "github.com/devtron-labs/devtron/pkg/cluster/repository" "github.com/devtron-labs/devtron/pkg/sql" "github.com/go-pg/pg" + "github.com/go-pg/pg/orm" "go.opentelemetry.io/otel" "go.uber.org/zap" "k8s.io/utils/pointer" @@ -126,6 +127,7 @@ type PipelineRepository interface { // FindWithEnvironmentByCiIds Possibility of duplicate environment names when filtered by unique pipeline ids FindWithEnvironmentByCiIds(ctx context.Context, cIPipelineIds []int) ([]*Pipeline, error) FindDeploymentAppTypeByAppIdAndEnvId(appId, envId int) (string, error) + FindByEnvToAppIdMapping(envToAppIdMap map[int]int) ([]*Pipeline, error) } type CiArtifactDTO struct { @@ -793,3 +795,16 @@ func (impl PipelineRepositoryImpl) FindDeploymentAppTypeByAppIdAndEnvId(appId, e Select(&deploymentAppType) return deploymentAppType, err } + +func (impl PipelineRepositoryImpl) FindByEnvToAppIdMapping(envToAppIdMap map[int]int) ([]*Pipeline, error) { + var pipelines []*Pipeline + err := impl.dbConnection.Model(&pipelines). + WhereGroup(func(query *orm.Query) (*orm.Query, error) { + for envId, appId := range envToAppIdMap { + query = query.Where("environment_id = ? and app_id = ? ", envId, appId) + } + return query, nil + }). + Select() + return pipelines, err +} diff --git a/pkg/appStore/installedApp/repository/InstalledAppRepository.go b/pkg/appStore/installedApp/repository/InstalledAppRepository.go index 5e641abae8..f916bd7e7c 100644 --- a/pkg/appStore/installedApp/repository/InstalledAppRepository.go +++ b/pkg/appStore/installedApp/repository/InstalledAppRepository.go @@ -159,6 +159,7 @@ type InstalledAppRepository interface { FindInstalledAppByIds(ids []int) ([]*InstalledApps, error) // FindInstalledAppsByAppId returns multiple installed apps for an appId, this only happens for external-apps with same name installed in diff namespaces FindInstalledAppsByAppId(appId int) ([]*InstalledApps, error) + FindInstalledAppByAppIds(appIds []int) ([]*InstalledApps, error) } type InstalledAppRepositoryImpl struct { @@ -921,3 +922,15 @@ func (impl InstalledAppRepositoryImpl) FindInstalledAppsByAppId(appId int) ([]*I } return installedApps, err } + +func (impl InstalledAppRepositoryImpl) FindInstalledAppByAppIds(appIds []int) ([]*InstalledApps, error) { + var installedApps []*InstalledApps + if len(appIds) == 0 { + return nil, nil + } + err := impl.dbConnection.Model(&installedApps). + Column("installed_apps.*"). + Where("installed_apps.app_id in (?)", appIds). + Select() + return installedApps, err +} diff --git a/pkg/deployment/common/bean/bean.go b/pkg/deployment/common/bean/bean.go index af6ad8404d..71b48a8b1d 100644 --- a/pkg/deployment/common/bean/bean.go +++ b/pkg/deployment/common/bean/bean.go @@ -15,6 +15,11 @@ type DeploymentConfig struct { Active bool } +type DeploymentConfigSelector struct { + AppId int + EnvironmentId int +} + type DeploymentConfigType string const ( diff --git a/pkg/deployment/common/deploymentConfigService.go b/pkg/deployment/common/deploymentConfigService.go index a5bd9e0a3b..f12125c635 100644 --- a/pkg/deployment/common/deploymentConfigService.go +++ b/pkg/deployment/common/deploymentConfigService.go @@ -3,6 +3,7 @@ package common import ( appRepository "github.com/devtron-labs/devtron/internal/sql/repository/app" "github.com/devtron-labs/devtron/internal/sql/repository/deploymentConfig" + "github.com/devtron-labs/devtron/internal/sql/repository/helper" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" "github.com/devtron-labs/devtron/pkg/appStore/installedApp/repository" bean3 "github.com/devtron-labs/devtron/pkg/auth/user/bean" @@ -19,6 +20,7 @@ type DeploymentConfigService interface { CreateOrUpdateConfig(tx *pg.Tx, config *bean.DeploymentConfig, userId int32) (*bean.DeploymentConfig, error) GetDeploymentConfig(appId, envId int) (*bean.DeploymentConfig, error) GetDeploymentConfigForHelmApp(appId, envId int) (*bean.DeploymentConfig, error) + GetDeploymentConfigInBulk(configSelector []*bean.DeploymentConfigSelector) ([]*bean.DeploymentConfig, error) } type DeploymentConfigServiceImpl struct { @@ -252,3 +254,286 @@ func (impl *DeploymentConfigServiceImpl) migrateHelmAppDataToDeploymentConfig(ap } return helmDeploymentConfig, nil } + +func (impl *DeploymentConfigServiceImpl) GetDeploymentConfigInBulk(configSelector []*bean.DeploymentConfigSelector) ([]*bean.DeploymentConfig, error) { + + appIds := make([]*int, len(configSelector)) + appIdToEnvIdMapping := make(map[int]int) + for _, s := range configSelector { + appIds = append(appIds, &s.AppId) + if s.EnvironmentId > 0 { + appIdToEnvIdMapping[s.AppId] = s.EnvironmentId + } + } + + apps, err := impl.appRepository.FindByIds(appIds) + if err != nil { + impl.logger.Errorw("error in fetching apps by ids", "ids", appIds, "err", err) + return nil, err + } + + devtronAppIds := make([]int, len(appIds)) + helmAppIds := make([]int, len(appIds)) + devtronAppsEnvToAppIdMapping := make(map[int]int) + helmAppsEnvToAppIdMapping := make(map[int]int) + + for _, a := range apps { + switch a.AppType { + case helper.CustomApp: + devtronAppIds = append(devtronAppIds, a.Id) + devtronAppsEnvToAppIdMapping[appIdToEnvIdMapping[a.Id]] = a.Id + case helper.ChartStoreApp: + helmAppIds = append(helmAppIds, a.Id) + helmAppsEnvToAppIdMapping[appIdToEnvIdMapping[a.Id]] = a.Id + } + } + + devtronAppConfigs, err := impl.GetDevtronAppConfigInBulk(devtronAppIds, devtronAppsEnvToAppIdMapping) + if err != nil { + impl.logger.Errorw("error in getting deployment config for devtron apps", "devtronAppsEnvToAppIdMapping", devtronAppsEnvToAppIdMapping, "err", err) + return nil, err + } + + helmAppConfigs, err := impl.GetHelmAppConfigInBulk(helmAppIds, helmAppsEnvToAppIdMapping) + if err != nil { + impl.logger.Errorw("error in getting deployment config for helm apps", "devtronAppsEnvToAppIdMapping", devtronAppsEnvToAppIdMapping, "err", err) + return nil, err + } + + allConfigs := make([]*bean.DeploymentConfig, len(devtronAppConfigs)+len(helmAppConfigs)) + allConfigs = append(allConfigs, devtronAppConfigs...) + allConfigs = append(allConfigs, helmAppConfigs...) + + return allConfigs, nil +} + +func (impl *DeploymentConfigServiceImpl) GetDevtronAppConfigInBulk(appIds []int, envIdToAppIdMapping map[int]int) ([]*bean.DeploymentConfig, error) { + + AppLevelDeploymentConfigs, err := impl.deploymentConfigRepository.GetAppLevelConfigByAppIds(appIds) + if err != nil { + impl.logger.Errorw("error in fetching deployment configs by appIds", "appIds", appIds, "err", err) + return nil, err + } + + AllAppLevelConfigs := make([]*deploymentConfig.DeploymentConfig, len(appIds)) + AllAppLevelConfigs = append(AllAppLevelConfigs, AppLevelDeploymentConfigs...) + + if len(AppLevelDeploymentConfigs) < len(appIds) { + presentAppIds := make(map[int]bool) + for _, c := range AppLevelDeploymentConfigs { + presentAppIds[c.AppId] = true + } + + notFoundAppIds := make([]int, len(appIds)) + for _, id := range appIds { + if _, ok := presentAppIds[id]; !ok { + notFoundAppIds = append(notFoundAppIds, id) + } + } + migratedAppLevelDeploymentConfigs, err := impl.migrateAppLevelDataTODeploymentConfigInBulk(notFoundAppIds) + if err != nil { + impl.logger.Errorw("error in migrating all level deployment configs", "appIds", notFoundAppIds, "err", err) + return nil, err + } + AllAppLevelConfigs = append(AllAppLevelConfigs, migratedAppLevelDeploymentConfigs...) + } + + appIdToAppLevelConfigMapping := make(map[int]*deploymentConfig.DeploymentConfig, len(appIds)) + for _, appLevelConfig := range AllAppLevelConfigs { + appIdToAppLevelConfigMapping[appLevelConfig.AppId] = appLevelConfig + } + + AllEnvLevelConfigs := make([]*deploymentConfig.DeploymentConfig, len(envIdToAppIdMapping)) + + if len(envIdToAppIdMapping) > 0 { + envLevelConfig, err := impl.deploymentConfigRepository.GetAppAndEnvLevelConfigsInBulk(envIdToAppIdMapping) + if err != nil { + impl.logger.Errorw("error in getting and and env level config in bulk", "envIdToAppIdMapping", envIdToAppIdMapping, "err", err) + return nil, err + } + AllEnvLevelConfigs = append(AllEnvLevelConfigs, envLevelConfig...) + if len(envLevelConfig) < len(envIdToAppIdMapping) { + notFoundEnvIdToAppIdsMap := make(map[int]int, len(envIdToAppIdMapping)) + presentEnvIds := make(map[int]bool) + for _, c := range envLevelConfig { + presentEnvIds[c.EnvironmentId] = true + } + for envId, appId := range envIdToAppIdMapping { + if _, ok := presentEnvIds[envId]; !ok { + notFoundEnvIdToAppIdsMap[envId] = appId + } + } + migratedEnvLevelDeploymentConfigs, err := impl.migrateEnvLevelDataTODeploymentConfigInBulk(notFoundEnvIdToAppIdsMap, envIdToAppIdMapping, appIdToAppLevelConfigMapping) + if err != nil { + impl.logger.Errorw("error in migrating env level configs", "envIdToAppIdMapping", envIdToAppIdMapping, "err", err) + return nil, err + } + AllEnvLevelConfigs = append(AllEnvLevelConfigs, migratedEnvLevelDeploymentConfigs...) + } + } + + allConfigs := make([]*bean.DeploymentConfig, len(appIds)+len(envIdToAppIdMapping)) + for _, c := range AllAppLevelConfigs { + allConfigs = append(allConfigs, ConvertDeploymentConfigDbObjToDTO(c)) + } + for _, c := range AllEnvLevelConfigs { + allConfigs = append(allConfigs, ConvertDeploymentConfigDbObjToDTO(c)) + } + + return allConfigs, err +} + +func (impl *DeploymentConfigServiceImpl) migrateAppLevelDataTODeploymentConfigInBulk(appIds []int) ([]*deploymentConfig.DeploymentConfig, error) { + charts, err := impl.chartRepository.FindLatestChartByAppIds(appIds) + if err != nil { + impl.logger.Errorw("error in fetching latest chart by appIds", "appIds", appIds, "err", err) + return nil, err + } + configDBObjects := make([]*deploymentConfig.DeploymentConfig, len(appIds)) + for _, c := range charts { + dbObj := &deploymentConfig.DeploymentConfig{ + ConfigType: GetDeploymentConfigType(c.IsCustomGitRepository), + AppId: c.AppId, + Active: true, + RepoUrl: c.GitRepoUrl, + ChartLocation: c.ChartLocation, + } + dbObj.AuditLog.CreateAuditLog(bean3.SYSTEM_USER_ID) + configDBObjects = append(configDBObjects, dbObj) + } + configDBObjects, err = impl.deploymentConfigRepository.SaveAll(nil, configDBObjects) + if err != nil { + impl.logger.Errorw("error in saving deployment config in DB", "appIds", appIds, "err", err) + return nil, err + } + return configDBObjects, nil +} + +func (impl *DeploymentConfigServiceImpl) migrateEnvLevelDataTODeploymentConfigInBulk(notFoundEnvToAppIdMapping map[int]int, envToAppIdMapping map[int]int, appIdToAppLevelConfigMapping map[int]*deploymentConfig.DeploymentConfig) ([]*deploymentConfig.DeploymentConfig, error) { + + pipelines, err := impl.pipelineRepository.FindByEnvToAppIdMapping(notFoundEnvToAppIdMapping) + if err != nil { + impl.logger.Errorw("error in fetching pipeline by envToAppId mapping", "envToAppIdMapping", envToAppIdMapping, "err", err) + return nil, err + } + + deploymentAppTypeMap := make(map[int]map[int]string) + for _, p := range pipelines { + deploymentAppTypeMap[p.AppId][p.EnvironmentId] = p.DeploymentAppType + } + + allRepoUrls := make([]string, 0) + for _, c := range appIdToAppLevelConfigMapping { + allRepoUrls = append(allRepoUrls, c.RepoUrl) + } + + repoUrlToConfigMapping, err := impl.gitOpsConfigReadService.GetGitOpsProviderMapByRepoURL(allRepoUrls) + if err != nil { + impl.logger.Errorw("error in fetching repoUrl to config mapping", "err", err) + return nil, err + } + + configDBObjects := make([]*deploymentConfig.DeploymentConfig, len(notFoundEnvToAppIdMapping)) + + for envId, appId := range notFoundEnvToAppIdMapping { + deploymentAppType := deploymentAppTypeMap[appId][envId] + appLevelConfig := appIdToAppLevelConfigMapping[appId] + configDbObj := &deploymentConfig.DeploymentConfig{ + AppId: appId, + EnvironmentId: envId, + ConfigType: appLevelConfig.ConfigType, + RepoUrl: appLevelConfig.RepoUrl, + ChartLocation: appLevelConfig.ChartLocation, + Active: true, + DeploymentAppType: deploymentAppType, + CredentialType: bean.GitOps.String(), + CredentialIdInt: repoUrlToConfigMapping[appLevelConfig.RepoUrl].Id, + } + configDbObj.AuditLog.CreateAuditLog(bean3.SYSTEM_USER_ID) + configDBObjects = append(configDBObjects, configDbObj) + } + + configDBObjects, err = impl.deploymentConfigRepository.SaveAll(nil, configDBObjects) + if err != nil { + impl.logger.Errorw("error in saving deployment config in DB", "envIdToAppIdMapping", notFoundEnvToAppIdMapping, "err", err) + return nil, err + } + + return configDBObjects, nil +} + +func (impl *DeploymentConfigServiceImpl) GetHelmAppConfigInBulk(appIds []int, envIdToAppIdMapping map[int]int) ([]*bean.DeploymentConfig, error) { + + allDeploymentConfigs := make([]*deploymentConfig.DeploymentConfig, len(appIds)) + + helmDeploymentConfig, err := impl.deploymentConfigRepository.GetAppAndEnvLevelConfigsInBulk(envIdToAppIdMapping) + if err != nil && err != pg.ErrNoRows { + impl.logger.Errorw("error in fetching deployment config by by appId and envId", "envIdToAppIdMapping", envIdToAppIdMapping, "err", err) + return nil, err + } + + allDeploymentConfigs = append(allDeploymentConfigs, helmDeploymentConfig...) + + if len(helmDeploymentConfig) < len(envIdToAppIdMapping) { + notFoundEnvIdToAppIdsMap := make(map[int]int, len(envIdToAppIdMapping)) + presentEnvIds := make(map[int]bool) + for _, c := range helmDeploymentConfig { + presentEnvIds[c.EnvironmentId] = true + } + for envId, appId := range envIdToAppIdMapping { + if _, ok := presentEnvIds[envId]; !ok { + notFoundEnvIdToAppIdsMap[envId] = appId + } + } + migratedConfigs, err := impl.migrateDeploymentConfigInBulkForHelmApps(appIds, notFoundEnvIdToAppIdsMap) + if err != nil { + impl.logger.Errorw("error in migrating helm apps config data in bult to deployment config", "migratedEnvToAppIdMapping", notFoundEnvIdToAppIdsMap, "err", err) + return nil, err + } + allDeploymentConfigs = append(allDeploymentConfigs, migratedConfigs...) + } + + deploymentConfigsResult := make([]*bean.DeploymentConfig, len(appIds)) + for _, c := range allDeploymentConfigs { + deploymentConfigsResult = append(deploymentConfigsResult, ConvertDeploymentConfigDbObjToDTO(c)) + } + return deploymentConfigsResult, nil +} + +func (impl *DeploymentConfigServiceImpl) migrateDeploymentConfigInBulkForHelmApps(appIds []int, envIdToAppIdMapping map[int]int) ([]*deploymentConfig.DeploymentConfig, error) { + installedApps, err := impl.installedAppRepository.FindInstalledAppByIds(appIds) + if err != nil { + impl.logger.Errorw("error in getting installed app by appId", "appIds", appIds, "err", err) + return nil, err + } + + appIdToInstalledAppMapping := make(map[int]*repository.InstalledApps) + + for _, ia := range installedApps { + appIdToInstalledAppMapping[ia.AppId] = ia + } + + configDBObjects := make([]*deploymentConfig.DeploymentConfig, len(envIdToAppIdMapping)) + + for envId, appId := range envIdToAppIdMapping { + installedApp := appIdToInstalledAppMapping[appId] + helmDeploymentConfig := &deploymentConfig.DeploymentConfig{ + AppId: appId, + EnvironmentId: envId, + DeploymentAppType: installedApp.DeploymentAppType, + ConfigType: GetDeploymentConfigType(installedApp.IsCustomRepository), + RepoUrl: installedApp.GitOpsRepoUrl, + RepoName: installedApp.GitOpsRepoName, + Active: true, + } + helmDeploymentConfig.AuditLog.CreateAuditLog(bean3.SYSTEM_USER_ID) + configDBObjects = append(configDBObjects, helmDeploymentConfig) + } + + configDBObjects, err = impl.deploymentConfigRepository.SaveAll(nil, configDBObjects) + if err != nil { + impl.logger.Errorw("error in saving deployment config in DB", "envIdToAppIdMapping", envIdToAppIdMapping, "err", err) + return nil, err + } + return configDBObjects, nil +} diff --git a/pkg/deployment/gitOps/config/GitOpsConfigReadService.go b/pkg/deployment/gitOps/config/GitOpsConfigReadService.go index 6bf8fcc2dc..5f46fb6252 100644 --- a/pkg/deployment/gitOps/config/GitOpsConfigReadService.go +++ b/pkg/deployment/gitOps/config/GitOpsConfigReadService.go @@ -39,6 +39,7 @@ type GitOpsConfigReadService interface { GetGitOpsConfigActive() (*bean2.GitOpsConfigDto, error) GetConfiguredGitOpsCount() (int, error) GetGitOpsProviderByRepoURL(gitRepoUrl string) (*bean2.GitOpsConfigDto, error) + GetGitOpsProviderMapByRepoURL(allGitRepoUrls []string) (map[string]*bean2.GitOpsConfigDto, error) } type GitOpsConfigReadServiceImpl struct { @@ -215,3 +216,64 @@ func (impl *GitOpsConfigReadServiceImpl) GetGitOpsProviderByRepoURL(gitRepoUrl s return gitOpsConfig, nil } + +func (impl *GitOpsConfigReadServiceImpl) GetGitOpsProviderMapByRepoURL(allGitRepoUrls []string) (map[string]*bean2.GitOpsConfigDto, error) { + + models, err := impl.gitOpsRepository.GetAllGitOpsConfig() + if err != nil { + impl.logger.Errorw("error, GetGitOpsConfigActive", "err", err) + return nil, err + } + + modelHostToConfigMapping := make(map[string]*bean2.GitOpsConfigDto) + for _, model := range models { + host, err := util.GetHost(model.Host) + if err != nil { + return nil, fmt.Errorf("unable to parse host from repo URL: %s", model.Host) + } + gitOpsConfig := &bean2.GitOpsConfigDto{ + Id: model.Id, + Provider: model.Provider, + GitHubOrgId: model.GitHubOrgId, + GitLabGroupId: model.GitLabGroupId, + Active: model.Active, + Token: model.Token, + Host: model.Host, + Username: model.Username, + UserId: model.CreatedBy, + AzureProjectName: model.AzureProject, + BitBucketWorkspaceId: model.BitBucketWorkspaceId, + BitBucketProjectKey: model.BitBucketProjectKey, + AllowCustomRepository: model.AllowCustomRepository, + } + modelHostToConfigMapping[host] = gitOpsConfig + } + + activeConfig, err := impl.GetGitOpsConfigActive() + if err != nil { + impl.logger.Errorw("error in getting active gitOps config", "err", err) + return nil, err + } + + repoUrlTOConfigMap := make(map[string]*bean2.GitOpsConfigDto) + + for _, gitRepoUrl := range allGitRepoUrls { + if gitRepoUrl == bean2.GIT_REPO_NOT_CONFIGURED { + repoUrlTOConfigMap[gitRepoUrl] = activeConfig + continue + } + requestHost, err := util.GetHost(gitRepoUrl) + if err != nil { + return nil, fmt.Errorf("unable to parse host from repo URL: %s", gitRepoUrl) + } + + if config, ok := modelHostToConfigMapping[requestHost]; ok { + repoUrlTOConfigMap[gitRepoUrl] = config + } else if !ok { + impl.logger.Infow("no gitops config found in DB for given url", "repoURL", gitRepoUrl) + repoUrlTOConfigMap[gitRepoUrl] = activeConfig // default behaviour + } + } + + return repoUrlTOConfigMap, nil +} From 7b601eb4838891ba1f8763d2da93cf002f930f8d Mon Sep 17 00:00:00 2001 From: ayushmaheshwari Date: Tue, 2 Jul 2024 07:58:43 +0530 Subject: [PATCH 07/45] wip: bulk logic fixes --- .../common/deploymentConfigService.go | 24 +++++++++++++++---- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/pkg/deployment/common/deploymentConfigService.go b/pkg/deployment/common/deploymentConfigService.go index f12125c635..2998a1ef58 100644 --- a/pkg/deployment/common/deploymentConfigService.go +++ b/pkg/deployment/common/deploymentConfigService.go @@ -10,7 +10,6 @@ import ( chartRepoRepository "github.com/devtron-labs/devtron/pkg/chartRepo/repository" "github.com/devtron-labs/devtron/pkg/deployment/common/bean" "github.com/devtron-labs/devtron/pkg/deployment/gitOps/config" - "github.com/devtron-labs/devtron/pkg/deployment/gitOps/git" bean2 "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/bean" "github.com/go-pg/pg" "go.uber.org/zap" @@ -31,7 +30,6 @@ type DeploymentConfigServiceImpl struct { gitOpsConfigReadService config.GitOpsConfigReadService appRepository appRepository.AppRepository installedAppRepository repository.InstalledAppRepository - git.GitOperationService } func NewDeploymentConfigServiceImpl( @@ -323,7 +321,6 @@ func (impl *DeploymentConfigServiceImpl) GetDevtronAppConfigInBulk(appIds []int, for _, c := range AppLevelDeploymentConfigs { presentAppIds[c.AppId] = true } - notFoundAppIds := make([]int, len(appIds)) for _, id := range appIds { if _, ok := presentAppIds[id]; !ok { @@ -446,8 +443,11 @@ func (impl *DeploymentConfigServiceImpl) migrateEnvLevelDataTODeploymentConfigIn ChartLocation: appLevelConfig.ChartLocation, Active: true, DeploymentAppType: deploymentAppType, - CredentialType: bean.GitOps.String(), - CredentialIdInt: repoUrlToConfigMapping[appLevelConfig.RepoUrl].Id, + } + switch deploymentAppType { + case bean2.ArgoCd: + configDbObj.CredentialType = bean.GitOps.String() + configDbObj.CredentialIdInt = repoUrlToConfigMapping[appLevelConfig.RepoUrl].Id } configDbObj.AuditLog.CreateAuditLog(bean3.SYSTEM_USER_ID) configDBObjects = append(configDBObjects, configDbObj) @@ -509,8 +509,17 @@ func (impl *DeploymentConfigServiceImpl) migrateDeploymentConfigInBulkForHelmApp appIdToInstalledAppMapping := make(map[int]*repository.InstalledApps) + allRepoURLS := make([]string, len(installedApps)) + for _, ia := range installedApps { appIdToInstalledAppMapping[ia.AppId] = ia + allRepoURLS = append(allRepoURLS, ia.GitOpsRepoUrl) + } + + repoUrlToConfigMapping, err := impl.gitOpsConfigReadService.GetGitOpsProviderMapByRepoURL(allRepoURLS) + if err != nil { + impl.logger.Errorw("error in fetching repoUrl to config mapping", "err", err) + return nil, err } configDBObjects := make([]*deploymentConfig.DeploymentConfig, len(envIdToAppIdMapping)) @@ -526,6 +535,11 @@ func (impl *DeploymentConfigServiceImpl) migrateDeploymentConfigInBulkForHelmApp RepoName: installedApp.GitOpsRepoName, Active: true, } + switch installedApp.DeploymentAppType { + case bean2.ArgoCd: + helmDeploymentConfig.CredentialType = bean.GitOps.String() + helmDeploymentConfig.CredentialIdInt = repoUrlToConfigMapping[installedApp.GitOpsRepoUrl].Id + } helmDeploymentConfig.AuditLog.CreateAuditLog(bean3.SYSTEM_USER_ID) configDBObjects = append(configDBObjects, helmDeploymentConfig) } From 847da98ccc4e1800496fa36b7142a63642f3c3f1 Mon Sep 17 00:00:00 2001 From: ayushmaheshwari Date: Tue, 2 Jul 2024 10:05:35 +0530 Subject: [PATCH 08/45] wip: bulk fixes --- .../repository/deploymentConfig/repository.go | 2 +- .../repository/InstalledAppRepository.go | 2 +- .../common/deploymentConfigService.go | 22 +++++++++---------- wire_gen.go | 5 +++++ 4 files changed, 18 insertions(+), 13 deletions(-) diff --git a/internal/sql/repository/deploymentConfig/repository.go b/internal/sql/repository/deploymentConfig/repository.go index cd7e373ed0..240e1ce711 100644 --- a/internal/sql/repository/deploymentConfig/repository.go +++ b/internal/sql/repository/deploymentConfig/repository.go @@ -101,7 +101,7 @@ func (impl RepositoryImpl) GetAppLevelConfig(appId int) (*DeploymentConfig, erro func (impl RepositoryImpl) GetAppLevelConfigByAppIds(appIds []int) ([]*DeploymentConfig, error) { var result []*DeploymentConfig err := impl.dbConnection.Model(&result). - Where("app_id in (?) ", appIds). + Where("app_id in (?) and environment_id is NULL ", pg.In(appIds)). Where("active = ?", true). Select() return result, err diff --git a/pkg/appStore/installedApp/repository/InstalledAppRepository.go b/pkg/appStore/installedApp/repository/InstalledAppRepository.go index f916bd7e7c..329d93ccde 100644 --- a/pkg/appStore/installedApp/repository/InstalledAppRepository.go +++ b/pkg/appStore/installedApp/repository/InstalledAppRepository.go @@ -930,7 +930,7 @@ func (impl InstalledAppRepositoryImpl) FindInstalledAppByAppIds(appIds []int) ([ } err := impl.dbConnection.Model(&installedApps). Column("installed_apps.*"). - Where("installed_apps.app_id in (?)", appIds). + Where("installed_apps.app_id in (?)", pg.In(appIds)). Select() return installedApps, err } diff --git a/pkg/deployment/common/deploymentConfigService.go b/pkg/deployment/common/deploymentConfigService.go index 2998a1ef58..5167c60bbe 100644 --- a/pkg/deployment/common/deploymentConfigService.go +++ b/pkg/deployment/common/deploymentConfigService.go @@ -313,7 +313,7 @@ func (impl *DeploymentConfigServiceImpl) GetDevtronAppConfigInBulk(appIds []int, return nil, err } - AllAppLevelConfigs := make([]*deploymentConfig.DeploymentConfig, len(appIds)) + AllAppLevelConfigs := make([]*deploymentConfig.DeploymentConfig, 0, len(appIds)) AllAppLevelConfigs = append(AllAppLevelConfigs, AppLevelDeploymentConfigs...) if len(AppLevelDeploymentConfigs) < len(appIds) { @@ -321,7 +321,7 @@ func (impl *DeploymentConfigServiceImpl) GetDevtronAppConfigInBulk(appIds []int, for _, c := range AppLevelDeploymentConfigs { presentAppIds[c.AppId] = true } - notFoundAppIds := make([]int, len(appIds)) + notFoundAppIds := make([]int, 0, len(appIds)) for _, id := range appIds { if _, ok := presentAppIds[id]; !ok { notFoundAppIds = append(notFoundAppIds, id) @@ -340,7 +340,7 @@ func (impl *DeploymentConfigServiceImpl) GetDevtronAppConfigInBulk(appIds []int, appIdToAppLevelConfigMapping[appLevelConfig.AppId] = appLevelConfig } - AllEnvLevelConfigs := make([]*deploymentConfig.DeploymentConfig, len(envIdToAppIdMapping)) + AllEnvLevelConfigs := make([]*deploymentConfig.DeploymentConfig, 0, len(envIdToAppIdMapping)) if len(envIdToAppIdMapping) > 0 { envLevelConfig, err := impl.deploymentConfigRepository.GetAppAndEnvLevelConfigsInBulk(envIdToAppIdMapping) @@ -369,7 +369,7 @@ func (impl *DeploymentConfigServiceImpl) GetDevtronAppConfigInBulk(appIds []int, } } - allConfigs := make([]*bean.DeploymentConfig, len(appIds)+len(envIdToAppIdMapping)) + allConfigs := make([]*bean.DeploymentConfig, 0, len(appIds)+len(envIdToAppIdMapping)) for _, c := range AllAppLevelConfigs { allConfigs = append(allConfigs, ConvertDeploymentConfigDbObjToDTO(c)) } @@ -386,7 +386,7 @@ func (impl *DeploymentConfigServiceImpl) migrateAppLevelDataTODeploymentConfigIn impl.logger.Errorw("error in fetching latest chart by appIds", "appIds", appIds, "err", err) return nil, err } - configDBObjects := make([]*deploymentConfig.DeploymentConfig, len(appIds)) + configDBObjects := make([]*deploymentConfig.DeploymentConfig, 0, len(appIds)) for _, c := range charts { dbObj := &deploymentConfig.DeploymentConfig{ ConfigType: GetDeploymentConfigType(c.IsCustomGitRepository), @@ -430,7 +430,7 @@ func (impl *DeploymentConfigServiceImpl) migrateEnvLevelDataTODeploymentConfigIn return nil, err } - configDBObjects := make([]*deploymentConfig.DeploymentConfig, len(notFoundEnvToAppIdMapping)) + configDBObjects := make([]*deploymentConfig.DeploymentConfig, 0, len(notFoundEnvToAppIdMapping)) for envId, appId := range notFoundEnvToAppIdMapping { deploymentAppType := deploymentAppTypeMap[appId][envId] @@ -464,7 +464,7 @@ func (impl *DeploymentConfigServiceImpl) migrateEnvLevelDataTODeploymentConfigIn func (impl *DeploymentConfigServiceImpl) GetHelmAppConfigInBulk(appIds []int, envIdToAppIdMapping map[int]int) ([]*bean.DeploymentConfig, error) { - allDeploymentConfigs := make([]*deploymentConfig.DeploymentConfig, len(appIds)) + allDeploymentConfigs := make([]*deploymentConfig.DeploymentConfig, 0, len(appIds)) helmDeploymentConfig, err := impl.deploymentConfigRepository.GetAppAndEnvLevelConfigsInBulk(envIdToAppIdMapping) if err != nil && err != pg.ErrNoRows { @@ -475,7 +475,7 @@ func (impl *DeploymentConfigServiceImpl) GetHelmAppConfigInBulk(appIds []int, en allDeploymentConfigs = append(allDeploymentConfigs, helmDeploymentConfig...) if len(helmDeploymentConfig) < len(envIdToAppIdMapping) { - notFoundEnvIdToAppIdsMap := make(map[int]int, len(envIdToAppIdMapping)) + notFoundEnvIdToAppIdsMap := make(map[int]int) presentEnvIds := make(map[int]bool) for _, c := range helmDeploymentConfig { presentEnvIds[c.EnvironmentId] = true @@ -493,7 +493,7 @@ func (impl *DeploymentConfigServiceImpl) GetHelmAppConfigInBulk(appIds []int, en allDeploymentConfigs = append(allDeploymentConfigs, migratedConfigs...) } - deploymentConfigsResult := make([]*bean.DeploymentConfig, len(appIds)) + deploymentConfigsResult := make([]*bean.DeploymentConfig, 0, len(appIds)) for _, c := range allDeploymentConfigs { deploymentConfigsResult = append(deploymentConfigsResult, ConvertDeploymentConfigDbObjToDTO(c)) } @@ -509,7 +509,7 @@ func (impl *DeploymentConfigServiceImpl) migrateDeploymentConfigInBulkForHelmApp appIdToInstalledAppMapping := make(map[int]*repository.InstalledApps) - allRepoURLS := make([]string, len(installedApps)) + allRepoURLS := make([]string, 0, len(installedApps)) for _, ia := range installedApps { appIdToInstalledAppMapping[ia.AppId] = ia @@ -522,7 +522,7 @@ func (impl *DeploymentConfigServiceImpl) migrateDeploymentConfigInBulkForHelmApp return nil, err } - configDBObjects := make([]*deploymentConfig.DeploymentConfig, len(envIdToAppIdMapping)) + configDBObjects := make([]*deploymentConfig.DeploymentConfig, 0, len(envIdToAppIdMapping)) for envId, appId := range envIdToAppIdMapping { installedApp := appIdToInstalledAppMapping[appId] diff --git a/wire_gen.go b/wire_gen.go index 65ce614493..43ef5d02ff 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -948,6 +948,11 @@ func InitializeApp() (*App, error) { if err != nil { return nil, err } + deploymentConfigs, err := deploymentConfigServiceImpl.GetDevtronAppConfigInBulk([]int{1,6}, map[int]int{1:1,2:1,3:1,4:1} ) + if err!=nil{ + + } + _ = len(deploymentConfigs) mainApp := NewApp(muxRouter, sugaredLogger, sseSSE, syncedEnforcer, db, sessionManager, posthogClient, loggingMiddlewareImpl, centralEventProcessor, pubSubClientServiceImpl, workflowEventProcessorImpl) return mainApp, nil } From 81cd5512d4d5ca35c3ae51035a022b2f4ba1deab Mon Sep 17 00:00:00 2001 From: kartik-579 <84493919+kartik-579@users.noreply.github.com> Date: Tue, 2 Jul 2024 19:34:58 +0530 Subject: [PATCH 09/45] feat: Cd pipeline migration sql and query usage update (#5416) * Added sql * removed unused mthod * wip * wip * wip * wip * wip * wip * QUERY : updated GetOfLatestCdWfrByCdPipelineId * QUERY : GetOfLatestInstalledAppVersionHistoryByInstalledAppVersionId * QUERY : fix GetOfLatestCdWfrByCdPipelineId * QUERY : fix GetOfLatestInstalledAppVersionHistoryByInstalledAppVersionId * QUERY : updated GetLatestTriggersOfHelmPipelinesStuckInNonTerminalStatuses * QUERY : updated GetArgoPipelineByArgoAppName * QUERY : updated GetArgoPipelinesHavingLatestTriggerStuckInNonTerminalStatuses * QUERY : updated GetArgoPipelinesHavingTriggersStuckInLastPossibleNonTerminalTimelines * QUERY : updated GetAppAndEnvDetailsForDeploymentAppTypePipeline * QUERY : updated FindActiveByEnvIdAndDeploymentType * QUERY : updated GetActiveInstalledAppByEnvIdAndDeploymentType * QUERY : updated GetArgoPipelinesHavingTriggersStuckInLastPossibleNonTerminalTimelinesForAppStore * QUERY : updated GetArgoPipelinesHavingLatestTriggerStuckInNonTerminalStatusesForAppStore * QUERY : updated GetInstalledAppByInstalledAppVersionId * QUERY : updated GetInstalledAppByAppIdAndDeploymentType * QUERY : updated GetAppAndEnvDetailsForDeploymentAppTypeInstalledApps * QUERY : updated FindLatestByAppIdAndEnvId * QUERY : updated GetInstalledAppByGitRepoUrl * QUERY : updated FindChartByGitRepoUrl --- api/helm-app/service/HelmAppService.go | 2 +- .../chartConfig/PipelineOverrideRepository.go | 21 +---------- .../pipelineConfig/CdWorfkflowRepository.go | 7 ++-- .../pipelineConfig/PipelineRepository.go | 37 +++++++------------ .../PipelineStatusSyncDetailRepository.go | 11 ++++-- .../repository/InstalledAppRepository.go | 34 ++++++++++------- pkg/chartRepo/repository/ChartsRepository.go | 3 +- .../sql/265_migration_deployed_app.down.sql | 1 + scripts/sql/265_migration_deployed_app.up.sql | 19 ++++++++++ 9 files changed, 70 insertions(+), 65 deletions(-) create mode 100644 scripts/sql/265_migration_deployed_app.down.sql create mode 100644 scripts/sql/265_migration_deployed_app.up.sql diff --git a/api/helm-app/service/HelmAppService.go b/api/helm-app/service/HelmAppService.go index 7c9b2dd24e..0400f05a3c 100644 --- a/api/helm-app/service/HelmAppService.go +++ b/api/helm-app/service/HelmAppService.go @@ -169,7 +169,7 @@ func (impl *HelmAppServiceImpl) ListHelmApplications(ctx context.Context, cluste impl.logger.Errorw("error in fetching helm app list from DB created using cd_pipelines", "clusters", clusterIds, "err", err) } - // if not hyperion mode, then fetch from installed_apps whose deployment_app_type is helm (as in hyperion mode, these apps should be treated as external-apps) + // if not hyperion mode, then fetch from installed_apps whose deploymentAppType is helm (as in hyperion mode, these apps should be treated as external-apps) if !util2.IsBaseStack() { newCtx, span = otel.Tracer("pipelineRepository").Start(newCtx, "GetAppAndEnvDetailsForDeploymentAppTypePipeline") start = time.Now() diff --git a/internal/sql/repository/chartConfig/PipelineOverrideRepository.go b/internal/sql/repository/chartConfig/PipelineOverrideRepository.go index 0e9a29f4de..56e1ce06c6 100644 --- a/internal/sql/repository/chartConfig/PipelineOverrideRepository.go +++ b/internal/sql/repository/chartConfig/PipelineOverrideRepository.go @@ -18,14 +18,11 @@ package chartConfig import ( "context" - "github.com/devtron-labs/devtron/api/bean" - argoApplication "github.com/devtron-labs/devtron/client/argocdServer/bean" "github.com/devtron-labs/devtron/internal/sql/models" "github.com/devtron-labs/devtron/internal/sql/repository" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" "github.com/devtron-labs/devtron/internal/util" "github.com/devtron-labs/devtron/pkg/sql" - deploymentStatus "github.com/devtron-labs/devtron/util" "github.com/go-pg/pg" "github.com/go-pg/pg/orm" "github.com/juju/errors" @@ -76,7 +73,6 @@ type PipelineOverrideRepository interface { GetByDeployedImage(appId, environmentId int, images []string) (pipelineOverride *PipelineOverride, err error) GetLatestReleaseByPipelineIds(pipelineIds []int) (pipelineOverrides []*PipelineOverride, err error) GetLatestReleaseDeploymentType(pipelineIds []int) ([]*PipelineOverride, error) - FetchHelmTypePipelineOverridesForStatusUpdate() (pipelines []*PipelineOverride, err error) FindLatestByAppIdAndEnvId(appId, environmentId int, deploymentAppType string) (pipelineOverrides *PipelineOverride, err error) FindLatestByCdWorkflowId(cdWorkflowId int) (pipelineOverride *PipelineOverride, err error) } @@ -296,28 +292,15 @@ func (impl PipelineOverrideRepositoryImpl) FindById(id int) (*PipelineOverride, return &pipelineOverride, err } -func (impl PipelineOverrideRepositoryImpl) FetchHelmTypePipelineOverridesForStatusUpdate() (pipelines []*PipelineOverride, err error) { - err = impl.dbConnection.Model(&pipelines). - Column("pipeline_override.*", "Pipeline", "Pipeline.App", "Pipeline.Environment"). - Join("inner join pipeline p on p.id = pipeline_override.pipeline_id"). - Join("inner join cd_workflow cdwf on cdwf.pipeline_id = p.id"). - Join("inner join cd_workflow_runner cdwfr on cdwfr.cd_workflow_id = cdwf.id"). - Where("p.deployment_app_type = ?", util.PIPELINE_DEPLOYMENT_TYPE_HELM). - Where("cdwfr.status not in (?)", pg.In([]string{argoApplication.Degraded, argoApplication.HIBERNATING, argoApplication.Healthy, deploymentStatus.WorkflowFailed, deploymentStatus.WorkflowAborted})). - Where("cdwfr.workflow_type = ?", bean.CD_WORKFLOW_TYPE_DEPLOY). - Where("p.deleted = ?", false). - Select() - return pipelines, err -} - func (impl PipelineOverrideRepositoryImpl) FindLatestByAppIdAndEnvId(appId, environmentId int, deploymentAppType string) (pipelineOverrides *PipelineOverride, err error) { var override PipelineOverride err = impl.dbConnection.Model(&override). Column("pipeline_override.*", "Pipeline"). Join("inner join pipeline p on p.id = pipeline_override.pipeline_id"). + Join("LEFT JOIN deployment_config dc on dc.app_id = p.app_id and dc.environment_id=p.environment_id"). Where("pipeline.app_id =? ", appId). Where("pipeline.environment_id =?", environmentId). - Where("p.deployment_app_type = ?", deploymentAppType). + Where("(p.deployment_app_type=? or dc.deployment_app_type=?)", deploymentAppType, deploymentAppType). Where("p.deleted = ?", false). Order("id DESC").Limit(1). Select() diff --git a/internal/sql/repository/pipelineConfig/CdWorfkflowRepository.go b/internal/sql/repository/pipelineConfig/CdWorfkflowRepository.go index 859c094ab2..966444605f 100644 --- a/internal/sql/repository/pipelineConfig/CdWorfkflowRepository.go +++ b/internal/sql/repository/pipelineConfig/CdWorfkflowRepository.go @@ -488,7 +488,7 @@ func (impl *CdWorkflowRepositoryImpl) FindLastUnFailedProcessedRunner(appId int, wfr := &CdWorkflowRunner{} err := impl.dbConnection. Model(wfr). - Column("cd_workflow_runner.*", "CdWorkflow.Pipeline.id", "CdWorkflow.Pipeline.deployment_app_delete_request", "CdWorkflow.Pipeline.deployment_app_type"). + Column("cd_workflow_runner.*", "CdWorkflow.Pipeline.id", "CdWorkflow.Pipeline.deployment_app_delete_request"). Where("p.environment_id = ?", environmentId). Where("p.app_id = ?", appId). Where("cd_workflow_runner.workflow_type = ?", apiBean.CD_WORKFLOW_TYPE_DEPLOY). @@ -752,10 +752,11 @@ func (impl *CdWorkflowRepositoryImpl) GetLatestTriggersOfHelmPipelinesStuckInNon excludedStatusList = append(excludedStatusList, WorkflowInitiated, WorkflowInQueue, WorkflowStarting) err := impl.dbConnection. Model(&wfrList). - Column("cd_workflow_runner.*", "CdWorkflow.id", "CdWorkflow.pipeline_id", "CdWorkflow.Pipeline.id", "CdWorkflow.Pipeline.deployment_app_name", "CdWorkflow.Pipeline.deployment_app_type", "CdWorkflow.Pipeline.deleted", "CdWorkflow.Pipeline.Environment"). + Column("cd_workflow_runner.*", "CdWorkflow.id", "CdWorkflow.pipeline_id", "CdWorkflow.Pipeline.id", "CdWorkflow.Pipeline.deployment_app_name", "CdWorkflow.Pipeline.deleted", "CdWorkflow.Pipeline.Environment"). Join("INNER JOIN cd_workflow wf on wf.id = cd_workflow_runner.cd_workflow_id"). Join("INNER JOIN pipeline p on p.id = wf.pipeline_id"). Join("INNER JOIN environment e on e.id = p.environment_id"). + Join("LEFT JOIN deployment_config dc on dc.app_id = p.app_id and dc.environment_id=p.environment_id"). Where("cd_workflow_runner.workflow_type=?", apiBean.CD_WORKFLOW_TYPE_DEPLOY). Where("cd_workflow_runner.status not in (?)", pg.In(excludedStatusList)). Where("cd_workflow_runner.cd_workflow_id in"+ @@ -764,7 +765,7 @@ func (impl *CdWorkflowRepositoryImpl) GetLatestTriggersOfHelmPipelinesStuckInNon " WHERE cd_workflow_runner.status != ?"+ " GROUP BY cd_workflow.pipeline_id"+ " ORDER BY cd_workflow.pipeline_id desc)", WorkflowInQueue). - Where("p.deployment_app_type = ?", util.PIPELINE_DEPLOYMENT_TYPE_HELM). + Where("(p.deployment_app_type=? or dc.deployment_app_type=?)", util.PIPELINE_DEPLOYMENT_TYPE_HELM, util.PIPELINE_DEPLOYMENT_TYPE_HELM). Where("cd_workflow_runner.started_on > NOW() - INTERVAL '? hours'", getPipelineDeployedWithinHours). Where("p.deleted=?", false). Order("cd_workflow_runner.id DESC"). diff --git a/internal/sql/repository/pipelineConfig/PipelineRepository.go b/internal/sql/repository/pipelineConfig/PipelineRepository.go index 6aaf65a0b9..9e8707058a 100644 --- a/internal/sql/repository/pipelineConfig/PipelineRepository.go +++ b/internal/sql/repository/pipelineConfig/PipelineRepository.go @@ -118,7 +118,6 @@ type PipelineRepository interface { FindIdsByProjectIdsAndEnvironmentIds(projectIds, environmentIds []int) ([]int, error) GetArgoPipelineByArgoAppName(argoAppName string) (Pipeline, error) - GetPartiallyDeletedPipelineByStatus(appId int, envId int) (Pipeline, error) FindActiveByAppIds(appIds []int) (pipelines []*Pipeline, err error) FindAppAndEnvironmentAndProjectByPipelineIds(pipelineIds []int) (pipelines []*Pipeline, err error) FilterDeploymentDeleteRequestedPipelineIds(cdPipelineIds []int) (map[int]bool, error) @@ -373,8 +372,9 @@ func (impl PipelineRepositoryImpl) FindActiveByEnvIdAndDeploymentType(environmen Model(&pipelines). Column("pipeline.*", "App", "Environment"). Join("inner join app a on pipeline.app_id = a.id"). + Join("LEFT JOIN deployment_config dc on dc.app_id = pipeline.app_id and dc.environment_id=pipeline.environment_id"). Where("pipeline.environment_id = ?", environmentId). - Where("pipeline.deployment_app_type = ?", deploymentAppType). + Where("(pipeline.deployment_app_type=? or dc.deployment_app_type=?)", deploymentAppType, deploymentAppType). Where("pipeline.deleted = ?", false) if len(exclusionListString) > 0 { @@ -560,7 +560,7 @@ func (impl PipelineRepositoryImpl) SetDeploymentAppCreatedInPipeline(deploymentA } // UpdateCdPipelineDeploymentAppInFilter takes in deployment app type and list of cd pipeline ids and -// updates the deployment_app_type and sets deployment_app_created to false in the table for given ids. +// updates the deploymentAppType and sets deployment_app_created to false in the table for given ids. func (impl PipelineRepositoryImpl) UpdateCdPipelineDeploymentAppInFilter(deploymentAppType string, cdPipelineIdIncludes []int, userId int32, deploymentAppCreated bool, isDeleted bool) error { query := "update pipeline set deployment_app_created = ?, deployment_app_type = ?, " + @@ -601,10 +601,11 @@ func (impl PipelineRepositoryImpl) GetAppAndEnvDetailsForDeploymentAppTypePipeli Column("pipeline.id", "App.app_name", "Environment.cluster_id", "Environment.namespace", "Environment.environment_name"). Join("inner join app a on pipeline.app_id = a.id"). Join("inner join environment e on pipeline.environment_id = e.id"). + Join("LEFT JOIN deployment_config dc on dc.app_id = pipeline.app_id and dc.environment_id=pipeline.environment_id"). Where("e.cluster_id in (?)", pg.In(clusterIds)). Where("a.active = ?", true). Where("pipeline.deleted = ?", false). - Where("pipeline.deployment_app_type = ?", deploymentAppType). + Where("(pipeline.deployment_app_type=? or dc.deployment_app_type=?)", deploymentAppType, deploymentAppType). Select() return pipelines, err } @@ -613,16 +614,17 @@ func (impl PipelineRepositoryImpl) GetArgoPipelinesHavingTriggersStuckInLastPoss var pipelines []*Pipeline queryString := `select p.* from pipeline p inner join cd_workflow cw on cw.pipeline_id = p.id inner join cd_workflow_runner cwr on cwr.cd_workflow_id=cw.id + left join deployment_config dc on dc.app_id = p.app_id and dc.environment_id=p.environment_id where cwr.id in (select cd_workflow_runner_id from pipeline_status_timeline where id in (select DISTINCT ON (cd_workflow_runner_id) max(id) as id from pipeline_status_timeline group by cd_workflow_runner_id, id order by cd_workflow_runner_id,id desc) and status in (?) and status_time < NOW() - INTERVAL '? seconds') - and cwr.started_on > NOW() - INTERVAL '? minutes' and p.deployment_app_type=? and p.deleted=?;` + and cwr.started_on > NOW() - INTERVAL '? minutes' and (p.deployment_app_type=? or dc.deployment_app_type=?) and p.deleted=?;` _, err := impl.dbConnection.Query(&pipelines, queryString, pg.In([]timelineStatus.TimelineStatus{timelineStatus.TIMELINE_STATUS_KUBECTL_APPLY_SYNCED, timelineStatus.TIMELINE_STATUS_FETCH_TIMED_OUT, timelineStatus.TIMELINE_STATUS_UNABLE_TO_FETCH_STATUS}), - pendingSinceSeconds, timeForDegradation, util.PIPELINE_DEPLOYMENT_TYPE_ACD, false) + pendingSinceSeconds, timeForDegradation, util.PIPELINE_DEPLOYMENT_TYPE_ACD, util.PIPELINE_DEPLOYMENT_TYPE_ACD, false) if err != nil { impl.logger.Errorw("error in GetArgoPipelinesHavingTriggersStuckInLastPossibleNonTerminalTimelines", "err", err) return nil, err @@ -634,14 +636,15 @@ func (impl PipelineRepositoryImpl) GetArgoPipelinesHavingLatestTriggerStuckInNon var pipelines []*Pipeline queryString := `select p.id from pipeline p inner join cd_workflow cw on cw.pipeline_id = p.id inner join cd_workflow_runner cwr on cwr.cd_workflow_id=cw.id + left join deployment_config dc on dc.app_id = p.app_id and dc.environment_id=p.environment_id where cwr.id in (select id from cd_workflow_runner where started_on < NOW() - INTERVAL '? minutes' and started_on > NOW() - INTERVAL '? hours' and status not in (?) and workflow_type=? and cd_workflow_id in (select DISTINCT ON (pipeline_id) max(id) as id from cd_workflow group by pipeline_id, id order by pipeline_id, id desc)) - and p.deployment_app_type=? and p.deleted=?;` + and (p.deployment_app_type=? or dc.deployment_app_type=?) and p.deleted=?;` _, err := impl.dbConnection.Query(&pipelines, queryString, getPipelineDeployedBeforeMinutes, getPipelineDeployedWithinHours, pg.In(append(WfrTerminalStatusList, WorkflowInitiated, WorkflowInQueue)), - bean.CD_WORKFLOW_TYPE_DEPLOY, util.PIPELINE_DEPLOYMENT_TYPE_ACD, false) + bean.CD_WORKFLOW_TYPE_DEPLOY, util.PIPELINE_DEPLOYMENT_TYPE_ACD, util.PIPELINE_DEPLOYMENT_TYPE_ACD, false) if err != nil { impl.logger.Errorw("error in GetArgoPipelinesHavingLatestTriggerStuckInNonTerminalStatuses", "err", err) return nil, err @@ -674,9 +677,10 @@ func (impl PipelineRepositoryImpl) FindIdsByProjectIdsAndEnvironmentIds(projectI func (impl PipelineRepositoryImpl) GetArgoPipelineByArgoAppName(argoAppName string) (Pipeline, error) { var pipeline Pipeline err := impl.dbConnection.Model(&pipeline). + Join("LEFT JOIN deployment_config dc on dc.app_id = pipeline.app_id and dc.environment_id=pipeline.environment_id"). Column("pipeline.*", "Environment"). Where("deployment_app_name = ?", argoAppName). - Where("deployment_app_type = ?", util.PIPELINE_DEPLOYMENT_TYPE_ACD). + Where("(pipeline.deployment_app_type=? or dc.deployment_app_type=?)", util.PIPELINE_DEPLOYMENT_TYPE_ACD, util.PIPELINE_DEPLOYMENT_TYPE_ACD). Where("deleted = ?", false). Select() if err != nil { @@ -686,21 +690,6 @@ func (impl PipelineRepositoryImpl) GetArgoPipelineByArgoAppName(argoAppName stri return pipeline, nil } -func (impl PipelineRepositoryImpl) GetPartiallyDeletedPipelineByStatus(appId int, envId int) (Pipeline, error) { - var pipeline Pipeline - err := impl.dbConnection.Model(&pipeline). - Column("pipeline.*", "App.app_name", "Environment.namespace"). - Where("app_id = ?", appId). - Where("environment_id = ?", envId). - Where("deployment_app_delete_request = ?", true). - Where("deployment_app_type = ?", util.PIPELINE_DEPLOYMENT_TYPE_ACD). - Where("deleted = ?", false).Select() - if err != nil && err != pg.ErrNoRows { - impl.logger.Errorw("error in updating argo pipeline delete status") - } - return pipeline, err -} - func (impl PipelineRepositoryImpl) FindActiveByAppIds(appIds []int) (pipelines []*Pipeline, err error) { err = impl.dbConnection.Model(&pipelines). Column("pipeline.*", "App", "Environment"). diff --git a/internal/sql/repository/pipelineConfig/PipelineStatusSyncDetailRepository.go b/internal/sql/repository/pipelineConfig/PipelineStatusSyncDetailRepository.go index 33cea5f8ca..b427c466b2 100644 --- a/internal/sql/repository/pipelineConfig/PipelineStatusSyncDetailRepository.go +++ b/internal/sql/repository/pipelineConfig/PipelineStatusSyncDetailRepository.go @@ -98,8 +98,10 @@ func (impl *PipelineStatusSyncDetailRepositoryImpl) GetOfLatestCdWfrByCdPipeline var model PipelineStatusSyncDetail query := `select * from pipeline_status_timeline_sync_detail where cd_workflow_runner_id = (select cwr.id from cd_workflow_runner cwr inner join cd_workflow cw on cw.id=cwr.cd_workflow_id - inner join pipeline p on p.id=cw.pipeline_id where p.id=? and p.deleted=? and p.deployment_app_type=? order by cwr.id desc limit ?);` - _, err := impl.dbConnection.Query(&model, query, pipelineId, false, util.PIPELINE_DEPLOYMENT_TYPE_ACD, 1) + inner join pipeline p on p.id=cw.pipeline_id + left join deployment_config dc on dc.app_id = p.app_id and dc.environment_id=p.environment_id + where p.id=? and p.deleted=? and (p.deployment_app_type=? or dc.deployment_app_type=?) order by cwr.id desc limit ?);` + _, err := impl.dbConnection.Query(&model, query, pipelineId, false, util.PIPELINE_DEPLOYMENT_TYPE_ACD, util.PIPELINE_DEPLOYMENT_TYPE_ACD, 1) if err != nil { impl.logger.Errorw("error in getting cd pipeline status sync detail of latest cdWfr by pipelineId", "err", err, "pipelineId", pipelineId) return nil, err @@ -113,9 +115,10 @@ func (impl *PipelineStatusSyncDetailRepositoryImpl) GetOfLatestInstalledAppVersi where installed_app_version_history_id = (select iavh.id from installed_app_version_history iavh inner join installed_app_versions iav on iavh.installed_app_version_id=iav.id inner join installed_apps ia on iav.installed_app_id=ia.id - where iav.id=? and iav.active=? and ia.deployment_app_type=? + left join deployment_config dc on dc.app_id = ia.app_id and dc.environment_id=ia.environment_id + where iav.id=? and iav.active=? and (ia.deployment_app_type=? or dc.deployment_app_type=?) order by iavh.id desc limit ?);` - _, err := impl.dbConnection.Query(&model, query, installedAppVersionId, true, util.PIPELINE_DEPLOYMENT_TYPE_ACD, 1) + _, err := impl.dbConnection.Query(&model, query, installedAppVersionId, true, util.PIPELINE_DEPLOYMENT_TYPE_ACD, util.PIPELINE_DEPLOYMENT_TYPE_ACD, 1) if err != nil { impl.logger.Errorw("error in getting cd pipeline status sync detail of latest cdWfr by pipelineId", "err", err, "installedAppVersionId", installedAppVersionId) return nil, err diff --git a/pkg/appStore/installedApp/repository/InstalledAppRepository.go b/pkg/appStore/installedApp/repository/InstalledAppRepository.go index 329d93ccde..707386f8a5 100644 --- a/pkg/appStore/installedApp/repository/InstalledAppRepository.go +++ b/pkg/appStore/installedApp/repository/InstalledAppRepository.go @@ -692,8 +692,9 @@ func (impl InstalledAppRepositoryImpl) GetAppAndEnvDetailsForDeploymentAppTypeIn err := impl.dbConnection. Model(&installedApps). Column("installed_apps.id", "App.app_name", "App.display_name", "Environment.cluster_id", "Environment.namespace"). + Join("LEFT JOIN deployment_config dc on dc.app_id = installed_apps.app_id and dc.environment_id=installed_apps.environment_id"). Where("environment.cluster_id in (?)", pg.In(clusterIds)). - Where("installed_apps.deployment_app_type = ?", deploymentAppType). + Where("(installed_apps.deployment_app_type = ? or dc.deployment_app_type = ?)", deploymentAppType, deploymentAppType). Where("app.active = ?", true). Where("installed_apps.active = ?", true). Select() @@ -743,8 +744,10 @@ func (impl InstalledAppRepositoryImpl) GetInstalledAppByGitHash(gitHash string) func (impl InstalledAppRepositoryImpl) GetInstalledAppByAppIdAndDeploymentType(appId int, deploymentAppType string) (InstalledApps, error) { var installedApps InstalledApps - queryString := `select * from installed_apps where active=? and app_id=? and deployment_app_type=?;` - _, err := impl.dbConnection.Query(&installedApps, queryString, true, appId, deploymentAppType) + queryString := `select * from installed_apps + left join deployment_config dc on dc.app_id = installed_apps.app_id and dc.environment_id=installed_apps.environment_id + where active=? and app_id=? and (installed_apps.deployment_app_type=? or dc.deployment_app_type=?);` + _, err := impl.dbConnection.Query(&installedApps, queryString, true, appId, deploymentAppType, deploymentAppType) if err != nil { impl.Logger.Errorw("error in fetching InstalledApp", "err", err) return installedApps, err @@ -767,8 +770,9 @@ func (impl InstalledAppRepositoryImpl) GetInstalledAppByAppName(appName string) func (impl InstalledAppRepositoryImpl) GetInstalledAppByInstalledAppVersionId(installedAppVersionId int) (InstalledApps, error) { var installedApps InstalledApps queryString := `select ia.* from installed_apps ia inner join installed_app_versions iav on ia.id=iav.installed_app_id - where iav.active=? and iav.id=? and ia.deployment_app_type=?;` - _, err := impl.dbConnection.Query(&installedApps, queryString, true, installedAppVersionId, util2.PIPELINE_DEPLOYMENT_TYPE_ACD) + left join deployment_config dc on dc.app_id = ia.app_id and dc.environment_id=ia.environment_id + where iav.active=? and iav.id=? and (ia.deployment_app_type=? or dc.deployment_app_type=?);` + _, err := impl.dbConnection.Query(&installedApps, queryString, true, installedAppVersionId, util2.PIPELINE_DEPLOYMENT_TYPE_ACD, util2.PIPELINE_DEPLOYMENT_TYPE_ACD) if err != nil { impl.Logger.Errorw("error in fetching InstalledApp", "err", err) return installedApps, err @@ -794,7 +798,8 @@ func (impl InstalledAppRepositoryImpl) GetInstalledAppByGitRepoUrl(repoName, rep model := &InstalledApps{} err := impl.dbConnection.Model(model). Column("installed_apps.*", "App"). - Where("installed_apps.git_ops_repo_name = ? OR installed_apps.git_ops_repo_url = ?", repoName, repoUrl). + Join("LEFT JOIN deployment_config dc on dc.app_id = installed_apps.app_id and dc.environment_id=installed_apps.environment_id"). + Where("installed_apps.git_ops_repo_name = ? OR (installed_apps.git_ops_repo_url = ? or dc.repo_url = ?)", repoName, repoUrl, repoUrl). Where("installed_apps.active = true"). Where("app.active = true"). Limit(1). @@ -807,13 +812,14 @@ func (impl InstalledAppRepositoryImpl) GetArgoPipelinesHavingLatestTriggerStuckI queryString := `select iav.* from installed_app_versions iav inner join installed_apps ia on iav.installed_app_id=ia.id inner join installed_app_version_history iavh on iavh.installed_app_version_id=iav.id - where iavh.id in (select DISTINCT ON (installed_app_version_id) max(id) as id from installed_app_version_history + left join deployment_config dc on dc.app_id = ia.app_id and dc.environment_id=ia.environment_id + where iavh.id in (select DISTINCT ON (installed_app_version_id) max(id) as id from installed_app_version_history where updated_on < NOW() - INTERVAL '? minutes' and updated_on > NOW() - INTERVAL '? hours' and status not in (?) - group by installed_app_version_id, id order by installed_app_version_id, id desc ) and ia.deployment_app_type=? and iav.active=?;` + group by installed_app_version_id, id order by installed_app_version_id, id desc ) and (ia.deployment_app_type=? or dc.deployment_app_type=?) and iav.active=?;` _, err := impl.dbConnection.Query(&installedAppVersions, queryString, getPipelineDeployedBeforeMinutes, getPipelineDeployedWithinHours, pg.In([]string{pipelineConfig.WorkflowAborted, pipelineConfig.WorkflowFailed, pipelineConfig.WorkflowSucceeded, string(health.HealthStatusHealthy), string(health.HealthStatusDegraded)}), - util2.PIPELINE_DEPLOYMENT_TYPE_ACD, true) + util2.PIPELINE_DEPLOYMENT_TYPE_ACD, util2.PIPELINE_DEPLOYMENT_TYPE_ACD, true) if err != nil { impl.Logger.Errorw("error in GetArgoPipelinesHavingLatestTriggerStuckInNonTerminalStatusesForAppStore", "err", err) return nil, err @@ -825,15 +831,16 @@ func (impl InstalledAppRepositoryImpl) GetArgoPipelinesHavingTriggersStuckInLast var installedAppVersions []*InstalledAppVersions queryString := `select iav.* from installed_app_versions iav inner join installed_apps ia on iav.installed_app_id=ia.id inner join installed_app_version_history iavh on iavh.installed_app_version_id=iav.id + left join deployment_config dc on dc.app_id = ia.app_id and dc.environment_id=ia.environment_id where iavh.id in (select DISTINCT ON (installed_app_version_history_id) max(id) as id from pipeline_status_timeline where status in (?) and status_time < NOW() - INTERVAL '? seconds' group by installed_app_version_history_id, id order by installed_app_version_history_id, id desc) - and iavh.updated_on > NOW() - INTERVAL '? minutes' and ia.deployment_app_type=? and iav.active=?;` + and iavh.updated_on > NOW() - INTERVAL '? minutes' and (ia.deployment_app_type=? or dc.deployment_app_type=?) and iav.active=?;` _, err := impl.dbConnection.Query(&installedAppVersions, queryString, pg.In([]timelineStatus.TimelineStatus{timelineStatus.TIMELINE_STATUS_KUBECTL_APPLY_SYNCED, timelineStatus.TIMELINE_STATUS_FETCH_TIMED_OUT, timelineStatus.TIMELINE_STATUS_UNABLE_TO_FETCH_STATUS}), - pendingSinceSeconds, timeForDegradation, util2.PIPELINE_DEPLOYMENT_TYPE_ACD, true) + pendingSinceSeconds, timeForDegradation, util2.PIPELINE_DEPLOYMENT_TYPE_ACD, util2.PIPELINE_DEPLOYMENT_TYPE_ACD, true) if err != nil { impl.Logger.Errorw("error in GetArgoPipelinesHavingTriggersStuckInLastPossibleNonTerminalTimelinesForAppStore", "err", err) return nil, err @@ -863,8 +870,9 @@ func (impl InstalledAppRepositoryImpl) GetActiveInstalledAppByEnvIdAndDeployment Model(&installedApps). Column("installed_apps.*", "App", "Environment"). Join("inner join app a on installed_apps.app_id = a.id"). + Join("LEFT JOIN deployment_config dc on dc.app_id = installed_apps.app_id and dc.environment_id=installed_apps.environment_id"). Where("installed_apps.environment_id = ?", envId). - Where("installed_apps.deployment_app_type = ?", deploymentType). + Where("installed_apps.deployment_app_type = ? or dc.deployment_app_type = ?", deploymentType, deploymentType). Where("installed_apps.active = ?", true) if len(excludeAppIds) > 0 { @@ -883,7 +891,7 @@ func (impl InstalledAppRepositoryImpl) GetActiveInstalledAppByEnvIdAndDeployment } // UpdateDeploymentAppTypeInInstalledApp takes in deploymentAppType and list of installedAppIds and -// updates the deployment_app_type in the table for given ids. +// updates the deploymentAppType in the table for given ids. func (impl InstalledAppRepositoryImpl) UpdateDeploymentAppTypeInInstalledApp(deploymentAppType string, installedAppIdIncludes []int, userId int32, deployStatus int) error { query := "update installed_apps set deployment_app_type = ?,updated_by = ?, updated_on = ?, status = ? where id in (?);" var installedApp *InstalledApps diff --git a/pkg/chartRepo/repository/ChartsRepository.go b/pkg/chartRepo/repository/ChartsRepository.go index 672877b891..f6b8e6fdec 100644 --- a/pkg/chartRepo/repository/ChartsRepository.go +++ b/pkg/chartRepo/repository/ChartsRepository.go @@ -244,8 +244,9 @@ func (repositoryImpl ChartRepositoryImpl) FindChartByGitRepoUrl(gitRepoUrl strin var chart Chart err := repositoryImpl.dbConnection.Model(&chart). Join("INNER JOIN app ON app.id=app_id"). + Join("LEFT JOIN deployment_config dc on dc.app_id = chart.app_id and dc.environment_id is null"). Where("app.active = ?", true). - Where("chart.git_repo_url = ?", gitRepoUrl). + Where("(chart.git_repo_url = ? or dc.repo_url = ?)", gitRepoUrl, gitRepoUrl). Where("chart.active = ?", true). Limit(1). Select() diff --git a/scripts/sql/265_migration_deployed_app.down.sql b/scripts/sql/265_migration_deployed_app.down.sql new file mode 100644 index 0000000000..672cbfc361 --- /dev/null +++ b/scripts/sql/265_migration_deployed_app.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS deployment_app_migration_history; \ No newline at end of file diff --git a/scripts/sql/265_migration_deployed_app.up.sql b/scripts/sql/265_migration_deployed_app.up.sql new file mode 100644 index 0000000000..90b1ebd42e --- /dev/null +++ b/scripts/sql/265_migration_deployed_app.up.sql @@ -0,0 +1,19 @@ +CREATE SEQUENCE IF NOT EXISTS id_seq_deployment_app_migration_history; +CREATE TABLE "public"."deployment_app_migration_history" ( + "id" integer NOT NULL default nextval('id_seq_deployment_app_migration_history'::regclass), + "app_id" integer, + "env_id" integer, + "is_migration_active" bool NOT NULL, + "migrate_to" text , + "migrate_from" text, + "current_status" integer, + "error_status" integer, + "error_encountered" text, + "created_on" timestamptz NOT NULL, + "created_by" int4 NOT NULL, + "updated_on" timestamptz, + "updated_by" int4, + PRIMARY KEY ("id"), + CONSTRAINT "deployment_app_migration_history_app_id_fk" FOREIGN KEY ("app_id") REFERENCES "public"."app" ("id"), + CONSTRAINT "deployment_app_migration_history_env_id_fk" FOREIGN KEY ("env_id") REFERENCES "public"."environment" ("id") +); \ No newline at end of file From 6c74b690fbf5b7ed16ade498a86b06d1533ad3db Mon Sep 17 00:00:00 2001 From: ayushmaheshwari Date: Tue, 2 Jul 2024 23:47:42 +0530 Subject: [PATCH 10/45] dev testing fixes --- .../repository/deploymentConfig/repository.go | 17 +- .../pipelineConfig/PipelineRepository.go | 11 +- .../repository/InstalledAppRepository.go | 13 -- pkg/deployment/common/bean/bean.go | 25 +++ .../common/deploymentConfigService.go | 203 +++++++++++------- pkg/deployment/common/helper.go | 15 +- scripts/sql/260_deployment_config.down.sql | 3 +- wire_gen.go | 5 - 8 files changed, 184 insertions(+), 108 deletions(-) diff --git a/internal/sql/repository/deploymentConfig/repository.go b/internal/sql/repository/deploymentConfig/repository.go index 240e1ce711..3276c95e3e 100644 --- a/internal/sql/repository/deploymentConfig/repository.go +++ b/internal/sql/repository/deploymentConfig/repository.go @@ -19,7 +19,7 @@ type DeploymentConfig struct { CredentialType string `sql:"credential_type"` CredentialIdInt int `sql:"credential_id_int"` CredentialIdString string `sql:"credential_id_string"` - Active bool `sql:"active"` + Active bool `sql:"active,notnull"` sql.AuditLog } @@ -31,7 +31,7 @@ type Repository interface { GetByAppIdAndEnvId(appId, envId int) (*DeploymentConfig, error) GetAppLevelConfig(appId int) (*DeploymentConfig, error) GetAppLevelConfigByAppIds(appIds []int) ([]*DeploymentConfig, error) - GetAppAndEnvLevelConfigsInBulk(envIdToAppIdMap map[int]int) ([]*DeploymentConfig, error) + GetAppAndEnvLevelConfigsInBulk(appIdToEnvIdsMap map[int][]int) ([]*DeploymentConfig, error) } type RepositoryImpl struct { @@ -55,9 +55,9 @@ func (impl RepositoryImpl) Save(tx *pg.Tx, config *DeploymentConfig) (*Deploymen func (impl RepositoryImpl) SaveAll(tx *pg.Tx, configs []*DeploymentConfig) ([]*DeploymentConfig, error) { var err error if tx != nil { - err = tx.Insert(configs) + err = tx.Insert(&configs) } else { - err = impl.dbConnection.Insert(configs) + err = impl.dbConnection.Insert(&configs) } return configs, err } @@ -107,12 +107,15 @@ func (impl RepositoryImpl) GetAppLevelConfigByAppIds(appIds []int) ([]*Deploymen return result, err } -func (impl RepositoryImpl) GetAppAndEnvLevelConfigsInBulk(envIdToAppIdMap map[int]int) ([]*DeploymentConfig, error) { +func (impl RepositoryImpl) GetAppAndEnvLevelConfigsInBulk(appIdToEnvIdsMap map[int][]int) ([]*DeploymentConfig, error) { var result []*DeploymentConfig err := impl.dbConnection.Model(&result). WhereGroup(func(query *orm.Query) (*orm.Query, error) { - for envId, appId := range envIdToAppIdMap { - query = query.Where("environment_id = ? and app_id = ? ", envId, appId) + for appId, envIds := range appIdToEnvIdsMap { + if len(envIds) == 0 { + continue + } + query = query.WhereOr(" app_id = ? and environment_id in (?) and active=true ", appId, pg.In(envIds)) } return query, nil }). diff --git a/internal/sql/repository/pipelineConfig/PipelineRepository.go b/internal/sql/repository/pipelineConfig/PipelineRepository.go index 6aaf65a0b9..492e108c62 100644 --- a/internal/sql/repository/pipelineConfig/PipelineRepository.go +++ b/internal/sql/repository/pipelineConfig/PipelineRepository.go @@ -127,7 +127,7 @@ type PipelineRepository interface { // FindWithEnvironmentByCiIds Possibility of duplicate environment names when filtered by unique pipeline ids FindWithEnvironmentByCiIds(ctx context.Context, cIPipelineIds []int) ([]*Pipeline, error) FindDeploymentAppTypeByAppIdAndEnvId(appId, envId int) (string, error) - FindByEnvToAppIdMapping(envToAppIdMap map[int]int) ([]*Pipeline, error) + FindByAppIdToEnvIdsMapping(appIdToEnvIds map[int][]int) ([]*Pipeline, error) } type CiArtifactDTO struct { @@ -796,12 +796,15 @@ func (impl PipelineRepositoryImpl) FindDeploymentAppTypeByAppIdAndEnvId(appId, e return deploymentAppType, err } -func (impl PipelineRepositoryImpl) FindByEnvToAppIdMapping(envToAppIdMap map[int]int) ([]*Pipeline, error) { +func (impl PipelineRepositoryImpl) FindByAppIdToEnvIdsMapping(appIdToEnvIds map[int][]int) ([]*Pipeline, error) { var pipelines []*Pipeline err := impl.dbConnection.Model(&pipelines). WhereGroup(func(query *orm.Query) (*orm.Query, error) { - for envId, appId := range envToAppIdMap { - query = query.Where("environment_id = ? and app_id = ? ", envId, appId) + for appId, envIds := range appIdToEnvIds { + if len(envIds) == 0 { + continue + } + query = query.WhereOr("app_id = ? and environment_id in (?) and deleted=false", appId, pg.In(envIds)) } return query, nil }). diff --git a/pkg/appStore/installedApp/repository/InstalledAppRepository.go b/pkg/appStore/installedApp/repository/InstalledAppRepository.go index 329d93ccde..5e641abae8 100644 --- a/pkg/appStore/installedApp/repository/InstalledAppRepository.go +++ b/pkg/appStore/installedApp/repository/InstalledAppRepository.go @@ -159,7 +159,6 @@ type InstalledAppRepository interface { FindInstalledAppByIds(ids []int) ([]*InstalledApps, error) // FindInstalledAppsByAppId returns multiple installed apps for an appId, this only happens for external-apps with same name installed in diff namespaces FindInstalledAppsByAppId(appId int) ([]*InstalledApps, error) - FindInstalledAppByAppIds(appIds []int) ([]*InstalledApps, error) } type InstalledAppRepositoryImpl struct { @@ -922,15 +921,3 @@ func (impl InstalledAppRepositoryImpl) FindInstalledAppsByAppId(appId int) ([]*I } return installedApps, err } - -func (impl InstalledAppRepositoryImpl) FindInstalledAppByAppIds(appIds []int) ([]*InstalledApps, error) { - var installedApps []*InstalledApps - if len(appIds) == 0 { - return nil, nil - } - err := impl.dbConnection.Model(&installedApps). - Column("installed_apps.*"). - Where("installed_apps.app_id in (?)", pg.In(appIds)). - Select() - return installedApps, err -} diff --git a/pkg/deployment/common/bean/bean.go b/pkg/deployment/common/bean/bean.go index 71b48a8b1d..df4a645e70 100644 --- a/pkg/deployment/common/bean/bean.go +++ b/pkg/deployment/common/bean/bean.go @@ -1,5 +1,11 @@ package bean +import ( + "fmt" + "strconv" + "strings" +) + type DeploymentConfig struct { Id int AppId int @@ -15,11 +21,30 @@ type DeploymentConfig struct { Active bool } +type UniqueDeploymentConfigIdentifier string + type DeploymentConfigSelector struct { AppId int EnvironmentId int } +func (u UniqueDeploymentConfigIdentifier) String() string { + return string(u) +} + +func GetConfigUniqueIdentifier(appId, envId int) UniqueDeploymentConfigIdentifier { + return UniqueDeploymentConfigIdentifier(fmt.Sprintf("%d-%d", appId, envId)) + +} + +func (u *UniqueDeploymentConfigIdentifier) GetAppAndEnvId() (appId, envId int) { + splitArr := strings.Split(u.String(), "-") + appIdStr, envIdStr := splitArr[0], splitArr[1] + appId, _ = strconv.Atoi(appIdStr) + envId, _ = strconv.Atoi(envIdStr) + return appId, envId +} + type DeploymentConfigType string const ( diff --git a/pkg/deployment/common/deploymentConfigService.go b/pkg/deployment/common/deploymentConfigService.go index 5167c60bbe..f823571a17 100644 --- a/pkg/deployment/common/deploymentConfigService.go +++ b/pkg/deployment/common/deploymentConfigService.go @@ -256,12 +256,8 @@ func (impl *DeploymentConfigServiceImpl) migrateHelmAppDataToDeploymentConfig(ap func (impl *DeploymentConfigServiceImpl) GetDeploymentConfigInBulk(configSelector []*bean.DeploymentConfigSelector) ([]*bean.DeploymentConfig, error) { appIds := make([]*int, len(configSelector)) - appIdToEnvIdMapping := make(map[int]int) for _, s := range configSelector { appIds = append(appIds, &s.AppId) - if s.EnvironmentId > 0 { - appIdToEnvIdMapping[s.AppId] = s.EnvironmentId - } } apps, err := impl.appRepository.FindByIds(appIds) @@ -272,29 +268,41 @@ func (impl *DeploymentConfigServiceImpl) GetDeploymentConfigInBulk(configSelecto devtronAppIds := make([]int, len(appIds)) helmAppIds := make([]int, len(appIds)) - devtronAppsEnvToAppIdMapping := make(map[int]int) - helmAppsEnvToAppIdMapping := make(map[int]int) + devtronAppIdMap := make(map[int]bool) + helmAppIdMap := make(map[int]bool) for _, a := range apps { switch a.AppType { case helper.CustomApp: devtronAppIds = append(devtronAppIds, a.Id) - devtronAppsEnvToAppIdMapping[appIdToEnvIdMapping[a.Id]] = a.Id + devtronAppIdMap[a.Id] = true case helper.ChartStoreApp: helmAppIds = append(helmAppIds, a.Id) - helmAppsEnvToAppIdMapping[appIdToEnvIdMapping[a.Id]] = a.Id + helmAppIdMap[a.Id] = true } } - devtronAppConfigs, err := impl.GetDevtronAppConfigInBulk(devtronAppIds, devtronAppsEnvToAppIdMapping) + devtronAppSelectors := make([]*bean.DeploymentConfigSelector, len(appIds)) + helmAppSelectors := make([]*bean.DeploymentConfigSelector, len(appIds)) + + for _, c := range configSelector { + if _, ok := devtronAppIdMap[c.AppId]; ok { + devtronAppSelectors = append(devtronAppSelectors, c) + } + if _, ok := helmAppIdMap[c.AppId]; ok { + helmAppSelectors = append(helmAppSelectors, c) + } + } + + devtronAppConfigs, err := impl.GetDevtronAppConfigInBulk(devtronAppIds, devtronAppSelectors) if err != nil { - impl.logger.Errorw("error in getting deployment config for devtron apps", "devtronAppsEnvToAppIdMapping", devtronAppsEnvToAppIdMapping, "err", err) + impl.logger.Errorw("error in getting deployment config for devtron apps", "devtronAppSelectors", devtronAppSelectors, "err", err) return nil, err } - helmAppConfigs, err := impl.GetHelmAppConfigInBulk(helmAppIds, helmAppsEnvToAppIdMapping) + helmAppConfigs, err := impl.GetHelmAppConfigInBulk(helmAppIds, helmAppSelectors) if err != nil { - impl.logger.Errorw("error in getting deployment config for helm apps", "devtronAppsEnvToAppIdMapping", devtronAppsEnvToAppIdMapping, "err", err) + impl.logger.Errorw("error in getting deployment config for helm apps", "helmAppSelectors", helmAppSelectors, "err", err) return nil, err } @@ -305,7 +313,7 @@ func (impl *DeploymentConfigServiceImpl) GetDeploymentConfigInBulk(configSelecto return allConfigs, nil } -func (impl *DeploymentConfigServiceImpl) GetDevtronAppConfigInBulk(appIds []int, envIdToAppIdMapping map[int]int) ([]*bean.DeploymentConfig, error) { +func (impl *DeploymentConfigServiceImpl) GetDevtronAppConfigInBulk(appIds []int, configSelector []*bean.DeploymentConfigSelector) ([]*bean.DeploymentConfig, error) { AppLevelDeploymentConfigs, err := impl.deploymentConfigRepository.GetAppLevelConfigByAppIds(appIds) if err != nil { @@ -340,36 +348,44 @@ func (impl *DeploymentConfigServiceImpl) GetDevtronAppConfigInBulk(appIds []int, appIdToAppLevelConfigMapping[appLevelConfig.AppId] = appLevelConfig } - AllEnvLevelConfigs := make([]*deploymentConfig.DeploymentConfig, 0, len(envIdToAppIdMapping)) + AllEnvLevelConfigs := make([]*deploymentConfig.DeploymentConfig, 0, len(configSelector)) + + if len(configSelector) > 0 { - if len(envIdToAppIdMapping) > 0 { - envLevelConfig, err := impl.deploymentConfigRepository.GetAppAndEnvLevelConfigsInBulk(envIdToAppIdMapping) + appIdToEnvIdsMap := GetAppIdToEnvIsMappingFromConfigSelectors(configSelector) + + envLevelConfig, err := impl.deploymentConfigRepository.GetAppAndEnvLevelConfigsInBulk(appIdToEnvIdsMap) if err != nil { - impl.logger.Errorw("error in getting and and env level config in bulk", "envIdToAppIdMapping", envIdToAppIdMapping, "err", err) + impl.logger.Errorw("error in getting and and env level config in bulk", "appIdToEnvIdsMap", appIdToEnvIdsMap, "err", err) return nil, err } AllEnvLevelConfigs = append(AllEnvLevelConfigs, envLevelConfig...) - if len(envLevelConfig) < len(envIdToAppIdMapping) { - notFoundEnvIdToAppIdsMap := make(map[int]int, len(envIdToAppIdMapping)) - presentEnvIds := make(map[int]bool) + if len(envLevelConfig) < len(configSelector) { + + notFoundDeploymentConfigMap := make(map[bean.UniqueDeploymentConfigIdentifier]bool) + presentDeploymentConfigMap := make(map[bean.UniqueDeploymentConfigIdentifier]bool) + for _, c := range envLevelConfig { - presentEnvIds[c.EnvironmentId] = true + presentDeploymentConfigMap[bean.GetConfigUniqueIdentifier(c.AppId, c.EnvironmentId)] = true } - for envId, appId := range envIdToAppIdMapping { - if _, ok := presentEnvIds[envId]; !ok { - notFoundEnvIdToAppIdsMap[envId] = appId + + for _, c := range configSelector { + key := bean.GetConfigUniqueIdentifier(c.AppId, c.EnvironmentId) + if _, ok := presentDeploymentConfigMap[key]; !ok { + notFoundDeploymentConfigMap[key] = true } } - migratedEnvLevelDeploymentConfigs, err := impl.migrateEnvLevelDataTODeploymentConfigInBulk(notFoundEnvIdToAppIdsMap, envIdToAppIdMapping, appIdToAppLevelConfigMapping) + + migratedEnvLevelDeploymentConfigs, err := impl.migrateEnvLevelDataTODeploymentConfigInBulk(notFoundDeploymentConfigMap, appIdToAppLevelConfigMapping) if err != nil { - impl.logger.Errorw("error in migrating env level configs", "envIdToAppIdMapping", envIdToAppIdMapping, "err", err) + impl.logger.Errorw("error in migrating env level configs", "notFoundDeploymentConfigMap", notFoundDeploymentConfigMap, "err", err) return nil, err } AllEnvLevelConfigs = append(AllEnvLevelConfigs, migratedEnvLevelDeploymentConfigs...) } } - allConfigs := make([]*bean.DeploymentConfig, 0, len(appIds)+len(envIdToAppIdMapping)) + allConfigs := make([]*bean.DeploymentConfig, 0) for _, c := range AllAppLevelConfigs { allConfigs = append(allConfigs, ConvertDeploymentConfigDbObjToDTO(c)) } @@ -381,19 +397,34 @@ func (impl *DeploymentConfigServiceImpl) GetDevtronAppConfigInBulk(appIds []int, } func (impl *DeploymentConfigServiceImpl) migrateAppLevelDataTODeploymentConfigInBulk(appIds []int) ([]*deploymentConfig.DeploymentConfig, error) { + charts, err := impl.chartRepository.FindLatestChartByAppIds(appIds) if err != nil { impl.logger.Errorw("error in fetching latest chart by appIds", "appIds", appIds, "err", err) return nil, err } + + allRepoUrls := make([]string, 0) + for _, c := range charts { + allRepoUrls = append(allRepoUrls, c.GitRepoUrl) + } + + repoUrlToConfigMapping, err := impl.gitOpsConfigReadService.GetGitOpsProviderMapByRepoURL(allRepoUrls) + if err != nil { + impl.logger.Errorw("error in fetching repoUrl to config mapping", "err", err) + return nil, err + } + configDBObjects := make([]*deploymentConfig.DeploymentConfig, 0, len(appIds)) for _, c := range charts { dbObj := &deploymentConfig.DeploymentConfig{ - ConfigType: GetDeploymentConfigType(c.IsCustomGitRepository), - AppId: c.AppId, - Active: true, - RepoUrl: c.GitRepoUrl, - ChartLocation: c.ChartLocation, + ConfigType: GetDeploymentConfigType(c.IsCustomGitRepository), + AppId: c.AppId, + Active: true, + RepoUrl: c.GitRepoUrl, + ChartLocation: c.ChartLocation, + CredentialType: bean.GitOps.String(), + CredentialIdInt: repoUrlToConfigMapping[c.GitRepoUrl].Id, } dbObj.AuditLog.CreateAuditLog(bean3.SYSTEM_USER_ID) configDBObjects = append(configDBObjects, dbObj) @@ -403,41 +434,48 @@ func (impl *DeploymentConfigServiceImpl) migrateAppLevelDataTODeploymentConfigIn impl.logger.Errorw("error in saving deployment config in DB", "appIds", appIds, "err", err) return nil, err } + return configDBObjects, nil } -func (impl *DeploymentConfigServiceImpl) migrateEnvLevelDataTODeploymentConfigInBulk(notFoundEnvToAppIdMapping map[int]int, envToAppIdMapping map[int]int, appIdToAppLevelConfigMapping map[int]*deploymentConfig.DeploymentConfig) ([]*deploymentConfig.DeploymentConfig, error) { +func (impl *DeploymentConfigServiceImpl) migrateEnvLevelDataTODeploymentConfigInBulk(notFoundDeploymentConfigMap map[bean.UniqueDeploymentConfigIdentifier]bool, appIdToAppLevelConfigMapping map[int]*deploymentConfig.DeploymentConfig) ([]*deploymentConfig.DeploymentConfig, error) { - pipelines, err := impl.pipelineRepository.FindByEnvToAppIdMapping(notFoundEnvToAppIdMapping) - if err != nil { - impl.logger.Errorw("error in fetching pipeline by envToAppId mapping", "envToAppIdMapping", envToAppIdMapping, "err", err) - return nil, err - } - - deploymentAppTypeMap := make(map[int]map[int]string) - for _, p := range pipelines { - deploymentAppTypeMap[p.AppId][p.EnvironmentId] = p.DeploymentAppType - } - - allRepoUrls := make([]string, 0) - for _, c := range appIdToAppLevelConfigMapping { - allRepoUrls = append(allRepoUrls, c.RepoUrl) + notFoundAppIdEnvIdsMap := make(map[int][]int) + for uniqueKey, _ := range notFoundDeploymentConfigMap { + appId, envId := uniqueKey.GetAppAndEnvId() + if _, ok := notFoundAppIdEnvIdsMap[appId]; !ok { + notFoundAppIdEnvIdsMap[appId] = make([]int, 0) + } + notFoundAppIdEnvIdsMap[appId] = append(notFoundAppIdEnvIdsMap[appId], envId) } - repoUrlToConfigMapping, err := impl.gitOpsConfigReadService.GetGitOpsProviderMapByRepoURL(allRepoUrls) + pipelines, err := impl.pipelineRepository.FindByAppIdToEnvIdsMapping(notFoundAppIdEnvIdsMap) if err != nil { - impl.logger.Errorw("error in fetching repoUrl to config mapping", "err", err) + impl.logger.Errorw("error in fetching pipeline by envToAppId mapping", "notFoundAppIdEnvIdsMap", notFoundAppIdEnvIdsMap, "err", err) + return nil, err + } + if len(pipelines) == 0 { + // pipelines are deleted return nil, err } - configDBObjects := make([]*deploymentConfig.DeploymentConfig, 0, len(notFoundEnvToAppIdMapping)) + pipelineMap := make(map[bean.UniqueDeploymentConfigIdentifier]*pipelineConfig.Pipeline) + for _, p := range pipelines { + pipelineMap[bean.GetConfigUniqueIdentifier(p.AppId, p.EnvironmentId)] = p + } - for envId, appId := range notFoundEnvToAppIdMapping { - deploymentAppType := deploymentAppTypeMap[appId][envId] - appLevelConfig := appIdToAppLevelConfigMapping[appId] + configDBObjects := make([]*deploymentConfig.DeploymentConfig, 0, len(notFoundDeploymentConfigMap)) + for uniqueKey, _ := range notFoundDeploymentConfigMap { + if _, ok := pipelineMap[uniqueKey]; !ok { + //skipping for deleted pipelines + continue + } + pipeline := pipelineMap[uniqueKey] + deploymentAppType := pipelineMap[uniqueKey].DeploymentAppType + appLevelConfig := appIdToAppLevelConfigMapping[pipeline.AppId] configDbObj := &deploymentConfig.DeploymentConfig{ - AppId: appId, - EnvironmentId: envId, + AppId: pipeline.AppId, + EnvironmentId: pipeline.EnvironmentId, ConfigType: appLevelConfig.ConfigType, RepoUrl: appLevelConfig.RepoUrl, ChartLocation: appLevelConfig.ChartLocation, @@ -446,48 +484,57 @@ func (impl *DeploymentConfigServiceImpl) migrateEnvLevelDataTODeploymentConfigIn } switch deploymentAppType { case bean2.ArgoCd: - configDbObj.CredentialType = bean.GitOps.String() - configDbObj.CredentialIdInt = repoUrlToConfigMapping[appLevelConfig.RepoUrl].Id + configDbObj.CredentialType = appLevelConfig.CredentialType + configDbObj.CredentialIdInt = appLevelConfig.CredentialIdInt } configDbObj.AuditLog.CreateAuditLog(bean3.SYSTEM_USER_ID) configDBObjects = append(configDBObjects, configDbObj) } - configDBObjects, err = impl.deploymentConfigRepository.SaveAll(nil, configDBObjects) - if err != nil { - impl.logger.Errorw("error in saving deployment config in DB", "envIdToAppIdMapping", notFoundEnvToAppIdMapping, "err", err) - return nil, err + if len(configDBObjects) > 0 { + configDBObjects, err = impl.deploymentConfigRepository.SaveAll(nil, configDBObjects) + if err != nil { + impl.logger.Errorw("error in saving deployment config in DB", "notFoundDeploymentConfigMap", notFoundDeploymentConfigMap, "err", err) + return nil, err + } } return configDBObjects, nil } -func (impl *DeploymentConfigServiceImpl) GetHelmAppConfigInBulk(appIds []int, envIdToAppIdMapping map[int]int) ([]*bean.DeploymentConfig, error) { +func (impl *DeploymentConfigServiceImpl) GetHelmAppConfigInBulk(appIds []int, configSelector []*bean.DeploymentConfigSelector) ([]*bean.DeploymentConfig, error) { allDeploymentConfigs := make([]*deploymentConfig.DeploymentConfig, 0, len(appIds)) - helmDeploymentConfig, err := impl.deploymentConfigRepository.GetAppAndEnvLevelConfigsInBulk(envIdToAppIdMapping) + appIdToEnvIdsMap := GetAppIdToEnvIsMappingFromConfigSelectors(configSelector) + + helmDeploymentConfig, err := impl.deploymentConfigRepository.GetAppAndEnvLevelConfigsInBulk(appIdToEnvIdsMap) if err != nil && err != pg.ErrNoRows { - impl.logger.Errorw("error in fetching deployment config by by appId and envId", "envIdToAppIdMapping", envIdToAppIdMapping, "err", err) + impl.logger.Errorw("error in fetching deployment config by by appId and envId", "envIdToAppIdMapping", appIdToEnvIdsMap, "err", err) return nil, err } allDeploymentConfigs = append(allDeploymentConfigs, helmDeploymentConfig...) - if len(helmDeploymentConfig) < len(envIdToAppIdMapping) { - notFoundEnvIdToAppIdsMap := make(map[int]int) - presentEnvIds := make(map[int]bool) + if len(helmDeploymentConfig) < len(configSelector) { + + notFoundDeploymentConfigMap := make(map[bean.UniqueDeploymentConfigIdentifier]bool) + presentDeploymentConfigMap := make(map[bean.UniqueDeploymentConfigIdentifier]bool) + for _, c := range helmDeploymentConfig { - presentEnvIds[c.EnvironmentId] = true + presentDeploymentConfigMap[bean.GetConfigUniqueIdentifier(c.AppId, c.EnvironmentId)] = true } - for envId, appId := range envIdToAppIdMapping { - if _, ok := presentEnvIds[envId]; !ok { - notFoundEnvIdToAppIdsMap[envId] = appId + + for _, c := range configSelector { + key := bean.GetConfigUniqueIdentifier(c.AppId, c.EnvironmentId) + if _, ok := presentDeploymentConfigMap[key]; !ok { + notFoundDeploymentConfigMap[key] = true } } - migratedConfigs, err := impl.migrateDeploymentConfigInBulkForHelmApps(appIds, notFoundEnvIdToAppIdsMap) + + migratedConfigs, err := impl.migrateDeploymentConfigInBulkForHelmApps(appIds, notFoundDeploymentConfigMap) if err != nil { - impl.logger.Errorw("error in migrating helm apps config data in bult to deployment config", "migratedEnvToAppIdMapping", notFoundEnvIdToAppIdsMap, "err", err) + impl.logger.Errorw("error in migrating helm apps config data in bult to deployment config", "notFoundDeploymentConfigMap", notFoundDeploymentConfigMap, "err", err) return nil, err } allDeploymentConfigs = append(allDeploymentConfigs, migratedConfigs...) @@ -500,7 +547,7 @@ func (impl *DeploymentConfigServiceImpl) GetHelmAppConfigInBulk(appIds []int, en return deploymentConfigsResult, nil } -func (impl *DeploymentConfigServiceImpl) migrateDeploymentConfigInBulkForHelmApps(appIds []int, envIdToAppIdMapping map[int]int) ([]*deploymentConfig.DeploymentConfig, error) { +func (impl *DeploymentConfigServiceImpl) migrateDeploymentConfigInBulkForHelmApps(appIds []int, notFoundDeploymentConfigMap map[bean.UniqueDeploymentConfigIdentifier]bool) ([]*deploymentConfig.DeploymentConfig, error) { installedApps, err := impl.installedAppRepository.FindInstalledAppByIds(appIds) if err != nil { impl.logger.Errorw("error in getting installed app by appId", "appIds", appIds, "err", err) @@ -522,9 +569,11 @@ func (impl *DeploymentConfigServiceImpl) migrateDeploymentConfigInBulkForHelmApp return nil, err } - configDBObjects := make([]*deploymentConfig.DeploymentConfig, 0, len(envIdToAppIdMapping)) + configDBObjects := make([]*deploymentConfig.DeploymentConfig, 0, len(notFoundDeploymentConfigMap)) + + for appEnvUniqueKey, _ := range notFoundDeploymentConfigMap { - for envId, appId := range envIdToAppIdMapping { + appId, envId := appEnvUniqueKey.GetAppAndEnvId() installedApp := appIdToInstalledAppMapping[appId] helmDeploymentConfig := &deploymentConfig.DeploymentConfig{ AppId: appId, @@ -546,7 +595,7 @@ func (impl *DeploymentConfigServiceImpl) migrateDeploymentConfigInBulkForHelmApp configDBObjects, err = impl.deploymentConfigRepository.SaveAll(nil, configDBObjects) if err != nil { - impl.logger.Errorw("error in saving deployment config in DB", "envIdToAppIdMapping", envIdToAppIdMapping, "err", err) + impl.logger.Errorw("error in saving deployment config in DB", "notFoundDeploymentConfigMap", notFoundDeploymentConfigMap, "err", err) return nil, err } return configDBObjects, nil diff --git a/pkg/deployment/common/helper.go b/pkg/deployment/common/helper.go index 71667ef6c6..7355fe03ac 100644 --- a/pkg/deployment/common/helper.go +++ b/pkg/deployment/common/helper.go @@ -1,6 +1,8 @@ package common -import "github.com/devtron-labs/devtron/pkg/deployment/common/bean" +import ( + "github.com/devtron-labs/devtron/pkg/deployment/common/bean" +) func GetDeploymentConfigType(isCustomGitOpsRepo bool) string { if isCustomGitOpsRepo { @@ -12,3 +14,14 @@ func GetDeploymentConfigType(isCustomGitOpsRepo bool) string { func IsCustomGitOpsRepo(deploymentConfigType string) bool { return deploymentConfigType == bean.CUSTOM.String() } + +func GetAppIdToEnvIsMappingFromConfigSelectors(configSelector []*bean.DeploymentConfigSelector) map[int][]int { + appIdToEnvIdsMap := make(map[int][]int) + for _, c := range configSelector { + if _, ok := appIdToEnvIdsMap[c.AppId]; !ok { + appIdToEnvIdsMap[c.AppId] = make([]int, 0) + } + appIdToEnvIdsMap[c.AppId] = append(appIdToEnvIdsMap[c.AppId], c.EnvironmentId) + } + return appIdToEnvIdsMap +} diff --git a/scripts/sql/260_deployment_config.down.sql b/scripts/sql/260_deployment_config.down.sql index a4f8551daa..5d3262a271 100644 --- a/scripts/sql/260_deployment_config.down.sql +++ b/scripts/sql/260_deployment_config.down.sql @@ -1 +1,2 @@ -DROP TABLE IF EXISTS deployment_config; \ No newline at end of file +DROP TABLE IF EXISTS deployment_config; +DROP SEQUENCE id_seq_deployment_config; \ No newline at end of file diff --git a/wire_gen.go b/wire_gen.go index 43ef5d02ff..65ce614493 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -948,11 +948,6 @@ func InitializeApp() (*App, error) { if err != nil { return nil, err } - deploymentConfigs, err := deploymentConfigServiceImpl.GetDevtronAppConfigInBulk([]int{1,6}, map[int]int{1:1,2:1,3:1,4:1} ) - if err!=nil{ - - } - _ = len(deploymentConfigs) mainApp := NewApp(muxRouter, sugaredLogger, sseSSE, syncedEnforcer, db, sessionManager, posthogClient, loggingMiddlewareImpl, centralEventProcessor, pubSubClientServiceImpl, workflowEventProcessorImpl) return mainApp, nil } From 63cebe75672df4d87953d973dc69618ab8347628 Mon Sep 17 00:00:00 2001 From: ayushmaheshwari Date: Wed, 3 Jul 2024 00:07:51 +0530 Subject: [PATCH 11/45] QUERY : UPDATED buildAppListingWhereCondition and FIXED deploymentDetailsByAppIdAndEnvId --- internal/sql/repository/AppListingRepository.go | 15 ++++++++++++++- .../helper/AppListingRepositoryQueryBuilder.go | 16 ++-------------- 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/internal/sql/repository/AppListingRepository.go b/internal/sql/repository/AppListingRepository.go index 3116b0b820..cf202f8589 100644 --- a/internal/sql/repository/AppListingRepository.go +++ b/internal/sql/repository/AppListingRepository.go @@ -26,6 +26,7 @@ import ( "github.com/devtron-labs/devtron/api/bean/gitOps" "github.com/devtron-labs/devtron/internal/middleware" appWorkflow2 "github.com/devtron-labs/devtron/internal/sql/repository/appWorkflow" + "github.com/devtron-labs/devtron/internal/sql/repository/deploymentConfig" repository2 "github.com/devtron-labs/devtron/pkg/cluster/repository" "go.opentelemetry.io/otel" "strings" @@ -88,6 +89,7 @@ type AppListingRepositoryImpl struct { environmentRepository repository2.EnvironmentRepository gitOpsRepository GitOpsConfigRepository appWorkflowRepository appWorkflow2.AppWorkflowRepository + deploymentConfigRepository deploymentConfig.Repository } func NewAppListingRepositoryImpl( @@ -95,7 +97,8 @@ func NewAppListingRepositoryImpl( dbConnection *pg.DB, appListingRepositoryQueryBuilder helper.AppListingRepositoryQueryBuilder, environmentRepository repository2.EnvironmentRepository, - gitOpsRepository GitOpsConfigRepository, appWorkflowRepository appWorkflow2.AppWorkflowRepository) *AppListingRepositoryImpl { + gitOpsRepository GitOpsConfigRepository, appWorkflowRepository appWorkflow2.AppWorkflowRepository, + deploymentConfigRepository deploymentConfig.Repository) *AppListingRepositoryImpl { return &AppListingRepositoryImpl{ dbConnection: dbConnection, Logger: Logger, @@ -103,6 +106,7 @@ func NewAppListingRepositoryImpl( environmentRepository: environmentRepository, gitOpsRepository: gitOpsRepository, appWorkflowRepository: appWorkflowRepository, + deploymentConfigRepository: deploymentConfigRepository, } } @@ -374,6 +378,15 @@ func (impl AppListingRepositoryImpl) deploymentDetailsByAppIdAndEnvId(ctx contex return deploymentDetail, err } deploymentDetail.EnvironmentId = envId + if len(deploymentDetail.DeploymentAppType) == 0 { + dc, err := impl.deploymentConfigRepository.GetByAppIdAndEnvId(appId, envId) + if err != nil { + impl.Logger.Errorw("error in getting deployment config by appId and envId", "appId", appId, "envId", envId, "err", err) + return deploymentDetail, err + } + deploymentDetail.DeploymentAppType = dc.DeploymentAppType + } + return deploymentDetail, nil } diff --git a/internal/sql/repository/helper/AppListingRepositoryQueryBuilder.go b/internal/sql/repository/helper/AppListingRepositoryQueryBuilder.go index 5df6c71f36..0d48c16f7c 100644 --- a/internal/sql/repository/helper/AppListingRepositoryQueryBuilder.go +++ b/internal/sql/repository/helper/AppListingRepositoryQueryBuilder.go @@ -129,19 +129,6 @@ func getAppListingCommonQueryString() string { " LEFT JOIN app_status aps on aps.app_id = a.id and p.environment_id = aps.env_id " } -func (impl AppListingRepositoryQueryBuilder) BuildAppListingQuery(appListingFilter AppListingFilter) string { - whereCondition := impl.buildAppListingWhereCondition(appListingFilter) - orderByClause := impl.buildAppListingSortBy(appListingFilter) - query := "SELECT env.id AS environment_id, env.environment_name,env.namespace as namespace ,a.id AS app_id, a.app_name, env.default,aps.status as app_status," + - " p.id as pipeline_id, env.active, a.team_id, t.name as team_name" + - " , cluster.cluster_name as cluster_name" + getAppListingCommonQueryString() - if appListingFilter.DeploymentGroupId != 0 { - query = query + " INNER JOIN deployment_group_app dga ON a.id = dga.app_id " - } - query = query + whereCondition + orderByClause - return query -} - func (impl AppListingRepositoryQueryBuilder) GetQueryForAppEnvContainerss(appListingFilter AppListingFilter) string { query := "SELECT p.environment_id , a.id AS app_id, a.app_name,p.id as pipeline_id, a.team_id ,aps.status as app_status " @@ -154,6 +141,7 @@ func (impl AppListingRepositoryQueryBuilder) CommonJoinSubQuery(appListingFilter whereCondition := impl.buildAppListingWhereCondition(appListingFilter) query := " LEFT JOIN pipeline p ON a.id=p.app_id and p.deleted=false " + + " LEFT JOIN deployment_config dc ON ( p.app_id=dc.app_id and p.environment_id=dc.environment_id )" + " LEFT JOIN app_status aps on aps.app_id = a.id and p.environment_id = aps.env_id " if appListingFilter.DeploymentGroupId != 0 { @@ -258,7 +246,7 @@ func (impl AppListingRepositoryQueryBuilder) buildAppListingWhereCondition(appLi appStatuses := util.ProcessAppStatuses(appStatusExcludingNotDeployed) if isNotDeployedFilterApplied { deploymentAppType := "manifest_download" - whereCondition += fmt.Sprintf(" and (p.deployment_app_created=%v and p.deployment_app_type != '%s' or a.id NOT IN (SELECT app_id from pipeline) ", false, deploymentAppType) + whereCondition += fmt.Sprintf(" and (p.deployment_app_created=%v and (p.deployment_app_type != '%s' || dc.deployment_app_type != '%s' ) or a.id NOT IN (SELECT app_id from pipeline) ", false, deploymentAppType, deploymentAppType) if len(appStatuses) > 0 { whereCondition += fmt.Sprintf(" or aps.status IN ( %s ) ", appStatuses) } From ffc626f10bb8542357369343761de9980a14756c Mon Sep 17 00:00:00 2001 From: ayushmaheshwari Date: Wed, 3 Jul 2024 01:32:19 +0530 Subject: [PATCH 12/45] function for bulk update config --- .../repository/deploymentConfig/repository.go | 11 +++ pkg/deployment/common/bean/bean.go | 12 +++ .../common/deploymentConfigService.go | 76 ++++++++++++++++++- pkg/deployment/common/helper.go | 13 ++-- 4 files changed, 105 insertions(+), 7 deletions(-) diff --git a/internal/sql/repository/deploymentConfig/repository.go b/internal/sql/repository/deploymentConfig/repository.go index 3276c95e3e..2dd77296da 100644 --- a/internal/sql/repository/deploymentConfig/repository.go +++ b/internal/sql/repository/deploymentConfig/repository.go @@ -27,6 +27,7 @@ type Repository interface { Save(tx *pg.Tx, config *DeploymentConfig) (*DeploymentConfig, error) SaveAll(tx *pg.Tx, configs []*DeploymentConfig) ([]*DeploymentConfig, error) Update(tx *pg.Tx, config *DeploymentConfig) (*DeploymentConfig, error) + UpdateAll(tx *pg.Tx, config []*DeploymentConfig) ([]*DeploymentConfig, error) GetById(id int) (*DeploymentConfig, error) GetByAppIdAndEnvId(appId, envId int) (*DeploymentConfig, error) GetAppLevelConfig(appId int) (*DeploymentConfig, error) @@ -72,6 +73,16 @@ func (impl RepositoryImpl) Update(tx *pg.Tx, config *DeploymentConfig) (*Deploym return config, err } +func (impl RepositoryImpl) UpdateAll(tx *pg.Tx, config []*DeploymentConfig) ([]*DeploymentConfig, error) { + var err error + if tx != nil { + err = tx.Update(config) + } else { + err = impl.dbConnection.Update(config) + } + return config, err +} + func (impl RepositoryImpl) GetById(id int) (*DeploymentConfig, error) { result := &DeploymentConfig{} err := impl.dbConnection.Model(result).Where("id = ?", id).Where("active = ?", true).Select() diff --git a/pkg/deployment/common/bean/bean.go b/pkg/deployment/common/bean/bean.go index df4a645e70..b7939f45a9 100644 --- a/pkg/deployment/common/bean/bean.go +++ b/pkg/deployment/common/bean/bean.go @@ -28,6 +28,18 @@ type DeploymentConfigSelector struct { EnvironmentId int } +func (d *DeploymentConfigSelector) GetAppIdSelector() func(config DeploymentConfig) int { + return func(config DeploymentConfig) int { + return config.AppId + } +} + +func (d *DeploymentConfigSelector) GetEnvIdSelector() func(config DeploymentConfig) int { + return func(config DeploymentConfig) int { + return config.EnvironmentId + } +} + func (u UniqueDeploymentConfigIdentifier) String() string { return string(u) } diff --git a/pkg/deployment/common/deploymentConfigService.go b/pkg/deployment/common/deploymentConfigService.go index f823571a17..299390311c 100644 --- a/pkg/deployment/common/deploymentConfigService.go +++ b/pkg/deployment/common/deploymentConfigService.go @@ -17,6 +17,7 @@ import ( type DeploymentConfigService interface { CreateOrUpdateConfig(tx *pg.Tx, config *bean.DeploymentConfig, userId int32) (*bean.DeploymentConfig, error) + CreateOrUpdateConfigsInBulk(tx *pg.Tx, configs []*bean.DeploymentConfig, userId int32) ([]*bean.DeploymentConfig, error) GetDeploymentConfig(appId, envId int) (*bean.DeploymentConfig, error) GetDeploymentConfigForHelmApp(appId, envId int) (*bean.DeploymentConfig, error) GetDeploymentConfigInBulk(configSelector []*bean.DeploymentConfigSelector) ([]*bean.DeploymentConfig, error) @@ -82,6 +83,71 @@ func (impl *DeploymentConfigServiceImpl) CreateOrUpdateConfig(tx *pg.Tx, config return ConvertDeploymentConfigDbObjToDTO(newDBObj), nil } +func (impl *DeploymentConfigServiceImpl) CreateOrUpdateConfigsInBulk(tx *pg.Tx, configs []*bean.DeploymentConfig, userId int32) ([]*bean.DeploymentConfig, error) { + + appIdToEnvIdsMap := GetAppIdToEnvIsMapping( + configs, + func(config *bean.DeploymentConfig) int { return config.AppId }, + func(config *bean.DeploymentConfig) int { return config.EnvironmentId }) + + deploymentConfigDbObj, err := impl.deploymentConfigRepository.GetAppAndEnvLevelConfigsInBulk(appIdToEnvIdsMap) + if err != nil { + impl.logger.Errorw("error in getting deployment configs by appIdToEnvIds", "appIdToEnvIdsMap", appIdToEnvIdsMap, "err", err) + return nil, err + } + + foundDeploymentConfigsMap := make(map[bean.UniqueDeploymentConfigIdentifier]*deploymentConfig.DeploymentConfig) + for _, savedConfig := range deploymentConfigDbObj { + foundDeploymentConfigsMap[bean.GetConfigUniqueIdentifier(savedConfig.AppId, savedConfig.EnvironmentId)] = savedConfig + } + + oldDeploymentConfigs := make([]*deploymentConfig.DeploymentConfig, 0) + newDeploymentConfigs := make([]*deploymentConfig.DeploymentConfig, 0) + + for _, requestConfig := range configs { + + if oldConfig, isConfigPresent := foundDeploymentConfigsMap[bean.GetConfigUniqueIdentifier(requestConfig.AppId, requestConfig.EnvironmentId)]; isConfigPresent { + + oldConfigDbObject := ConvertDeploymentConfigDTOToDbObj(requestConfig) + oldConfigDbObject.Id = oldConfig.Id + oldConfigDbObject.AuditLog.UpdateAuditLog(userId) + oldDeploymentConfigs = append(oldDeploymentConfigs, oldConfigDbObject) + + } else if !isConfigPresent { + + newConfigDbObject := ConvertDeploymentConfigDTOToDbObj(requestConfig) + newConfigDbObject.AuditLog.CreateAuditLog(userId) + newDeploymentConfigs = append(newDeploymentConfigs, ConvertDeploymentConfigDTOToDbObj(requestConfig)) + + } + } + + if len(newDeploymentConfigs) > 0 { + newDeploymentConfigs, err = impl.deploymentConfigRepository.SaveAll(tx, newDeploymentConfigs) + if err != nil { + impl.logger.Errorw("error in saving deployment config in db", "err", err) + return nil, err + } + } + if len(oldDeploymentConfigs) > 0 { + oldDeploymentConfigs, err = impl.deploymentConfigRepository.UpdateAll(tx, oldDeploymentConfigs) + if err != nil { + impl.logger.Errorw("error in saving deployment config in db", "err", err) + return nil, err + } + } + + allDeploymentConfigs := make([]*bean.DeploymentConfig, 0, len(oldDeploymentConfigs)+len(newDeploymentConfigs)) + + for _, c := range oldDeploymentConfigs { + allDeploymentConfigs = append(allDeploymentConfigs, ConvertDeploymentConfigDbObjToDTO(c)) + } + for _, c := range newDeploymentConfigs { + allDeploymentConfigs = append(allDeploymentConfigs, ConvertDeploymentConfigDbObjToDTO(c)) + } + return allDeploymentConfigs, nil +} + func (impl *DeploymentConfigServiceImpl) GetDeploymentConfig(appId, envId int) (*bean.DeploymentConfig, error) { appLevelConfigDbObj, err := impl.deploymentConfigRepository.GetAppLevelConfig(appId) @@ -352,7 +418,10 @@ func (impl *DeploymentConfigServiceImpl) GetDevtronAppConfigInBulk(appIds []int, if len(configSelector) > 0 { - appIdToEnvIdsMap := GetAppIdToEnvIsMappingFromConfigSelectors(configSelector) + appIdToEnvIdsMap := GetAppIdToEnvIsMapping( + configSelector, + func(config *bean.DeploymentConfigSelector) int { return config.AppId }, + func(config *bean.DeploymentConfigSelector) int { return config.EnvironmentId }) envLevelConfig, err := impl.deploymentConfigRepository.GetAppAndEnvLevelConfigsInBulk(appIdToEnvIdsMap) if err != nil { @@ -506,7 +575,10 @@ func (impl *DeploymentConfigServiceImpl) GetHelmAppConfigInBulk(appIds []int, co allDeploymentConfigs := make([]*deploymentConfig.DeploymentConfig, 0, len(appIds)) - appIdToEnvIdsMap := GetAppIdToEnvIsMappingFromConfigSelectors(configSelector) + appIdToEnvIdsMap := GetAppIdToEnvIsMapping( + configSelector, + func(config *bean.DeploymentConfigSelector) int { return config.AppId }, + func(config *bean.DeploymentConfigSelector) int { return config.EnvironmentId }) helmDeploymentConfig, err := impl.deploymentConfigRepository.GetAppAndEnvLevelConfigsInBulk(appIdToEnvIdsMap) if err != nil && err != pg.ErrNoRows { diff --git a/pkg/deployment/common/helper.go b/pkg/deployment/common/helper.go index 7355fe03ac..46e553146b 100644 --- a/pkg/deployment/common/helper.go +++ b/pkg/deployment/common/helper.go @@ -15,13 +15,16 @@ func IsCustomGitOpsRepo(deploymentConfigType string) bool { return deploymentConfigType == bean.CUSTOM.String() } -func GetAppIdToEnvIsMappingFromConfigSelectors(configSelector []*bean.DeploymentConfigSelector) map[int][]int { +func GetAppIdToEnvIsMapping[T any](configArr []T, appIdSelector func(config T) int, envIdSelector func(config T) int) map[int][]int { + appIdToEnvIdsMap := make(map[int][]int) - for _, c := range configSelector { - if _, ok := appIdToEnvIdsMap[c.AppId]; !ok { - appIdToEnvIdsMap[c.AppId] = make([]int, 0) + for _, c := range configArr { + appId := appIdSelector(c) + envId := envIdSelector(c) + if _, ok := appIdToEnvIdsMap[appId]; !ok { + appIdToEnvIdsMap[appId] = make([]int, 0) } - appIdToEnvIdsMap[c.AppId] = append(appIdToEnvIdsMap[c.AppId], c.EnvironmentId) + appIdToEnvIdsMap[appId] = append(appIdToEnvIdsMap[appId], envId) } return appIdToEnvIdsMap } From f4d16b93c5a453ff46267265c23eed99b1748bce Mon Sep 17 00:00:00 2001 From: ayushmaheshwari Date: Wed, 3 Jul 2024 16:44:10 +0530 Subject: [PATCH 13/45] query changes --- .../service/AppStoreDeploymentDBService.go | 20 +++- ...InstalledAppDeploymentTypeChangeService.go | 33 +++++- pkg/deployment/common/bean/bean.go | 12 -- .../common/deploymentConfigService.go | 104 ++++++++++++------ .../AppDeploymentTypeChangeManager.go | 86 +++++++++++---- 5 files changed, 186 insertions(+), 69 deletions(-) diff --git a/pkg/appStore/installedApp/service/AppStoreDeploymentDBService.go b/pkg/appStore/installedApp/service/AppStoreDeploymentDBService.go index cfa9531125..5ea15e2c41 100644 --- a/pkg/appStore/installedApp/service/AppStoreDeploymentDBService.go +++ b/pkg/appStore/installedApp/service/AppStoreDeploymentDBService.go @@ -36,6 +36,7 @@ import ( clusterService "github.com/devtron-labs/devtron/pkg/cluster" clutserBean "github.com/devtron-labs/devtron/pkg/cluster/repository/bean" "github.com/devtron-labs/devtron/pkg/deployment/common" + bean2 "github.com/devtron-labs/devtron/pkg/deployment/common/bean" "github.com/devtron-labs/devtron/pkg/deployment/gitOps/config" gitOpsBean "github.com/devtron-labs/devtron/pkg/deployment/gitOps/config/bean" validationBean "github.com/devtron-labs/devtron/pkg/deployment/gitOps/validation/bean" @@ -325,8 +326,24 @@ func (impl *AppStoreDeploymentDBServiceImpl) GetAllInstalledAppsByAppStoreId(app impl.logger.Error(err) return nil, err } + + deploymentSelectors := make([]*bean2.DeploymentConfigSelector, 0) + for _, ia := range installedApps { + deploymentSelectors = append(deploymentSelectors, &bean2.DeploymentConfigSelector{ + AppId: ia.AppId, + EnvironmentId: ia.EnvironmentId, + }) + } + + deploymentConfigMap, err := impl.deploymentConfigService.GetDeploymentConfigInBulk(deploymentSelectors) + if err != nil { + impl.logger.Errorw("error in getting deployment config by deployment selectors", "deploymentSelectors", deploymentSelectors, "err", err) + return nil, err + } + var installedAppsEnvResponse []appStoreBean.InstalledAppsResponse for _, a := range installedApps { + deploymentConfig := deploymentConfigMap[bean2.GetConfigUniqueIdentifier(a.AppId, a.EnvironmentId)] installedAppRes := appStoreBean.InstalledAppsResponse{ EnvironmentName: a.EnvironmentName, AppName: a.AppName, @@ -338,9 +355,8 @@ func (impl *AppStoreDeploymentDBServiceImpl) GetAllInstalledAppsByAppStoreId(app InstalledAppsId: a.InstalledAppId, EnvironmentId: a.EnvironmentId, AppOfferingMode: a.AppOfferingMode, - DeploymentAppType: a.DeploymentAppType, + DeploymentAppType: deploymentConfig.DeploymentAppType, } - // if hyperion mode app, then fill clusterId and namespace if globalUtil.IsHelmApp(a.AppOfferingMode) { environment, err := impl.environmentService.FindById(a.EnvironmentId) diff --git a/pkg/appStore/installedApp/service/FullMode/deploymentTypeChange/InstalledAppDeploymentTypeChangeService.go b/pkg/appStore/installedApp/service/FullMode/deploymentTypeChange/InstalledAppDeploymentTypeChangeService.go index 4554df574f..08036acbf7 100644 --- a/pkg/appStore/installedApp/service/FullMode/deploymentTypeChange/InstalledAppDeploymentTypeChangeService.go +++ b/pkg/appStore/installedApp/service/FullMode/deploymentTypeChange/InstalledAppDeploymentTypeChangeService.go @@ -221,7 +221,38 @@ func (impl *InstalledAppDeploymentTypeChangeServiceImpl) MigrateDeploymentType(c func (impl *InstalledAppDeploymentTypeChangeServiceImpl) performDbOperationsAfterMigrations(desiredDeploymentType bean2.DeploymentType, successInstalledAppIds []int, successAppIds []*int, userId int32, deployStatus int) error { - err := impl.installedAppRepository.UpdateDeploymentAppTypeInInstalledApp(desiredDeploymentType, successInstalledAppIds, userId, deployStatus) + installedApps, err := impl.installedAppRepository.FindInstalledAppByIds(successInstalledAppIds) + if err != nil { + impl.logger.Errorw("error in getting installed apps by ids", "installedAppIds", successInstalledAppIds, "err", err) + return err + } + + deploymentConfigSelector := make([]*bean3.DeploymentConfigSelector, len(successInstalledAppIds)) + for _, ia := range installedApps { + deploymentConfigSelector = append(deploymentConfigSelector, &bean3.DeploymentConfigSelector{ + AppId: ia.AppId, + EnvironmentId: ia.EnvironmentId, + }) + } + + deploymentConfigs, err := impl.deploymentConfigService.GetDeploymentConfigInBulk(deploymentConfigSelector) + if err != nil { + impl.logger.Errorw("error in getting deployment config", "deploymentConfigSelector", deploymentConfigSelector, "err", err) + return err + } + updatedDeploymentConfigs := make([]*bean3.DeploymentConfig, 0) + for _, c := range deploymentConfigs { + c.DeploymentAppType = desiredDeploymentType + updatedDeploymentConfigs = append(updatedDeploymentConfigs, c) + } + + updatedDeploymentConfigs, err = impl.deploymentConfigService.UpdateConfigs(nil, updatedDeploymentConfigs, userId) + if err != nil { + impl.logger.Errorw("error in updating configs", "err", err) + return err + } + + err = impl.installedAppRepository.UpdateDeploymentAppTypeInInstalledApp(desiredDeploymentType, successInstalledAppIds, userId, deployStatus) if err != nil { impl.logger.Errorw("failed to update deployment app type for successfully deleted installed apps in db", "successfully deleted installedApp ids", successInstalledAppIds, diff --git a/pkg/deployment/common/bean/bean.go b/pkg/deployment/common/bean/bean.go index b7939f45a9..df4a645e70 100644 --- a/pkg/deployment/common/bean/bean.go +++ b/pkg/deployment/common/bean/bean.go @@ -28,18 +28,6 @@ type DeploymentConfigSelector struct { EnvironmentId int } -func (d *DeploymentConfigSelector) GetAppIdSelector() func(config DeploymentConfig) int { - return func(config DeploymentConfig) int { - return config.AppId - } -} - -func (d *DeploymentConfigSelector) GetEnvIdSelector() func(config DeploymentConfig) int { - return func(config DeploymentConfig) int { - return config.EnvironmentId - } -} - func (u UniqueDeploymentConfigIdentifier) String() string { return string(u) } diff --git a/pkg/deployment/common/deploymentConfigService.go b/pkg/deployment/common/deploymentConfigService.go index 299390311c..8dd8a5c1f8 100644 --- a/pkg/deployment/common/deploymentConfigService.go +++ b/pkg/deployment/common/deploymentConfigService.go @@ -18,9 +18,10 @@ import ( type DeploymentConfigService interface { CreateOrUpdateConfig(tx *pg.Tx, config *bean.DeploymentConfig, userId int32) (*bean.DeploymentConfig, error) CreateOrUpdateConfigsInBulk(tx *pg.Tx, configs []*bean.DeploymentConfig, userId int32) ([]*bean.DeploymentConfig, error) + UpdateConfigs(tx *pg.Tx, configs []*bean.DeploymentConfig, userId int32) ([]*bean.DeploymentConfig, error) GetDeploymentConfig(appId, envId int) (*bean.DeploymentConfig, error) GetDeploymentConfigForHelmApp(appId, envId int) (*bean.DeploymentConfig, error) - GetDeploymentConfigInBulk(configSelector []*bean.DeploymentConfigSelector) ([]*bean.DeploymentConfig, error) + GetDeploymentConfigInBulk(configSelector []*bean.DeploymentConfigSelector) (map[bean.UniqueDeploymentConfigIdentifier]*bean.DeploymentConfig, error) } type DeploymentConfigServiceImpl struct { @@ -96,9 +97,9 @@ func (impl *DeploymentConfigServiceImpl) CreateOrUpdateConfigsInBulk(tx *pg.Tx, return nil, err } - foundDeploymentConfigsMap := make(map[bean.UniqueDeploymentConfigIdentifier]*deploymentConfig.DeploymentConfig) + savedDeploymentConfigsMap := make(map[bean.UniqueDeploymentConfigIdentifier]*deploymentConfig.DeploymentConfig) for _, savedConfig := range deploymentConfigDbObj { - foundDeploymentConfigsMap[bean.GetConfigUniqueIdentifier(savedConfig.AppId, savedConfig.EnvironmentId)] = savedConfig + savedDeploymentConfigsMap[bean.GetConfigUniqueIdentifier(savedConfig.AppId, savedConfig.EnvironmentId)] = savedConfig } oldDeploymentConfigs := make([]*deploymentConfig.DeploymentConfig, 0) @@ -106,7 +107,7 @@ func (impl *DeploymentConfigServiceImpl) CreateOrUpdateConfigsInBulk(tx *pg.Tx, for _, requestConfig := range configs { - if oldConfig, isConfigPresent := foundDeploymentConfigsMap[bean.GetConfigUniqueIdentifier(requestConfig.AppId, requestConfig.EnvironmentId)]; isConfigPresent { + if oldConfig, isConfigPresent := savedDeploymentConfigsMap[bean.GetConfigUniqueIdentifier(requestConfig.AppId, requestConfig.EnvironmentId)]; isConfigPresent { oldConfigDbObject := ConvertDeploymentConfigDTOToDbObj(requestConfig) oldConfigDbObject.Id = oldConfig.Id @@ -148,11 +149,35 @@ func (impl *DeploymentConfigServiceImpl) CreateOrUpdateConfigsInBulk(tx *pg.Tx, return allDeploymentConfigs, nil } +func (impl *DeploymentConfigServiceImpl) UpdateConfigs(tx *pg.Tx, configs []*bean.DeploymentConfig, userId int32) ([]*bean.DeploymentConfig, error) { + + configDbObject := make([]*deploymentConfig.DeploymentConfig, 0) + for _, c := range configs { + dbObj := ConvertDeploymentConfigDTOToDbObj(c) + dbObj.AuditLog.UpdateAuditLog(userId) + configDbObject = append(configDbObject) + } + + var err error + configDbObject, err = impl.deploymentConfigRepository.UpdateAll(tx, configDbObject) + if err != nil { + impl.logger.Errorw("error in saving deployment config in db", "err", err) + return nil, err + } + + updatedConfigs := make([]*bean.DeploymentConfig, 0) + for _, c := range configDbObject { + updatedConfigs = append(updatedConfigs, ConvertDeploymentConfigDbObjToDTO(c)) + } + + return updatedConfigs, nil +} + func (impl *DeploymentConfigServiceImpl) GetDeploymentConfig(appId, envId int) (*bean.DeploymentConfig, error) { appLevelConfigDbObj, err := impl.deploymentConfigRepository.GetAppLevelConfig(appId) if err != nil && err != pg.ErrNoRows { - impl.logger.Errorw("error in getiting deployment config db object by appId", "appId", appId, "err", err) + impl.logger.Errorw("error in getting deployment config db object by appId", "appId", appId, "err", err) return nil, err } if err == pg.ErrNoRows { @@ -166,7 +191,7 @@ func (impl *DeploymentConfigServiceImpl) GetDeploymentConfig(appId, envId int) ( if envId > 0 { appAndEnvLevelConfig, err := impl.deploymentConfigRepository.GetByAppIdAndEnvId(appId, envId) if err != nil && err != pg.ErrNoRows { - impl.logger.Errorw("error in getiting deployment config db object by appId and envId", "appId", appId, "envId", envId, "err", err) + impl.logger.Errorw("error in getting deployment config db object by appId and envId", "appId", appId, "envId", envId, "err", err) return nil, err } if err == pg.ErrNoRows { @@ -251,13 +276,13 @@ func (impl *DeploymentConfigServiceImpl) GetConfigDBObj(appId, envId int) (*depl if envId == 0 { configDbObj, err = impl.deploymentConfigRepository.GetAppLevelConfig(appId) if err != nil { - impl.logger.Errorw("error in getiting deployment config db object by appId", "appId", configDbObj.AppId, "err", err) + impl.logger.Errorw("error in getting deployment config db object by appId", "appId", configDbObj.AppId, "err", err) return nil, err } } else { configDbObj, err = impl.deploymentConfigRepository.GetByAppIdAndEnvId(appId, envId) if err != nil { - impl.logger.Errorw("error in getiting deployment config db object by appId and envId", "appId", configDbObj.AppId, "envId", configDbObj.EnvironmentId, "err", err) + impl.logger.Errorw("error in getting deployment config db object by appId and envId", "appId", configDbObj.AppId, "envId", configDbObj.EnvironmentId, "err", err) return nil, err } } @@ -319,7 +344,7 @@ func (impl *DeploymentConfigServiceImpl) migrateHelmAppDataToDeploymentConfig(ap return helmDeploymentConfig, nil } -func (impl *DeploymentConfigServiceImpl) GetDeploymentConfigInBulk(configSelector []*bean.DeploymentConfigSelector) ([]*bean.DeploymentConfig, error) { +func (impl *DeploymentConfigServiceImpl) GetDeploymentConfigInBulk(configSelector []*bean.DeploymentConfigSelector) (map[bean.UniqueDeploymentConfigIdentifier]*bean.DeploymentConfig, error) { appIds := make([]*int, len(configSelector)) for _, s := range configSelector { @@ -372,27 +397,33 @@ func (impl *DeploymentConfigServiceImpl) GetDeploymentConfigInBulk(configSelecto return nil, err } - allConfigs := make([]*bean.DeploymentConfig, len(devtronAppConfigs)+len(helmAppConfigs)) - allConfigs = append(allConfigs, devtronAppConfigs...) - allConfigs = append(allConfigs, helmAppConfigs...) + allConfigs := make(map[bean.UniqueDeploymentConfigIdentifier]*bean.DeploymentConfig) + + for _, c := range devtronAppConfigs { + allConfigs[bean.GetConfigUniqueIdentifier(c.AppId, c.EnvironmentId)] = c + } + + for _, c := range helmAppConfigs { + allConfigs[bean.GetConfigUniqueIdentifier(c.AppId, c.EnvironmentId)] = c + } return allConfigs, nil } func (impl *DeploymentConfigServiceImpl) GetDevtronAppConfigInBulk(appIds []int, configSelector []*bean.DeploymentConfigSelector) ([]*bean.DeploymentConfig, error) { - AppLevelDeploymentConfigs, err := impl.deploymentConfigRepository.GetAppLevelConfigByAppIds(appIds) + appLevelDeploymentConfigs, err := impl.deploymentConfigRepository.GetAppLevelConfigByAppIds(appIds) if err != nil { impl.logger.Errorw("error in fetching deployment configs by appIds", "appIds", appIds, "err", err) return nil, err } - AllAppLevelConfigs := make([]*deploymentConfig.DeploymentConfig, 0, len(appIds)) - AllAppLevelConfigs = append(AllAppLevelConfigs, AppLevelDeploymentConfigs...) + allAppLevelConfigs := make([]*deploymentConfig.DeploymentConfig, 0, len(appIds)) + allAppLevelConfigs = append(allAppLevelConfigs, appLevelDeploymentConfigs...) - if len(AppLevelDeploymentConfigs) < len(appIds) { + if len(appLevelDeploymentConfigs) < len(appIds) { presentAppIds := make(map[int]bool) - for _, c := range AppLevelDeploymentConfigs { + for _, c := range appLevelDeploymentConfigs { presentAppIds[c.AppId] = true } notFoundAppIds := make([]int, 0, len(appIds)) @@ -406,15 +437,15 @@ func (impl *DeploymentConfigServiceImpl) GetDevtronAppConfigInBulk(appIds []int, impl.logger.Errorw("error in migrating all level deployment configs", "appIds", notFoundAppIds, "err", err) return nil, err } - AllAppLevelConfigs = append(AllAppLevelConfigs, migratedAppLevelDeploymentConfigs...) + allAppLevelConfigs = append(allAppLevelConfigs, migratedAppLevelDeploymentConfigs...) } appIdToAppLevelConfigMapping := make(map[int]*deploymentConfig.DeploymentConfig, len(appIds)) - for _, appLevelConfig := range AllAppLevelConfigs { + for _, appLevelConfig := range allAppLevelConfigs { appIdToAppLevelConfigMapping[appLevelConfig.AppId] = appLevelConfig } - AllEnvLevelConfigs := make([]*deploymentConfig.DeploymentConfig, 0, len(configSelector)) + allEnvLevelConfigs := make([]*deploymentConfig.DeploymentConfig, 0, len(configSelector)) if len(configSelector) > 0 { @@ -428,7 +459,7 @@ func (impl *DeploymentConfigServiceImpl) GetDevtronAppConfigInBulk(appIds []int, impl.logger.Errorw("error in getting and and env level config in bulk", "appIdToEnvIdsMap", appIdToEnvIdsMap, "err", err) return nil, err } - AllEnvLevelConfigs = append(AllEnvLevelConfigs, envLevelConfig...) + allEnvLevelConfigs = append(allEnvLevelConfigs, envLevelConfig...) if len(envLevelConfig) < len(configSelector) { notFoundDeploymentConfigMap := make(map[bean.UniqueDeploymentConfigIdentifier]bool) @@ -445,21 +476,28 @@ func (impl *DeploymentConfigServiceImpl) GetDevtronAppConfigInBulk(appIds []int, } } - migratedEnvLevelDeploymentConfigs, err := impl.migrateEnvLevelDataTODeploymentConfigInBulk(notFoundDeploymentConfigMap, appIdToAppLevelConfigMapping) + migratedEnvLevelDeploymentConfigs, err := impl.migrateEnvLevelDataToDeploymentConfigInBulk(notFoundDeploymentConfigMap, appIdToAppLevelConfigMapping) if err != nil { impl.logger.Errorw("error in migrating env level configs", "notFoundDeploymentConfigMap", notFoundDeploymentConfigMap, "err", err) return nil, err } - AllEnvLevelConfigs = append(AllEnvLevelConfigs, migratedEnvLevelDeploymentConfigs...) + allEnvLevelConfigs = append(allEnvLevelConfigs, migratedEnvLevelDeploymentConfigs...) } } - allConfigs := make([]*bean.DeploymentConfig, 0) - for _, c := range AllAppLevelConfigs { - allConfigs = append(allConfigs, ConvertDeploymentConfigDbObjToDTO(c)) + envLevelConfigMapping := make(map[bean.UniqueDeploymentConfigIdentifier]*bean.DeploymentConfig) + for _, c := range envLevelConfigMapping { + envLevelConfigMapping[bean.GetConfigUniqueIdentifier(c.AppId, c.EnvironmentId)] = c } - for _, c := range AllEnvLevelConfigs { - allConfigs = append(allConfigs, ConvertDeploymentConfigDbObjToDTO(c)) + + allConfigs := make([]*bean.DeploymentConfig, 0) + for _, c := range configSelector { + if c.AppId > 0 && c.EnvironmentId > 0 { + allConfigs = append(allConfigs, envLevelConfigMapping[bean.GetConfigUniqueIdentifier(c.AppId, c.EnvironmentId)]) + } else if c.AppId > 0 { + // if user has sent only appId in config selector then only app level + allConfigs = append(allConfigs, ConvertDeploymentConfigDbObjToDTO(appIdToAppLevelConfigMapping[c.AppId])) + } } return allConfigs, err @@ -478,7 +516,7 @@ func (impl *DeploymentConfigServiceImpl) migrateAppLevelDataTODeploymentConfigIn allRepoUrls = append(allRepoUrls, c.GitRepoUrl) } - repoUrlToConfigMapping, err := impl.gitOpsConfigReadService.GetGitOpsProviderMapByRepoURL(allRepoUrls) + repoUrlToGitOpsConfigMapping, err := impl.gitOpsConfigReadService.GetGitOpsProviderMapByRepoURL(allRepoUrls) if err != nil { impl.logger.Errorw("error in fetching repoUrl to config mapping", "err", err) return nil, err @@ -493,7 +531,7 @@ func (impl *DeploymentConfigServiceImpl) migrateAppLevelDataTODeploymentConfigIn RepoUrl: c.GitRepoUrl, ChartLocation: c.ChartLocation, CredentialType: bean.GitOps.String(), - CredentialIdInt: repoUrlToConfigMapping[c.GitRepoUrl].Id, + CredentialIdInt: repoUrlToGitOpsConfigMapping[c.GitRepoUrl].Id, } dbObj.AuditLog.CreateAuditLog(bean3.SYSTEM_USER_ID) configDBObjects = append(configDBObjects, dbObj) @@ -507,7 +545,7 @@ func (impl *DeploymentConfigServiceImpl) migrateAppLevelDataTODeploymentConfigIn return configDBObjects, nil } -func (impl *DeploymentConfigServiceImpl) migrateEnvLevelDataTODeploymentConfigInBulk(notFoundDeploymentConfigMap map[bean.UniqueDeploymentConfigIdentifier]bool, appIdToAppLevelConfigMapping map[int]*deploymentConfig.DeploymentConfig) ([]*deploymentConfig.DeploymentConfig, error) { +func (impl *DeploymentConfigServiceImpl) migrateEnvLevelDataToDeploymentConfigInBulk(notFoundDeploymentConfigMap map[bean.UniqueDeploymentConfigIdentifier]bool, appIdToAppLevelConfigMapping map[int]*deploymentConfig.DeploymentConfig) ([]*deploymentConfig.DeploymentConfig, error) { notFoundAppIdEnvIdsMap := make(map[int][]int) for uniqueKey, _ := range notFoundDeploymentConfigMap { @@ -635,7 +673,7 @@ func (impl *DeploymentConfigServiceImpl) migrateDeploymentConfigInBulkForHelmApp allRepoURLS = append(allRepoURLS, ia.GitOpsRepoUrl) } - repoUrlToConfigMapping, err := impl.gitOpsConfigReadService.GetGitOpsProviderMapByRepoURL(allRepoURLS) + repoUrlToGitOpsConfigMapping, err := impl.gitOpsConfigReadService.GetGitOpsProviderMapByRepoURL(allRepoURLS) if err != nil { impl.logger.Errorw("error in fetching repoUrl to config mapping", "err", err) return nil, err @@ -659,7 +697,7 @@ func (impl *DeploymentConfigServiceImpl) migrateDeploymentConfigInBulkForHelmApp switch installedApp.DeploymentAppType { case bean2.ArgoCd: helmDeploymentConfig.CredentialType = bean.GitOps.String() - helmDeploymentConfig.CredentialIdInt = repoUrlToConfigMapping[installedApp.GitOpsRepoUrl].Id + helmDeploymentConfig.CredentialIdInt = repoUrlToGitOpsConfigMapping[installedApp.GitOpsRepoUrl].Id } helmDeploymentConfig.AuditLog.CreateAuditLog(bean3.SYSTEM_USER_ID) configDBObjects = append(configDBObjects, helmDeploymentConfig) diff --git a/pkg/pipeline/AppDeploymentTypeChangeManager.go b/pkg/pipeline/AppDeploymentTypeChangeManager.go index fdea822217..ea47a9252a 100644 --- a/pkg/pipeline/AppDeploymentTypeChangeManager.go +++ b/pkg/pipeline/AppDeploymentTypeChangeManager.go @@ -136,23 +136,57 @@ func (impl *AppDeploymentTypeChangeManagerImpl) ChangeDeploymentType(ctx context response.TriggeredPipelines = make([]*bean.CdPipelineTrigger, 0) // Update the deployment app type to Helm and toggle deployment_app_created to false in db - var cdPipelineIds []int + pipelineIds := make([]int, 0, len(response.SuccessfulPipelines)) + for _, item := range response.SuccessfulPipelines { - cdPipelineIds = append(cdPipelineIds, item.PipelineId) + pipelineIds = append(pipelineIds, item.PipelineId) } + // Get all pipelines + pipelines, err := impl.pipelineRepository.FindByIdsIn(pipelineIds) + if err != nil { + impl.logger.Errorw("failed to fetch pipeline details", + "ids", pipelineIds, + "err", err) + + return response, nil + } // If nothing to update in db - if len(cdPipelineIds) == 0 { + if len(pipelineIds) == 0 { return response, nil } + deploymentConfigSelector := make([]*bean4.DeploymentConfigSelector, len(pipelineIds)) + for _, pipeline := range pipelines { + deploymentConfigSelector = append(deploymentConfigSelector, &bean4.DeploymentConfigSelector{ + AppId: pipeline.AppId, + EnvironmentId: pipeline.EnvironmentId, + }) + } + + deploymentConfigs, err := impl.deploymentConfigService.GetDeploymentConfigInBulk(deploymentConfigSelector) + if err != nil { + impl.logger.Errorw("error in getting deployment config", "deploymentConfigSelector", deploymentConfigSelector, "err", err) + return nil, err + } + updatedDeploymentConfigs := make([]*bean4.DeploymentConfig, 0) + for _, c := range deploymentConfigs { + c.DeploymentAppType = request.DesiredDeploymentType + updatedDeploymentConfigs = append(updatedDeploymentConfigs, c) + } + + updatedDeploymentConfigs, err = impl.deploymentConfigService.UpdateConfigs(nil, updatedDeploymentConfigs, request.UserId) + if err != nil { + impl.logger.Errorw("error in updating configs", "err", err) + return nil, err + } + // Update in db err = impl.pipelineRepository.UpdateCdPipelineDeploymentAppInFilter(string(request.DesiredDeploymentType), - cdPipelineIds, request.UserId, false, true) - + pipelineIds, request.UserId, false, true) if err != nil { impl.logger.Errorw("failed to update deployment app type in db", - "pipeline ids", cdPipelineIds, + "pipeline ids", pipelineIds, "desired deployment type", request.DesiredDeploymentType, "err", err) @@ -166,21 +200,6 @@ func (impl *AppDeploymentTypeChangeManagerImpl) ChangeDeploymentType(ctx context // Bulk trigger all the successfully changed pipelines (async) bulkTriggerRequest := make([]*bean2.BulkTriggerRequest, 0) - pipelineIds := make([]int, 0, len(response.SuccessfulPipelines)) - for _, item := range response.SuccessfulPipelines { - pipelineIds = append(pipelineIds, item.PipelineId) - } - - // Get all pipelines - pipelines, err := impl.pipelineRepository.FindByIdsIn(pipelineIds) - if err != nil { - impl.logger.Errorw("failed to fetch pipeline details", - "ids", pipelineIds, - "err", err) - - return response, nil - } - for _, pipeline := range pipelines { artifactDetails, err := impl.appArtifactManager.RetrieveArtifactsByCDPipeline(pipeline, "DEPLOY") @@ -396,6 +415,31 @@ func (impl *AppDeploymentTypeChangeManagerImpl) TriggerDeploymentAfterTypeChange "err", err) } + deploymentConfigSelector := make([]*bean4.DeploymentConfigSelector, len(pipelineIds)) + for _, pipeline := range pipelines { + deploymentConfigSelector = append(deploymentConfigSelector, &bean4.DeploymentConfigSelector{ + AppId: pipeline.AppId, + EnvironmentId: pipeline.EnvironmentId, + }) + } + + deploymentConfigs, err := impl.deploymentConfigService.GetDeploymentConfigInBulk(deploymentConfigSelector) + if err != nil { + impl.logger.Errorw("error in getting deployment config", "deploymentConfigSelector", deploymentConfigSelector, "err", err) + return nil, err + } + updatedDeploymentConfigs := make([]*bean4.DeploymentConfig, 0) + for _, c := range deploymentConfigs { + c.DeploymentAppType = request.DesiredDeploymentType + updatedDeploymentConfigs = append(updatedDeploymentConfigs, c) + } + + updatedDeploymentConfigs, err = impl.deploymentConfigService.UpdateConfigs(nil, updatedDeploymentConfigs, request.UserId) + if err != nil { + impl.logger.Errorw("error in updating configs", "err", err) + return nil, err + } + err = impl.pipelineRepository.UpdateCdPipelineAfterDeployment(string(request.DesiredDeploymentType), successPipelines, request.UserId, false) From b01aa801814d5c5641292db79e3288bc14f1a20c Mon Sep 17 00:00:00 2001 From: ayushmaheshwari Date: Wed, 3 Jul 2024 17:23:25 +0530 Subject: [PATCH 14/45] remaining query changes --- internal/sql/repository/deploymentConfig/repository.go | 4 ++-- internal/sql/repository/pipelineConfig/PipelineRepository.go | 2 +- pkg/deployment/common/deploymentConfigService.go | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/internal/sql/repository/deploymentConfig/repository.go b/internal/sql/repository/deploymentConfig/repository.go index 2dd77296da..5956b83528 100644 --- a/internal/sql/repository/deploymentConfig/repository.go +++ b/internal/sql/repository/deploymentConfig/repository.go @@ -30,7 +30,7 @@ type Repository interface { UpdateAll(tx *pg.Tx, config []*DeploymentConfig) ([]*DeploymentConfig, error) GetById(id int) (*DeploymentConfig, error) GetByAppIdAndEnvId(appId, envId int) (*DeploymentConfig, error) - GetAppLevelConfig(appId int) (*DeploymentConfig, error) + GetAppLevelConfigForDevtronApps(appId int) (*DeploymentConfig, error) GetAppLevelConfigByAppIds(appIds []int) ([]*DeploymentConfig, error) GetAppAndEnvLevelConfigsInBulk(appIdToEnvIdsMap map[int][]int) ([]*DeploymentConfig, error) } @@ -99,7 +99,7 @@ func (impl RepositoryImpl) GetByAppIdAndEnvId(appId, envId int) (*DeploymentConf return result, err } -func (impl RepositoryImpl) GetAppLevelConfig(appId int) (*DeploymentConfig, error) { +func (impl RepositoryImpl) GetAppLevelConfigForDevtronApps(appId int) (*DeploymentConfig, error) { result := &DeploymentConfig{} err := impl.dbConnection.Model(result). Where("app_id = ? ", appId). diff --git a/internal/sql/repository/pipelineConfig/PipelineRepository.go b/internal/sql/repository/pipelineConfig/PipelineRepository.go index e705e29964..b17cb07b37 100644 --- a/internal/sql/repository/pipelineConfig/PipelineRepository.go +++ b/internal/sql/repository/pipelineConfig/PipelineRepository.go @@ -64,7 +64,7 @@ type Pipeline struct { RunPreStageInEnv bool `sql:"run_pre_stage_in_env"` // secret names RunPostStageInEnv bool `sql:"run_post_stage_in_env"` // secret names DeploymentAppCreated bool `sql:"deployment_app_created,notnull"` - DeploymentAppType string `sql:"deployment_app_type,notnull"` //helm, acd + DeploymentAppType string `sql:"deployment_app_type,notnull"` // Deprecated; DeploymentAppName string `sql:"deployment_app_name"` DeploymentAppDeleteRequest bool `sql:"deployment_app_delete_request,notnull"` Environment repository.Environment diff --git a/pkg/deployment/common/deploymentConfigService.go b/pkg/deployment/common/deploymentConfigService.go index 8dd8a5c1f8..5a04305be5 100644 --- a/pkg/deployment/common/deploymentConfigService.go +++ b/pkg/deployment/common/deploymentConfigService.go @@ -175,7 +175,7 @@ func (impl *DeploymentConfigServiceImpl) UpdateConfigs(tx *pg.Tx, configs []*bea func (impl *DeploymentConfigServiceImpl) GetDeploymentConfig(appId, envId int) (*bean.DeploymentConfig, error) { - appLevelConfigDbObj, err := impl.deploymentConfigRepository.GetAppLevelConfig(appId) + appLevelConfigDbObj, err := impl.deploymentConfigRepository.GetAppLevelConfigForDevtronApps(appId) if err != nil && err != pg.ErrNoRows { impl.logger.Errorw("error in getting deployment config db object by appId", "appId", appId, "err", err) return nil, err @@ -274,7 +274,7 @@ func (impl *DeploymentConfigServiceImpl) GetConfigDBObj(appId, envId int) (*depl var configDbObj *deploymentConfig.DeploymentConfig var err error if envId == 0 { - configDbObj, err = impl.deploymentConfigRepository.GetAppLevelConfig(appId) + configDbObj, err = impl.deploymentConfigRepository.GetAppLevelConfigForDevtronApps(appId) if err != nil { impl.logger.Errorw("error in getting deployment config db object by appId", "appId", configDbObj.AppId, "err", err) return nil, err From a3335bc94ad0f0642574316cd573c5976f956a7c Mon Sep 17 00:00:00 2001 From: ayushmaheshwari Date: Thu, 4 Jul 2024 21:03:35 +0530 Subject: [PATCH 15/45] PR review changes and logical fixes --- Wire.go | 6 +- .../app/appList/AppListingRestHandler.go | 6 +- env_gen.md | 2 +- .../repository/deploymentConfig/repository.go | 56 +++-- pkg/app/AppService.go | 19 +- .../status/PipelineStatusTimelineService.go | 4 +- pkg/appStore/bean/bean.go | 4 +- .../service/AppStoreDeploymentDBService.go | 11 +- .../service/AppStoreDeploymentService.go | 8 +- .../service/EAMode/InstalledAppDBService.go | 8 +- .../deployment/FullModeDeploymentService.go | 2 +- .../deployment/InstalledAppGitOpsService.go | 2 +- ...InstalledAppDeploymentTypeChangeService.go | 2 +- .../service/FullMode/resource/NotesService.go | 2 +- pkg/appWorkflow/AppWorkflowService.go | 2 +- pkg/chart/ChartService.go | 45 ++-- .../DevtronAppGitOpsConfigService.go | 2 +- pkg/deployment/common/adapter.go | 38 ++-- pkg/deployment/common/bean/bean.go | 21 +- .../common/deploymentConfigService.go | 209 +++++++++++------- .../manifest/ManifestCreationService.go | 8 - .../devtronApps/PostStageTriggerService.go | 10 +- .../devtronApps/PreStageTriggerService.go | 13 +- .../trigger/devtronApps/TriggerService.go | 69 +++--- ...eployedApplicationEventProcessorService.go | 4 + .../in/WorkflowEventProcessorService.go | 14 +- .../AppDeploymentTypeChangeManager.go | 120 +++++----- pkg/pipeline/CdHandler.go | 2 +- pkg/pipeline/CiCdPipelineOrchestrator.go | 4 +- .../DeploymentPipelineConfigService.go | 31 ++- pkg/workflow/cd/CdWorkflowCommonService.go | 4 +- pkg/workflow/dag/WorkflowDagExecutor.go | 9 +- pkg/workflow/status/WorkflowStatusService.go | 2 +- wire_gen.go | 54 ++--- 34 files changed, 425 insertions(+), 368 deletions(-) diff --git a/Wire.go b/Wire.go index 97f8acfd7b..18e59fdb02 100644 --- a/Wire.go +++ b/Wire.go @@ -208,13 +208,13 @@ func InitializeApp() (*App, error) { wire.Bind(new(gitSensor.Client), new(*gitSensor.ClientImpl)), // ------- helper.NewAppListingRepositoryQueryBuilder, - // sql.GetConfig, + // sql.GetConfigForDevtronApps, eClient.GetEventClientConfig, // sql.NewDbConnection, // app.GetACDAuthConfig, util3.GetACDAuthConfig, connection.SettingsManager, - // auth.GetConfig, + // auth.GetConfigForDevtronApps, connection.GetConfig, wire.Bind(new(session2.ServiceClient), new(*middleware.LoginService)), @@ -431,7 +431,7 @@ func InitializeApp() (*App, error) { wire.Bind(new(repository2.ServiceClient), new(*repository2.ServiceClientImpl)), wire.Bind(new(connector.Pump), new(*connector.PumpImpl)), - //app.GetConfig, + //app.GetConfigForDevtronApps, pipeline.GetEcrConfig, // otel.NewOtelTracingServiceImpl, diff --git a/api/restHandler/app/appList/AppListingRestHandler.go b/api/restHandler/app/appList/AppListingRestHandler.go index 7a9133e14c..ee8441de94 100644 --- a/api/restHandler/app/appList/AppListingRestHandler.go +++ b/api/restHandler/app/appList/AppListingRestHandler.go @@ -548,7 +548,7 @@ func (handler AppListingRestHandlerImpl) FetchAppDetails(w http.ResponseWriter, common.WriteJsonResp(w, fmt.Errorf("error in getting acd token"), nil, http.StatusInternalServerError) return } - envDeploymentConfig, err := handler.deploymentConfigService.GetDeploymentConfig(appId, envId) + envDeploymentConfig, err := handler.deploymentConfigService.GetConfigForDevtronApps(appId, envId) if err != nil { handler.logger.Errorw("error in fetching deployment config", "appId", appId, "envId", envId, "err", err) common.WriteJsonResp(w, fmt.Errorf("error in getting deployment config for env"), nil, http.StatusInternalServerError) @@ -647,7 +647,7 @@ func (handler AppListingRestHandlerImpl) FetchResourceTree(w http.ResponseWriter common.WriteJsonResp(w, fmt.Errorf("error in getting acd token"), nil, http.StatusInternalServerError) return } - envDeploymentConfig, err := handler.deploymentConfigService.GetDeploymentConfig(appId, envId) + envDeploymentConfig, err := handler.deploymentConfigService.GetConfigForDevtronApps(appId, envId) if err != nil { handler.logger.Errorw("error in fetching deployment config", "appId", appId, "envId", envId, "err", err) common.WriteJsonResp(w, fmt.Errorf("error in getting deployment config for env"), nil, http.StatusInternalServerError) @@ -961,7 +961,7 @@ func (handler AppListingRestHandlerImpl) GetHostUrlsByBatch(w http.ResponseWrite } cdPipeline := pipelines[0] - envDeploymentConfig, err := handler.deploymentConfigService.GetDeploymentConfig(appId, envId) + envDeploymentConfig, err := handler.deploymentConfigService.GetConfigForDevtronApps(appId, envId) if err != nil { handler.logger.Errorw("error in fetching deployment config", "appId", appId, "envId", envId, "err", err) common.WriteJsonResp(w, fmt.Errorf("error in getting deployment config for env"), nil, http.StatusInternalServerError) diff --git a/env_gen.md b/env_gen.md index 1ed120c5f6..a91e8b6655 100644 --- a/env_gen.md +++ b/env_gen.md @@ -245,8 +245,8 @@ | USE_BLOB_STORAGE_CONFIG_IN_CD_WORKFLOW | true | | | USE_BLOB_STORAGE_CONFIG_IN_CI_WORKFLOW | true | | | USE_BUILDX | false | | - | USE_CUSTOM_HTTP_TRANSPORT | false | | | USE_CASBIN_V2 | false | | + | USE_CUSTOM_HTTP_TRANSPORT | false | | | USE_EXTERNAL_NODE | false | | | USE_GIT_CLI | false | | | USE_IMAGE_TAG_FROM_GIT_PROVIDER_FOR_TAG_BASED_BUILD | false | | diff --git a/internal/sql/repository/deploymentConfig/repository.go b/internal/sql/repository/deploymentConfig/repository.go index 5956b83528..bd0e522f92 100644 --- a/internal/sql/repository/deploymentConfig/repository.go +++ b/internal/sql/repository/deploymentConfig/repository.go @@ -6,20 +6,31 @@ import ( "github.com/go-pg/pg/orm" ) +type DeploymentAppType string + +const ( + Argo DeploymentAppType = "argo_cd" + Helm DeploymentAppType = "helm" +) + +type ConfigType string + +const ( + Custom ConfigType = "custom" + SystemGenerated = "system_generated" +) + type DeploymentConfig struct { - tableName struct{} `sql:"deployment_config" pg:",discard_unknown_columns"` - Id int `sql:"id,pk"` - AppId int `sql:"app_id"` - EnvironmentId int `sql:"environment_id"` - DeploymentAppType string `sql:"deployment_app_type"` - ConfigType string `sql:"config_type"` - RepoUrl string `sql:"repo_url"` - RepoName string `sql:"repo_name"` - ChartLocation string `sql:"chart_location"` - CredentialType string `sql:"credential_type"` - CredentialIdInt int `sql:"credential_id_int"` - CredentialIdString string `sql:"credential_id_string"` - Active bool `sql:"active,notnull"` + tableName struct{} `sql:"deployment_config" pg:",discard_unknown_columns"` + Id int `sql:"id,pk"` + AppId int `sql:"app_id"` + EnvironmentId int `sql:"environment_id"` + DeploymentAppType string `sql:"deployment_app_type"` + ConfigType string `sql:"config_type"` + RepoUrl string `sql:"repo_url"` + RepoName string `sql:"repo_name"` + ChartLocation string `sql:"chart_location"` + Active bool `sql:"active,notnull"` sql.AuditLog } @@ -33,6 +44,7 @@ type Repository interface { GetAppLevelConfigForDevtronApps(appId int) (*DeploymentConfig, error) GetAppLevelConfigByAppIds(appIds []int) ([]*DeploymentConfig, error) GetAppAndEnvLevelConfigsInBulk(appIdToEnvIdsMap map[int][]int) ([]*DeploymentConfig, error) + GetByAppIdAndEnvIdEvenIfInactive(appId, envId int) (*DeploymentConfig, error) } type RepositoryImpl struct { @@ -95,6 +107,7 @@ func (impl RepositoryImpl) GetByAppIdAndEnvId(appId, envId int) (*DeploymentConf Where("app_id = ?", appId). Where("environment_id = ? ", envId). Where("active = ?", true). + Order("id DESC").Limit(1). Select() return result, err } @@ -134,3 +147,20 @@ func (impl RepositoryImpl) GetAppAndEnvLevelConfigsInBulk(appIdToEnvIdsMap map[i Select() return result, err } + +func (impl RepositoryImpl) GetByAppIdAndEnvIdEvenIfInactive(appId, envId int) (*DeploymentConfig, error) { + result := &DeploymentConfig{} + err := impl.dbConnection.Model(result). + WhereGroup(func(query *orm.Query) (*orm.Query, error) { + query = query.Where("app_id = ?", appId) + if envId == 0 { + query = query.Where("environment_id is NULL") + } else { + query = query.Where("environment_id = ? ", envId) + } + return query, nil + }). + Order("id DESC").Limit(1). + Select() + return result, err +} diff --git a/pkg/app/AppService.go b/pkg/app/AppService.go index a43e7a0b20..fb0cea3dc3 100644 --- a/pkg/app/AppService.go +++ b/pkg/app/AppService.go @@ -40,7 +40,6 @@ import ( "strconv" "time" - application2 "github.com/argoproj/argo-cd/v2/pkg/apiclient/application" "github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1" "github.com/caarlos0/env" k8sCommonBean "github.com/devtron-labs/common-lib/utils/k8s/commonBean" @@ -1040,22 +1039,6 @@ func (impl *AppServiceImpl) GetConfigMapAndSecretJson(appId int, envId int, pipe return merged, nil } -func (impl *AppServiceImpl) synchCD(pipeline *pipelineConfig.Pipeline, ctx context.Context, - overrideRequest *bean.ValuesOverrideRequest, envOverride *chartConfig.EnvConfigOverride) { - req := new(application2.ApplicationSyncRequest) - pipelineName := pipeline.App.AppName + "-" + envOverride.Environment.Name - req.Name = &pipelineName - prune := true - req.Prune = &prune - if ctx == nil { - impl.logger.Errorw("err in syncing ACD, ctx is NULL", "pipelineId", overrideRequest.PipelineId) - return - } - if _, err := impl.acdClient.Sync(ctx, req); err != nil { - impl.logger.Errorw("err in syncing ACD", "pipelineId", overrideRequest.PipelineId, "err", err) - } -} - type DeploymentEvent struct { ApplicationId int EnvironmentId int @@ -1155,7 +1138,7 @@ func (impl *AppServiceImpl) UpdateCdWorkflowRunnerByACDObject(app *v1alpha1.Appl } appId := wfr.CdWorkflow.Pipeline.AppId envId := wfr.CdWorkflow.Pipeline.EnvironmentId - envDeploymentConfig, err := impl.deploymentConfigService.GetDeploymentConfig(appId, envId) + envDeploymentConfig, err := impl.deploymentConfigService.GetConfigForDevtronApps(appId, envId) if err != nil { impl.logger.Errorw("error in fetching environment deployment config by appId and envId", "appId", appId, "envId", envId, "err", err) return err diff --git a/pkg/app/status/PipelineStatusTimelineService.go b/pkg/app/status/PipelineStatusTimelineService.go index 2706e577df..ae0bd986c1 100644 --- a/pkg/app/status/PipelineStatusTimelineService.go +++ b/pkg/app/status/PipelineStatusTimelineService.go @@ -225,7 +225,7 @@ func (impl *PipelineStatusTimelineServiceImpl) FetchTimelines(appId, envId, wfrI triggeredBy = wfr.TriggeredBy wfrStatus = wfr.Status - envDeploymentConfig, err := impl.deploymentConfigService.GetDeploymentConfig(appId, envId) + envDeploymentConfig, err := impl.deploymentConfigService.GetConfigForDevtronApps(appId, envId) if err != nil { impl.logger.Errorw("error in fetching environment deployment config by appId and envId", "appId", appId, "envId", envId, "err", err) return nil, err @@ -302,7 +302,7 @@ func (impl *PipelineStatusTimelineServiceImpl) FetchTimelinesForAppStore(install impl.logger.Errorw("error in getting installed_app_version by appId and envId", "err", err, "appId", installedAppId, "envId", envId) return nil, err } - deploymentConfig, err := impl.deploymentConfigService.GetDeploymentConfigForHelmApp(installedAppVersion.InstalledApp.AppId, installedAppVersion.InstalledApp.EnvironmentId) + deploymentConfig, err := impl.deploymentConfigService.GetConfigForHelmApps(installedAppVersion.InstalledApp.AppId, installedAppVersion.InstalledApp.EnvironmentId) if err != nil { impl.logger.Errorw("error in getiting deployment config db object by appId and envId", "appId", installedAppVersion.InstalledApp.AppId, "envId", installedAppVersion.InstalledApp.EnvironmentId, "err", err) return nil, err diff --git a/pkg/appStore/bean/bean.go b/pkg/appStore/bean/bean.go index c02c8af36a..84a7a32360 100644 --- a/pkg/appStore/bean/bean.go +++ b/pkg/appStore/bean/bean.go @@ -172,7 +172,7 @@ type InstallAppVersionChartRepoDTO struct { Password string `json:"-"` } -func (chart *InstallAppVersionDTO) GetDeploymentConfig(gitOpsConfigId int) *bean2.DeploymentConfig { +func (chart *InstallAppVersionDTO) GetDeploymentConfig() *bean2.DeploymentConfig { var configType string if chart.IsCustomRepository { configType = bean2.CUSTOM.String() @@ -186,8 +186,6 @@ func (chart *InstallAppVersionDTO) GetDeploymentConfig(gitOpsConfigId int) *bean DeploymentAppType: chart.DeploymentAppType, RepoURL: chart.GitOpsRepoURL, RepoName: gitUtil.GetGitRepoNameFromGitRepoUrl(chart.GitOpsRepoURL), - CredentialType: bean2.GitOps.String(), - CredentialIdInt: gitOpsConfigId, Active: true, } } diff --git a/pkg/appStore/installedApp/service/AppStoreDeploymentDBService.go b/pkg/appStore/installedApp/service/AppStoreDeploymentDBService.go index 5ea15e2c41..9db52d0d06 100644 --- a/pkg/appStore/installedApp/service/AppStoreDeploymentDBService.go +++ b/pkg/appStore/installedApp/service/AppStoreDeploymentDBService.go @@ -212,14 +212,7 @@ func (impl *AppStoreDeploymentDBServiceImpl) AppStoreDeployOperationDB(installRe installRequest.InstalledAppId = installedApp.Id // Stage 3: ends - //save deployment config - gitOpsConfig, err := impl.gitOpsConfigReadService.GetGitOpsConfigActive() - if err != nil { - impl.logger.Errorw("error in getting active gitops config", "err", err) - return nil, err - } - - deploymentConfig := installRequest.GetDeploymentConfig(gitOpsConfig.Id) + deploymentConfig := installRequest.GetDeploymentConfig() deploymentConfig, err = impl.deploymentConfigService.CreateOrUpdateConfig(tx, deploymentConfig, installRequest.UserId) if err != nil { impl.logger.Errorw("error in creating deployment config for installed app", "appId", installedApp.AppId, "envId", installedApp.EnvironmentId, "err", err) @@ -311,7 +304,7 @@ func (impl *AppStoreDeploymentDBServiceImpl) GetInstalledApp(id int) (*appStoreB impl.logger.Errorw("error while fetching from db", "error", err) return nil, err } - deploymentConfig, err := impl.deploymentConfigService.GetDeploymentConfigForHelmApp(app.AppId, app.EnvironmentId) + deploymentConfig, err := impl.deploymentConfigService.GetConfigForHelmApps(app.AppId, app.EnvironmentId) if err != nil { impl.logger.Errorw("error in getiting deployment config db object by appId and envId", "appId", app.AppId, "envId", app.EnvironmentId, "err", err) return nil, err diff --git a/pkg/appStore/installedApp/service/AppStoreDeploymentService.go b/pkg/appStore/installedApp/service/AppStoreDeploymentService.go index 2db93e7759..464adb2496 100644 --- a/pkg/appStore/installedApp/service/AppStoreDeploymentService.go +++ b/pkg/appStore/installedApp/service/AppStoreDeploymentService.go @@ -250,7 +250,7 @@ func (impl *AppStoreDeploymentServiceImpl) DeleteInstalledApp(ctx context.Contex return nil, err } - deploymentConfig, err := impl.deploymentConfigService.GetDeploymentConfigForHelmApp(model.AppId, model.EnvironmentId) + deploymentConfig, err := impl.deploymentConfigService.GetConfigForHelmApps(model.AppId, model.EnvironmentId) if err != nil { impl.logger.Errorw("error in getiting deployment config db object by appId and envId", "appId", model.AppId, "envId", model.EnvironmentId, "err", err) return nil, err @@ -458,7 +458,7 @@ func (impl *AppStoreDeploymentServiceImpl) RollbackApplication(ctx context.Conte impl.logger.Errorw("error while fetching installed app", "error", err) return false, err } - deploymentConfig, err := impl.deploymentConfigService.GetDeploymentConfigForHelmApp(installedApp.AppId, installedApp.EnvironmentId) + deploymentConfig, err := impl.deploymentConfigService.GetAndMigrateConfigIfAbsentForHelmApp(installedApp.AppId, installedApp.EnvironmentId) if err != nil { impl.logger.Errorw("error in getiting deployment config db object by appId and envId", "appId", installedApp.AppId, "envId", installedApp.EnvironmentId, "err", err) return false, err @@ -613,7 +613,7 @@ func (impl *AppStoreDeploymentServiceImpl) UpdateInstalledApp(ctx context.Contex installedApp.Environment.ClusterId: installedApp.Environment.Namespace, } - deploymentConfig, err := impl.deploymentConfigService.GetDeploymentConfigForHelmApp(installedApp.AppId, installedApp.EnvironmentId) + deploymentConfig, err := impl.deploymentConfigService.GetAndMigrateConfigIfAbsentForHelmApp(installedApp.AppId, installedApp.EnvironmentId) if err != nil { impl.logger.Errorw("error in getting deploymentConfig by appId and envId", "appId", installedApp.AppId, "envId", installedApp.EnvironmentId, "err", err) return nil, err @@ -863,7 +863,7 @@ func (impl *AppStoreDeploymentServiceImpl) MarkGitOpsInstalledAppsDeletedIfArgoA apiError.InternalMessage = "error in fetching partially deleted argoCd apps from installed app repo" return apiError } - deploymentConfig, err := impl.deploymentConfigService.GetDeploymentConfig(installedAppId, envId) + deploymentConfig, err := impl.deploymentConfigService.GetConfigForHelmApps(installedAppId, envId) if err != nil { impl.logger.Errorw("error in getting deployment config by appId and envId", "appId", installedAppId, "envId", envId, "err", err) apiError.HttpStatusCode = http.StatusInternalServerError diff --git a/pkg/appStore/installedApp/service/EAMode/InstalledAppDBService.go b/pkg/appStore/installedApp/service/EAMode/InstalledAppDBService.go index 2e83752ab5..e3e9d2a6f7 100644 --- a/pkg/appStore/installedApp/service/EAMode/InstalledAppDBService.go +++ b/pkg/appStore/installedApp/service/EAMode/InstalledAppDBService.go @@ -181,7 +181,7 @@ func (impl *InstalledAppDBServiceImpl) FindAppDetailsForAppstoreApplication(inst return bean2.AppDetailContainer{}, err } - deploymentConfig, err := impl.deploymentConfigService.GetDeploymentConfigForHelmApp(installedAppVerison.InstalledApp.AppId, installedAppVerison.InstalledApp.EnvironmentId) + deploymentConfig, err := impl.deploymentConfigService.GetConfigForHelmApps(installedAppVerison.InstalledApp.AppId, installedAppVerison.InstalledApp.EnvironmentId) if err != nil { impl.Logger.Errorw("error in getiting deployment config db object by appId and envId", "appId", installedAppVerison.InstalledApp.AppId, "envId", installedAppVerison.InstalledApp.EnvironmentId, "err", err) return bean2.AppDetailContainer{}, err @@ -258,7 +258,7 @@ func (impl *InstalledAppDBServiceImpl) GetInstalledAppByClusterNamespaceAndName( if err != nil { return nil, err } - deploymentConfig, err := impl.deploymentConfigService.GetDeploymentConfigForHelmApp(installedApp.AppId, installedApp.EnvironmentId) + deploymentConfig, err := impl.deploymentConfigService.GetConfigForHelmApps(installedApp.AppId, installedApp.EnvironmentId) if err != nil { impl.Logger.Errorw("error in getiting deployment config db object by appId and envId", "appId", installedApp.AppId, "envId", installedApp.EnvironmentId, "err", err) return nil, err @@ -275,7 +275,7 @@ func (impl *InstalledAppDBServiceImpl) GetInstalledAppByInstalledAppId(installed return nil, err } installedApp := &installedAppVersion.InstalledApp - deploymentConfig, err := impl.deploymentConfigService.GetDeploymentConfigForHelmApp(installedApp.AppId, installedApp.EnvironmentId) + deploymentConfig, err := impl.deploymentConfigService.GetConfigForHelmApps(installedApp.AppId, installedApp.EnvironmentId) if err != nil { impl.Logger.Errorw("error in getiting deployment config db object by appId and envId", "appId", installedApp.AppId, "envId", installedApp.EnvironmentId, "err", err) return nil, err @@ -292,7 +292,7 @@ func (impl *InstalledAppDBServiceImpl) GetInstalledAppVersion(id int, userId int impl.Logger.Errorw("error while fetching from db", "error", err) return nil, err } - deploymentConfig, err := impl.deploymentConfigService.GetDeploymentConfigForHelmApp(model.InstalledApp.AppId, model.InstalledApp.EnvironmentId) + deploymentConfig, err := impl.deploymentConfigService.GetAndMigrateConfigIfAbsentForHelmApp(model.InstalledApp.AppId, model.InstalledApp.EnvironmentId) if err != nil { impl.Logger.Errorw("error in getiting deployment config db object by appId and envId", "appId", model.InstalledApp.AppId, "envId", model.InstalledApp.EnvironmentId, "err", err) return nil, err diff --git a/pkg/appStore/installedApp/service/FullMode/deployment/FullModeDeploymentService.go b/pkg/appStore/installedApp/service/FullMode/deployment/FullModeDeploymentService.go index 0a3741196b..dadcb89dbf 100644 --- a/pkg/appStore/installedApp/service/FullMode/deployment/FullModeDeploymentService.go +++ b/pkg/appStore/installedApp/service/FullMode/deployment/FullModeDeploymentService.go @@ -248,7 +248,7 @@ func (impl *FullModeDeploymentServiceImpl) RollbackRelease(ctx context.Context, return installedApp, false, err } - deploymentConfig, err := impl.deploymentConfigService.GetDeploymentConfigForHelmApp(installedApp.AppId, installedApp.EnvironmentId) + deploymentConfig, err := impl.deploymentConfigService.GetAndMigrateConfigIfAbsentForHelmApp(installedApp.AppId, installedApp.EnvironmentId) if err != nil { impl.Logger.Errorw("error in getiting deployment config db object by appId and envId", "appId", installedApp.AppId, "envId", installedApp.EnvironmentId, "err", err) return installedApp, false, err diff --git a/pkg/appStore/installedApp/service/FullMode/deployment/InstalledAppGitOpsService.go b/pkg/appStore/installedApp/service/FullMode/deployment/InstalledAppGitOpsService.go index 2b04fdf907..eff7c27a92 100644 --- a/pkg/appStore/installedApp/service/FullMode/deployment/InstalledAppGitOpsService.go +++ b/pkg/appStore/installedApp/service/FullMode/deployment/InstalledAppGitOpsService.go @@ -320,7 +320,7 @@ func (impl *FullModeDeploymentServiceImpl) getGitCommitConfig(installAppVersionR apiErr := &util.ApiError{HttpStatusCode: http.StatusNotFound, Code: strconv.Itoa(http.StatusNotFound), InternalMessage: "Invalid request! No InstalledApp found.", UserMessage: "Invalid request! No InstalledApp found."} return nil, apiErr } - deploymentConfig, err := impl.deploymentConfigService.GetDeploymentConfigForHelmApp(InstalledApp.AppId, InstalledApp.EnvironmentId) + deploymentConfig, err := impl.deploymentConfigService.GetConfigForHelmApps(InstalledApp.AppId, InstalledApp.EnvironmentId) if err != nil { impl.Logger.Errorw("error in getiting deployment config db object by appId and envId", "appId", InstalledApp.AppId, "envId", InstalledApp.EnvironmentId, "err", err) return nil, err diff --git a/pkg/appStore/installedApp/service/FullMode/deploymentTypeChange/InstalledAppDeploymentTypeChangeService.go b/pkg/appStore/installedApp/service/FullMode/deploymentTypeChange/InstalledAppDeploymentTypeChangeService.go index 08036acbf7..ccac059316 100644 --- a/pkg/appStore/installedApp/service/FullMode/deploymentTypeChange/InstalledAppDeploymentTypeChangeService.go +++ b/pkg/appStore/installedApp/service/FullMode/deploymentTypeChange/InstalledAppDeploymentTypeChangeService.go @@ -329,7 +329,7 @@ func (impl *InstalledAppDeploymentTypeChangeServiceImpl) deleteInstalledApps(ctx for _, installedApp := range installedApps { installedApp.Environment.Cluster = cluster - deploymentConfig, err := impl.deploymentConfigService.GetDeploymentConfigForHelmApp(installedApp.AppId, installedApp.EnvironmentId) + deploymentConfig, err := impl.deploymentConfigService.GetAndMigrateConfigIfAbsentForHelmApp(installedApp.AppId, installedApp.EnvironmentId) if err != nil { impl.logger.Errorw("error in getiting deployment config db object by appId and envId", "appId", installedApp.AppId, "envId", installedApp.EnvironmentId, "err", err) return nil, err diff --git a/pkg/appStore/installedApp/service/FullMode/resource/NotesService.go b/pkg/appStore/installedApp/service/FullMode/resource/NotesService.go index 0bdcdc91ab..d8f4452635 100644 --- a/pkg/appStore/installedApp/service/FullMode/resource/NotesService.go +++ b/pkg/appStore/installedApp/service/FullMode/resource/NotesService.go @@ -83,7 +83,7 @@ func (impl *InstalledAppResourceServiceImpl) findNotesForArgoApplication(install } var notes string - deploymentConfig, err := impl.deploymentConfigurationService.GetDeploymentConfigForHelmApp(installedAppVerison.InstalledApp.AppId, installedAppVerison.InstalledApp.EnvironmentId) + deploymentConfig, err := impl.deploymentConfigurationService.GetConfigForHelmApps(installedAppVerison.InstalledApp.AppId, installedAppVerison.InstalledApp.EnvironmentId) if err != nil { impl.logger.Errorw("error in getiting deployment config db object by appId and envId", "appId", installedAppVerison.InstalledApp.AppId, "envId", installedAppVerison.InstalledApp.EnvironmentId, "err", err) return "", err diff --git a/pkg/appWorkflow/AppWorkflowService.go b/pkg/appWorkflow/AppWorkflowService.go index 184473c26e..5f76759cf7 100644 --- a/pkg/appWorkflow/AppWorkflowService.go +++ b/pkg/appWorkflow/AppWorkflowService.go @@ -857,7 +857,7 @@ func (impl AppWorkflowServiceImpl) FindCdPipelinesByAppId(appId int) (*bean.CdPi for _, pipeline := range dbPipelines { - envDeploymentConfig, err := impl.deploymentConfigService.GetDeploymentConfig(appId, pipeline.EnvironmentId) + envDeploymentConfig, err := impl.deploymentConfigService.GetConfigForDevtronApps(appId, pipeline.EnvironmentId) if err != nil { impl.Logger.Errorw("error in fetching environment deployment config by appId and envId", "appId", appId, "envId", pipeline.EnvironmentId, "err", err) return nil, err diff --git a/pkg/chart/ChartService.go b/pkg/chart/ChartService.go index fcd31aa757..64176a673a 100644 --- a/pkg/chart/ChartService.go +++ b/pkg/chart/ChartService.go @@ -281,21 +281,12 @@ func (impl *ChartServiceImpl) Create(templateRequest TemplateRequest, ctx contex return nil, err } - // create deployment config - activeGitOpsConfig, err := impl.gitOpsConfigReadService.GetGitOpsConfigActive() - if err != nil { - impl.logger.Errorw("error in getting active gitOps config", "err", err) - return nil, err - } - deploymentConfig := &bean2.DeploymentConfig{ - AppId: templateRequest.AppId, - ConfigType: common.GetDeploymentConfigType(templateRequest.IsCustomGitRepository), - RepoURL: gitRepoUrl, - ChartLocation: chartLocation, - CredentialType: bean2.GitOps.String(), - CredentialIdInt: activeGitOpsConfig.Id, - Active: true, + AppId: templateRequest.AppId, + ConfigType: common.GetDeploymentConfigType(templateRequest.IsCustomGitRepository), + RepoURL: gitRepoUrl, + ChartLocation: chartLocation, + Active: true, } deploymentConfig, err = impl.deploymentConfigService.CreateOrUpdateConfig(nil, deploymentConfig, templateRequest.UserId) if err != nil { @@ -375,7 +366,7 @@ func (impl *ChartServiceImpl) CreateChartFromEnvOverride(templateRequest Templat return nil, err } - currentDeploymentConfig, err := impl.deploymentConfigService.GetDeploymentConfig(templateRequest.AppId, 0) + currentDeploymentConfig, err := impl.deploymentConfigService.GetAndMigrateConfigIfAbsentForDevtronApps(templateRequest.AppId, 0) if err != nil && err != pg.ErrNoRows { return nil, err } @@ -386,20 +377,12 @@ func (impl *ChartServiceImpl) CreateChartFromEnvOverride(templateRequest Templat gitRepoUrl = currentDeploymentConfig.RepoURL } - // create deployment config - activeGitOpsConfig, err := impl.gitOpsConfigReadService.GetGitOpsConfigActive() - if err != nil { - impl.logger.Errorw("error in getting active gitOps config", "err", err) - return nil, err - } deploymentConfig := &bean2.DeploymentConfig{ - AppId: templateRequest.AppId, - ConfigType: common.GetDeploymentConfigType(templateRequest.IsCustomGitRepository), - RepoURL: gitRepoUrl, - ChartLocation: chartLocation, - CredentialType: bean2.GitOps.String(), - CredentialIdInt: activeGitOpsConfig.Id, - Active: true, + AppId: templateRequest.AppId, + ConfigType: common.GetDeploymentConfigType(templateRequest.IsCustomGitRepository), + RepoURL: gitRepoUrl, + ChartLocation: chartLocation, + Active: true, } deploymentConfig, err = impl.deploymentConfigService.CreateOrUpdateConfig(nil, deploymentConfig, templateRequest.UserId) if err != nil { @@ -588,7 +571,7 @@ func (impl *ChartServiceImpl) FindLatestChartForAppByAppId(appId int) (chartTemp return nil, err } - deploymentConfig, err := impl.deploymentConfigService.GetDeploymentConfig(appId, 0) + deploymentConfig, err := impl.deploymentConfigService.GetConfigForDevtronApps(appId, 0) if err != nil { impl.logger.Errorw("error in fetching deployment config by appId", "appId", appId, "err", err) return nil, err @@ -614,7 +597,7 @@ func (impl *ChartServiceImpl) GetByAppIdAndChartRefId(appId int, chartRefId int) impl.logger.Errorw("error in fetching app-metrics", "appId", appId, "err", err) return nil, err } - deploymentConfig, err := impl.deploymentConfigService.GetDeploymentConfig(appId, 0) + deploymentConfig, err := impl.deploymentConfigService.GetConfigForDevtronApps(appId, 0) if err != nil { impl.logger.Errorw("error in fetching deployment config by appId", "appId", appId, "err", err) return nil, err @@ -850,7 +833,7 @@ func (impl *ChartServiceImpl) FindPreviousChartByAppId(appId int) (chartTemplate impl.logger.Errorw("error in fetching chart ", "appId", appId, "err", err) return nil, err } - deploymentConfig, err := impl.deploymentConfigService.GetDeploymentConfig(appId, 0) + deploymentConfig, err := impl.deploymentConfigService.GetConfigForDevtronApps(appId, 0) if err != nil { impl.logger.Errorw("error in fetching deployment config by appId", "appId", appId, "err", err) return nil, err diff --git a/pkg/chart/gitOpsConfig/DevtronAppGitOpsConfigService.go b/pkg/chart/gitOpsConfig/DevtronAppGitOpsConfigService.go index c5d568d99f..bbd28bca63 100644 --- a/pkg/chart/gitOpsConfig/DevtronAppGitOpsConfigService.go +++ b/pkg/chart/gitOpsConfig/DevtronAppGitOpsConfigService.go @@ -196,7 +196,7 @@ func (impl *DevtronAppGitOpConfigServiceImpl) GetAppLevelGitOpsConfiguration(app } func (impl *DevtronAppGitOpConfigServiceImpl) isGitRepoUrlPresent(appId int) bool { - deploymentConfig, err := impl.deploymentConfigService.GetDeploymentConfig(appId, 0) + deploymentConfig, err := impl.deploymentConfigService.GetConfigForDevtronApps(appId, 0) if err != nil { impl.logger.Errorw("error fetching git repo url from deploymentConfig for latest chart") return false diff --git a/pkg/deployment/common/adapter.go b/pkg/deployment/common/adapter.go index 6b8df190c8..8555000e70 100644 --- a/pkg/deployment/common/adapter.go +++ b/pkg/deployment/common/adapter.go @@ -7,32 +7,26 @@ import ( func ConvertDeploymentConfigDTOToDbObj(config *bean.DeploymentConfig) *deploymentConfig.DeploymentConfig { return &deploymentConfig.DeploymentConfig{ - Id: config.Id, - AppId: config.AppId, - EnvironmentId: config.EnvironmentId, - DeploymentAppType: config.DeploymentAppType, - ConfigType: config.ConfigType, - RepoUrl: config.RepoURL, - ChartLocation: config.ChartLocation, - CredentialType: config.CredentialType, - CredentialIdInt: config.CredentialIdInt, - CredentialIdString: config.CredentialIdString, - Active: config.Active, + Id: config.Id, + AppId: config.AppId, + EnvironmentId: config.EnvironmentId, + DeploymentAppType: config.DeploymentAppType, + ConfigType: config.ConfigType, + RepoUrl: config.RepoURL, + ChartLocation: config.ChartLocation, + Active: config.Active, } } func ConvertDeploymentConfigDbObjToDTO(dbObj *deploymentConfig.DeploymentConfig) *bean.DeploymentConfig { return &bean.DeploymentConfig{ - Id: dbObj.Id, - AppId: dbObj.AppId, - EnvironmentId: dbObj.EnvironmentId, - DeploymentAppType: dbObj.DeploymentAppType, - ConfigType: dbObj.ConfigType, - RepoURL: dbObj.RepoUrl, - ChartLocation: dbObj.ChartLocation, - CredentialType: dbObj.CredentialType, - CredentialIdInt: dbObj.CredentialIdInt, - CredentialIdString: dbObj.CredentialIdString, - Active: dbObj.Active, + Id: dbObj.Id, + AppId: dbObj.AppId, + EnvironmentId: dbObj.EnvironmentId, + DeploymentAppType: dbObj.DeploymentAppType, + ConfigType: dbObj.ConfigType, + RepoURL: dbObj.RepoUrl, + ChartLocation: dbObj.ChartLocation, + Active: dbObj.Active, } } diff --git a/pkg/deployment/common/bean/bean.go b/pkg/deployment/common/bean/bean.go index df4a645e70..81bba2303c 100644 --- a/pkg/deployment/common/bean/bean.go +++ b/pkg/deployment/common/bean/bean.go @@ -7,18 +7,15 @@ import ( ) type DeploymentConfig struct { - Id int - AppId int - EnvironmentId int - ConfigType string - DeploymentAppType string - RepoURL string - RepoName string - ChartLocation string - CredentialType string - CredentialIdInt int - CredentialIdString string - Active bool + Id int + AppId int + EnvironmentId int + ConfigType string + DeploymentAppType string + RepoURL string + RepoName string + ChartLocation string + Active bool } type UniqueDeploymentConfigIdentifier string diff --git a/pkg/deployment/common/deploymentConfigService.go b/pkg/deployment/common/deploymentConfigService.go index 5a04305be5..6cc4fb5bdb 100644 --- a/pkg/deployment/common/deploymentConfigService.go +++ b/pkg/deployment/common/deploymentConfigService.go @@ -9,8 +9,6 @@ import ( bean3 "github.com/devtron-labs/devtron/pkg/auth/user/bean" chartRepoRepository "github.com/devtron-labs/devtron/pkg/chartRepo/repository" "github.com/devtron-labs/devtron/pkg/deployment/common/bean" - "github.com/devtron-labs/devtron/pkg/deployment/gitOps/config" - bean2 "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/bean" "github.com/go-pg/pg" "go.uber.org/zap" ) @@ -19,8 +17,11 @@ type DeploymentConfigService interface { CreateOrUpdateConfig(tx *pg.Tx, config *bean.DeploymentConfig, userId int32) (*bean.DeploymentConfig, error) CreateOrUpdateConfigsInBulk(tx *pg.Tx, configs []*bean.DeploymentConfig, userId int32) ([]*bean.DeploymentConfig, error) UpdateConfigs(tx *pg.Tx, configs []*bean.DeploymentConfig, userId int32) ([]*bean.DeploymentConfig, error) - GetDeploymentConfig(appId, envId int) (*bean.DeploymentConfig, error) - GetDeploymentConfigForHelmApp(appId, envId int) (*bean.DeploymentConfig, error) + GetConfigForDevtronApps(appId, envId int) (*bean.DeploymentConfig, error) + GetAndMigrateConfigIfAbsentForDevtronApps(appId, envId int) (*bean.DeploymentConfig, error) + GetConfigForHelmApps(appId, envId int) (*bean.DeploymentConfig, error) + GetConfigEvenIfInactive(appId, envId int) (*bean.DeploymentConfig, error) + GetAndMigrateConfigIfAbsentForHelmApp(appId, envId int) (*bean.DeploymentConfig, error) GetDeploymentConfigInBulk(configSelector []*bean.DeploymentConfigSelector) (map[bean.UniqueDeploymentConfigIdentifier]*bean.DeploymentConfig, error) } @@ -29,7 +30,6 @@ type DeploymentConfigServiceImpl struct { logger *zap.SugaredLogger chartRepository chartRepoRepository.ChartRepository pipelineRepository pipelineConfig.PipelineRepository - gitOpsConfigReadService config.GitOpsConfigReadService appRepository appRepository.AppRepository installedAppRepository repository.InstalledAppRepository } @@ -39,7 +39,6 @@ func NewDeploymentConfigServiceImpl( logger *zap.SugaredLogger, chartRepository chartRepoRepository.ChartRepository, pipelineRepository pipelineConfig.PipelineRepository, - gitOpsConfigReadService config.GitOpsConfigReadService, appRepository appRepository.AppRepository, installedAppRepository repository.InstalledAppRepository, ) *DeploymentConfigServiceImpl { @@ -48,7 +47,6 @@ func NewDeploymentConfigServiceImpl( logger: logger, chartRepository: chartRepository, pipelineRepository: pipelineRepository, - gitOpsConfigReadService: gitOpsConfigReadService, appRepository: appRepository, installedAppRepository: installedAppRepository, } @@ -173,7 +171,39 @@ func (impl *DeploymentConfigServiceImpl) UpdateConfigs(tx *pg.Tx, configs []*bea return updatedConfigs, nil } -func (impl *DeploymentConfigServiceImpl) GetDeploymentConfig(appId, envId int) (*bean.DeploymentConfig, error) { +func (impl *DeploymentConfigServiceImpl) GetConfigForDevtronApps(appId, envId int) (*bean.DeploymentConfig, error) { + appLevelConfigDbObj, err := impl.deploymentConfigRepository.GetAppLevelConfigForDevtronApps(appId) + if err != nil && err != pg.ErrNoRows { + impl.logger.Errorw("error in getting deployment config db object by appId", "appId", appId, "err", err) + return nil, err + } + if err == pg.ErrNoRows { + appLevelConfigDbObj, err = impl.parseAppLevelConfigForDevtronApps(appId) + if err != nil { + impl.logger.Errorw("error in migrating app level config to deployment config", "appId", appId, "err", err) + return nil, err + } + } + if envId > 0 { + // if envId>0 then only env level config will be returned, for getting app level config envId should be zero + appAndEnvLevelConfig, err := impl.deploymentConfigRepository.GetByAppIdAndEnvId(appId, envId) + if err != nil && err != pg.ErrNoRows { + impl.logger.Errorw("error in getting deployment config db object by appId and envId", "appId", appId, "envId", envId, "err", err) + return nil, err + } + if err == pg.ErrNoRows { + appAndEnvLevelConfig, err = impl.parseEnvLevelConfigForDevtronApps(appLevelConfigDbObj, appId, envId) + if err != nil { + impl.logger.Errorw("error in migrating app level config to deployment config", "appId", appId, "err", err) + return nil, err + } + } + return ConvertDeploymentConfigDbObjToDTO(appAndEnvLevelConfig), nil + } + return ConvertDeploymentConfigDbObjToDTO(appLevelConfigDbObj), nil +} + +func (impl *DeploymentConfigServiceImpl) GetAndMigrateConfigIfAbsentForDevtronApps(appId, envId int) (*bean.DeploymentConfig, error) { appLevelConfigDbObj, err := impl.deploymentConfigRepository.GetAppLevelConfigForDevtronApps(appId) if err != nil && err != pg.ErrNoRows { @@ -181,7 +211,8 @@ func (impl *DeploymentConfigServiceImpl) GetDeploymentConfig(appId, envId int) ( return nil, err } if err == pg.ErrNoRows { - appLevelConfigDbObj, err = impl.migrateAppLevelDataTODeploymentConfig(appId) + impl.logger.Infow("app level deployment config not found, migrating data from charts to deployment_config", "appId", appId, "err", err) + appLevelConfigDbObj, err = impl.migrateChartsDataToDeploymentConfig(appId) if err != nil { impl.logger.Errorw("error in migrating app level config to deployment config", "appId", appId, "err", err) return nil, err @@ -195,7 +226,8 @@ func (impl *DeploymentConfigServiceImpl) GetDeploymentConfig(appId, envId int) ( return nil, err } if err == pg.ErrNoRows { - appAndEnvLevelConfig, err = impl.migrateAppAndEnvLevelDataToDeploymentConfig(appLevelConfigDbObj, appId, envId) + impl.logger.Infow("env level deployment config not found, migrating data from pipeline to deployment_config", "appId", appId, "envId", envId, "err", err) + appAndEnvLevelConfig, err = impl.migrateDevtronAppsPipelineDataToDeploymentConfig(appLevelConfigDbObj, appId, envId) if err != nil { impl.logger.Errorw("error in migrating app level config to deployment config", "appId", appId, "err", err) return nil, err @@ -207,7 +239,23 @@ func (impl *DeploymentConfigServiceImpl) GetDeploymentConfig(appId, envId int) ( return ConvertDeploymentConfigDbObjToDTO(appLevelConfigDbObj), nil } -func (impl *DeploymentConfigServiceImpl) migrateAppLevelDataTODeploymentConfig(appId int) (*deploymentConfig.DeploymentConfig, error) { +func (impl *DeploymentConfigServiceImpl) migrateChartsDataToDeploymentConfig(appId int) (*deploymentConfig.DeploymentConfig, error) { + + configDbObj, err := impl.parseAppLevelConfigForDevtronApps(appId) + if err != nil { + impl.logger.Errorw("error in parsing charts data for devtron apps", "appId", appId, "err", err) + return nil, err + } + configDbObj.AuditLog.CreateAuditLog(1) + configDbObj, err = impl.deploymentConfigRepository.Save(nil, configDbObj) + if err != nil { + impl.logger.Errorw("error in saving deployment config in DB", "appId", appId, "err", err) + return nil, err + } + return configDbObj, nil +} + +func (impl *DeploymentConfigServiceImpl) parseAppLevelConfigForDevtronApps(appId int) (*deploymentConfig.DeploymentConfig, error) { chart, err := impl.chartRepository.FindLatestChartForAppByAppId(appId) if err != nil { impl.logger.Errorw("error in fetch chart for git repo migration by appId", "appId", appId, "err", err) @@ -220,16 +268,28 @@ func (impl *DeploymentConfigServiceImpl) migrateAppLevelDataTODeploymentConfig(a RepoUrl: chart.GitRepoUrl, ChartLocation: chart.ChartLocation, } - ConfigDbObj.AuditLog.CreateAuditLog(1) - ConfigDbObj, err = impl.deploymentConfigRepository.Save(nil, ConfigDbObj) + return ConfigDbObj, nil +} + +func (impl *DeploymentConfigServiceImpl) migrateDevtronAppsPipelineDataToDeploymentConfig(appLevelConfig *deploymentConfig.DeploymentConfig, appId int, envId int) (*deploymentConfig.DeploymentConfig, error) { + + configDbObj, err := impl.parseEnvLevelConfigForDevtronApps(appLevelConfig, appId, envId) if err != nil { - impl.logger.Errorw("error in saving deployment config in DB", "appId", appId, "err", err) + impl.logger.Errorw("error in parsing config for cd pipeline from appId and envId", "appId", appId, "envId", envId, "err", err) return nil, err } - return ConfigDbObj, nil + + configDbObj.AuditLog.CreateAuditLog(bean3.SYSTEM_USER_ID) + configDbObj, err = impl.deploymentConfigRepository.Save(nil, configDbObj) + if err != nil { + impl.logger.Errorw("error in saving deployment config in DB", "appId", appId, "envId", envId, "err", err) + return nil, err + } + + return configDbObj, nil } -func (impl *DeploymentConfigServiceImpl) migrateAppAndEnvLevelDataToDeploymentConfig(appLevelConfig *deploymentConfig.DeploymentConfig, appId int, envId int) (*deploymentConfig.DeploymentConfig, error) { +func (impl *DeploymentConfigServiceImpl) parseEnvLevelConfigForDevtronApps(appLevelConfig *deploymentConfig.DeploymentConfig, appId int, envId int) (*deploymentConfig.DeploymentConfig, error) { configDbObj := &deploymentConfig.DeploymentConfig{ AppId: appId, @@ -245,28 +305,8 @@ func (impl *DeploymentConfigServiceImpl) migrateAppAndEnvLevelDataToDeploymentCo impl.logger.Errorw("error in getting deployment app type by appId and envId", "appId", appId, "envId", envId, "err", err) return nil, err } - configDbObj.DeploymentAppType = deploymentAppType - switch configDbObj.DeploymentAppType { - //TODO: handling for other deployment app type in future - case bean2.ArgoCd: - configDbObj.CredentialType = bean.GitOps.String() - gitOpsConfig, err := impl.gitOpsConfigReadService.GetGitOpsProviderByRepoURL(configDbObj.RepoUrl) - if err != nil { - impl.logger.Infow("error in fetching gitOps config by repoUrl, skipping migration to deployment config", "repoURL", configDbObj.RepoUrl) - return configDbObj, err - } - configDbObj.CredentialIdInt = gitOpsConfig.Id - } - - configDbObj.AuditLog.CreateAuditLog(bean3.SYSTEM_USER_ID) - configDbObj, err = impl.deploymentConfigRepository.Save(nil, configDbObj) - if err != nil { - impl.logger.Errorw("error in saving deployment config in DB", "appId", appId, "envId", envId, "err", err) - return nil, err - } - return configDbObj, nil } @@ -289,7 +329,34 @@ func (impl *DeploymentConfigServiceImpl) GetConfigDBObj(appId, envId int) (*depl return configDbObj, nil } -func (impl *DeploymentConfigServiceImpl) GetDeploymentConfigForHelmApp(appId, envId int) (*bean.DeploymentConfig, error) { +func (impl *DeploymentConfigServiceImpl) GetConfigForHelmApps(appId, envId int) (*bean.DeploymentConfig, error) { + + helmDeploymentConfig, err := impl.deploymentConfigRepository.GetByAppIdAndEnvId(appId, envId) + if err != nil && err != pg.ErrNoRows { + impl.logger.Errorw("error in fetching deployment config by by appId and envId", "appId", appId, "envId", envId, "err", err) + return nil, err + } + + if err == pg.ErrNoRows { + helmDeploymentConfig, err = impl.parseConfigForHelmApps(appId, envId) + if err != nil { + impl.logger.Errorw("error in migrating helm deployment config", "appId", appId, "envId", envId, "err", err) + return nil, err + } + } + return ConvertDeploymentConfigDbObjToDTO(helmDeploymentConfig), nil +} + +func (impl *DeploymentConfigServiceImpl) GetConfigEvenIfInactive(appId, envId int) (*bean.DeploymentConfig, error) { + config, err := impl.deploymentConfigRepository.GetByAppIdAndEnvIdEvenIfInactive(appId, envId) + if err != nil { + impl.logger.Errorw("error in getting deployment config by appId and envId", "appId", appId, "envId", envId, "err", err) + return nil, err + } + return ConvertDeploymentConfigDbObjToDTO(config), nil +} + +func (impl *DeploymentConfigServiceImpl) GetAndMigrateConfigIfAbsentForHelmApp(appId, envId int) (*bean.DeploymentConfig, error) { helmDeploymentConfig, err := impl.deploymentConfigRepository.GetByAppIdAndEnvId(appId, envId) if err != nil && err != pg.ErrNoRows { @@ -309,12 +376,27 @@ func (impl *DeploymentConfigServiceImpl) GetDeploymentConfigForHelmApp(appId, en func (impl *DeploymentConfigServiceImpl) migrateHelmAppDataToDeploymentConfig(appId, envId int) (*deploymentConfig.DeploymentConfig, error) { + helmDeploymentConfig, err := impl.parseConfigForHelmApps(appId, envId) + if err != nil { + impl.logger.Errorw("error in parsing deployment config for helm app", "appId", appId, "envId", envId, "err", err) + return helmDeploymentConfig, err + } + + helmDeploymentConfig.CreateAuditLog(bean3.SYSTEM_USER_ID) + helmDeploymentConfig, err = impl.deploymentConfigRepository.Save(nil, helmDeploymentConfig) + if err != nil { + impl.logger.Errorw("error in saving deployment config for helm app", "appId", appId, "envId", envId, "err", err) + return nil, err + } + return helmDeploymentConfig, nil +} + +func (impl *DeploymentConfigServiceImpl) parseConfigForHelmApps(appId int, envId int) (*deploymentConfig.DeploymentConfig, error) { installedApp, err := impl.installedAppRepository.GetInstalledAppsByAppId(appId) if err != nil { impl.logger.Errorw("error in getting installed app by appId", "appId", appId, "err", err) return nil, err } - helmDeploymentConfig := &deploymentConfig.DeploymentConfig{ AppId: appId, EnvironmentId: envId, @@ -324,23 +406,6 @@ func (impl *DeploymentConfigServiceImpl) migrateHelmAppDataToDeploymentConfig(ap RepoName: installedApp.GitOpsRepoName, Active: true, } - - switch helmDeploymentConfig.DeploymentAppType { - case bean2.ArgoCd: - gitOpsConfig, err := impl.gitOpsConfigReadService.GetGitOpsProviderByRepoURL(installedApp.GitOpsRepoUrl) - if err != nil { - impl.logger.Infow("error in fetching gitOps config by repoUrl, skipping migration to deployment config", "repoURL", installedApp.GitOpsRepoUrl) - return nil, err - } - helmDeploymentConfig.ConfigType = bean.SYSTEM_GENERATED.String() - helmDeploymentConfig.CredentialIdInt = gitOpsConfig.Id - } - helmDeploymentConfig.CreateAuditLog(bean3.SYSTEM_USER_ID) - helmDeploymentConfig, err = impl.deploymentConfigRepository.Save(nil, helmDeploymentConfig) - if err != nil { - impl.logger.Errorw("error in saving deployment config for helm app", "appId", appId, "envId", envId, "err", err) - return nil, err - } return helmDeploymentConfig, nil } @@ -516,22 +581,14 @@ func (impl *DeploymentConfigServiceImpl) migrateAppLevelDataTODeploymentConfigIn allRepoUrls = append(allRepoUrls, c.GitRepoUrl) } - repoUrlToGitOpsConfigMapping, err := impl.gitOpsConfigReadService.GetGitOpsProviderMapByRepoURL(allRepoUrls) - if err != nil { - impl.logger.Errorw("error in fetching repoUrl to config mapping", "err", err) - return nil, err - } - configDBObjects := make([]*deploymentConfig.DeploymentConfig, 0, len(appIds)) for _, c := range charts { dbObj := &deploymentConfig.DeploymentConfig{ - ConfigType: GetDeploymentConfigType(c.IsCustomGitRepository), - AppId: c.AppId, - Active: true, - RepoUrl: c.GitRepoUrl, - ChartLocation: c.ChartLocation, - CredentialType: bean.GitOps.String(), - CredentialIdInt: repoUrlToGitOpsConfigMapping[c.GitRepoUrl].Id, + ConfigType: GetDeploymentConfigType(c.IsCustomGitRepository), + AppId: c.AppId, + Active: true, + RepoUrl: c.GitRepoUrl, + ChartLocation: c.ChartLocation, } dbObj.AuditLog.CreateAuditLog(bean3.SYSTEM_USER_ID) configDBObjects = append(configDBObjects, dbObj) @@ -589,11 +646,6 @@ func (impl *DeploymentConfigServiceImpl) migrateEnvLevelDataToDeploymentConfigIn Active: true, DeploymentAppType: deploymentAppType, } - switch deploymentAppType { - case bean2.ArgoCd: - configDbObj.CredentialType = appLevelConfig.CredentialType - configDbObj.CredentialIdInt = appLevelConfig.CredentialIdInt - } configDbObj.AuditLog.CreateAuditLog(bean3.SYSTEM_USER_ID) configDBObjects = append(configDBObjects, configDbObj) } @@ -673,12 +725,6 @@ func (impl *DeploymentConfigServiceImpl) migrateDeploymentConfigInBulkForHelmApp allRepoURLS = append(allRepoURLS, ia.GitOpsRepoUrl) } - repoUrlToGitOpsConfigMapping, err := impl.gitOpsConfigReadService.GetGitOpsProviderMapByRepoURL(allRepoURLS) - if err != nil { - impl.logger.Errorw("error in fetching repoUrl to config mapping", "err", err) - return nil, err - } - configDBObjects := make([]*deploymentConfig.DeploymentConfig, 0, len(notFoundDeploymentConfigMap)) for appEnvUniqueKey, _ := range notFoundDeploymentConfigMap { @@ -694,11 +740,6 @@ func (impl *DeploymentConfigServiceImpl) migrateDeploymentConfigInBulkForHelmApp RepoName: installedApp.GitOpsRepoName, Active: true, } - switch installedApp.DeploymentAppType { - case bean2.ArgoCd: - helmDeploymentConfig.CredentialType = bean.GitOps.String() - helmDeploymentConfig.CredentialIdInt = repoUrlToGitOpsConfigMapping[installedApp.GitOpsRepoUrl].Id - } helmDeploymentConfig.AuditLog.CreateAuditLog(bean3.SYSTEM_USER_ID) configDBObjects = append(configDBObjects, helmDeploymentConfig) } diff --git a/pkg/deployment/manifest/ManifestCreationService.go b/pkg/deployment/manifest/ManifestCreationService.go index 92c2952706..2de8dfd189 100644 --- a/pkg/deployment/manifest/ManifestCreationService.go +++ b/pkg/deployment/manifest/ManifestCreationService.go @@ -222,14 +222,6 @@ func (impl *ManifestCreationServiceImpl) GetValuesOverrideForTrigger(overrideReq } valuesOverrideResponse.EnvOverride = envOverride - envDeploymentConfig, err := impl.deploymentConfigService.GetDeploymentConfig(pipeline.AppId, pipeline.EnvironmentId) - if err != nil { - impl.logger.Errorw("error in getting deployment config by appId and envId", "appId", pipeline.AppId, "envId", pipeline.EnvironmentId, "err", err) - return valuesOverrideResponse, err - } - - valuesOverrideResponse.DeploymentConfig = envDeploymentConfig - // Conditional Block based on PipelineOverrideCreated --> start if !isPipelineOverrideCreated { pipelineOverride, err = impl.savePipelineOverride(newCtx, overrideRequest, envOverride.Id, triggeredAt) diff --git a/pkg/deployment/trigger/devtronApps/PostStageTriggerService.go b/pkg/deployment/trigger/devtronApps/PostStageTriggerService.go index a510b607ed..413e51b5fc 100644 --- a/pkg/deployment/trigger/devtronApps/PostStageTriggerService.go +++ b/pkg/deployment/trigger/devtronApps/PostStageTriggerService.go @@ -62,8 +62,14 @@ func (impl *TriggerServiceImpl) TriggerPostStage(request bean.TriggerRequest) er } } + envDevploymentConfig, err := impl.deploymentConfigService.GetAndMigrateConfigIfAbsentForDevtronApps(pipeline.AppId, pipeline.EnvironmentId) + if err != nil { + impl.logger.Errorw("error in fetching deployment config by appId and envId", "appId", pipeline.AppId, "envId", pipeline.EnvironmentId, "err", err) + return err + } + // custom GitOps repo url validation --> Start - err = impl.handleCustomGitOpsRepoValidation(runner, pipeline, triggeredBy) + err = impl.handleCustomGitOpsRepoValidation(runner, pipeline, envDevploymentConfig, triggeredBy) if err != nil { impl.logger.Errorw("custom GitOps repository validation error, TriggerPreStage", "err", err) return err @@ -75,7 +81,7 @@ func (impl *TriggerServiceImpl) TriggerPostStage(request bean.TriggerRequest) er impl.logger.Errorw("error, checkVulnerabilityStatusAndFailWfIfNeeded", "err", err, "runner", runner) return err } - cdStageWorkflowRequest, err := impl.buildWFRequest(runner, cdWf, pipeline, triggeredBy) + cdStageWorkflowRequest, err := impl.buildWFRequest(runner, cdWf, pipeline, envDevploymentConfig, triggeredBy) if err != nil { impl.logger.Errorw("error in building wfRequest", "err", err, "runner", runner, "cdWf", cdWf, "pipeline", pipeline) return err diff --git a/pkg/deployment/trigger/devtronApps/PreStageTriggerService.go b/pkg/deployment/trigger/devtronApps/PreStageTriggerService.go index 71f84474e4..945b896de9 100644 --- a/pkg/deployment/trigger/devtronApps/PreStageTriggerService.go +++ b/pkg/deployment/trigger/devtronApps/PreStageTriggerService.go @@ -28,6 +28,7 @@ import ( "github.com/devtron-labs/devtron/internal/util" bean4 "github.com/devtron-labs/devtron/pkg/bean" repository2 "github.com/devtron-labs/devtron/pkg/cluster/repository" + bean5 "github.com/devtron-labs/devtron/pkg/deployment/common/bean" adapter2 "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/adapter" "github.com/devtron-labs/devtron/pkg/deployment/trigger/devtronApps/bean" "github.com/devtron-labs/devtron/pkg/imageDigestPolicy" @@ -84,8 +85,14 @@ func (impl *TriggerServiceImpl) TriggerPreStage(request bean.TriggerRequest) err return err } + envDeploymentConfig, err := impl.deploymentConfigService.GetAndMigrateConfigIfAbsentForDevtronApps(pipeline.AppId, pipeline.EnvironmentId) + if err != nil { + impl.logger.Errorw("error in fetching deployment config by appId and envId", "appId", pipeline.AppId, "envId", pipeline.EnvironmentId, "err", err) + return err + } + // custom GitOps repo url validation --> Start - err = impl.handleCustomGitOpsRepoValidation(runner, pipeline, triggeredBy) + err = impl.handleCustomGitOpsRepoValidation(runner, pipeline, envDeploymentConfig, triggeredBy) if err != nil { impl.logger.Errorw("custom GitOps repository validation error, TriggerPreStage", "err", err) return err @@ -99,7 +106,7 @@ func (impl *TriggerServiceImpl) TriggerPreStage(request bean.TriggerRequest) err return err } _, span := otel.Tracer("orchestrator").Start(ctx, "buildWFRequest") - cdStageWorkflowRequest, err := impl.buildWFRequest(runner, cdWf, pipeline, triggeredBy) + cdStageWorkflowRequest, err := impl.buildWFRequest(runner, cdWf, pipeline, envDeploymentConfig, triggeredBy) span.End() if err != nil { return err @@ -321,7 +328,7 @@ func (impl *TriggerServiceImpl) SetCopyContainerImagePluginDataInWorkflowRequest return imagePathReservationIds, nil } -func (impl *TriggerServiceImpl) buildWFRequest(runner *pipelineConfig.CdWorkflowRunner, cdWf *pipelineConfig.CdWorkflow, cdPipeline *pipelineConfig.Pipeline, triggeredBy int32) (*types.WorkflowRequest, error) { +func (impl *TriggerServiceImpl) buildWFRequest(runner *pipelineConfig.CdWorkflowRunner, cdWf *pipelineConfig.CdWorkflow, cdPipeline *pipelineConfig.Pipeline, envDeploymentConfig *bean5.DeploymentConfig, triggeredBy int32) (*types.WorkflowRequest, error) { if cdPipeline.App.Id == 0 { appModel, err := impl.appRepository.FindById(cdPipeline.AppId) if err != nil { diff --git a/pkg/deployment/trigger/devtronApps/TriggerService.go b/pkg/deployment/trigger/devtronApps/TriggerService.go index b80d56446f..dce7e3e885 100644 --- a/pkg/deployment/trigger/devtronApps/TriggerService.go +++ b/pkg/deployment/trigger/devtronApps/TriggerService.go @@ -102,7 +102,7 @@ type TriggerService interface { ManualCdTrigger(triggerContext bean.TriggerContext, overrideRequest *bean3.ValuesOverrideRequest) (int, error) TriggerAutomaticDeployment(request bean.TriggerRequest) error - TriggerRelease(overrideRequest *bean3.ValuesOverrideRequest, ctx context.Context, triggeredAt time.Time, triggeredBy int32) (releaseNo int, err error) + TriggerRelease(overrideRequest *bean3.ValuesOverrideRequest, envDeploymentConfig *bean9.DeploymentConfig, ctx context.Context, triggeredAt time.Time, triggeredBy int32) (releaseNo int, err error) } type TriggerServiceImpl struct { @@ -328,11 +328,11 @@ func (impl *TriggerServiceImpl) getCdPipelineForManualCdTrigger(ctx context.Cont } func (impl *TriggerServiceImpl) validateDeploymentTriggerRequest(ctx context.Context, runner *pipelineConfig.CdWorkflowRunner, - cdPipeline *pipelineConfig.Pipeline, imageDigest string, triggeredBy int32) error { + cdPipeline *pipelineConfig.Pipeline, imageDigest string, envDeploymentConfig *bean9.DeploymentConfig, triggeredBy int32) error { newCtx, span := otel.Tracer("orchestrator").Start(ctx, "TriggerServiceImpl.validateDeploymentTriggerRequest") defer span.End() // custom GitOps repo url validation --> Start - err := impl.handleCustomGitOpsRepoValidation(runner, cdPipeline, triggeredBy) + err := impl.handleCustomGitOpsRepoValidation(runner, cdPipeline, envDeploymentConfig, triggeredBy) if err != nil { impl.logger.Errorw("custom GitOps repository validation error, TriggerStage", "err", err) return err @@ -368,7 +368,7 @@ func (impl *TriggerServiceImpl) ManualCdTrigger(triggerContext bean.TriggerConte if err != nil { return 0, err } - envDeploymentConfig, err := impl.deploymentConfigService.GetDeploymentConfig(cdPipeline.AppId, cdPipeline.EnvironmentId) + envDeploymentConfig, err := impl.deploymentConfigService.GetAndMigrateConfigIfAbsentForDevtronApps(cdPipeline.AppId, cdPipeline.EnvironmentId) if err != nil { impl.logger.Errorw("error in fetching environment deployment config by appId and envId", "appId", cdPipeline.AppId, "envId", cdPipeline.EnvironmentId, "err", err) return 0, err @@ -479,7 +479,7 @@ func (impl *TriggerServiceImpl) ManualCdTrigger(triggerContext bean.TriggerConte } } if isNotHibernateRequest(overrideRequest.DeploymentType) { - validationErr := impl.validateDeploymentTriggerRequest(ctx, runner, cdPipeline, artifact.ImageDigest, overrideRequest.UserId) + validationErr := impl.validateDeploymentTriggerRequest(ctx, runner, cdPipeline, artifact.ImageDigest, envDeploymentConfig, overrideRequest.UserId) if validationErr != nil { impl.logger.Errorw("validation error deployment request", "cdWfr", runner.Id, "err", validationErr) return 0, validationErr @@ -487,7 +487,7 @@ func (impl *TriggerServiceImpl) ManualCdTrigger(triggerContext bean.TriggerConte } // Deploy the release var releaseErr error - releaseId, releaseErr = impl.handleCDTriggerRelease(ctx, overrideRequest, triggeredAt, overrideRequest.UserId) + releaseId, releaseErr = impl.handleCDTriggerRelease(ctx, overrideRequest, envDeploymentConfig, triggeredAt, overrideRequest.UserId) // if releaseErr found, then the mark current deployment Failed and return if releaseErr != nil { err := impl.cdWorkflowCommonService.MarkCurrentDeploymentFailed(runner, releaseErr, overrideRequest.UserId) @@ -607,13 +607,18 @@ func (impl *TriggerServiceImpl) TriggerAutomaticDeployment(request bean.TriggerR if err != nil { impl.logger.Errorw("error in creating timeline status for deployment initiation", "err", err, "timeline", timeline) } + envDeploymentConfig, err := impl.deploymentConfigService.GetAndMigrateConfigIfAbsentForDevtronApps(pipeline.AppId, pipeline.EnvironmentId) + if err != nil { + impl.logger.Errorw("error in fetching environment deployment config by appId and envId", "appId", pipeline.AppId, "envId", pipeline.EnvironmentId, "err", err) + return err + } // setting triggeredBy as 1(system user) since case of auto trigger - validationErr := impl.validateDeploymentTriggerRequest(ctx, runner, pipeline, artifact.ImageDigest, 1) + validationErr := impl.validateDeploymentTriggerRequest(ctx, runner, pipeline, artifact.ImageDigest, envDeploymentConfig, 1) if validationErr != nil { impl.logger.Errorw("validation error deployment request", "cdWfr", runner.Id, "err", validationErr) return validationErr } - releaseErr := impl.TriggerCD(ctx, artifact, cdWf.Id, savedWfr.Id, pipeline, triggeredAt) + releaseErr := impl.TriggerCD(ctx, artifact, cdWf.Id, savedWfr.Id, pipeline, envDeploymentConfig, triggeredAt) // if releaseErr found, then the mark current deployment Failed and return if releaseErr != nil { err := impl.cdWorkflowCommonService.MarkCurrentDeploymentFailed(runner, releaseErr, triggeredBy) @@ -625,14 +630,14 @@ func (impl *TriggerServiceImpl) TriggerAutomaticDeployment(request bean.TriggerR return nil } -func (impl *TriggerServiceImpl) TriggerCD(ctx context.Context, artifact *repository3.CiArtifact, cdWorkflowId, wfrId int, pipeline *pipelineConfig.Pipeline, triggeredAt time.Time) error { +func (impl *TriggerServiceImpl) TriggerCD(ctx context.Context, artifact *repository3.CiArtifact, cdWorkflowId, wfrId int, pipeline *pipelineConfig.Pipeline, envDeploymentConfig *bean9.DeploymentConfig, triggeredAt time.Time) error { impl.logger.Debugw("automatic pipeline trigger attempt async", "artifactId", artifact.Id) - return impl.triggerReleaseAsync(ctx, artifact, cdWorkflowId, wfrId, pipeline, triggeredAt) + return impl.triggerReleaseAsync(ctx, artifact, cdWorkflowId, wfrId, pipeline, envDeploymentConfig, triggeredAt) } -func (impl *TriggerServiceImpl) triggerReleaseAsync(ctx context.Context, artifact *repository3.CiArtifact, cdWorkflowId, wfrId int, pipeline *pipelineConfig.Pipeline, triggeredAt time.Time) error { - err := impl.validateAndTrigger(ctx, pipeline, artifact, cdWorkflowId, wfrId, triggeredAt) +func (impl *TriggerServiceImpl) triggerReleaseAsync(ctx context.Context, artifact *repository3.CiArtifact, cdWorkflowId, wfrId int, pipeline *pipelineConfig.Pipeline, envDeploymentConfig *bean9.DeploymentConfig, triggeredAt time.Time) error { + err := impl.validateAndTrigger(ctx, pipeline, envDeploymentConfig, artifact, cdWorkflowId, wfrId, triggeredAt) if err != nil { impl.logger.Errorw("error in trigger for pipeline", "pipelineId", strconv.Itoa(pipeline.Id)) } @@ -640,7 +645,7 @@ func (impl *TriggerServiceImpl) triggerReleaseAsync(ctx context.Context, artifac return err } -func (impl *TriggerServiceImpl) validateAndTrigger(ctx context.Context, p *pipelineConfig.Pipeline, artifact *repository3.CiArtifact, cdWorkflowId, wfrId int, triggeredAt time.Time) error { +func (impl *TriggerServiceImpl) validateAndTrigger(ctx context.Context, p *pipelineConfig.Pipeline, envDeploymentConfig *bean9.DeploymentConfig, artifact *repository3.CiArtifact, cdWorkflowId, wfrId int, triggeredAt time.Time) error { //TODO: verify this logic object := impl.enforcerUtil.GetAppRBACNameByAppId(p.AppId) envApp := strings.Split(object, "/") @@ -648,11 +653,11 @@ func (impl *TriggerServiceImpl) validateAndTrigger(ctx context.Context, p *pipel impl.logger.Error("invalid req, app and env not found from rbac") return errors.New("invalid req, app and env not found from rbac") } - err := impl.releasePipeline(ctx, p, artifact, cdWorkflowId, wfrId, triggeredAt) + err := impl.releasePipeline(ctx, p, envDeploymentConfig, artifact, cdWorkflowId, wfrId, triggeredAt) return err } -func (impl *TriggerServiceImpl) releasePipeline(ctx context.Context, pipeline *pipelineConfig.Pipeline, artifact *repository3.CiArtifact, cdWorkflowId, wfrId int, triggeredAt time.Time) error { +func (impl *TriggerServiceImpl) releasePipeline(ctx context.Context, pipeline *pipelineConfig.Pipeline, envDeploymentConfig *bean9.DeploymentConfig, artifact *repository3.CiArtifact, cdWorkflowId, wfrId int, triggeredAt time.Time) error { startTime := time.Now() defer func() { impl.logger.Debugw("auto trigger release process completed", "timeTaken", time.Since(startTime), "cdPipelineId", pipeline.Id, "artifactId", artifact.Id, "wfrId", wfrId) @@ -674,11 +679,7 @@ func (impl *TriggerServiceImpl) releasePipeline(ctx context.Context, pipeline *p DeploymentWithConfig: bean3.DEPLOYMENT_CONFIG_TYPE_LAST_SAVED, WfrId: wfrId, } - envDeploymentConfig, err := impl.deploymentConfigService.GetDeploymentConfig(pipeline.AppId, pipeline.EnvironmentId) - if err != nil { - impl.logger.Errorw("error in fetching environment deployment config by appId and envId", "appId", pipeline.AppId, "envId", pipeline.EnvironmentId, "err", err) - return err - } + adapter.SetPipelineFieldsInOverrideRequest(request, pipeline, envDeploymentConfig) releaseCtx, err := impl.argoUserService.GetACDContext(ctx) @@ -687,7 +688,7 @@ func (impl *TriggerServiceImpl) releasePipeline(ctx context.Context, pipeline *p return err } // setting deployedBy as 1(system user) since case of auto trigger - id, err := impl.handleCDTriggerRelease(releaseCtx, request, triggeredAt, 1) + id, err := impl.handleCDTriggerRelease(releaseCtx, request, envDeploymentConfig, triggeredAt, 1) if err != nil { impl.logger.Errorw("error in auto cd pipeline trigger", "pipelineId", pipeline.Id, "artifactId", artifact.Id, "err", err) } else { @@ -716,7 +717,7 @@ func (impl *TriggerServiceImpl) triggerAsyncRelease(userDeploymentRequestId int, return impl.workflowEventPublishService.TriggerAsyncRelease(userDeploymentRequestId, overrideRequest, valuesOverrideResponse, newCtx, deployedBy) } -func (impl *TriggerServiceImpl) handleCDTriggerRelease(ctx context.Context, overrideRequest *bean3.ValuesOverrideRequest, triggeredAt time.Time, deployedBy int32) (releaseNo int, err error) { +func (impl *TriggerServiceImpl) handleCDTriggerRelease(ctx context.Context, overrideRequest *bean3.ValuesOverrideRequest, envDeploymentConfig *bean9.DeploymentConfig, triggeredAt time.Time, deployedBy int32) (releaseNo int, err error) { newCtx, span := otel.Tracer("orchestrator").Start(ctx, "TriggerServiceImpl.handleCDTriggerRelease") defer span.End() // Handling for auto trigger @@ -749,7 +750,7 @@ func (impl *TriggerServiceImpl) handleCDTriggerRelease(ctx context.Context, over return impl.triggerAsyncRelease(userDeploymentRequestId, overrideRequest, newCtx, triggeredAt, deployedBy) } // synchronous mode of installation starts - return impl.TriggerRelease(overrideRequest, newCtx, triggeredAt, deployedBy) + return impl.TriggerRelease(overrideRequest, envDeploymentConfig, newCtx, triggeredAt, deployedBy) } func (impl *TriggerServiceImpl) auditDeploymentTriggerHistory(cdWfrId int, valuesOverrideResponse *app.ValuesOverrideResponse, ctx context.Context, triggeredAt time.Time, triggeredBy int32) (err error) { @@ -766,7 +767,7 @@ func (impl *TriggerServiceImpl) auditDeploymentTriggerHistory(cdWfrId int, value } // TriggerRelease will trigger Install/Upgrade request for Devtron App releases synchronously -func (impl *TriggerServiceImpl) TriggerRelease(overrideRequest *bean3.ValuesOverrideRequest, ctx context.Context, triggeredAt time.Time, triggeredBy int32) (releaseNo int, err error) { +func (impl *TriggerServiceImpl) TriggerRelease(overrideRequest *bean3.ValuesOverrideRequest, envDeploymentConfig *bean9.DeploymentConfig, ctx context.Context, triggeredAt time.Time, triggeredBy int32) (releaseNo int, err error) { newCtx, span := otel.Tracer("orchestrator").Start(ctx, "TriggerServiceImpl.TriggerRelease") defer span.End() triggerEvent, skipRequest, err := impl.buildTriggerEventForOverrideRequest(overrideRequest, triggeredAt) @@ -779,6 +780,17 @@ func (impl *TriggerServiceImpl) TriggerRelease(overrideRequest *bean3.ValuesOver } // build merged values and save PCO history for the release valuesOverrideResponse, builtChartPath, err := impl.manifestCreationService.BuildManifestForTrigger(overrideRequest, triggeredAt, newCtx) + + if envDeploymentConfig == nil || (envDeploymentConfig != nil && envDeploymentConfig.Id == 0) { + envDeploymentConfig, err = impl.deploymentConfigService.GetAndMigrateConfigIfAbsentForDevtronApps(overrideRequest.AppId, overrideRequest.EnvId) + if err != nil { + impl.logger.Errorw("error in getting deployment config by appId and envId", "appId", overrideRequest.AppId, "envId", overrideRequest.EnvId, "err", err) + return releaseNo, err + } + valuesOverrideResponse.DeploymentConfig = envDeploymentConfig + } + valuesOverrideResponse.DeploymentConfig = envDeploymentConfig + // auditDeploymentTriggerHistory is performed irrespective of BuildManifestForTrigger error - for auditing purposes historyErr := impl.auditDeploymentTriggerHistory(overrideRequest.WfrId, valuesOverrideResponse, newCtx, triggeredAt, triggeredBy) if historyErr != nil { @@ -1445,7 +1457,7 @@ func (impl *TriggerServiceImpl) deleteCorruptedPipelineStage(pipelineStage *repo return nil, false } -func (impl *TriggerServiceImpl) handleCustomGitOpsRepoValidation(runner *pipelineConfig.CdWorkflowRunner, pipeline *pipelineConfig.Pipeline, triggeredBy int32) error { +func (impl *TriggerServiceImpl) handleCustomGitOpsRepoValidation(runner *pipelineConfig.CdWorkflowRunner, pipeline *pipelineConfig.Pipeline, envDeploymentConfig *bean9.DeploymentConfig, triggeredBy int32) error { if !util.IsAcdApp(pipeline.DeploymentAppName) { return nil } @@ -1463,12 +1475,7 @@ func (impl *TriggerServiceImpl) handleCustomGitOpsRepoValidation(runner *pipelin // impl.logger.Errorw("error in fetching latest chart for app by appId", "err", err, "appId", pipeline.AppId) // return err //} - deploymentConfig, err := impl.deploymentConfigService.GetDeploymentConfig(pipeline.AppId, pipeline.EnvironmentId) - if err != nil { - impl.logger.Errorw("error in getting deployment config by appId and envId", "appId", pipeline.AppId, "envId", pipeline.EnvironmentId, "err", err) - return err - } - if gitOps.IsGitOpsRepoNotConfigured(deploymentConfig.RepoURL) { + if gitOps.IsGitOpsRepoNotConfigured(envDeploymentConfig.RepoURL) { if err = impl.cdWorkflowCommonService.MarkCurrentDeploymentFailed(runner, errors.New(pipelineConfig.GITOPS_REPO_NOT_CONFIGURED), triggeredBy); err != nil { impl.logger.Errorw("error while updating current runner status to failed, TriggerDeployment", "wfrId", runner.Id, "err", err) } diff --git a/pkg/eventProcessor/in/DeployedApplicationEventProcessorService.go b/pkg/eventProcessor/in/DeployedApplicationEventProcessorService.go index cbf7eaa73a..e3680d6828 100644 --- a/pkg/eventProcessor/in/DeployedApplicationEventProcessorService.go +++ b/pkg/eventProcessor/in/DeployedApplicationEventProcessorService.go @@ -262,6 +262,10 @@ func (impl *DeployedApplicationEventProcessorImpl) updateArgoAppDeleteStatus(app } } else { // devtron app + if pipeline.DeploymentAppDeleteRequest == false { + impl.logger.Infow("Deployment delete not requested for app, not deleting app from DB", "appName", app.Name, "app", app) + return nil + } _, err = impl.pipelineBuilder.DeleteCdPipeline(&pipeline, context.Background(), bean.FORCE_DELETE, false, 1) if err != nil { impl.logger.Errorw("error in deleting cd pipeline", "err", err) diff --git a/pkg/eventProcessor/in/WorkflowEventProcessorService.go b/pkg/eventProcessor/in/WorkflowEventProcessorService.go index 7ea34cb9d4..bd2f8adf44 100644 --- a/pkg/eventProcessor/in/WorkflowEventProcessorService.go +++ b/pkg/eventProcessor/in/WorkflowEventProcessorService.go @@ -757,12 +757,18 @@ func (impl *WorkflowEventProcessorImpl) SubscribeCDPipelineDeleteEvent() error { impl.logger.Errorw("error in fetching pipeline by pipelineId", "err", err, "pipelineId", cdPipelineDeleteEvent.PipelineId) return } - envDeploymentConfig, err := impl.deploymentConfigService.GetDeploymentConfig(pipeline.AppId, pipeline.EnvironmentId) - if err != nil { + envDeploymentConfig, err := impl.deploymentConfigService.GetConfigEvenIfInactive(pipeline.AppId, pipeline.EnvironmentId) + if err != nil && err != pg.ErrNoRows { impl.logger.Errorw("error in fetching environment deployment config by appId and envId", "appId", pipeline.AppId, "envId", pipeline.EnvironmentId, "err", err) return } - if util3.IsHelmApp(envDeploymentConfig.DeploymentAppType) || util3.IsAcdApp(envDeploymentConfig.DeploymentAppType) { + var deploymentAppType string + if err == pg.ErrNoRows { + deploymentAppType = pipeline.DeploymentAppType + } else { + deploymentAppType = envDeploymentConfig.DeploymentAppType + } + if util3.IsHelmApp(deploymentAppType) || util3.IsAcdApp(deploymentAppType) { impl.RemoveReleaseContextForPipeline(cdPipelineDeleteEvent.PipelineId, cdPipelineDeleteEvent.TriggeredBy) // there is a possibility that when the pipeline was deleted, async request nats message was not consumed completely and could have led to dangling deployment app // trying to delete deployment app once @@ -1035,7 +1041,7 @@ func (impl *WorkflowEventProcessorImpl) setAdditionalDataInAsyncInstallReq(ctx c if err != nil { return err } - envDeploymentConfig, err := impl.deploymentConfigService.GetDeploymentConfig(pipelineModel.AppId, pipelineModel.EnvironmentId) + envDeploymentConfig, err := impl.deploymentConfigService.GetConfigForDevtronApps(pipelineModel.AppId, pipelineModel.EnvironmentId) if err != nil { impl.logger.Errorw("error in fetching environment deployment config by appId and envId", "appId", pipelineModel.AppId, "envId", pipelineModel.EnvironmentId, "err", err) return err diff --git a/pkg/pipeline/AppDeploymentTypeChangeManager.go b/pkg/pipeline/AppDeploymentTypeChangeManager.go index ea47a9252a..02bd5a69a0 100644 --- a/pkg/pipeline/AppDeploymentTypeChangeManager.go +++ b/pkg/pipeline/AppDeploymentTypeChangeManager.go @@ -57,7 +57,7 @@ type AppDeploymentTypeChangeManager interface { // TriggerDeploymentAfterTypeChange : triggers a new deployment after type change TriggerDeploymentAfterTypeChange(ctx context.Context, request *bean.DeploymentAppTypeChangeRequest) (*bean.DeploymentAppTypeChangeResponse, error) // DeleteDeploymentApps : takes in a list of pipelines and delete the applications - DeleteDeploymentApps(ctx context.Context, pipelines []*pipelineConfig.Pipeline, userId int32) *bean.DeploymentAppTypeChangeResponse + DeleteDeploymentApps(ctx context.Context, pipelines []*pipelineConfig.Pipeline, deploymentConfig []*bean4.DeploymentConfig, userId int32) *bean.DeploymentAppTypeChangeResponse // DeleteDeploymentAppsForEnvironment : takes in environment id and current deployment app type // and deletes all the cd pipelines for that deployment type in all apps that belongs to // that environment. @@ -156,23 +156,15 @@ func (impl *AppDeploymentTypeChangeManagerImpl) ChangeDeploymentType(ctx context return response, nil } - deploymentConfigSelector := make([]*bean4.DeploymentConfigSelector, len(pipelineIds)) - for _, pipeline := range pipelines { - deploymentConfigSelector = append(deploymentConfigSelector, &bean4.DeploymentConfigSelector{ - AppId: pipeline.AppId, - EnvironmentId: pipeline.EnvironmentId, - }) - } - - deploymentConfigs, err := impl.deploymentConfigService.GetDeploymentConfigInBulk(deploymentConfigSelector) - if err != nil { - impl.logger.Errorw("error in getting deployment config", "deploymentConfigSelector", deploymentConfigSelector, "err", err) - return nil, err - } updatedDeploymentConfigs := make([]*bean4.DeploymentConfig, 0) - for _, c := range deploymentConfigs { - c.DeploymentAppType = request.DesiredDeploymentType - updatedDeploymentConfigs = append(updatedDeploymentConfigs, c) + for _, pipeline := range pipelines { + deploymentConfig, err := impl.deploymentConfigService.GetAndMigrateConfigIfAbsentForDevtronApps(pipeline.AppId, pipeline.EnvironmentId) + if err != nil { + impl.logger.Errorw("error in getting deployment config by appId and envId", "appId", pipeline.AppId, "envId", pipeline.EnvironmentId, "err", err) + return nil, err + } + deploymentConfig.DeploymentAppType = request.DesiredDeploymentType + updatedDeploymentConfigs = append(updatedDeploymentConfigs, deploymentConfig) } updatedDeploymentConfigs, err = impl.deploymentConfigService.UpdateConfigs(nil, updatedDeploymentConfigs, request.UserId) @@ -182,16 +174,16 @@ func (impl *AppDeploymentTypeChangeManagerImpl) ChangeDeploymentType(ctx context } // Update in db - err = impl.pipelineRepository.UpdateCdPipelineDeploymentAppInFilter(string(request.DesiredDeploymentType), - pipelineIds, request.UserId, false, true) - if err != nil { - impl.logger.Errorw("failed to update deployment app type in db", - "pipeline ids", pipelineIds, - "desired deployment type", request.DesiredDeploymentType, - "err", err) - - return response, nil - } + //err = impl.pipelineRepository.UpdateCdPipelineDeploymentAppInFilter(string(request.DesiredDeploymentType), + // pipelineIds, request.UserId, false, true) + //if err != nil { + // impl.logger.Errorw("failed to update deployment app type in db", + // "pipeline ids", pipelineIds, + // "desired deployment type", request.DesiredDeploymentType, + // "err", err) + // + // return response, nil + //} if !request.AutoTriggerDeployment { return response, nil @@ -280,18 +272,28 @@ func (impl *AppDeploymentTypeChangeManagerImpl) ChangePipelineDeploymentType(ctx return response, nil } - err = impl.pipelineRepository.UpdateCdPipelineDeploymentAppInFilter(string(request.DesiredDeploymentType), - pipelineIds, request.UserId, false, true) - - if err != nil { - impl.logger.Errorw("failed to update deployment app type in db", - "pipeline ids", pipelineIds, - "desired deployment type", request.DesiredDeploymentType, - "err", err) - - return response, nil - } - deleteResponse := impl.DeleteDeploymentApps(ctx, pipelines, request.UserId) + deploymentConfigs := make([]*bean4.DeploymentConfig, 0) + for _, p := range pipelines { + envDeploymentConfig, err := impl.deploymentConfigService.GetAndMigrateConfigIfAbsentForDevtronApps(p.AppId, p.EnvironmentId) + if err != nil { + impl.logger.Errorw("error in fetching environment deployment config by appId and envId", "appId", p.AppId, "envId", p.EnvironmentId, "err", err) + return response, err + } + deploymentConfigs = append(deploymentConfigs, envDeploymentConfig) + } + + //err = impl.pipelineRepository.UpdateCdPipelineDeploymentAppInFilter(string(request.DesiredDeploymentType), + // pipelineIds, request.UserId, false, true) + // + //if err != nil { + // impl.logger.Errorw("failed to update deployment app type in db", + // "pipeline ids", pipelineIds, + // "desired deployment type", request.DesiredDeploymentType, + // "err", err) + // + // return response, nil + //} + deleteResponse := impl.DeleteDeploymentApps(ctx, pipelines, deploymentConfigs, request.UserId) response.SuccessfulPipelines = deleteResponse.SuccessfulPipelines response.FailedPipelines = deleteResponse.FailedPipelines @@ -452,21 +454,22 @@ func (impl *AppDeploymentTypeChangeManagerImpl) TriggerDeploymentAfterTypeChange } func (impl *AppDeploymentTypeChangeManagerImpl) DeleteDeploymentApps(ctx context.Context, - pipelines []*pipelineConfig.Pipeline, userId int32) *bean.DeploymentAppTypeChangeResponse { + pipelines []*pipelineConfig.Pipeline, deploymentConfig []*bean4.DeploymentConfig, userId int32) *bean.DeploymentAppTypeChangeResponse { successfulPipelines := make([]*bean.DeploymentChangeStatus, 0) failedPipelines := make([]*bean.DeploymentChangeStatus, 0) gitOpsConfigurationStatus, gitOpsConfigErr := impl.gitOpsConfigReadService.IsGitOpsConfigured() + deploymentConfigMap := make(map[bean4.UniqueDeploymentConfigIdentifier]*bean4.DeploymentConfig) + for _, c := range deploymentConfig { + deploymentConfigMap[bean4.GetConfigUniqueIdentifier(c.AppId, c.EnvironmentId)] = c + } + // Iterate over all the pipelines in the environment for given deployment app type for _, pipeline := range pipelines { - envDeploymentConfig, err := impl.deploymentConfigService.GetDeploymentConfig(pipeline.AppId, pipeline.EnvironmentId) - if err != nil { - impl.logger.Errorw("error in fetching environment deployment config by appId and envId", "appId", pipeline.AppId, "envId", pipeline.EnvironmentId, "err", err) - continue - } + envDeploymentConfig := deploymentConfigMap[bean4.GetConfigUniqueIdentifier(pipeline.AppId, pipeline.EnvironmentId)] var isValid bool // check if pipeline info like app name and environment is empty or not @@ -484,6 +487,7 @@ func (impl *AppDeploymentTypeChangeManagerImpl) DeleteDeploymentApps(ctx context deploymentAppName := fmt.Sprintf("%s-%s", pipeline.App.AppName, pipeline.Environment.Name) // delete request + var err error if envDeploymentConfig.DeploymentAppType == bean3.ArgoCd { err = impl.deleteArgoCdApp(ctx, pipeline, deploymentAppName, true) @@ -520,17 +524,9 @@ func (impl *AppDeploymentTypeChangeManagerImpl) DeleteDeploymentApps(ctx context if RepoURLUpdateErr != nil { impl.logger.Errorw("error in updating git repo url in charts", "err", RepoURLUpdateErr) } - //configure gitOps repoUrl for ENV - activeGitOpsConfig, err := impl.gitOpsConfigReadService.GetGitOpsConfigActive() - if err != nil { - impl.logger.Errorw("error in getting active gitOps config", "err", err) - } - envDeploymentConfig.DeploymentAppType = bean3.ArgoCd envDeploymentConfig.ConfigType = common.GetDeploymentConfigType(chart.IsCustomGitRepository) envDeploymentConfig.RepoURL = chartGitAttr.RepoUrl envDeploymentConfig.ChartLocation = chartGitAttr.ChartLocation - envDeploymentConfig.CredentialIdInt = activeGitOpsConfig.Id - envDeploymentConfig.CredentialType = bean4.GitOps.String() envDeploymentConfig, RepoURLUpdateErr = impl.deploymentConfigService.CreateOrUpdateConfig(nil, envDeploymentConfig, userId) if RepoURLUpdateErr != nil { @@ -614,6 +610,20 @@ func (impl *AppDeploymentTypeChangeManagerImpl) DeleteDeploymentAppsForEnvironme pipelines, err := impl.pipelineRepository.FindActiveByEnvIdAndDeploymentType(environmentId, string(currentDeploymentAppType), exclusionList, includeApps) + deploymentConfigs := make([]*bean4.DeploymentConfig, 0) + for _, p := range pipelines { + envDeploymentConfig, err := impl.deploymentConfigService.GetAndMigrateConfigIfAbsentForDevtronApps(p.AppId, p.EnvironmentId) + if err != nil { + impl.logger.Errorw("error in fetching environment deployment config by appId and envId", "appId", p.AppId, "envId", p.EnvironmentId, "err", err) + return &bean.DeploymentAppTypeChangeResponse{ + EnvId: environmentId, + SuccessfulPipelines: []*bean.DeploymentChangeStatus{}, + FailedPipelines: []*bean.DeploymentChangeStatus{}, + }, err + } + deploymentConfigs = append(deploymentConfigs, envDeploymentConfig) + } + if err != nil { impl.logger.Errorw("Error fetching cd pipelines", "environmentId", environmentId, @@ -628,7 +638,7 @@ func (impl *AppDeploymentTypeChangeManagerImpl) DeleteDeploymentAppsForEnvironme } // Currently deleting apps only in argocd is supported - return impl.DeleteDeploymentApps(ctx, pipelines, userId), nil + return impl.DeleteDeploymentApps(ctx, pipelines, deploymentConfigs, userId), nil } func (impl *AppDeploymentTypeChangeManagerImpl) isPipelineInfoValid(pipeline *pipelineConfig.Pipeline, @@ -725,7 +735,7 @@ func (impl *AppDeploymentTypeChangeManagerImpl) fetchDeletedApp(ctx context.Cont failedPipelines := make([]*bean.DeploymentChangeStatus, 0) // Iterate over all the pipelines in the environment for given deployment app type for _, pipeline := range pipelines { - envDeploymentConfig, err := impl.deploymentConfigService.GetDeploymentConfig(pipeline.AppId, pipeline.EnvironmentId) + envDeploymentConfig, err := impl.deploymentConfigService.GetConfigForDevtronApps(pipeline.AppId, pipeline.EnvironmentId) if err != nil { impl.logger.Errorw("error in fetching environment deployment config by appId and envId", "appId", pipeline.AppId, "envId", pipeline.EnvironmentId, "err", err) } diff --git a/pkg/pipeline/CdHandler.go b/pkg/pipeline/CdHandler.go index 15dcec283f..bc8c1cef0b 100644 --- a/pkg/pipeline/CdHandler.go +++ b/pkg/pipeline/CdHandler.go @@ -251,7 +251,7 @@ func (impl *CdHandlerImpl) UpdateWorkflow(workflowStatus v1alpha1.WorkflowStatus } appId := savedWorkflow.CdWorkflow.Pipeline.AppId envId := savedWorkflow.CdWorkflow.Pipeline.EnvironmentId - envDeploymentConfig, err := impl.deploymentConfigService.GetDeploymentConfig(appId, envId) + envDeploymentConfig, err := impl.deploymentConfigService.GetConfigForDevtronApps(appId, envId) if err != nil { impl.Logger.Errorw("error in fetching environment deployment config by appId and envId", "appId", appId, "envId", envId, "err", err) return 0, "", err diff --git a/pkg/pipeline/CiCdPipelineOrchestrator.go b/pkg/pipeline/CiCdPipelineOrchestrator.go index 629fa540e6..7502cfc3b0 100644 --- a/pkg/pipeline/CiCdPipelineOrchestrator.go +++ b/pkg/pipeline/CiCdPipelineOrchestrator.go @@ -1783,7 +1783,7 @@ func (impl CiCdPipelineOrchestratorImpl) GetCdPipelinesForApp(appId int) (cdPipe var pipelines []*bean.CDPipelineConfigObject for _, dbPipeline := range dbPipelines { - envDeploymentConfig, err := impl.deploymentConfigService.GetDeploymentConfig(dbPipeline.AppId, dbPipeline.EnvironmentId) + envDeploymentConfig, err := impl.deploymentConfigService.GetConfigForDevtronApps(dbPipeline.AppId, dbPipeline.EnvironmentId) if err != nil { impl.logger.Errorw("error in fetching environment deployment config by appId and envId", "appId", appId, "envId", dbPipeline.EnvironmentId, "err", err) return nil, err @@ -1897,7 +1897,7 @@ func (impl CiCdPipelineOrchestratorImpl) GetCdPipelinesForEnv(envId int, request impl.logger.Errorw("error in fetching latest chart details for app by appId") return nil, err } - envDeploymentConfig, err := impl.deploymentConfigService.GetDeploymentConfig(dbPipeline.AppId, dbPipeline.EnvironmentId) + envDeploymentConfig, err := impl.deploymentConfigService.GetConfigForDevtronApps(dbPipeline.AppId, dbPipeline.EnvironmentId) if err != nil { impl.logger.Errorw("error in fetching environment deployment config by appId and envId", "appId", dbPipeline.AppId, "envId", dbPipeline.EnvironmentId, "err", err) return nil, err diff --git a/pkg/pipeline/DeploymentPipelineConfigService.go b/pkg/pipeline/DeploymentPipelineConfigService.go index 4529b6e4ea..1e05fc380f 100644 --- a/pkg/pipeline/DeploymentPipelineConfigService.go +++ b/pkg/pipeline/DeploymentPipelineConfigService.go @@ -327,7 +327,7 @@ func (impl *CdPipelineConfigServiceImpl) GetCdPipelineById(pipelineId int) (cdPi return nil, err } - envDeploymentConfig, err := impl.deploymentConfigService.GetDeploymentConfig(dbPipeline.AppId, dbPipeline.EnvironmentId) + envDeploymentConfig, err := impl.deploymentConfigService.GetConfigForDevtronApps(dbPipeline.AppId, dbPipeline.EnvironmentId) if err != nil { impl.logger.Errorw("error in fetching environment deployment config by appId and envId", "appId", dbPipeline.AppId, "envId", dbPipeline.EnvironmentId, "err", err) return nil, err @@ -414,7 +414,7 @@ func (impl *CdPipelineConfigServiceImpl) CreateCdPipelines(pipelineCreateRequest // TODO: creating git repo for all apps irrespective of acd or helm if gitOpsConfigurationStatus.IsGitOpsConfigured && isGitOpsRequiredForCD && !pipelineCreateRequest.IsCloneAppReq { - AppDeploymentConfig, err := impl.deploymentConfigService.GetDeploymentConfig(app.Id, 0) + AppDeploymentConfig, err := impl.deploymentConfigService.GetAndMigrateConfigIfAbsentForDevtronApps(app.Id, 0) if err != nil { impl.logger.Errorw("error in fetching deployment config by appId", "appId", app.Id, "err", err) return nil, err @@ -448,17 +448,14 @@ func (impl *CdPipelineConfigServiceImpl) CreateCdPipelines(pipelineCreateRequest } for _, pipeline := range pipelineCreateRequest.Pipelines { envDeploymentConfig := &bean4.DeploymentConfig{ - AppId: app.Id, - EnvironmentId: pipeline.EnvironmentId, - ConfigType: AppDeploymentConfig.ConfigType, - DeploymentAppType: pipeline.DeploymentAppType, - RepoURL: AppDeploymentConfig.RepoURL, - RepoName: AppDeploymentConfig.RepoName, - ChartLocation: AppDeploymentConfig.ChartLocation, - CredentialType: AppDeploymentConfig.CredentialType, - CredentialIdInt: AppDeploymentConfig.CredentialIdInt, - CredentialIdString: AppDeploymentConfig.CredentialIdString, - Active: true, + AppId: app.Id, + EnvironmentId: pipeline.EnvironmentId, + ConfigType: AppDeploymentConfig.ConfigType, + DeploymentAppType: pipeline.DeploymentAppType, + RepoURL: AppDeploymentConfig.RepoURL, + RepoName: AppDeploymentConfig.RepoName, + ChartLocation: AppDeploymentConfig.ChartLocation, + Active: true, } envDeploymentConfig, err := impl.deploymentConfigService.CreateOrUpdateConfig(nil, envDeploymentConfig, pipelineCreateRequest.UserId) if err != nil { @@ -855,7 +852,7 @@ func (impl *CdPipelineConfigServiceImpl) DeleteCdPipeline(pipeline *pipelineConf impl.logger.Errorw("error in deleting imageDigestPolicy for pipeline", "err", err, "pipelineId", pipeline.Id) return nil, err } - envDeploymentConfig, err := impl.deploymentConfigService.GetDeploymentConfig(pipeline.AppId, pipeline.EnvironmentId) + envDeploymentConfig, err := impl.deploymentConfigService.GetAndMigrateConfigIfAbsentForDevtronApps(pipeline.AppId, pipeline.EnvironmentId) if err != nil { impl.logger.Errorw("error in fetching environment deployment config by appId and envId", "appId", pipeline.AppId, "envId", pipeline.EnvironmentId, "err", err) return nil, err @@ -955,7 +952,7 @@ func (impl *CdPipelineConfigServiceImpl) DeleteACDAppCdPipelineWithNonCascade(pi _, err := impl.DeleteCdPipeline(pipeline, ctx, bean.FORCE_DELETE, false, userId) return err } - envDeploymentConfig, err := impl.deploymentConfigService.GetDeploymentConfig(pipeline.AppId, pipeline.EnvironmentId) + envDeploymentConfig, err := impl.deploymentConfigService.GetConfigForDevtronApps(pipeline.AppId, pipeline.EnvironmentId) if err != nil { impl.logger.Errorw("error in fetching environment deployment config by appId and envId", "appId", pipeline.AppId, "envId", pipeline.EnvironmentId, "err", err) return err @@ -1340,7 +1337,7 @@ func (impl *CdPipelineConfigServiceImpl) GetCdPipelinesByEnvironmentMin(request continue } - envDeploymentConfig, err := impl.deploymentConfigService.GetDeploymentConfig(dbPipeline.AppId, dbPipeline.EnvironmentId) + envDeploymentConfig, err := impl.deploymentConfigService.GetConfigForDevtronApps(dbPipeline.AppId, dbPipeline.EnvironmentId) if err != nil { impl.logger.Errorw("error in fetching environment deployment config by appId and envId", "appId", dbPipeline.AppId, "envId", dbPipeline.EnvironmentId, "err", err) return nil, err @@ -2041,7 +2038,7 @@ func (impl *CdPipelineConfigServiceImpl) DeleteCdPipelinePartial(pipeline *pipel //delete app from argo cd, if created if pipeline.DeploymentAppCreated && !pipeline.DeploymentAppDeleteRequest { - envDeploymentConfig, err := impl.deploymentConfigService.GetDeploymentConfig(pipeline.AppId, pipeline.EnvironmentId) + envDeploymentConfig, err := impl.deploymentConfigService.GetAndMigrateConfigIfAbsentForDevtronApps(pipeline.AppId, pipeline.EnvironmentId) if err != nil { impl.logger.Errorw("error in fetching environment deployment config by appId and envId", "appId", pipeline.AppId, "envId", pipeline.EnvironmentId, "err", err) return deleteResponse, err diff --git a/pkg/workflow/cd/CdWorkflowCommonService.go b/pkg/workflow/cd/CdWorkflowCommonService.go index 6b9e217257..03c116348c 100644 --- a/pkg/workflow/cd/CdWorkflowCommonService.go +++ b/pkg/workflow/cd/CdWorkflowCommonService.go @@ -191,7 +191,7 @@ func (impl *CdWorkflowCommonServiceImpl) MarkCurrentDeploymentFailed(runner *pip } appId := runner.CdWorkflow.Pipeline.AppId envId := runner.CdWorkflow.Pipeline.EnvironmentId - envDeploymentConfig, err := impl.deploymentConfigService.GetDeploymentConfig(appId, envId) + envDeploymentConfig, err := impl.deploymentConfigService.GetConfigForDevtronApps(appId, envId) if err != nil { impl.logger.Errorw("error in fetching environment deployment config by appId and envId", "appId", appId, "envId", envId, "err", err) return err @@ -256,7 +256,7 @@ func (impl *CdWorkflowCommonServiceImpl) UpdatePreviousQueuedRunnerStatus(cdWfrI appId := queuedRunner.CdWorkflow.Pipeline.AppId envId := queuedRunner.CdWorkflow.Pipeline.EnvironmentId - envDeploymentConfig, err := impl.deploymentConfigService.GetDeploymentConfig(appId, envId) + envDeploymentConfig, err := impl.deploymentConfigService.GetConfigForDevtronApps(appId, envId) if err != nil { impl.logger.Errorw("error in fetching environment deployment config by appId and envId", "appId", appId, "envId", envId, "err", err) return err diff --git a/pkg/workflow/dag/WorkflowDagExecutor.go b/pkg/workflow/dag/WorkflowDagExecutor.go index 73eb007a7a..1870b1085a 100644 --- a/pkg/workflow/dag/WorkflowDagExecutor.go +++ b/pkg/workflow/dag/WorkflowDagExecutor.go @@ -329,7 +329,7 @@ func (impl *WorkflowDagExecutorImpl) handleAsyncTriggerReleaseError(ctx context. if util4.IsTerminalRunnerStatus(cdWfr.Status) { appId := cdWfr.CdWorkflow.Pipeline.AppId envId := cdWfr.CdWorkflow.Pipeline.EnvironmentId - envDeploymentConfig, err := impl.deploymentConfigService.GetDeploymentConfig(appId, envId) + envDeploymentConfig, err := impl.deploymentConfigService.GetConfigForDevtronApps(appId, envId) if err != nil { impl.logger.Errorw("error in fetching environment deployment config by appId and envId", "appId", appId, "envId", envId, "err", err) } @@ -373,7 +373,12 @@ func (impl *WorkflowDagExecutorImpl) ProcessDevtronAsyncInstallRequest(cdAsyncIn return err } } - releaseId, releaseErr := impl.cdTriggerService.TriggerRelease(overrideRequest, newCtx, cdAsyncInstallReq.TriggeredAt, cdAsyncInstallReq.TriggeredBy) + envDeploymentConfig, err := impl.deploymentConfigService.GetAndMigrateConfigIfAbsentForDevtronApps(overrideRequest.AppId, overrideRequest.EnvId) + if err != nil { + impl.logger.Errorw("error in getting deployment config by appId and envId", "appId", overrideRequest.AppId, "envId", overrideRequest.EnvId, "err", err) + return err + } + releaseId, releaseErr := impl.cdTriggerService.TriggerRelease(overrideRequest, envDeploymentConfig, newCtx, cdAsyncInstallReq.TriggeredAt, cdAsyncInstallReq.TriggeredBy) if releaseErr != nil { impl.handleAsyncTriggerReleaseError(newCtx, releaseErr, cdWfr, overrideRequest) return releaseErr diff --git a/pkg/workflow/status/WorkflowStatusService.go b/pkg/workflow/status/WorkflowStatusService.go index 3f7d3ff9cf..19c3d9214d 100644 --- a/pkg/workflow/status/WorkflowStatusService.go +++ b/pkg/workflow/status/WorkflowStatusService.go @@ -181,7 +181,7 @@ func (impl *WorkflowStatusServiceImpl) CheckHelmAppStatusPeriodicallyAndUpdateIn } appId := wfr.CdWorkflow.Pipeline.AppId envId := wfr.CdWorkflow.Pipeline.EnvironmentId - envDeploymentConfig, err := impl.deploymentConfigService.GetDeploymentConfig(appId, envId) + envDeploymentConfig, err := impl.deploymentConfigService.GetConfigForDevtronApps(appId, envId) if err != nil { impl.logger.Errorw("error in fetching environment deployment config by appId and envId", "appId", appId, "envId", envId, "err", err) return err diff --git a/wire_gen.go b/wire_gen.go index 7710e02bd3..b0378556d5 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -84,6 +84,7 @@ import ( "github.com/devtron-labs/devtron/internal/sql/repository/appWorkflow" "github.com/devtron-labs/devtron/internal/sql/repository/bulkUpdate" "github.com/devtron-labs/devtron/internal/sql/repository/chartConfig" + "github.com/devtron-labs/devtron/internal/sql/repository/deploymentConfig" repository5 "github.com/devtron-labs/devtron/internal/sql/repository/dockerRegistry" "github.com/devtron-labs/devtron/internal/sql/repository/helper" repository12 "github.com/devtron-labs/devtron/internal/sql/repository/imageTagging" @@ -131,6 +132,7 @@ import ( "github.com/devtron-labs/devtron/pkg/clusterTerminalAccess" "github.com/devtron-labs/devtron/pkg/commonService" delete2 "github.com/devtron-labs/devtron/pkg/delete" + "github.com/devtron-labs/devtron/pkg/deployment/common" "github.com/devtron-labs/devtron/pkg/deployment/deployedApp" "github.com/devtron-labs/devtron/pkg/deployment/gitOps/config" "github.com/devtron-labs/devtron/pkg/deployment/gitOps/git" @@ -426,7 +428,9 @@ func InitializeApp() (*App, error) { pipelineStatusSyncDetailRepositoryImpl := pipelineConfig.NewPipelineStatusSyncDetailRepositoryImpl(db, sugaredLogger) pipelineStatusSyncDetailServiceImpl := status.NewPipelineStatusSyncDetailServiceImpl(sugaredLogger, pipelineStatusSyncDetailRepositoryImpl) installedAppVersionHistoryRepositoryImpl := repository3.NewInstalledAppVersionHistoryRepositoryImpl(sugaredLogger, db) - pipelineStatusTimelineServiceImpl := status.NewPipelineStatusTimelineServiceImpl(sugaredLogger, pipelineStatusTimelineRepositoryImpl, cdWorkflowRepositoryImpl, userServiceImpl, pipelineStatusTimelineResourcesServiceImpl, pipelineStatusSyncDetailServiceImpl, installedAppRepositoryImpl, installedAppVersionHistoryRepositoryImpl) + repositoryImpl := deploymentConfig.NewRepositoryImpl(db) + deploymentConfigServiceImpl := common.NewDeploymentConfigServiceImpl(repositoryImpl, sugaredLogger, chartRepositoryImpl, pipelineRepositoryImpl, appRepositoryImpl, installedAppRepositoryImpl) + pipelineStatusTimelineServiceImpl := status.NewPipelineStatusTimelineServiceImpl(sugaredLogger, pipelineStatusTimelineRepositoryImpl, cdWorkflowRepositoryImpl, userServiceImpl, pipelineStatusTimelineResourcesServiceImpl, pipelineStatusSyncDetailServiceImpl, installedAppRepositoryImpl, installedAppVersionHistoryRepositoryImpl, deploymentConfigServiceImpl) appServiceConfig, err := app2.GetAppServiceConfig() if err != nil { return nil, err @@ -480,7 +484,7 @@ func InitializeApp() (*App, error) { deploymentTemplateServiceImpl := deploymentTemplate.NewDeploymentTemplateServiceImpl(sugaredLogger, chartRefServiceImpl, chartTemplateServiceImpl, chartRepositoryImpl) appListingRepositoryQueryBuilder := helper.NewAppListingRepositoryQueryBuilder(sugaredLogger) appWorkflowRepositoryImpl := appWorkflow.NewAppWorkflowRepositoryImpl(sugaredLogger, db) - appListingRepositoryImpl := repository2.NewAppListingRepositoryImpl(sugaredLogger, db, appListingRepositoryQueryBuilder, environmentRepositoryImpl, gitOpsConfigRepositoryImpl, appWorkflowRepositoryImpl) + appListingRepositoryImpl := repository2.NewAppListingRepositoryImpl(sugaredLogger, db, appListingRepositoryQueryBuilder, environmentRepositoryImpl, gitOpsConfigRepositoryImpl, appWorkflowRepositoryImpl, repositoryImpl) appListingViewBuilderImpl := app2.NewAppListingViewBuilderImpl(sugaredLogger) linkoutsRepositoryImpl := repository2.NewLinkoutsRepositoryImpl(sugaredLogger, db) ciTemplateOverrideRepositoryImpl := pipelineConfig.NewCiTemplateOverrideRepositoryImpl(db, sugaredLogger) @@ -489,7 +493,7 @@ func InitializeApp() (*App, error) { envLevelAppMetricsRepositoryImpl := repository9.NewEnvLevelAppMetricsRepositoryImpl(db, sugaredLogger) deployedAppMetricsServiceImpl := deployedAppMetrics.NewDeployedAppMetricsServiceImpl(sugaredLogger, appLevelMetricsRepositoryImpl, envLevelAppMetricsRepositoryImpl, chartRefServiceImpl) appListingServiceImpl := app2.NewAppListingServiceImpl(sugaredLogger, appListingRepositoryImpl, applicationServiceClientImpl, appRepositoryImpl, appListingViewBuilderImpl, pipelineRepositoryImpl, linkoutsRepositoryImpl, cdWorkflowRepositoryImpl, pipelineOverrideRepositoryImpl, environmentRepositoryImpl, argoUserServiceImpl, envConfigOverrideRepositoryImpl, chartRepositoryImpl, ciPipelineRepositoryImpl, dockerRegistryIpsConfigServiceImpl, userRepositoryImpl, deployedAppMetricsServiceImpl, ciArtifactRepositoryImpl) - appServiceImpl := app2.NewAppService(envConfigOverrideRepositoryImpl, pipelineOverrideRepositoryImpl, mergeUtil, sugaredLogger, pipelineRepositoryImpl, eventRESTClientImpl, eventSimpleFactoryImpl, applicationServiceClientImpl, appRepositoryImpl, configMapRepositoryImpl, chartRepositoryImpl, cdWorkflowRepositoryImpl, commonServiceImpl, chartTemplateServiceImpl, argoUserServiceImpl, pipelineStatusTimelineRepositoryImpl, pipelineStatusTimelineResourcesServiceImpl, pipelineStatusSyncDetailServiceImpl, pipelineStatusTimelineServiceImpl, appServiceConfig, appStatusServiceImpl, installedAppRepositoryImpl, installedAppVersionHistoryRepositoryImpl, scopedVariableCMCSManagerImpl, acdConfig, gitOpsConfigReadServiceImpl, gitOperationServiceImpl, deploymentTemplateServiceImpl, appListingServiceImpl) + appServiceImpl := app2.NewAppService(envConfigOverrideRepositoryImpl, pipelineOverrideRepositoryImpl, mergeUtil, sugaredLogger, pipelineRepositoryImpl, eventRESTClientImpl, eventSimpleFactoryImpl, applicationServiceClientImpl, appRepositoryImpl, configMapRepositoryImpl, chartRepositoryImpl, cdWorkflowRepositoryImpl, commonServiceImpl, chartTemplateServiceImpl, argoUserServiceImpl, pipelineStatusTimelineRepositoryImpl, pipelineStatusTimelineResourcesServiceImpl, pipelineStatusSyncDetailServiceImpl, pipelineStatusTimelineServiceImpl, appServiceConfig, appStatusServiceImpl, installedAppRepositoryImpl, installedAppVersionHistoryRepositoryImpl, scopedVariableCMCSManagerImpl, acdConfig, gitOpsConfigReadServiceImpl, gitOperationServiceImpl, deploymentTemplateServiceImpl, appListingServiceImpl, deploymentConfigServiceImpl) globalCMCSRepositoryImpl := repository2.NewGlobalCMCSRepositoryImpl(sugaredLogger, db) globalCMCSServiceImpl := pipeline.NewGlobalCMCSServiceImpl(sugaredLogger, globalCMCSRepositoryImpl) argoWorkflowExecutorImpl := executors.NewArgoWorkflowExecutorImpl(sugaredLogger) @@ -518,7 +522,7 @@ func InitializeApp() (*App, error) { ciTemplateServiceImpl := pipeline.NewCiTemplateServiceImpl(sugaredLogger, ciBuildConfigServiceImpl, ciTemplateRepositoryImpl, ciTemplateOverrideRepositoryImpl) appLabelRepositoryImpl := pipelineConfig.NewAppLabelRepositoryImpl(db) materialRepositoryImpl := pipelineConfig.NewMaterialRepositoryImpl(db) - installedAppDBServiceImpl := EAMode.NewInstalledAppDBServiceImpl(sugaredLogger, installedAppRepositoryImpl, appRepositoryImpl, userServiceImpl, environmentServiceImpl, installedAppVersionHistoryRepositoryImpl) + installedAppDBServiceImpl := EAMode.NewInstalledAppDBServiceImpl(sugaredLogger, installedAppRepositoryImpl, appRepositoryImpl, userServiceImpl, environmentServiceImpl, installedAppVersionHistoryRepositoryImpl, deploymentConfigServiceImpl) crudOperationServiceConfig, err := app2.GetCrudOperationServiceConfig() if err != nil { return nil, err @@ -563,8 +567,8 @@ func InitializeApp() (*App, error) { configMapServiceImpl := pipeline.NewConfigMapServiceImpl(chartRepositoryImpl, sugaredLogger, chartRepoRepositoryImpl, utilMergeUtil, pipelineConfigRepositoryImpl, configMapRepositoryImpl, envConfigOverrideRepositoryImpl, commonServiceImpl, appRepositoryImpl, configMapHistoryServiceImpl, environmentRepositoryImpl, scopedVariableCMCSManagerImpl) deploymentTemplateHistoryRepositoryImpl := repository13.NewDeploymentTemplateHistoryRepositoryImpl(sugaredLogger, db) deploymentTemplateHistoryServiceImpl := history.NewDeploymentTemplateHistoryServiceImpl(sugaredLogger, deploymentTemplateHistoryRepositoryImpl, pipelineRepositoryImpl, chartRepositoryImpl, userServiceImpl, cdWorkflowRepositoryImpl, scopedVariableManagerImpl, deployedAppMetricsServiceImpl, chartRefServiceImpl) - chartServiceImpl := chart.NewChartServiceImpl(chartRepositoryImpl, sugaredLogger, chartTemplateServiceImpl, chartRepoRepositoryImpl, appRepositoryImpl, utilMergeUtil, envConfigOverrideRepositoryImpl, pipelineConfigRepositoryImpl, environmentRepositoryImpl, deploymentTemplateHistoryServiceImpl, scopedVariableManagerImpl, deployedAppMetricsServiceImpl, chartRefServiceImpl, gitOpsConfigReadServiceImpl) - ciCdPipelineOrchestratorImpl := pipeline.NewCiCdPipelineOrchestrator(appRepositoryImpl, sugaredLogger, materialRepositoryImpl, pipelineRepositoryImpl, ciPipelineRepositoryImpl, ciPipelineMaterialRepositoryImpl, cdWorkflowRepositoryImpl, clientImpl, ciCdConfig, appWorkflowRepositoryImpl, environmentRepositoryImpl, attributesServiceImpl, appCrudOperationServiceImpl, userAuthServiceImpl, prePostCdScriptHistoryServiceImpl, pipelineStageServiceImpl, gitMaterialHistoryServiceImpl, ciPipelineHistoryServiceImpl, ciTemplateServiceImpl, dockerArtifactStoreRepositoryImpl, ciArtifactRepositoryImpl, configMapServiceImpl, customTagServiceImpl, genericNoteServiceImpl, chartServiceImpl, transactionUtilImpl) + chartServiceImpl := chart.NewChartServiceImpl(chartRepositoryImpl, sugaredLogger, chartTemplateServiceImpl, chartRepoRepositoryImpl, appRepositoryImpl, utilMergeUtil, envConfigOverrideRepositoryImpl, pipelineConfigRepositoryImpl, environmentRepositoryImpl, deploymentTemplateHistoryServiceImpl, scopedVariableManagerImpl, deployedAppMetricsServiceImpl, chartRefServiceImpl, gitOpsConfigReadServiceImpl, deploymentConfigServiceImpl) + ciCdPipelineOrchestratorImpl := pipeline.NewCiCdPipelineOrchestrator(appRepositoryImpl, sugaredLogger, materialRepositoryImpl, pipelineRepositoryImpl, ciPipelineRepositoryImpl, ciPipelineMaterialRepositoryImpl, cdWorkflowRepositoryImpl, clientImpl, ciCdConfig, appWorkflowRepositoryImpl, environmentRepositoryImpl, attributesServiceImpl, appCrudOperationServiceImpl, userAuthServiceImpl, prePostCdScriptHistoryServiceImpl, pipelineStageServiceImpl, gitMaterialHistoryServiceImpl, ciPipelineHistoryServiceImpl, ciTemplateServiceImpl, dockerArtifactStoreRepositoryImpl, ciArtifactRepositoryImpl, configMapServiceImpl, customTagServiceImpl, genericNoteServiceImpl, chartServiceImpl, transactionUtilImpl, gitOpsConfigReadServiceImpl, deploymentConfigServiceImpl) ecrConfig, err := pipeline.GetEcrConfig() if err != nil { return nil, err @@ -583,12 +587,12 @@ func InitializeApp() (*App, error) { imageDigestPolicyServiceImpl := imageDigestPolicy.NewImageDigestPolicyServiceImpl(sugaredLogger, qualifierMappingServiceImpl, devtronResourceSearchableKeyServiceImpl) pipelineConfigEventPublishServiceImpl := out.NewPipelineConfigEventPublishServiceImpl(sugaredLogger, pubSubClientServiceImpl) deploymentTypeOverrideServiceImpl := providerConfig.NewDeploymentTypeOverrideServiceImpl(sugaredLogger, environmentVariables, attributesServiceImpl) - cdPipelineConfigServiceImpl := pipeline.NewCdPipelineConfigServiceImpl(sugaredLogger, pipelineRepositoryImpl, environmentRepositoryImpl, pipelineConfigRepositoryImpl, appWorkflowRepositoryImpl, pipelineStageServiceImpl, appRepositoryImpl, appServiceImpl, deploymentGroupRepositoryImpl, ciCdPipelineOrchestratorImpl, appStatusRepositoryImpl, ciPipelineRepositoryImpl, prePostCdScriptHistoryServiceImpl, clusterRepositoryImpl, helmAppServiceImpl, enforcerUtilImpl, pipelineStrategyHistoryServiceImpl, chartRepositoryImpl, resourceGroupServiceImpl, propertiesConfigServiceImpl, deploymentTemplateHistoryServiceImpl, scopedVariableManagerImpl, environmentVariables, applicationServiceClientImpl, customTagServiceImpl, ciPipelineConfigServiceImpl, buildPipelineSwitchServiceImpl, argoClientWrapperServiceImpl, deployedAppMetricsServiceImpl, gitOpsConfigReadServiceImpl, gitOperationServiceImpl, chartServiceImpl, imageDigestPolicyServiceImpl, pipelineConfigEventPublishServiceImpl, deploymentTypeOverrideServiceImpl) + cdPipelineConfigServiceImpl := pipeline.NewCdPipelineConfigServiceImpl(sugaredLogger, pipelineRepositoryImpl, environmentRepositoryImpl, pipelineConfigRepositoryImpl, appWorkflowRepositoryImpl, pipelineStageServiceImpl, appRepositoryImpl, appServiceImpl, deploymentGroupRepositoryImpl, ciCdPipelineOrchestratorImpl, appStatusRepositoryImpl, ciPipelineRepositoryImpl, prePostCdScriptHistoryServiceImpl, clusterRepositoryImpl, helmAppServiceImpl, enforcerUtilImpl, pipelineStrategyHistoryServiceImpl, chartRepositoryImpl, resourceGroupServiceImpl, propertiesConfigServiceImpl, deploymentTemplateHistoryServiceImpl, scopedVariableManagerImpl, environmentVariables, applicationServiceClientImpl, customTagServiceImpl, ciPipelineConfigServiceImpl, buildPipelineSwitchServiceImpl, argoClientWrapperServiceImpl, deployedAppMetricsServiceImpl, gitOpsConfigReadServiceImpl, gitOperationServiceImpl, chartServiceImpl, imageDigestPolicyServiceImpl, pipelineConfigEventPublishServiceImpl, deploymentTypeOverrideServiceImpl, deploymentConfigServiceImpl) appArtifactManagerImpl := pipeline.NewAppArtifactManagerImpl(sugaredLogger, cdWorkflowRepositoryImpl, userServiceImpl, imageTaggingServiceImpl, ciArtifactRepositoryImpl, ciWorkflowRepositoryImpl, pipelineStageServiceImpl, cdPipelineConfigServiceImpl, dockerArtifactStoreRepositoryImpl, ciPipelineRepositoryImpl, ciTemplateServiceImpl) devtronAppCMCSServiceImpl := pipeline.NewDevtronAppCMCSServiceImpl(sugaredLogger, appServiceImpl, attributesRepositoryImpl) globalStrategyMetadataChartRefMappingRepositoryImpl := chartRepoRepository.NewGlobalStrategyMetadataChartRefMappingRepositoryImpl(db, sugaredLogger) devtronAppStrategyServiceImpl := pipeline.NewDevtronAppStrategyServiceImpl(sugaredLogger, chartRepositoryImpl, globalStrategyMetadataChartRefMappingRepositoryImpl, ciCdPipelineOrchestratorImpl, cdPipelineConfigServiceImpl) - cdWorkflowCommonServiceImpl, err := cd.NewCdWorkflowCommonServiceImpl(sugaredLogger, cdWorkflowRepositoryImpl, pipelineStatusTimelineServiceImpl, pipelineRepositoryImpl, pipelineStatusTimelineRepositoryImpl) + cdWorkflowCommonServiceImpl, err := cd.NewCdWorkflowCommonServiceImpl(sugaredLogger, cdWorkflowRepositoryImpl, pipelineStatusTimelineServiceImpl, pipelineRepositoryImpl, pipelineStatusTimelineRepositoryImpl, deploymentConfigServiceImpl) if err != nil { return nil, err } @@ -601,15 +605,15 @@ func InitializeApp() (*App, error) { if err != nil { return nil, err } - appDeploymentTypeChangeManagerImpl := pipeline.NewAppDeploymentTypeChangeManagerImpl(sugaredLogger, pipelineRepositoryImpl, appServiceImpl, appStatusRepositoryImpl, helmAppServiceImpl, applicationServiceClientImpl, appArtifactManagerImpl, cdPipelineConfigServiceImpl, gitOpsConfigReadServiceImpl, chartServiceImpl, workflowEventPublishServiceImpl) + appDeploymentTypeChangeManagerImpl := pipeline.NewAppDeploymentTypeChangeManagerImpl(sugaredLogger, pipelineRepositoryImpl, appServiceImpl, appStatusRepositoryImpl, helmAppServiceImpl, applicationServiceClientImpl, appArtifactManagerImpl, cdPipelineConfigServiceImpl, gitOpsConfigReadServiceImpl, chartServiceImpl, workflowEventPublishServiceImpl, deploymentConfigServiceImpl) devtronAppConfigServiceImpl := pipeline.NewDevtronAppConfigServiceImpl(sugaredLogger, ciCdPipelineOrchestratorImpl, appRepositoryImpl, pipelineRepositoryImpl, resourceGroupServiceImpl, enforcerUtilImpl, ciMaterialConfigServiceImpl) pipelineBuilderImpl := pipeline.NewPipelineBuilderImpl(sugaredLogger, materialRepositoryImpl, chartRepositoryImpl, ciPipelineConfigServiceImpl, ciMaterialConfigServiceImpl, appArtifactManagerImpl, devtronAppCMCSServiceImpl, devtronAppStrategyServiceImpl, appDeploymentTypeChangeManagerImpl, cdPipelineConfigServiceImpl, devtronAppConfigServiceImpl) deploymentTemplateValidationServiceImpl := deploymentTemplate.NewDeploymentTemplateValidationServiceImpl(sugaredLogger, chartRefServiceImpl, scopedVariableManagerImpl) installedAppDBExtendedServiceImpl := FullMode.NewInstalledAppDBExtendedServiceImpl(installedAppDBServiceImpl, appStatusServiceImpl, gitOpsConfigReadServiceImpl) gitOpsValidationServiceImpl := validation.NewGitOpsValidationServiceImpl(sugaredLogger, gitFactory, gitOperationServiceImpl, gitOpsConfigReadServiceImpl, chartTemplateServiceImpl, chartServiceImpl, installedAppDBExtendedServiceImpl) - devtronAppGitOpConfigServiceImpl := gitOpsConfig.NewDevtronAppGitOpConfigServiceImpl(sugaredLogger, chartRepositoryImpl, chartServiceImpl, gitOpsConfigReadServiceImpl, gitOpsValidationServiceImpl, argoClientWrapperServiceImpl) - cdHandlerImpl := pipeline.NewCdHandlerImpl(sugaredLogger, userServiceImpl, cdWorkflowRepositoryImpl, ciLogServiceImpl, ciArtifactRepositoryImpl, ciPipelineMaterialRepositoryImpl, pipelineRepositoryImpl, environmentRepositoryImpl, ciWorkflowRepositoryImpl, enforcerUtilImpl, resourceGroupServiceImpl, imageTaggingServiceImpl, k8sServiceImpl, workflowServiceImpl, clusterServiceImplExtended, blobStorageConfigServiceImpl, customTagServiceImpl) - appWorkflowServiceImpl := appWorkflow2.NewAppWorkflowServiceImpl(sugaredLogger, appWorkflowRepositoryImpl, ciCdPipelineOrchestratorImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, enforcerUtilImpl, resourceGroupServiceImpl, appRepositoryImpl, userAuthServiceImpl, chartServiceImpl) + devtronAppGitOpConfigServiceImpl := gitOpsConfig.NewDevtronAppGitOpConfigServiceImpl(sugaredLogger, chartRepositoryImpl, chartServiceImpl, gitOpsConfigReadServiceImpl, gitOpsValidationServiceImpl, argoClientWrapperServiceImpl, deploymentConfigServiceImpl) + cdHandlerImpl := pipeline.NewCdHandlerImpl(sugaredLogger, userServiceImpl, cdWorkflowRepositoryImpl, ciLogServiceImpl, ciArtifactRepositoryImpl, ciPipelineMaterialRepositoryImpl, pipelineRepositoryImpl, environmentRepositoryImpl, ciWorkflowRepositoryImpl, enforcerUtilImpl, resourceGroupServiceImpl, imageTaggingServiceImpl, k8sServiceImpl, workflowServiceImpl, clusterServiceImplExtended, blobStorageConfigServiceImpl, customTagServiceImpl, deploymentConfigServiceImpl) + appWorkflowServiceImpl := appWorkflow2.NewAppWorkflowServiceImpl(sugaredLogger, appWorkflowRepositoryImpl, ciCdPipelineOrchestratorImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, enforcerUtilImpl, resourceGroupServiceImpl, appRepositoryImpl, userAuthServiceImpl, chartServiceImpl, deploymentConfigServiceImpl) appCloneServiceImpl := appClone.NewAppCloneServiceImpl(sugaredLogger, pipelineBuilderImpl, attributesServiceImpl, chartServiceImpl, configMapServiceImpl, appWorkflowServiceImpl, appListingServiceImpl, propertiesConfigServiceImpl, pipelineStageServiceImpl, ciTemplateServiceImpl, appRepositoryImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, ciPipelineConfigServiceImpl, gitOpsConfigReadServiceImpl) deploymentTemplateRepositoryImpl := repository2.NewDeploymentTemplateRepositoryImpl(db, sugaredLogger) generateManifestDeploymentTemplateServiceImpl, err := generateManifest.NewDeploymentTemplateServiceImpl(sugaredLogger, chartServiceImpl, appListingServiceImpl, deploymentTemplateRepositoryImpl, helmAppServiceImpl, chartTemplateServiceImpl, helmAppClientImpl, k8sServiceImpl, propertiesConfigServiceImpl, deploymentTemplateHistoryServiceImpl, environmentRepositoryImpl, appRepositoryImpl, scopedVariableManagerImpl, chartRefServiceImpl, pipelineOverrideRepositoryImpl, chartRepositoryImpl) @@ -626,19 +630,19 @@ func InitializeApp() (*App, error) { pipelineConfigRestHandlerImpl := configure.NewPipelineRestHandlerImpl(pipelineBuilderImpl, sugaredLogger, deploymentTemplateValidationServiceImpl, chartServiceImpl, devtronAppGitOpConfigServiceImpl, propertiesConfigServiceImpl, userServiceImpl, teamServiceImpl, enforcerImpl, ciHandlerImpl, validate, clientImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, enforcerUtilImpl, dockerRegistryConfigImpl, cdHandlerImpl, appCloneServiceImpl, generateManifestDeploymentTemplateServiceImpl, appWorkflowServiceImpl, materialRepositoryImpl, policyServiceImpl, imageScanResultRepositoryImpl, gitProviderRepositoryImpl, argoUserServiceImpl, ciPipelineMaterialRepositoryImpl, imageTaggingServiceImpl, ciArtifactRepositoryImpl, deployedAppMetricsServiceImpl, chartRefServiceImpl, ciCdPipelineOrchestratorImpl) gitOpsManifestPushServiceImpl := publish.NewGitOpsManifestPushServiceImpl(sugaredLogger, pipelineStatusTimelineServiceImpl, pipelineOverrideRepositoryImpl, acdConfig, chartRefServiceImpl, gitOpsConfigReadServiceImpl, chartServiceImpl, gitOperationServiceImpl, argoClientWrapperServiceImpl, transactionUtilImpl) argoK8sClientImpl := argocdServer.NewArgoK8sClientImpl(sugaredLogger, k8sServiceImpl) - manifestCreationServiceImpl := manifest.NewManifestCreationServiceImpl(sugaredLogger, dockerRegistryIpsConfigServiceImpl, chartRefServiceImpl, scopedVariableCMCSManagerImpl, k8sCommonServiceImpl, deployedAppMetricsServiceImpl, imageDigestPolicyServiceImpl, mergeUtil, appCrudOperationServiceImpl, deploymentTemplateServiceImpl, applicationServiceClientImpl, configMapHistoryRepositoryImpl, configMapRepositoryImpl, chartRepositoryImpl, envConfigOverrideRepositoryImpl, environmentRepositoryImpl, pipelineRepositoryImpl, ciArtifactRepositoryImpl, pipelineOverrideRepositoryImpl, pipelineStrategyHistoryRepositoryImpl, pipelineConfigRepositoryImpl, deploymentTemplateHistoryRepositoryImpl) + manifestCreationServiceImpl := manifest.NewManifestCreationServiceImpl(sugaredLogger, dockerRegistryIpsConfigServiceImpl, chartRefServiceImpl, scopedVariableCMCSManagerImpl, k8sCommonServiceImpl, deployedAppMetricsServiceImpl, imageDigestPolicyServiceImpl, mergeUtil, appCrudOperationServiceImpl, deploymentTemplateServiceImpl, applicationServiceClientImpl, configMapHistoryRepositoryImpl, configMapRepositoryImpl, chartRepositoryImpl, envConfigOverrideRepositoryImpl, environmentRepositoryImpl, pipelineRepositoryImpl, ciArtifactRepositoryImpl, pipelineOverrideRepositoryImpl, pipelineStrategyHistoryRepositoryImpl, pipelineConfigRepositoryImpl, deploymentTemplateHistoryRepositoryImpl, deploymentConfigServiceImpl) deployedConfigurationHistoryServiceImpl := history.NewDeployedConfigurationHistoryServiceImpl(sugaredLogger, userServiceImpl, deploymentTemplateHistoryServiceImpl, pipelineStrategyHistoryServiceImpl, configMapHistoryServiceImpl, cdWorkflowRepositoryImpl, scopedVariableCMCSManagerImpl) userDeploymentRequestRepositoryImpl := repository15.NewUserDeploymentRequestRepositoryImpl(db, transactionUtilImpl) userDeploymentRequestServiceImpl := service2.NewUserDeploymentRequestServiceImpl(sugaredLogger, userDeploymentRequestRepositoryImpl) manifestPushConfigRepositoryImpl := repository10.NewManifestPushConfigRepository(sugaredLogger, db) scanToolExecutionHistoryMappingRepositoryImpl := security.NewScanToolExecutionHistoryMappingRepositoryImpl(db, sugaredLogger) imageScanServiceImpl := security2.NewImageScanServiceImpl(sugaredLogger, imageScanHistoryRepositoryImpl, imageScanResultRepositoryImpl, imageScanObjectMetaRepositoryImpl, cveStoreRepositoryImpl, imageScanDeployInfoRepositoryImpl, userServiceImpl, teamRepositoryImpl, appRepositoryImpl, environmentServiceImpl, ciArtifactRepositoryImpl, policyServiceImpl, pipelineRepositoryImpl, ciPipelineRepositoryImpl, scanToolMetadataRepositoryImpl, scanToolExecutionHistoryMappingRepositoryImpl, cvePolicyRepositoryImpl) - triggerServiceImpl, err := devtronApps.NewTriggerServiceImpl(sugaredLogger, cdWorkflowCommonServiceImpl, gitOpsManifestPushServiceImpl, gitOpsConfigReadServiceImpl, argoK8sClientImpl, acdConfig, argoClientWrapperServiceImpl, pipelineStatusTimelineServiceImpl, chartTemplateServiceImpl, workflowEventPublishServiceImpl, manifestCreationServiceImpl, deployedConfigurationHistoryServiceImpl, argoUserServiceImpl, pipelineStageServiceImpl, globalPluginServiceImpl, customTagServiceImpl, pluginInputVariableParserImpl, prePostCdScriptHistoryServiceImpl, scopedVariableCMCSManagerImpl, workflowServiceImpl, imageDigestPolicyServiceImpl, userServiceImpl, clientImpl, helmAppServiceImpl, enforcerUtilImpl, userDeploymentRequestServiceImpl, helmAppClientImpl, eventSimpleFactoryImpl, eventRESTClientImpl, environmentVariables, appRepositoryImpl, ciPipelineMaterialRepositoryImpl, imageScanHistoryRepositoryImpl, imageScanDeployInfoRepositoryImpl, pipelineRepositoryImpl, pipelineOverrideRepositoryImpl, manifestPushConfigRepositoryImpl, chartRepositoryImpl, environmentRepositoryImpl, cdWorkflowRepositoryImpl, ciWorkflowRepositoryImpl, ciArtifactRepositoryImpl, ciTemplateServiceImpl, materialRepositoryImpl, appLabelRepositoryImpl, ciPipelineRepositoryImpl, appWorkflowRepositoryImpl, dockerArtifactStoreRepositoryImpl, imageScanServiceImpl, k8sServiceImpl, transactionUtilImpl) + triggerServiceImpl, err := devtronApps.NewTriggerServiceImpl(sugaredLogger, cdWorkflowCommonServiceImpl, gitOpsManifestPushServiceImpl, gitOpsConfigReadServiceImpl, argoK8sClientImpl, acdConfig, argoClientWrapperServiceImpl, pipelineStatusTimelineServiceImpl, chartTemplateServiceImpl, workflowEventPublishServiceImpl, manifestCreationServiceImpl, deployedConfigurationHistoryServiceImpl, argoUserServiceImpl, pipelineStageServiceImpl, globalPluginServiceImpl, customTagServiceImpl, pluginInputVariableParserImpl, prePostCdScriptHistoryServiceImpl, scopedVariableCMCSManagerImpl, workflowServiceImpl, imageDigestPolicyServiceImpl, userServiceImpl, clientImpl, helmAppServiceImpl, enforcerUtilImpl, userDeploymentRequestServiceImpl, helmAppClientImpl, eventSimpleFactoryImpl, eventRESTClientImpl, environmentVariables, appRepositoryImpl, ciPipelineMaterialRepositoryImpl, imageScanHistoryRepositoryImpl, imageScanDeployInfoRepositoryImpl, pipelineRepositoryImpl, pipelineOverrideRepositoryImpl, manifestPushConfigRepositoryImpl, chartRepositoryImpl, environmentRepositoryImpl, cdWorkflowRepositoryImpl, ciWorkflowRepositoryImpl, ciArtifactRepositoryImpl, ciTemplateServiceImpl, materialRepositoryImpl, appLabelRepositoryImpl, ciPipelineRepositoryImpl, appWorkflowRepositoryImpl, dockerArtifactStoreRepositoryImpl, imageScanServiceImpl, k8sServiceImpl, transactionUtilImpl, deploymentConfigServiceImpl) if err != nil { return nil, err } commonArtifactServiceImpl := artifacts.NewCommonArtifactServiceImpl(sugaredLogger, ciArtifactRepositoryImpl) - workflowDagExecutorImpl := dag.NewWorkflowDagExecutorImpl(sugaredLogger, pipelineRepositoryImpl, cdWorkflowRepositoryImpl, ciArtifactRepositoryImpl, enforcerUtilImpl, appWorkflowRepositoryImpl, pipelineStageServiceImpl, ciWorkflowRepositoryImpl, ciPipelineRepositoryImpl, pipelineStageRepositoryImpl, globalPluginRepositoryImpl, eventRESTClientImpl, eventSimpleFactoryImpl, customTagServiceImpl, pipelineStatusTimelineServiceImpl, helmAppServiceImpl, cdWorkflowCommonServiceImpl, triggerServiceImpl, userDeploymentRequestServiceImpl, manifestCreationServiceImpl, commonArtifactServiceImpl) + workflowDagExecutorImpl := dag.NewWorkflowDagExecutorImpl(sugaredLogger, pipelineRepositoryImpl, cdWorkflowRepositoryImpl, ciArtifactRepositoryImpl, enforcerUtilImpl, appWorkflowRepositoryImpl, pipelineStageServiceImpl, ciWorkflowRepositoryImpl, ciPipelineRepositoryImpl, pipelineStageRepositoryImpl, globalPluginRepositoryImpl, eventRESTClientImpl, eventSimpleFactoryImpl, customTagServiceImpl, pipelineStatusTimelineServiceImpl, helmAppServiceImpl, cdWorkflowCommonServiceImpl, triggerServiceImpl, userDeploymentRequestServiceImpl, manifestCreationServiceImpl, commonArtifactServiceImpl, deploymentConfigServiceImpl) externalCiRestHandlerImpl := restHandler.NewExternalCiRestHandlerImpl(sugaredLogger, validate, userServiceImpl, enforcerImpl, workflowDagExecutorImpl) pubSubClientRestHandlerImpl := restHandler.NewPubSubClientRestHandlerImpl(pubSubClientServiceImpl, sugaredLogger, ciCdConfig) webhookRouterImpl := router.NewWebhookRouterImpl(gitWebhookRestHandlerImpl, pipelineConfigRestHandlerImpl, externalCiRestHandlerImpl, pubSubClientRestHandlerImpl) @@ -692,7 +696,7 @@ func InitializeApp() (*App, error) { if err != nil { return nil, err } - installedAppResourceServiceImpl := resource.NewInstalledAppResourceServiceImpl(sugaredLogger, installedAppRepositoryImpl, appStoreApplicationVersionRepositoryImpl, applicationServiceClientImpl, acdAuthConfig, installedAppVersionHistoryRepositoryImpl, argoUserServiceImpl, helmAppClientImpl, helmAppServiceImpl, appStatusServiceImpl, k8sCommonServiceImpl, k8sApplicationServiceImpl, k8sServiceImpl) + installedAppResourceServiceImpl := resource.NewInstalledAppResourceServiceImpl(sugaredLogger, installedAppRepositoryImpl, appStoreApplicationVersionRepositoryImpl, applicationServiceClientImpl, acdAuthConfig, installedAppVersionHistoryRepositoryImpl, argoUserServiceImpl, helmAppClientImpl, helmAppServiceImpl, appStatusServiceImpl, k8sCommonServiceImpl, k8sApplicationServiceImpl, k8sServiceImpl, deploymentConfigServiceImpl) chartGroupEntriesRepositoryImpl := repository17.NewChartGroupEntriesRepositoryImpl(db, sugaredLogger) chartGroupReposotoryImpl := repository17.NewChartGroupReposotoryImpl(db, sugaredLogger) chartGroupDeploymentRepositoryImpl := repository17.NewChartGroupDeploymentRepositoryImpl(db, sugaredLogger) @@ -701,24 +705,24 @@ func InitializeApp() (*App, error) { clusterInstalledAppsRepositoryImpl := repository3.NewClusterInstalledAppsRepositoryImpl(db, sugaredLogger) appStoreValuesServiceImpl := service3.NewAppStoreValuesServiceImpl(sugaredLogger, appStoreApplicationVersionRepositoryImpl, installedAppRepositoryImpl, appStoreVersionValuesRepositoryImpl, userServiceImpl) appStoreDeploymentCommonServiceImpl := appStoreDeploymentCommon.NewAppStoreDeploymentCommonServiceImpl(sugaredLogger, appStoreApplicationVersionRepositoryImpl, chartTemplateServiceImpl) - fullModeDeploymentServiceImpl := deployment.NewFullModeDeploymentServiceImpl(sugaredLogger, applicationServiceClientImpl, argoK8sClientImpl, acdAuthConfig, chartGroupDeploymentRepositoryImpl, installedAppRepositoryImpl, installedAppVersionHistoryRepositoryImpl, argoUserServiceImpl, appStoreDeploymentCommonServiceImpl, helmAppServiceImpl, appStatusServiceImpl, pipelineStatusTimelineServiceImpl, userServiceImpl, pipelineStatusTimelineRepositoryImpl, appStoreApplicationVersionRepositoryImpl, argoClientWrapperServiceImpl, acdConfig, gitOperationServiceImpl, gitOpsConfigReadServiceImpl, gitOpsValidationServiceImpl, environmentRepositoryImpl) + fullModeDeploymentServiceImpl := deployment.NewFullModeDeploymentServiceImpl(sugaredLogger, applicationServiceClientImpl, argoK8sClientImpl, acdAuthConfig, chartGroupDeploymentRepositoryImpl, installedAppRepositoryImpl, installedAppVersionHistoryRepositoryImpl, argoUserServiceImpl, appStoreDeploymentCommonServiceImpl, helmAppServiceImpl, appStatusServiceImpl, pipelineStatusTimelineServiceImpl, userServiceImpl, pipelineStatusTimelineRepositoryImpl, appStoreApplicationVersionRepositoryImpl, argoClientWrapperServiceImpl, acdConfig, gitOperationServiceImpl, gitOpsConfigReadServiceImpl, gitOpsValidationServiceImpl, environmentRepositoryImpl, deploymentConfigServiceImpl) appStoreValidatorImpl := service4.NewAppAppStoreValidatorImpl(sugaredLogger) - appStoreDeploymentDBServiceImpl := service4.NewAppStoreDeploymentDBServiceImpl(sugaredLogger, installedAppRepositoryImpl, appStoreApplicationVersionRepositoryImpl, appRepositoryImpl, environmentServiceImpl, clusterServiceImplExtended, installedAppVersionHistoryRepositoryImpl, environmentVariables, gitOpsConfigReadServiceImpl, deploymentTypeOverrideServiceImpl, fullModeDeploymentServiceImpl, appStoreValidatorImpl, installedAppDBServiceImpl) + appStoreDeploymentDBServiceImpl := service4.NewAppStoreDeploymentDBServiceImpl(sugaredLogger, installedAppRepositoryImpl, appStoreApplicationVersionRepositoryImpl, appRepositoryImpl, environmentServiceImpl, clusterServiceImplExtended, installedAppVersionHistoryRepositoryImpl, environmentVariables, gitOpsConfigReadServiceImpl, deploymentTypeOverrideServiceImpl, fullModeDeploymentServiceImpl, appStoreValidatorImpl, installedAppDBServiceImpl, deploymentConfigServiceImpl) eaModeDeploymentServiceImpl := EAMode.NewEAModeDeploymentServiceImpl(sugaredLogger, helmAppServiceImpl, appStoreApplicationVersionRepositoryImpl, helmAppClientImpl, installedAppRepositoryImpl, ociRegistryConfigRepositoryImpl) deletePostProcessorImpl := service4.NewDeletePostProcessorImpl(sugaredLogger) - appStoreDeploymentServiceImpl := service4.NewAppStoreDeploymentServiceImpl(sugaredLogger, installedAppRepositoryImpl, installedAppDBServiceImpl, appStoreDeploymentDBServiceImpl, chartGroupDeploymentRepositoryImpl, appStoreApplicationVersionRepositoryImpl, appRepositoryImpl, eaModeDeploymentServiceImpl, fullModeDeploymentServiceImpl, environmentServiceImpl, helmAppServiceImpl, installedAppVersionHistoryRepositoryImpl, environmentVariables, acdConfig, gitOpsConfigReadServiceImpl, deletePostProcessorImpl, appStoreValidatorImpl) + appStoreDeploymentServiceImpl := service4.NewAppStoreDeploymentServiceImpl(sugaredLogger, installedAppRepositoryImpl, installedAppDBServiceImpl, appStoreDeploymentDBServiceImpl, chartGroupDeploymentRepositoryImpl, appStoreApplicationVersionRepositoryImpl, appRepositoryImpl, eaModeDeploymentServiceImpl, fullModeDeploymentServiceImpl, environmentServiceImpl, helmAppServiceImpl, installedAppVersionHistoryRepositoryImpl, environmentVariables, acdConfig, gitOpsConfigReadServiceImpl, deletePostProcessorImpl, appStoreValidatorImpl, deploymentConfigServiceImpl) appStoreAppsEventPublishServiceImpl := out.NewAppStoreAppsEventPublishServiceImpl(sugaredLogger, pubSubClientServiceImpl) chartGroupServiceImpl, err := chartGroup.NewChartGroupServiceImpl(sugaredLogger, chartGroupEntriesRepositoryImpl, chartGroupReposotoryImpl, chartGroupDeploymentRepositoryImpl, installedAppRepositoryImpl, appStoreVersionValuesRepositoryImpl, appStoreRepositoryImpl, userAuthServiceImpl, appStoreApplicationVersionRepositoryImpl, environmentServiceImpl, teamRepositoryImpl, clusterInstalledAppsRepositoryImpl, appStoreValuesServiceImpl, appStoreDeploymentServiceImpl, appStoreDeploymentDBServiceImpl, argoUserServiceImpl, pipelineStatusTimelineServiceImpl, acdConfig, fullModeDeploymentServiceImpl, gitOperationServiceImpl, installedAppDBExtendedServiceImpl, appStoreAppsEventPublishServiceImpl) if err != nil { return nil, err } cdPipelineEventPublishServiceImpl := out.NewCDPipelineEventPublishServiceImpl(sugaredLogger, pubSubClientServiceImpl) - workflowStatusServiceImpl, err := status2.NewWorkflowStatusServiceImpl(sugaredLogger, workflowDagExecutorImpl, pipelineStatusTimelineServiceImpl, appServiceImpl, appStatusServiceImpl, acdConfig, appServiceConfig, argoUserServiceImpl, pipelineStatusSyncDetailServiceImpl, argoClientWrapperServiceImpl, cdPipelineEventPublishServiceImpl, cdWorkflowRepositoryImpl, pipelineOverrideRepositoryImpl, installedAppVersionHistoryRepositoryImpl, appRepositoryImpl, environmentRepositoryImpl, installedAppRepositoryImpl, pipelineStatusTimelineRepositoryImpl, pipelineRepositoryImpl, applicationServiceClientImpl, appListingServiceImpl) + workflowStatusServiceImpl, err := status2.NewWorkflowStatusServiceImpl(sugaredLogger, workflowDagExecutorImpl, pipelineStatusTimelineServiceImpl, appServiceImpl, appStatusServiceImpl, acdConfig, appServiceConfig, argoUserServiceImpl, pipelineStatusSyncDetailServiceImpl, argoClientWrapperServiceImpl, cdPipelineEventPublishServiceImpl, cdWorkflowRepositoryImpl, pipelineOverrideRepositoryImpl, installedAppVersionHistoryRepositoryImpl, appRepositoryImpl, environmentRepositoryImpl, installedAppRepositoryImpl, pipelineStatusTimelineRepositoryImpl, pipelineRepositoryImpl, applicationServiceClientImpl, appListingServiceImpl, deploymentConfigServiceImpl) if err != nil { return nil, err } cdApplicationStatusUpdateHandlerImpl := cron2.NewCdApplicationStatusUpdateHandlerImpl(sugaredLogger, appServiceImpl, workflowDagExecutorImpl, installedAppDBServiceImpl, appServiceConfig, pipelineStatusTimelineRepositoryImpl, eventRESTClientImpl, appListingRepositoryImpl, cdWorkflowRepositoryImpl, pipelineRepositoryImpl, installedAppVersionHistoryRepositoryImpl, installedAppRepositoryImpl, cronLoggerImpl, cdWorkflowCommonServiceImpl, workflowStatusServiceImpl) - installedAppDeploymentTypeChangeServiceImpl := deploymentTypeChange.NewInstalledAppDeploymentTypeChangeServiceImpl(sugaredLogger, installedAppRepositoryImpl, installedAppVersionHistoryRepositoryImpl, appStatusRepositoryImpl, gitOpsConfigReadServiceImpl, environmentRepositoryImpl, applicationServiceClientImpl, k8sCommonServiceImpl, k8sServiceImpl, fullModeDeploymentServiceImpl, eaModeDeploymentServiceImpl, argoClientWrapperServiceImpl, chartGroupServiceImpl, helmAppServiceImpl, argoUserServiceImpl, clusterServiceImplExtended, appRepositoryImpl) + installedAppDeploymentTypeChangeServiceImpl := deploymentTypeChange.NewInstalledAppDeploymentTypeChangeServiceImpl(sugaredLogger, installedAppRepositoryImpl, installedAppVersionHistoryRepositoryImpl, appStatusRepositoryImpl, gitOpsConfigReadServiceImpl, environmentRepositoryImpl, applicationServiceClientImpl, k8sCommonServiceImpl, k8sServiceImpl, fullModeDeploymentServiceImpl, eaModeDeploymentServiceImpl, argoClientWrapperServiceImpl, chartGroupServiceImpl, helmAppServiceImpl, argoUserServiceImpl, clusterServiceImplExtended, appRepositoryImpl, deploymentConfigServiceImpl) installedAppRestHandlerImpl := appStore.NewInstalledAppRestHandlerImpl(sugaredLogger, userServiceImpl, enforcerImpl, enforcerUtilImpl, enforcerUtilHelmImpl, installedAppDBExtendedServiceImpl, installedAppResourceServiceImpl, chartGroupServiceImpl, validate, clusterServiceImplExtended, applicationServiceClientImpl, appStoreDeploymentServiceImpl, appStoreDeploymentDBServiceImpl, helmAppClientImpl, argoUserServiceImpl, cdApplicationStatusUpdateHandlerImpl, installedAppRepositoryImpl, appCrudOperationServiceImpl, installedAppDeploymentTypeChangeServiceImpl) appStoreValuesRestHandlerImpl := appStoreValues.NewAppStoreValuesRestHandlerImpl(sugaredLogger, userServiceImpl, appStoreValuesServiceImpl) appStoreValuesRouterImpl := appStoreValues.NewAppStoreValuesRouterImpl(appStoreValuesRestHandlerImpl) @@ -816,12 +820,12 @@ func InitializeApp() (*App, error) { webhookListenerRouterImpl := router.NewWebhookListenerRouterImpl(webhookEventHandlerImpl) appFilteringRestHandlerImpl := appList.NewAppFilteringRestHandlerImpl(sugaredLogger, teamServiceImpl, enforcerImpl, userServiceImpl, clusterServiceImplExtended, environmentServiceImpl) appFilteringRouterImpl := appList2.NewAppFilteringRouterImpl(appFilteringRestHandlerImpl) - appListingRestHandlerImpl := appList.NewAppListingRestHandlerImpl(applicationServiceClientImpl, appListingServiceImpl, enforcerImpl, pipelineBuilderImpl, sugaredLogger, enforcerUtilImpl, deploymentGroupServiceImpl, userServiceImpl, helmAppClientImpl, helmAppServiceImpl, argoUserServiceImpl, k8sCommonServiceImpl, installedAppDBExtendedServiceImpl, installedAppResourceServiceImpl, cdApplicationStatusUpdateHandlerImpl, pipelineRepositoryImpl, appStatusServiceImpl, installedAppRepositoryImpl, genericNoteServiceImpl, k8sApplicationServiceImpl, generateManifestDeploymentTemplateServiceImpl) + appListingRestHandlerImpl := appList.NewAppListingRestHandlerImpl(applicationServiceClientImpl, appListingServiceImpl, enforcerImpl, pipelineBuilderImpl, sugaredLogger, enforcerUtilImpl, deploymentGroupServiceImpl, userServiceImpl, helmAppClientImpl, helmAppServiceImpl, argoUserServiceImpl, k8sCommonServiceImpl, installedAppDBExtendedServiceImpl, installedAppResourceServiceImpl, cdApplicationStatusUpdateHandlerImpl, pipelineRepositoryImpl, appStatusServiceImpl, installedAppRepositoryImpl, genericNoteServiceImpl, k8sApplicationServiceImpl, generateManifestDeploymentTemplateServiceImpl, deploymentConfigServiceImpl) appListingRouterImpl := appList2.NewAppListingRouterImpl(appListingRestHandlerImpl) appInfoRestHandlerImpl := appInfo.NewAppInfoRestHandlerImpl(sugaredLogger, appCrudOperationServiceImpl, userServiceImpl, validate, enforcerUtilImpl, enforcerImpl, helmAppServiceImpl, enforcerUtilHelmImpl, genericNoteServiceImpl) appInfoRouterImpl := appInfo2.NewAppInfoRouterImpl(sugaredLogger, appInfoRestHandlerImpl) - deploymentConfigServiceImpl := pipeline.NewDeploymentConfigServiceImpl(sugaredLogger, envConfigOverrideRepositoryImpl, chartRepositoryImpl, pipelineRepositoryImpl, pipelineConfigRepositoryImpl, configMapRepositoryImpl, configMapHistoryServiceImpl, scopedVariableCMCSManagerImpl, deployedAppMetricsServiceImpl, chartRefServiceImpl) - pipelineTriggerRestHandlerImpl := trigger.NewPipelineRestHandler(appServiceImpl, userServiceImpl, validate, enforcerImpl, teamServiceImpl, sugaredLogger, enforcerUtilImpl, deploymentGroupServiceImpl, argoUserServiceImpl, deploymentConfigServiceImpl, deployedAppServiceImpl, triggerServiceImpl, workflowEventPublishServiceImpl) + pipelineDeploymentConfigServiceImpl := pipeline.NewPipelineDeploymentConfigServiceImpl(sugaredLogger, envConfigOverrideRepositoryImpl, chartRepositoryImpl, pipelineRepositoryImpl, pipelineConfigRepositoryImpl, configMapRepositoryImpl, configMapHistoryServiceImpl, scopedVariableCMCSManagerImpl, deployedAppMetricsServiceImpl, chartRefServiceImpl) + pipelineTriggerRestHandlerImpl := trigger.NewPipelineRestHandler(appServiceImpl, userServiceImpl, validate, enforcerImpl, teamServiceImpl, sugaredLogger, enforcerUtilImpl, deploymentGroupServiceImpl, argoUserServiceImpl, pipelineDeploymentConfigServiceImpl, deployedAppServiceImpl, triggerServiceImpl, workflowEventPublishServiceImpl) sseSSE := sse.NewSSE() pipelineTriggerRouterImpl := trigger2.NewPipelineTriggerRouter(pipelineTriggerRestHandlerImpl, sseSSE) webhookDataRestHandlerImpl := webhook.NewWebhookDataRestHandlerImpl(sugaredLogger, userServiceImpl, ciPipelineMaterialRepositoryImpl, enforcerUtilImpl, enforcerImpl, clientImpl, webhookEventDataConfigImpl) @@ -936,7 +940,7 @@ func InitializeApp() (*App, error) { cdWorkflowServiceImpl := cd.NewCdWorkflowServiceImpl(sugaredLogger, cdWorkflowRepositoryImpl) cdWorkflowRunnerServiceImpl := cd.NewCdWorkflowRunnerServiceImpl(sugaredLogger, cdWorkflowRepositoryImpl) webhookServiceImpl := pipeline.NewWebhookServiceImpl(ciArtifactRepositoryImpl, sugaredLogger, ciPipelineRepositoryImpl, ciWorkflowRepositoryImpl, cdWorkflowCommonServiceImpl) - workflowEventProcessorImpl, err := in.NewWorkflowEventProcessorImpl(sugaredLogger, pubSubClientServiceImpl, cdWorkflowServiceImpl, cdWorkflowRunnerServiceImpl, workflowDagExecutorImpl, argoUserServiceImpl, ciHandlerImpl, cdHandlerImpl, eventSimpleFactoryImpl, eventRESTClientImpl, triggerServiceImpl, deployedAppServiceImpl, webhookServiceImpl, validate, environmentVariables, cdWorkflowCommonServiceImpl, cdPipelineConfigServiceImpl, userDeploymentRequestServiceImpl, pipelineRepositoryImpl, ciArtifactRepositoryImpl, cdWorkflowRepositoryImpl) + workflowEventProcessorImpl, err := in.NewWorkflowEventProcessorImpl(sugaredLogger, pubSubClientServiceImpl, cdWorkflowServiceImpl, cdWorkflowRunnerServiceImpl, workflowDagExecutorImpl, argoUserServiceImpl, ciHandlerImpl, cdHandlerImpl, eventSimpleFactoryImpl, eventRESTClientImpl, triggerServiceImpl, deployedAppServiceImpl, webhookServiceImpl, validate, environmentVariables, cdWorkflowCommonServiceImpl, cdPipelineConfigServiceImpl, userDeploymentRequestServiceImpl, pipelineRepositoryImpl, ciArtifactRepositoryImpl, cdWorkflowRepositoryImpl, deploymentConfigServiceImpl) if err != nil { return nil, err } From d194d1016d2931b824ca10845cbc9c543e8dddc3 Mon Sep 17 00:00:00 2001 From: ayushmaheshwari Date: Thu, 4 Jul 2024 22:21:26 +0530 Subject: [PATCH 16/45] nil pointer fix --- .../installedApp/service/EAMode/InstalledAppDBService.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/appStore/installedApp/service/EAMode/InstalledAppDBService.go b/pkg/appStore/installedApp/service/EAMode/InstalledAppDBService.go index e3e9d2a6f7..7a95e9a011 100644 --- a/pkg/appStore/installedApp/service/EAMode/InstalledAppDBService.go +++ b/pkg/appStore/installedApp/service/EAMode/InstalledAppDBService.go @@ -84,6 +84,7 @@ func NewInstalledAppDBServiceImpl(logger *zap.SugaredLogger, UserService: userService, EnvironmentService: environmentService, InstalledAppRepositoryHistory: installedAppRepositoryHistory, + deploymentConfigService: deploymentConfigService, } } From ff5ea7bb48411299c80cf6efd1d8cd4a136471f9 Mon Sep 17 00:00:00 2001 From: ayushmaheshwari Date: Fri, 5 Jul 2024 00:32:19 +0530 Subject: [PATCH 17/45] fix cd pipeline helm app create --- .../DeploymentPipelineConfigService.go | 47 +++++++++---------- 1 file changed, 22 insertions(+), 25 deletions(-) diff --git a/pkg/pipeline/DeploymentPipelineConfigService.go b/pkg/pipeline/DeploymentPipelineConfigService.go index 1e05fc380f..8e8ca746be 100644 --- a/pkg/pipeline/DeploymentPipelineConfigService.go +++ b/pkg/pipeline/DeploymentPipelineConfigService.go @@ -411,15 +411,15 @@ func (impl *CdPipelineConfigServiceImpl) CreateCdPipelines(pipelineCreateRequest return nil, err } + AppDeploymentConfig, err := impl.deploymentConfigService.GetAndMigrateConfigIfAbsentForDevtronApps(app.Id, 0) + if err != nil { + impl.logger.Errorw("error in fetching deployment config by appId", "appId", app.Id, "err", err) + return nil, err + } + // TODO: creating git repo for all apps irrespective of acd or helm if gitOpsConfigurationStatus.IsGitOpsConfigured && isGitOpsRequiredForCD && !pipelineCreateRequest.IsCloneAppReq { - AppDeploymentConfig, err := impl.deploymentConfigService.GetAndMigrateConfigIfAbsentForDevtronApps(app.Id, 0) - if err != nil { - impl.logger.Errorw("error in fetching deployment config by appId", "appId", app.Id, "err", err) - return nil, err - } - if gitOps.IsGitOpsRepoNotConfigured(AppDeploymentConfig.RepoURL) { if gitOpsConfigurationStatus.AllowCustomRepository || AppDeploymentConfig.ConfigType == bean4.CUSTOM.String() { apiErr := &util.ApiError{ @@ -446,29 +446,26 @@ func (impl *CdPipelineConfigServiceImpl) CreateCdPipelines(pipelineCreateRequest return nil, err } } - for _, pipeline := range pipelineCreateRequest.Pipelines { - envDeploymentConfig := &bean4.DeploymentConfig{ - AppId: app.Id, - EnvironmentId: pipeline.EnvironmentId, - ConfigType: AppDeploymentConfig.ConfigType, - DeploymentAppType: pipeline.DeploymentAppType, - RepoURL: AppDeploymentConfig.RepoURL, - RepoName: AppDeploymentConfig.RepoName, - ChartLocation: AppDeploymentConfig.ChartLocation, - Active: true, - } - envDeploymentConfig, err := impl.deploymentConfigService.CreateOrUpdateConfig(nil, envDeploymentConfig, pipelineCreateRequest.UserId) - if err != nil { - impl.logger.Errorw("error in fetching creating env config", "appId", app.Id, "envId", pipeline.EnvironmentId, "err", err) - return nil, err - } - - } - } for _, pipeline := range pipelineCreateRequest.Pipelines { + envDeploymentConfig := &bean4.DeploymentConfig{ + AppId: app.Id, + EnvironmentId: pipeline.EnvironmentId, + ConfigType: AppDeploymentConfig.ConfigType, + DeploymentAppType: pipeline.DeploymentAppType, + RepoURL: AppDeploymentConfig.RepoURL, + RepoName: AppDeploymentConfig.RepoName, + ChartLocation: AppDeploymentConfig.ChartLocation, + Active: true, + } + envDeploymentConfig, err := impl.deploymentConfigService.CreateOrUpdateConfig(nil, envDeploymentConfig, pipelineCreateRequest.UserId) + if err != nil { + impl.logger.Errorw("error in fetching creating env config", "appId", app.Id, "envId", pipeline.EnvironmentId, "err", err) + return nil, err + } + id, err := impl.createCdPipeline(ctx, app, pipeline, pipelineCreateRequest.UserId) if err != nil { impl.logger.Errorw("error in creating pipeline", "name", pipeline.Name, "err", err) From 27b1743cffa4334480c87802f428a07e1562e7be Mon Sep 17 00:00:00 2001 From: ayushmaheshwari Date: Fri, 5 Jul 2024 08:07:03 +0530 Subject: [PATCH 18/45] dev testing fixes --- ...InstalledAppDeploymentTypeChangeService.go | 24 +++++---------- .../AppDeploymentTypeChangeManager.go | 29 ++++++++++--------- 2 files changed, 23 insertions(+), 30 deletions(-) diff --git a/pkg/appStore/installedApp/service/FullMode/deploymentTypeChange/InstalledAppDeploymentTypeChangeService.go b/pkg/appStore/installedApp/service/FullMode/deploymentTypeChange/InstalledAppDeploymentTypeChangeService.go index ccac059316..d786978b26 100644 --- a/pkg/appStore/installedApp/service/FullMode/deploymentTypeChange/InstalledAppDeploymentTypeChangeService.go +++ b/pkg/appStore/installedApp/service/FullMode/deploymentTypeChange/InstalledAppDeploymentTypeChangeService.go @@ -227,23 +227,15 @@ func (impl *InstalledAppDeploymentTypeChangeServiceImpl) performDbOperationsAfte return err } - deploymentConfigSelector := make([]*bean3.DeploymentConfigSelector, len(successInstalledAppIds)) - for _, ia := range installedApps { - deploymentConfigSelector = append(deploymentConfigSelector, &bean3.DeploymentConfigSelector{ - AppId: ia.AppId, - EnvironmentId: ia.EnvironmentId, - }) - } - - deploymentConfigs, err := impl.deploymentConfigService.GetDeploymentConfigInBulk(deploymentConfigSelector) - if err != nil { - impl.logger.Errorw("error in getting deployment config", "deploymentConfigSelector", deploymentConfigSelector, "err", err) - return err - } updatedDeploymentConfigs := make([]*bean3.DeploymentConfig, 0) - for _, c := range deploymentConfigs { - c.DeploymentAppType = desiredDeploymentType - updatedDeploymentConfigs = append(updatedDeploymentConfigs, c) + for _, ia := range installedApps { + deploymentConfig, err := impl.deploymentConfigService.GetAndMigrateConfigIfAbsentForDevtronApps(ia.AppId, ia.EnvironmentId) + if err != nil { + impl.logger.Errorw("error in getting deployment config by appId and envId", "appId", ia.AppId, "envId", ia.EnvironmentId, "err", err) + return err + } + deploymentConfig.DeploymentAppType = desiredDeploymentType + updatedDeploymentConfigs = append(updatedDeploymentConfigs, deploymentConfig) } updatedDeploymentConfigs, err = impl.deploymentConfigService.UpdateConfigs(nil, updatedDeploymentConfigs, userId) diff --git a/pkg/pipeline/AppDeploymentTypeChangeManager.go b/pkg/pipeline/AppDeploymentTypeChangeManager.go index 02bd5a69a0..ef2a83ad11 100644 --- a/pkg/pipeline/AppDeploymentTypeChangeManager.go +++ b/pkg/pipeline/AppDeploymentTypeChangeManager.go @@ -142,6 +142,11 @@ func (impl *AppDeploymentTypeChangeManagerImpl) ChangeDeploymentType(ctx context pipelineIds = append(pipelineIds, item.PipelineId) } + // If nothing to update in db + if len(pipelineIds) == 0 { + return response, nil + } + // Get all pipelines pipelines, err := impl.pipelineRepository.FindByIdsIn(pipelineIds) if err != nil { @@ -151,10 +156,6 @@ func (impl *AppDeploymentTypeChangeManagerImpl) ChangeDeploymentType(ctx context return response, nil } - // If nothing to update in db - if len(pipelineIds) == 0 { - return response, nil - } updatedDeploymentConfigs := make([]*bean4.DeploymentConfig, 0) for _, pipeline := range pipelines { @@ -425,18 +426,18 @@ func (impl *AppDeploymentTypeChangeManagerImpl) TriggerDeploymentAfterTypeChange }) } - deploymentConfigs, err := impl.deploymentConfigService.GetDeploymentConfigInBulk(deploymentConfigSelector) - if err != nil { - impl.logger.Errorw("error in getting deployment config", "deploymentConfigSelector", deploymentConfigSelector, "err", err) - return nil, err - } - updatedDeploymentConfigs := make([]*bean4.DeploymentConfig, 0) - for _, c := range deploymentConfigs { - c.DeploymentAppType = request.DesiredDeploymentType - updatedDeploymentConfigs = append(updatedDeploymentConfigs, c) + deploymentConfigs := make([]*bean4.DeploymentConfig, 0) + for _, p := range pipelines { + envDeploymentConfig, err := impl.deploymentConfigService.GetAndMigrateConfigIfAbsentForDevtronApps(p.AppId, p.EnvironmentId) + if err != nil { + impl.logger.Errorw("error in fetching environment deployment config by appId and envId", "appId", p.AppId, "envId", p.EnvironmentId, "err", err) + return response, err + } + envDeploymentConfig.DeploymentAppType = request.DesiredDeploymentType + deploymentConfigs = append(deploymentConfigs, envDeploymentConfig) } - updatedDeploymentConfigs, err = impl.deploymentConfigService.UpdateConfigs(nil, updatedDeploymentConfigs, request.UserId) + deploymentConfigs, err = impl.deploymentConfigService.UpdateConfigs(nil, deploymentConfigs, request.UserId) if err != nil { impl.logger.Errorw("error in updating configs", "err", err) return nil, err From b0d450539f2035c0d682ceae0624fce09fcf6fa6 Mon Sep 17 00:00:00 2001 From: ayushmaheshwari Date: Fri, 5 Jul 2024 13:17:59 +0530 Subject: [PATCH 19/45] updating app stages status check --- .../sql/repository/AppListingRepository.go | 24 ++++++++++--------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/internal/sql/repository/AppListingRepository.go b/internal/sql/repository/AppListingRepository.go index cf202f8589..776487b305 100644 --- a/internal/sql/repository/AppListingRepository.go +++ b/internal/sql/repository/AppListingRepository.go @@ -551,23 +551,25 @@ func (impl AppListingRepositoryImpl) FetchAppStageStatus(appId int, appType int) var appStageStatus []bean.AppStageStatus var stages struct { - AppId int `json:"app_id,omitempty"` - CiTemplateId int `json:"ci_template_id,omitempty"` - CiPipelineId int `json:"ci_pipeline_id,omitempty"` - ChartId int `json:"chart_id,omitempty"` - ChartGitRepoUrl string `json:"chart_git_repo_url,omitempty"` - PipelineId int `json:"pipeline_id,omitempty"` - YamlStatus int `json:"yaml_status,omitempty"` - YamlReviewed bool `json:"yaml_reviewed,omitempty"` + AppId int `json:"app_id,omitempty"` + CiTemplateId int `json:"ci_template_id,omitempty"` + CiPipelineId int `json:"ci_pipeline_id,omitempty"` + ChartId int `json:"chart_id,omitempty"` + ChartGitRepoUrl string `json:"chart_git_repo_url,omitempty"` + PipelineId int `json:"pipeline_id,omitempty"` + YamlStatus int `json:"yaml_status,omitempty"` + YamlReviewed bool `json:"yaml_reviewed,omitempty"` + DeploymentConfigRepoURL string `json:"deployment_config_repo_url"` } query := "SELECT " + - " app.id as app_id, ct.id as ci_template_id, cp.id as ci_pipeline_id, ch.id as chart_id, ch.git_repo_url as chart_git_repo_url," + + " app.id as app_id, ct.id as ci_template_id, cp.id as ci_pipeline_id, ch.id as chart_id, ch.git_repo_url as chart_git_repo_url, dc.repo_url as deployment_config_repo_url, " + " p.id as pipeline_id, ceco.status as yaml_status, ceco.reviewed as yaml_reviewed " + " FROM app app" + " LEFT JOIN ci_template ct on ct.app_id=app.id" + " LEFT JOIN ci_pipeline cp on cp.app_id=app.id" + " LEFT JOIN charts ch on ch.app_id=app.id" + + " LEFT JOIN deployment_config dc on dc.app_id=app.id" + " LEFT JOIN pipeline p on p.app_id=app.id" + " LEFT JOIN chart_env_config_override ceco on ceco.chart_id=ch.id" + " WHERE app.id=? and app.app_type = ? limit 1;" @@ -602,7 +604,7 @@ func (impl AppListingRepositoryImpl) FetchAppStageStatus(appId int, appType int) if model != nil && model.Id > 0 && model.AllowCustomRepository { isCustomGitopsRepoUrl = true } - if gitOps.IsGitOpsRepoNotConfigured(stages.ChartGitRepoUrl) && stages.CiPipelineId == 0 { + if (gitOps.IsGitOpsRepoNotConfigured(stages.ChartGitRepoUrl) && gitOps.IsGitOpsRepoNotConfigured(stages.DeploymentConfigRepoURL)) && stages.CiPipelineId == 0 { stages.ChartGitRepoUrl = "" } appStageStatus = append(appStageStatus, impl.makeAppStageStatus(0, "APP", stages.AppId, true), @@ -610,7 +612,7 @@ func (impl AppListingRepositoryImpl) FetchAppStageStatus(appId int, appType int) impl.makeAppStageStatus(2, "TEMPLATE", stages.CiTemplateId, true), impl.makeAppStageStatus(3, "CI_PIPELINE", stages.CiPipelineId, true), impl.makeAppStageStatus(4, "CHART", stages.ChartId, true), - impl.makeAppStageStatus(5, "GITOPS_CONFIG", len(stages.ChartGitRepoUrl), isCustomGitopsRepoUrl), + impl.makeAppStageStatus(5, "GITOPS_CONFIG", len(stages.ChartGitRepoUrl)+len(stages.DeploymentConfigRepoURL), isCustomGitopsRepoUrl), impl.makeAppStageStatus(6, "CD_PIPELINE", stages.PipelineId, true), impl.makeAppStageChartEnvConfigStatus(7, "CHART_ENV_CONFIG", stages.YamlStatus == 3 && stages.YamlReviewed), ) From a8d9228a122a61f1e4a1c68af02ecffdfcfe8cad Mon Sep 17 00:00:00 2001 From: ayushmaheshwari Date: Fri, 5 Jul 2024 19:35:43 +0530 Subject: [PATCH 20/45] query active check for deployment config --- .../chartConfig/PipelineOverrideRepository.go | 2 +- .../helper/AppListingRepositoryQueryBuilder.go | 2 +- .../pipelineConfig/CdWorfkflowRepository.go | 2 +- .../pipelineConfig/PipelineRepository.go | 10 +++++----- .../PipelineStatusSyncDetailRepository.go | 4 ++-- .../repository/InstalledAppRepository.go | 14 +++++++------- .../service/AppStoreDeploymentDBService.go | 18 +++++++----------- .../trigger/devtronApps/TriggerService.go | 1 - .../107_create_k8s_resource_history.down.sql | 15 ++++++++++++++- 9 files changed, 38 insertions(+), 30 deletions(-) diff --git a/internal/sql/repository/chartConfig/PipelineOverrideRepository.go b/internal/sql/repository/chartConfig/PipelineOverrideRepository.go index 56e1ce06c6..aba3d7554c 100644 --- a/internal/sql/repository/chartConfig/PipelineOverrideRepository.go +++ b/internal/sql/repository/chartConfig/PipelineOverrideRepository.go @@ -297,7 +297,7 @@ func (impl PipelineOverrideRepositoryImpl) FindLatestByAppIdAndEnvId(appId, envi err = impl.dbConnection.Model(&override). Column("pipeline_override.*", "Pipeline"). Join("inner join pipeline p on p.id = pipeline_override.pipeline_id"). - Join("LEFT JOIN deployment_config dc on dc.app_id = p.app_id and dc.environment_id=p.environment_id"). + Join("LEFT JOIN deployment_config dc on dc.app_id = p.app_id and dc.environment_id=p.environment_id and dc.active=true"). Where("pipeline.app_id =? ", appId). Where("pipeline.environment_id =?", environmentId). Where("(p.deployment_app_type=? or dc.deployment_app_type=?)", deploymentAppType, deploymentAppType). diff --git a/internal/sql/repository/helper/AppListingRepositoryQueryBuilder.go b/internal/sql/repository/helper/AppListingRepositoryQueryBuilder.go index 0d48c16f7c..a0e8f5e498 100644 --- a/internal/sql/repository/helper/AppListingRepositoryQueryBuilder.go +++ b/internal/sql/repository/helper/AppListingRepositoryQueryBuilder.go @@ -141,7 +141,7 @@ func (impl AppListingRepositoryQueryBuilder) CommonJoinSubQuery(appListingFilter whereCondition := impl.buildAppListingWhereCondition(appListingFilter) query := " LEFT JOIN pipeline p ON a.id=p.app_id and p.deleted=false " + - " LEFT JOIN deployment_config dc ON ( p.app_id=dc.app_id and p.environment_id=dc.environment_id )" + + " LEFT JOIN deployment_config dc ON ( p.app_id=dc.app_id and p.environment_id=dc.environment_id and dc.active=true )" + " LEFT JOIN app_status aps on aps.app_id = a.id and p.environment_id = aps.env_id " if appListingFilter.DeploymentGroupId != 0 { diff --git a/internal/sql/repository/pipelineConfig/CdWorfkflowRepository.go b/internal/sql/repository/pipelineConfig/CdWorfkflowRepository.go index 966444605f..cd3ec276ee 100644 --- a/internal/sql/repository/pipelineConfig/CdWorfkflowRepository.go +++ b/internal/sql/repository/pipelineConfig/CdWorfkflowRepository.go @@ -756,7 +756,7 @@ func (impl *CdWorkflowRepositoryImpl) GetLatestTriggersOfHelmPipelinesStuckInNon Join("INNER JOIN cd_workflow wf on wf.id = cd_workflow_runner.cd_workflow_id"). Join("INNER JOIN pipeline p on p.id = wf.pipeline_id"). Join("INNER JOIN environment e on e.id = p.environment_id"). - Join("LEFT JOIN deployment_config dc on dc.app_id = p.app_id and dc.environment_id=p.environment_id"). + Join("LEFT JOIN deployment_config dc on dc.active=true and dc.app_id = p.app_id and dc.environment_id=p.environment_id"). Where("cd_workflow_runner.workflow_type=?", apiBean.CD_WORKFLOW_TYPE_DEPLOY). Where("cd_workflow_runner.status not in (?)", pg.In(excludedStatusList)). Where("cd_workflow_runner.cd_workflow_id in"+ diff --git a/internal/sql/repository/pipelineConfig/PipelineRepository.go b/internal/sql/repository/pipelineConfig/PipelineRepository.go index b17cb07b37..f5d6134666 100644 --- a/internal/sql/repository/pipelineConfig/PipelineRepository.go +++ b/internal/sql/repository/pipelineConfig/PipelineRepository.go @@ -372,7 +372,7 @@ func (impl PipelineRepositoryImpl) FindActiveByEnvIdAndDeploymentType(environmen Model(&pipelines). Column("pipeline.*", "App", "Environment"). Join("inner join app a on pipeline.app_id = a.id"). - Join("LEFT JOIN deployment_config dc on dc.app_id = pipeline.app_id and dc.environment_id=pipeline.environment_id"). + Join("LEFT JOIN deployment_config dc on dc.active=true and dc.app_id = pipeline.app_id and dc.environment_id=pipeline.environment_id"). Where("pipeline.environment_id = ?", environmentId). Where("(pipeline.deployment_app_type=? or dc.deployment_app_type=?)", deploymentAppType, deploymentAppType). Where("pipeline.deleted = ?", false) @@ -601,7 +601,7 @@ func (impl PipelineRepositoryImpl) GetAppAndEnvDetailsForDeploymentAppTypePipeli Column("pipeline.id", "App.app_name", "Environment.cluster_id", "Environment.namespace", "Environment.environment_name"). Join("inner join app a on pipeline.app_id = a.id"). Join("inner join environment e on pipeline.environment_id = e.id"). - Join("LEFT JOIN deployment_config dc on dc.app_id = pipeline.app_id and dc.environment_id=pipeline.environment_id"). + Join("LEFT JOIN deployment_config dc on dc.active=true and dc.app_id = pipeline.app_id and dc.environment_id=pipeline.environment_id"). Where("e.cluster_id in (?)", pg.In(clusterIds)). Where("a.active = ?", true). Where("pipeline.deleted = ?", false). @@ -614,7 +614,7 @@ func (impl PipelineRepositoryImpl) GetArgoPipelinesHavingTriggersStuckInLastPoss var pipelines []*Pipeline queryString := `select p.* from pipeline p inner join cd_workflow cw on cw.pipeline_id = p.id inner join cd_workflow_runner cwr on cwr.cd_workflow_id=cw.id - left join deployment_config dc on dc.app_id = p.app_id and dc.environment_id=p.environment_id + left join deployment_config dc on dc.active=true and dc.app_id = p.app_id and dc.environment_id=p.environment_id where cwr.id in (select cd_workflow_runner_id from pipeline_status_timeline where id in (select DISTINCT ON (cd_workflow_runner_id) max(id) as id from pipeline_status_timeline @@ -636,7 +636,7 @@ func (impl PipelineRepositoryImpl) GetArgoPipelinesHavingLatestTriggerStuckInNon var pipelines []*Pipeline queryString := `select p.id from pipeline p inner join cd_workflow cw on cw.pipeline_id = p.id inner join cd_workflow_runner cwr on cwr.cd_workflow_id=cw.id - left join deployment_config dc on dc.app_id = p.app_id and dc.environment_id=p.environment_id + left join deployment_config dc on dc.active=true and dc.app_id = p.app_id and dc.environment_id=p.environment_id where cwr.id in (select id from cd_workflow_runner where started_on < NOW() - INTERVAL '? minutes' and started_on > NOW() - INTERVAL '? hours' and status not in (?) and workflow_type=? and cd_workflow_id in (select DISTINCT ON (pipeline_id) max(id) as id from cd_workflow @@ -677,7 +677,7 @@ func (impl PipelineRepositoryImpl) FindIdsByProjectIdsAndEnvironmentIds(projectI func (impl PipelineRepositoryImpl) GetArgoPipelineByArgoAppName(argoAppName string) (Pipeline, error) { var pipeline Pipeline err := impl.dbConnection.Model(&pipeline). - Join("LEFT JOIN deployment_config dc on dc.app_id = pipeline.app_id and dc.environment_id=pipeline.environment_id"). + Join("LEFT JOIN deployment_config dc on dc.active=true and dc.app_id = pipeline.app_id and dc.environment_id=pipeline.environment_id and dc.active=true"). Column("pipeline.*", "Environment"). Where("deployment_app_name = ?", argoAppName). Where("(pipeline.deployment_app_type=? or dc.deployment_app_type=?)", util.PIPELINE_DEPLOYMENT_TYPE_ACD, util.PIPELINE_DEPLOYMENT_TYPE_ACD). diff --git a/internal/sql/repository/pipelineConfig/PipelineStatusSyncDetailRepository.go b/internal/sql/repository/pipelineConfig/PipelineStatusSyncDetailRepository.go index b427c466b2..e342823ce7 100644 --- a/internal/sql/repository/pipelineConfig/PipelineStatusSyncDetailRepository.go +++ b/internal/sql/repository/pipelineConfig/PipelineStatusSyncDetailRepository.go @@ -99,7 +99,7 @@ func (impl *PipelineStatusSyncDetailRepositoryImpl) GetOfLatestCdWfrByCdPipeline query := `select * from pipeline_status_timeline_sync_detail where cd_workflow_runner_id = (select cwr.id from cd_workflow_runner cwr inner join cd_workflow cw on cw.id=cwr.cd_workflow_id inner join pipeline p on p.id=cw.pipeline_id - left join deployment_config dc on dc.app_id = p.app_id and dc.environment_id=p.environment_id + left join deployment_config dc on dc.active=true and dc.app_id = p.app_id and dc.environment_id=p.environment_id where p.id=? and p.deleted=? and (p.deployment_app_type=? or dc.deployment_app_type=?) order by cwr.id desc limit ?);` _, err := impl.dbConnection.Query(&model, query, pipelineId, false, util.PIPELINE_DEPLOYMENT_TYPE_ACD, util.PIPELINE_DEPLOYMENT_TYPE_ACD, 1) if err != nil { @@ -115,7 +115,7 @@ func (impl *PipelineStatusSyncDetailRepositoryImpl) GetOfLatestInstalledAppVersi where installed_app_version_history_id = (select iavh.id from installed_app_version_history iavh inner join installed_app_versions iav on iavh.installed_app_version_id=iav.id inner join installed_apps ia on iav.installed_app_id=ia.id - left join deployment_config dc on dc.app_id = ia.app_id and dc.environment_id=ia.environment_id + left join deployment_config dc on dc.active=true and dc.app_id = ia.app_id and dc.environment_id=ia.environment_id where iav.id=? and iav.active=? and (ia.deployment_app_type=? or dc.deployment_app_type=?) order by iavh.id desc limit ?);` _, err := impl.dbConnection.Query(&model, query, installedAppVersionId, true, util.PIPELINE_DEPLOYMENT_TYPE_ACD, util.PIPELINE_DEPLOYMENT_TYPE_ACD, 1) diff --git a/pkg/appStore/installedApp/repository/InstalledAppRepository.go b/pkg/appStore/installedApp/repository/InstalledAppRepository.go index 080e5ac57b..a1a64f3200 100644 --- a/pkg/appStore/installedApp/repository/InstalledAppRepository.go +++ b/pkg/appStore/installedApp/repository/InstalledAppRepository.go @@ -691,7 +691,7 @@ func (impl InstalledAppRepositoryImpl) GetAppAndEnvDetailsForDeploymentAppTypeIn err := impl.dbConnection. Model(&installedApps). Column("installed_apps.id", "App.app_name", "App.display_name", "Environment.cluster_id", "Environment.namespace"). - Join("LEFT JOIN deployment_config dc on dc.app_id = installed_apps.app_id and dc.environment_id=installed_apps.environment_id"). + Join("LEFT JOIN deployment_config dc on dc.active=true and dc.app_id = installed_apps.app_id and dc.environment_id=installed_apps.environment_id"). Where("environment.cluster_id in (?)", pg.In(clusterIds)). Where("(installed_apps.deployment_app_type = ? or dc.deployment_app_type = ?)", deploymentAppType, deploymentAppType). Where("app.active = ?", true). @@ -744,7 +744,7 @@ func (impl InstalledAppRepositoryImpl) GetInstalledAppByGitHash(gitHash string) func (impl InstalledAppRepositoryImpl) GetInstalledAppByAppIdAndDeploymentType(appId int, deploymentAppType string) (InstalledApps, error) { var installedApps InstalledApps queryString := `select * from installed_apps - left join deployment_config dc on dc.app_id = installed_apps.app_id and dc.environment_id=installed_apps.environment_id + left join deployment_config dc on dc.active=true and dc.app_id = installed_apps.app_id and dc.environment_id=installed_apps.environment_id where active=? and app_id=? and (installed_apps.deployment_app_type=? or dc.deployment_app_type=?);` _, err := impl.dbConnection.Query(&installedApps, queryString, true, appId, deploymentAppType, deploymentAppType) if err != nil { @@ -769,8 +769,8 @@ func (impl InstalledAppRepositoryImpl) GetInstalledAppByAppName(appName string) func (impl InstalledAppRepositoryImpl) GetInstalledAppByInstalledAppVersionId(installedAppVersionId int) (InstalledApps, error) { var installedApps InstalledApps queryString := `select ia.* from installed_apps ia inner join installed_app_versions iav on ia.id=iav.installed_app_id - left join deployment_config dc on dc.app_id = ia.app_id and dc.environment_id=ia.environment_id - where iav.active=? and iav.id=? and (ia.deployment_app_type=? or dc.deployment_app_type=?);` + left join deployment_config dc on dc.active=true and dc.app_id = ia.app_id and dc.environment_id=ia.environment_id + where iav.active=? and iav.id=? and (ia.deployment_app_type=? or dc.deployment_app_type=? );` _, err := impl.dbConnection.Query(&installedApps, queryString, true, installedAppVersionId, util2.PIPELINE_DEPLOYMENT_TYPE_ACD, util2.PIPELINE_DEPLOYMENT_TYPE_ACD) if err != nil { impl.Logger.Errorw("error in fetching InstalledApp", "err", err) @@ -811,7 +811,7 @@ func (impl InstalledAppRepositoryImpl) GetArgoPipelinesHavingLatestTriggerStuckI queryString := `select iav.* from installed_app_versions iav inner join installed_apps ia on iav.installed_app_id=ia.id inner join installed_app_version_history iavh on iavh.installed_app_version_id=iav.id - left join deployment_config dc on dc.app_id = ia.app_id and dc.environment_id=ia.environment_id + left join deployment_config dc on dc.active=true and dc.app_id = ia.app_id and dc.environment_id=ia.environment_id where iavh.id in (select DISTINCT ON (installed_app_version_id) max(id) as id from installed_app_version_history where updated_on < NOW() - INTERVAL '? minutes' and updated_on > NOW() - INTERVAL '? hours' and status not in (?) group by installed_app_version_id, id order by installed_app_version_id, id desc ) and (ia.deployment_app_type=? or dc.deployment_app_type=?) and iav.active=?;` @@ -830,7 +830,7 @@ func (impl InstalledAppRepositoryImpl) GetArgoPipelinesHavingTriggersStuckInLast var installedAppVersions []*InstalledAppVersions queryString := `select iav.* from installed_app_versions iav inner join installed_apps ia on iav.installed_app_id=ia.id inner join installed_app_version_history iavh on iavh.installed_app_version_id=iav.id - left join deployment_config dc on dc.app_id = ia.app_id and dc.environment_id=ia.environment_id + left join deployment_config dc on dc.active=true and dc.app_id = ia.app_id and dc.environment_id=ia.environment_id where iavh.id in (select DISTINCT ON (installed_app_version_history_id) max(id) as id from pipeline_status_timeline where status in (?) and status_time < NOW() - INTERVAL '? seconds' group by installed_app_version_history_id, id order by installed_app_version_history_id, id desc) @@ -869,7 +869,7 @@ func (impl InstalledAppRepositoryImpl) GetActiveInstalledAppByEnvIdAndDeployment Model(&installedApps). Column("installed_apps.*", "App", "Environment"). Join("inner join app a on installed_apps.app_id = a.id"). - Join("LEFT JOIN deployment_config dc on dc.app_id = installed_apps.app_id and dc.environment_id=installed_apps.environment_id"). + Join("LEFT JOIN deployment_config dc on dc.active=true and dc.app_id = installed_apps.app_id and dc.environment_id=installed_apps.environment_id"). Where("installed_apps.environment_id = ?", envId). Where("installed_apps.deployment_app_type = ? or dc.deployment_app_type = ?", deploymentType, deploymentType). Where("installed_apps.active = ?", true) diff --git a/pkg/appStore/installedApp/service/AppStoreDeploymentDBService.go b/pkg/appStore/installedApp/service/AppStoreDeploymentDBService.go index 9db52d0d06..d21c42c1ca 100644 --- a/pkg/appStore/installedApp/service/AppStoreDeploymentDBService.go +++ b/pkg/appStore/installedApp/service/AppStoreDeploymentDBService.go @@ -320,18 +320,14 @@ func (impl *AppStoreDeploymentDBServiceImpl) GetAllInstalledAppsByAppStoreId(app return nil, err } - deploymentSelectors := make([]*bean2.DeploymentConfigSelector, 0) + deploymentConfigMap := make(map[bean2.UniqueDeploymentConfigIdentifier]*bean2.DeploymentConfig) for _, ia := range installedApps { - deploymentSelectors = append(deploymentSelectors, &bean2.DeploymentConfigSelector{ - AppId: ia.AppId, - EnvironmentId: ia.EnvironmentId, - }) - } - - deploymentConfigMap, err := impl.deploymentConfigService.GetDeploymentConfigInBulk(deploymentSelectors) - if err != nil { - impl.logger.Errorw("error in getting deployment config by deployment selectors", "deploymentSelectors", deploymentSelectors, "err", err) - return nil, err + envDeploymentConfig, err := impl.deploymentConfigService.GetConfigForHelmApps(ia.AppId, ia.EnvironmentId) + if err != nil { + impl.logger.Errorw("error in fetching deployment config by appId and envId", "appId", ia.AppId, "envId", ia.EnvironmentId, "err", err) + return nil, err + } + deploymentConfigMap[bean2.GetConfigUniqueIdentifier(envDeploymentConfig.AppId, envDeploymentConfig.EnvironmentId)] = envDeploymentConfig } var installedAppsEnvResponse []appStoreBean.InstalledAppsResponse diff --git a/pkg/deployment/trigger/devtronApps/TriggerService.go b/pkg/deployment/trigger/devtronApps/TriggerService.go index dce7e3e885..da1bb28ab9 100644 --- a/pkg/deployment/trigger/devtronApps/TriggerService.go +++ b/pkg/deployment/trigger/devtronApps/TriggerService.go @@ -1152,7 +1152,6 @@ func (impl *TriggerServiceImpl) deployArgoCdApp(ctx context.Context, overrideReq return err } impl.logger.Debugw("ArgoCd application created", "name", name) - updateAppInArgoCd, err := impl.updateArgoPipeline(newCtx, valuesOverrideResponse.Pipeline, valuesOverrideResponse.EnvOverride, valuesOverrideResponse.DeploymentConfig) if err != nil { impl.logger.Errorw("error in updating argocd app ", "err", err) diff --git a/scripts/sql/107_create_k8s_resource_history.down.sql b/scripts/sql/107_create_k8s_resource_history.down.sql index ab0e40549d..31e54ccab0 100644 --- a/scripts/sql/107_create_k8s_resource_history.down.sql +++ b/scripts/sql/107_create_k8s_resource_history.down.sql @@ -1,2 +1,15 @@ DROP SEQUENCE IF EXISTS "id_seq_k8s_resource_history_sequence"; -DROP TABLE IF EXISTS "kubernetes_resource_history"; \ No newline at end of file +DROP TABLE IF EXISTS "kubernetes_resource_history"; + + +SELECT "cd_workflow_runner".*, + "cd_workflow"."id" AS "cd_workflow__id", + "cd_workflow"."pipeline_id" AS "cd_workflow__pipeline_id", + "cd_workflow__pipeline"."id" AS "cd_workflow__pipeline__id", + "cd_workflow__pipeline"."deployment_app_name" AS "cd_workflow__pipeline__deployment_app_name", + "cd_workflow__pipeline"."deleted" AS "cd_workflow__pipeline__deleted", + "cd_workflow__pipeline__environment"."id" AS "cd_workflow__pipeline__environment__id", + "cd_workflow__pipeline__environment"."environment_name" AS "cd_workflow__pipeline__environment__environment_name", + "cd_workflow__pipeline__environment"."cluster_id" AS "cd_workflow__pipeline__environment__cluster_id", + "cd_workflow__pipeline__environment"."active" AS "cd_workflow__pipeline__environment__active", + "cd_workflow__pipeline__environment"."default" AS "cd_workflow__pipeline__environment__default", "cd_workflow__pipeline__environment"."grafana_datasource_id" AS "cd_workflow__pipeline__environment__grafana_datasource_id", "cd_workflow__pipeline__environment"."namespace" AS "cd_workflow__pipeline__environment__namespace", "cd_workflow__pipeline__environment"."environment_identifier" AS "cd_workflow__pipeline__environment__environment_identifier", "cd_workflow__pipeline__environment"."description" AS "cd_workflow__pipeline__environment__description", "cd_workflow__pipeline__environment"."is_virtual_environment" AS "cd_workflow__pipeline__environment__is_virtual_environment", "cd_workflow__pipeline__environment"."created_on" AS "cd_workflow__pipeline__environment__created_on", "cd_workflow__pipeline__environment"."created_by" AS "cd_workflow__pipeline__environment__created_by", "cd_workflow__pipeline__environment"."updated_on" AS "cd_workflow__pipeline__environment__updated_on", "cd_workflow__pipeline__environment"."updated_by" AS "cd_workflow__pipeline__environment__updated_by" FROM cd_workflow_runner AS "cd_workflow_runner" LEFT JOIN cd_workflow AS "cd_workflow" ON "cd_workflow"."id" = "cd_workflow_runner"."cd_workflow_id" LEFT JOIN pipeline AS "cd_workflow__pipeline" ON "cd_workflow__pipeline"."id" = "cd_workflow"."pipeline_id" LEFT JOIN environment AS "cd_workflow__pipeline__environment" ON "cd_workflow__pipeline__environment"."id" = "cd_workflow__pipeline"."environment_id" INNER JOIN cd_workflow wf on wf.id = cd_workflow_runner.cd_workflow_id INNER JOIN pipeline p on p.id = wf.pipeline_id INNER JOIN environment e on e.id = p.environment_id LEFT JOIN deployment_config dc on dc.app_id = p.app_id and dc.environment_id=p.environment_id WHERE (cd_workflow_runner.workflow_type='DEPLOY') AND (cd_workflow_runner.status not in ('Aborted','Failed','Succeeded','HIBERNATING','Healthy','Degraded','Initiating','Queued','Starting')) AND (cd_workflow_runner.cd_workflow_id in (SELECT max(cd_workflow.id) as id from cd_workflow INNER JOIN cd_workflow_runner on cd_workflow.id = cd_workflow_runner.cd_workflow_id WHERE cd_workflow_runner.status != 'Queued' GROUP BY cd_workflow.pipeline_id ORDER BY cd_workflow.pipeline_id desc)) AND ((p.deployment_app_type='helm' or dc.deployment_app_type='helm')) AND (cd_workflow_runner.started_on > NOW() - INTERVAL '12 hours') AND (p.deleted=FALSE) ORDER BY "cd_workflow_runner"."id" DESC \ No newline at end of file From f7463ae25f5718cd089f54958ca1139c082659c5 Mon Sep 17 00:00:00 2001 From: ayushmaheshwari Date: Fri, 5 Jul 2024 19:46:13 +0530 Subject: [PATCH 21/45] wip --- .../repository/pipelineConfig/PipelineRepository.go | 2 +- .../sql/107_create_k8s_resource_history.down.sql | 13 ------------- 2 files changed, 1 insertion(+), 14 deletions(-) diff --git a/internal/sql/repository/pipelineConfig/PipelineRepository.go b/internal/sql/repository/pipelineConfig/PipelineRepository.go index f5d6134666..ae33d765fd 100644 --- a/internal/sql/repository/pipelineConfig/PipelineRepository.go +++ b/internal/sql/repository/pipelineConfig/PipelineRepository.go @@ -677,7 +677,7 @@ func (impl PipelineRepositoryImpl) FindIdsByProjectIdsAndEnvironmentIds(projectI func (impl PipelineRepositoryImpl) GetArgoPipelineByArgoAppName(argoAppName string) (Pipeline, error) { var pipeline Pipeline err := impl.dbConnection.Model(&pipeline). - Join("LEFT JOIN deployment_config dc on dc.active=true and dc.app_id = pipeline.app_id and dc.environment_id=pipeline.environment_id and dc.active=true"). + Join("LEFT JOIN deployment_config dc on dc.app_id = pipeline.app_id and dc.environment_id=pipeline.environment_id and dc.active=true"). Column("pipeline.*", "Environment"). Where("deployment_app_name = ?", argoAppName). Where("(pipeline.deployment_app_type=? or dc.deployment_app_type=?)", util.PIPELINE_DEPLOYMENT_TYPE_ACD, util.PIPELINE_DEPLOYMENT_TYPE_ACD). diff --git a/scripts/sql/107_create_k8s_resource_history.down.sql b/scripts/sql/107_create_k8s_resource_history.down.sql index 31e54ccab0..baf9dd27e7 100644 --- a/scripts/sql/107_create_k8s_resource_history.down.sql +++ b/scripts/sql/107_create_k8s_resource_history.down.sql @@ -1,15 +1,2 @@ DROP SEQUENCE IF EXISTS "id_seq_k8s_resource_history_sequence"; DROP TABLE IF EXISTS "kubernetes_resource_history"; - - -SELECT "cd_workflow_runner".*, - "cd_workflow"."id" AS "cd_workflow__id", - "cd_workflow"."pipeline_id" AS "cd_workflow__pipeline_id", - "cd_workflow__pipeline"."id" AS "cd_workflow__pipeline__id", - "cd_workflow__pipeline"."deployment_app_name" AS "cd_workflow__pipeline__deployment_app_name", - "cd_workflow__pipeline"."deleted" AS "cd_workflow__pipeline__deleted", - "cd_workflow__pipeline__environment"."id" AS "cd_workflow__pipeline__environment__id", - "cd_workflow__pipeline__environment"."environment_name" AS "cd_workflow__pipeline__environment__environment_name", - "cd_workflow__pipeline__environment"."cluster_id" AS "cd_workflow__pipeline__environment__cluster_id", - "cd_workflow__pipeline__environment"."active" AS "cd_workflow__pipeline__environment__active", - "cd_workflow__pipeline__environment"."default" AS "cd_workflow__pipeline__environment__default", "cd_workflow__pipeline__environment"."grafana_datasource_id" AS "cd_workflow__pipeline__environment__grafana_datasource_id", "cd_workflow__pipeline__environment"."namespace" AS "cd_workflow__pipeline__environment__namespace", "cd_workflow__pipeline__environment"."environment_identifier" AS "cd_workflow__pipeline__environment__environment_identifier", "cd_workflow__pipeline__environment"."description" AS "cd_workflow__pipeline__environment__description", "cd_workflow__pipeline__environment"."is_virtual_environment" AS "cd_workflow__pipeline__environment__is_virtual_environment", "cd_workflow__pipeline__environment"."created_on" AS "cd_workflow__pipeline__environment__created_on", "cd_workflow__pipeline__environment"."created_by" AS "cd_workflow__pipeline__environment__created_by", "cd_workflow__pipeline__environment"."updated_on" AS "cd_workflow__pipeline__environment__updated_on", "cd_workflow__pipeline__environment"."updated_by" AS "cd_workflow__pipeline__environment__updated_by" FROM cd_workflow_runner AS "cd_workflow_runner" LEFT JOIN cd_workflow AS "cd_workflow" ON "cd_workflow"."id" = "cd_workflow_runner"."cd_workflow_id" LEFT JOIN pipeline AS "cd_workflow__pipeline" ON "cd_workflow__pipeline"."id" = "cd_workflow"."pipeline_id" LEFT JOIN environment AS "cd_workflow__pipeline__environment" ON "cd_workflow__pipeline__environment"."id" = "cd_workflow__pipeline"."environment_id" INNER JOIN cd_workflow wf on wf.id = cd_workflow_runner.cd_workflow_id INNER JOIN pipeline p on p.id = wf.pipeline_id INNER JOIN environment e on e.id = p.environment_id LEFT JOIN deployment_config dc on dc.app_id = p.app_id and dc.environment_id=p.environment_id WHERE (cd_workflow_runner.workflow_type='DEPLOY') AND (cd_workflow_runner.status not in ('Aborted','Failed','Succeeded','HIBERNATING','Healthy','Degraded','Initiating','Queued','Starting')) AND (cd_workflow_runner.cd_workflow_id in (SELECT max(cd_workflow.id) as id from cd_workflow INNER JOIN cd_workflow_runner on cd_workflow.id = cd_workflow_runner.cd_workflow_id WHERE cd_workflow_runner.status != 'Queued' GROUP BY cd_workflow.pipeline_id ORDER BY cd_workflow.pipeline_id desc)) AND ((p.deployment_app_type='helm' or dc.deployment_app_type='helm')) AND (cd_workflow_runner.started_on > NOW() - INTERVAL '12 hours') AND (p.deleted=FALSE) ORDER BY "cd_workflow_runner"."id" DESC \ No newline at end of file From f9171468d85f017913e65c5f23ce1489cf1ff350 Mon Sep 17 00:00:00 2001 From: ayushmaheshwari Date: Tue, 9 Jul 2024 01:57:28 +0530 Subject: [PATCH 22/45] adding flag for migration and removing bulk code --- .../service/AppStoreDeploymentService.go | 2 +- ...InstalledAppDeploymentTypeChangeService.go | 13 +- pkg/chart/ChartService.go | 92 ++-- .../DevtronAppGitOpsConfigService.go | 2 +- .../common/deploymentConfigService.go | 490 ++---------------- .../AppDeploymentTypeChangeManager.go | 81 ++- .../DeploymentPipelineConfigService.go | 6 +- util/GlobalConfig.go | 1 + 8 files changed, 167 insertions(+), 520 deletions(-) diff --git a/pkg/appStore/installedApp/service/AppStoreDeploymentService.go b/pkg/appStore/installedApp/service/AppStoreDeploymentService.go index eb3cf58e02..231522f16c 100644 --- a/pkg/appStore/installedApp/service/AppStoreDeploymentService.go +++ b/pkg/appStore/installedApp/service/AppStoreDeploymentService.go @@ -628,7 +628,7 @@ func (impl *AppStoreDeploymentServiceImpl) UpdateInstalledApp(ctx context.Contex installedAppDeploymentAction := adapter.NewInstalledAppDeploymentAction(deploymentConfig.DeploymentAppType) // migrate installedApp.GitOpsRepoName to installedApp.GitOpsRepoUrl if util.IsAcdApp(deploymentConfig.DeploymentAppType) && - len(deploymentConfig.GitOpsRepoUrl) == 0 { + len(deploymentConfig.RepoURL) == 0 { gitRepoUrl, err := impl.fullModeDeploymentService.GetAcdAppGitOpsRepoURL(installedApp.App.AppName, installedApp.Environment.Name) if err != nil { impl.logger.Errorw("error in GitOps repository url migration", "err", err) diff --git a/pkg/appStore/installedApp/service/FullMode/deploymentTypeChange/InstalledAppDeploymentTypeChangeService.go b/pkg/appStore/installedApp/service/FullMode/deploymentTypeChange/InstalledAppDeploymentTypeChangeService.go index d786978b26..b038d2f36c 100644 --- a/pkg/appStore/installedApp/service/FullMode/deploymentTypeChange/InstalledAppDeploymentTypeChangeService.go +++ b/pkg/appStore/installedApp/service/FullMode/deploymentTypeChange/InstalledAppDeploymentTypeChangeService.go @@ -227,7 +227,6 @@ func (impl *InstalledAppDeploymentTypeChangeServiceImpl) performDbOperationsAfte return err } - updatedDeploymentConfigs := make([]*bean3.DeploymentConfig, 0) for _, ia := range installedApps { deploymentConfig, err := impl.deploymentConfigService.GetAndMigrateConfigIfAbsentForDevtronApps(ia.AppId, ia.EnvironmentId) if err != nil { @@ -235,13 +234,11 @@ func (impl *InstalledAppDeploymentTypeChangeServiceImpl) performDbOperationsAfte return err } deploymentConfig.DeploymentAppType = desiredDeploymentType - updatedDeploymentConfigs = append(updatedDeploymentConfigs, deploymentConfig) - } - - updatedDeploymentConfigs, err = impl.deploymentConfigService.UpdateConfigs(nil, updatedDeploymentConfigs, userId) - if err != nil { - impl.logger.Errorw("error in updating configs", "err", err) - return err + deploymentConfig, err = impl.deploymentConfigService.CreateOrUpdateConfig(nil, deploymentConfig, userId) + if err != nil { + impl.logger.Errorw("error in updating deployment config", "appId", ia.AppId, "envId", ia.EnvironmentId, "err", err) + return err + } } err = impl.installedAppRepository.UpdateDeploymentAppTypeInInstalledApp(desiredDeploymentType, successInstalledAppIds, userId, deployStatus) diff --git a/pkg/chart/ChartService.go b/pkg/chart/ChartService.go index 325ee07b23..003f06f9b7 100644 --- a/pkg/chart/ChartService.go +++ b/pkg/chart/ChartService.go @@ -65,7 +65,7 @@ type ChartService interface { ChartRefAutocompleteForAppOrEnv(appId int, envId int) (*chartRefBean.ChartRefAutocompleteResponse, error) - ConfigureGitOpsRepoUrlForApp(appId int, repoUrl, chartLocation string, isCustomRepo bool, userId int32) error + ConfigureGitOpsRepoUrlForApp(appId int, repoUrl, chartLocation string, isCustomRepo bool, userId int32) (*bean2.DeploymentConfig, error) IsGitOpsRepoConfiguredForDevtronApps(appId int) (bool, error) IsGitOpsRepoAlreadyRegistered(gitOpsRepoUrl string) (bool, error) @@ -211,10 +211,9 @@ func (impl *ChartServiceImpl) Create(templateRequest TemplateRequest, ctx contex if err != nil { return nil, err } - - //if currentLatestChart.GitRepoUrl != "" { - // gitRepoUrl = currentLatestChart.GitRepoUrl - //} + if currentLatestChart.GitRepoUrl != "" { + gitRepoUrl = currentLatestChart.GitRepoUrl + } } // ENDS @@ -263,15 +262,15 @@ func (impl *ChartServiceImpl) Create(templateRequest TemplateRequest, ctx contex ChartVersion: chartMeta.Version, Status: models.CHARTSTATUS_NEW, Active: true, - //ChartLocation: chartLocation, - //GitRepoUrl: gitRepoUrl, - ReferenceTemplate: templateName, - ChartRefId: templateRequest.ChartRefId, - Latest: true, - Previous: false, - IsBasicViewLocked: templateRequest.IsBasicViewLocked, - CurrentViewEditor: templateRequest.CurrentViewEditor, - AuditLog: sql.AuditLog{CreatedBy: templateRequest.UserId, CreatedOn: time.Now(), UpdatedOn: time.Now(), UpdatedBy: templateRequest.UserId}, + ChartLocation: chartLocation, + GitRepoUrl: gitRepoUrl, + ReferenceTemplate: templateName, + ChartRefId: templateRequest.ChartRefId, + Latest: true, + Previous: false, + IsBasicViewLocked: templateRequest.IsBasicViewLocked, + CurrentViewEditor: templateRequest.CurrentViewEditor, + AuditLog: sql.AuditLog{CreatedBy: templateRequest.UserId, CreatedOn: time.Now(), UpdatedOn: time.Now(), UpdatedBy: templateRequest.UserId}, } err = impl.chartRepository.Save(chart) @@ -366,15 +365,14 @@ func (impl *ChartServiceImpl) CreateChartFromEnvOverride(templateRequest Templat return nil, err } - currentDeploymentConfig, err := impl.deploymentConfigService.GetAndMigrateConfigIfAbsentForDevtronApps(templateRequest.AppId, 0) - if err != nil && err != pg.ErrNoRows { + currentLatestChart, err := impl.chartRepository.FindLatestChartForAppByAppId(templateRequest.AppId) + if err != nil && pg.ErrNoRows != err { return nil, err } - chartLocation := filepath.Join(templateName, version) gitRepoUrl := apiGitOpsBean.GIT_REPO_NOT_CONFIGURED - if currentDeploymentConfig != nil && currentDeploymentConfig.Id > 0 && len(currentDeploymentConfig.RepoURL) > 0 { - gitRepoUrl = currentDeploymentConfig.RepoURL + if currentLatestChart.Id > 0 && currentLatestChart.GitRepoUrl != "" { + gitRepoUrl = currentLatestChart.GitRepoUrl } deploymentConfig := &bean2.DeploymentConfig{ @@ -423,15 +421,15 @@ func (impl *ChartServiceImpl) CreateChartFromEnvOverride(templateRequest Templat ChartVersion: chartMeta.Version, Status: models.CHARTSTATUS_NEW, Active: true, - //ChartLocation: chartLocation, - //GitRepoUrl: gitRepoUrl, - ReferenceTemplate: templateName, - ChartRefId: templateRequest.ChartRefId, - Latest: false, - Previous: false, - IsBasicViewLocked: templateRequest.IsBasicViewLocked, - CurrentViewEditor: templateRequest.CurrentViewEditor, - AuditLog: sql.AuditLog{CreatedBy: templateRequest.UserId, CreatedOn: time.Now(), UpdatedOn: time.Now(), UpdatedBy: templateRequest.UserId}, + ChartLocation: chartLocation, + GitRepoUrl: gitRepoUrl, + ReferenceTemplate: templateName, + ChartRefId: templateRequest.ChartRefId, + Latest: false, + Previous: false, + IsBasicViewLocked: templateRequest.IsBasicViewLocked, + CurrentViewEditor: templateRequest.CurrentViewEditor, + AuditLog: sql.AuditLog{CreatedBy: templateRequest.UserId, CreatedOn: time.Now(), UpdatedOn: time.Now(), UpdatedBy: templateRequest.UserId}, } err = impl.chartRepository.Save(chart) @@ -942,7 +940,37 @@ func (impl *ChartServiceImpl) CheckIfChartRefUserUploadedByAppId(id int) (bool, return chartData.UserUploaded, err } -func (impl *ChartServiceImpl) ConfigureGitOpsRepoUrlForApp(appId int, repoUrl, chartLocation string, isCustomRepo bool, userId int32) error { +func (impl *ChartServiceImpl) ConfigureGitOpsRepoUrlForApp(appId int, repoUrl, chartLocation string, isCustomRepo bool, userId int32) (*bean2.DeploymentConfig, error) { + + //update in both charts and deployment config + + charts, err := impl.chartRepository.FindActiveChartsByAppId(appId) + if err != nil { + return nil, err + } + tx, err := impl.chartRepository.StartTx() + if err != nil { + impl.logger.Errorw("error in starting transaction to update charts", "error", err) + return nil, err + } + defer impl.chartRepository.RollbackTx(tx) + var updatedCharts []*chartRepoRepository.Chart + for _, ch := range charts { + if !ch.IsCustomGitRepository { + ch.GitRepoUrl = repoUrl + ch.UpdateAuditLog(userId) + updatedCharts = append(updatedCharts, ch) + } + } + err = impl.chartRepository.UpdateAllInTx(tx, updatedCharts) + if err != nil { + return nil, err + } + err = impl.chartRepository.CommitTx(tx) + if err != nil { + impl.logger.Errorw("error in committing transaction to update charts", "error", err) + return nil, err + } deploymentConfig := &bean2.DeploymentConfig{ AppId: appId, @@ -951,12 +979,12 @@ func (impl *ChartServiceImpl) ConfigureGitOpsRepoUrlForApp(appId int, repoUrl, c ChartLocation: chartLocation, Active: true, } - deploymentConfig, err := impl.deploymentConfigService.CreateOrUpdateConfig(nil, deploymentConfig, userId) + deploymentConfig, err = impl.deploymentConfigService.CreateOrUpdateConfig(nil, deploymentConfig, userId) if err != nil { impl.logger.Errorw("error in saving deployment config for app", "appId", appId, "err", err) - return err + return nil, err } - return nil + return deploymentConfig, nil } //func (impl *ChartServiceImpl) OverrideGitOpsRepoUrl(appId int, repoUrl string, userId int32) error { diff --git a/pkg/chart/gitOpsConfig/DevtronAppGitOpsConfigService.go b/pkg/chart/gitOpsConfig/DevtronAppGitOpsConfigService.go index bbd28bca63..d85e99bf69 100644 --- a/pkg/chart/gitOpsConfig/DevtronAppGitOpsConfigService.go +++ b/pkg/chart/gitOpsConfig/DevtronAppGitOpsConfigService.go @@ -142,7 +142,7 @@ func (impl *DevtronAppGitOpConfigServiceImpl) SaveAppLevelGitOpsConfiguration(ap return err } isCustomGitOpsRepo := gitOpsConfigurationStatus.AllowCustomRepository && appGitOpsRequest.GitOpsRepoURL != apiGitOpsBean.GIT_REPO_DEFAULT - err = impl.chartService.ConfigureGitOpsRepoUrlForApp(appGitOpsRequest.AppId, chartGitAttr.RepoUrl, chartGitAttr.ChartLocation, isCustomGitOpsRepo, appGitOpsRequest.UserId) + _, err = impl.chartService.ConfigureGitOpsRepoUrlForApp(appGitOpsRequest.AppId, chartGitAttr.RepoUrl, chartGitAttr.ChartLocation, isCustomGitOpsRepo, appGitOpsRequest.UserId) if err != nil { impl.logger.Errorw("error in updating git repo url in charts", "err", err) return err diff --git a/pkg/deployment/common/deploymentConfigService.go b/pkg/deployment/common/deploymentConfigService.go index 6cc4fb5bdb..7776476482 100644 --- a/pkg/deployment/common/deploymentConfigService.go +++ b/pkg/deployment/common/deploymentConfigService.go @@ -3,35 +3,33 @@ package common import ( appRepository "github.com/devtron-labs/devtron/internal/sql/repository/app" "github.com/devtron-labs/devtron/internal/sql/repository/deploymentConfig" - "github.com/devtron-labs/devtron/internal/sql/repository/helper" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" "github.com/devtron-labs/devtron/pkg/appStore/installedApp/repository" bean3 "github.com/devtron-labs/devtron/pkg/auth/user/bean" chartRepoRepository "github.com/devtron-labs/devtron/pkg/chartRepo/repository" "github.com/devtron-labs/devtron/pkg/deployment/common/bean" + "github.com/devtron-labs/devtron/util" "github.com/go-pg/pg" "go.uber.org/zap" ) type DeploymentConfigService interface { CreateOrUpdateConfig(tx *pg.Tx, config *bean.DeploymentConfig, userId int32) (*bean.DeploymentConfig, error) - CreateOrUpdateConfigsInBulk(tx *pg.Tx, configs []*bean.DeploymentConfig, userId int32) ([]*bean.DeploymentConfig, error) - UpdateConfigs(tx *pg.Tx, configs []*bean.DeploymentConfig, userId int32) ([]*bean.DeploymentConfig, error) GetConfigForDevtronApps(appId, envId int) (*bean.DeploymentConfig, error) GetAndMigrateConfigIfAbsentForDevtronApps(appId, envId int) (*bean.DeploymentConfig, error) GetConfigForHelmApps(appId, envId int) (*bean.DeploymentConfig, error) GetConfigEvenIfInactive(appId, envId int) (*bean.DeploymentConfig, error) GetAndMigrateConfigIfAbsentForHelmApp(appId, envId int) (*bean.DeploymentConfig, error) - GetDeploymentConfigInBulk(configSelector []*bean.DeploymentConfigSelector) (map[bean.UniqueDeploymentConfigIdentifier]*bean.DeploymentConfig, error) } type DeploymentConfigServiceImpl struct { - deploymentConfigRepository deploymentConfig.Repository - logger *zap.SugaredLogger - chartRepository chartRepoRepository.ChartRepository - pipelineRepository pipelineConfig.PipelineRepository - appRepository appRepository.AppRepository - installedAppRepository repository.InstalledAppRepository + deploymentConfigRepository deploymentConfig.Repository + logger *zap.SugaredLogger + chartRepository chartRepoRepository.ChartRepository + pipelineRepository pipelineConfig.PipelineRepository + appRepository appRepository.AppRepository + installedAppRepository repository.InstalledAppRepository + deploymentServiceTypeConfig util.DeploymentServiceTypeConfig } func NewDeploymentConfigServiceImpl( @@ -41,14 +39,16 @@ func NewDeploymentConfigServiceImpl( pipelineRepository pipelineConfig.PipelineRepository, appRepository appRepository.AppRepository, installedAppRepository repository.InstalledAppRepository, + deploymentServiceTypeConfig util.DeploymentServiceTypeConfig, ) *DeploymentConfigServiceImpl { return &DeploymentConfigServiceImpl{ - deploymentConfigRepository: deploymentConfigRepository, - logger: logger, - chartRepository: chartRepository, - pipelineRepository: pipelineRepository, - appRepository: appRepository, - installedAppRepository: installedAppRepository, + deploymentConfigRepository: deploymentConfigRepository, + logger: logger, + chartRepository: chartRepository, + pipelineRepository: pipelineRepository, + appRepository: appRepository, + installedAppRepository: installedAppRepository, + deploymentServiceTypeConfig: deploymentServiceTypeConfig, } } @@ -82,96 +82,19 @@ func (impl *DeploymentConfigServiceImpl) CreateOrUpdateConfig(tx *pg.Tx, config return ConvertDeploymentConfigDbObjToDTO(newDBObj), nil } -func (impl *DeploymentConfigServiceImpl) CreateOrUpdateConfigsInBulk(tx *pg.Tx, configs []*bean.DeploymentConfig, userId int32) ([]*bean.DeploymentConfig, error) { - - appIdToEnvIdsMap := GetAppIdToEnvIsMapping( - configs, - func(config *bean.DeploymentConfig) int { return config.AppId }, - func(config *bean.DeploymentConfig) int { return config.EnvironmentId }) - - deploymentConfigDbObj, err := impl.deploymentConfigRepository.GetAppAndEnvLevelConfigsInBulk(appIdToEnvIdsMap) - if err != nil { - impl.logger.Errorw("error in getting deployment configs by appIdToEnvIds", "appIdToEnvIdsMap", appIdToEnvIdsMap, "err", err) - return nil, err - } - - savedDeploymentConfigsMap := make(map[bean.UniqueDeploymentConfigIdentifier]*deploymentConfig.DeploymentConfig) - for _, savedConfig := range deploymentConfigDbObj { - savedDeploymentConfigsMap[bean.GetConfigUniqueIdentifier(savedConfig.AppId, savedConfig.EnvironmentId)] = savedConfig - } - - oldDeploymentConfigs := make([]*deploymentConfig.DeploymentConfig, 0) - newDeploymentConfigs := make([]*deploymentConfig.DeploymentConfig, 0) - - for _, requestConfig := range configs { - - if oldConfig, isConfigPresent := savedDeploymentConfigsMap[bean.GetConfigUniqueIdentifier(requestConfig.AppId, requestConfig.EnvironmentId)]; isConfigPresent { - - oldConfigDbObject := ConvertDeploymentConfigDTOToDbObj(requestConfig) - oldConfigDbObject.Id = oldConfig.Id - oldConfigDbObject.AuditLog.UpdateAuditLog(userId) - oldDeploymentConfigs = append(oldDeploymentConfigs, oldConfigDbObject) - - } else if !isConfigPresent { - - newConfigDbObject := ConvertDeploymentConfigDTOToDbObj(requestConfig) - newConfigDbObject.AuditLog.CreateAuditLog(userId) - newDeploymentConfigs = append(newDeploymentConfigs, ConvertDeploymentConfigDTOToDbObj(requestConfig)) - - } - } +func (impl *DeploymentConfigServiceImpl) GetConfigForDevtronApps(appId, envId int) (*bean.DeploymentConfig, error) { - if len(newDeploymentConfigs) > 0 { - newDeploymentConfigs, err = impl.deploymentConfigRepository.SaveAll(tx, newDeploymentConfigs) - if err != nil { - impl.logger.Errorw("error in saving deployment config in db", "err", err) - return nil, err - } - } - if len(oldDeploymentConfigs) > 0 { - oldDeploymentConfigs, err = impl.deploymentConfigRepository.UpdateAll(tx, oldDeploymentConfigs) + if !impl.deploymentServiceTypeConfig.UseDeploymentConfigData { + configFromOldData, err := impl.parseFromOldTablesForDevtronApps(appId, envId) if err != nil { - impl.logger.Errorw("error in saving deployment config in db", "err", err) + impl.logger.Errorw("error in parsing config from charts and pipeline repository", "appId", appId, "envId", envId, "err", err) return nil, err } + return configFromOldData, nil } - allDeploymentConfigs := make([]*bean.DeploymentConfig, 0, len(oldDeploymentConfigs)+len(newDeploymentConfigs)) - - for _, c := range oldDeploymentConfigs { - allDeploymentConfigs = append(allDeploymentConfigs, ConvertDeploymentConfigDbObjToDTO(c)) - } - for _, c := range newDeploymentConfigs { - allDeploymentConfigs = append(allDeploymentConfigs, ConvertDeploymentConfigDbObjToDTO(c)) - } - return allDeploymentConfigs, nil -} - -func (impl *DeploymentConfigServiceImpl) UpdateConfigs(tx *pg.Tx, configs []*bean.DeploymentConfig, userId int32) ([]*bean.DeploymentConfig, error) { - - configDbObject := make([]*deploymentConfig.DeploymentConfig, 0) - for _, c := range configs { - dbObj := ConvertDeploymentConfigDTOToDbObj(c) - dbObj.AuditLog.UpdateAuditLog(userId) - configDbObject = append(configDbObject) - } + // if USE_DEPLOYMENT_CONFIG_DATA is true, first try to fetch data from deployment_config table and if not found use charts and pipeline respectively - var err error - configDbObject, err = impl.deploymentConfigRepository.UpdateAll(tx, configDbObject) - if err != nil { - impl.logger.Errorw("error in saving deployment config in db", "err", err) - return nil, err - } - - updatedConfigs := make([]*bean.DeploymentConfig, 0) - for _, c := range configDbObject { - updatedConfigs = append(updatedConfigs, ConvertDeploymentConfigDbObjToDTO(c)) - } - - return updatedConfigs, nil -} - -func (impl *DeploymentConfigServiceImpl) GetConfigForDevtronApps(appId, envId int) (*bean.DeploymentConfig, error) { appLevelConfigDbObj, err := impl.deploymentConfigRepository.GetAppLevelConfigForDevtronApps(appId) if err != nil && err != pg.ErrNoRows { impl.logger.Errorw("error in getting deployment config db object by appId", "appId", appId, "err", err) @@ -205,6 +128,15 @@ func (impl *DeploymentConfigServiceImpl) GetConfigForDevtronApps(appId, envId in func (impl *DeploymentConfigServiceImpl) GetAndMigrateConfigIfAbsentForDevtronApps(appId, envId int) (*bean.DeploymentConfig, error) { + if !impl.deploymentServiceTypeConfig.UseDeploymentConfigData { + configFromOldData, err := impl.parseFromOldTablesForDevtronApps(appId, envId) + if err != nil { + impl.logger.Errorw("error in parsing config from charts and pipeline repository", "appId", appId, "envId", envId, "err", err) + return nil, err + } + return configFromOldData, nil + } + appLevelConfigDbObj, err := impl.deploymentConfigRepository.GetAppLevelConfigForDevtronApps(appId) if err != nil && err != pg.ErrNoRows { impl.logger.Errorw("error in getting deployment config db object by appId", "appId", appId, "err", err) @@ -331,6 +263,15 @@ func (impl *DeploymentConfigServiceImpl) GetConfigDBObj(appId, envId int) (*depl func (impl *DeploymentConfigServiceImpl) GetConfigForHelmApps(appId, envId int) (*bean.DeploymentConfig, error) { + if !impl.deploymentServiceTypeConfig.UseDeploymentConfigData { + configFromOldData, err := impl.parseConfigForHelmApps(appId, envId) + if err != nil { + impl.logger.Errorw("error in parsing config from charts and pipeline repository", "appId", appId, "envId", envId, "err", err) + return nil, err + } + return ConvertDeploymentConfigDbObjToDTO(configFromOldData), nil + } + helmDeploymentConfig, err := impl.deploymentConfigRepository.GetByAppIdAndEnvId(appId, envId) if err != nil && err != pg.ErrNoRows { impl.logger.Errorw("error in fetching deployment config by by appId and envId", "appId", appId, "envId", envId, "err", err) @@ -358,6 +299,15 @@ func (impl *DeploymentConfigServiceImpl) GetConfigEvenIfInactive(appId, envId in func (impl *DeploymentConfigServiceImpl) GetAndMigrateConfigIfAbsentForHelmApp(appId, envId int) (*bean.DeploymentConfig, error) { + if !impl.deploymentServiceTypeConfig.UseDeploymentConfigData { + configFromOldData, err := impl.parseConfigForHelmApps(appId, envId) + if err != nil { + impl.logger.Errorw("error in parsing config from charts and pipeline repository", "appId", appId, "envId", envId, "err", err) + return nil, err + } + return ConvertDeploymentConfigDbObjToDTO(configFromOldData), nil + } + helmDeploymentConfig, err := impl.deploymentConfigRepository.GetByAppIdAndEnvId(appId, envId) if err != nil && err != pg.ErrNoRows { impl.logger.Errorw("error in fetching deployment config by by appId and envId", "appId", appId, "envId", envId, "err", err) @@ -409,345 +359,19 @@ func (impl *DeploymentConfigServiceImpl) parseConfigForHelmApps(appId int, envId return helmDeploymentConfig, nil } -func (impl *DeploymentConfigServiceImpl) GetDeploymentConfigInBulk(configSelector []*bean.DeploymentConfigSelector) (map[bean.UniqueDeploymentConfigIdentifier]*bean.DeploymentConfig, error) { - - appIds := make([]*int, len(configSelector)) - for _, s := range configSelector { - appIds = append(appIds, &s.AppId) - } - - apps, err := impl.appRepository.FindByIds(appIds) +func (impl *DeploymentConfigServiceImpl) parseFromOldTablesForDevtronApps(appId, envId int) (*bean.DeploymentConfig, error) { + appLevelConfig, err := impl.parseAppLevelConfigForDevtronApps(appId) if err != nil { - impl.logger.Errorw("error in fetching apps by ids", "ids", appIds, "err", err) + impl.logger.Errorw("error in parsing charts data to deployment config", "appId", appId, "err", err) return nil, err } - - devtronAppIds := make([]int, len(appIds)) - helmAppIds := make([]int, len(appIds)) - devtronAppIdMap := make(map[int]bool) - helmAppIdMap := make(map[int]bool) - - for _, a := range apps { - switch a.AppType { - case helper.CustomApp: - devtronAppIds = append(devtronAppIds, a.Id) - devtronAppIdMap[a.Id] = true - case helper.ChartStoreApp: - helmAppIds = append(helmAppIds, a.Id) - helmAppIdMap[a.Id] = true - } - } - - devtronAppSelectors := make([]*bean.DeploymentConfigSelector, len(appIds)) - helmAppSelectors := make([]*bean.DeploymentConfigSelector, len(appIds)) - - for _, c := range configSelector { - if _, ok := devtronAppIdMap[c.AppId]; ok { - devtronAppSelectors = append(devtronAppSelectors, c) - } - if _, ok := helmAppIdMap[c.AppId]; ok { - helmAppSelectors = append(helmAppSelectors, c) - } - } - - devtronAppConfigs, err := impl.GetDevtronAppConfigInBulk(devtronAppIds, devtronAppSelectors) - if err != nil { - impl.logger.Errorw("error in getting deployment config for devtron apps", "devtronAppSelectors", devtronAppSelectors, "err", err) - return nil, err - } - - helmAppConfigs, err := impl.GetHelmAppConfigInBulk(helmAppIds, helmAppSelectors) - if err != nil { - impl.logger.Errorw("error in getting deployment config for helm apps", "helmAppSelectors", helmAppSelectors, "err", err) - return nil, err - } - - allConfigs := make(map[bean.UniqueDeploymentConfigIdentifier]*bean.DeploymentConfig) - - for _, c := range devtronAppConfigs { - allConfigs[bean.GetConfigUniqueIdentifier(c.AppId, c.EnvironmentId)] = c - } - - for _, c := range helmAppConfigs { - allConfigs[bean.GetConfigUniqueIdentifier(c.AppId, c.EnvironmentId)] = c - } - - return allConfigs, nil -} - -func (impl *DeploymentConfigServiceImpl) GetDevtronAppConfigInBulk(appIds []int, configSelector []*bean.DeploymentConfigSelector) ([]*bean.DeploymentConfig, error) { - - appLevelDeploymentConfigs, err := impl.deploymentConfigRepository.GetAppLevelConfigByAppIds(appIds) - if err != nil { - impl.logger.Errorw("error in fetching deployment configs by appIds", "appIds", appIds, "err", err) - return nil, err - } - - allAppLevelConfigs := make([]*deploymentConfig.DeploymentConfig, 0, len(appIds)) - allAppLevelConfigs = append(allAppLevelConfigs, appLevelDeploymentConfigs...) - - if len(appLevelDeploymentConfigs) < len(appIds) { - presentAppIds := make(map[int]bool) - for _, c := range appLevelDeploymentConfigs { - presentAppIds[c.AppId] = true - } - notFoundAppIds := make([]int, 0, len(appIds)) - for _, id := range appIds { - if _, ok := presentAppIds[id]; !ok { - notFoundAppIds = append(notFoundAppIds, id) - } - } - migratedAppLevelDeploymentConfigs, err := impl.migrateAppLevelDataTODeploymentConfigInBulk(notFoundAppIds) - if err != nil { - impl.logger.Errorw("error in migrating all level deployment configs", "appIds", notFoundAppIds, "err", err) - return nil, err - } - allAppLevelConfigs = append(allAppLevelConfigs, migratedAppLevelDeploymentConfigs...) - } - - appIdToAppLevelConfigMapping := make(map[int]*deploymentConfig.DeploymentConfig, len(appIds)) - for _, appLevelConfig := range allAppLevelConfigs { - appIdToAppLevelConfigMapping[appLevelConfig.AppId] = appLevelConfig - } - - allEnvLevelConfigs := make([]*deploymentConfig.DeploymentConfig, 0, len(configSelector)) - - if len(configSelector) > 0 { - - appIdToEnvIdsMap := GetAppIdToEnvIsMapping( - configSelector, - func(config *bean.DeploymentConfigSelector) int { return config.AppId }, - func(config *bean.DeploymentConfigSelector) int { return config.EnvironmentId }) - - envLevelConfig, err := impl.deploymentConfigRepository.GetAppAndEnvLevelConfigsInBulk(appIdToEnvIdsMap) - if err != nil { - impl.logger.Errorw("error in getting and and env level config in bulk", "appIdToEnvIdsMap", appIdToEnvIdsMap, "err", err) - return nil, err - } - allEnvLevelConfigs = append(allEnvLevelConfigs, envLevelConfig...) - if len(envLevelConfig) < len(configSelector) { - - notFoundDeploymentConfigMap := make(map[bean.UniqueDeploymentConfigIdentifier]bool) - presentDeploymentConfigMap := make(map[bean.UniqueDeploymentConfigIdentifier]bool) - - for _, c := range envLevelConfig { - presentDeploymentConfigMap[bean.GetConfigUniqueIdentifier(c.AppId, c.EnvironmentId)] = true - } - - for _, c := range configSelector { - key := bean.GetConfigUniqueIdentifier(c.AppId, c.EnvironmentId) - if _, ok := presentDeploymentConfigMap[key]; !ok { - notFoundDeploymentConfigMap[key] = true - } - } - - migratedEnvLevelDeploymentConfigs, err := impl.migrateEnvLevelDataToDeploymentConfigInBulk(notFoundDeploymentConfigMap, appIdToAppLevelConfigMapping) - if err != nil { - impl.logger.Errorw("error in migrating env level configs", "notFoundDeploymentConfigMap", notFoundDeploymentConfigMap, "err", err) - return nil, err - } - allEnvLevelConfigs = append(allEnvLevelConfigs, migratedEnvLevelDeploymentConfigs...) - } - } - - envLevelConfigMapping := make(map[bean.UniqueDeploymentConfigIdentifier]*bean.DeploymentConfig) - for _, c := range envLevelConfigMapping { - envLevelConfigMapping[bean.GetConfigUniqueIdentifier(c.AppId, c.EnvironmentId)] = c - } - - allConfigs := make([]*bean.DeploymentConfig, 0) - for _, c := range configSelector { - if c.AppId > 0 && c.EnvironmentId > 0 { - allConfigs = append(allConfigs, envLevelConfigMapping[bean.GetConfigUniqueIdentifier(c.AppId, c.EnvironmentId)]) - } else if c.AppId > 0 { - // if user has sent only appId in config selector then only app level - allConfigs = append(allConfigs, ConvertDeploymentConfigDbObjToDTO(appIdToAppLevelConfigMapping[c.AppId])) - } - } - - return allConfigs, err -} - -func (impl *DeploymentConfigServiceImpl) migrateAppLevelDataTODeploymentConfigInBulk(appIds []int) ([]*deploymentConfig.DeploymentConfig, error) { - - charts, err := impl.chartRepository.FindLatestChartByAppIds(appIds) - if err != nil { - impl.logger.Errorw("error in fetching latest chart by appIds", "appIds", appIds, "err", err) - return nil, err - } - - allRepoUrls := make([]string, 0) - for _, c := range charts { - allRepoUrls = append(allRepoUrls, c.GitRepoUrl) - } - - configDBObjects := make([]*deploymentConfig.DeploymentConfig, 0, len(appIds)) - for _, c := range charts { - dbObj := &deploymentConfig.DeploymentConfig{ - ConfigType: GetDeploymentConfigType(c.IsCustomGitRepository), - AppId: c.AppId, - Active: true, - RepoUrl: c.GitRepoUrl, - ChartLocation: c.ChartLocation, - } - dbObj.AuditLog.CreateAuditLog(bean3.SYSTEM_USER_ID) - configDBObjects = append(configDBObjects, dbObj) - } - configDBObjects, err = impl.deploymentConfigRepository.SaveAll(nil, configDBObjects) - if err != nil { - impl.logger.Errorw("error in saving deployment config in DB", "appIds", appIds, "err", err) - return nil, err - } - - return configDBObjects, nil -} - -func (impl *DeploymentConfigServiceImpl) migrateEnvLevelDataToDeploymentConfigInBulk(notFoundDeploymentConfigMap map[bean.UniqueDeploymentConfigIdentifier]bool, appIdToAppLevelConfigMapping map[int]*deploymentConfig.DeploymentConfig) ([]*deploymentConfig.DeploymentConfig, error) { - - notFoundAppIdEnvIdsMap := make(map[int][]int) - for uniqueKey, _ := range notFoundDeploymentConfigMap { - appId, envId := uniqueKey.GetAppAndEnvId() - if _, ok := notFoundAppIdEnvIdsMap[appId]; !ok { - notFoundAppIdEnvIdsMap[appId] = make([]int, 0) - } - notFoundAppIdEnvIdsMap[appId] = append(notFoundAppIdEnvIdsMap[appId], envId) - } - - pipelines, err := impl.pipelineRepository.FindByAppIdToEnvIdsMapping(notFoundAppIdEnvIdsMap) - if err != nil { - impl.logger.Errorw("error in fetching pipeline by envToAppId mapping", "notFoundAppIdEnvIdsMap", notFoundAppIdEnvIdsMap, "err", err) - return nil, err - } - if len(pipelines) == 0 { - // pipelines are deleted - return nil, err - } - - pipelineMap := make(map[bean.UniqueDeploymentConfigIdentifier]*pipelineConfig.Pipeline) - for _, p := range pipelines { - pipelineMap[bean.GetConfigUniqueIdentifier(p.AppId, p.EnvironmentId)] = p - } - - configDBObjects := make([]*deploymentConfig.DeploymentConfig, 0, len(notFoundDeploymentConfigMap)) - for uniqueKey, _ := range notFoundDeploymentConfigMap { - if _, ok := pipelineMap[uniqueKey]; !ok { - //skipping for deleted pipelines - continue - } - pipeline := pipelineMap[uniqueKey] - deploymentAppType := pipelineMap[uniqueKey].DeploymentAppType - appLevelConfig := appIdToAppLevelConfigMapping[pipeline.AppId] - configDbObj := &deploymentConfig.DeploymentConfig{ - AppId: pipeline.AppId, - EnvironmentId: pipeline.EnvironmentId, - ConfigType: appLevelConfig.ConfigType, - RepoUrl: appLevelConfig.RepoUrl, - ChartLocation: appLevelConfig.ChartLocation, - Active: true, - DeploymentAppType: deploymentAppType, - } - configDbObj.AuditLog.CreateAuditLog(bean3.SYSTEM_USER_ID) - configDBObjects = append(configDBObjects, configDbObj) - } - - if len(configDBObjects) > 0 { - configDBObjects, err = impl.deploymentConfigRepository.SaveAll(nil, configDBObjects) - if err != nil { - impl.logger.Errorw("error in saving deployment config in DB", "notFoundDeploymentConfigMap", notFoundDeploymentConfigMap, "err", err) - return nil, err - } - } - - return configDBObjects, nil -} - -func (impl *DeploymentConfigServiceImpl) GetHelmAppConfigInBulk(appIds []int, configSelector []*bean.DeploymentConfigSelector) ([]*bean.DeploymentConfig, error) { - - allDeploymentConfigs := make([]*deploymentConfig.DeploymentConfig, 0, len(appIds)) - - appIdToEnvIdsMap := GetAppIdToEnvIsMapping( - configSelector, - func(config *bean.DeploymentConfigSelector) int { return config.AppId }, - func(config *bean.DeploymentConfigSelector) int { return config.EnvironmentId }) - - helmDeploymentConfig, err := impl.deploymentConfigRepository.GetAppAndEnvLevelConfigsInBulk(appIdToEnvIdsMap) - if err != nil && err != pg.ErrNoRows { - impl.logger.Errorw("error in fetching deployment config by by appId and envId", "envIdToAppIdMapping", appIdToEnvIdsMap, "err", err) - return nil, err - } - - allDeploymentConfigs = append(allDeploymentConfigs, helmDeploymentConfig...) - - if len(helmDeploymentConfig) < len(configSelector) { - - notFoundDeploymentConfigMap := make(map[bean.UniqueDeploymentConfigIdentifier]bool) - presentDeploymentConfigMap := make(map[bean.UniqueDeploymentConfigIdentifier]bool) - - for _, c := range helmDeploymentConfig { - presentDeploymentConfigMap[bean.GetConfigUniqueIdentifier(c.AppId, c.EnvironmentId)] = true - } - - for _, c := range configSelector { - key := bean.GetConfigUniqueIdentifier(c.AppId, c.EnvironmentId) - if _, ok := presentDeploymentConfigMap[key]; !ok { - notFoundDeploymentConfigMap[key] = true - } - } - - migratedConfigs, err := impl.migrateDeploymentConfigInBulkForHelmApps(appIds, notFoundDeploymentConfigMap) + if envId > 0 { + appAndEnvLevelConfig, err := impl.parseEnvLevelConfigForDevtronApps(appLevelConfig, appId, envId) if err != nil { - impl.logger.Errorw("error in migrating helm apps config data in bult to deployment config", "notFoundDeploymentConfigMap", notFoundDeploymentConfigMap, "err", err) + impl.logger.Errorw("error in parsing env level config to deployment config", "appId", appId, "err", err) return nil, err } - allDeploymentConfigs = append(allDeploymentConfigs, migratedConfigs...) - } - - deploymentConfigsResult := make([]*bean.DeploymentConfig, 0, len(appIds)) - for _, c := range allDeploymentConfigs { - deploymentConfigsResult = append(deploymentConfigsResult, ConvertDeploymentConfigDbObjToDTO(c)) - } - return deploymentConfigsResult, nil -} - -func (impl *DeploymentConfigServiceImpl) migrateDeploymentConfigInBulkForHelmApps(appIds []int, notFoundDeploymentConfigMap map[bean.UniqueDeploymentConfigIdentifier]bool) ([]*deploymentConfig.DeploymentConfig, error) { - installedApps, err := impl.installedAppRepository.FindInstalledAppByIds(appIds) - if err != nil { - impl.logger.Errorw("error in getting installed app by appId", "appIds", appIds, "err", err) - return nil, err - } - - appIdToInstalledAppMapping := make(map[int]*repository.InstalledApps) - - allRepoURLS := make([]string, 0, len(installedApps)) - - for _, ia := range installedApps { - appIdToInstalledAppMapping[ia.AppId] = ia - allRepoURLS = append(allRepoURLS, ia.GitOpsRepoUrl) - } - - configDBObjects := make([]*deploymentConfig.DeploymentConfig, 0, len(notFoundDeploymentConfigMap)) - - for appEnvUniqueKey, _ := range notFoundDeploymentConfigMap { - - appId, envId := appEnvUniqueKey.GetAppAndEnvId() - installedApp := appIdToInstalledAppMapping[appId] - helmDeploymentConfig := &deploymentConfig.DeploymentConfig{ - AppId: appId, - EnvironmentId: envId, - DeploymentAppType: installedApp.DeploymentAppType, - ConfigType: GetDeploymentConfigType(installedApp.IsCustomRepository), - RepoUrl: installedApp.GitOpsRepoUrl, - RepoName: installedApp.GitOpsRepoName, - Active: true, - } - helmDeploymentConfig.AuditLog.CreateAuditLog(bean3.SYSTEM_USER_ID) - configDBObjects = append(configDBObjects, helmDeploymentConfig) - } - - configDBObjects, err = impl.deploymentConfigRepository.SaveAll(nil, configDBObjects) - if err != nil { - impl.logger.Errorw("error in saving deployment config in DB", "notFoundDeploymentConfigMap", notFoundDeploymentConfigMap, "err", err) - return nil, err + return ConvertDeploymentConfigDbObjToDTO(appAndEnvLevelConfig), nil } - return configDBObjects, nil + return ConvertDeploymentConfigDbObjToDTO(appLevelConfig), nil } diff --git a/pkg/pipeline/AppDeploymentTypeChangeManager.go b/pkg/pipeline/AppDeploymentTypeChangeManager.go index ef2a83ad11..8813071ca1 100644 --- a/pkg/pipeline/AppDeploymentTypeChangeManager.go +++ b/pkg/pipeline/AppDeploymentTypeChangeManager.go @@ -157,7 +157,18 @@ func (impl *AppDeploymentTypeChangeManagerImpl) ChangeDeploymentType(ctx context return response, nil } - updatedDeploymentConfigs := make([]*bean4.DeploymentConfig, 0) + //Update in db + err = impl.pipelineRepository.UpdateCdPipelineDeploymentAppInFilter(string(request.DesiredDeploymentType), + pipelineIds, request.UserId, false, true) + if err != nil { + impl.logger.Errorw("failed to update deployment app type in db", + "pipeline ids", pipelineIds, + "desired deployment type", request.DesiredDeploymentType, + "err", err) + + return response, nil + } + for _, pipeline := range pipelines { deploymentConfig, err := impl.deploymentConfigService.GetAndMigrateConfigIfAbsentForDevtronApps(pipeline.AppId, pipeline.EnvironmentId) if err != nil { @@ -165,27 +176,13 @@ func (impl *AppDeploymentTypeChangeManagerImpl) ChangeDeploymentType(ctx context return nil, err } deploymentConfig.DeploymentAppType = request.DesiredDeploymentType - updatedDeploymentConfigs = append(updatedDeploymentConfigs, deploymentConfig) - } - - updatedDeploymentConfigs, err = impl.deploymentConfigService.UpdateConfigs(nil, updatedDeploymentConfigs, request.UserId) - if err != nil { - impl.logger.Errorw("error in updating configs", "err", err) - return nil, err + deploymentConfig, err = impl.deploymentConfigService.CreateOrUpdateConfig(nil, deploymentConfig, request.UserId) + if err != nil { + impl.logger.Errorw("error in updating configs", "err", err) + return nil, err + } } - // Update in db - //err = impl.pipelineRepository.UpdateCdPipelineDeploymentAppInFilter(string(request.DesiredDeploymentType), - // pipelineIds, request.UserId, false, true) - //if err != nil { - // impl.logger.Errorw("failed to update deployment app type in db", - // "pipeline ids", pipelineIds, - // "desired deployment type", request.DesiredDeploymentType, - // "err", err) - // - // return response, nil - //} - if !request.AutoTriggerDeployment { return response, nil } @@ -283,24 +280,13 @@ func (impl *AppDeploymentTypeChangeManagerImpl) ChangePipelineDeploymentType(ctx deploymentConfigs = append(deploymentConfigs, envDeploymentConfig) } - //err = impl.pipelineRepository.UpdateCdPipelineDeploymentAppInFilter(string(request.DesiredDeploymentType), - // pipelineIds, request.UserId, false, true) - // - //if err != nil { - // impl.logger.Errorw("failed to update deployment app type in db", - // "pipeline ids", pipelineIds, - // "desired deployment type", request.DesiredDeploymentType, - // "err", err) - // - // return response, nil - //} deleteResponse := impl.DeleteDeploymentApps(ctx, pipelines, deploymentConfigs, request.UserId) response.SuccessfulPipelines = deleteResponse.SuccessfulPipelines response.FailedPipelines = deleteResponse.FailedPipelines var cdPipelineIds []int - for _, item := range response.FailedPipelines { + for _, item := range response.SuccessfulPipelines { cdPipelineIds = append(cdPipelineIds, item.PipelineId) } @@ -308,7 +294,7 @@ func (impl *AppDeploymentTypeChangeManagerImpl) ChangePipelineDeploymentType(ctx return response, nil } - err = impl.pipelineRepository.UpdateCdPipelineDeploymentAppInFilter(string(deleteDeploymentType), + err = impl.pipelineRepository.UpdateCdPipelineDeploymentAppInFilter(string(request.DesiredDeploymentType), cdPipelineIds, request.UserId, true, false) if err != nil { @@ -320,6 +306,20 @@ func (impl *AppDeploymentTypeChangeManagerImpl) ChangePipelineDeploymentType(ctx return response, nil } + for _, item := range response.SuccessfulPipelines { + envDeploymentConfig, err := impl.deploymentConfigService.GetConfigForDevtronApps(item.AppId, item.EnvId) + if err != nil { + impl.logger.Errorw("error in fetching environment deployment config by appId and envId", "appId", item.AppId, "envId", item.EnvId, "err", err) + return response, err + } + envDeploymentConfig.DeploymentAppType = request.DesiredDeploymentType + envDeploymentConfig, err = impl.deploymentConfigService.CreateOrUpdateConfig(nil, envDeploymentConfig, request.UserId) + if err != nil { + impl.logger.Errorw("error in updating deployment config", "err", err) + return nil, err + } + } + return response, nil } @@ -426,7 +426,6 @@ func (impl *AppDeploymentTypeChangeManagerImpl) TriggerDeploymentAfterTypeChange }) } - deploymentConfigs := make([]*bean4.DeploymentConfig, 0) for _, p := range pipelines { envDeploymentConfig, err := impl.deploymentConfigService.GetAndMigrateConfigIfAbsentForDevtronApps(p.AppId, p.EnvironmentId) if err != nil { @@ -434,13 +433,11 @@ func (impl *AppDeploymentTypeChangeManagerImpl) TriggerDeploymentAfterTypeChange return response, err } envDeploymentConfig.DeploymentAppType = request.DesiredDeploymentType - deploymentConfigs = append(deploymentConfigs, envDeploymentConfig) - } - - deploymentConfigs, err = impl.deploymentConfigService.UpdateConfigs(nil, deploymentConfigs, request.UserId) - if err != nil { - impl.logger.Errorw("error in updating configs", "err", err) - return nil, err + envDeploymentConfig, err = impl.deploymentConfigService.CreateOrUpdateConfig(nil, envDeploymentConfig, request.UserId) + if err != nil { + impl.logger.Errorw("error in updating configs", "err", err) + return nil, err + } } err = impl.pipelineRepository.UpdateCdPipelineAfterDeployment(string(request.DesiredDeploymentType), @@ -521,7 +518,7 @@ func (impl *AppDeploymentTypeChangeManagerImpl) DeleteDeploymentApps(ctx context impl.logger.Errorw("error in registering acd app", "err", AcdRegisterErr) } if AcdRegisterErr == nil { - RepoURLUpdateErr = impl.chartService.ConfigureGitOpsRepoUrlForApp(pipeline.AppId, chartGitAttr.RepoUrl, chartGitAttr.ChartLocation, false, userId) + _, RepoURLUpdateErr = impl.chartService.ConfigureGitOpsRepoUrlForApp(pipeline.AppId, chartGitAttr.RepoUrl, chartGitAttr.ChartLocation, false, userId) if RepoURLUpdateErr != nil { impl.logger.Errorw("error in updating git repo url in charts", "err", RepoURLUpdateErr) } diff --git a/pkg/pipeline/DeploymentPipelineConfigService.go b/pkg/pipeline/DeploymentPipelineConfigService.go index 8e8ca746be..055537f8e8 100644 --- a/pkg/pipeline/DeploymentPipelineConfigService.go +++ b/pkg/pipeline/DeploymentPipelineConfigService.go @@ -439,10 +439,10 @@ func (impl *CdPipelineConfigServiceImpl) CreateCdPipelines(pipelineCreateRequest impl.logger.Errorw("error in registering app in acd", "err", err) return nil, err } - AppDeploymentConfig.RepoURL = chartGitAttr.RepoUrl - AppDeploymentConfig, err = impl.deploymentConfigService.CreateOrUpdateConfig(nil, AppDeploymentConfig, pipelineCreateRequest.UserId) + // below function will update gitRepoUrl for charts if user has not already provided gitOps repoURL + AppDeploymentConfig, err = impl.chartService.ConfigureGitOpsRepoUrlForApp(pipelineCreateRequest.AppId, chartGitAttr.RepoUrl, chartGitAttr.ChartLocation, false, pipelineCreateRequest.UserId) if err != nil { - impl.logger.Errorw("error in fetching creating env config", "appId", app.Id, "err", err) + impl.logger.Errorw("error in updating git repo url in charts", "err", err) return nil, err } } diff --git a/util/GlobalConfig.go b/util/GlobalConfig.go index d41f50eaf6..e980325d29 100644 --- a/util/GlobalConfig.go +++ b/util/GlobalConfig.go @@ -30,6 +30,7 @@ type EnvironmentVariables struct { type DeploymentServiceTypeConfig struct { ExternallyManagedDeploymentType bool `env:"IS_INTERNAL_USE" envDefault:"false"` HelmInstallASyncMode bool `env:"RUN_HELM_INSTALL_IN_ASYNC_MODE_HELM_APPS" envDefault:"false"` + UseDeploymentConfigData bool `env:"USE_DEPLOYMENT_CONFIG_DATA" envDefault:"false"` } type GlobalEnvVariables struct { From 0b952955c79338315512306b13695229c3d41b5b Mon Sep 17 00:00:00 2001 From: ayushmaheshwari Date: Tue, 9 Jul 2024 02:35:15 +0530 Subject: [PATCH 23/45] build fix --- env_gen.md | 1 + pkg/deployment/common/deploymentConfigService.go | 6 +++--- pkg/deployment/manifest/publish/ManifestPushService.go | 2 +- wire_gen.go | 2 +- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/env_gen.md b/env_gen.md index 7a76a63bc3..1c9b60bf2b 100644 --- a/env_gen.md +++ b/env_gen.md @@ -252,6 +252,7 @@ | USE_BUILDX | false | | | USE_CASBIN_V2 | false | | | USE_CUSTOM_HTTP_TRANSPORT | false | | + | USE_DEPLOYMENT_CONFIG_DATA | false | | | USE_EXTERNAL_NODE | false | | | USE_GIT_CLI | false | | | USE_IMAGE_TAG_FROM_GIT_PROVIDER_FOR_TAG_BASED_BUILD | false | | diff --git a/pkg/deployment/common/deploymentConfigService.go b/pkg/deployment/common/deploymentConfigService.go index 7776476482..ac4705273f 100644 --- a/pkg/deployment/common/deploymentConfigService.go +++ b/pkg/deployment/common/deploymentConfigService.go @@ -29,7 +29,7 @@ type DeploymentConfigServiceImpl struct { pipelineRepository pipelineConfig.PipelineRepository appRepository appRepository.AppRepository installedAppRepository repository.InstalledAppRepository - deploymentServiceTypeConfig util.DeploymentServiceTypeConfig + deploymentServiceTypeConfig *util.DeploymentServiceTypeConfig } func NewDeploymentConfigServiceImpl( @@ -39,7 +39,7 @@ func NewDeploymentConfigServiceImpl( pipelineRepository pipelineConfig.PipelineRepository, appRepository appRepository.AppRepository, installedAppRepository repository.InstalledAppRepository, - deploymentServiceTypeConfig util.DeploymentServiceTypeConfig, + envVariables *util.EnvironmentVariables, ) *DeploymentConfigServiceImpl { return &DeploymentConfigServiceImpl{ deploymentConfigRepository: deploymentConfigRepository, @@ -48,7 +48,7 @@ func NewDeploymentConfigServiceImpl( pipelineRepository: pipelineRepository, appRepository: appRepository, installedAppRepository: installedAppRepository, - deploymentServiceTypeConfig: deploymentServiceTypeConfig, + deploymentServiceTypeConfig: envVariables.DeploymentServiceTypeConfig, } } diff --git a/pkg/deployment/manifest/publish/ManifestPushService.go b/pkg/deployment/manifest/publish/ManifestPushService.go index 7817a79226..c477122525 100644 --- a/pkg/deployment/manifest/publish/ManifestPushService.go +++ b/pkg/deployment/manifest/publish/ManifestPushService.go @@ -144,7 +144,7 @@ func (impl *GitOpsManifestPushServiceImpl) PushChart(ctx context.Context, manife manifestPushTemplate.RepoUrl = newGitRepoUrl manifestPushResponse.NewGitRepoUrl = newGitRepoUrl // below function will override gitRepoUrl for charts even if user has already configured gitOps repoURL - err = impl.chartService.ConfigureGitOpsRepoUrlForApp(manifestPushTemplate.AppId, newGitRepoUrl, manifestPushTemplate.ChartLocation, manifestPushTemplate.IsCustomGitRepository, manifestPushTemplate.UserId) + _, err = impl.chartService.ConfigureGitOpsRepoUrlForApp(manifestPushTemplate.AppId, newGitRepoUrl, manifestPushTemplate.ChartLocation, manifestPushTemplate.IsCustomGitRepository, manifestPushTemplate.UserId) if err != nil { impl.logger.Errorw("error in updating git repo url in charts", "err", err) manifestPushResponse.Error = fmt.Errorf("No repository configured for Gitops! Error while migrating gitops repository: '%s'", newGitRepoUrl) diff --git a/wire_gen.go b/wire_gen.go index 7e6a5d0182..9e7d9d55b8 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -434,7 +434,7 @@ func InitializeApp() (*App, error) { pipelineStatusSyncDetailServiceImpl := status.NewPipelineStatusSyncDetailServiceImpl(sugaredLogger, pipelineStatusSyncDetailRepositoryImpl) installedAppVersionHistoryRepositoryImpl := repository3.NewInstalledAppVersionHistoryRepositoryImpl(sugaredLogger, db) repositoryImpl := deploymentConfig.NewRepositoryImpl(db) - deploymentConfigServiceImpl := common.NewDeploymentConfigServiceImpl(repositoryImpl, sugaredLogger, chartRepositoryImpl, pipelineRepositoryImpl, appRepositoryImpl, installedAppRepositoryImpl) + deploymentConfigServiceImpl := common.NewDeploymentConfigServiceImpl(repositoryImpl, sugaredLogger, chartRepositoryImpl, pipelineRepositoryImpl, appRepositoryImpl, installedAppRepositoryImpl, environmentVariables) pipelineStatusTimelineServiceImpl := status.NewPipelineStatusTimelineServiceImpl(sugaredLogger, pipelineStatusTimelineRepositoryImpl, cdWorkflowRepositoryImpl, userServiceImpl, pipelineStatusTimelineResourcesServiceImpl, pipelineStatusSyncDetailServiceImpl, installedAppRepositoryImpl, installedAppVersionHistoryRepositoryImpl, deploymentConfigServiceImpl) appServiceConfig, err := app2.GetAppServiceConfig() if err != nil { From 75d75c51dcf038a15859e13964c62bc6b7d634c0 Mon Sep 17 00:00:00 2001 From: ayushmaheshwari Date: Tue, 9 Jul 2024 12:58:37 +0530 Subject: [PATCH 24/45] wip: modifying get function --- .../common/deploymentConfigService.go | 26 +++++++++++-------- pkg/pipeline/CiCdPipelineOrchestrator.go | 6 ++--- 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/pkg/deployment/common/deploymentConfigService.go b/pkg/deployment/common/deploymentConfigService.go index ac4705273f..e2cf315086 100644 --- a/pkg/deployment/common/deploymentConfigService.go +++ b/pkg/deployment/common/deploymentConfigService.go @@ -128,15 +128,6 @@ func (impl *DeploymentConfigServiceImpl) GetConfigForDevtronApps(appId, envId in func (impl *DeploymentConfigServiceImpl) GetAndMigrateConfigIfAbsentForDevtronApps(appId, envId int) (*bean.DeploymentConfig, error) { - if !impl.deploymentServiceTypeConfig.UseDeploymentConfigData { - configFromOldData, err := impl.parseFromOldTablesForDevtronApps(appId, envId) - if err != nil { - impl.logger.Errorw("error in parsing config from charts and pipeline repository", "appId", appId, "envId", envId, "err", err) - return nil, err - } - return configFromOldData, nil - } - appLevelConfigDbObj, err := impl.deploymentConfigRepository.GetAppLevelConfigForDevtronApps(appId) if err != nil && err != pg.ErrNoRows { impl.logger.Errorw("error in getting deployment config db object by appId", "appId", appId, "err", err) @@ -150,7 +141,7 @@ func (impl *DeploymentConfigServiceImpl) GetAndMigrateConfigIfAbsentForDevtronAp return nil, err } } - + var envLevelConfig *bean.DeploymentConfig if envId > 0 { appAndEnvLevelConfig, err := impl.deploymentConfigRepository.GetByAppIdAndEnvId(appId, envId) if err != nil && err != pg.ErrNoRows { @@ -165,7 +156,20 @@ func (impl *DeploymentConfigServiceImpl) GetAndMigrateConfigIfAbsentForDevtronAp return nil, err } } - return ConvertDeploymentConfigDbObjToDTO(appAndEnvLevelConfig), nil + envLevelConfig = ConvertDeploymentConfigDbObjToDTO(appAndEnvLevelConfig) + } + + if !impl.deploymentServiceTypeConfig.UseDeploymentConfigData { + configFromOldData, err := impl.parseFromOldTablesForDevtronApps(appId, envId) + if err != nil { + impl.logger.Errorw("error in parsing config from charts and pipeline repository", "appId", appId, "envId", envId, "err", err) + return nil, err + } + return configFromOldData, nil + } + + if envId > 0 { + return envLevelConfig, nil } return ConvertDeploymentConfigDbObjToDTO(appLevelConfigDbObj), nil diff --git a/pkg/pipeline/CiCdPipelineOrchestrator.go b/pkg/pipeline/CiCdPipelineOrchestrator.go index 7502cfc3b0..4b58a25968 100644 --- a/pkg/pipeline/CiCdPipelineOrchestrator.go +++ b/pkg/pipeline/CiCdPipelineOrchestrator.go @@ -1619,9 +1619,9 @@ func (impl CiCdPipelineOrchestratorImpl) CreateCDPipelines(pipelineRequest *bean RunPreStageInEnv: pipelineRequest.RunPreStageInEnv, RunPostStageInEnv: pipelineRequest.RunPostStageInEnv, DeploymentAppCreated: false, - //DeploymentAppType: pipelineRequest.DeploymentAppType, - DeploymentAppName: fmt.Sprintf("%s-%s", appName, env.Name), - AuditLog: sql.AuditLog{UpdatedBy: userId, CreatedBy: userId, UpdatedOn: time.Now(), CreatedOn: time.Now()}, + DeploymentAppType: pipelineRequest.DeploymentAppType, + DeploymentAppName: fmt.Sprintf("%s-%s", appName, env.Name), + AuditLog: sql.AuditLog{UpdatedBy: userId, CreatedBy: userId, UpdatedOn: time.Now(), CreatedOn: time.Now()}, } err = impl.pipelineRepository.Save([]*pipelineConfig.Pipeline{pipeline}, tx) if err != nil { From 5c913e6f3290c7e0fdd66867c71966fe0eb7c894 Mon Sep 17 00:00:00 2001 From: ayushmaheshwari Date: Tue, 9 Jul 2024 14:07:03 +0530 Subject: [PATCH 25/45] handling custom gitops case for app clone --- api/bean/gitOps/GitOpsConfig.go | 4 ++++ .../common/deploymentConfigService.go | 18 ++++++++++++++ .../trigger/devtronApps/TriggerService.go | 24 ++++++++++++++++++- 3 files changed, 45 insertions(+), 1 deletion(-) diff --git a/api/bean/gitOps/GitOpsConfig.go b/api/bean/gitOps/GitOpsConfig.go index 22960800a1..c9015c95de 100644 --- a/api/bean/gitOps/GitOpsConfig.go +++ b/api/bean/gitOps/GitOpsConfig.go @@ -69,3 +69,7 @@ const ( func IsGitOpsRepoNotConfigured(gitRepoUrl string) bool { return len(gitRepoUrl) == 0 || gitRepoUrl == GIT_REPO_NOT_CONFIGURED } + +func IsGitOpsRepoConfigured(gitRepoUrl string) bool { + return len(gitRepoUrl) != 0 || gitRepoUrl != GIT_REPO_NOT_CONFIGURED +} diff --git a/pkg/deployment/common/deploymentConfigService.go b/pkg/deployment/common/deploymentConfigService.go index e2cf315086..1000efad30 100644 --- a/pkg/deployment/common/deploymentConfigService.go +++ b/pkg/deployment/common/deploymentConfigService.go @@ -20,6 +20,7 @@ type DeploymentConfigService interface { GetConfigForHelmApps(appId, envId int) (*bean.DeploymentConfig, error) GetConfigEvenIfInactive(appId, envId int) (*bean.DeploymentConfig, error) GetAndMigrateConfigIfAbsentForHelmApp(appId, envId int) (*bean.DeploymentConfig, error) + GetAppLevelConfigForDevtronApp(appId int) (*bean.DeploymentConfig, error) } type DeploymentConfigServiceImpl struct { @@ -379,3 +380,20 @@ func (impl *DeploymentConfigServiceImpl) parseFromOldTablesForDevtronApps(appId, } return ConvertDeploymentConfigDbObjToDTO(appLevelConfig), nil } + +func (impl *DeploymentConfigServiceImpl) GetAppLevelConfigForDevtronApp(appId int) (*bean.DeploymentConfig, error) { + appLevelConfigDbObj, err := impl.deploymentConfigRepository.GetAppLevelConfigForDevtronApps(appId) + if err != nil && err != pg.ErrNoRows { + impl.logger.Errorw("error in getting deployment config db object by appId", "appId", appId, "err", err) + return nil, err + } + if err == pg.ErrNoRows { + impl.logger.Infow("app level deployment config not found, migrating data from charts to deployment_config", "appId", appId, "err", err) + appLevelConfigDbObj, err = impl.migrateChartsDataToDeploymentConfig(appId) + if err != nil { + impl.logger.Errorw("error in migrating app level config to deployment config", "appId", appId, "err", err) + return nil, err + } + } + return ConvertDeploymentConfigDbObjToDTO(appLevelConfigDbObj), nil +} diff --git a/pkg/deployment/trigger/devtronApps/TriggerService.go b/pkg/deployment/trigger/devtronApps/TriggerService.go index e9b2e073fa..422974caa2 100644 --- a/pkg/deployment/trigger/devtronApps/TriggerService.go +++ b/pkg/deployment/trigger/devtronApps/TriggerService.go @@ -160,6 +160,7 @@ type TriggerServiceImpl struct { K8sUtil *util5.K8sServiceImpl transactionUtilImpl *sql.TransactionUtilImpl deploymentConfigService common.DeploymentConfigService + deploymentServiceTypeConfig *util3.DeploymentServiceTypeConfig } func NewTriggerServiceImpl(logger *zap.SugaredLogger, @@ -213,7 +214,9 @@ func NewTriggerServiceImpl(logger *zap.SugaredLogger, imageScanService security2.ImageScanService, K8sUtil *util5.K8sServiceImpl, transactionUtilImpl *sql.TransactionUtilImpl, - deploymentConfigService common.DeploymentConfigService) (*TriggerServiceImpl, error) { + deploymentConfigService common.DeploymentConfigService, + environmentVariables *util3.EnvironmentVariables, +) (*TriggerServiceImpl, error) { impl := &TriggerServiceImpl{ logger: logger, cdWorkflowCommonService: cdWorkflowCommonService, @@ -267,6 +270,7 @@ func NewTriggerServiceImpl(logger *zap.SugaredLogger, K8sUtil: K8sUtil, transactionUtilImpl: transactionUtilImpl, deploymentConfigService: deploymentConfigService, + deploymentServiceTypeConfig: envVariables.DeploymentServiceTypeConfig, } config, err := types.GetCdConfig() if err != nil { @@ -791,6 +795,24 @@ func (impl *TriggerServiceImpl) TriggerRelease(overrideRequest *bean3.ValuesOver } valuesOverrideResponse.DeploymentConfig = envDeploymentConfig + if impl.deploymentServiceTypeConfig.UseDeploymentConfigData && gitOps.IsGitOpsRepoNotConfigured(valuesOverrideResponse.DeploymentConfig.RepoURL) { + appLevelConfig, err := impl.deploymentConfigService.GetAppLevelConfigForDevtronApp(overrideRequest.AppId) + if err != nil { + impl.logger.Errorw("error in fetching app level config for devtron apps", "appId", overrideRequest.AppId, "err", err) + return releaseNo, err + } + // if url is present at app level and not at env level then copy app level url to env level config + if gitOps.IsGitOpsRepoConfigured(valuesOverrideResponse.DeploymentConfig.RepoURL) { + valuesOverrideResponse.DeploymentConfig.RepoURL = appLevelConfig.RepoURL + valuesOverrideResponse.DeploymentConfig, err = impl.deploymentConfigService.CreateOrUpdateConfig(nil, valuesOverrideResponse.DeploymentConfig, overrideRequest.UserId) + if err != nil { + impl.logger.Errorw("error in copying app level url to env level config", "appId", overrideRequest.AppId, "envId", overrideRequest.EnvId, "err", err) + return 0, err + } + + } + } + // auditDeploymentTriggerHistory is performed irrespective of BuildManifestForTrigger error - for auditing purposes historyErr := impl.auditDeploymentTriggerHistory(overrideRequest.WfrId, valuesOverrideResponse, newCtx, triggeredAt, triggeredBy) if historyErr != nil { From 0d354864f5b66870b723868603873053364a3aa1 Mon Sep 17 00:00:00 2001 From: ayushmaheshwari Date: Tue, 9 Jul 2024 14:44:30 +0530 Subject: [PATCH 26/45] handling app clone with custom gitops for system generated url --- .../repository/deploymentConfig/repository.go | 29 ++++++++++++------- .../common/deploymentConfigService.go | 10 +++++++ .../manifest/publish/ManifestPushService.go | 13 ++++++++- 3 files changed, 41 insertions(+), 11 deletions(-) diff --git a/internal/sql/repository/deploymentConfig/repository.go b/internal/sql/repository/deploymentConfig/repository.go index bd0e522f92..7ca699c72f 100644 --- a/internal/sql/repository/deploymentConfig/repository.go +++ b/internal/sql/repository/deploymentConfig/repository.go @@ -45,6 +45,7 @@ type Repository interface { GetAppLevelConfigByAppIds(appIds []int) ([]*DeploymentConfig, error) GetAppAndEnvLevelConfigsInBulk(appIdToEnvIdsMap map[int][]int) ([]*DeploymentConfig, error) GetByAppIdAndEnvIdEvenIfInactive(appId, envId int) (*DeploymentConfig, error) + UpdateRepoUrlByAppIdAndEnvId(repoUrl string, appId, envId int) error } type RepositoryImpl struct { @@ -55,7 +56,7 @@ func NewRepositoryImpl(dbConnection *pg.DB) *RepositoryImpl { return &RepositoryImpl{dbConnection: dbConnection} } -func (impl RepositoryImpl) Save(tx *pg.Tx, config *DeploymentConfig) (*DeploymentConfig, error) { +func (impl *RepositoryImpl) Save(tx *pg.Tx, config *DeploymentConfig) (*DeploymentConfig, error) { var err error if tx != nil { err = tx.Insert(config) @@ -65,7 +66,7 @@ func (impl RepositoryImpl) Save(tx *pg.Tx, config *DeploymentConfig) (*Deploymen return config, err } -func (impl RepositoryImpl) SaveAll(tx *pg.Tx, configs []*DeploymentConfig) ([]*DeploymentConfig, error) { +func (impl *RepositoryImpl) SaveAll(tx *pg.Tx, configs []*DeploymentConfig) ([]*DeploymentConfig, error) { var err error if tx != nil { err = tx.Insert(&configs) @@ -75,7 +76,7 @@ func (impl RepositoryImpl) SaveAll(tx *pg.Tx, configs []*DeploymentConfig) ([]*D return configs, err } -func (impl RepositoryImpl) Update(tx *pg.Tx, config *DeploymentConfig) (*DeploymentConfig, error) { +func (impl *RepositoryImpl) Update(tx *pg.Tx, config *DeploymentConfig) (*DeploymentConfig, error) { var err error if tx != nil { err = tx.Update(config) @@ -85,7 +86,7 @@ func (impl RepositoryImpl) Update(tx *pg.Tx, config *DeploymentConfig) (*Deploym return config, err } -func (impl RepositoryImpl) UpdateAll(tx *pg.Tx, config []*DeploymentConfig) ([]*DeploymentConfig, error) { +func (impl *RepositoryImpl) UpdateAll(tx *pg.Tx, config []*DeploymentConfig) ([]*DeploymentConfig, error) { var err error if tx != nil { err = tx.Update(config) @@ -95,13 +96,13 @@ func (impl RepositoryImpl) UpdateAll(tx *pg.Tx, config []*DeploymentConfig) ([]* return config, err } -func (impl RepositoryImpl) GetById(id int) (*DeploymentConfig, error) { +func (impl *RepositoryImpl) GetById(id int) (*DeploymentConfig, error) { result := &DeploymentConfig{} err := impl.dbConnection.Model(result).Where("id = ?", id).Where("active = ?", true).Select() return result, err } -func (impl RepositoryImpl) GetByAppIdAndEnvId(appId, envId int) (*DeploymentConfig, error) { +func (impl *RepositoryImpl) GetByAppIdAndEnvId(appId, envId int) (*DeploymentConfig, error) { result := &DeploymentConfig{} err := impl.dbConnection.Model(result). Where("app_id = ?", appId). @@ -112,7 +113,7 @@ func (impl RepositoryImpl) GetByAppIdAndEnvId(appId, envId int) (*DeploymentConf return result, err } -func (impl RepositoryImpl) GetAppLevelConfigForDevtronApps(appId int) (*DeploymentConfig, error) { +func (impl *RepositoryImpl) GetAppLevelConfigForDevtronApps(appId int) (*DeploymentConfig, error) { result := &DeploymentConfig{} err := impl.dbConnection.Model(result). Where("app_id = ? ", appId). @@ -122,7 +123,7 @@ func (impl RepositoryImpl) GetAppLevelConfigForDevtronApps(appId int) (*Deployme return result, err } -func (impl RepositoryImpl) GetAppLevelConfigByAppIds(appIds []int) ([]*DeploymentConfig, error) { +func (impl *RepositoryImpl) GetAppLevelConfigByAppIds(appIds []int) ([]*DeploymentConfig, error) { var result []*DeploymentConfig err := impl.dbConnection.Model(&result). Where("app_id in (?) and environment_id is NULL ", pg.In(appIds)). @@ -131,7 +132,7 @@ func (impl RepositoryImpl) GetAppLevelConfigByAppIds(appIds []int) ([]*Deploymen return result, err } -func (impl RepositoryImpl) GetAppAndEnvLevelConfigsInBulk(appIdToEnvIdsMap map[int][]int) ([]*DeploymentConfig, error) { +func (impl *RepositoryImpl) GetAppAndEnvLevelConfigsInBulk(appIdToEnvIdsMap map[int][]int) ([]*DeploymentConfig, error) { var result []*DeploymentConfig err := impl.dbConnection.Model(&result). WhereGroup(func(query *orm.Query) (*orm.Query, error) { @@ -148,7 +149,7 @@ func (impl RepositoryImpl) GetAppAndEnvLevelConfigsInBulk(appIdToEnvIdsMap map[i return result, err } -func (impl RepositoryImpl) GetByAppIdAndEnvIdEvenIfInactive(appId, envId int) (*DeploymentConfig, error) { +func (impl *RepositoryImpl) GetByAppIdAndEnvIdEvenIfInactive(appId, envId int) (*DeploymentConfig, error) { result := &DeploymentConfig{} err := impl.dbConnection.Model(result). WhereGroup(func(query *orm.Query) (*orm.Query, error) { @@ -164,3 +165,11 @@ func (impl RepositoryImpl) GetByAppIdAndEnvIdEvenIfInactive(appId, envId int) (* Select() return result, err } + +func (impl *RepositoryImpl) UpdateRepoUrlByAppIdAndEnvId(repoUrl string, appId, envId int) error { + _, err := impl.dbConnection. + Model((*DeploymentConfig)(nil)). + Where("app_id = ? and environment_id = ? ", appId, envId). + Update("repo_url = ? ", repoUrl) + return err +} diff --git a/pkg/deployment/common/deploymentConfigService.go b/pkg/deployment/common/deploymentConfigService.go index 1000efad30..42a3083569 100644 --- a/pkg/deployment/common/deploymentConfigService.go +++ b/pkg/deployment/common/deploymentConfigService.go @@ -21,6 +21,7 @@ type DeploymentConfigService interface { GetConfigEvenIfInactive(appId, envId int) (*bean.DeploymentConfig, error) GetAndMigrateConfigIfAbsentForHelmApp(appId, envId int) (*bean.DeploymentConfig, error) GetAppLevelConfigForDevtronApp(appId int) (*bean.DeploymentConfig, error) + UpdateRepoUrlForAppAndEnvId(repoURL string, appId, envId int) error } type DeploymentConfigServiceImpl struct { @@ -397,3 +398,12 @@ func (impl *DeploymentConfigServiceImpl) GetAppLevelConfigForDevtronApp(appId in } return ConvertDeploymentConfigDbObjToDTO(appLevelConfigDbObj), nil } + +func (impl *DeploymentConfigServiceImpl) UpdateRepoUrlForAppAndEnvId(repoURL string, appId, envId int) error { + err := impl.deploymentConfigRepository.UpdateRepoUrlByAppIdAndEnvId(repoURL, appId, envId) + if err != nil { + impl.logger.Errorw("error in updating repoUrl by app-id and env-id", "appId", appId, "envId", envId, "err", err) + return err + } + return nil +} diff --git a/pkg/deployment/manifest/publish/ManifestPushService.go b/pkg/deployment/manifest/publish/ManifestPushService.go index c477122525..26699f0b1a 100644 --- a/pkg/deployment/manifest/publish/ManifestPushService.go +++ b/pkg/deployment/manifest/publish/ManifestPushService.go @@ -27,6 +27,7 @@ import ( "github.com/devtron-labs/devtron/pkg/app/bean" "github.com/devtron-labs/devtron/pkg/app/status" chartService "github.com/devtron-labs/devtron/pkg/chart" + "github.com/devtron-labs/devtron/pkg/deployment/common" "github.com/devtron-labs/devtron/pkg/deployment/gitOps/config" gitOpsBean "github.com/devtron-labs/devtron/pkg/deployment/gitOps/config/bean" "github.com/devtron-labs/devtron/pkg/deployment/gitOps/git" @@ -55,6 +56,7 @@ type GitOpsManifestPushServiceImpl struct { chartService chartService.ChartService gitOperationService git.GitOperationService argoClientWrapperService argocdServer.ArgoClientWrapperService + deploymentConfigService common.DeploymentConfigService *sql.TransactionUtilImpl } @@ -67,7 +69,8 @@ func NewGitOpsManifestPushServiceImpl(logger *zap.SugaredLogger, chartService chartService.ChartService, gitOperationService git.GitOperationService, argoClientWrapperService argocdServer.ArgoClientWrapperService, - transactionUtilImpl *sql.TransactionUtilImpl) *GitOpsManifestPushServiceImpl { + transactionUtilImpl *sql.TransactionUtilImpl, + deploymentConfigService common.DeploymentConfigService) *GitOpsManifestPushServiceImpl { return &GitOpsManifestPushServiceImpl{ logger: logger, pipelineStatusTimelineService: pipelineStatusTimelineService, @@ -79,6 +82,7 @@ func NewGitOpsManifestPushServiceImpl(logger *zap.SugaredLogger, gitOperationService: gitOperationService, argoClientWrapperService: argoClientWrapperService, TransactionUtilImpl: transactionUtilImpl, + deploymentConfigService: deploymentConfigService, } } @@ -152,6 +156,13 @@ func (impl *GitOpsManifestPushServiceImpl) PushChart(ctx context.Context, manife return manifestPushResponse } + err := impl.deploymentConfigService.UpdateRepoUrlForAppAndEnvId(newGitRepoUrl, manifestPushTemplate.AppId, manifestPushTemplate.EnvironmentId) + if err != nil { + impl.logger.Errorw("error in updating repo url in env config", "appId", manifestPushTemplate.AppId, "envId", manifestPushTemplate.EnvironmentId, "err", err) + manifestPushResponse.Error = err + return manifestPushResponse + } + } // 4. Push Chart to Git Repository err = impl.pushChartToGitRepo(newCtx, manifestPushTemplate) From f2a311e5d15df509d9fcbcbf4ffd8d8d5946e58b Mon Sep 17 00:00:00 2001 From: ayushmaheshwari Date: Tue, 9 Jul 2024 15:20:46 +0530 Subject: [PATCH 27/45] fix: wrong call for helm app deployment app type change --- .../InstalledAppDeploymentTypeChangeService.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/appStore/installedApp/service/FullMode/deploymentTypeChange/InstalledAppDeploymentTypeChangeService.go b/pkg/appStore/installedApp/service/FullMode/deploymentTypeChange/InstalledAppDeploymentTypeChangeService.go index b038d2f36c..436fc98600 100644 --- a/pkg/appStore/installedApp/service/FullMode/deploymentTypeChange/InstalledAppDeploymentTypeChangeService.go +++ b/pkg/appStore/installedApp/service/FullMode/deploymentTypeChange/InstalledAppDeploymentTypeChangeService.go @@ -228,7 +228,7 @@ func (impl *InstalledAppDeploymentTypeChangeServiceImpl) performDbOperationsAfte } for _, ia := range installedApps { - deploymentConfig, err := impl.deploymentConfigService.GetAndMigrateConfigIfAbsentForDevtronApps(ia.AppId, ia.EnvironmentId) + deploymentConfig, err := impl.deploymentConfigService.GetAndMigrateConfigIfAbsentForHelmApp(ia.AppId, ia.EnvironmentId) if err != nil { impl.logger.Errorw("error in getting deployment config by appId and envId", "appId", ia.AppId, "envId", ia.EnvironmentId, "err", err) return err From 27b60f7c4a19c6333016f3217785027d6cca22d7 Mon Sep 17 00:00:00 2001 From: ayushmaheshwari Date: Tue, 9 Jul 2024 19:16:16 +0530 Subject: [PATCH 28/45] fix migration function for helm app --- .../common/deploymentConfigService.go | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/pkg/deployment/common/deploymentConfigService.go b/pkg/deployment/common/deploymentConfigService.go index 42a3083569..521187e9eb 100644 --- a/pkg/deployment/common/deploymentConfigService.go +++ b/pkg/deployment/common/deploymentConfigService.go @@ -305,15 +305,6 @@ func (impl *DeploymentConfigServiceImpl) GetConfigEvenIfInactive(appId, envId in func (impl *DeploymentConfigServiceImpl) GetAndMigrateConfigIfAbsentForHelmApp(appId, envId int) (*bean.DeploymentConfig, error) { - if !impl.deploymentServiceTypeConfig.UseDeploymentConfigData { - configFromOldData, err := impl.parseConfigForHelmApps(appId, envId) - if err != nil { - impl.logger.Errorw("error in parsing config from charts and pipeline repository", "appId", appId, "envId", envId, "err", err) - return nil, err - } - return ConvertDeploymentConfigDbObjToDTO(configFromOldData), nil - } - helmDeploymentConfig, err := impl.deploymentConfigRepository.GetByAppIdAndEnvId(appId, envId) if err != nil && err != pg.ErrNoRows { impl.logger.Errorw("error in fetching deployment config by by appId and envId", "appId", appId, "envId", envId, "err", err) @@ -327,6 +318,16 @@ func (impl *DeploymentConfigServiceImpl) GetAndMigrateConfigIfAbsentForHelmApp(a return nil, err } } + + if !impl.deploymentServiceTypeConfig.UseDeploymentConfigData { + configFromOldData, err := impl.parseConfigForHelmApps(appId, envId) + if err != nil { + impl.logger.Errorw("error in parsing config from charts and pipeline repository", "appId", appId, "envId", envId, "err", err) + return nil, err + } + return ConvertDeploymentConfigDbObjToDTO(configFromOldData), nil + } + return ConvertDeploymentConfigDbObjToDTO(helmDeploymentConfig), nil } From 4c7aefca2a617c01528e915531860022b2af9496 Mon Sep 17 00:00:00 2001 From: ayushmaheshwari Date: Tue, 9 Jul 2024 21:42:48 +0530 Subject: [PATCH 29/45] wip: updating dc on template update --- pkg/chart/ChartService.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/pkg/chart/ChartService.go b/pkg/chart/ChartService.go index af457bddd0..c9ce2f54bf 100644 --- a/pkg/chart/ChartService.go +++ b/pkg/chart/ChartService.go @@ -707,6 +707,20 @@ func (impl *ChartServiceImpl) UpdateAppOverride(ctx context.Context, templateReq return nil, err } + deploymentConfig := &bean2.DeploymentConfig{ + AppId: template.AppId, + ConfigType: common.GetDeploymentConfigType(template.IsCustomGitRepository), + RepoURL: template.GitRepoUrl, + ChartLocation: template.ChartLocation, + Active: true, + } + + deploymentConfig, err = impl.deploymentConfigService.CreateOrUpdateConfig(nil, deploymentConfig, templateRequest.UserId) + if err != nil { + impl.logger.Errorw("error in creating or updating deploymentConfig", "appId", templateRequest.AppId, "err", err) + return nil, err + } + appLevelMetricsUpdateReq := &bean.DeployedAppMetricsRequest{ EnableMetrics: templateRequest.IsAppMetricsEnabled, AppId: templateRequest.AppId, From e62aef180c02eb619759d558998edb66900bff50 Mon Sep 17 00:00:00 2001 From: ayushmaheshwari Date: Tue, 9 Jul 2024 21:50:20 +0530 Subject: [PATCH 30/45] fix: replacing migrate with get call --- .../installedApp/service/EAMode/InstalledAppDBService.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/appStore/installedApp/service/EAMode/InstalledAppDBService.go b/pkg/appStore/installedApp/service/EAMode/InstalledAppDBService.go index b7568b946f..da3bc324ea 100644 --- a/pkg/appStore/installedApp/service/EAMode/InstalledAppDBService.go +++ b/pkg/appStore/installedApp/service/EAMode/InstalledAppDBService.go @@ -293,7 +293,7 @@ func (impl *InstalledAppDBServiceImpl) GetInstalledAppVersion(id int, userId int impl.Logger.Errorw("error while fetching from db", "error", err) return nil, err } - deploymentConfig, err := impl.deploymentConfigService.GetAndMigrateConfigIfAbsentForHelmApp(model.InstalledApp.AppId, model.InstalledApp.EnvironmentId) + deploymentConfig, err := impl.deploymentConfigService.GetConfigForDevtronApps(model.InstalledApp.AppId, model.InstalledApp.EnvironmentId) if err != nil { impl.Logger.Errorw("error in getiting deployment config db object by appId and envId", "appId", model.InstalledApp.AppId, "envId", model.InstalledApp.EnvironmentId, "err", err) return nil, err From b7d00871e13f1524face6059092519d947989ff4 Mon Sep 17 00:00:00 2001 From: ayushmaheshwari Date: Tue, 9 Jul 2024 22:38:28 +0530 Subject: [PATCH 31/45] fix: wrong call for helm app --- .../installedApp/service/EAMode/InstalledAppDBService.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/appStore/installedApp/service/EAMode/InstalledAppDBService.go b/pkg/appStore/installedApp/service/EAMode/InstalledAppDBService.go index da3bc324ea..6eb4156e24 100644 --- a/pkg/appStore/installedApp/service/EAMode/InstalledAppDBService.go +++ b/pkg/appStore/installedApp/service/EAMode/InstalledAppDBService.go @@ -293,7 +293,7 @@ func (impl *InstalledAppDBServiceImpl) GetInstalledAppVersion(id int, userId int impl.Logger.Errorw("error while fetching from db", "error", err) return nil, err } - deploymentConfig, err := impl.deploymentConfigService.GetConfigForDevtronApps(model.InstalledApp.AppId, model.InstalledApp.EnvironmentId) + deploymentConfig, err := impl.deploymentConfigService.GetConfigForHelmApps(model.InstalledApp.AppId, model.InstalledApp.EnvironmentId) if err != nil { impl.Logger.Errorw("error in getiting deployment config db object by appId and envId", "appId", model.InstalledApp.AppId, "envId", model.InstalledApp.EnvironmentId, "err", err) return nil, err From e72b79bdcc58efa90a283af6e9ad4a1610f438d5 Mon Sep 17 00:00:00 2001 From: ayushmaheshwari Date: Wed, 10 Jul 2024 02:39:11 +0530 Subject: [PATCH 32/45] minor fixes --- pkg/chart/ChartService.go | 4 +++- pkg/pipeline/AppDeploymentTypeChangeManager.go | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/pkg/chart/ChartService.go b/pkg/chart/ChartService.go index c9ce2f54bf..53318843a3 100644 --- a/pkg/chart/ChartService.go +++ b/pkg/chart/ChartService.go @@ -970,8 +970,10 @@ func (impl *ChartServiceImpl) ConfigureGitOpsRepoUrlForApp(appId int, repoUrl, c } defer impl.chartRepository.RollbackTx(tx) var updatedCharts []*chartRepoRepository.Chart + var isCustom bool for _, ch := range charts { if !ch.IsCustomGitRepository { + isCustom = ch.IsCustomGitRepository ch.GitRepoUrl = repoUrl ch.UpdateAuditLog(userId) updatedCharts = append(updatedCharts, ch) @@ -989,7 +991,7 @@ func (impl *ChartServiceImpl) ConfigureGitOpsRepoUrlForApp(appId int, repoUrl, c deploymentConfig := &bean2.DeploymentConfig{ AppId: appId, - ConfigType: common.GetDeploymentConfigType(isCustomRepo), + ConfigType: common.GetDeploymentConfigType(isCustom), RepoURL: repoUrl, ChartLocation: chartLocation, Active: true, diff --git a/pkg/pipeline/AppDeploymentTypeChangeManager.go b/pkg/pipeline/AppDeploymentTypeChangeManager.go index 8813071ca1..ccfab729e4 100644 --- a/pkg/pipeline/AppDeploymentTypeChangeManager.go +++ b/pkg/pipeline/AppDeploymentTypeChangeManager.go @@ -159,7 +159,7 @@ func (impl *AppDeploymentTypeChangeManagerImpl) ChangeDeploymentType(ctx context //Update in db err = impl.pipelineRepository.UpdateCdPipelineDeploymentAppInFilter(string(request.DesiredDeploymentType), - pipelineIds, request.UserId, false, true) + pipelineIds, request.UserId, false, false) if err != nil { impl.logger.Errorw("failed to update deployment app type in db", "pipeline ids", pipelineIds, @@ -295,7 +295,7 @@ func (impl *AppDeploymentTypeChangeManagerImpl) ChangePipelineDeploymentType(ctx } err = impl.pipelineRepository.UpdateCdPipelineDeploymentAppInFilter(string(request.DesiredDeploymentType), - cdPipelineIds, request.UserId, true, false) + cdPipelineIds, request.UserId, false, false) if err != nil { impl.logger.Errorw("failed to update deployment app type in db", From 4d94230b0e2ec63b1ab7b63c0cd1c8defb491795 Mon Sep 17 00:00:00 2001 From: ayushmaheshwari Date: Wed, 10 Jul 2024 02:52:11 +0530 Subject: [PATCH 33/45] renaming chart path --- .../repository/deploymentConfig/repository.go | 2 +- pkg/chart/ChartService.go | 40 +++++++++---------- pkg/deployment/common/adapter.go | 4 +- pkg/deployment/common/bean/bean.go | 2 +- .../common/deploymentConfigService.go | 12 +++--- .../trigger/devtronApps/TriggerService.go | 4 +- .../AppDeploymentTypeChangeManager.go | 2 +- .../DeploymentPipelineConfigService.go | 2 +- scripts/sql/266_migration_deployed_app.up.sql | 8 ++-- 9 files changed, 39 insertions(+), 37 deletions(-) diff --git a/internal/sql/repository/deploymentConfig/repository.go b/internal/sql/repository/deploymentConfig/repository.go index 7ca699c72f..f48b76e5ee 100644 --- a/internal/sql/repository/deploymentConfig/repository.go +++ b/internal/sql/repository/deploymentConfig/repository.go @@ -29,7 +29,7 @@ type DeploymentConfig struct { ConfigType string `sql:"config_type"` RepoUrl string `sql:"repo_url"` RepoName string `sql:"repo_name"` - ChartLocation string `sql:"chart_location"` + ChartPath string `sql:"chart_path"` Active bool `sql:"active,notnull"` sql.AuditLog } diff --git a/pkg/chart/ChartService.go b/pkg/chart/ChartService.go index 53318843a3..09a62edbee 100644 --- a/pkg/chart/ChartService.go +++ b/pkg/chart/ChartService.go @@ -281,11 +281,11 @@ func (impl *ChartServiceImpl) Create(templateRequest TemplateRequest, ctx contex } deploymentConfig := &bean2.DeploymentConfig{ - AppId: templateRequest.AppId, - ConfigType: common.GetDeploymentConfigType(templateRequest.IsCustomGitRepository), - RepoURL: gitRepoUrl, - ChartLocation: chartLocation, - Active: true, + AppId: templateRequest.AppId, + ConfigType: common.GetDeploymentConfigType(templateRequest.IsCustomGitRepository), + RepoURL: gitRepoUrl, + ChartPath: chartLocation, + Active: true, } deploymentConfig, err = impl.deploymentConfigService.CreateOrUpdateConfig(nil, deploymentConfig, templateRequest.UserId) if err != nil { @@ -376,11 +376,11 @@ func (impl *ChartServiceImpl) CreateChartFromEnvOverride(templateRequest Templat } deploymentConfig := &bean2.DeploymentConfig{ - AppId: templateRequest.AppId, - ConfigType: common.GetDeploymentConfigType(templateRequest.IsCustomGitRepository), - RepoURL: gitRepoUrl, - ChartLocation: chartLocation, - Active: true, + AppId: templateRequest.AppId, + ConfigType: common.GetDeploymentConfigType(templateRequest.IsCustomGitRepository), + RepoURL: gitRepoUrl, + ChartPath: chartLocation, + Active: true, } deploymentConfig, err = impl.deploymentConfigService.CreateOrUpdateConfig(nil, deploymentConfig, templateRequest.UserId) if err != nil { @@ -708,11 +708,11 @@ func (impl *ChartServiceImpl) UpdateAppOverride(ctx context.Context, templateReq } deploymentConfig := &bean2.DeploymentConfig{ - AppId: template.AppId, - ConfigType: common.GetDeploymentConfigType(template.IsCustomGitRepository), - RepoURL: template.GitRepoUrl, - ChartLocation: template.ChartLocation, - Active: true, + AppId: template.AppId, + ConfigType: common.GetDeploymentConfigType(template.IsCustomGitRepository), + RepoURL: template.GitRepoUrl, + ChartPath: template.ChartLocation, + Active: true, } deploymentConfig, err = impl.deploymentConfigService.CreateOrUpdateConfig(nil, deploymentConfig, templateRequest.UserId) @@ -990,11 +990,11 @@ func (impl *ChartServiceImpl) ConfigureGitOpsRepoUrlForApp(appId int, repoUrl, c } deploymentConfig := &bean2.DeploymentConfig{ - AppId: appId, - ConfigType: common.GetDeploymentConfigType(isCustom), - RepoURL: repoUrl, - ChartLocation: chartLocation, - Active: true, + AppId: appId, + ConfigType: common.GetDeploymentConfigType(isCustom), + RepoURL: repoUrl, + ChartPath: chartLocation, + Active: true, } deploymentConfig, err = impl.deploymentConfigService.CreateOrUpdateConfig(nil, deploymentConfig, userId) if err != nil { diff --git a/pkg/deployment/common/adapter.go b/pkg/deployment/common/adapter.go index 8555000e70..2cace99f1c 100644 --- a/pkg/deployment/common/adapter.go +++ b/pkg/deployment/common/adapter.go @@ -13,7 +13,7 @@ func ConvertDeploymentConfigDTOToDbObj(config *bean.DeploymentConfig) *deploymen DeploymentAppType: config.DeploymentAppType, ConfigType: config.ConfigType, RepoUrl: config.RepoURL, - ChartLocation: config.ChartLocation, + ChartPath: config.ChartPath, Active: config.Active, } } @@ -26,7 +26,7 @@ func ConvertDeploymentConfigDbObjToDTO(dbObj *deploymentConfig.DeploymentConfig) DeploymentAppType: dbObj.DeploymentAppType, ConfigType: dbObj.ConfigType, RepoURL: dbObj.RepoUrl, - ChartLocation: dbObj.ChartLocation, + ChartPath: dbObj.ChartPath, Active: dbObj.Active, } } diff --git a/pkg/deployment/common/bean/bean.go b/pkg/deployment/common/bean/bean.go index 81bba2303c..cbf4da9a99 100644 --- a/pkg/deployment/common/bean/bean.go +++ b/pkg/deployment/common/bean/bean.go @@ -14,7 +14,7 @@ type DeploymentConfig struct { DeploymentAppType string RepoURL string RepoName string - ChartLocation string + ChartPath string Active bool } diff --git a/pkg/deployment/common/deploymentConfigService.go b/pkg/deployment/common/deploymentConfigService.go index 521187e9eb..f39f912065 100644 --- a/pkg/deployment/common/deploymentConfigService.go +++ b/pkg/deployment/common/deploymentConfigService.go @@ -200,11 +200,11 @@ func (impl *DeploymentConfigServiceImpl) parseAppLevelConfigForDevtronApps(appId return nil, err } ConfigDbObj := &deploymentConfig.DeploymentConfig{ - ConfigType: GetDeploymentConfigType(chart.IsCustomGitRepository), - AppId: appId, - Active: true, - RepoUrl: chart.GitRepoUrl, - ChartLocation: chart.ChartLocation, + ConfigType: GetDeploymentConfigType(chart.IsCustomGitRepository), + AppId: appId, + Active: true, + RepoUrl: chart.GitRepoUrl, + ChartPath: chart.ChartLocation, } return ConfigDbObj, nil } @@ -234,7 +234,7 @@ func (impl *DeploymentConfigServiceImpl) parseEnvLevelConfigForDevtronApps(appLe EnvironmentId: envId, ConfigType: appLevelConfig.ConfigType, RepoUrl: appLevelConfig.RepoUrl, - ChartLocation: appLevelConfig.ChartLocation, + ChartPath: appLevelConfig.ChartPath, Active: true, } diff --git a/pkg/deployment/trigger/devtronApps/TriggerService.go b/pkg/deployment/trigger/devtronApps/TriggerService.go index 422974caa2..f61af61249 100644 --- a/pkg/deployment/trigger/devtronApps/TriggerService.go +++ b/pkg/deployment/trigger/devtronApps/TriggerService.go @@ -965,7 +965,7 @@ func (impl *TriggerServiceImpl) buildManifestPushTemplate(overrideRequest *bean3 manifestPushTemplate.ChartReferenceTemplate = valuesOverrideResponse.EnvOverride.Chart.ReferenceTemplate manifestPushTemplate.ChartName = valuesOverrideResponse.EnvOverride.Chart.ChartName manifestPushTemplate.ChartVersion = valuesOverrideResponse.EnvOverride.Chart.ChartVersion - manifestPushTemplate.ChartLocation = valuesOverrideResponse.DeploymentConfig.ChartLocation + manifestPushTemplate.ChartLocation = valuesOverrideResponse.DeploymentConfig.ChartPath manifestPushTemplate.RepoUrl = valuesOverrideResponse.DeploymentConfig.RepoURL manifestPushTemplate.IsCustomGitRepository = common.IsCustomGitOpsRepo(valuesOverrideResponse.DeploymentConfig.ConfigType) } @@ -1228,7 +1228,7 @@ func (impl *TriggerServiceImpl) updateArgoPipeline(ctx context.Context, pipeline if impl.argoClientWrapperService.IsArgoAppPatchRequired(argoApplication.Spec.Source, deploymentConfig.RepoURL, envOverride.Chart.ChartLocation) { patchRequestDto := &bean7.ArgoCdAppPatchReqDto{ ArgoAppName: argoAppName, - ChartLocation: deploymentConfig.ChartLocation, + ChartLocation: deploymentConfig.ChartPath, GitRepoUrl: deploymentConfig.RepoURL, TargetRevision: bean7.TargetRevisionMaster, PatchType: bean7.PatchTypeMerge, diff --git a/pkg/pipeline/AppDeploymentTypeChangeManager.go b/pkg/pipeline/AppDeploymentTypeChangeManager.go index ccfab729e4..6d70ea011d 100644 --- a/pkg/pipeline/AppDeploymentTypeChangeManager.go +++ b/pkg/pipeline/AppDeploymentTypeChangeManager.go @@ -524,7 +524,7 @@ func (impl *AppDeploymentTypeChangeManagerImpl) DeleteDeploymentApps(ctx context } envDeploymentConfig.ConfigType = common.GetDeploymentConfigType(chart.IsCustomGitRepository) envDeploymentConfig.RepoURL = chartGitAttr.RepoUrl - envDeploymentConfig.ChartLocation = chartGitAttr.ChartLocation + envDeploymentConfig.ChartPath = chartGitAttr.ChartLocation envDeploymentConfig, RepoURLUpdateErr = impl.deploymentConfigService.CreateOrUpdateConfig(nil, envDeploymentConfig, userId) if RepoURLUpdateErr != nil { diff --git a/pkg/pipeline/DeploymentPipelineConfigService.go b/pkg/pipeline/DeploymentPipelineConfigService.go index 055537f8e8..b718908c58 100644 --- a/pkg/pipeline/DeploymentPipelineConfigService.go +++ b/pkg/pipeline/DeploymentPipelineConfigService.go @@ -457,7 +457,7 @@ func (impl *CdPipelineConfigServiceImpl) CreateCdPipelines(pipelineCreateRequest DeploymentAppType: pipeline.DeploymentAppType, RepoURL: AppDeploymentConfig.RepoURL, RepoName: AppDeploymentConfig.RepoName, - ChartLocation: AppDeploymentConfig.ChartLocation, + ChartPath: AppDeploymentConfig.ChartPath, Active: true, } envDeploymentConfig, err := impl.deploymentConfigService.CreateOrUpdateConfig(nil, envDeploymentConfig, pipelineCreateRequest.UserId) diff --git a/scripts/sql/266_migration_deployed_app.up.sql b/scripts/sql/266_migration_deployed_app.up.sql index 873c2fd9b2..829999f9d8 100644 --- a/scripts/sql/266_migration_deployed_app.up.sql +++ b/scripts/sql/266_migration_deployed_app.up.sql @@ -29,7 +29,7 @@ CREATE TABLE IF NOT EXISTS "public"."deployment_config" "config_type" VARCHAR(100), "repo_url" VARCHAR(250), "repo_name" VARCHAR(200), - "chart_location" VARCHAR(250), + "chart_path" VARCHAR(250), "credential_type" VARCHAR(100), "credential_id_int" int, "credential_id_string" VARCHAR(100), @@ -38,5 +38,7 @@ CREATE TABLE IF NOT EXISTS "public"."deployment_config" "created_by" integer, "updated_on" timestamptz, "updated_by" integer, - PRIMARY KEY ("id") - ); \ No newline at end of file + PRIMARY KEY ("id"), + CONSTRAINT "deployment_config_app_id_fk" FOREIGN KEY ("app_id") REFERENCES "public"."app" ("id"), + CONSTRAINT "deployment_config_env_id_fk" FOREIGN KEY ("environment_id") REFERENCES "public"."environment" ("id") +); \ No newline at end of file From e3be2efefcf6fe284b083d185538da4159f1fcdb Mon Sep 17 00:00:00 2001 From: ayushmaheshwari Date: Wed, 10 Jul 2024 03:26:53 +0530 Subject: [PATCH 34/45] updating deprecated columns --- .../installedApp/repository/InstalledAppRepository.go | 4 ++-- pkg/chart/ChartService.go | 2 +- pkg/chartRepo/repository/ChartsRepository.go | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pkg/appStore/installedApp/repository/InstalledAppRepository.go b/pkg/appStore/installedApp/repository/InstalledAppRepository.go index a1a64f3200..c9ffb1e00f 100644 --- a/pkg/appStore/installedApp/repository/InstalledAppRepository.go +++ b/pkg/appStore/installedApp/repository/InstalledAppRepository.go @@ -45,9 +45,9 @@ type InstalledApps struct { EnvironmentId int `sql:"environment_id,notnull"` Active bool `sql:"active, notnull"` GitOpsRepoName string `sql:"git_ops_repo_name"` // Deprecated; currently in use for backward compatibility - GitOpsRepoUrl string `sql:"git_ops_repo_url"` // GitOpsRepoName has been migrated to GitOpsRepoUrl; Make sure to migrate from GitOpsRepoName for future flows + GitOpsRepoUrl string `sql:"git_ops_repo_url"` // Deprecated; use deployment_config table instead GitOpsRepoName has been migrated to GitOpsRepoUrl; Make sure to migrate from GitOpsRepoName for future flows IsCustomRepository bool `sql:"is_custom_repository"` - DeploymentAppType string `sql:"deployment_app_type"` + DeploymentAppType string `sql:"deployment_app_type"` // Deprecated; use deployment_config table instead Status appStoreBean.AppstoreDeploymentStatus `sql:"status"` DeploymentAppDeleteRequest bool `sql:"deployment_app_delete_request"` Notes string `json:"notes"` diff --git a/pkg/chart/ChartService.go b/pkg/chart/ChartService.go index 09a62edbee..f731159f01 100644 --- a/pkg/chart/ChartService.go +++ b/pkg/chart/ChartService.go @@ -476,7 +476,7 @@ func (impl *ChartServiceImpl) chartAdaptor(chart *chartRepoRepository.Chart, isA IsBasicViewLocked: chart.IsBasicViewLocked, CurrentViewEditor: chart.CurrentViewEditor, GitRepoUrl: gitRepoUrl, - IsCustomGitRepository: chart.IsCustomGitRepository, + IsCustomGitRepository: deploymentConfig.ConfigType == bean2.CUSTOM.String(), ImageDescriptorTemplate: chart.ImageDescriptorTemplate, } if chart.Latest { diff --git a/pkg/chartRepo/repository/ChartsRepository.go b/pkg/chartRepo/repository/ChartsRepository.go index f6b8e6fdec..ae7521b8c4 100644 --- a/pkg/chartRepo/repository/ChartsRepository.go +++ b/pkg/chartRepo/repository/ChartsRepository.go @@ -37,8 +37,8 @@ type Chart struct { PipelineOverride string `sql:"pipeline_override"` //json format // pipeline values -> strategy values Status models.ChartStatus `sql:"status"` //(new , deployment-in-progress, deployed-To-production, error ) Active bool `sql:"active"` - GitRepoUrl string `sql:"git_repo_url"` //git repository where chart is stored - ChartLocation string `sql:"chart_location"` //location within git repo where current chart is pointing + GitRepoUrl string `sql:"git_repo_url"` // Deprecated; use deployment_config table instead //git repository where chart is stored + ChartLocation string `sql:"chart_location"` // Deprecated; use deployment_config table instead //location within git repo where current chart is pointing ReferenceTemplate string `sql:"reference_template"` ImageDescriptorTemplate string `sql:"image_descriptor_template"` ChartRefId int `sql:"chart_ref_id"` @@ -47,7 +47,7 @@ type Chart struct { ReferenceChart []byte `sql:"reference_chart"` IsBasicViewLocked bool `sql:"is_basic_view_locked,notnull"` CurrentViewEditor models.ChartsViewEditorType `sql:"current_view_editor"` - IsCustomGitRepository bool `sql:"is_custom_repository"` + IsCustomGitRepository bool `sql:"is_custom_repository"` // Deprecated; use deployment_config table instead ResolvedGlobalOverride string `sql:"-"` sql.AuditLog } From 9ad5acb359cde7b6f425a751e1675ff8b2cda9a2 Mon Sep 17 00:00:00 2001 From: ayushmaheshwari Date: Wed, 10 Jul 2024 03:30:27 +0530 Subject: [PATCH 35/45] updating dc.active --- pkg/chartRepo/repository/ChartsRepository.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/chartRepo/repository/ChartsRepository.go b/pkg/chartRepo/repository/ChartsRepository.go index ae7521b8c4..ce0435ee34 100644 --- a/pkg/chartRepo/repository/ChartsRepository.go +++ b/pkg/chartRepo/repository/ChartsRepository.go @@ -244,7 +244,7 @@ func (repositoryImpl ChartRepositoryImpl) FindChartByGitRepoUrl(gitRepoUrl strin var chart Chart err := repositoryImpl.dbConnection.Model(&chart). Join("INNER JOIN app ON app.id=app_id"). - Join("LEFT JOIN deployment_config dc on dc.app_id = chart.app_id and dc.environment_id is null"). + Join("LEFT JOIN deployment_config dc on dc.active=true and dc.app_id = chart.app_id and dc.environment_id is null"). Where("app.active = ?", true). Where("(chart.git_repo_url = ? or dc.repo_url = ?)", gitRepoUrl, gitRepoUrl). Where("chart.active = ?", true). From 3673aac5bd235a5a8964fa829c5281df9ead1494 Mon Sep 17 00:00:00 2001 From: ayushmaheshwari Date: Wed, 10 Jul 2024 03:32:43 +0530 Subject: [PATCH 36/45] updating dc.active --- pkg/appStore/installedApp/repository/InstalledAppRepository.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/appStore/installedApp/repository/InstalledAppRepository.go b/pkg/appStore/installedApp/repository/InstalledAppRepository.go index c9ffb1e00f..bdf7d04751 100644 --- a/pkg/appStore/installedApp/repository/InstalledAppRepository.go +++ b/pkg/appStore/installedApp/repository/InstalledAppRepository.go @@ -797,7 +797,7 @@ func (impl InstalledAppRepositoryImpl) GetInstalledAppByGitRepoUrl(repoName, rep model := &InstalledApps{} err := impl.dbConnection.Model(model). Column("installed_apps.*", "App"). - Join("LEFT JOIN deployment_config dc on dc.app_id = installed_apps.app_id and dc.environment_id=installed_apps.environment_id"). + Join("LEFT JOIN deployment_config dc on dc.active=true and dc.app_id = installed_apps.app_id and dc.environment_id=installed_apps.environment_id"). Where("installed_apps.git_ops_repo_name = ? OR (installed_apps.git_ops_repo_url = ? or dc.repo_url = ?)", repoName, repoUrl, repoUrl). Where("installed_apps.active = true"). Where("app.active = true"). From 641cb35a5438f886432ef4863d3d670b3c6f0614 Mon Sep 17 00:00:00 2001 From: ayushmaheshwari Date: Wed, 10 Jul 2024 17:44:42 +0530 Subject: [PATCH 37/45] fix redirection --- internal/sql/repository/AppListingRepository.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/sql/repository/AppListingRepository.go b/internal/sql/repository/AppListingRepository.go index 7d5c4f0095..94193e4836 100644 --- a/internal/sql/repository/AppListingRepository.go +++ b/internal/sql/repository/AppListingRepository.go @@ -606,6 +606,7 @@ func (impl AppListingRepositoryImpl) FetchAppStageStatus(appId int, appType int) } if (gitOps.IsGitOpsRepoNotConfigured(stages.ChartGitRepoUrl) && gitOps.IsGitOpsRepoNotConfigured(stages.DeploymentConfigRepoURL)) && stages.CiPipelineId == 0 { stages.ChartGitRepoUrl = "" + stages.DeploymentConfigRepoURL = "" } appStageStatus = append(appStageStatus, impl.makeAppStageStatus(0, "APP", stages.AppId, true), impl.makeAppStageStatus(1, "MATERIAL", materialExists, true), From 7b485a002c2ef70072bec33e0d5f3bd2f2037d87 Mon Sep 17 00:00:00 2001 From: ayushmaheshwari Date: Thu, 11 Jul 2024 15:06:09 +0530 Subject: [PATCH 38/45] removing chart path --- internal/sql/repository/deploymentConfig/repository.go | 1 - pkg/chart/ChartService.go | 4 ---- pkg/chartRepo/repository/ChartsRepository.go | 2 +- pkg/deployment/common/adapter.go | 2 -- pkg/deployment/common/bean/bean.go | 1 - pkg/deployment/common/deploymentConfigService.go | 2 -- pkg/deployment/trigger/devtronApps/TriggerService.go | 5 ++--- pkg/pipeline/AppDeploymentTypeChangeManager.go | 1 - pkg/pipeline/DeploymentPipelineConfigService.go | 1 - wire_gen.go | 2 +- 10 files changed, 4 insertions(+), 17 deletions(-) diff --git a/internal/sql/repository/deploymentConfig/repository.go b/internal/sql/repository/deploymentConfig/repository.go index f48b76e5ee..d61b51a75d 100644 --- a/internal/sql/repository/deploymentConfig/repository.go +++ b/internal/sql/repository/deploymentConfig/repository.go @@ -29,7 +29,6 @@ type DeploymentConfig struct { ConfigType string `sql:"config_type"` RepoUrl string `sql:"repo_url"` RepoName string `sql:"repo_name"` - ChartPath string `sql:"chart_path"` Active bool `sql:"active,notnull"` sql.AuditLog } diff --git a/pkg/chart/ChartService.go b/pkg/chart/ChartService.go index f731159f01..686c0bce65 100644 --- a/pkg/chart/ChartService.go +++ b/pkg/chart/ChartService.go @@ -284,7 +284,6 @@ func (impl *ChartServiceImpl) Create(templateRequest TemplateRequest, ctx contex AppId: templateRequest.AppId, ConfigType: common.GetDeploymentConfigType(templateRequest.IsCustomGitRepository), RepoURL: gitRepoUrl, - ChartPath: chartLocation, Active: true, } deploymentConfig, err = impl.deploymentConfigService.CreateOrUpdateConfig(nil, deploymentConfig, templateRequest.UserId) @@ -379,7 +378,6 @@ func (impl *ChartServiceImpl) CreateChartFromEnvOverride(templateRequest Templat AppId: templateRequest.AppId, ConfigType: common.GetDeploymentConfigType(templateRequest.IsCustomGitRepository), RepoURL: gitRepoUrl, - ChartPath: chartLocation, Active: true, } deploymentConfig, err = impl.deploymentConfigService.CreateOrUpdateConfig(nil, deploymentConfig, templateRequest.UserId) @@ -711,7 +709,6 @@ func (impl *ChartServiceImpl) UpdateAppOverride(ctx context.Context, templateReq AppId: template.AppId, ConfigType: common.GetDeploymentConfigType(template.IsCustomGitRepository), RepoURL: template.GitRepoUrl, - ChartPath: template.ChartLocation, Active: true, } @@ -993,7 +990,6 @@ func (impl *ChartServiceImpl) ConfigureGitOpsRepoUrlForApp(appId int, repoUrl, c AppId: appId, ConfigType: common.GetDeploymentConfigType(isCustom), RepoURL: repoUrl, - ChartPath: chartLocation, Active: true, } deploymentConfig, err = impl.deploymentConfigService.CreateOrUpdateConfig(nil, deploymentConfig, userId) diff --git a/pkg/chartRepo/repository/ChartsRepository.go b/pkg/chartRepo/repository/ChartsRepository.go index ce0435ee34..7d82b84844 100644 --- a/pkg/chartRepo/repository/ChartsRepository.go +++ b/pkg/chartRepo/repository/ChartsRepository.go @@ -38,7 +38,7 @@ type Chart struct { Status models.ChartStatus `sql:"status"` //(new , deployment-in-progress, deployed-To-production, error ) Active bool `sql:"active"` GitRepoUrl string `sql:"git_repo_url"` // Deprecated; use deployment_config table instead //git repository where chart is stored - ChartLocation string `sql:"chart_location"` // Deprecated; use deployment_config table instead //location within git repo where current chart is pointing + ChartLocation string `sql:"chart_location"` //location within git repo where current chart is pointing ReferenceTemplate string `sql:"reference_template"` ImageDescriptorTemplate string `sql:"image_descriptor_template"` ChartRefId int `sql:"chart_ref_id"` diff --git a/pkg/deployment/common/adapter.go b/pkg/deployment/common/adapter.go index 2cace99f1c..a6f4618e5a 100644 --- a/pkg/deployment/common/adapter.go +++ b/pkg/deployment/common/adapter.go @@ -13,7 +13,6 @@ func ConvertDeploymentConfigDTOToDbObj(config *bean.DeploymentConfig) *deploymen DeploymentAppType: config.DeploymentAppType, ConfigType: config.ConfigType, RepoUrl: config.RepoURL, - ChartPath: config.ChartPath, Active: config.Active, } } @@ -26,7 +25,6 @@ func ConvertDeploymentConfigDbObjToDTO(dbObj *deploymentConfig.DeploymentConfig) DeploymentAppType: dbObj.DeploymentAppType, ConfigType: dbObj.ConfigType, RepoURL: dbObj.RepoUrl, - ChartPath: dbObj.ChartPath, Active: dbObj.Active, } } diff --git a/pkg/deployment/common/bean/bean.go b/pkg/deployment/common/bean/bean.go index cbf4da9a99..4e29a85533 100644 --- a/pkg/deployment/common/bean/bean.go +++ b/pkg/deployment/common/bean/bean.go @@ -14,7 +14,6 @@ type DeploymentConfig struct { DeploymentAppType string RepoURL string RepoName string - ChartPath string Active bool } diff --git a/pkg/deployment/common/deploymentConfigService.go b/pkg/deployment/common/deploymentConfigService.go index f39f912065..21ffafbf05 100644 --- a/pkg/deployment/common/deploymentConfigService.go +++ b/pkg/deployment/common/deploymentConfigService.go @@ -204,7 +204,6 @@ func (impl *DeploymentConfigServiceImpl) parseAppLevelConfigForDevtronApps(appId AppId: appId, Active: true, RepoUrl: chart.GitRepoUrl, - ChartPath: chart.ChartLocation, } return ConfigDbObj, nil } @@ -234,7 +233,6 @@ func (impl *DeploymentConfigServiceImpl) parseEnvLevelConfigForDevtronApps(appLe EnvironmentId: envId, ConfigType: appLevelConfig.ConfigType, RepoUrl: appLevelConfig.RepoUrl, - ChartPath: appLevelConfig.ChartPath, Active: true, } diff --git a/pkg/deployment/trigger/devtronApps/TriggerService.go b/pkg/deployment/trigger/devtronApps/TriggerService.go index f61af61249..797c5ac380 100644 --- a/pkg/deployment/trigger/devtronApps/TriggerService.go +++ b/pkg/deployment/trigger/devtronApps/TriggerService.go @@ -215,7 +215,6 @@ func NewTriggerServiceImpl(logger *zap.SugaredLogger, K8sUtil *util5.K8sServiceImpl, transactionUtilImpl *sql.TransactionUtilImpl, deploymentConfigService common.DeploymentConfigService, - environmentVariables *util3.EnvironmentVariables, ) (*TriggerServiceImpl, error) { impl := &TriggerServiceImpl{ logger: logger, @@ -965,7 +964,7 @@ func (impl *TriggerServiceImpl) buildManifestPushTemplate(overrideRequest *bean3 manifestPushTemplate.ChartReferenceTemplate = valuesOverrideResponse.EnvOverride.Chart.ReferenceTemplate manifestPushTemplate.ChartName = valuesOverrideResponse.EnvOverride.Chart.ChartName manifestPushTemplate.ChartVersion = valuesOverrideResponse.EnvOverride.Chart.ChartVersion - manifestPushTemplate.ChartLocation = valuesOverrideResponse.DeploymentConfig.ChartPath + manifestPushTemplate.ChartLocation = valuesOverrideResponse.EnvOverride.Chart.ChartLocation manifestPushTemplate.RepoUrl = valuesOverrideResponse.DeploymentConfig.RepoURL manifestPushTemplate.IsCustomGitRepository = common.IsCustomGitOpsRepo(valuesOverrideResponse.DeploymentConfig.ConfigType) } @@ -1228,7 +1227,7 @@ func (impl *TriggerServiceImpl) updateArgoPipeline(ctx context.Context, pipeline if impl.argoClientWrapperService.IsArgoAppPatchRequired(argoApplication.Spec.Source, deploymentConfig.RepoURL, envOverride.Chart.ChartLocation) { patchRequestDto := &bean7.ArgoCdAppPatchReqDto{ ArgoAppName: argoAppName, - ChartLocation: deploymentConfig.ChartPath, + ChartLocation: envOverride.Chart.ChartLocation, GitRepoUrl: deploymentConfig.RepoURL, TargetRevision: bean7.TargetRevisionMaster, PatchType: bean7.PatchTypeMerge, diff --git a/pkg/pipeline/AppDeploymentTypeChangeManager.go b/pkg/pipeline/AppDeploymentTypeChangeManager.go index 6d70ea011d..c29c7572ba 100644 --- a/pkg/pipeline/AppDeploymentTypeChangeManager.go +++ b/pkg/pipeline/AppDeploymentTypeChangeManager.go @@ -524,7 +524,6 @@ func (impl *AppDeploymentTypeChangeManagerImpl) DeleteDeploymentApps(ctx context } envDeploymentConfig.ConfigType = common.GetDeploymentConfigType(chart.IsCustomGitRepository) envDeploymentConfig.RepoURL = chartGitAttr.RepoUrl - envDeploymentConfig.ChartPath = chartGitAttr.ChartLocation envDeploymentConfig, RepoURLUpdateErr = impl.deploymentConfigService.CreateOrUpdateConfig(nil, envDeploymentConfig, userId) if RepoURLUpdateErr != nil { diff --git a/pkg/pipeline/DeploymentPipelineConfigService.go b/pkg/pipeline/DeploymentPipelineConfigService.go index b718908c58..463b9647c6 100644 --- a/pkg/pipeline/DeploymentPipelineConfigService.go +++ b/pkg/pipeline/DeploymentPipelineConfigService.go @@ -457,7 +457,6 @@ func (impl *CdPipelineConfigServiceImpl) CreateCdPipelines(pipelineCreateRequest DeploymentAppType: pipeline.DeploymentAppType, RepoURL: AppDeploymentConfig.RepoURL, RepoName: AppDeploymentConfig.RepoName, - ChartPath: AppDeploymentConfig.ChartPath, Active: true, } envDeploymentConfig, err := impl.deploymentConfigService.CreateOrUpdateConfig(nil, envDeploymentConfig, pipelineCreateRequest.UserId) diff --git a/wire_gen.go b/wire_gen.go index 9e7d9d55b8..a17e5b5da4 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -633,7 +633,7 @@ func InitializeApp() (*App, error) { cveStoreRepositoryImpl := security.NewCveStoreRepositoryImpl(db, sugaredLogger) policyServiceImpl := security2.NewPolicyServiceImpl(environmentServiceImpl, sugaredLogger, appRepositoryImpl, pipelineOverrideRepositoryImpl, cvePolicyRepositoryImpl, clusterServiceImplExtended, pipelineRepositoryImpl, imageScanResultRepositoryImpl, imageScanDeployInfoRepositoryImpl, imageScanObjectMetaRepositoryImpl, httpClient, ciArtifactRepositoryImpl, ciCdConfig, imageScanHistoryRepositoryImpl, cveStoreRepositoryImpl, ciTemplateRepositoryImpl) pipelineConfigRestHandlerImpl := configure.NewPipelineRestHandlerImpl(pipelineBuilderImpl, sugaredLogger, deploymentTemplateValidationServiceImpl, chartServiceImpl, devtronAppGitOpConfigServiceImpl, propertiesConfigServiceImpl, userServiceImpl, teamServiceImpl, enforcerImpl, ciHandlerImpl, validate, clientImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, enforcerUtilImpl, dockerRegistryConfigImpl, cdHandlerImpl, appCloneServiceImpl, generateManifestDeploymentTemplateServiceImpl, appWorkflowServiceImpl, materialRepositoryImpl, policyServiceImpl, imageScanResultRepositoryImpl, gitProviderRepositoryImpl, argoUserServiceImpl, ciPipelineMaterialRepositoryImpl, imageTaggingServiceImpl, ciArtifactRepositoryImpl, deployedAppMetricsServiceImpl, chartRefServiceImpl, ciCdPipelineOrchestratorImpl) - gitOpsManifestPushServiceImpl := publish.NewGitOpsManifestPushServiceImpl(sugaredLogger, pipelineStatusTimelineServiceImpl, pipelineOverrideRepositoryImpl, acdConfig, chartRefServiceImpl, gitOpsConfigReadServiceImpl, chartServiceImpl, gitOperationServiceImpl, argoClientWrapperServiceImpl, transactionUtilImpl) + gitOpsManifestPushServiceImpl := publish.NewGitOpsManifestPushServiceImpl(sugaredLogger, pipelineStatusTimelineServiceImpl, pipelineOverrideRepositoryImpl, acdConfig, chartRefServiceImpl, gitOpsConfigReadServiceImpl, chartServiceImpl, gitOperationServiceImpl, argoClientWrapperServiceImpl, transactionUtilImpl, deploymentConfigServiceImpl) argoK8sClientImpl := argocdServer.NewArgoK8sClientImpl(sugaredLogger, k8sServiceImpl) manifestCreationServiceImpl := manifest.NewManifestCreationServiceImpl(sugaredLogger, dockerRegistryIpsConfigServiceImpl, chartRefServiceImpl, scopedVariableCMCSManagerImpl, k8sCommonServiceImpl, deployedAppMetricsServiceImpl, imageDigestPolicyServiceImpl, mergeUtil, appCrudOperationServiceImpl, deploymentTemplateServiceImpl, applicationServiceClientImpl, configMapHistoryRepositoryImpl, configMapRepositoryImpl, chartRepositoryImpl, envConfigOverrideRepositoryImpl, environmentRepositoryImpl, pipelineRepositoryImpl, ciArtifactRepositoryImpl, pipelineOverrideRepositoryImpl, pipelineStrategyHistoryRepositoryImpl, pipelineConfigRepositoryImpl, deploymentTemplateHistoryRepositoryImpl, deploymentConfigServiceImpl) deployedConfigurationHistoryServiceImpl := history.NewDeployedConfigurationHistoryServiceImpl(sugaredLogger, userServiceImpl, deploymentTemplateHistoryServiceImpl, pipelineStrategyHistoryServiceImpl, configMapHistoryServiceImpl, cdWorkflowRepositoryImpl, scopedVariableCMCSManagerImpl) From 8a2614a98250e9300dceed76f908934d3e183f00 Mon Sep 17 00:00:00 2001 From: ayushmaheshwari Date: Thu, 11 Jul 2024 16:10:07 +0530 Subject: [PATCH 39/45] updating migration --- scripts/sql/266_migration_deployed_app.up.sql | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/sql/266_migration_deployed_app.up.sql b/scripts/sql/266_migration_deployed_app.up.sql index 829999f9d8..5346f709ed 100644 --- a/scripts/sql/266_migration_deployed_app.up.sql +++ b/scripts/sql/266_migration_deployed_app.up.sql @@ -29,7 +29,6 @@ CREATE TABLE IF NOT EXISTS "public"."deployment_config" "config_type" VARCHAR(100), "repo_url" VARCHAR(250), "repo_name" VARCHAR(200), - "chart_path" VARCHAR(250), "credential_type" VARCHAR(100), "credential_id_int" int, "credential_id_string" VARCHAR(100), From db939dc4d4a699a208990d5cd5ce4415d4d63c4d Mon Sep 17 00:00:00 2001 From: ayushmaheshwari Date: Thu, 11 Jul 2024 16:28:51 +0530 Subject: [PATCH 40/45] conditions in patch object --- client/argocdServer/adapter/adapter.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/client/argocdServer/adapter/adapter.go b/client/argocdServer/adapter/adapter.go index 54568ff977..a52dbdd116 100644 --- a/client/argocdServer/adapter/adapter.go +++ b/client/argocdServer/adapter/adapter.go @@ -22,13 +22,18 @@ import ( ) func GetArgoCdPatchReqFromDto(dto *bean.ArgoCdAppPatchReqDto) v1alpha1.Application { - return v1alpha1.Application{ + app := v1alpha1.Application{ Spec: v1alpha1.ApplicationSpec{ Source: &v1alpha1.ApplicationSource{ - Path: dto.ChartLocation, - RepoURL: dto.GitRepoUrl, TargetRevision: dto.TargetRevision, }, }, } + if len(dto.ChartLocation) > 0 { + app.Spec.Source.Path = dto.ChartLocation + } + if len(dto.GitRepoUrl) > 0 { + app.Spec.Source.RepoURL = dto.GitRepoUrl + } + return app } From efbaa75de6e0a1dfe9d64af4724ca467eb8fd7a8 Mon Sep 17 00:00:00 2001 From: ayushmaheshwari Date: Fri, 12 Jul 2024 16:13:33 +0530 Subject: [PATCH 41/45] fix error in flag off-> custom-gitops-> app-clone-> deploy-> flag-on-> migrate --- .../common/deploymentConfigService.go | 15 +++++++++++++++ .../trigger/devtronApps/TriggerService.go | 18 ------------------ 2 files changed, 15 insertions(+), 18 deletions(-) diff --git a/pkg/deployment/common/deploymentConfigService.go b/pkg/deployment/common/deploymentConfigService.go index 21ffafbf05..740ecc8ea2 100644 --- a/pkg/deployment/common/deploymentConfigService.go +++ b/pkg/deployment/common/deploymentConfigService.go @@ -1,6 +1,7 @@ package common import ( + "github.com/devtron-labs/devtron/api/bean/gitOps" appRepository "github.com/devtron-labs/devtron/internal/sql/repository/app" "github.com/devtron-labs/devtron/internal/sql/repository/deploymentConfig" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" @@ -122,7 +123,11 @@ func (impl *DeploymentConfigServiceImpl) GetConfigForDevtronApps(appId, envId in impl.logger.Errorw("error in migrating app level config to deployment config", "appId", appId, "err", err) return nil, err } + } else if gitOps.IsGitOpsRepoNotConfigured(appAndEnvLevelConfig.RepoUrl) && gitOps.IsGitOpsRepoConfigured(appLevelConfigDbObj.RepoUrl) { + // if url is present at app level and not at env level then copy app level url to env level config + appAndEnvLevelConfig.RepoUrl = appLevelConfigDbObj.RepoUrl } + return ConvertDeploymentConfigDbObjToDTO(appAndEnvLevelConfig), nil } return ConvertDeploymentConfigDbObjToDTO(appLevelConfigDbObj), nil @@ -157,6 +162,16 @@ func (impl *DeploymentConfigServiceImpl) GetAndMigrateConfigIfAbsentForDevtronAp impl.logger.Errorw("error in migrating app level config to deployment config", "appId", appId, "err", err) return nil, err } + } else if gitOps.IsGitOpsRepoNotConfigured(appAndEnvLevelConfig.RepoUrl) && gitOps.IsGitOpsRepoConfigured(appLevelConfigDbObj.RepoUrl) { + // if url is present at app level and not at env level then copy app level url to env level config + // will happen when custom gitOps is enabled and app is cloned. In this case when user configure app level gitOps , env level gitOps will not be updated + appAndEnvLevelConfig.RepoUrl = appLevelConfigDbObj.RepoUrl + appAndEnvLevelConfig.AuditLog.UpdateAuditLog(1) + appAndEnvLevelConfig, err = impl.deploymentConfigRepository.Update(nil, appAndEnvLevelConfig) + if err != nil { + impl.logger.Errorw("error in updating deploymentConfig", "appId", appAndEnvLevelConfig.AppId, "envId", appAndEnvLevelConfig.EnvironmentId, "err", err) + return nil, err + } } envLevelConfig = ConvertDeploymentConfigDbObjToDTO(appAndEnvLevelConfig) } diff --git a/pkg/deployment/trigger/devtronApps/TriggerService.go b/pkg/deployment/trigger/devtronApps/TriggerService.go index 797c5ac380..91ca8c24c6 100644 --- a/pkg/deployment/trigger/devtronApps/TriggerService.go +++ b/pkg/deployment/trigger/devtronApps/TriggerService.go @@ -794,24 +794,6 @@ func (impl *TriggerServiceImpl) TriggerRelease(overrideRequest *bean3.ValuesOver } valuesOverrideResponse.DeploymentConfig = envDeploymentConfig - if impl.deploymentServiceTypeConfig.UseDeploymentConfigData && gitOps.IsGitOpsRepoNotConfigured(valuesOverrideResponse.DeploymentConfig.RepoURL) { - appLevelConfig, err := impl.deploymentConfigService.GetAppLevelConfigForDevtronApp(overrideRequest.AppId) - if err != nil { - impl.logger.Errorw("error in fetching app level config for devtron apps", "appId", overrideRequest.AppId, "err", err) - return releaseNo, err - } - // if url is present at app level and not at env level then copy app level url to env level config - if gitOps.IsGitOpsRepoConfigured(valuesOverrideResponse.DeploymentConfig.RepoURL) { - valuesOverrideResponse.DeploymentConfig.RepoURL = appLevelConfig.RepoURL - valuesOverrideResponse.DeploymentConfig, err = impl.deploymentConfigService.CreateOrUpdateConfig(nil, valuesOverrideResponse.DeploymentConfig, overrideRequest.UserId) - if err != nil { - impl.logger.Errorw("error in copying app level url to env level config", "appId", overrideRequest.AppId, "envId", overrideRequest.EnvId, "err", err) - return 0, err - } - - } - } - // auditDeploymentTriggerHistory is performed irrespective of BuildManifestForTrigger error - for auditing purposes historyErr := impl.auditDeploymentTriggerHistory(overrideRequest.WfrId, valuesOverrideResponse, newCtx, triggeredAt, triggeredBy) if historyErr != nil { From 4429da2b6c506755f478fada32acbb17edc331b7 Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Fri, 12 Jul 2024 18:44:26 +0530 Subject: [PATCH 42/45] wire fix --- cmd/external-app/wire_gen.go | 6 +++--- .../installedApp/service/EAMode/EAModeDeploymentService.go | 4 ++++ 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/cmd/external-app/wire_gen.go b/cmd/external-app/wire_gen.go index 03b8aa9963..0977fd8d64 100644 --- a/cmd/external-app/wire_gen.go +++ b/cmd/external-app/wire_gen.go @@ -1,6 +1,6 @@ // Code generated by Wire. DO NOT EDIT. -//go:generate go run github.com/google/wire/cmd/wire +//go:generate go run -mod=mod github.com/google/wire/cmd/wire //go:build !wireinject // +build !wireinject @@ -267,10 +267,10 @@ func InitializeApp() (*App, error) { installedAppVersionHistoryRepositoryImpl := repository4.NewInstalledAppVersionHistoryRepositoryImpl(sugaredLogger, db) repositoryImpl := deploymentConfig.NewRepositoryImpl(db) chartRepositoryImpl := chartRepoRepository.NewChartRepository(db, transactionUtilImpl) + deploymentConfigServiceImpl := common.NewDeploymentConfigServiceImpl(repositoryImpl, sugaredLogger, chartRepositoryImpl, pipelineRepositoryImpl, appRepositoryImpl, installedAppRepositoryImpl, environmentVariables) + installedAppDBServiceImpl := EAMode.NewInstalledAppDBServiceImpl(sugaredLogger, installedAppRepositoryImpl, appRepositoryImpl, userServiceImpl, environmentServiceImpl, installedAppVersionHistoryRepositoryImpl, deploymentConfigServiceImpl) gitOpsConfigRepositoryImpl := repository3.NewGitOpsConfigRepositoryImpl(sugaredLogger, db) gitOpsConfigReadServiceImpl := config.NewGitOpsConfigReadServiceImpl(sugaredLogger, gitOpsConfigRepositoryImpl, userServiceImpl, environmentVariables) - deploymentConfigServiceImpl := common.NewDeploymentConfigServiceImpl(repositoryImpl, sugaredLogger, chartRepositoryImpl, pipelineRepositoryImpl, gitOpsConfigReadServiceImpl, appRepositoryImpl, installedAppRepositoryImpl) - installedAppDBServiceImpl := EAMode.NewInstalledAppDBServiceImpl(sugaredLogger, installedAppRepositoryImpl, appRepositoryImpl, userServiceImpl, environmentServiceImpl, installedAppVersionHistoryRepositoryImpl, deploymentConfigServiceImpl) attributesServiceImpl := attributes.NewAttributesServiceImpl(sugaredLogger, attributesRepositoryImpl) deploymentTypeOverrideServiceImpl := providerConfig.NewDeploymentTypeOverrideServiceImpl(sugaredLogger, environmentVariables, attributesServiceImpl) eaModeDeploymentServiceImpl := EAMode.NewEAModeDeploymentServiceImpl(sugaredLogger, helmAppServiceImpl, appStoreApplicationVersionRepositoryImpl, helmAppClientImpl, installedAppRepositoryImpl, ociRegistryConfigRepositoryImpl) diff --git a/pkg/appStore/installedApp/service/EAMode/EAModeDeploymentService.go b/pkg/appStore/installedApp/service/EAMode/EAModeDeploymentService.go index 4f06ae9ebc..35025a5a41 100644 --- a/pkg/appStore/installedApp/service/EAMode/EAModeDeploymentService.go +++ b/pkg/appStore/installedApp/service/EAMode/EAModeDeploymentService.go @@ -441,3 +441,7 @@ func (impl *EAModeDeploymentServiceImpl) GetChartBytesForParticularDeployment(in func (impl *EAModeDeploymentServiceImpl) DeleteACD(acdAppName string, ctx context.Context, isNonCascade bool) error { return errors.New("this is not implemented") } + +func (impl *EAModeDeploymentServiceImpl) GetAcdAppGitOpsRepoURL(appName string, environmentName string) (string, error) { + return "", errors.New("this is not implemented") +} From 18ebdc06b86914dc41cd9bd51b9220a252170db9 Mon Sep 17 00:00:00 2001 From: ayushmaheshwari Date: Mon, 15 Jul 2024 13:36:09 +0530 Subject: [PATCH 43/45] audit log fix --- pkg/deployment/common/deploymentConfigService.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/deployment/common/deploymentConfigService.go b/pkg/deployment/common/deploymentConfigService.go index 740ecc8ea2..0d4771180d 100644 --- a/pkg/deployment/common/deploymentConfigService.go +++ b/pkg/deployment/common/deploymentConfigService.go @@ -74,6 +74,7 @@ func (impl *DeploymentConfigServiceImpl) CreateOrUpdateConfig(tx *pg.Tx, config } } else { newDBObj.Id = configDbObj.Id + newDBObj.AuditLog = configDbObj.AuditLog newDBObj.AuditLog.UpdateAuditLog(userId) newDBObj, err = impl.deploymentConfigRepository.Update(tx, newDBObj) if err != nil { From cd123778bc0805fd36bcb6e969db47f0b3b07e01 Mon Sep 17 00:00:00 2001 From: ayushmaheshwari Date: Mon, 15 Jul 2024 14:32:11 +0530 Subject: [PATCH 44/45] migation update --- pkg/deployment/common/deploymentConfigService.go | 3 +++ scripts/sql/266_migration_deployed_app.up.sql | 3 --- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/deployment/common/deploymentConfigService.go b/pkg/deployment/common/deploymentConfigService.go index 0d4771180d..7f2c35b8e6 100644 --- a/pkg/deployment/common/deploymentConfigService.go +++ b/pkg/deployment/common/deploymentConfigService.go @@ -368,6 +368,9 @@ func (impl *DeploymentConfigServiceImpl) parseConfigForHelmApps(appId int, envId impl.logger.Errorw("error in getting installed app by appId", "appId", appId, "err", err) return nil, err } + if installedApp.EnvironmentId != envId { + return nil, pg.ErrNoRows + } helmDeploymentConfig := &deploymentConfig.DeploymentConfig{ AppId: appId, EnvironmentId: envId, diff --git a/scripts/sql/266_migration_deployed_app.up.sql b/scripts/sql/266_migration_deployed_app.up.sql index 5346f709ed..b152e295c8 100644 --- a/scripts/sql/266_migration_deployed_app.up.sql +++ b/scripts/sql/266_migration_deployed_app.up.sql @@ -29,9 +29,6 @@ CREATE TABLE IF NOT EXISTS "public"."deployment_config" "config_type" VARCHAR(100), "repo_url" VARCHAR(250), "repo_name" VARCHAR(200), - "credential_type" VARCHAR(100), - "credential_id_int" int, - "credential_id_string" VARCHAR(100), "active" bool, "created_on" timestamptz, "created_by" integer, From ec17f0930719fd14cdb1be78e44f9c565373df0a Mon Sep 17 00:00:00 2001 From: kartik-579 Date: Mon, 15 Jul 2024 15:23:10 +0530 Subject: [PATCH 45/45] reverted 107 sql newline add --- scripts/sql/107_create_k8s_resource_history.down.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/sql/107_create_k8s_resource_history.down.sql b/scripts/sql/107_create_k8s_resource_history.down.sql index baf9dd27e7..ab0e40549d 100644 --- a/scripts/sql/107_create_k8s_resource_history.down.sql +++ b/scripts/sql/107_create_k8s_resource_history.down.sql @@ -1,2 +1,2 @@ DROP SEQUENCE IF EXISTS "id_seq_k8s_resource_history_sequence"; -DROP TABLE IF EXISTS "kubernetes_resource_history"; +DROP TABLE IF EXISTS "kubernetes_resource_history"; \ No newline at end of file