fix: query optimisations for app group cd listing and ci pipeline blockage state

api/restHandler/app/workflow/AppWorkflowRestHandler.go

@@ -280,9 +280,9 @@ func (impl AppWorkflowRestHandlerImpl) FindAppWorkflow(w http.ResponseWriter, r
 	} else {
 		workflows[bean3.Workflows] = []appWorkflow.AppWorkflowDto{}
 	}
-	isAppLevelGitOpsConfigured, err := impl.chartService.IsGitOpsRepoConfiguredForDevtronApps(appId)
+	isAppLevelGitOpsConfigured, err := impl.chartService.IsGitOpsRepoConfiguredForDevtronApp(appId)
 	if err != nil && !util.IsErrNoRows(err) {
-		impl.Logger.Errorw("service err, IsGitOpsRepoConfiguredForDevtronApps", "appId", appId, "envIds", envIds, "err", err)
+		impl.Logger.Errorw("service err, IsGitOpsRepoConfiguredForDevtronApp", "appId", appId, "envIds", envIds, "err", err)
 		common.WriteJsonResp(w, err, nil, http.StatusInternalServerError)
 		return
 	}

internal/sql/repository/deploymentConfig/repository.go

@@ -134,17 +134,15 @@ func (impl *RepositoryImpl) GetAppLevelConfigByAppIds(appIds []int) ([]*Deployme
 func (impl *RepositoryImpl) GetAppAndEnvLevelConfigsInBulk(appIdToEnvIdsMap map[int][]int) ([]*DeploymentConfig, error) {
 	var result []*DeploymentConfig
 	err := impl.dbConnection.Model(&result).
-		WhereGroup(func(query *orm.Query) (*orm.Query, error) {
+		WhereOrGroup(func(query *orm.Query) (*orm.Query, error) {
 			for appId, envIds := range appIdToEnvIdsMap {
 				if len(envIds) == 0 {
 					continue
 				}
-				query = query.WhereOr(" app_id = ? and environment_id in (?) and active=true ", appId, pg.In(envIds))
+				query = query.Where("app_id = ?", appId).Where("environment_id in (?)", pg.In((envIds))).Where("active = ?", true)
 			}
 			return query, nil
-		}).
-		Where("active = ?", true).
-		Select()
+		}).Select()
 	return result, err
 }
 

internal/sql/repository/pipelineConfig/PipelineRepository.go

@@ -100,9 +100,11 @@ type PipelineRepository interface {
 	FindAllPipelineCreatedCountInLast24Hour() (pipelineCount int, err error)
 	FindAllDeletedPipelineCountInLast24Hour() (pipelineCount int, err error)
 	FindActiveByEnvId(envId int) (pipelines []*Pipeline, err error)
+	FindActivePipelineAppIdsByEnvId(envId int) ([]int, error)
 	FindActivePipelineByEnvId(envId int) (pipelines []*Pipeline, err error)
 	FindActiveByEnvIds(envId []int) (pipelines []*Pipeline, err error)
 	FindActiveByInFilter(envId int, appIdIncludes []int) (pipelines []*Pipeline, err error)
+	FindActivePipelineAppIdsByInFilter(envId int, appIdIncludes []int) ([]int, error)
 	FindActiveByNotFilter(envId int, appIdExcludes []int) (pipelines []*Pipeline, err error)
 	FindAllPipelinesByChartsOverrideAndAppIdAndChartId(chartOverridden bool, appId int, chartId int) (pipelines []*Pipeline, err error)
 	FindActiveByAppIdAndPipelineId(appId int, pipelineId int) ([]*Pipeline, error)
@@ -128,6 +130,7 @@ type PipelineRepository interface {
 	FindWithEnvironmentByCiIds(ctx context.Context, cIPipelineIds []int) ([]*Pipeline, error)
 	FindDeploymentAppTypeByAppIdAndEnvId(appId, envId int) (string, error)
 	FindByAppIdToEnvIdsMapping(appIdToEnvIds map[int][]int) ([]*Pipeline, error)
+	FindDeploymentAppTypeByIds(ids []int) (pipelines []*Pipeline, err error)
 }
 
 type CiArtifactDTO struct {
@@ -483,6 +486,15 @@ func (impl PipelineRepositoryImpl) FindActiveByEnvId(envId int) (pipelines []*Pi
 	return pipelines, err
 }
 
+func (impl PipelineRepositoryImpl) FindActivePipelineAppIdsByEnvId(envId int) ([]int, error) {
+	var appIds []int
+	err := impl.dbConnection.Model((*Pipeline)(nil)).Column("app_id").
+		Where("environment_id = ?", envId).
+		Where("deleted = ?", false).
+		Select(&appIds)
+	return appIds, err
+}
+
 func (impl PipelineRepositoryImpl) FindActivePipelineByEnvId(envId int) (pipelines []*Pipeline, err error) {
 	err = impl.dbConnection.Model(&pipelines).Column("pipeline.*", "App", "Environment").
 		Where("environment_id = ?", envId).
@@ -509,6 +521,15 @@ func (impl PipelineRepositoryImpl) FindActiveByInFilter(envId int, appIdIncludes
 	return pipelines, err
 }
 
+func (impl PipelineRepositoryImpl) FindActivePipelineAppIdsByInFilter(envId int, appIdIncludes []int) ([]int, error) {
+	var appIds []int
+	err := impl.dbConnection.Model((*Pipeline)(nil)).Column("app_id").
+		Where("environment_id = ?", envId).
+		Where("app_id in (?)", pg.In(appIdIncludes)).
+		Where("deleted = ?", false).Select(&appIds)
+	return appIds, err
+}
+
 func (impl PipelineRepositoryImpl) FindActiveByNotFilter(envId int, appIdExcludes []int) (pipelines []*Pipeline, err error) {
 	err = impl.dbConnection.Model(&pipelines).Column("pipeline.*", "App", "Environment").
 		Where("environment_id = ?", envId).
@@ -812,3 +833,9 @@ func (impl PipelineRepositoryImpl) FindByAppIdToEnvIdsMapping(appIdToEnvIds map[
 		Select()
 	return pipelines, err
 }
+
+func (impl PipelineRepositoryImpl) FindDeploymentAppTypeByIds(ids []int) (pipelines []*Pipeline, err error) {
+	err = impl.dbConnection.Model(&pipelines).Column("id", "app_id", "env_id", "deployment_app_type").
+		Where("id in (?)", pg.In(ids)).Where("deleted = ?", false).Select()
+	return pipelines, err
+}

pkg/appWorkflow/AppWorkflowService.go

@@ -851,7 +851,7 @@ func (impl AppWorkflowServiceImpl) FindCdPipelinesByAppId(appId int) (*bean.CdPi
 		AppId: appId,
 	}
 
-	isAppLevelGitOpsConfigured, err := impl.chartService.IsGitOpsRepoConfiguredForDevtronApps(appId)
+	isAppLevelGitOpsConfigured, err := impl.chartService.IsGitOpsRepoConfiguredForDevtronApp(appId)
 	if err != nil {
 		impl.Logger.Errorw("error in fetching latest chart details for app by appId")
 		return nil, err

pkg/chart/ChartService.go

@@ -67,7 +67,8 @@ type ChartService interface {
 
 	ConfigureGitOpsRepoUrlForApp(appId int, repoUrl, chartLocation string, isCustomRepo bool, userId int32) (*bean2.DeploymentConfig, error)
 
-	IsGitOpsRepoConfiguredForDevtronApps(appId int) (bool, error)
+	IsGitOpsRepoConfiguredForDevtronApp(appId int) (bool, error)
+	IsGitOpsRepoConfiguredForDevtronApps(appIds []int) (map[int]bool, error)
 	IsGitOpsRepoAlreadyRegistered(gitOpsRepoUrl string) (bool, error)
 }
 
@@ -542,7 +543,7 @@ func (impl *ChartServiceImpl) getNewVersion(chartRepo, chartName, refChartLocati
 	return placeholders[0] + "." + placeholders[1] + "." + strconv.FormatInt(count, 10), nil
 }
 
-func (impl *ChartServiceImpl) IsGitOpsRepoConfiguredForDevtronApps(appId int) (bool, error) {
+func (impl *ChartServiceImpl) IsGitOpsRepoConfiguredForDevtronApp(appId int) (bool, error) {
 	gitOpsConfigStatus, err := impl.gitOpsConfigReadService.IsGitOpsConfigured()
 	if err != nil {
 		impl.logger.Errorw("error in fetching latest chart for app by appId")
@@ -560,6 +561,30 @@ func (impl *ChartServiceImpl) IsGitOpsRepoConfiguredForDevtronApps(appId int) (b
 	return !apiGitOpsBean.IsGitOpsRepoNotConfigured(latestChartConfiguredInApp.GitRepoUrl), nil
 }
 
+func (impl *ChartServiceImpl) IsGitOpsRepoConfiguredForDevtronApps(appIds []int) (map[int]bool, error) {
+	gitOpsConfigStatus, err := impl.gitOpsConfigReadService.IsGitOpsConfigured()
+	if err != nil {
+		impl.logger.Errorw("error in fetching latest chart for app by appId")
+		return nil, err
+	}
+	appIdRepoConfiguredMap := make(map[int]bool, len(appIds))
+	for _, appId := range appIds {
+		if !gitOpsConfigStatus.IsGitOpsConfigured {
+			appIdRepoConfiguredMap[appId] = false
+		} else if !gitOpsConfigStatus.AllowCustomRepository {
+			appIdRepoConfiguredMap[appId] = true
+		} else {
+			latestChartConfiguredInApp, err := impl.FindLatestChartForAppByAppId(appId)
+			if err != nil {
+				impl.logger.Errorw("error in fetching latest chart for app by appId")
+				return nil, err
+			}
+			appIdRepoConfiguredMap[appId] = !apiGitOpsBean.IsGitOpsRepoNotConfigured(latestChartConfiguredInApp.GitRepoUrl)
+		}
+	}
+	return appIdRepoConfiguredMap, nil
+}
+
 func (impl *ChartServiceImpl) FindLatestChartForAppByAppId(appId int) (chartTemplate *TemplateRequest, err error) {
 	chart, err := impl.chartRepository.FindLatestChartForAppByAppId(appId)
 	if err != nil {

pkg/deployment/common/bean/bean.go

@@ -22,6 +22,7 @@ type UniqueDeploymentConfigIdentifier string
 type DeploymentConfigSelector struct {
 	AppId         int
 	EnvironmentId int
+	CDPipelineId  int
 }
 
 func (u UniqueDeploymentConfigIdentifier) String() string {

pkg/deployment/common/deploymentConfigService.go

@@ -1,6 +1,7 @@
 package common
 
 import (
+	"fmt"
 	"github.com/devtron-labs/devtron/api/bean/gitOps"
 	appRepository "github.com/devtron-labs/devtron/internal/sql/repository/app"
 	"github.com/devtron-labs/devtron/internal/sql/repository/deploymentConfig"
@@ -24,6 +25,7 @@ type DeploymentConfigService interface {
 	GetAndMigrateConfigIfAbsentForHelmApp(appId, envId int) (*bean.DeploymentConfig, error)
 	GetAppLevelConfigForDevtronApp(appId int) (*bean.DeploymentConfig, error)
 	UpdateRepoUrlForAppAndEnvId(repoURL string, appId, envId int) error
+	GetDeploymentAppTypeForCDInBulk(pipelines []*pipelineConfig.Pipeline) (map[int]string, error)
 }
 
 type DeploymentConfigServiceImpl struct {
@@ -430,3 +432,30 @@ func (impl *DeploymentConfigServiceImpl) UpdateRepoUrlForAppAndEnvId(repoURL str
 	}
 	return nil
 }
+
+func (impl *DeploymentConfigServiceImpl) GetDeploymentAppTypeForCDInBulk(pipelines []*pipelineConfig.Pipeline) (map[int]string, error) {
+	resp := make(map[int]string, len(pipelines)) //map of pipelineId and deploymentAppType
+	if impl.deploymentServiceTypeConfig.UseDeploymentConfigData {
+		appIdEnvIdMapping := make(map[int][]int, len(pipelines))
+		appIdEnvIdKeyPipelineIdMap := make(map[string]int, len(pipelines))
+		for _, pipeline := range pipelines {
+			appIdEnvIdMapping[pipeline.AppId] = append(appIdEnvIdMapping[pipeline.AppId], pipeline.EnvironmentId)
+			appIdEnvIdKeyPipelineIdMap[fmt.Sprintf("%d-%d", pipeline.AppId, pipeline.EnvironmentId)] = pipeline.Id
+		}
+		configs, err := impl.deploymentConfigRepository.GetAppAndEnvLevelConfigsInBulk(appIdEnvIdMapping)
+		if err != nil {
+			impl.logger.Errorw("error, GetAppAndEnvLevelConfigsInBulk", "appIdEnvIdMapping", appIdEnvIdMapping, "err", err)
+			return nil, err
+		}
+		for _, config := range configs {
+			pipelineId := appIdEnvIdKeyPipelineIdMap[fmt.Sprintf("%d-%d", config.AppId, config.EnvironmentId)]
+			resp[pipelineId] = config.DeploymentAppType
+		}
+	}
+	for _, pipeline := range pipelines {
+		if _, ok := resp[pipeline.Id]; !ok { //not found in map, either flag is disabled or config not migrated yet. Getting from old data
+			resp[pipeline.Id] = pipeline.DeploymentAppType
+		}
+	}
+	return resp, nil
+}

pkg/pipeline/AppDeploymentTypeChangeManager.go

@@ -418,14 +418,6 @@ func (impl *AppDeploymentTypeChangeManagerImpl) TriggerDeploymentAfterTypeChange
 			"err", err)
 	}
 
-	deploymentConfigSelector := make([]*bean4.DeploymentConfigSelector, len(pipelineIds))
-	for _, pipeline := range pipelines {
-		deploymentConfigSelector = append(deploymentConfigSelector, &bean4.DeploymentConfigSelector{
-			AppId:         pipeline.AppId,
-			EnvironmentId: pipeline.EnvironmentId,
-		})
-	}
-
 	for _, p := range pipelines {
 		envDeploymentConfig, err := impl.deploymentConfigService.GetAndMigrateConfigIfAbsentForDevtronApps(p.AppId, p.EnvironmentId)
 		if err != nil {

pkg/pipeline/BuildPipelineConfigService.go

@@ -33,7 +33,9 @@ import (
 	pipelineConfigBean "github.com/devtron-labs/devtron/pkg/pipeline/bean"
 	"github.com/devtron-labs/devtron/pkg/pipeline/bean/CiPipeline"
 	"github.com/devtron-labs/devtron/pkg/pipeline/history"
+	"github.com/devtron-labs/devtron/pkg/pipeline/repository"
 	"github.com/devtron-labs/devtron/pkg/pipeline/types"
+	repository2 "github.com/devtron-labs/devtron/pkg/plugin/repository"
 	resourceGroup2 "github.com/devtron-labs/devtron/pkg/resourceGroup"
 	"github.com/devtron-labs/devtron/pkg/sql"
 	"github.com/devtron-labs/devtron/util/rbac"
@@ -133,6 +135,8 @@ type CiPipelineConfigServiceImpl struct {
 	customTagService              CustomTagService
 	cdWorkflowRepository          pipelineConfig.CdWorkflowRepository
 	buildPipelineSwitchService    BuildPipelineSwitchService
+	pipelineStageRepository       repository.PipelineStageRepository
+	globalPluginRepository        repository2.GlobalPluginRepository
 }
 
 func NewCiPipelineConfigServiceImpl(logger *zap.SugaredLogger,
@@ -157,8 +161,8 @@ func NewCiPipelineConfigServiceImpl(logger *zap.SugaredLogger,
 	customTagService CustomTagService,
 	cdWorkflowRepository pipelineConfig.CdWorkflowRepository,
 	buildPipelineSwitchService BuildPipelineSwitchService,
-) *CiPipelineConfigServiceImpl {
-
+	pipelineStageRepository repository.PipelineStageRepository,
+	globalPluginRepository repository2.GlobalPluginRepository) *CiPipelineConfigServiceImpl {
 	securityConfig := &SecurityConfig{}
 	err := env.Parse(securityConfig)
 	if err != nil {
@@ -188,6 +192,8 @@ func NewCiPipelineConfigServiceImpl(logger *zap.SugaredLogger,
 		customTagService:              customTagService,
 		cdWorkflowRepository:          cdWorkflowRepository,
 		buildPipelineSwitchService:    buildPipelineSwitchService,
+		pipelineStageRepository:       pipelineStageRepository,
+		globalPluginRepository:        globalPluginRepository,
 	}
 }