fix: ci-cd count per day in telemetry data

client/telemetry/TelemetryEventClientExtended.go

@@ -131,8 +131,12 @@ type TelemetryEventDto struct {
 	UserCount                            int                `json:"userCount,omitempty"`
 	EnvironmentCount                     int                `json:"environmentCount,omitempty"`
 	ClusterCount                         int                `json:"clusterCount,omitempty"`
-	CiCountPerDay                        int                `json:"ciCountPerDay,omitempty"`
-	CdCountPerDay                        int                `json:"cdCountPerDay,omitempty"`
+	CiCreatedPerDay                      int                `json:"ciCreatedPerDay"`
+	CdCreatedPerDay                      int                `json:"cdCreatedPerDay"`
+	CiDeletedPerDay                      int                `json:"ciDeletedPerDay"`
+	CdDeletedPerDay                      int                `json:"cdDeletedPerDay"`
+	CiTriggeredPerDay                    int                `json:"ciTriggeredPerDay"`
+	CdTriggeredPerDay                    int                `json:"cdTriggeredPerDay"`
 	HelmChartCount                       int                `json:"helmChartCount,omitempty"`
 	SecurityScanCountPerDay              int                `json:"securityScanCountPerDay,omitempty"`
 	GitAccountsCount                     int                `json:"gitAccountsCount,omitempty"`
@@ -200,52 +204,71 @@ func (impl *TelemetryEventClientImplExtended) SendSummaryEvent(eventType string)
 	payload := &TelemetryEventDto{UCID: ucid, Timestamp: time.Now(), EventType: TelemetryEventType(eventType), DevtronVersion: "v1"}
 	payload.ServerVersion = k8sServerVersion.String()
 
-	environments, err := impl.environmentService.GetAllActive()
+	environmentCount, err := impl.environmentService.GetAllActiveEnvironmentCount()
 	if err != nil && err != pg.ErrNoRows {
-		impl.logger.Errorw("exception caught inside telemetry summary event", "err", err)
-		return err
+		impl.logger.Errorw("exception caught inside telemetry summary event, setting value to -1", "err", err)
+		environmentCount = -1
 	}
 
 	prodApps, err := impl.appListingRepository.FindAppCount(true)
 	if err != nil && err != pg.ErrNoRows {
-		impl.logger.Errorw("exception caught inside telemetry summary event", "err", err)
-		return err
+		impl.logger.Errorw("exception caught inside telemetry summary event, setting value to -1", "err", err)
+		prodApps = -1
 	}
 
 	nonProdApps, err := impl.appListingRepository.FindAppCount(false)
 	if err != nil && err != pg.ErrNoRows {
-		impl.logger.Errorw("exception caught inside telemetry summary event", "err", err)
-		return err
+		impl.logger.Errorw("exception caught inside telemetry summary event, setting value to -1", "err", err)
+		nonProdApps = -1
 	}
 
-	ciPipeline, err := impl.ciPipelineRepository.FindAllPipelineInLast24Hour()
+	ciPipelineCount, err := impl.ciPipelineRepository.FindAllPipelineCountInLast24Hour()
 	if err != nil && err != pg.ErrNoRows {
-		impl.logger.Errorw("exception caught inside telemetry summary event", "err", err)
-		return err
+		impl.logger.Errorw("exception caught inside telemetry summary event, setting value to -1", "err", err)
+		ciPipelineCount = -1
 	}
-
-	cdPipeline, err := impl.pipelineRepository.FindAllPipelineInLast24Hour()
+	ciPipelineDeletedCount, err := impl.ciPipelineRepository.FindAllDeletedPipelineCountInLast24Hour()
 	if err != nil && err != pg.ErrNoRows {
-		impl.logger.Errorw("exception caught inside telemetry summary event", "err", err)
-		return err
+		impl.logger.Errorw("exception caught inside telemetry summary event, setting value to -1", "err", err)
+		ciPipelineDeletedCount = -1
+	}
+	ciPipelineTriggeredCount, err := impl.ciWorkflowRepository.FindAllTriggeredWorkflowCountInLast24Hour()
+	if err != nil && err != pg.ErrNoRows {
+		impl.logger.Errorw("exception caught inside telemetry summary event, setting value to -1", "err", err)
+		ciPipelineTriggeredCount = -1
 	}
 
-	gitAccounts, err := impl.gitProviderRepository.FindAll()
+	cdPipelineCount, err := impl.pipelineRepository.FindAllPipelineCountInLast24Hour()
 	if err != nil && err != pg.ErrNoRows {
-		impl.logger.Errorw("exception caught inside telemetry summary event", "err", err)
-		return err
+		impl.logger.Errorw("exception caught inside telemetry summary event, setting value to -1", "err", err)
+		cdPipelineCount = -1
+	}
+	cdPipelineDeletedCount, err := impl.pipelineRepository.FindAllDeletedPipelineCountInLast24Hour()
+	if err != nil && err != pg.ErrNoRows {
+		impl.logger.Errorw("exception caught inside telemetry summary event, setting value to -1", "err", err)
+		cdPipelineDeletedCount = -1
+	}
+	cdPipelineTriggeredCount, err := impl.cdWorkflowRepository.FindAllTriggeredWorkflowCountInLast24Hour()
+	if err != nil && err != pg.ErrNoRows {
+		impl.logger.Errorw("exception caught inside telemetry summary event, setting value to -1", "err", err)
+		cdPipelineTriggeredCount = -1
+	}
+	gitAccounts, err := impl.gitProviderRepository.FindAllGitProviderCount()
+	if err != nil && err != pg.ErrNoRows {
+		impl.logger.Errorw("exception caught inside telemetry summary event, setting value to -1", "err", err)
+		gitAccounts = -1
 	}
 
 	gitOpsCount, err := impl.gitOpsConfigReadService.GetConfiguredGitOpsCount()
 	if err != nil {
-		impl.logger.Errorw("exception caught inside telemetry summary event", "err", err)
-		return err
+		impl.logger.Errorw("exception caught inside telemetry summary event,setting value to -1", "err", err)
+		gitOpsCount = -1
 	}
 
-	containerRegistry, err := impl.dockerArtifactStoreRepository.FindAll()
+	containerRegistryCount, err := impl.dockerArtifactStoreRepository.FindAllDockerArtifactCount()
 	if err != nil && err != pg.ErrNoRows {
-		impl.logger.Errorw("exception caught inside telemetry summary event", "err", err)
-		return err
+		impl.logger.Errorw("exception caught inside telemetry summary event,setting value to -1", "err", err)
+		containerRegistryCount = -1
 	}
 
 	//appSetup := false
@@ -302,15 +325,18 @@ func (impl *TelemetryEventClientImplExtended) SendSummaryEvent(eventType string)
 	devtronVersion := util.GetDevtronVersion()
 	payload.ProdAppCount = prodApps
 	payload.NonProdAppCount = nonProdApps
-	payload.RegistryCount = len(containerRegistry)
+	payload.RegistryCount = containerRegistryCount
 	payload.SSOLogin = ssoSetup
 	payload.UserCount = len(users)
-	payload.EnvironmentCount = len(environments)
+	payload.EnvironmentCount = environmentCount
 	payload.ClusterCount = len(clusters)
-	payload.CiCountPerDay = len(ciPipeline)
-
-	payload.CdCountPerDay = len(cdPipeline)
-	payload.GitAccountsCount = len(gitAccounts)
+	payload.CiCreatedPerDay = ciPipelineCount
+	payload.CiDeletedPerDay = ciPipelineDeletedCount
+	payload.CiTriggeredPerDay = ciPipelineTriggeredCount
+	payload.CdCreatedPerDay = cdPipelineCount
+	payload.CdDeletedPerDay = cdPipelineDeletedCount
+	payload.CdTriggeredPerDay = cdPipelineTriggeredCount
+	payload.GitAccountsCount = gitAccounts
 	payload.GitOpsCount = gitOpsCount
 	payload.HostURL = hostURL
 	payload.DevtronGitVersion = devtronVersion.GitCommit

internal/sql/repository/GitOpsConfigRepository.go

@@ -28,6 +28,7 @@ type GitOpsConfigRepository interface {
 	UpdateGitOpsConfig(model *GitOpsConfig, tx *pg.Tx) error
 	GetGitOpsConfigById(id int) (*GitOpsConfig, error)
 	GetAllGitOpsConfig() ([]*GitOpsConfig, error)
+	GetAllGitOpsConfigCount() (int, error)
 	GetGitOpsConfigByProvider(provider string) (*GitOpsConfig, error)
 	GetGitOpsConfigActive() (*GitOpsConfig, error)
 	GetConnection() *pg.DB
@@ -91,6 +92,10 @@ func (impl *GitOpsConfigRepositoryImpl) GetAllGitOpsConfig() ([]*GitOpsConfig, e
 	err := impl.dbConnection.Model(&userModel).Order("updated_on desc").Select()
 	return userModel, err
 }
+func (impl *GitOpsConfigRepositoryImpl) GetAllGitOpsConfigCount() (int, error) {
+	cnt, err := impl.dbConnection.Model(&GitOpsConfig{}).Order("updated_on desc").Count()
+	return cnt, err
+}
 func (impl *GitOpsConfigRepositoryImpl) GetGitOpsConfigByProvider(provider string) (*GitOpsConfig, error) {
 	var model GitOpsConfig
 	err := impl.dbConnection.Model(&model).Where("provider = ?", provider).Select()

internal/sql/repository/GitProviderRepository.go

@@ -52,6 +52,7 @@ type GitProviderRepository interface {
 	ProviderExists(url string) (bool, error)
 	FindAllActiveForAutocomplete() ([]GitProvider, error)
 	FindAll() ([]GitProvider, error)
+	FindAllGitProviderCount() (int, error)
 	FindOne(providerId string) (GitProvider, error)
 	FindByUrl(providerUrl string) (GitProvider, error)
 	Update(gitProvider *GitProvider) error
@@ -95,6 +96,11 @@ func (impl GitProviderRepositoryImpl) FindAll() ([]GitProvider, error) {
 		Where("deleted = ?", false).Select()
 	return providers, err
 }
+func (impl GitProviderRepositoryImpl) FindAllGitProviderCount() (int, error) {
+	gitProviderCount, err := impl.dbConnection.Model(&GitProvider{}).
+		Where("deleted = ?", false).Count()
+	return gitProviderCount, err
+}
 
 func (impl GitProviderRepositoryImpl) FindOne(providerId string) (GitProvider, error) {
 	var provider GitProvider

internal/sql/repository/dockerRegistry/DockerArtifactStoreRepository.go

@@ -92,6 +92,7 @@ type DockerArtifactStoreRepository interface {
 	FindActiveDefaultStore() (*DockerArtifactStore, error)
 	FindAllActiveForAutocomplete() ([]DockerArtifactStore, error)
 	FindAll() ([]DockerArtifactStore, error)
+	FindAllDockerArtifactCount() (int, error)
 	FindAllChartProviders() ([]DockerArtifactStore, error)
 	FindOne(storeId string) (*DockerArtifactStore, error)
 	FindOneWithDeploymentCount(storeId string) (*DockerArtifactStoreExt, error)
@@ -169,7 +170,19 @@ func (impl DockerArtifactStoreRepositoryImpl) FindAll() ([]DockerArtifactStore,
 		Select()
 	return providers, err
 }
-
+func (impl DockerArtifactStoreRepositoryImpl) FindAllDockerArtifactCount() (int, error) {
+	dockerArtifactCount, err := impl.dbConnection.Model(&DockerArtifactStore{}).
+		Column("docker_artifact_store.*", "IpsConfig", "OCIRegistryConfig").
+		Where("docker_artifact_store.active = ?", true).
+		Relation("OCIRegistryConfig", func(q *orm.Query) (query *orm.Query, err error) {
+			return q.Where("deleted IS FALSE"), nil
+		}).
+		Relation("IpsConfig", func(q *orm.Query) (query *orm.Query, err error) {
+			return q.JoinOn("(ips_config.active=true or ips_config is null)"), nil
+		}).
+		Count()
+	return dockerArtifactCount, err
+}
 func (impl DockerArtifactStoreRepositoryImpl) FindAllChartProviders() ([]DockerArtifactStore, error) {
 	var providers []DockerArtifactStore
 	err := impl.dbConnection.Model(&providers).

internal/sql/repository/pipelineConfig/CdWorfkflowRepository.go

@@ -57,7 +57,7 @@ type CdWorkflowRepository interface {
 	FindLatestWfrByAppIdAndEnvironmentId(appId int, environmentId int) (*CdWorkflowRunner, error)
 	IsLatestCDWfr(pipelineId, wfrId int) (bool, error)
 	FindLatestCdWorkflowRunnerByEnvironmentIdAndRunnerType(appId int, environmentId int, runnerType bean.WorkflowType) (CdWorkflowRunner, error)
-
+	FindAllTriggeredWorkflowCountInLast24Hour() (cdWorkflowCount int, err error)
 	GetConnection() *pg.DB
 
 	FindLastPreOrPostTriggeredByPipelineId(pipelineId int) (CdWorkflowRunner, error)
@@ -357,7 +357,19 @@ func (impl *CdWorkflowRepositoryImpl) FindLatestCdWorkflowByPipelineIdV2(pipelin
 	// TODO - Group By Environment And Pipeline will get latest pipeline from top
 	return cdWorkflow, err
 }
-
+func (impl *CdWorkflowRepositoryImpl) FindAllTriggeredWorkflowCountInLast24Hour() (cdWorkflowCount int, err error) {
+	cnt, err := impl.dbConnection.
+		Model(&CdWorkflow{}).
+		ColumnExpr("DISTINCT pipeline_id").
+		Join("JOIN cd_workflow_runner ON cd_workflow.id = cd_workflow_runner.cd_workflow_id").
+		Where("cd_workflow_runner.workflow_type = ? AND cd_workflow_runner.started_on > ?", bean.CD_WORKFLOW_TYPE_DEPLOY, time.Now().AddDate(0, 0, -1)).
+		Group("cd_workflow.pipeline_id").
+		Count()
+	if err != nil {
+		impl.logger.Errorw("error occurred while fetching cd workflow", "err", err)
+	}
+	return cnt, err
+}
 func (impl *CdWorkflowRepositoryImpl) FindCdWorkflowMetaByEnvironmentId(appId int, environmentId int, offset int, limit int) ([]CdWorkflowRunner, error) {
 	var wfrList []CdWorkflowRunner
 	err := impl.dbConnection.

internal/sql/repository/pipelineConfig/CiPipelineRepository.go

@@ -128,7 +128,8 @@ type CiPipelineRepository interface {
 	FetchParentCiPipelinesForDG() ([]*bean.CiPipelinesMap, error)
 	FetchCiPipelinesForDG(parentId int, childCiPipelineIds []int) (*CiPipeline, int, error)
 	FinDByParentCiPipelineAndAppId(parentCiPipeline int, appIds []int) ([]*CiPipeline, error)
-	FindAllPipelineInLast24Hour() (pipelines []*CiPipeline, err error)
+	FindAllPipelineCountInLast24Hour() (pipelineCount int, err error)
+	FindAllDeletedPipelineCountInLast24Hour() (pipelineCount int, err error)
 	FindNumberOfAppsWithCiPipeline(appIds []int) (count int, err error)
 	FindAppAndProjectByCiPipelineIds(ciPipelineIds []int) ([]*CiPipeline, error)
 	FindCiPipelineConfigsByIds(ids []int) ([]*CiPipeline, error)
@@ -502,12 +503,19 @@ func (impl *CiPipelineRepositoryImpl) FinDByParentCiPipelineAndAppId(parentCiPip
 	return ciPipelines, err
 }
 
-func (impl *CiPipelineRepositoryImpl) FindAllPipelineInLast24Hour() (pipelines []*CiPipeline, err error) {
-	err = impl.dbConnection.Model(&pipelines).
+func (impl *CiPipelineRepositoryImpl) FindAllPipelineCountInLast24Hour() (pipelineCount int, err error) {
+	pipelineCount, err = impl.dbConnection.Model(&CiPipeline{}).
 		Column("ci_pipeline.*").
 		Where("created_on > ?", time.Now().AddDate(0, 0, -1)).
-		Select()
-	return pipelines, err
+		Count()
+	return pipelineCount, err
+}
+func (impl *CiPipelineRepositoryImpl) FindAllDeletedPipelineCountInLast24Hour() (pipelineCount int, err error) {
+	pipelineCount, err = impl.dbConnection.Model(&CiPipeline{}).
+		Column("ci_pipeline.*").
+		Where("created_on > ? and deleted=?", time.Now().AddDate(0, 0, -1), true).
+		Count()
+	return pipelineCount, err
 }
 
 func (impl *CiPipelineRepositoryImpl) FindNumberOfAppsWithCiPipeline(appIds []int) (count int, err error) {

internal/sql/repository/pipelineConfig/CiWorkflowRepository.go

@@ -43,6 +43,7 @@ type CiWorkflowRepository interface {
 	FindLastTriggeredWorkflowByCiIds(pipelineId []int) (ciWorkflow []*CiWorkflow, err error)
 	FindLastTriggeredWorkflowByArtifactId(ciArtifactId int) (ciWorkflow *CiWorkflow, err error)
 	FindAllLastTriggeredWorkflowByArtifactId(ciArtifactId []int) (ciWorkflow []*CiWorkflow, err error)
+	FindAllTriggeredWorkflowCountInLast24Hour() (ciWorkflowCount int, err error)
 	FindLastTriggeredWorkflowGitTriggersByArtifactId(ciArtifactId int) (ciWorkflow *CiWorkflow, err error)
 	FindLastTriggeredWorkflowGitTriggersByArtifactIds(ciArtifactIds []int) ([]*WorkflowWithArtifact, error)
 	ExistsByStatus(status string) (bool, error)
@@ -286,7 +287,16 @@ func (impl *CiWorkflowRepositoryImpl) FindLastTriggeredWorkflowByArtifactId(ciAr
 		Select()
 	return workflow, err
 }
-
+func (impl *CiWorkflowRepositoryImpl) FindAllTriggeredWorkflowCountInLast24Hour() (ciWorkflowCount int, err error) {
+	cnt, err := impl.dbConnection.Model(&CiWorkflow{}).
+		ColumnExpr("DISTINCT ci_pipeline_id").
+		Where("started_on > ? ", time.Now().AddDate(0, 0, -1)).
+		Count()
+	if err != nil {
+		impl.logger.Errorw("error occurred while fetching ci workflow", "err", err)
+	}
+	return cnt, err
+}
 func (impl *CiWorkflowRepositoryImpl) FindAllLastTriggeredWorkflowByArtifactId(ciArtifactIds []int) (ciWorkflows []*CiWorkflow, err error) {
 	err = impl.dbConnection.Model(&ciWorkflows).
 		Column("ci_workflow.git_triggers", "ci_workflow.ci_pipeline_id", "CiPipeline", "cia.id").

internal/sql/repository/pipelineConfig/PipelineRepository.go

@@ -93,7 +93,8 @@ type PipelineRepository interface {
 	FindByIdsInAndEnvironment(ids []int, environmentId int) ([]*Pipeline, error)
 	FindActiveByAppIdAndEnvironmentIdV2() (pipelines []*Pipeline, err error)
 	GetConnection() *pg.DB
-	FindAllPipelineInLast24Hour() (pipelines []*Pipeline, err error)
+	FindAllPipelineCountInLast24Hour() (pipelineCount int, err error)
+	FindAllDeletedPipelineCountInLast24Hour() (pipelineCount int, err error)
 	FindActiveByEnvId(envId int) (pipelines []*Pipeline, err error)
 	FindActivePipelineByEnvId(envId int) (pipelines []*Pipeline, err error)
 	FindActiveByEnvIds(envId []int) (pipelines []*Pipeline, err error)
@@ -447,12 +448,19 @@ func (impl PipelineRepositoryImpl) FindByPipelineTriggerGitHash(gitHash string)
 	return pipelines, err
 }
 
-func (impl PipelineRepositoryImpl) FindAllPipelineInLast24Hour() (pipelines []*Pipeline, err error) {
-	err = impl.dbConnection.Model(&pipelines).
+func (impl PipelineRepositoryImpl) FindAllPipelineCountInLast24Hour() (pipelineCount int, err error) {
+	pipelineCount, err = impl.dbConnection.Model(&Pipeline{}).
 		Column("pipeline.*").
 		Where("created_on > ?", time.Now().AddDate(0, 0, -1)).
-		Select()
-	return pipelines, err
+		Count()
+	return pipelineCount, err
+}
+func (impl PipelineRepositoryImpl) FindAllDeletedPipelineCountInLast24Hour() (pipelineCount int, err error) {
+	pipelineCount, err = impl.dbConnection.Model(&Pipeline{}).
+		Column("pipeline.*").
+		Where("created_on > ? and deleted=?", time.Now().AddDate(0, 0, -1), true).
+		Count()
+	return pipelineCount, err
 }
 func (impl PipelineRepositoryImpl) FindActiveByEnvId(envId int) (pipelines []*Pipeline, err error) {
 	err = impl.dbConnection.Model(&pipelines).Column("pipeline.*", "App", "Environment").

pkg/cluster/EnvironmentService.go

@@ -46,6 +46,7 @@ type EnvironmentService interface {
 	Update(mappings *bean2.EnvironmentBean, userId int32) (*bean2.EnvironmentBean, error)
 	GetAll() ([]bean2.EnvironmentBean, error)
 	GetAllActive() ([]bean2.EnvironmentBean, error)
+	GetAllActiveEnvironmentCount() (int, error)
 	Delete(deleteReq *bean2.EnvironmentBean, userId int32) error
 	FindClusterByEnvId(id int) (*ClusterBean, error)
 	// FindById provides an exposed struct of bean.EnvironmentBean;
@@ -231,7 +232,10 @@ func (impl EnvironmentServiceImpl) GetAllActive() ([]bean2.EnvironmentBean, erro
 	}
 	return beans, nil
 }
-
+func (impl EnvironmentServiceImpl) GetAllActiveEnvironmentCount() (int, error) {
+	cnt, err := impl.environmentRepository.FindAllActiveEnvironmentCount()
+	return cnt, err
+}
 func (impl EnvironmentServiceImpl) FindById(id int) (*bean2.EnvironmentBean, error) {
 	model, err := impl.environmentRepository.FindById(id)
 	if err != nil {

pkg/cluster/repository/EnvironmentRepository.go

@@ -54,6 +54,7 @@ type EnvironmentRepository interface {
 	Create(mappings *Environment) error
 	FindAll() ([]Environment, error)
 	FindAllActive() ([]*Environment, error)
+	FindAllActiveEnvironmentCount() (int, error)
 	MarkEnvironmentDeleted(mappings *Environment, tx *pg.Tx) error
 	GetConnection() (dbConnection *pg.DB)
 	FindAllActiveEnvOnlyDetails() ([]*Environment, error)
@@ -256,6 +257,15 @@ func (repositoryImpl EnvironmentRepositoryImpl) FindAllActive() ([]*Environment,
 		Select()
 	return mappings, err
 }
+func (repositoryImpl EnvironmentRepositoryImpl) FindAllActiveEnvironmentCount() (int, error) {
+	cnt, err := repositoryImpl.
+		dbConnection.Model(&Environment{}).
+		Where("environment.active = ?", true).
+		Column("environment.*", "Cluster").
+		Join("inner join cluster c on environment.cluster_id = c.id").
+		Count()
+	return cnt, err
+}
 func (repositoryImpl EnvironmentRepositoryImpl) FindAllActiveEnvOnlyDetails() ([]*Environment, error) {
 	var mappings []*Environment
 	err := repositoryImpl.

pkg/deployment/gitOps/config/GitOpsConfigReadService.go

@@ -139,12 +139,6 @@ func (impl *GitOpsConfigReadServiceImpl) GetGitOpsConfigActive() (*bean2.GitOpsC
 }
 
 func (impl *GitOpsConfigReadServiceImpl) GetConfiguredGitOpsCount() (int, error) {
-	count := 0
-	models, err := impl.gitOpsRepository.GetAllGitOpsConfig()
-	if err != nil && err != pg.ErrNoRows {
-		impl.logger.Errorw("error, GetGitOpsConfigActive", "err", err)
-		return count, err
-	}
-	count = len(models)
-	return count, nil
+	count, err := impl.gitOpsRepository.GetAllGitOpsConfigCount()
+	return count, err
 }