From 72a18a0c1012fa5925f2ea5ef95af4c53f12d47f Mon Sep 17 00:00:00 2001 From: komalreddy3 Date: Fri, 12 Apr 2024 10:15:23 +0530 Subject: [PATCH 01/14] fix: ci-cd count --- .../telemetry/TelemetryEventClientExtended.go | 38 ++++++++++++++++--- .../pipelineConfig/CdWorfkflowRepository.go | 12 +++++- .../pipelineConfig/CiPipelineRepository.go | 8 ++++ .../pipelineConfig/CiWorkflowRepository.go | 10 +++++ .../pipelineConfig/PipelineRepository.go | 8 ++++ 5 files changed, 68 insertions(+), 8 deletions(-) diff --git a/client/telemetry/TelemetryEventClientExtended.go b/client/telemetry/TelemetryEventClientExtended.go index d80134097a..7b0869f739 100644 --- a/client/telemetry/TelemetryEventClientExtended.go +++ b/client/telemetry/TelemetryEventClientExtended.go @@ -131,8 +131,12 @@ type TelemetryEventDto struct { UserCount int `json:"userCount,omitempty"` EnvironmentCount int `json:"environmentCount,omitempty"` ClusterCount int `json:"clusterCount,omitempty"` - CiCountPerDay int `json:"ciCountPerDay,omitempty"` - CdCountPerDay int `json:"cdCountPerDay,omitempty"` + CiCreatedPerDay int `json:"ciCreatedPerDay,omitempty"` + CdCreatedPerDay int `json:"cdCreatedPerDay,omitempty"` + CiDeletedPerDay int `json:"ciDeletedPerDay,omitempty"` + CdDeletedPerDay int `json:"cdDeletedPerDay,omitempty"` + CiTriggeredPerDay int `json:"ciTriggeredPerDay,omitempty"` + CdTriggeredPerDay int `json:"cdTriggeredPerDay,omitempty"` HelmChartCount int `json:"helmChartCount,omitempty"` SecurityScanCountPerDay int `json:"securityScanCountPerDay,omitempty"` GitAccountsCount int `json:"gitAccountsCount,omitempty"` @@ -223,13 +227,32 @@ func (impl *TelemetryEventClientImplExtended) SendSummaryEvent(eventType string) impl.logger.Errorw("exception caught inside telemetry summary event", "err", err) return err } + ciPipelineDeleted, err := impl.ciPipelineRepository.FindAllDeletedPipelineInLast24Hour() + if err != nil && err != pg.ErrNoRows { + impl.logger.Errorw("exception caught inside telemetry summary event", "err", err) + return err + } + ciPipelineTriggered, err := impl.ciWorkflowRepository.FindAllTriggeredWorkflowInLast24Hour() + if err != nil && err != pg.ErrNoRows { + impl.logger.Errorw("exception caught inside telemetry summary event", "err", err) + return err + } cdPipeline, err := impl.pipelineRepository.FindAllPipelineInLast24Hour() if err != nil && err != pg.ErrNoRows { impl.logger.Errorw("exception caught inside telemetry summary event", "err", err) return err } - + cdPipelineDeleted, err := impl.pipelineRepository.FindAllDeletedPipelineInLast24Hour() + if err != nil && err != pg.ErrNoRows { + impl.logger.Errorw("exception caught inside telemetry summary event", "err", err) + return err + } + cdPipelineTriggered, err := impl.cdWorkflowRepository.FindAllTriggeredWorkflowInLast24Hour() + if err != nil && err != pg.ErrNoRows { + impl.logger.Errorw("exception caught inside telemetry summary event", "err", err) + return err + } gitAccounts, err := impl.gitProviderRepository.FindAll() if err != nil && err != pg.ErrNoRows { impl.logger.Errorw("exception caught inside telemetry summary event", "err", err) @@ -307,9 +330,12 @@ func (impl *TelemetryEventClientImplExtended) SendSummaryEvent(eventType string) payload.UserCount = len(users) payload.EnvironmentCount = len(environments) payload.ClusterCount = len(clusters) - payload.CiCountPerDay = len(ciPipeline) - - payload.CdCountPerDay = len(cdPipeline) + payload.CiCreatedPerDay = len(ciPipeline) + payload.CiDeletedPerDay = len(ciPipelineDeleted) + payload.CiTriggeredPerDay = ciPipelineTriggered + payload.CdCreatedPerDay = len(cdPipeline) + payload.CdDeletedPerDay = len(cdPipelineDeleted) + payload.CdTriggeredPerDay = cdPipelineTriggered payload.GitAccountsCount = len(gitAccounts) payload.GitOpsCount = gitOpsCount payload.HostURL = hostURL diff --git a/internal/sql/repository/pipelineConfig/CdWorfkflowRepository.go b/internal/sql/repository/pipelineConfig/CdWorfkflowRepository.go index b24c79934a..f1d5c70534 100644 --- a/internal/sql/repository/pipelineConfig/CdWorfkflowRepository.go +++ b/internal/sql/repository/pipelineConfig/CdWorfkflowRepository.go @@ -57,7 +57,7 @@ type CdWorkflowRepository interface { FindLatestWfrByAppIdAndEnvironmentId(appId int, environmentId int) (*CdWorkflowRunner, error) IsLatestCDWfr(pipelineId, wfrId int) (bool, error) FindLatestCdWorkflowRunnerByEnvironmentIdAndRunnerType(appId int, environmentId int, runnerType bean.WorkflowType) (CdWorkflowRunner, error) - + FindAllTriggeredWorkflowInLast24Hour() (cdWorkflowCount int, err error) GetConnection() *pg.DB FindLastPreOrPostTriggeredByPipelineId(pipelineId int) (CdWorkflowRunner, error) @@ -357,7 +357,15 @@ func (impl *CdWorkflowRepositoryImpl) FindLatestCdWorkflowByPipelineIdV2(pipelin // TODO - Group By Environment And Pipeline will get latest pipeline from top return cdWorkflow, err } - +func (impl *CdWorkflowRepositoryImpl) FindAllTriggeredWorkflowInLast24Hour() (cdWorkflowCount int, err error) { + var workflowCount int + query := "SELECT count(DISTINCT(cd_workflow_id)) FROM cd_workflow_runner WHERE started_on > now() - interval '1 day'" + _, err = impl.dbConnection.Query(&workflowCount, query) + if err != nil { + impl.logger.Errorw("error occurred while fetching ci workflow", "err", err) + } + return workflowCount, err +} func (impl *CdWorkflowRepositoryImpl) FindCdWorkflowMetaByEnvironmentId(appId int, environmentId int, offset int, limit int) ([]CdWorkflowRunner, error) { var wfrList []CdWorkflowRunner err := impl.dbConnection. diff --git a/internal/sql/repository/pipelineConfig/CiPipelineRepository.go b/internal/sql/repository/pipelineConfig/CiPipelineRepository.go index 495b4b7d64..ed8f75ded1 100644 --- a/internal/sql/repository/pipelineConfig/CiPipelineRepository.go +++ b/internal/sql/repository/pipelineConfig/CiPipelineRepository.go @@ -129,6 +129,7 @@ type CiPipelineRepository interface { FetchCiPipelinesForDG(parentId int, childCiPipelineIds []int) (*CiPipeline, int, error) FinDByParentCiPipelineAndAppId(parentCiPipeline int, appIds []int) ([]*CiPipeline, error) FindAllPipelineInLast24Hour() (pipelines []*CiPipeline, err error) + FindAllDeletedPipelineInLast24Hour() (pipelines []*CiPipeline, err error) FindNumberOfAppsWithCiPipeline(appIds []int) (count int, err error) FindAppAndProjectByCiPipelineIds(ciPipelineIds []int) ([]*CiPipeline, error) FindCiPipelineConfigsByIds(ids []int) ([]*CiPipeline, error) @@ -509,6 +510,13 @@ func (impl *CiPipelineRepositoryImpl) FindAllPipelineInLast24Hour() (pipelines [ Select() return pipelines, err } +func (impl *CiPipelineRepositoryImpl) FindAllDeletedPipelineInLast24Hour() (pipelines []*CiPipeline, err error) { + err = impl.dbConnection.Model(&pipelines). + Column("ci_pipeline.*"). + Where("created_on > ? and deleted=?", time.Now().AddDate(0, 0, -1), true). + Select() + return pipelines, err +} func (impl *CiPipelineRepositoryImpl) FindNumberOfAppsWithCiPipeline(appIds []int) (count int, err error) { var ciPipelines []*CiPipeline diff --git a/internal/sql/repository/pipelineConfig/CiWorkflowRepository.go b/internal/sql/repository/pipelineConfig/CiWorkflowRepository.go index 338032227a..c85bfa77e4 100644 --- a/internal/sql/repository/pipelineConfig/CiWorkflowRepository.go +++ b/internal/sql/repository/pipelineConfig/CiWorkflowRepository.go @@ -43,6 +43,7 @@ type CiWorkflowRepository interface { FindLastTriggeredWorkflowByCiIds(pipelineId []int) (ciWorkflow []*CiWorkflow, err error) FindLastTriggeredWorkflowByArtifactId(ciArtifactId int) (ciWorkflow *CiWorkflow, err error) FindAllLastTriggeredWorkflowByArtifactId(ciArtifactId []int) (ciWorkflow []*CiWorkflow, err error) + FindAllTriggeredWorkflowInLast24Hour() (ciWorkflowCount int, err error) FindLastTriggeredWorkflowGitTriggersByArtifactId(ciArtifactId int) (ciWorkflow *CiWorkflow, err error) FindLastTriggeredWorkflowGitTriggersByArtifactIds(ciArtifactIds []int) ([]*WorkflowWithArtifact, error) ExistsByStatus(status string) (bool, error) @@ -286,7 +287,16 @@ func (impl *CiWorkflowRepositoryImpl) FindLastTriggeredWorkflowByArtifactId(ciAr Select() return workflow, err } +func (impl *CiWorkflowRepositoryImpl) FindAllTriggeredWorkflowInLast24Hour() (ciWorkflowCount int, err error) { + var workflowCount int + query := "SELECT count(DISTINCT(ci_pipeline_id)) FROM ci_workflow WHERE started_on > now() - interval '1 day'" + _, err = impl.dbConnection.Query(&workflowCount, query) + if err != nil { + impl.logger.Errorw("error occurred while fetching ci workflow", "err", err) + } + return workflowCount, err +} func (impl *CiWorkflowRepositoryImpl) FindAllLastTriggeredWorkflowByArtifactId(ciArtifactIds []int) (ciWorkflows []*CiWorkflow, err error) { err = impl.dbConnection.Model(&ciWorkflows). Column("ci_workflow.git_triggers", "ci_workflow.ci_pipeline_id", "CiPipeline", "cia.id"). diff --git a/internal/sql/repository/pipelineConfig/PipelineRepository.go b/internal/sql/repository/pipelineConfig/PipelineRepository.go index 0d8fbbb7e8..06868a77e4 100644 --- a/internal/sql/repository/pipelineConfig/PipelineRepository.go +++ b/internal/sql/repository/pipelineConfig/PipelineRepository.go @@ -94,6 +94,7 @@ type PipelineRepository interface { FindActiveByAppIdAndEnvironmentIdV2() (pipelines []*Pipeline, err error) GetConnection() *pg.DB FindAllPipelineInLast24Hour() (pipelines []*Pipeline, err error) + FindAllDeletedPipelineInLast24Hour() (pipelines []*Pipeline, err error) FindActiveByEnvId(envId int) (pipelines []*Pipeline, err error) FindActivePipelineByEnvId(envId int) (pipelines []*Pipeline, err error) FindActiveByEnvIds(envId []int) (pipelines []*Pipeline, err error) @@ -454,6 +455,13 @@ func (impl PipelineRepositoryImpl) FindAllPipelineInLast24Hour() (pipelines []*P Select() return pipelines, err } +func (impl PipelineRepositoryImpl) FindAllDeletedPipelineInLast24Hour() (pipelines []*Pipeline, err error) { + err = impl.dbConnection.Model(&pipelines). + Column("pipeline.*"). + Where("created_on > ? and deleted=?", time.Now().AddDate(0, 0, -1), true). + Select() + return pipelines, err +} func (impl PipelineRepositoryImpl) FindActiveByEnvId(envId int) (pipelines []*Pipeline, err error) { err = impl.dbConnection.Model(&pipelines).Column("pipeline.*", "App", "Environment"). Where("environment_id = ?", envId). From 917e2200227c52ea93a4e47aaf682529e975c7d4 Mon Sep 17 00:00:00 2001 From: komalreddy3 Date: Fri, 12 Apr 2024 15:24:05 +0530 Subject: [PATCH 02/14] update queries --- .../pipelineConfig/CdWorfkflowRepository.go | 14 +++++++++----- .../pipelineConfig/CiWorkflowRepository.go | 12 +++++++----- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/internal/sql/repository/pipelineConfig/CdWorfkflowRepository.go b/internal/sql/repository/pipelineConfig/CdWorfkflowRepository.go index f1d5c70534..76c45074c1 100644 --- a/internal/sql/repository/pipelineConfig/CdWorfkflowRepository.go +++ b/internal/sql/repository/pipelineConfig/CdWorfkflowRepository.go @@ -358,13 +358,17 @@ func (impl *CdWorkflowRepositoryImpl) FindLatestCdWorkflowByPipelineIdV2(pipelin return cdWorkflow, err } func (impl *CdWorkflowRepositoryImpl) FindAllTriggeredWorkflowInLast24Hour() (cdWorkflowCount int, err error) { - var workflowCount int - query := "SELECT count(DISTINCT(cd_workflow_id)) FROM cd_workflow_runner WHERE started_on > now() - interval '1 day'" - _, err = impl.dbConnection.Query(&workflowCount, query) + var wfrList []CdWorkflowRunner + err = impl.dbConnection. + Model(&wfrList). + Column("cd_workflow_runner.*", "CdWorkflow"). + ColumnExpr("DISTINCT(cd_workflow.pipeline_id)"). + Where("cd_workflow_runner.workflow_type = ? and cd_workflow_runner.started_on > ?", bean.CD_WORKFLOW_TYPE_DEPLOY, time.Now().AddDate(0, 0, -1)). + Select() if err != nil { - impl.logger.Errorw("error occurred while fetching ci workflow", "err", err) + impl.logger.Errorw("error occurred while fetching cd workflow", "err", err) } - return workflowCount, err + return len(wfrList), err } func (impl *CdWorkflowRepositoryImpl) FindCdWorkflowMetaByEnvironmentId(appId int, environmentId int, offset int, limit int) ([]CdWorkflowRunner, error) { var wfrList []CdWorkflowRunner diff --git a/internal/sql/repository/pipelineConfig/CiWorkflowRepository.go b/internal/sql/repository/pipelineConfig/CiWorkflowRepository.go index c85bfa77e4..4b8bfe4122 100644 --- a/internal/sql/repository/pipelineConfig/CiWorkflowRepository.go +++ b/internal/sql/repository/pipelineConfig/CiWorkflowRepository.go @@ -288,14 +288,16 @@ func (impl *CiWorkflowRepositoryImpl) FindLastTriggeredWorkflowByArtifactId(ciAr return workflow, err } func (impl *CiWorkflowRepositoryImpl) FindAllTriggeredWorkflowInLast24Hour() (ciWorkflowCount int, err error) { - - var workflowCount int - query := "SELECT count(DISTINCT(ci_pipeline_id)) FROM ci_workflow WHERE started_on > now() - interval '1 day'" - _, err = impl.dbConnection.Query(&workflowCount, query) + var wfrList []CiWorkflow + err = impl.dbConnection.Model(&wfrList). + Column("ci_workflow.*"). + ColumnExpr("DISTINCT(ci_workflow.ci_pipeline_id)"). + Where("ci_workflow.started_on > ? ", time.Now().AddDate(0, 0, -1)). + Select() if err != nil { impl.logger.Errorw("error occurred while fetching ci workflow", "err", err) } - return workflowCount, err + return len(wfrList), err } func (impl *CiWorkflowRepositoryImpl) FindAllLastTriggeredWorkflowByArtifactId(ciArtifactIds []int) (ciWorkflows []*CiWorkflow, err error) { err = impl.dbConnection.Model(&ciWorkflows). From 660fba59c3c601ff8da52220cf25638245fb2e39 Mon Sep 17 00:00:00 2001 From: komalreddy3 Date: Fri, 12 Apr 2024 19:24:38 +0530 Subject: [PATCH 03/14] update queries to go-pg instead of raw --- .../pipelineConfig/CdWorfkflowRepository.go | 13 +++++++------ .../pipelineConfig/CiWorkflowRepository.go | 5 ++--- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/internal/sql/repository/pipelineConfig/CdWorfkflowRepository.go b/internal/sql/repository/pipelineConfig/CdWorfkflowRepository.go index 76c45074c1..96bed91f8e 100644 --- a/internal/sql/repository/pipelineConfig/CdWorfkflowRepository.go +++ b/internal/sql/repository/pipelineConfig/CdWorfkflowRepository.go @@ -358,13 +358,14 @@ func (impl *CdWorkflowRepositoryImpl) FindLatestCdWorkflowByPipelineIdV2(pipelin return cdWorkflow, err } func (impl *CdWorkflowRepositoryImpl) FindAllTriggeredWorkflowInLast24Hour() (cdWorkflowCount int, err error) { - var wfrList []CdWorkflowRunner + var wfrList []int err = impl.dbConnection. - Model(&wfrList). - Column("cd_workflow_runner.*", "CdWorkflow"). - ColumnExpr("DISTINCT(cd_workflow.pipeline_id)"). - Where("cd_workflow_runner.workflow_type = ? and cd_workflow_runner.started_on > ?", bean.CD_WORKFLOW_TYPE_DEPLOY, time.Now().AddDate(0, 0, -1)). - Select() + Model(&CdWorkflow{}). + ColumnExpr("DISTINCT pipeline_id"). + Join("JOIN cd_workflow_runner ON cd_workflow.id = cd_workflow_runner.cd_workflow_id"). + Where("cd_workflow_runner.workflow_type = ? AND cd_workflow_runner.started_on > ?", bean.CD_WORKFLOW_TYPE_DEPLOY, time.Now().AddDate(0, 0, -1)). + Group("cd_workflow.pipeline_id"). + Select(&wfrList) if err != nil { impl.logger.Errorw("error occurred while fetching cd workflow", "err", err) } diff --git a/internal/sql/repository/pipelineConfig/CiWorkflowRepository.go b/internal/sql/repository/pipelineConfig/CiWorkflowRepository.go index 4b8bfe4122..6acdfb621f 100644 --- a/internal/sql/repository/pipelineConfig/CiWorkflowRepository.go +++ b/internal/sql/repository/pipelineConfig/CiWorkflowRepository.go @@ -290,9 +290,8 @@ func (impl *CiWorkflowRepositoryImpl) FindLastTriggeredWorkflowByArtifactId(ciAr func (impl *CiWorkflowRepositoryImpl) FindAllTriggeredWorkflowInLast24Hour() (ciWorkflowCount int, err error) { var wfrList []CiWorkflow err = impl.dbConnection.Model(&wfrList). - Column("ci_workflow.*"). - ColumnExpr("DISTINCT(ci_workflow.ci_pipeline_id)"). - Where("ci_workflow.started_on > ? ", time.Now().AddDate(0, 0, -1)). + ColumnExpr("DISTINCT ci_pipeline_id"). + Where("started_on > ? ", time.Now().AddDate(0, 0, -1)). Select() if err != nil { impl.logger.Errorw("error occurred while fetching ci workflow", "err", err) From ee71147c1639711d4766a6088eab36a8e9f51a93 Mon Sep 17 00:00:00 2001 From: komalreddy3 Date: Mon, 15 Apr 2024 11:29:07 +0530 Subject: [PATCH 04/14] remove omit empty --- client/telemetry/TelemetryEventClientExtended.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/client/telemetry/TelemetryEventClientExtended.go b/client/telemetry/TelemetryEventClientExtended.go index 7b0869f739..86471e3d66 100644 --- a/client/telemetry/TelemetryEventClientExtended.go +++ b/client/telemetry/TelemetryEventClientExtended.go @@ -131,12 +131,12 @@ type TelemetryEventDto struct { UserCount int `json:"userCount,omitempty"` EnvironmentCount int `json:"environmentCount,omitempty"` ClusterCount int `json:"clusterCount,omitempty"` - CiCreatedPerDay int `json:"ciCreatedPerDay,omitempty"` - CdCreatedPerDay int `json:"cdCreatedPerDay,omitempty"` - CiDeletedPerDay int `json:"ciDeletedPerDay,omitempty"` - CdDeletedPerDay int `json:"cdDeletedPerDay,omitempty"` - CiTriggeredPerDay int `json:"ciTriggeredPerDay,omitempty"` - CdTriggeredPerDay int `json:"cdTriggeredPerDay,omitempty"` + CiCreatedPerDay int `json:"ciCreatedPerDay"` + CdCreatedPerDay int `json:"cdCreatedPerDay"` + CiDeletedPerDay int `json:"ciDeletedPerDay"` + CdDeletedPerDay int `json:"cdDeletedPerDay"` + CiTriggeredPerDay int `json:"ciTriggeredPerDay"` + CdTriggeredPerDay int `json:"cdTriggeredPerDay"` HelmChartCount int `json:"helmChartCount,omitempty"` SecurityScanCountPerDay int `json:"securityScanCountPerDay,omitempty"` GitAccountsCount int `json:"gitAccountsCount,omitempty"` From c6f955502818a395d0619ed4cf3d86aafeb1f06e Mon Sep 17 00:00:00 2001 From: komalreddy3 Date: Tue, 16 Apr 2024 13:47:16 +0530 Subject: [PATCH 05/14] summarycron expr change --- client/telemetry/PosthogClient.go | 1 - client/telemetry/TelemetryEventClient.go | 3 ++- client/telemetry/TelemetryEventClientExtended.go | 3 ++- cmd/external-app/wire_gen.go | 4 ++-- util/GlobalConfig.go | 6 +++++- wire_gen.go | 4 ++-- 6 files changed, 13 insertions(+), 8 deletions(-) diff --git a/client/telemetry/PosthogClient.go b/client/telemetry/PosthogClient.go index 41302aeefb..c0567de5fc 100644 --- a/client/telemetry/PosthogClient.go +++ b/client/telemetry/PosthogClient.go @@ -36,7 +36,6 @@ type PosthogClient struct { var ( PosthogApiKey string = "" PosthogEndpoint string = "https://app.posthog.com" - SummaryCronExpr string = "0 0 * * *" // Run once a day, midnight HeartbeatCronExpr string = "0 0/6 * * *" CacheExpiry int = 1440 PosthogEncodedApiKey string = "" diff --git a/client/telemetry/TelemetryEventClient.go b/client/telemetry/TelemetryEventClient.go index 7e76aea91a..0434411757 100644 --- a/client/telemetry/TelemetryEventClient.go +++ b/client/telemetry/TelemetryEventClient.go @@ -75,7 +75,7 @@ func NewTelemetryEventClientImpl(logger *zap.SugaredLogger, client *http.Client, K8sUtil *k8s.K8sServiceImpl, aCDAuthConfig *util3.ACDAuthConfig, userService user2.UserService, attributeRepo repository.AttributesRepository, ssoLoginService sso.SSOLoginService, PosthogClient *PosthogClient, moduleRepository moduleRepo.ModuleRepository, serverDataStore *serverDataStore.ServerDataStore, userAuditService user2.UserAuditService, helmAppClient gRPC.HelmAppClient, InstalledAppRepository repository2.InstalledAppRepository, - cloudProviderIdentifierService cloudProviderIdentifier.ProviderIdentifierService, cronLogger *cron3.CronLoggerImpl) (*TelemetryEventClientImpl, error) { + cloudProviderIdentifierService cloudProviderIdentifier.ProviderIdentifierService, cronLogger *cron3.CronLoggerImpl, envVariables *util.EnvironmentVariables) (*TelemetryEventClientImpl, error) { cron := cron.New( cron.WithChain(cron.Recover(cronLogger))) cron.Start() @@ -97,6 +97,7 @@ func NewTelemetryEventClientImpl(logger *zap.SugaredLogger, client *http.Client, } watcher.HeartbeatEventForTelemetry() + SummaryCronExpr := envVariables.PostHogClientConfig.SummaryCron _, err := cron.AddFunc(SummaryCronExpr, watcher.SummaryEventForTelemetryEA) if err != nil { logger.Errorw("error in starting summery event", "err", err) diff --git a/client/telemetry/TelemetryEventClientExtended.go b/client/telemetry/TelemetryEventClientExtended.go index 86471e3d66..49f3871850 100644 --- a/client/telemetry/TelemetryEventClientExtended.go +++ b/client/telemetry/TelemetryEventClientExtended.go @@ -64,7 +64,7 @@ func NewTelemetryEventClientImplExtended(logger *zap.SugaredLogger, client *http ciBuildConfigService pipeline.CiBuildConfigService, moduleRepository moduleRepo.ModuleRepository, serverDataStore *serverDataStore.ServerDataStore, helmAppClient client.HelmAppClient, InstalledAppRepository repository2.InstalledAppRepository, userAttributesRepository repository.UserAttributesRepository, cloudProviderIdentifierService cloudProviderIdentifier.ProviderIdentifierService, cronLogger *cron3.CronLoggerImpl, - gitOpsConfigReadService config.GitOpsConfigReadService) (*TelemetryEventClientImplExtended, error) { + gitOpsConfigReadService config.GitOpsConfigReadService, envVariables *util.EnvironmentVariables) (*TelemetryEventClientImplExtended, error) { cron := cron.New( cron.WithChain(cron.Recover(cronLogger))) @@ -107,6 +107,7 @@ func NewTelemetryEventClientImplExtended(logger *zap.SugaredLogger, client *http } watcher.HeartbeatEventForTelemetry() + SummaryCronExpr := envVariables.PostHogClientConfig.SummaryCron _, err := cron.AddFunc(SummaryCronExpr, watcher.SummaryEventForTelemetry) if err != nil { logger.Errorw("error in starting summery event", "err", err) diff --git a/cmd/external-app/wire_gen.go b/cmd/external-app/wire_gen.go index 4019867777..d4545a0710 100644 --- a/cmd/external-app/wire_gen.go +++ b/cmd/external-app/wire_gen.go @@ -1,6 +1,6 @@ // Code generated by Wire. DO NOT EDIT. -//go:generate go run github.com/google/wire/cmd/wire +//go:generate go run -mod=mod github.com/google/wire/cmd/wire //go:build !wireinject // +build !wireinject @@ -298,7 +298,7 @@ func InitializeApp() (*App, error) { moduleRepositoryImpl := moduleRepo.NewModuleRepositoryImpl(db) providerIdentifierServiceImpl := providerIdentifier.NewProviderIdentifierServiceImpl(sugaredLogger) cronLoggerImpl := cron.NewCronLoggerImpl(sugaredLogger) - telemetryEventClientImpl, err := telemetry.NewTelemetryEventClientImpl(sugaredLogger, httpClient, clusterServiceImpl, k8sServiceImpl, acdAuthConfig, userServiceImpl, attributesRepositoryImpl, ssoLoginServiceImpl, posthogClient, moduleRepositoryImpl, serverDataStoreServerDataStore, userAuditServiceImpl, helmAppClientImpl, installedAppRepositoryImpl, providerIdentifierServiceImpl, cronLoggerImpl) + telemetryEventClientImpl, err := telemetry.NewTelemetryEventClientImpl(sugaredLogger, httpClient, clusterServiceImpl, k8sServiceImpl, acdAuthConfig, userServiceImpl, attributesRepositoryImpl, ssoLoginServiceImpl, posthogClient, moduleRepositoryImpl, serverDataStoreServerDataStore, userAuditServiceImpl, helmAppClientImpl, installedAppRepositoryImpl, providerIdentifierServiceImpl, cronLoggerImpl, environmentVariables) if err != nil { return nil, err } diff --git a/util/GlobalConfig.go b/util/GlobalConfig.go index 5728015bef..0a106123a9 100644 --- a/util/GlobalConfig.go +++ b/util/GlobalConfig.go @@ -8,8 +8,11 @@ type EnvironmentVariables struct { GlobalEnvVariables *GlobalEnvVariables DevtronSecretConfig *DevtronSecretConfig DeploymentServiceTypeConfig *DeploymentServiceTypeConfig + PostHogClientConfig *PostHogClientConfig +} +type PostHogClientConfig struct { + SummaryCron string `env:"POSTHOG_SUMMARY_CRON_EXPR" envDefault:"0 0 * * *"` // Default Run once a day, midnight } - type DeploymentServiceTypeConfig struct { ExternallyManagedDeploymentType bool `env:"IS_INTERNAL_USE" envDefault:"false"` HelmInstallASyncMode bool `env:"RUN_HELM_INSTALL_IN_ASYNC_MODE_HELM_APPS" envDefault:"false"` @@ -31,6 +34,7 @@ func GetEnvironmentVariables() (*EnvironmentVariables, error) { GlobalEnvVariables: &GlobalEnvVariables{}, DevtronSecretConfig: &DevtronSecretConfig{}, DeploymentServiceTypeConfig: &DeploymentServiceTypeConfig{}, + PostHogClientConfig: &PostHogClientConfig{}, } err := env.Parse(cfg) if err != nil { diff --git a/wire_gen.go b/wire_gen.go index 079bbb60af..a497d2deb1 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -1,6 +1,6 @@ // Code generated by Wire. DO NOT EDIT. -//go:generate go run github.com/google/wire/cmd/wire +//go:generate go run -mod=mod github.com/google/wire/cmd/wire //go:build !wireinject // +build !wireinject @@ -745,7 +745,7 @@ func InitializeApp() (*App, error) { return nil, err } providerIdentifierServiceImpl := providerIdentifier.NewProviderIdentifierServiceImpl(sugaredLogger) - telemetryEventClientImplExtended, err := telemetry.NewTelemetryEventClientImplExtended(sugaredLogger, httpClient, clusterServiceImplExtended, k8sServiceImpl, acdAuthConfig, environmentServiceImpl, userServiceImpl, appListingRepositoryImpl, posthogClient, ciPipelineRepositoryImpl, pipelineRepositoryImpl, gitProviderRepositoryImpl, attributesRepositoryImpl, ssoLoginServiceImpl, appRepositoryImpl, ciWorkflowRepositoryImpl, cdWorkflowRepositoryImpl, dockerArtifactStoreRepositoryImpl, materialRepositoryImpl, ciTemplateRepositoryImpl, chartRepositoryImpl, userAuditServiceImpl, ciBuildConfigServiceImpl, moduleRepositoryImpl, serverDataStoreServerDataStore, helmAppClientImpl, installedAppRepositoryImpl, userAttributesRepositoryImpl, providerIdentifierServiceImpl, cronLoggerImpl, gitOpsConfigReadServiceImpl) + telemetryEventClientImplExtended, err := telemetry.NewTelemetryEventClientImplExtended(sugaredLogger, httpClient, clusterServiceImplExtended, k8sServiceImpl, acdAuthConfig, environmentServiceImpl, userServiceImpl, appListingRepositoryImpl, posthogClient, ciPipelineRepositoryImpl, pipelineRepositoryImpl, gitProviderRepositoryImpl, attributesRepositoryImpl, ssoLoginServiceImpl, appRepositoryImpl, ciWorkflowRepositoryImpl, cdWorkflowRepositoryImpl, dockerArtifactStoreRepositoryImpl, materialRepositoryImpl, ciTemplateRepositoryImpl, chartRepositoryImpl, userAuditServiceImpl, ciBuildConfigServiceImpl, moduleRepositoryImpl, serverDataStoreServerDataStore, helmAppClientImpl, installedAppRepositoryImpl, userAttributesRepositoryImpl, providerIdentifierServiceImpl, cronLoggerImpl, gitOpsConfigReadServiceImpl, environmentVariables) if err != nil { return nil, err } From ade898b84b62051d39a90f17af3d6f722251f645 Mon Sep 17 00:00:00 2001 From: komalreddy3 Date: Thu, 18 Apr 2024 16:46:36 +0530 Subject: [PATCH 06/14] refactor --- client/telemetry/PosthogClient.go | 1 + client/telemetry/TelemetryEventClient.go | 3 +- .../telemetry/TelemetryEventClientExtended.go | 87 +++++++++---------- cmd/external-app/wire_gen.go | 2 +- .../sql/repository/GitOpsConfigRepository.go | 5 ++ .../sql/repository/GitProviderRepository.go | 6 ++ .../DockerArtifactStoreRepository.go | 15 +++- .../pipelineConfig/CdWorfkflowRepository.go | 12 +-- .../pipelineConfig/CiPipelineRepository.go | 20 ++--- .../pipelineConfig/CiWorkflowRepository.go | 12 +-- .../pipelineConfig/PipelineRepository.go | 20 ++--- pkg/cluster/EnvironmentService.go | 6 +- .../repository/EnvironmentRepository.go | 10 +++ .../gitOps/config/GitOpsConfigReadService.go | 17 ++-- util/GlobalConfig.go | 6 +- wire_gen.go | 2 +- 16 files changed, 129 insertions(+), 95 deletions(-) diff --git a/client/telemetry/PosthogClient.go b/client/telemetry/PosthogClient.go index c0567de5fc..3ee4694100 100644 --- a/client/telemetry/PosthogClient.go +++ b/client/telemetry/PosthogClient.go @@ -36,6 +36,7 @@ type PosthogClient struct { var ( PosthogApiKey string = "" PosthogEndpoint string = "https://app.posthog.com" + SummaryCronExpr string = "0 0 * * *" // Default Run once a day, midnight HeartbeatCronExpr string = "0 0/6 * * *" CacheExpiry int = 1440 PosthogEncodedApiKey string = "" diff --git a/client/telemetry/TelemetryEventClient.go b/client/telemetry/TelemetryEventClient.go index 0434411757..7e76aea91a 100644 --- a/client/telemetry/TelemetryEventClient.go +++ b/client/telemetry/TelemetryEventClient.go @@ -75,7 +75,7 @@ func NewTelemetryEventClientImpl(logger *zap.SugaredLogger, client *http.Client, K8sUtil *k8s.K8sServiceImpl, aCDAuthConfig *util3.ACDAuthConfig, userService user2.UserService, attributeRepo repository.AttributesRepository, ssoLoginService sso.SSOLoginService, PosthogClient *PosthogClient, moduleRepository moduleRepo.ModuleRepository, serverDataStore *serverDataStore.ServerDataStore, userAuditService user2.UserAuditService, helmAppClient gRPC.HelmAppClient, InstalledAppRepository repository2.InstalledAppRepository, - cloudProviderIdentifierService cloudProviderIdentifier.ProviderIdentifierService, cronLogger *cron3.CronLoggerImpl, envVariables *util.EnvironmentVariables) (*TelemetryEventClientImpl, error) { + cloudProviderIdentifierService cloudProviderIdentifier.ProviderIdentifierService, cronLogger *cron3.CronLoggerImpl) (*TelemetryEventClientImpl, error) { cron := cron.New( cron.WithChain(cron.Recover(cronLogger))) cron.Start() @@ -97,7 +97,6 @@ func NewTelemetryEventClientImpl(logger *zap.SugaredLogger, client *http.Client, } watcher.HeartbeatEventForTelemetry() - SummaryCronExpr := envVariables.PostHogClientConfig.SummaryCron _, err := cron.AddFunc(SummaryCronExpr, watcher.SummaryEventForTelemetryEA) if err != nil { logger.Errorw("error in starting summery event", "err", err) diff --git a/client/telemetry/TelemetryEventClientExtended.go b/client/telemetry/TelemetryEventClientExtended.go index 49f3871850..154714ee46 100644 --- a/client/telemetry/TelemetryEventClientExtended.go +++ b/client/telemetry/TelemetryEventClientExtended.go @@ -64,7 +64,7 @@ func NewTelemetryEventClientImplExtended(logger *zap.SugaredLogger, client *http ciBuildConfigService pipeline.CiBuildConfigService, moduleRepository moduleRepo.ModuleRepository, serverDataStore *serverDataStore.ServerDataStore, helmAppClient client.HelmAppClient, InstalledAppRepository repository2.InstalledAppRepository, userAttributesRepository repository.UserAttributesRepository, cloudProviderIdentifierService cloudProviderIdentifier.ProviderIdentifierService, cronLogger *cron3.CronLoggerImpl, - gitOpsConfigReadService config.GitOpsConfigReadService, envVariables *util.EnvironmentVariables) (*TelemetryEventClientImplExtended, error) { + gitOpsConfigReadService config.GitOpsConfigReadService) (*TelemetryEventClientImplExtended, error) { cron := cron.New( cron.WithChain(cron.Recover(cronLogger))) @@ -107,7 +107,6 @@ func NewTelemetryEventClientImplExtended(logger *zap.SugaredLogger, client *http } watcher.HeartbeatEventForTelemetry() - SummaryCronExpr := envVariables.PostHogClientConfig.SummaryCron _, err := cron.AddFunc(SummaryCronExpr, watcher.SummaryEventForTelemetry) if err != nil { logger.Errorw("error in starting summery event", "err", err) @@ -205,71 +204,71 @@ func (impl *TelemetryEventClientImplExtended) SendSummaryEvent(eventType string) payload := &TelemetryEventDto{UCID: ucid, Timestamp: time.Now(), EventType: TelemetryEventType(eventType), DevtronVersion: "v1"} payload.ServerVersion = k8sServerVersion.String() - environments, err := impl.environmentService.GetAllActive() + environmentCount, err := impl.environmentService.GetAllActiveEnvironmentCount() if err != nil && err != pg.ErrNoRows { - impl.logger.Errorw("exception caught inside telemetry summary event", "err", err) - return err + impl.logger.Errorw("exception caught inside telemetry summary event, setting value to -1", "err", err) + environmentCount = -1 } prodApps, err := impl.appListingRepository.FindAppCount(true) if err != nil && err != pg.ErrNoRows { - impl.logger.Errorw("exception caught inside telemetry summary event", "err", err) - return err + impl.logger.Errorw("exception caught inside telemetry summary event, setting value to -1", "err", err) + prodApps = -1 } nonProdApps, err := impl.appListingRepository.FindAppCount(false) if err != nil && err != pg.ErrNoRows { - impl.logger.Errorw("exception caught inside telemetry summary event", "err", err) - return err + impl.logger.Errorw("exception caught inside telemetry summary event, setting value to -1", "err", err) + nonProdApps = -1 } - ciPipeline, err := impl.ciPipelineRepository.FindAllPipelineInLast24Hour() + ciPipelineCount, err := impl.ciPipelineRepository.FindAllPipelineCountInLast24Hour() if err != nil && err != pg.ErrNoRows { - impl.logger.Errorw("exception caught inside telemetry summary event", "err", err) - return err + impl.logger.Errorw("exception caught inside telemetry summary event, setting value to -1", "err", err) + ciPipelineCount = -1 } - ciPipelineDeleted, err := impl.ciPipelineRepository.FindAllDeletedPipelineInLast24Hour() + ciPipelineDeletedCount, err := impl.ciPipelineRepository.FindAllDeletedPipelineCountInLast24Hour() if err != nil && err != pg.ErrNoRows { - impl.logger.Errorw("exception caught inside telemetry summary event", "err", err) - return err + impl.logger.Errorw("exception caught inside telemetry summary event, setting value to -1", "err", err) + ciPipelineDeletedCount = -1 } - ciPipelineTriggered, err := impl.ciWorkflowRepository.FindAllTriggeredWorkflowInLast24Hour() + ciPipelineTriggeredCount, err := impl.ciWorkflowRepository.FindAllTriggeredWorkflowCountInLast24Hour() if err != nil && err != pg.ErrNoRows { - impl.logger.Errorw("exception caught inside telemetry summary event", "err", err) - return err + impl.logger.Errorw("exception caught inside telemetry summary event, setting value to -1", "err", err) + ciPipelineTriggeredCount = -1 } - cdPipeline, err := impl.pipelineRepository.FindAllPipelineInLast24Hour() + cdPipelineCount, err := impl.pipelineRepository.FindAllPipelineCountInLast24Hour() if err != nil && err != pg.ErrNoRows { - impl.logger.Errorw("exception caught inside telemetry summary event", "err", err) - return err + impl.logger.Errorw("exception caught inside telemetry summary event, setting value to -1", "err", err) + cdPipelineCount = -1 } - cdPipelineDeleted, err := impl.pipelineRepository.FindAllDeletedPipelineInLast24Hour() + cdPipelineDeletedCount, err := impl.pipelineRepository.FindAllDeletedPipelineCountInLast24Hour() if err != nil && err != pg.ErrNoRows { - impl.logger.Errorw("exception caught inside telemetry summary event", "err", err) - return err + impl.logger.Errorw("exception caught inside telemetry summary event, setting value to -1", "err", err) + cdPipelineDeletedCount = -1 } - cdPipelineTriggered, err := impl.cdWorkflowRepository.FindAllTriggeredWorkflowInLast24Hour() + cdPipelineTriggeredCount, err := impl.cdWorkflowRepository.FindAllTriggeredWorkflowCountInLast24Hour() if err != nil && err != pg.ErrNoRows { - impl.logger.Errorw("exception caught inside telemetry summary event", "err", err) - return err + impl.logger.Errorw("exception caught inside telemetry summary event, setting value to -1", "err", err) + cdPipelineTriggeredCount = -1 } - gitAccounts, err := impl.gitProviderRepository.FindAll() + gitAccounts, err := impl.gitProviderRepository.FindAllGitProviderCount() if err != nil && err != pg.ErrNoRows { - impl.logger.Errorw("exception caught inside telemetry summary event", "err", err) - return err + impl.logger.Errorw("exception caught inside telemetry summary event, setting value to -1", "err", err) + gitAccounts = -1 } gitOpsCount, err := impl.gitOpsConfigReadService.GetConfiguredGitOpsCount() if err != nil { - impl.logger.Errorw("exception caught inside telemetry summary event", "err", err) - return err + impl.logger.Errorw("exception caught inside telemetry summary event,setting value to -1", "err", err) + gitOpsCount = -1 } - containerRegistry, err := impl.dockerArtifactStoreRepository.FindAll() + containerRegistryCount, err := impl.dockerArtifactStoreRepository.FindAllDockerArtifactCount() if err != nil && err != pg.ErrNoRows { - impl.logger.Errorw("exception caught inside telemetry summary event", "err", err) - return err + impl.logger.Errorw("exception caught inside telemetry summary event,setting value to -1", "err", err) + containerRegistryCount = -1 } //appSetup := false @@ -326,18 +325,18 @@ func (impl *TelemetryEventClientImplExtended) SendSummaryEvent(eventType string) devtronVersion := util.GetDevtronVersion() payload.ProdAppCount = prodApps payload.NonProdAppCount = nonProdApps - payload.RegistryCount = len(containerRegistry) + payload.RegistryCount = containerRegistryCount payload.SSOLogin = ssoSetup payload.UserCount = len(users) - payload.EnvironmentCount = len(environments) + payload.EnvironmentCount = environmentCount payload.ClusterCount = len(clusters) - payload.CiCreatedPerDay = len(ciPipeline) - payload.CiDeletedPerDay = len(ciPipelineDeleted) - payload.CiTriggeredPerDay = ciPipelineTriggered - payload.CdCreatedPerDay = len(cdPipeline) - payload.CdDeletedPerDay = len(cdPipelineDeleted) - payload.CdTriggeredPerDay = cdPipelineTriggered - payload.GitAccountsCount = len(gitAccounts) + payload.CiCreatedPerDay = ciPipelineCount + payload.CiDeletedPerDay = ciPipelineDeletedCount + payload.CiTriggeredPerDay = ciPipelineTriggeredCount + payload.CdCreatedPerDay = cdPipelineCount + payload.CdDeletedPerDay = cdPipelineDeletedCount + payload.CdTriggeredPerDay = cdPipelineTriggeredCount + payload.GitAccountsCount = gitAccounts payload.GitOpsCount = gitOpsCount payload.HostURL = hostURL payload.DevtronGitVersion = devtronVersion.GitCommit diff --git a/cmd/external-app/wire_gen.go b/cmd/external-app/wire_gen.go index d4545a0710..d82b8c6fa1 100644 --- a/cmd/external-app/wire_gen.go +++ b/cmd/external-app/wire_gen.go @@ -298,7 +298,7 @@ func InitializeApp() (*App, error) { moduleRepositoryImpl := moduleRepo.NewModuleRepositoryImpl(db) providerIdentifierServiceImpl := providerIdentifier.NewProviderIdentifierServiceImpl(sugaredLogger) cronLoggerImpl := cron.NewCronLoggerImpl(sugaredLogger) - telemetryEventClientImpl, err := telemetry.NewTelemetryEventClientImpl(sugaredLogger, httpClient, clusterServiceImpl, k8sServiceImpl, acdAuthConfig, userServiceImpl, attributesRepositoryImpl, ssoLoginServiceImpl, posthogClient, moduleRepositoryImpl, serverDataStoreServerDataStore, userAuditServiceImpl, helmAppClientImpl, installedAppRepositoryImpl, providerIdentifierServiceImpl, cronLoggerImpl, environmentVariables) + telemetryEventClientImpl, err := telemetry.NewTelemetryEventClientImpl(sugaredLogger, httpClient, clusterServiceImpl, k8sServiceImpl, acdAuthConfig, userServiceImpl, attributesRepositoryImpl, ssoLoginServiceImpl, posthogClient, moduleRepositoryImpl, serverDataStoreServerDataStore, userAuditServiceImpl, helmAppClientImpl, installedAppRepositoryImpl, providerIdentifierServiceImpl, cronLoggerImpl) if err != nil { return nil, err } diff --git a/internal/sql/repository/GitOpsConfigRepository.go b/internal/sql/repository/GitOpsConfigRepository.go index 8f59497342..4d6f30787a 100644 --- a/internal/sql/repository/GitOpsConfigRepository.go +++ b/internal/sql/repository/GitOpsConfigRepository.go @@ -28,6 +28,7 @@ type GitOpsConfigRepository interface { UpdateGitOpsConfig(model *GitOpsConfig, tx *pg.Tx) error GetGitOpsConfigById(id int) (*GitOpsConfig, error) GetAllGitOpsConfig() ([]*GitOpsConfig, error) + GetAllGitOpsConfigCount() (int, error) GetGitOpsConfigByProvider(provider string) (*GitOpsConfig, error) GetGitOpsConfigActive() (*GitOpsConfig, error) GetConnection() *pg.DB @@ -91,6 +92,10 @@ func (impl *GitOpsConfigRepositoryImpl) GetAllGitOpsConfig() ([]*GitOpsConfig, e err := impl.dbConnection.Model(&userModel).Order("updated_on desc").Select() return userModel, err } +func (impl *GitOpsConfigRepositoryImpl) GetAllGitOpsConfigCount() (int, error) { + cnt, err := impl.dbConnection.Model(&GitOpsConfig{}).Order("updated_on desc").Count() + return cnt, err +} func (impl *GitOpsConfigRepositoryImpl) GetGitOpsConfigByProvider(provider string) (*GitOpsConfig, error) { var model GitOpsConfig err := impl.dbConnection.Model(&model).Where("provider = ?", provider).Select() diff --git a/internal/sql/repository/GitProviderRepository.go b/internal/sql/repository/GitProviderRepository.go index f2b73d96b3..b75a5806a2 100644 --- a/internal/sql/repository/GitProviderRepository.go +++ b/internal/sql/repository/GitProviderRepository.go @@ -52,6 +52,7 @@ type GitProviderRepository interface { ProviderExists(url string) (bool, error) FindAllActiveForAutocomplete() ([]GitProvider, error) FindAll() ([]GitProvider, error) + FindAllGitProviderCount() (int, error) FindOne(providerId string) (GitProvider, error) FindByUrl(providerUrl string) (GitProvider, error) Update(gitProvider *GitProvider) error @@ -95,6 +96,11 @@ func (impl GitProviderRepositoryImpl) FindAll() ([]GitProvider, error) { Where("deleted = ?", false).Select() return providers, err } +func (impl GitProviderRepositoryImpl) FindAllGitProviderCount() (int, error) { + gitProviderCount, err := impl.dbConnection.Model(&GitProvider{}). + Where("deleted = ?", false).Count() + return gitProviderCount, err +} func (impl GitProviderRepositoryImpl) FindOne(providerId string) (GitProvider, error) { var provider GitProvider diff --git a/internal/sql/repository/dockerRegistry/DockerArtifactStoreRepository.go b/internal/sql/repository/dockerRegistry/DockerArtifactStoreRepository.go index ebb20f79f3..74f777a39e 100644 --- a/internal/sql/repository/dockerRegistry/DockerArtifactStoreRepository.go +++ b/internal/sql/repository/dockerRegistry/DockerArtifactStoreRepository.go @@ -92,6 +92,7 @@ type DockerArtifactStoreRepository interface { FindActiveDefaultStore() (*DockerArtifactStore, error) FindAllActiveForAutocomplete() ([]DockerArtifactStore, error) FindAll() ([]DockerArtifactStore, error) + FindAllDockerArtifactCount() (int, error) FindAllChartProviders() ([]DockerArtifactStore, error) FindOne(storeId string) (*DockerArtifactStore, error) FindOneWithDeploymentCount(storeId string) (*DockerArtifactStoreExt, error) @@ -169,7 +170,19 @@ func (impl DockerArtifactStoreRepositoryImpl) FindAll() ([]DockerArtifactStore, Select() return providers, err } - +func (impl DockerArtifactStoreRepositoryImpl) FindAllDockerArtifactCount() (int, error) { + dockerArtifactCount, err := impl.dbConnection.Model(&DockerArtifactStore{}). + Column("docker_artifact_store.*", "IpsConfig", "OCIRegistryConfig"). + Where("docker_artifact_store.active = ?", true). + Relation("OCIRegistryConfig", func(q *orm.Query) (query *orm.Query, err error) { + return q.Where("deleted IS FALSE"), nil + }). + Relation("IpsConfig", func(q *orm.Query) (query *orm.Query, err error) { + return q.JoinOn("(ips_config.active=true or ips_config is null)"), nil + }). + Count() + return dockerArtifactCount, err +} func (impl DockerArtifactStoreRepositoryImpl) FindAllChartProviders() ([]DockerArtifactStore, error) { var providers []DockerArtifactStore err := impl.dbConnection.Model(&providers). diff --git a/internal/sql/repository/pipelineConfig/CdWorfkflowRepository.go b/internal/sql/repository/pipelineConfig/CdWorfkflowRepository.go index 96bed91f8e..809bf6654c 100644 --- a/internal/sql/repository/pipelineConfig/CdWorfkflowRepository.go +++ b/internal/sql/repository/pipelineConfig/CdWorfkflowRepository.go @@ -57,7 +57,7 @@ type CdWorkflowRepository interface { FindLatestWfrByAppIdAndEnvironmentId(appId int, environmentId int) (*CdWorkflowRunner, error) IsLatestCDWfr(pipelineId, wfrId int) (bool, error) FindLatestCdWorkflowRunnerByEnvironmentIdAndRunnerType(appId int, environmentId int, runnerType bean.WorkflowType) (CdWorkflowRunner, error) - FindAllTriggeredWorkflowInLast24Hour() (cdWorkflowCount int, err error) + FindAllTriggeredWorkflowCountInLast24Hour() (cdWorkflowCount int, err error) GetConnection() *pg.DB FindLastPreOrPostTriggeredByPipelineId(pipelineId int) (CdWorkflowRunner, error) @@ -357,19 +357,19 @@ func (impl *CdWorkflowRepositoryImpl) FindLatestCdWorkflowByPipelineIdV2(pipelin // TODO - Group By Environment And Pipeline will get latest pipeline from top return cdWorkflow, err } -func (impl *CdWorkflowRepositoryImpl) FindAllTriggeredWorkflowInLast24Hour() (cdWorkflowCount int, err error) { - var wfrList []int - err = impl.dbConnection. +func (impl *CdWorkflowRepositoryImpl) FindAllTriggeredWorkflowCountInLast24Hour() (cdWorkflowCount int, err error) { + //var wfrList []int + cnt, err := impl.dbConnection. Model(&CdWorkflow{}). ColumnExpr("DISTINCT pipeline_id"). Join("JOIN cd_workflow_runner ON cd_workflow.id = cd_workflow_runner.cd_workflow_id"). Where("cd_workflow_runner.workflow_type = ? AND cd_workflow_runner.started_on > ?", bean.CD_WORKFLOW_TYPE_DEPLOY, time.Now().AddDate(0, 0, -1)). Group("cd_workflow.pipeline_id"). - Select(&wfrList) + Count() if err != nil { impl.logger.Errorw("error occurred while fetching cd workflow", "err", err) } - return len(wfrList), err + return cnt, err } func (impl *CdWorkflowRepositoryImpl) FindCdWorkflowMetaByEnvironmentId(appId int, environmentId int, offset int, limit int) ([]CdWorkflowRunner, error) { var wfrList []CdWorkflowRunner diff --git a/internal/sql/repository/pipelineConfig/CiPipelineRepository.go b/internal/sql/repository/pipelineConfig/CiPipelineRepository.go index ed8f75ded1..d9cef59870 100644 --- a/internal/sql/repository/pipelineConfig/CiPipelineRepository.go +++ b/internal/sql/repository/pipelineConfig/CiPipelineRepository.go @@ -128,8 +128,8 @@ type CiPipelineRepository interface { FetchParentCiPipelinesForDG() ([]*bean.CiPipelinesMap, error) FetchCiPipelinesForDG(parentId int, childCiPipelineIds []int) (*CiPipeline, int, error) FinDByParentCiPipelineAndAppId(parentCiPipeline int, appIds []int) ([]*CiPipeline, error) - FindAllPipelineInLast24Hour() (pipelines []*CiPipeline, err error) - FindAllDeletedPipelineInLast24Hour() (pipelines []*CiPipeline, err error) + FindAllPipelineCountInLast24Hour() (pipelineCount int, err error) + FindAllDeletedPipelineCountInLast24Hour() (pipelineCount int, err error) FindNumberOfAppsWithCiPipeline(appIds []int) (count int, err error) FindAppAndProjectByCiPipelineIds(ciPipelineIds []int) ([]*CiPipeline, error) FindCiPipelineConfigsByIds(ids []int) ([]*CiPipeline, error) @@ -503,19 +503,19 @@ func (impl *CiPipelineRepositoryImpl) FinDByParentCiPipelineAndAppId(parentCiPip return ciPipelines, err } -func (impl *CiPipelineRepositoryImpl) FindAllPipelineInLast24Hour() (pipelines []*CiPipeline, err error) { - err = impl.dbConnection.Model(&pipelines). +func (impl *CiPipelineRepositoryImpl) FindAllPipelineCountInLast24Hour() (pipelineCount int, err error) { + pipelineCount, err = impl.dbConnection.Model(&CiPipeline{}). Column("ci_pipeline.*"). Where("created_on > ?", time.Now().AddDate(0, 0, -1)). - Select() - return pipelines, err + Count() + return pipelineCount, err } -func (impl *CiPipelineRepositoryImpl) FindAllDeletedPipelineInLast24Hour() (pipelines []*CiPipeline, err error) { - err = impl.dbConnection.Model(&pipelines). +func (impl *CiPipelineRepositoryImpl) FindAllDeletedPipelineCountInLast24Hour() (pipelineCount int, err error) { + pipelineCount, err = impl.dbConnection.Model(&CiPipeline{}). Column("ci_pipeline.*"). Where("created_on > ? and deleted=?", time.Now().AddDate(0, 0, -1), true). - Select() - return pipelines, err + Count() + return pipelineCount, err } func (impl *CiPipelineRepositoryImpl) FindNumberOfAppsWithCiPipeline(appIds []int) (count int, err error) { diff --git a/internal/sql/repository/pipelineConfig/CiWorkflowRepository.go b/internal/sql/repository/pipelineConfig/CiWorkflowRepository.go index 6acdfb621f..e814e72523 100644 --- a/internal/sql/repository/pipelineConfig/CiWorkflowRepository.go +++ b/internal/sql/repository/pipelineConfig/CiWorkflowRepository.go @@ -43,7 +43,7 @@ type CiWorkflowRepository interface { FindLastTriggeredWorkflowByCiIds(pipelineId []int) (ciWorkflow []*CiWorkflow, err error) FindLastTriggeredWorkflowByArtifactId(ciArtifactId int) (ciWorkflow *CiWorkflow, err error) FindAllLastTriggeredWorkflowByArtifactId(ciArtifactId []int) (ciWorkflow []*CiWorkflow, err error) - FindAllTriggeredWorkflowInLast24Hour() (ciWorkflowCount int, err error) + FindAllTriggeredWorkflowCountInLast24Hour() (ciWorkflowCount int, err error) FindLastTriggeredWorkflowGitTriggersByArtifactId(ciArtifactId int) (ciWorkflow *CiWorkflow, err error) FindLastTriggeredWorkflowGitTriggersByArtifactIds(ciArtifactIds []int) ([]*WorkflowWithArtifact, error) ExistsByStatus(status string) (bool, error) @@ -287,16 +287,16 @@ func (impl *CiWorkflowRepositoryImpl) FindLastTriggeredWorkflowByArtifactId(ciAr Select() return workflow, err } -func (impl *CiWorkflowRepositoryImpl) FindAllTriggeredWorkflowInLast24Hour() (ciWorkflowCount int, err error) { - var wfrList []CiWorkflow - err = impl.dbConnection.Model(&wfrList). +func (impl *CiWorkflowRepositoryImpl) FindAllTriggeredWorkflowCountInLast24Hour() (ciWorkflowCount int, err error) { + //var wfrList []CiWorkflow + cnt, err := impl.dbConnection.Model(&CiWorkflow{}). ColumnExpr("DISTINCT ci_pipeline_id"). Where("started_on > ? ", time.Now().AddDate(0, 0, -1)). - Select() + Count() if err != nil { impl.logger.Errorw("error occurred while fetching ci workflow", "err", err) } - return len(wfrList), err + return cnt, err } func (impl *CiWorkflowRepositoryImpl) FindAllLastTriggeredWorkflowByArtifactId(ciArtifactIds []int) (ciWorkflows []*CiWorkflow, err error) { err = impl.dbConnection.Model(&ciWorkflows). diff --git a/internal/sql/repository/pipelineConfig/PipelineRepository.go b/internal/sql/repository/pipelineConfig/PipelineRepository.go index 06868a77e4..0c3bba54d8 100644 --- a/internal/sql/repository/pipelineConfig/PipelineRepository.go +++ b/internal/sql/repository/pipelineConfig/PipelineRepository.go @@ -93,8 +93,8 @@ type PipelineRepository interface { FindByIdsInAndEnvironment(ids []int, environmentId int) ([]*Pipeline, error) FindActiveByAppIdAndEnvironmentIdV2() (pipelines []*Pipeline, err error) GetConnection() *pg.DB - FindAllPipelineInLast24Hour() (pipelines []*Pipeline, err error) - FindAllDeletedPipelineInLast24Hour() (pipelines []*Pipeline, err error) + FindAllPipelineCountInLast24Hour() (pipelineCount int, err error) + FindAllDeletedPipelineCountInLast24Hour() (pipelineCount int, err error) FindActiveByEnvId(envId int) (pipelines []*Pipeline, err error) FindActivePipelineByEnvId(envId int) (pipelines []*Pipeline, err error) FindActiveByEnvIds(envId []int) (pipelines []*Pipeline, err error) @@ -448,19 +448,19 @@ func (impl PipelineRepositoryImpl) FindByPipelineTriggerGitHash(gitHash string) return pipelines, err } -func (impl PipelineRepositoryImpl) FindAllPipelineInLast24Hour() (pipelines []*Pipeline, err error) { - err = impl.dbConnection.Model(&pipelines). +func (impl PipelineRepositoryImpl) FindAllPipelineCountInLast24Hour() (pipelineCount int, err error) { + pipelineCount, err = impl.dbConnection.Model(&Pipeline{}). Column("pipeline.*"). Where("created_on > ?", time.Now().AddDate(0, 0, -1)). - Select() - return pipelines, err + Count() + return pipelineCount, err } -func (impl PipelineRepositoryImpl) FindAllDeletedPipelineInLast24Hour() (pipelines []*Pipeline, err error) { - err = impl.dbConnection.Model(&pipelines). +func (impl PipelineRepositoryImpl) FindAllDeletedPipelineCountInLast24Hour() (pipelineCount int, err error) { + pipelineCount, err = impl.dbConnection.Model(&Pipeline{}). Column("pipeline.*"). Where("created_on > ? and deleted=?", time.Now().AddDate(0, 0, -1), true). - Select() - return pipelines, err + Count() + return pipelineCount, err } func (impl PipelineRepositoryImpl) FindActiveByEnvId(envId int) (pipelines []*Pipeline, err error) { err = impl.dbConnection.Model(&pipelines).Column("pipeline.*", "App", "Environment"). diff --git a/pkg/cluster/EnvironmentService.go b/pkg/cluster/EnvironmentService.go index 43740137bd..02f4a16b1e 100644 --- a/pkg/cluster/EnvironmentService.go +++ b/pkg/cluster/EnvironmentService.go @@ -46,6 +46,7 @@ type EnvironmentService interface { Update(mappings *bean2.EnvironmentBean, userId int32) (*bean2.EnvironmentBean, error) GetAll() ([]bean2.EnvironmentBean, error) GetAllActive() ([]bean2.EnvironmentBean, error) + GetAllActiveEnvironmentCount() (int, error) Delete(deleteReq *bean2.EnvironmentBean, userId int32) error FindClusterByEnvId(id int) (*ClusterBean, error) // FindById provides an exposed struct of bean.EnvironmentBean; @@ -231,7 +232,10 @@ func (impl EnvironmentServiceImpl) GetAllActive() ([]bean2.EnvironmentBean, erro } return beans, nil } - +func (impl EnvironmentServiceImpl) GetAllActiveEnvironmentCount() (int, error) { + cnt, err := impl.environmentRepository.FindAllActiveEnvironmentCount() + return cnt, err +} func (impl EnvironmentServiceImpl) FindById(id int) (*bean2.EnvironmentBean, error) { model, err := impl.environmentRepository.FindById(id) if err != nil { diff --git a/pkg/cluster/repository/EnvironmentRepository.go b/pkg/cluster/repository/EnvironmentRepository.go index 19b9c9c3d5..d3d7a9ade9 100644 --- a/pkg/cluster/repository/EnvironmentRepository.go +++ b/pkg/cluster/repository/EnvironmentRepository.go @@ -54,6 +54,7 @@ type EnvironmentRepository interface { Create(mappings *Environment) error FindAll() ([]Environment, error) FindAllActive() ([]*Environment, error) + FindAllActiveEnvironmentCount() (int, error) MarkEnvironmentDeleted(mappings *Environment, tx *pg.Tx) error GetConnection() (dbConnection *pg.DB) FindAllActiveEnvOnlyDetails() ([]*Environment, error) @@ -256,6 +257,15 @@ func (repositoryImpl EnvironmentRepositoryImpl) FindAllActive() ([]*Environment, Select() return mappings, err } +func (repositoryImpl EnvironmentRepositoryImpl) FindAllActiveEnvironmentCount() (int, error) { + cnt, err := repositoryImpl. + dbConnection.Model(&Environment{}). + Where("environment.active = ?", true). + Column("environment.*", "Cluster"). + Join("inner join cluster c on environment.cluster_id = c.id"). + Count() + return cnt, err +} func (repositoryImpl EnvironmentRepositoryImpl) FindAllActiveEnvOnlyDetails() ([]*Environment, error) { var mappings []*Environment err := repositoryImpl. diff --git a/pkg/deployment/gitOps/config/GitOpsConfigReadService.go b/pkg/deployment/gitOps/config/GitOpsConfigReadService.go index 519f5e5cf0..d292bc4a59 100644 --- a/pkg/deployment/gitOps/config/GitOpsConfigReadService.go +++ b/pkg/deployment/gitOps/config/GitOpsConfigReadService.go @@ -139,12 +139,13 @@ func (impl *GitOpsConfigReadServiceImpl) GetGitOpsConfigActive() (*bean2.GitOpsC } func (impl *GitOpsConfigReadServiceImpl) GetConfiguredGitOpsCount() (int, error) { - count := 0 - models, err := impl.gitOpsRepository.GetAllGitOpsConfig() - if err != nil && err != pg.ErrNoRows { - impl.logger.Errorw("error, GetGitOpsConfigActive", "err", err) - return count, err - } - count = len(models) - return count, nil + //count := 0 + //models, err := impl.gitOpsRepository.GetAllGitOpsConfig() + //if err != nil && err != pg.ErrNoRows { + // impl.logger.Errorw("error, GetGitOpsConfigActive", "err", err) + // return count, err + //} + //count = len(models) + count, err := impl.gitOpsRepository.GetAllGitOpsConfigCount() + return count, err } diff --git a/util/GlobalConfig.go b/util/GlobalConfig.go index 0a106123a9..5728015bef 100644 --- a/util/GlobalConfig.go +++ b/util/GlobalConfig.go @@ -8,11 +8,8 @@ type EnvironmentVariables struct { GlobalEnvVariables *GlobalEnvVariables DevtronSecretConfig *DevtronSecretConfig DeploymentServiceTypeConfig *DeploymentServiceTypeConfig - PostHogClientConfig *PostHogClientConfig -} -type PostHogClientConfig struct { - SummaryCron string `env:"POSTHOG_SUMMARY_CRON_EXPR" envDefault:"0 0 * * *"` // Default Run once a day, midnight } + type DeploymentServiceTypeConfig struct { ExternallyManagedDeploymentType bool `env:"IS_INTERNAL_USE" envDefault:"false"` HelmInstallASyncMode bool `env:"RUN_HELM_INSTALL_IN_ASYNC_MODE_HELM_APPS" envDefault:"false"` @@ -34,7 +31,6 @@ func GetEnvironmentVariables() (*EnvironmentVariables, error) { GlobalEnvVariables: &GlobalEnvVariables{}, DevtronSecretConfig: &DevtronSecretConfig{}, DeploymentServiceTypeConfig: &DeploymentServiceTypeConfig{}, - PostHogClientConfig: &PostHogClientConfig{}, } err := env.Parse(cfg) if err != nil { diff --git a/wire_gen.go b/wire_gen.go index a497d2deb1..8717462e38 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -745,7 +745,7 @@ func InitializeApp() (*App, error) { return nil, err } providerIdentifierServiceImpl := providerIdentifier.NewProviderIdentifierServiceImpl(sugaredLogger) - telemetryEventClientImplExtended, err := telemetry.NewTelemetryEventClientImplExtended(sugaredLogger, httpClient, clusterServiceImplExtended, k8sServiceImpl, acdAuthConfig, environmentServiceImpl, userServiceImpl, appListingRepositoryImpl, posthogClient, ciPipelineRepositoryImpl, pipelineRepositoryImpl, gitProviderRepositoryImpl, attributesRepositoryImpl, ssoLoginServiceImpl, appRepositoryImpl, ciWorkflowRepositoryImpl, cdWorkflowRepositoryImpl, dockerArtifactStoreRepositoryImpl, materialRepositoryImpl, ciTemplateRepositoryImpl, chartRepositoryImpl, userAuditServiceImpl, ciBuildConfigServiceImpl, moduleRepositoryImpl, serverDataStoreServerDataStore, helmAppClientImpl, installedAppRepositoryImpl, userAttributesRepositoryImpl, providerIdentifierServiceImpl, cronLoggerImpl, gitOpsConfigReadServiceImpl, environmentVariables) + telemetryEventClientImplExtended, err := telemetry.NewTelemetryEventClientImplExtended(sugaredLogger, httpClient, clusterServiceImplExtended, k8sServiceImpl, acdAuthConfig, environmentServiceImpl, userServiceImpl, appListingRepositoryImpl, posthogClient, ciPipelineRepositoryImpl, pipelineRepositoryImpl, gitProviderRepositoryImpl, attributesRepositoryImpl, ssoLoginServiceImpl, appRepositoryImpl, ciWorkflowRepositoryImpl, cdWorkflowRepositoryImpl, dockerArtifactStoreRepositoryImpl, materialRepositoryImpl, ciTemplateRepositoryImpl, chartRepositoryImpl, userAuditServiceImpl, ciBuildConfigServiceImpl, moduleRepositoryImpl, serverDataStoreServerDataStore, helmAppClientImpl, installedAppRepositoryImpl, userAttributesRepositoryImpl, providerIdentifierServiceImpl, cronLoggerImpl, gitOpsConfigReadServiceImpl) if err != nil { return nil, err } From 4dde44771416476c3e8adf1e4b9438126cefca2d Mon Sep 17 00:00:00 2001 From: KomalReddyK Date: Thu, 18 Apr 2024 16:56:31 +0530 Subject: [PATCH 07/14] type fix --- client/telemetry/PosthogClient.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/telemetry/PosthogClient.go b/client/telemetry/PosthogClient.go index 3ee4694100..41302aeefb 100644 --- a/client/telemetry/PosthogClient.go +++ b/client/telemetry/PosthogClient.go @@ -36,7 +36,7 @@ type PosthogClient struct { var ( PosthogApiKey string = "" PosthogEndpoint string = "https://app.posthog.com" - SummaryCronExpr string = "0 0 * * *" // Default Run once a day, midnight + SummaryCronExpr string = "0 0 * * *" // Run once a day, midnight HeartbeatCronExpr string = "0 0/6 * * *" CacheExpiry int = 1440 PosthogEncodedApiKey string = "" From 398c78d36ba7ea1c614f6b819ff3d5fce2c9f00b Mon Sep 17 00:00:00 2001 From: komalreddy3 Date: Thu, 18 Apr 2024 17:01:21 +0530 Subject: [PATCH 08/14] remove comments --- .../sql/repository/pipelineConfig/CdWorfkflowRepository.go | 1 - .../sql/repository/pipelineConfig/CiWorkflowRepository.go | 1 - pkg/deployment/gitOps/config/GitOpsConfigReadService.go | 7 ------- 3 files changed, 9 deletions(-) diff --git a/internal/sql/repository/pipelineConfig/CdWorfkflowRepository.go b/internal/sql/repository/pipelineConfig/CdWorfkflowRepository.go index 809bf6654c..9e5cf2abcd 100644 --- a/internal/sql/repository/pipelineConfig/CdWorfkflowRepository.go +++ b/internal/sql/repository/pipelineConfig/CdWorfkflowRepository.go @@ -358,7 +358,6 @@ func (impl *CdWorkflowRepositoryImpl) FindLatestCdWorkflowByPipelineIdV2(pipelin return cdWorkflow, err } func (impl *CdWorkflowRepositoryImpl) FindAllTriggeredWorkflowCountInLast24Hour() (cdWorkflowCount int, err error) { - //var wfrList []int cnt, err := impl.dbConnection. Model(&CdWorkflow{}). ColumnExpr("DISTINCT pipeline_id"). diff --git a/internal/sql/repository/pipelineConfig/CiWorkflowRepository.go b/internal/sql/repository/pipelineConfig/CiWorkflowRepository.go index e814e72523..7044f77ee6 100644 --- a/internal/sql/repository/pipelineConfig/CiWorkflowRepository.go +++ b/internal/sql/repository/pipelineConfig/CiWorkflowRepository.go @@ -288,7 +288,6 @@ func (impl *CiWorkflowRepositoryImpl) FindLastTriggeredWorkflowByArtifactId(ciAr return workflow, err } func (impl *CiWorkflowRepositoryImpl) FindAllTriggeredWorkflowCountInLast24Hour() (ciWorkflowCount int, err error) { - //var wfrList []CiWorkflow cnt, err := impl.dbConnection.Model(&CiWorkflow{}). ColumnExpr("DISTINCT ci_pipeline_id"). Where("started_on > ? ", time.Now().AddDate(0, 0, -1)). diff --git a/pkg/deployment/gitOps/config/GitOpsConfigReadService.go b/pkg/deployment/gitOps/config/GitOpsConfigReadService.go index d292bc4a59..4f98046e05 100644 --- a/pkg/deployment/gitOps/config/GitOpsConfigReadService.go +++ b/pkg/deployment/gitOps/config/GitOpsConfigReadService.go @@ -139,13 +139,6 @@ func (impl *GitOpsConfigReadServiceImpl) GetGitOpsConfigActive() (*bean2.GitOpsC } func (impl *GitOpsConfigReadServiceImpl) GetConfiguredGitOpsCount() (int, error) { - //count := 0 - //models, err := impl.gitOpsRepository.GetAllGitOpsConfig() - //if err != nil && err != pg.ErrNoRows { - // impl.logger.Errorw("error, GetGitOpsConfigActive", "err", err) - // return count, err - //} - //count = len(models) count, err := impl.gitOpsRepository.GetAllGitOpsConfigCount() return count, err } From 21fedc63377232e6ef54cf37cd7f2abb60fdc80a Mon Sep 17 00:00:00 2001 From: komalreddy3 Date: Fri, 19 Apr 2024 13:26:43 +0530 Subject: [PATCH 09/14] refactor query --- .../dockerRegistry/DockerArtifactStoreRepository.go | 8 +------- .../sql/repository/pipelineConfig/CiPipelineRepository.go | 6 +++--- .../sql/repository/pipelineConfig/PipelineRepository.go | 4 ++-- pkg/cluster/repository/EnvironmentRepository.go | 3 +-- 4 files changed, 7 insertions(+), 14 deletions(-) diff --git a/internal/sql/repository/dockerRegistry/DockerArtifactStoreRepository.go b/internal/sql/repository/dockerRegistry/DockerArtifactStoreRepository.go index 74f777a39e..d5fd286ff6 100644 --- a/internal/sql/repository/dockerRegistry/DockerArtifactStoreRepository.go +++ b/internal/sql/repository/dockerRegistry/DockerArtifactStoreRepository.go @@ -172,14 +172,8 @@ func (impl DockerArtifactStoreRepositoryImpl) FindAll() ([]DockerArtifactStore, } func (impl DockerArtifactStoreRepositoryImpl) FindAllDockerArtifactCount() (int, error) { dockerArtifactCount, err := impl.dbConnection.Model(&DockerArtifactStore{}). - Column("docker_artifact_store.*", "IpsConfig", "OCIRegistryConfig"). + Column("docker_artifact_store.*"). Where("docker_artifact_store.active = ?", true). - Relation("OCIRegistryConfig", func(q *orm.Query) (query *orm.Query, err error) { - return q.Where("deleted IS FALSE"), nil - }). - Relation("IpsConfig", func(q *orm.Query) (query *orm.Query, err error) { - return q.JoinOn("(ips_config.active=true or ips_config is null)"), nil - }). Count() return dockerArtifactCount, err } diff --git a/internal/sql/repository/pipelineConfig/CiPipelineRepository.go b/internal/sql/repository/pipelineConfig/CiPipelineRepository.go index d9cef59870..6443b39d34 100644 --- a/internal/sql/repository/pipelineConfig/CiPipelineRepository.go +++ b/internal/sql/repository/pipelineConfig/CiPipelineRepository.go @@ -35,7 +35,7 @@ import ( ) type CiPipeline struct { - tableName struct{} `sql:"ci_pipeline" pg:",discard_unknown_columns"` + tableName struct{} `sql:"ci_pipelinegggg" pg:",discard_unknown_columns"` Id int `sql:"id,pk"` AppId int `sql:"app_id"` App *app.App @@ -505,14 +505,14 @@ func (impl *CiPipelineRepositoryImpl) FinDByParentCiPipelineAndAppId(parentCiPip func (impl *CiPipelineRepositoryImpl) FindAllPipelineCountInLast24Hour() (pipelineCount int, err error) { pipelineCount, err = impl.dbConnection.Model(&CiPipeline{}). - Column("ci_pipeline.*"). + Column("ci_pipeline.id"). Where("created_on > ?", time.Now().AddDate(0, 0, -1)). Count() return pipelineCount, err } func (impl *CiPipelineRepositoryImpl) FindAllDeletedPipelineCountInLast24Hour() (pipelineCount int, err error) { pipelineCount, err = impl.dbConnection.Model(&CiPipeline{}). - Column("ci_pipeline.*"). + Column("ci_pipeline.id"). Where("created_on > ? and deleted=?", time.Now().AddDate(0, 0, -1), true). Count() return pipelineCount, err diff --git a/internal/sql/repository/pipelineConfig/PipelineRepository.go b/internal/sql/repository/pipelineConfig/PipelineRepository.go index 0c3bba54d8..4877be3048 100644 --- a/internal/sql/repository/pipelineConfig/PipelineRepository.go +++ b/internal/sql/repository/pipelineConfig/PipelineRepository.go @@ -450,14 +450,14 @@ func (impl PipelineRepositoryImpl) FindByPipelineTriggerGitHash(gitHash string) func (impl PipelineRepositoryImpl) FindAllPipelineCountInLast24Hour() (pipelineCount int, err error) { pipelineCount, err = impl.dbConnection.Model(&Pipeline{}). - Column("pipeline.*"). + Column("pipeline.id"). Where("created_on > ?", time.Now().AddDate(0, 0, -1)). Count() return pipelineCount, err } func (impl PipelineRepositoryImpl) FindAllDeletedPipelineCountInLast24Hour() (pipelineCount int, err error) { pipelineCount, err = impl.dbConnection.Model(&Pipeline{}). - Column("pipeline.*"). + Column("pipeline.id"). Where("created_on > ? and deleted=?", time.Now().AddDate(0, 0, -1), true). Count() return pipelineCount, err diff --git a/pkg/cluster/repository/EnvironmentRepository.go b/pkg/cluster/repository/EnvironmentRepository.go index d3d7a9ade9..94677042d6 100644 --- a/pkg/cluster/repository/EnvironmentRepository.go +++ b/pkg/cluster/repository/EnvironmentRepository.go @@ -261,8 +261,7 @@ func (repositoryImpl EnvironmentRepositoryImpl) FindAllActiveEnvironmentCount() cnt, err := repositoryImpl. dbConnection.Model(&Environment{}). Where("environment.active = ?", true). - Column("environment.*", "Cluster"). - Join("inner join cluster c on environment.cluster_id = c.id"). + Column("environment.id"). Count() return cnt, err } From 19b9503f28a90f3ce917d53c6e2df29f95f6708b Mon Sep 17 00:00:00 2001 From: komalreddy3 Date: Fri, 19 Apr 2024 13:35:25 +0530 Subject: [PATCH 10/14] typo --- internal/sql/repository/pipelineConfig/CiPipelineRepository.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/sql/repository/pipelineConfig/CiPipelineRepository.go b/internal/sql/repository/pipelineConfig/CiPipelineRepository.go index 6443b39d34..a9a2b1d342 100644 --- a/internal/sql/repository/pipelineConfig/CiPipelineRepository.go +++ b/internal/sql/repository/pipelineConfig/CiPipelineRepository.go @@ -35,7 +35,7 @@ import ( ) type CiPipeline struct { - tableName struct{} `sql:"ci_pipelinegggg" pg:",discard_unknown_columns"` + tableName struct{} `sql:"ci_pipeline" pg:",discard_unknown_columns"` Id int `sql:"id,pk"` AppId int `sql:"app_id"` App *app.App From abf8262782136857385e35270f6b18deee01eb52 Mon Sep 17 00:00:00 2001 From: komalreddy3 Date: Fri, 19 Apr 2024 13:43:17 +0530 Subject: [PATCH 11/14] docker query --- .../repository/dockerRegistry/DockerArtifactStoreRepository.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/sql/repository/dockerRegistry/DockerArtifactStoreRepository.go b/internal/sql/repository/dockerRegistry/DockerArtifactStoreRepository.go index d5fd286ff6..e6838f91ee 100644 --- a/internal/sql/repository/dockerRegistry/DockerArtifactStoreRepository.go +++ b/internal/sql/repository/dockerRegistry/DockerArtifactStoreRepository.go @@ -172,7 +172,7 @@ func (impl DockerArtifactStoreRepositoryImpl) FindAll() ([]DockerArtifactStore, } func (impl DockerArtifactStoreRepositoryImpl) FindAllDockerArtifactCount() (int, error) { dockerArtifactCount, err := impl.dbConnection.Model(&DockerArtifactStore{}). - Column("docker_artifact_store.*"). + Column("docker_artifact_store.id"). Where("docker_artifact_store.active = ?", true). Count() return dockerArtifactCount, err From 55e512045575dccbcbd30b8925bfbc4db9e6f4e2 Mon Sep 17 00:00:00 2001 From: komalreddy3 Date: Fri, 19 Apr 2024 15:56:18 +0530 Subject: [PATCH 12/14] refactor --- .../telemetry/TelemetryEventClientExtended.go | 42 +++++++++---------- .../sql/repository/GitOpsConfigRepository.go | 2 +- .../pipelineConfig/CiPipelineRepository.go | 6 +-- .../pipelineConfig/PipelineRepository.go | 6 +-- .../repository/EnvironmentRepository.go | 1 - 5 files changed, 26 insertions(+), 31 deletions(-) diff --git a/client/telemetry/TelemetryEventClientExtended.go b/client/telemetry/TelemetryEventClientExtended.go index 154714ee46..424512691d 100644 --- a/client/telemetry/TelemetryEventClientExtended.go +++ b/client/telemetry/TelemetryEventClientExtended.go @@ -191,7 +191,7 @@ func (impl *TelemetryEventClientImplExtended) SendSummaryEvent(eventType string) impl.logger.Infow("sending summary event", "eventType", eventType) ucid, err := impl.getUCID() if err != nil { - impl.logger.Errorw("exception caught inside telemetry summary event", "err", err) + impl.logger.Errorw("exception caught inside telemetry summary event while fetching ucid", "err", err) return err } @@ -206,75 +206,75 @@ func (impl *TelemetryEventClientImplExtended) SendSummaryEvent(eventType string) environmentCount, err := impl.environmentService.GetAllActiveEnvironmentCount() if err != nil && err != pg.ErrNoRows { - impl.logger.Errorw("exception caught inside telemetry summary event, setting value to -1", "err", err) + impl.logger.Errorw("exception caught inside telemetry summary event while retrieving environmentCount, setting its value to -1", "err", err) environmentCount = -1 } prodApps, err := impl.appListingRepository.FindAppCount(true) if err != nil && err != pg.ErrNoRows { - impl.logger.Errorw("exception caught inside telemetry summary event, setting value to -1", "err", err) + impl.logger.Errorw("exception caught inside telemetry summary event, while retrieving prodApps, setting its value to -1", "err", err) prodApps = -1 } nonProdApps, err := impl.appListingRepository.FindAppCount(false) if err != nil && err != pg.ErrNoRows { - impl.logger.Errorw("exception caught inside telemetry summary event, setting value to -1", "err", err) + impl.logger.Errorw("exception caught inside telemetry summary event,while retrieving nonProdApps, setting its value to -1", "err", err) nonProdApps = -1 } - ciPipelineCount, err := impl.ciPipelineRepository.FindAllPipelineCountInLast24Hour() + ciPipelineCount, err := impl.ciPipelineRepository.FindAllPipelineCreatedCountInLast24Hour() if err != nil && err != pg.ErrNoRows { - impl.logger.Errorw("exception caught inside telemetry summary event, setting value to -1", "err", err) + impl.logger.Errorw("exception caught inside telemetry summary event, while retrieving ciPipelineCount, setting its value to -1", "err", err) ciPipelineCount = -1 } ciPipelineDeletedCount, err := impl.ciPipelineRepository.FindAllDeletedPipelineCountInLast24Hour() if err != nil && err != pg.ErrNoRows { - impl.logger.Errorw("exception caught inside telemetry summary event, setting value to -1", "err", err) + impl.logger.Errorw("exception caught inside telemetry summary event, while retrieving ciPipelineDeletedCount, setting its value to -1", "err", err) ciPipelineDeletedCount = -1 } ciPipelineTriggeredCount, err := impl.ciWorkflowRepository.FindAllTriggeredWorkflowCountInLast24Hour() if err != nil && err != pg.ErrNoRows { - impl.logger.Errorw("exception caught inside telemetry summary event, setting value to -1", "err", err) + impl.logger.Errorw("exception caught inside telemetry summary event, while retrieving ciPipelineTriggeredCount, setting its value to -1", "err", err) ciPipelineTriggeredCount = -1 } - cdPipelineCount, err := impl.pipelineRepository.FindAllPipelineCountInLast24Hour() + cdPipelineCount, err := impl.pipelineRepository.FindAllPipelineCreatedCountInLast24Hour() if err != nil && err != pg.ErrNoRows { - impl.logger.Errorw("exception caught inside telemetry summary event, setting value to -1", "err", err) + impl.logger.Errorw("exception caught inside telemetry summary event, while retrieving cdPipelineCount, setting its value to -1", "err", err) cdPipelineCount = -1 } cdPipelineDeletedCount, err := impl.pipelineRepository.FindAllDeletedPipelineCountInLast24Hour() if err != nil && err != pg.ErrNoRows { - impl.logger.Errorw("exception caught inside telemetry summary event, setting value to -1", "err", err) + impl.logger.Errorw("exception caught inside telemetry summary event, while retrieving cdPipelineDeletedCount, setting its value to -1", "err", err) cdPipelineDeletedCount = -1 } cdPipelineTriggeredCount, err := impl.cdWorkflowRepository.FindAllTriggeredWorkflowCountInLast24Hour() if err != nil && err != pg.ErrNoRows { - impl.logger.Errorw("exception caught inside telemetry summary event, setting value to -1", "err", err) + impl.logger.Errorw("exception caught inside telemetry summary event, while retrieving cdPipelineTriggeredCount, setting its value to -1", "err", err) cdPipelineTriggeredCount = -1 } gitAccounts, err := impl.gitProviderRepository.FindAllGitProviderCount() if err != nil && err != pg.ErrNoRows { - impl.logger.Errorw("exception caught inside telemetry summary event, setting value to -1", "err", err) + impl.logger.Errorw("exception caught inside telemetry summary event, while retrieving gitAccounts, setting its value to -1", "err", err) gitAccounts = -1 } gitOpsCount, err := impl.gitOpsConfigReadService.GetConfiguredGitOpsCount() if err != nil { - impl.logger.Errorw("exception caught inside telemetry summary event,setting value to -1", "err", err) + impl.logger.Errorw("exception caught inside telemetry summary event,while retrieving gitOpsCount, setting its value to -1", "err", err) gitOpsCount = -1 } containerRegistryCount, err := impl.dockerArtifactStoreRepository.FindAllDockerArtifactCount() if err != nil && err != pg.ErrNoRows { - impl.logger.Errorw("exception caught inside telemetry summary event,setting value to -1", "err", err) + impl.logger.Errorw("exception caught inside telemetry summary event,while retrieving containerRegistryCount, setting its value to -1", "err", err) containerRegistryCount = -1 } //appSetup := false apps, err := impl.appRepository.FindAll() if err != nil { - impl.logger.Errorw("exception caught inside telemetry summary event", "err", err) + impl.logger.Errorw("exception caught inside telemetry summary event,while retrieving apps", "err", err) return err } @@ -287,24 +287,24 @@ func (impl *TelemetryEventClientImplExtended) SendSummaryEvent(eventType string) if len(appIds) < AppsCount { payload.AppsWithGitRepoConfigured, err = impl.materialRepository.FindNumberOfAppsWithGitRepo(appIds) if err != nil { - impl.logger.Errorw("exception caught inside telemetry summary event", "err", err) + impl.logger.Errorw("exception caught inside telemetry summary event,while retrieving AppsWithGitRepoConfigured", "err", err) } payload.AppsWithDockerConfigured, err = impl.ciTemplateRepository.FindNumberOfAppsWithDockerConfigured(appIds) if err != nil { - impl.logger.Errorw("exception caught inside telemetry summary event", "err", err) + impl.logger.Errorw("exception caught inside telemetry summary event,while retrieving AppsWithDockerConfigured", "err", err) } payload.AppsWithDeploymentTemplateConfigured, err = impl.chartRepository.FindNumberOfAppsWithDeploymentTemplate(appIds) if err != nil { - impl.logger.Errorw("exception caught inside telemetry summary event", "err", err) + impl.logger.Errorw("exception caught inside telemetry summary event,while retrieving AppsWithDeploymentTemplateConfigured", "err", err) } payload.AppsWithCiPipelineConfigured, err = impl.ciPipelineRepository.FindNumberOfAppsWithCiPipeline(appIds) if err != nil { - impl.logger.Errorw("exception caught inside telemetry summary event", "err", err) + impl.logger.Errorw("exception caught inside telemetry summary event,while retrieving AppsWithCiPipelineConfigured", "err", err) } payload.AppsWithCdPipelineConfigured, err = impl.pipelineRepository.FindNumberOfAppsWithCdPipeline(appIds) if err != nil { - impl.logger.Errorw("exception caught inside telemetry summary event", "err", err) + impl.logger.Errorw("exception caught inside telemetry summary event,while retrieving AppsWithCdPipelineConfigured", "err", err) } } diff --git a/internal/sql/repository/GitOpsConfigRepository.go b/internal/sql/repository/GitOpsConfigRepository.go index 4d6f30787a..4cb3a23046 100644 --- a/internal/sql/repository/GitOpsConfigRepository.go +++ b/internal/sql/repository/GitOpsConfigRepository.go @@ -93,7 +93,7 @@ func (impl *GitOpsConfigRepositoryImpl) GetAllGitOpsConfig() ([]*GitOpsConfig, e return userModel, err } func (impl *GitOpsConfigRepositoryImpl) GetAllGitOpsConfigCount() (int, error) { - cnt, err := impl.dbConnection.Model(&GitOpsConfig{}).Order("updated_on desc").Count() + cnt, err := impl.dbConnection.Model(&GitOpsConfig{}).Count() return cnt, err } func (impl *GitOpsConfigRepositoryImpl) GetGitOpsConfigByProvider(provider string) (*GitOpsConfig, error) { diff --git a/internal/sql/repository/pipelineConfig/CiPipelineRepository.go b/internal/sql/repository/pipelineConfig/CiPipelineRepository.go index a9a2b1d342..f16a4e6cf1 100644 --- a/internal/sql/repository/pipelineConfig/CiPipelineRepository.go +++ b/internal/sql/repository/pipelineConfig/CiPipelineRepository.go @@ -128,7 +128,7 @@ type CiPipelineRepository interface { FetchParentCiPipelinesForDG() ([]*bean.CiPipelinesMap, error) FetchCiPipelinesForDG(parentId int, childCiPipelineIds []int) (*CiPipeline, int, error) FinDByParentCiPipelineAndAppId(parentCiPipeline int, appIds []int) ([]*CiPipeline, error) - FindAllPipelineCountInLast24Hour() (pipelineCount int, err error) + FindAllPipelineCreatedCountInLast24Hour() (pipelineCount int, err error) FindAllDeletedPipelineCountInLast24Hour() (pipelineCount int, err error) FindNumberOfAppsWithCiPipeline(appIds []int) (count int, err error) FindAppAndProjectByCiPipelineIds(ciPipelineIds []int) ([]*CiPipeline, error) @@ -503,16 +503,14 @@ func (impl *CiPipelineRepositoryImpl) FinDByParentCiPipelineAndAppId(parentCiPip return ciPipelines, err } -func (impl *CiPipelineRepositoryImpl) FindAllPipelineCountInLast24Hour() (pipelineCount int, err error) { +func (impl *CiPipelineRepositoryImpl) FindAllPipelineCreatedCountInLast24Hour() (pipelineCount int, err error) { pipelineCount, err = impl.dbConnection.Model(&CiPipeline{}). - Column("ci_pipeline.id"). Where("created_on > ?", time.Now().AddDate(0, 0, -1)). Count() return pipelineCount, err } func (impl *CiPipelineRepositoryImpl) FindAllDeletedPipelineCountInLast24Hour() (pipelineCount int, err error) { pipelineCount, err = impl.dbConnection.Model(&CiPipeline{}). - Column("ci_pipeline.id"). Where("created_on > ? and deleted=?", time.Now().AddDate(0, 0, -1), true). Count() return pipelineCount, err diff --git a/internal/sql/repository/pipelineConfig/PipelineRepository.go b/internal/sql/repository/pipelineConfig/PipelineRepository.go index 4877be3048..d2176e4ec4 100644 --- a/internal/sql/repository/pipelineConfig/PipelineRepository.go +++ b/internal/sql/repository/pipelineConfig/PipelineRepository.go @@ -93,7 +93,7 @@ type PipelineRepository interface { FindByIdsInAndEnvironment(ids []int, environmentId int) ([]*Pipeline, error) FindActiveByAppIdAndEnvironmentIdV2() (pipelines []*Pipeline, err error) GetConnection() *pg.DB - FindAllPipelineCountInLast24Hour() (pipelineCount int, err error) + FindAllPipelineCreatedCountInLast24Hour() (pipelineCount int, err error) FindAllDeletedPipelineCountInLast24Hour() (pipelineCount int, err error) FindActiveByEnvId(envId int) (pipelines []*Pipeline, err error) FindActivePipelineByEnvId(envId int) (pipelines []*Pipeline, err error) @@ -448,16 +448,14 @@ func (impl PipelineRepositoryImpl) FindByPipelineTriggerGitHash(gitHash string) return pipelines, err } -func (impl PipelineRepositoryImpl) FindAllPipelineCountInLast24Hour() (pipelineCount int, err error) { +func (impl PipelineRepositoryImpl) FindAllPipelineCreatedCountInLast24Hour() (pipelineCount int, err error) { pipelineCount, err = impl.dbConnection.Model(&Pipeline{}). - Column("pipeline.id"). Where("created_on > ?", time.Now().AddDate(0, 0, -1)). Count() return pipelineCount, err } func (impl PipelineRepositoryImpl) FindAllDeletedPipelineCountInLast24Hour() (pipelineCount int, err error) { pipelineCount, err = impl.dbConnection.Model(&Pipeline{}). - Column("pipeline.id"). Where("created_on > ? and deleted=?", time.Now().AddDate(0, 0, -1), true). Count() return pipelineCount, err diff --git a/pkg/cluster/repository/EnvironmentRepository.go b/pkg/cluster/repository/EnvironmentRepository.go index 94677042d6..7ccd1845d7 100644 --- a/pkg/cluster/repository/EnvironmentRepository.go +++ b/pkg/cluster/repository/EnvironmentRepository.go @@ -261,7 +261,6 @@ func (repositoryImpl EnvironmentRepositoryImpl) FindAllActiveEnvironmentCount() cnt, err := repositoryImpl. dbConnection.Model(&Environment{}). Where("environment.active = ?", true). - Column("environment.id"). Count() return cnt, err } From 9279efa56149ede4cc0ab0e913f64b41bbeb3867 Mon Sep 17 00:00:00 2001 From: komalreddy3 Date: Fri, 19 Apr 2024 15:57:59 +0530 Subject: [PATCH 13/14] fetching to retrieving --- client/telemetry/TelemetryEventClientExtended.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/telemetry/TelemetryEventClientExtended.go b/client/telemetry/TelemetryEventClientExtended.go index 424512691d..24bf093577 100644 --- a/client/telemetry/TelemetryEventClientExtended.go +++ b/client/telemetry/TelemetryEventClientExtended.go @@ -191,7 +191,7 @@ func (impl *TelemetryEventClientImplExtended) SendSummaryEvent(eventType string) impl.logger.Infow("sending summary event", "eventType", eventType) ucid, err := impl.getUCID() if err != nil { - impl.logger.Errorw("exception caught inside telemetry summary event while fetching ucid", "err", err) + impl.logger.Errorw("exception caught inside telemetry summary event while retrieving ucid", "err", err) return err } From ea6224cb158fe089845d759ba98c90f183d28d5a Mon Sep 17 00:00:00 2001 From: komalreddy3 Date: Fri, 19 Apr 2024 16:00:43 +0530 Subject: [PATCH 14/14] remove unnecessary column --- .../repository/dockerRegistry/DockerArtifactStoreRepository.go | 1 - 1 file changed, 1 deletion(-) diff --git a/internal/sql/repository/dockerRegistry/DockerArtifactStoreRepository.go b/internal/sql/repository/dockerRegistry/DockerArtifactStoreRepository.go index e6838f91ee..d2810cb083 100644 --- a/internal/sql/repository/dockerRegistry/DockerArtifactStoreRepository.go +++ b/internal/sql/repository/dockerRegistry/DockerArtifactStoreRepository.go @@ -172,7 +172,6 @@ func (impl DockerArtifactStoreRepositoryImpl) FindAll() ([]DockerArtifactStore, } func (impl DockerArtifactStoreRepositoryImpl) FindAllDockerArtifactCount() (int, error) { dockerArtifactCount, err := impl.dbConnection.Model(&DockerArtifactStore{}). - Column("docker_artifact_store.id"). Where("docker_artifact_store.active = ?", true). Count() return dockerArtifactCount, err