Skip to content

fix: ci-cd count per day in telemetry data #4931

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 26 commits into from
Apr 22, 2024
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
26 commits
Select commit Hold shift + click to select a range
72a18a0
fix: ci-cd count
komalreddy3 Apr 12, 2024
cffd27d
Merge branch 'main' into fix-ci-cd-count
komalreddy3 Apr 12, 2024
4686c48
Merge branch 'main' into fix-ci-cd-count
komalreddy3 Apr 12, 2024
d115038
Merge branch 'fix-ci-cd-count' of https://github.com/devtron-labs/dev…
komalreddy3 Apr 12, 2024
917e220
update queries
komalreddy3 Apr 12, 2024
9c7257b
Merge branch 'main' into fix-ci-cd-count
komalreddy3 Apr 12, 2024
660fba5
update queries to go-pg instead of raw
komalreddy3 Apr 12, 2024
ee71147
remove omit empty
komalreddy3 Apr 15, 2024
c6f9555
summarycron expr change
komalreddy3 Apr 16, 2024
ade898b
refactor
komalreddy3 Apr 18, 2024
0d4e8db
Merge branch 'main' into fix-ci-cd-count
komalreddy3 Apr 18, 2024
4dde447
type fix
komalreddy3 Apr 18, 2024
398c78d
remove comments
komalreddy3 Apr 18, 2024
8c05e67
Merge branch 'fix-ci-cd-count' of https://github.com/devtron-labs/dev…
komalreddy3 Apr 18, 2024
58f99e2
Merge branch 'main' into fix-ci-cd-count
komalreddy3 Apr 18, 2024
21fedc6
refactor query
komalreddy3 Apr 19, 2024
19b9503
typo
komalreddy3 Apr 19, 2024
7098b15
Merge branch 'fix-ci-cd-count' of https://github.com/devtron-labs/dev…
komalreddy3 Apr 19, 2024
abf8262
docker query
komalreddy3 Apr 19, 2024
55e5120
refactor
komalreddy3 Apr 19, 2024
a3dddcf
Merge branch 'main' into fix-ci-cd-count
komalreddy3 Apr 19, 2024
9279efa
fetching to retrieving
komalreddy3 Apr 19, 2024
a762f30
Merge branch 'fix-ci-cd-count' of https://github.com/devtron-labs/dev…
komalreddy3 Apr 19, 2024
ea6224c
remove unnecessary column
komalreddy3 Apr 19, 2024
1cfa504
Merge branch 'main' into fix-ci-cd-count
komalreddy3 Apr 22, 2024
f849e06
Merge branch 'main' into fix-ci-cd-count
komalreddy3 Apr 22, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 32 additions & 6 deletions client/telemetry/TelemetryEventClientExtended.go
Original file line number Diff line number Diff line change
Expand Up @@ -131,8 +131,12 @@ type TelemetryEventDto struct {
UserCount int `json:"userCount,omitempty"`
EnvironmentCount int `json:"environmentCount,omitempty"`
ClusterCount int `json:"clusterCount,omitempty"`
CiCountPerDay int `json:"ciCountPerDay,omitempty"`
CdCountPerDay int `json:"cdCountPerDay,omitempty"`
CiCreatedPerDay int `json:"ciCreatedPerDay,omitempty"`
CdCreatedPerDay int `json:"cdCreatedPerDay,omitempty"`
CiDeletedPerDay int `json:"ciDeletedPerDay,omitempty"`
CdDeletedPerDay int `json:"cdDeletedPerDay,omitempty"`
CiTriggeredPerDay int `json:"ciTriggeredPerDay,omitempty"`
CdTriggeredPerDay int `json:"cdTriggeredPerDay,omitempty"`
HelmChartCount int `json:"helmChartCount,omitempty"`
SecurityScanCountPerDay int `json:"securityScanCountPerDay,omitempty"`
GitAccountsCount int `json:"gitAccountsCount,omitempty"`
Expand Down Expand Up @@ -223,13 +227,32 @@ func (impl *TelemetryEventClientImplExtended) SendSummaryEvent(eventType string)
impl.logger.Errorw("exception caught inside telemetry summary event", "err", err)
return err
}
ciPipelineDeleted, err := impl.ciPipelineRepository.FindAllDeletedPipelineInLast24Hour()
if err != nil && err != pg.ErrNoRows {
impl.logger.Errorw("exception caught inside telemetry summary event", "err", err)
return err
}
ciPipelineTriggered, err := impl.ciWorkflowRepository.FindAllTriggeredWorkflowInLast24Hour()
if err != nil && err != pg.ErrNoRows {
impl.logger.Errorw("exception caught inside telemetry summary event", "err", err)
return err
}

cdPipeline, err := impl.pipelineRepository.FindAllPipelineInLast24Hour()
if err != nil && err != pg.ErrNoRows {
impl.logger.Errorw("exception caught inside telemetry summary event", "err", err)
return err
}

cdPipelineDeleted, err := impl.pipelineRepository.FindAllDeletedPipelineInLast24Hour()
if err != nil && err != pg.ErrNoRows {
impl.logger.Errorw("exception caught inside telemetry summary event", "err", err)
return err
}
cdPipelineTriggered, err := impl.cdWorkflowRepository.FindAllTriggeredWorkflowInLast24Hour()
if err != nil && err != pg.ErrNoRows {
impl.logger.Errorw("exception caught inside telemetry summary event", "err", err)
return err
}
gitAccounts, err := impl.gitProviderRepository.FindAll()
if err != nil && err != pg.ErrNoRows {
impl.logger.Errorw("exception caught inside telemetry summary event", "err", err)
Expand Down Expand Up @@ -307,9 +330,12 @@ func (impl *TelemetryEventClientImplExtended) SendSummaryEvent(eventType string)
payload.UserCount = len(users)
payload.EnvironmentCount = len(environments)
payload.ClusterCount = len(clusters)
payload.CiCountPerDay = len(ciPipeline)

payload.CdCountPerDay = len(cdPipeline)
payload.CiCreatedPerDay = len(ciPipeline)
payload.CiDeletedPerDay = len(ciPipelineDeleted)
payload.CiTriggeredPerDay = ciPipelineTriggered
payload.CdCreatedPerDay = len(cdPipeline)
payload.CdDeletedPerDay = len(cdPipelineDeleted)
payload.CdTriggeredPerDay = cdPipelineTriggered
payload.GitAccountsCount = len(gitAccounts)
payload.GitOpsCount = gitOpsCount
payload.HostURL = hostURL
Expand Down
12 changes: 10 additions & 2 deletions internal/sql/repository/pipelineConfig/CdWorfkflowRepository.go
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ type CdWorkflowRepository interface {
FindLatestWfrByAppIdAndEnvironmentId(appId int, environmentId int) (*CdWorkflowRunner, error)
IsLatestCDWfr(pipelineId, wfrId int) (bool, error)
FindLatestCdWorkflowRunnerByEnvironmentIdAndRunnerType(appId int, environmentId int, runnerType bean.WorkflowType) (CdWorkflowRunner, error)

FindAllTriggeredWorkflowInLast24Hour() (cdWorkflowCount int, err error)
GetConnection() *pg.DB

FindLastPreOrPostTriggeredByPipelineId(pipelineId int) (CdWorkflowRunner, error)
Expand Down Expand Up @@ -357,7 +357,15 @@ func (impl *CdWorkflowRepositoryImpl) FindLatestCdWorkflowByPipelineIdV2(pipelin
// TODO - Group By Environment And Pipeline will get latest pipeline from top
return cdWorkflow, err
}

func (impl *CdWorkflowRepositoryImpl) FindAllTriggeredWorkflowInLast24Hour() (cdWorkflowCount int, err error) {
var workflowCount int
query := "SELECT count(DISTINCT(cd_workflow_id)) FROM cd_workflow_runner WHERE started_on > now() - interval '1 day'"
_, err = impl.dbConnection.Query(&workflowCount, query)
if err != nil {
impl.logger.Errorw("error occurred while fetching ci workflow", "err", err)
}
return workflowCount, err
}
func (impl *CdWorkflowRepositoryImpl) FindCdWorkflowMetaByEnvironmentId(appId int, environmentId int, offset int, limit int) ([]CdWorkflowRunner, error) {
var wfrList []CdWorkflowRunner
err := impl.dbConnection.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -129,6 +129,7 @@ type CiPipelineRepository interface {
FetchCiPipelinesForDG(parentId int, childCiPipelineIds []int) (*CiPipeline, int, error)
FinDByParentCiPipelineAndAppId(parentCiPipeline int, appIds []int) ([]*CiPipeline, error)
FindAllPipelineInLast24Hour() (pipelines []*CiPipeline, err error)
FindAllDeletedPipelineInLast24Hour() (pipelines []*CiPipeline, err error)
FindNumberOfAppsWithCiPipeline(appIds []int) (count int, err error)
FindAppAndProjectByCiPipelineIds(ciPipelineIds []int) ([]*CiPipeline, error)
FindCiPipelineConfigsByIds(ids []int) ([]*CiPipeline, error)
Expand Down Expand Up @@ -509,6 +510,13 @@ func (impl *CiPipelineRepositoryImpl) FindAllPipelineInLast24Hour() (pipelines [
Select()
return pipelines, err
}
func (impl *CiPipelineRepositoryImpl) FindAllDeletedPipelineInLast24Hour() (pipelines []*CiPipeline, err error) {
err = impl.dbConnection.Model(&pipelines).
Column("ci_pipeline.*").
Where("created_on > ? and deleted=?", time.Now().AddDate(0, 0, -1), true).
Select()
return pipelines, err
}

func (impl *CiPipelineRepositoryImpl) FindNumberOfAppsWithCiPipeline(appIds []int) (count int, err error) {
var ciPipelines []*CiPipeline
Expand Down
10 changes: 10 additions & 0 deletions internal/sql/repository/pipelineConfig/CiWorkflowRepository.go
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ type CiWorkflowRepository interface {
FindLastTriggeredWorkflowByCiIds(pipelineId []int) (ciWorkflow []*CiWorkflow, err error)
FindLastTriggeredWorkflowByArtifactId(ciArtifactId int) (ciWorkflow *CiWorkflow, err error)
FindAllLastTriggeredWorkflowByArtifactId(ciArtifactId []int) (ciWorkflow []*CiWorkflow, err error)
FindAllTriggeredWorkflowInLast24Hour() (ciWorkflowCount int, err error)
FindLastTriggeredWorkflowGitTriggersByArtifactId(ciArtifactId int) (ciWorkflow *CiWorkflow, err error)
FindLastTriggeredWorkflowGitTriggersByArtifactIds(ciArtifactIds []int) ([]*WorkflowWithArtifact, error)
ExistsByStatus(status string) (bool, error)
Expand Down Expand Up @@ -286,7 +287,16 @@ func (impl *CiWorkflowRepositoryImpl) FindLastTriggeredWorkflowByArtifactId(ciAr
Select()
return workflow, err
}
func (impl *CiWorkflowRepositoryImpl) FindAllTriggeredWorkflowInLast24Hour() (ciWorkflowCount int, err error) {

var workflowCount int
query := "SELECT count(DISTINCT(ci_pipeline_id)) FROM ci_workflow WHERE started_on > now() - interval '1 day'"
_, err = impl.dbConnection.Query(&workflowCount, query)
if err != nil {
impl.logger.Errorw("error occurred while fetching ci workflow", "err", err)
}
return workflowCount, err
}
func (impl *CiWorkflowRepositoryImpl) FindAllLastTriggeredWorkflowByArtifactId(ciArtifactIds []int) (ciWorkflows []*CiWorkflow, err error) {
err = impl.dbConnection.Model(&ciWorkflows).
Column("ci_workflow.git_triggers", "ci_workflow.ci_pipeline_id", "CiPipeline", "cia.id").
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,7 @@ type PipelineRepository interface {
FindActiveByAppIdAndEnvironmentIdV2() (pipelines []*Pipeline, err error)
GetConnection() *pg.DB
FindAllPipelineInLast24Hour() (pipelines []*Pipeline, err error)
FindAllDeletedPipelineInLast24Hour() (pipelines []*Pipeline, err error)
FindActiveByEnvId(envId int) (pipelines []*Pipeline, err error)
FindActivePipelineByEnvId(envId int) (pipelines []*Pipeline, err error)
FindActiveByEnvIds(envId []int) (pipelines []*Pipeline, err error)
Expand Down Expand Up @@ -454,6 +455,13 @@ func (impl PipelineRepositoryImpl) FindAllPipelineInLast24Hour() (pipelines []*P
Select()
return pipelines, err
}
func (impl PipelineRepositoryImpl) FindAllDeletedPipelineInLast24Hour() (pipelines []*Pipeline, err error) {
err = impl.dbConnection.Model(&pipelines).
Column("pipeline.*").
Where("created_on > ? and deleted=?", time.Now().AddDate(0, 0, -1), true).
Select()
return pipelines, err
}
func (impl PipelineRepositoryImpl) FindActiveByEnvId(envId int) (pipelines []*Pipeline, err error) {
err = impl.dbConnection.Model(&pipelines).Column("pipeline.*", "App", "Environment").
Where("environment_id = ?", envId).
Expand Down